From b8b3e348e889cd18e7bb5a9a1092d9fd37531b58 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Tue, 13 Jun 2023 12:19:30 -0400 Subject: [PATCH] Improved atomicity --- src/services/denokv.ts | 26 +++++++++++++++++++------- src/services/interfaces.ts | 4 ++-- tests/fileservice_test.ts | 33 +++++++++++++++++++++++++++------ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/services/denokv.ts b/src/services/denokv.ts index e9f0781..63056ea 100644 --- a/src/services/denokv.ts +++ b/src/services/denokv.ts @@ -14,6 +14,7 @@ interface SerializedFile { path: string; contentSize: number; compression: CompressionAlgorithm; + creationTime: number; } interface SerializedFilePathIndex { @@ -43,8 +44,11 @@ export class DenoKVBlobService implements BlobService { } const blob_id = await this.sha1(content); - await this.kv.set(['blob_index', blobIndex, chunckIndex], blob_id); - await this.kv.set(['blob', blob_id], content); + await this.kv.atomic() + .check({key: ['blob_index', blobIndex, chunckIndex], versionstamp: null }) + .set(['blob_index', blobIndex, chunckIndex], blob_id) + .set(['blob', blob_id], content) + .commit(); } persistBlob(index: string, content: Uint8Array): Promise { @@ -107,12 +111,12 @@ export class DenoKVFileService implements FileService { createFromFullContent( path: string, content: string | Uint8Array, - ): Promise { + ): Promise { const reader = content instanceof Uint8Array ? new Buffer(content) : new StringReader(content); return this.createFromReader(path, reader); } - async createFromReader(path: string, content: Deno.Reader): Promise { + async createFromReader(path: string, content: Deno.Reader): Promise { const newId = crypto.randomUUID(); const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream(CompressionAlgorithm.GZIP)) @@ -123,12 +127,18 @@ export class DenoKVFileService implements FileService { path, contentSize: size, compression: CompressionAlgorithm.GZIP, + creationTime: Date.now() }; const idx: SerializedFilePathIndex = { fileId: newId }; - await this.kv.set(['file', newId], serializedFile); - await this.kv.set(['file_index_path', path], idx); - return { id: newId, path, contentSize: size }; + const ret = await this.kv.atomic() + .check({key: ['file_index_path', path], versionstamp: null }) + .set(['file', newId], serializedFile) + .set(['file_index_path', path], idx) + .commit(); + + + return ret.ok ? { id: newId, path, contentSize: size } : null; } async *list(): AsyncIterable { @@ -253,10 +263,12 @@ export class DenoKVFileService implements FileService { id: fileObject.value.id, contentSize: fileObject.value.contentSize, compression: fileObject.value.compression, + creationTime: fileObject.value.creationTime, }; await this.kv.atomic() .check(fileObject) .check(idx) + .check({key: ['file_index_path', newPath], versionstamp: null }) .delete(idx.key) .set(['file_index_path', newPath], idx.value) .set(fileObject.key, newFile) diff --git a/src/services/interfaces.ts b/src/services/interfaces.ts index e7edbc5..1ee29a6 100644 --- a/src/services/interfaces.ts +++ b/src/services/interfaces.ts @@ -10,9 +10,9 @@ export interface FileService { createFromFullContent( path: string, content: string | Uint8Array, - ): Promise; + ): Promise; - createFromReader(path: string, content: Deno.Reader): Promise; + createFromReader(path: string, content: Deno.Reader): Promise; getRawContentByPath(path: string): Promise; diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts index a029b9a..551df5b 100644 --- a/tests/fileservice_test.ts +++ b/tests/fileservice_test.ts @@ -45,7 +45,7 @@ Deno.test('DenoKVBlobService happy path', async (t) => { kv.close(); }); -Deno.test('DenoKVFileService happy path xxx ', async (t) => { +Deno.test('DenoKVFileService happy path ', async (t) => { const kv = await Deno.openKv(':memory:'); const fs: FileService = new DenoKVFileService(kv); const fileName1 = 'didier/file1.txt'; @@ -54,8 +54,8 @@ Deno.test('DenoKVFileService happy path xxx ', async (t) => { let file2: FileMetaData | undefined; await t.step('Create 2 files xyzabc', async () => { - file1 = await fs.createFromFullContent(fileName1, 'bob le chef'); - file2 = await fs.createFromFullContent(fileName2, 'le geant vert ☀️'); + file1 = (await fs.createFromFullContent(fileName1, 'bob le chef'))!; + file2 = (await fs.createFromFullContent(fileName2, 'le geant vert ☀️'))!; assertEquals(file1.path, fileName1); assertEquals(file2.contentSize, 47); @@ -97,7 +97,7 @@ Deno.test('DenoKVFileService happy path xxx ', async (t) => { const newPath = 'boris/streamed-file.txt'; const ret = await fs.createFromReader(newPath, new StringReader('hello world 18')); - assertEquals(ret.contentSize, 40); + assertEquals(ret!.contentSize, 40); const s = await fs.getRawContentByPath(newPath); assertEquals(new TextDecoder().decode(s!), 'hello world 18'); @@ -117,6 +117,27 @@ Deno.test('DenoKVFileService happy path xxx ', async (t) => { kv.close(); }); + +Deno.test('DenoKVFileService file duplication',async (t) => { + const kv = await Deno.openKv(':memory:'); + const fs: FileService = new DenoKVFileService(kv); + + await t.step("Create two files with same name", async () => { + const file1 = await fs.createFromFullContent('create/one', 'file 1'); + const file2 = await fs.createFromFullContent(file1!.path, 'file 2'); + assertEquals(file2, null); + }); + + await t.step('Rename file to path that already exists',async () => { + const file1 = await fs.createFromFullContent('rename/one', 'file 1'); + const file2 = await fs.createFromFullContent('rename/two', 'file 2'); + const ret = await fs.renameById(file1!.id, file2!.path); + assertEquals(ret!.path, file1!.path) // assert file did not move + }); + + kv.close(); + +}) Deno.test('DenoKVFileService happy path with byte array', async (t) => { const kv = await Deno.openKv(':memory:'); const fs: FileService = new DenoKVFileService(kv); @@ -124,8 +145,8 @@ Deno.test('DenoKVFileService happy path with byte array', async (t) => { const inputContent = new Uint8Array([240]); const file = await fs.createFromFullContent('hello', inputContent); - assertEquals(file.path, 'hello'); - assertEquals(file.contentSize, 27); + assertEquals(file!.path, 'hello'); + assertEquals(file!.contentSize, 27); const persistedContent = await fs.getRawContentByPath('hello'); assertEquals(persistedContent, inputContent);