From bbd705ee3b866dbfcef52ad909c897199216549d Mon Sep 17 00:00:00 2001 From: didier amyot Date: Tue, 13 Jun 2023 07:53:31 -0400 Subject: [PATCH] Refactored a bit the code; the original code was too experimental in style. --- src/fileservice.ts | 73 ++++++++++++++++++--------------------- src/restapi.ts | 2 +- tests/fileservice_test.ts | 40 +++++++++++---------- 3 files changed, 55 insertions(+), 60 deletions(-) diff --git a/src/fileservice.ts b/src/fileservice.ts index 39e5c62..2f6dff6 100644 --- a/src/fileservice.ts +++ b/src/fileservice.ts @@ -14,18 +14,16 @@ export interface FileService { list(): AsyncIterable; - createFromFullBody( + createFromFullContent( path: string, - body: string | Uint8Array | Deno.Reader, + content: string | Uint8Array, ): Promise; - createFromReader(path: string, body: Deno.Reader): Promise; + createFromReader(path: string, content: Deno.Reader): Promise; - getFullBodyByPath(path: string): Promise; + getRawContentByPath(path: string): Promise; - getRawBodyByPath(path: string): Promise; - - getBodyAsReader(path: string): Promise; + getContentAsReader(path: string): Promise; getMetaDataByID(id: string): Promise; @@ -42,14 +40,14 @@ export interface FileService { interface SerializedFile { id: string; path: string; - bodySize: number; + contentSize: number; compression: string; } export interface FileMetaData { path: string; id: string; - bodySize: number; + contentSize: number; } interface SerializedFilePathIndex { @@ -73,36 +71,36 @@ export class DenoKVBlobService { .join(""); // convert bytes to hex string } - private *splitText(body: Uint8Array): Generator { + private *splitText(content: Uint8Array): Generator { let blobCount = 0; - let txt = body.slice( + let txt = content.slice( blobCount * this.blobSize, (blobCount + 1) * this.blobSize, ); while (txt.length > 0) { yield txt; blobCount += 1; - txt = body.slice( + txt = content.slice( blobCount * this.blobSize, (blobCount + 1) * this.blobSize, ); } } - private async unsafePersistBlob(mainIndex: string, index: number, body: Uint8Array): Promise { - if(body.length > this.blobSize){ + private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise { + if(content.length > this.blobSize){ throw new Error('Persisted a too large blob'); } - const blob_id = await this.sha1(body); + const blob_id = await this.sha1(content); await this.kv.set(['blob_index', mainIndex, index], blob_id); - await this.kv.set(['blob', blob_id], body); + await this.kv.set(['blob', blob_id], content); } - async persistBlob(index: string, body: Uint8Array): Promise { + async persistBlob(index: string, content: Uint8Array): Promise { let blobCount = 0; let size = 0; - for (const txt of this.splitText(body)) { + for (const txt of this.splitText(content)) { await this.unsafePersistBlob(index, blobCount, txt); size += txt.length; blobCount += 1; @@ -110,10 +108,10 @@ export class DenoKVBlobService { return size; } - async persistReader(index: string, body: Deno.Reader): Promise{ + async persistReader(index: string, content: Deno.Reader): Promise{ let i = 0; let size = 0; - for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){ + for await (const chunk of iterateReader(content, {bufSize: this.blobSize})){ await this.unsafePersistBlob(index, i, chunk); size += chunk.length; i++; @@ -181,14 +179,14 @@ export class DenoKVFileService implements FileService { this.blobService = new DenoKVBlobService(kv, 32 * 1024); } - createFromFullBody( + createFromFullContent( path: string, - body: string | Uint8Array, + content: string | Uint8Array, ): Promise { - if (body instanceof Uint8Array){ - return this.createFromReader(path, new Buffer(body)); + if (content instanceof Uint8Array){ + return this.createFromReader(path, new Buffer(content)); } - return this.createFromReader(path, new StringReader(body)); + return this.createFromReader(path, new StringReader(content)); } async createFromReader(path: string, content: Deno.Reader): Promise{ @@ -199,14 +197,14 @@ export class DenoKVFileService implements FileService { const serializedFile: SerializedFile = { id: newId, path, - bodySize: size, + contentSize: size, compression: 'gzip' }; const idx: SerializedFilePathIndex = { fileId: newId }; await this.kv.set(['file', newId], serializedFile); await this.kv.set(['file_index_path', path], idx); - return { id: newId, path, bodySize: size }; + return { id: newId, path, contentSize: size }; } @@ -224,7 +222,7 @@ export class DenoKVFileService implements FileService { yield { path: ret.value.path, id: ret.value.id, - bodySize: ret.value.bodySize, + contentSize: ret.value.contentSize, }; } } @@ -252,7 +250,7 @@ export class DenoKVFileService implements FileService { return ret.value } - private decompressBody(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader { + private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader { if(serializedFile.compression == 'gzip'){ const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader(); return readerFromStreamReader(ret); @@ -262,18 +260,13 @@ export class DenoKVFileService implements FileService { } - async getFullBodyByPath(path: string): Promise { - const ret = await this.getRawBodyByPath(path); - return ret == null ? null : new TextDecoder().decode(ret); - } - - async getRawBodyByPath(path: string): Promise { - const ret = await this.getBodyAsReader(path); + async getRawContentByPath(path: string): Promise { + const ret = await this.getContentAsReader(path); return ret === null ? null : readAll(ret); } - async getBodyAsReader(path: string): Promise { + async getContentAsReader(path: string): Promise { const idx = await this.kv.get([ 'file_index_path', path, @@ -285,7 +278,7 @@ export class DenoKVFileService implements FileService { if(ser.value === null){ return null; } - return this.decompressBody(ser.value, this.blobService.getBlob(idx.value.fileId)); + return this.decompressContent(ser.value, this.blobService.getBlob(idx.value.fileId)); } @@ -297,7 +290,7 @@ export class DenoKVFileService implements FileService { return { path: fileObject.path, id: fileObject.id, - bodySize: fileObject.bodySize, + contentSize: fileObject.contentSize, }; } @@ -313,7 +306,7 @@ export class DenoKVFileService implements FileService { const newFile: SerializedFile = { path: newPath, id: fileObject.value.id, - bodySize: fileObject.value.bodySize, + contentSize: fileObject.value.contentSize, compression: fileObject.value.compression, }; await this.kv.atomic() diff --git a/src/restapi.ts b/src/restapi.ts index e83cdec..820af11 100644 --- a/src/restapi.ts +++ b/src/restapi.ts @@ -30,7 +30,7 @@ function getFileHandlerFactory(fileService: FileService): Handler { return async (ctx: Context) => { const untypedCtx: any = ctx; const fileId = untypedCtx.params.path; - const reader = await fileService.getBodyAsReader(fileId); + const reader = await fileService.getContentAsReader(fileId); if(reader == null){ ctx.response.status = 404; return; diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts index c000ac3..0ac03f7 100644 --- a/tests/fileservice_test.ts +++ b/tests/fileservice_test.ts @@ -24,8 +24,8 @@ Deno.test("DenoKVBlobService happy path", async (t) => { await t.step("read write small text", async () => { const size = await bs.persistBlob("0002", toUint8Array("hello world")); - const body = await bs.readAllBlob("0002"); - assertEquals(body, toUint8Array("hello world")); + const content = await bs.readAllBlob("0002"); + assertEquals(content, toUint8Array("hello world")); assertEquals(await bs.getNumberOfBlobs("0002"), 1); assertEquals(size, "hello world".length); }); @@ -33,8 +33,8 @@ Deno.test("DenoKVBlobService happy path", async (t) => { await t.step("read write large text", async () => { const largeText = toUint8Array(generateLargeText(33, 100)); const size = await bs.persistBlob("0001", largeText); - const body = await bs.readAllBlob("0001"); - assertEquals(body, largeText); + const content = await bs.readAllBlob("0001"); + assertEquals(content, largeText); assertEquals(await bs.getNumberOfBlobs("0001"), 21); assertEquals(size, largeText.length); }); @@ -61,15 +61,15 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => { let file2: FileMetaData | undefined; await t.step("Create 2 files xyzabc", async () => { - file1 = await fs.createFromFullBody(fileName1, "bob le chef"); - file2 = await fs.createFromFullBody(fileName2, "le geant vert ☀️"); + file1 = await fs.createFromFullContent(fileName1, "bob le chef"); + file2 = await fs.createFromFullContent(fileName2, "le geant vert ☀️"); assertEquals(file1.path, fileName1); - assertEquals(file2.bodySize, 47); + assertEquals(file2.contentSize, 47); }); await t.step("Get file metadata", async () => { - const ret = await fs.getMetaDataByID(file1?.id ?? "404"); + const ret = await fs.getMetaDataByID(file1!.id); assertEquals(ret, file1); }); @@ -80,11 +80,13 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => { }); await t.step("Get File by path", async () => { - assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef"); + const ret = await fs.getRawContentByPath(fileName1); + assertEquals(new TextDecoder().decode(ret!), "bob le chef"); }); await t.step("Get File by Id", async () => { - assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef"); + const ret = await fs.getRawContentByPath(fileName1); + assertEquals(new TextDecoder().decode(ret!), "bob le chef"); }); await t.step("Delete File", async () => { @@ -102,19 +104,19 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => { const newPath = "boris/streamed-file.txt"; const ret = await fs.createFromReader(newPath, new StringReader("hello world 18")); - assertEquals(ret.bodySize, 40); + assertEquals(ret.contentSize, 40); - const s = await fs.getRawBodyByPath(newPath); + const s = await fs.getRawContentByPath(newPath); assertEquals(new TextDecoder().decode(s!), "hello world 18") }); await t.step("Rename file", async () => { const newPath = "boris/file2.txt"; - const ret = await fs.renameById(file2?.id ?? "404", newPath); + const ret = await fs.renameById(file2!.id, newPath); assertEquals(ret?.path, newPath); assertEquals( - await fs.getFullBodyByPath(newPath), + new TextDecoder().decode((await fs.getRawContentByPath(newPath))!), 'le geant vert ☀️', ); }); @@ -126,14 +128,14 @@ Deno.test("DenoKVFileService happy path with byte array", async (t) => { const kv = await Deno.openKv(":memory:"); const fs: FileService = new DenoKVFileService(kv); await t.step("Create and read 1 binary file", async () => { - const inputBody = new Uint8Array([240]); - const file = await fs.createFromFullBody("hello", inputBody); + const inputContent = new Uint8Array([240]); + const file = await fs.createFromFullContent("hello", inputContent); assertEquals(file.path, "hello"); - assertEquals(file.bodySize, 27); + assertEquals(file.contentSize, 27); - const bodyString = await fs.getFullBodyByPath("hello"); - assertEquals(bodyString, new TextDecoder().decode(inputBody)); + const persistedContent = await fs.getRawContentByPath("hello"); + assertEquals(persistedContent, inputContent); }); kv.close();