From ce7b6ae52357f5166ff4f7fde72b901609f41213 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Sat, 10 Jun 2023 08:41:17 -0400 Subject: [PATCH] Added raw byte array in storage --- src/fileservice.ts | 37 ++++++++++++++++++++++++++----------- tests/fileservice_test.ts | 33 +++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/fileservice.ts b/src/fileservice.ts index 673fcf5..949dd53 100644 --- a/src/fileservice.ts +++ b/src/fileservice.ts @@ -2,10 +2,11 @@ import { sha1 } from "https://denopkg.com/chiefbiiko/sha1/mod.ts"; export interface FileService { - createFromFullBody(path: string, body: string): Promise; + createFromFullBody(path: string, body: string|Uint8Array): Promise; list(): AsyncIterable; deleteByPath(path: string): Promise; getFullBodyByPath(path: string): Promise; + getRawBodyByPath(path: string): Promise; getMetaDataByID(id: string): Promise; renameById(id: string, newPath: string): Promise; renameByPath(originalPath: string, newPath: string): Promise; @@ -37,18 +38,18 @@ export class DenoKVBlobService { this.blobSize = blobSize; } - private *splitText(body: string): Generator{ + private *splitText(body: Uint8Array): Generator{ let blobCount = 0; - let txt = body.substring(blobCount * this.blobSize, (blobCount + 1) * this.blobSize); + let txt = body.slice(blobCount * this.blobSize, (blobCount + 1) * this.blobSize); while(txt.length > 0 ){ yield txt; blobCount += 1; - txt = body.substring(blobCount * this.blobSize, (blobCount + 1) * this.blobSize); + txt = body.slice(blobCount * this.blobSize, (blobCount + 1) * this.blobSize); } } - async persistBlob(index: string, body: string): Promise{ + async persistBlob(index: string, body: Uint8Array): Promise{ let blobCount = 0; for(const txt of this.splitText(body)){ const blob_id = sha1(txt, "utf8", "hex"); @@ -58,14 +59,22 @@ export class DenoKVBlobService { } } - async getBlob(index: string): Promise{ - let ret = ""; + // todo: remove + private concatUint8Array(arr1: Uint8Array, arr2: Uint8Array): Uint8Array { + const mergedArray = new Uint8Array(arr1.length + arr2.length); + mergedArray.set(arr1); + mergedArray.set(arr2, arr1.length); + return mergedArray; + } + + async getBlob(index: string): Promise{ + let ret = new Uint8Array(); for await(const itm of this.kv.list({prefix: ['blob_index', index]})){ - const foo = await this.kv.get(['blob', itm.value]) + const foo = await this.kv.get(['blob', itm.value]) if(foo.value === null){ throw new Error(`Inconsistant blob index for key "${foo.value}", this is very bad.`); } - ret += foo.value; + ret = this.concatUint8Array(ret, foo.value); } return ret; } @@ -91,13 +100,14 @@ export class DenoKVFileService implements FileService { this.blobService = new DenoKVBlobService(kv, 500); } - async createFromFullBody(path: string, body: string): Promise { + async createFromFullBody(path: string, body: string|Uint8Array): Promise { const newId = crypto.randomUUID(); const serializedFile: SerializedFile = {id: newId, path, bodySize: body.length}; const idx: SerializedFilePathIndex = {fileId: newId}; await this.kv.set(['file', newId], serializedFile); await this.kv.set(['file_index_path', path], idx); - await this.blobService.persistBlob(newId, body); + const bodyBytes: Uint8Array = body instanceof Uint8Array ? body: new TextEncoder().encode(body); + await this.blobService.persistBlob(newId, bodyBytes); return {id: newId, path, bodySize: body.length}; } @@ -124,6 +134,11 @@ export class DenoKVFileService implements FileService { } async getFullBodyByPath(path: string): Promise { + const ret = await this.getRawBodyByPath(path); + return ret == null ? null : new TextDecoder().decode(ret); + } + + async getRawBodyByPath(path: string): Promise { const idx = await this.kv.get(["file_index_path", path]); if(idx.value === null){ return null; diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts index e978956..f4c06f4 100644 --- a/tests/fileservice_test.ts +++ b/tests/fileservice_test.ts @@ -53,15 +53,19 @@ Deno.test("DenoKVBlobService happy path", async (t) => { const bs = new DenoKVBlobService(kv, 50); + function toUint8Array(s: string): Uint8Array{ + return new TextEncoder().encode(s); + } + await t.step("read write small text", async () => { - await bs.persistBlob('0002', 'hello world'); + await bs.persistBlob('0002', toUint8Array('hello world')); const body = await bs.getBlob('0002'); - assertEquals(body, 'hello world'); + assertEquals(body, toUint8Array('hello world')); assertEquals(await bs.getNumberOfBlobs('0002'), 1); }); await t.step("read write large text", async () => { - const largeText = generateLargeText(33, 100); + const largeText = toUint8Array(generateLargeText(33, 100)); await bs.persistBlob('0001', largeText); const body = await bs.getBlob('0001'); assertEquals(body, largeText); @@ -128,7 +132,24 @@ Deno.test("DenoKVFileService happy path", async (t) => { }); kv.close(); - - - }); + + +Deno.test("DenoKVFileService happy path with byte array", async (t) => { + const kv = await Deno.openKv(":memory:"); + const fs: FileService = new DenoKVFileService(kv); + await t.step("Create and read 1 binary file", async () => { + const inputBody = new Uint8Array([240]); + const file = await fs.createFromFullBody('hello', inputBody); + + assertEquals(file.path, 'hello'); + assertEquals(file.bodySize, inputBody.length); + + const bodyString = await fs.getFullBodyByPath('hello'); + assertEquals(bodyString, new TextDecoder().decode(inputBody)); + }); + + + kv.close(); + +}); \ No newline at end of file