From 8dcd8cc425720b6473b1795264d77d6362b68b75 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Tue, 13 Jun 2023 08:53:26 -0400 Subject: [PATCH] Deleted dead code. --- src/restapi.ts | 21 ++++++++- src/services/denokv.ts | 91 ++++++++++++-------------------------- src/services/interfaces.ts | 3 +- tests/fileservice_test.ts | 6 +-- 4 files changed, 51 insertions(+), 70 deletions(-) diff --git a/src/restapi.ts b/src/restapi.ts index cf970fa..3015190 100644 --- a/src/restapi.ts +++ b/src/restapi.ts @@ -1,14 +1,31 @@ import { Context, Router } from './deps.ts'; -import { FileService } from './services/mod.ts'; +import { FileMetaData, FileService } from './services/mod.ts'; import { readerFromStreamReader } from './deps.ts'; import { asyncIterableToArray } from './utils.ts'; type Handler = (ctx: Context) => void; +class FileDTO { + readonly path: string; + readonly id: string; + readonly size: number; + + constructor(path: string, id: string, size: number){ + this.path = path; + this.id = id; + this.size = size; + } + + static fromFileMetaData(fm: FileMetaData): FileDTO{ + return new FileDTO(); + + } +} + function listFileHandlerFactory(fileService: FileService): Handler { return async (ctx: Context) => { ctx.response.type = 'application/json'; - ctx.response.body = await asyncIterableToArray(fileService.list()); + ctx.response.body = (await asyncIterableToArray(fileService.list())).map(FileDTO.fromFileMetaData); }; } diff --git a/src/services/denokv.ts b/src/services/denokv.ts index cbe8ef8..73db8ee 100644 --- a/src/services/denokv.ts +++ b/src/services/denokv.ts @@ -4,11 +4,16 @@ import { BlobService, FileMetaData, FileService } from './interfaces.ts'; import { readableStreamFromReader, readerFromStreamReader } from 'https://deno.land/std@0.191.0/streams/mod.ts'; +enum CompressionAlgorithm{ + UNCOMPRESSED = 'uncompressed', + GZIP = 'gzip', +} + interface SerializedFile { id: string; path: string; contentSize: number; - compression: string; + compression: CompressionAlgorithm; } interface SerializedFilePathIndex { @@ -24,65 +29,42 @@ export class DenoKVBlobService implements BlobService { this.blobSize = blobSize; } - private async sha1(msgUint8: Uint8Array): Promise { - const hashBuffer = await crypto.subtle.digest('SHA-1', msgUint8); + private async sha1(content: Uint8Array): Promise { + const hashBuffer = await crypto.subtle.digest('SHA-1', content); const hashArray = Array.from(new Uint8Array(hashBuffer)); return hashArray .map((b) => b.toString(16).padStart(2, '0')) .join(''); // convert bytes to hex string } - private *splitText(content: Uint8Array): Generator { - let blobCount = 0; - let txt = content.slice( - blobCount * this.blobSize, - (blobCount + 1) * this.blobSize, - ); - while (txt.length > 0) { - yield txt; - blobCount += 1; - txt = content.slice( - blobCount * this.blobSize, - (blobCount + 1) * this.blobSize, - ); - } - } - - private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise { + private async unsafePersistChunck(blobIndex: string, chunckIndex: number, content: Uint8Array): Promise { if (content.length > this.blobSize) { throw new Error('Persisted a too large blob'); } const blob_id = await this.sha1(content); - await this.kv.set(['blob_index', mainIndex, index], blob_id); + await this.kv.set(['blob_index', blobIndex, chunckIndex], blob_id); await this.kv.set(['blob', blob_id], content); } - async persistBlob(index: string, content: Uint8Array): Promise { - let blobCount = 0; - let size = 0; - for (const txt of this.splitText(content)) { - await this.unsafePersistBlob(index, blobCount, txt); - size += txt.length; - blobCount += 1; - } - return size; + persistBlob(index: string, content: Uint8Array): Promise { + return this.persistReader(index, new Buffer(content)); } async persistReader(index: string, content: Deno.Reader): Promise { let i = 0; let size = 0; - for await (const chunk of iterateReader(content, { bufSize: this.blobSize })) { - await this.unsafePersistBlob(index, i, chunk); - size += chunk.length; + for await (const chunck of iterateReader(content, { bufSize: this.blobSize })) { + await this.unsafePersistChunck(index, i, chunck); + size += chunck.length; i++; } return size; } - private async *iterBlob(index: string): AsyncIterable { + private async *iterBlob(blobIndex: string): AsyncIterable { for await ( - const itm of this.kv.list({ prefix: ['blob_index', index] }) + const itm of this.kv.list({ prefix: ['blob_index', blobIndex] }) ) { const foo = await this.kv.get(['blob', itm.value]); if (foo.value === null) { @@ -102,20 +84,7 @@ export class DenoKVBlobService implements BlobService { return readerFromIterable(this.iterBlob(index)); } - private async *iterRawBlobs(index: string): AsyncIterable { - const prefix = ['blob_index', index]; - for await (const itm of this.kv.list({ prefix })) { - const foo = await this.kv.get(['blob', itm.value]); - if (foo.value === null) { - throw new Error( - `Inconsistant blob index for key "${foo.value}", this is very bad.`, - ); - } - yield foo.value; - } - } - - async getNumberOfBlobs(index: string): Promise { + async getNumberOfChunks(index: string): Promise { let ret = 0; for await ( const _itm of this.kv.list({ prefix: ['blob_index', index] }) @@ -124,10 +93,6 @@ export class DenoKVBlobService implements BlobService { } return ret; } - - streamBlob(index: string): Deno.Reader { - return readerFromIterable(this.iterRawBlobs(index)); - } } export class DenoKVFileService implements FileService { @@ -143,23 +108,21 @@ export class DenoKVFileService implements FileService { path: string, content: string | Uint8Array, ): Promise { - if (content instanceof Uint8Array) { - return this.createFromReader(path, new Buffer(content)); - } - return this.createFromReader(path, new StringReader(content)); + const reader = content instanceof Uint8Array ? new Buffer(content) : new StringReader(content); + return this.createFromReader(path, reader); } async createFromReader(path: string, content: Deno.Reader): Promise { const newId = crypto.randomUUID(); - const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')) + const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream(CompressionAlgorithm.GZIP)) .getReader(); const size = await this.blobService.persistReader(newId, readerFromStreamReader(compressedContent)); const serializedFile: SerializedFile = { id: newId, path, contentSize: size, - compression: 'gzip', + compression: CompressionAlgorithm.GZIP, }; const idx: SerializedFilePathIndex = { fileId: newId }; await this.kv.set(['file', newId], serializedFile); @@ -211,11 +174,13 @@ export class DenoKVFileService implements FileService { } private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader { - if (serializedFile.compression == 'gzip') { - const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader(); - return readerFromStreamReader(ret); + if(serializedFile.compression === CompressionAlgorithm.UNCOMPRESSED){ + return content; } - return content; + + const decompresser = new DecompressionStream(serializedFile.compression); + const ret = readableStreamFromReader(content).pipeThrough(decompresser).getReader(); + return readerFromStreamReader(ret); } async getRawContentByPath(path: string): Promise { diff --git a/src/services/interfaces.ts b/src/services/interfaces.ts index a2a7537..6bd8cfd 100644 --- a/src/services/interfaces.ts +++ b/src/services/interfaces.ts @@ -35,6 +35,5 @@ export interface BlobService { persistReader(index: string, content: Deno.Reader): Promise; readAllBlob(index: string): Promise; getBlob(index: string): Deno.Reader; - getNumberOfBlobs(index: string): Promise; - streamBlob(index: string): Deno.Reader; + getNumberOfChunks(index: string): Promise; } diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts index 8991b06..19004fb 100644 --- a/tests/fileservice_test.ts +++ b/tests/fileservice_test.ts @@ -19,7 +19,7 @@ Deno.test('DenoKVBlobService happy path', async (t) => { const size = await bs.persistBlob('0002', toUint8Array('hello world')); const content = await bs.readAllBlob('0002'); assertEquals(content, toUint8Array('hello world')); - assertEquals(await bs.getNumberOfBlobs('0002'), 1); + assertEquals(await bs.getNumberOfChunks('0002'), 1); assertEquals(size, 'hello world'.length); }); @@ -28,7 +28,7 @@ Deno.test('DenoKVBlobService happy path', async (t) => { const size = await bs.persistBlob('0001', largeText); const content = await bs.readAllBlob('0001'); assertEquals(content, largeText); - assertEquals(await bs.getNumberOfBlobs('0001'), 21); + assertEquals(await bs.getNumberOfChunks('0001'), 21); assertEquals(size, largeText.length); }); @@ -37,7 +37,7 @@ Deno.test('DenoKVBlobService happy path', async (t) => { const f = await Deno.open(fileName); const size = await bs.persistReader('large-stream', f); - assertEquals(await bs.getNumberOfBlobs('large-stream'), 1030); + assertEquals(await bs.getNumberOfChunks('large-stream'), 1030); assertEquals((await Deno.readFile(fileName)).length, size); f.close(); });