diff --git a/src/deps.ts b/src/deps.ts index 891f7b6..157ca52 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,5 +1,10 @@ -export { Buffer, StringReader } from "https://deno.land/std@0.191.0/io/mod.ts"; -export { readAll, readerFromIterable, readableStreamFromReader, readerFromStreamReader, iterateReader, TextLineStream } from "https://deno.land/std@0.191.0/streams/mod.ts"; +export { Buffer, StringReader } from 'https://deno.land/std@0.191.0/io/mod.ts'; +export { + iterateReader, + readableStreamFromReader, + readAll, + readerFromIterable, + readerFromStreamReader, + TextLineStream, +} from 'https://deno.land/std@0.191.0/streams/mod.ts'; export { Application, Context, Router } from 'https://deno.land/x/oak/mod.ts'; - - diff --git a/src/restapi.ts b/src/restapi.ts index 131ffe2..cf970fa 100644 --- a/src/restapi.ts +++ b/src/restapi.ts @@ -1,15 +1,13 @@ -import { Context, Router } from "./deps.ts"; -import { FileService } from "./services/mod.ts"; -import { readerFromStreamReader } from "./deps.ts"; -import { asyncIterableToArray } from "./utils.ts"; - +import { Context, Router } from './deps.ts'; +import { FileService } from './services/mod.ts'; +import { readerFromStreamReader } from './deps.ts'; +import { asyncIterableToArray } from './utils.ts'; type Handler = (ctx: Context) => void; - function listFileHandlerFactory(fileService: FileService): Handler { return async (ctx: Context) => { - ctx.response.type = "application/json"; + ctx.response.type = 'application/json'; ctx.response.body = await asyncIterableToArray(fileService.list()); }; } @@ -20,8 +18,8 @@ function createFileHandlerFactory(fileService: FileService): Handler { ctx.response.body = 'This is a post'; const path = untypedCtx.request.url.searchParams.get('path'); - const reqBody = await ctx.request.body({ type: "stream" }).value; - const reader = readerFromStreamReader(reqBody.getReader()) + const reqBody = await ctx.request.body({ type: 'stream' }).value; + const reader = readerFromStreamReader(reqBody.getReader()); await fileService.createFromReader(path, reader); }; } @@ -31,22 +29,21 @@ function getFileHandlerFactory(fileService: FileService): Handler { const untypedCtx: any = ctx; const fileId = untypedCtx.params.path; const reader = await fileService.getContentAsReader(fileId); - if(reader == null){ + if (reader == null) { ctx.response.status = 404; return; } ctx.response.status = 200; - ctx.response.body = reader; + ctx.response.body = reader; }; } - function deleteFileHandlerFactory(fileService: FileService): Handler { return async (ctx: Context) => { const untypedCtx: any = ctx; const path = untypedCtx.params.path; const isDeleted = await fileService.deleteByPath(path); - if(!isDeleted){ + if (!isDeleted) { ctx.response.status = 404; return; } @@ -54,7 +51,6 @@ function deleteFileHandlerFactory(fileService: FileService): Handler { }; } - export function buildAPI(router: Router, fileService: FileService): Router { router.get('/files', listFileHandlerFactory(fileService)); router.get('/files/:path', getFileHandlerFactory(fileService)); @@ -62,4 +58,4 @@ export function buildAPI(router: Router, fileService: FileService): Router { router.post('/files', createFileHandlerFactory(fileService)); return router; -} \ No newline at end of file +} diff --git a/src/server.ts b/src/server.ts index 975409f..e488224 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,8 +1,8 @@ import { Application, Router } from './deps.ts'; import { buildAPI } from './restapi.ts'; -import { DenoKVFileService } from "./services/mod.ts"; +import { DenoKVFileService } from './services/mod.ts'; -async function main(){ +async function main() { const app = new Application(); const kv = await Deno.openKv(); const fileService = new DenoKVFileService(kv); diff --git a/src/services/denokv.ts b/src/services/denokv.ts index c7282c2..cbe8ef8 100644 --- a/src/services/denokv.ts +++ b/src/services/denokv.ts @@ -1,9 +1,8 @@ -import { readAll, readerFromIterable, iterateReader } from "../deps.ts"; -import { Buffer, StringReader } from "../deps.ts"; -import {FileService, FileMetaData, BlobService} from './interfaces.ts'; - -import { readerFromStreamReader, readableStreamFromReader } from "https://deno.land/std@0.191.0/streams/mod.ts"; +import { iterateReader, readAll, readerFromIterable } from '../deps.ts'; +import { Buffer, StringReader } from '../deps.ts'; +import { BlobService, FileMetaData, FileService } from './interfaces.ts'; +import { readableStreamFromReader, readerFromStreamReader } from 'https://deno.land/std@0.191.0/streams/mod.ts'; interface SerializedFile { id: string; @@ -12,7 +11,6 @@ interface SerializedFile { compression: string; } - interface SerializedFilePathIndex { fileId: string; } @@ -27,12 +25,12 @@ export class DenoKVBlobService implements BlobService { } private async sha1(msgUint8: Uint8Array): Promise { - const hashBuffer = await crypto.subtle.digest("SHA-1", msgUint8); + const hashBuffer = await crypto.subtle.digest('SHA-1', msgUint8); const hashArray = Array.from(new Uint8Array(hashBuffer)); return hashArray - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); // convert bytes to hex string - } + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); // convert bytes to hex string + } private *splitText(content: Uint8Array): Generator { let blobCount = 0; @@ -51,10 +49,10 @@ export class DenoKVBlobService implements BlobService { } private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise { - if(content.length > this.blobSize){ + if (content.length > this.blobSize) { throw new Error('Persisted a too large blob'); } - + const blob_id = await this.sha1(content); await this.kv.set(['blob_index', mainIndex, index], blob_id); await this.kv.set(['blob', blob_id], content); @@ -71,10 +69,10 @@ export class DenoKVBlobService implements BlobService { return size; } - async persistReader(index: string, content: Deno.Reader): Promise{ + async persistReader(index: string, content: Deno.Reader): Promise { let i = 0; let size = 0; - for await (const chunk of iterateReader(content, {bufSize: this.blobSize})){ + for await (const chunk of iterateReader(content, { bufSize: this.blobSize })) { await this.unsafePersistBlob(index, i, chunk); size += chunk.length; i++; @@ -100,11 +98,11 @@ export class DenoKVBlobService implements BlobService { return await readAll(this.getBlob(index)); } - getBlob(index: string): Deno.Reader{ + getBlob(index: string): Deno.Reader { return readerFromIterable(this.iterBlob(index)); } - private async *iterRawBlobs(index: string): AsyncIterable{ + private async *iterRawBlobs(index: string): AsyncIterable { const prefix = ['blob_index', index]; for await (const itm of this.kv.list({ prefix })) { const foo = await this.kv.get(['blob', itm.value]); @@ -127,8 +125,7 @@ export class DenoKVBlobService implements BlobService { return ret; } - streamBlob(index: string): Deno.Reader{ - + streamBlob(index: string): Deno.Reader { return readerFromIterable(this.iterRawBlobs(index)); } } @@ -146,29 +143,29 @@ export class DenoKVFileService implements FileService { path: string, content: string | Uint8Array, ): Promise { - if (content instanceof Uint8Array){ + if (content instanceof Uint8Array) { return this.createFromReader(path, new Buffer(content)); } return this.createFromReader(path, new StringReader(content)); } - async createFromReader(path: string, content: Deno.Reader): Promise{ + async createFromReader(path: string, content: Deno.Reader): Promise { const newId = crypto.randomUUID(); - const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')).getReader() + const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')) + .getReader(); const size = await this.blobService.persistReader(newId, readerFromStreamReader(compressedContent)); const serializedFile: SerializedFile = { id: newId, path, contentSize: size, - compression: 'gzip' + compression: 'gzip', }; const idx: SerializedFilePathIndex = { fileId: newId }; await this.kv.set(['file', newId], serializedFile); await this.kv.set(['file_index_path', path], idx); - - return { id: newId, path, contentSize: size }; + return { id: newId, path, contentSize: size }; } async *list(): AsyncIterable { @@ -206,21 +203,19 @@ export class DenoKVFileService implements FileService { .delete(idx.key) .commit(); return true; - } + } - private async getSerializedFile(fileId: string): Promise{ + private async getSerializedFile(fileId: string): Promise { const ret = await this.kv.get(['file', fileId]); - return ret.value + return ret.value; } private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader { - if(serializedFile.compression == 'gzip'){ + if (serializedFile.compression == 'gzip') { const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader(); return readerFromStreamReader(ret); - } - return content; - + return content; } async getRawContentByPath(path: string): Promise { @@ -228,7 +223,6 @@ export class DenoKVFileService implements FileService { return ret === null ? null : readAll(ret); } - async getContentAsReader(path: string): Promise { const idx = await this.kv.get([ 'file_index_path', @@ -238,11 +232,10 @@ export class DenoKVFileService implements FileService { return null; } const ser = await this.kv.get(['file', idx.value.fileId]); - if(ser.value === null){ + if (ser.value === null) { return null; } return this.decompressContent(ser.value, this.blobService.getBlob(idx.value.fileId)); - } async getMetaDataByID(id: string): Promise { diff --git a/src/services/interfaces.ts b/src/services/interfaces.ts index d0b0b4a..a2a7537 100644 --- a/src/services/interfaces.ts +++ b/src/services/interfaces.ts @@ -4,24 +4,22 @@ export interface FileMetaData { contentSize: number; } - export interface FileService { - list(): AsyncIterable; createFromFullContent( path: string, content: string | Uint8Array, ): Promise; - + createFromReader(path: string, content: Deno.Reader): Promise; - + getRawContentByPath(path: string): Promise; getContentAsReader(path: string): Promise; getMetaDataByID(id: string): Promise; - + renameById(id: string, newPath: string): Promise; renameByPath( @@ -32,7 +30,6 @@ export interface FileService { deleteByPath(path: string): Promise; } - export interface BlobService { persistBlob(index: string, content: Uint8Array): Promise; persistReader(index: string, content: Deno.Reader): Promise; @@ -40,4 +37,4 @@ export interface BlobService { getBlob(index: string): Deno.Reader; getNumberOfBlobs(index: string): Promise; streamBlob(index: string): Deno.Reader; -} \ No newline at end of file +} diff --git a/src/services/mod.ts b/src/services/mod.ts index f7ea77c..e8d5a61 100644 --- a/src/services/mod.ts +++ b/src/services/mod.ts @@ -1,2 +1,2 @@ export * from './interfaces.ts'; -export {DenoKVBlobService, DenoKVFileService} from './denokv.ts'; \ No newline at end of file +export { DenoKVBlobService, DenoKVFileService } from './denokv.ts'; diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts index 18c670e..8991b06 100644 --- a/tests/fileservice_test.ts +++ b/tests/fileservice_test.ts @@ -1,142 +1,135 @@ -import { assertEquals } from "https://deno.land/std/testing/asserts.ts"; +import { assertEquals } from 'https://deno.land/std/testing/asserts.ts'; -import { - DenoKVBlobService, - DenoKVFileService, - FileMetaData, - FileService, -} from "../src/services/mod.ts"; -import { asyncIterableToArray } from "../src/utils.ts"; -import { generateLargeText } from "./utils.ts"; -import { StringReader } from "https://deno.land/std@0.191.0/io/mod.ts"; +import { DenoKVBlobService, DenoKVFileService, FileMetaData, FileService } from '../src/services/mod.ts'; +import { asyncIterableToArray } from '../src/utils.ts'; +import { generateLargeText } from './utils.ts'; +import { StringReader } from 'https://deno.land/std@0.191.0/io/mod.ts'; +Deno.test('DenoKVBlobService happy path', async (t) => { + localStorage.clear(); + const kv = await Deno.openKv(':memory:'); + const bs = new DenoKVBlobService(kv, 50); -Deno.test("DenoKVBlobService happy path", async (t) => { - localStorage.clear(); - const kv = await Deno.openKv(":memory:"); + function toUint8Array(s: string): Uint8Array { + return new TextEncoder().encode(s); + } - const bs = new DenoKVBlobService(kv, 50); + await t.step('read write small text', async () => { + const size = await bs.persistBlob('0002', toUint8Array('hello world')); + const content = await bs.readAllBlob('0002'); + assertEquals(content, toUint8Array('hello world')); + assertEquals(await bs.getNumberOfBlobs('0002'), 1); + assertEquals(size, 'hello world'.length); + }); - function toUint8Array(s: string): Uint8Array { - return new TextEncoder().encode(s); - } + await t.step('read write large text', async () => { + const largeText = toUint8Array(generateLargeText(33, 100)); + const size = await bs.persistBlob('0001', largeText); + const content = await bs.readAllBlob('0001'); + assertEquals(content, largeText); + assertEquals(await bs.getNumberOfBlobs('0001'), 21); + assertEquals(size, largeText.length); + }); - await t.step("read write small text", async () => { - const size = await bs.persistBlob("0002", toUint8Array("hello world")); - const content = await bs.readAllBlob("0002"); - assertEquals(content, toUint8Array("hello world")); - assertEquals(await bs.getNumberOfBlobs("0002"), 1); - assertEquals(size, "hello world".length); - }); + await t.step('read write large file', async () => { + const fileName = 'tests/examples/teapot.obj'; + const f = await Deno.open(fileName); + const size = await bs.persistReader('large-stream', f); - await t.step("read write large text", async () => { - const largeText = toUint8Array(generateLargeText(33, 100)); - const size = await bs.persistBlob("0001", largeText); - const content = await bs.readAllBlob("0001"); - assertEquals(content, largeText); - assertEquals(await bs.getNumberOfBlobs("0001"), 21); - assertEquals(size, largeText.length); - }); + assertEquals(await bs.getNumberOfBlobs('large-stream'), 1030); + assertEquals((await Deno.readFile(fileName)).length, size); + f.close(); + }); - await t.step("read write large file", async () => { - const fileName = 'tests/examples/teapot.obj'; - const f = await Deno.open(fileName); - const size = await bs.persistReader("large-stream", f); - - assertEquals(await bs.getNumberOfBlobs("large-stream"), 1030); - assertEquals((await Deno.readFile(fileName)).length, size); - f.close(); - }); - - kv.close(); + kv.close(); }); -Deno.test("DenoKVFileService happy path xxx ", async (t) => { - const kv = await Deno.openKv(":memory:"); - const fs: FileService = new DenoKVFileService(kv); - const fileName1 = "didier/file1.txt"; - const fileName2 = "didier/file2.txt"; - let file1: FileMetaData | undefined; - let file2: FileMetaData | undefined; +Deno.test('DenoKVFileService happy path xxx ', async (t) => { + const kv = await Deno.openKv(':memory:'); + const fs: FileService = new DenoKVFileService(kv); + const fileName1 = 'didier/file1.txt'; + const fileName2 = 'didier/file2.txt'; + let file1: FileMetaData | undefined; + let file2: FileMetaData | undefined; - await t.step("Create 2 files xyzabc", async () => { - file1 = await fs.createFromFullContent(fileName1, "bob le chef"); - file2 = await fs.createFromFullContent(fileName2, "le geant vert ☀️"); + await t.step('Create 2 files xyzabc', async () => { + file1 = await fs.createFromFullContent(fileName1, 'bob le chef'); + file2 = await fs.createFromFullContent(fileName2, 'le geant vert ☀️'); - assertEquals(file1.path, fileName1); - assertEquals(file2.contentSize, 47); - }); + assertEquals(file1.path, fileName1); + assertEquals(file2.contentSize, 47); + }); - await t.step("Get file metadata", async () => { - const ret = await fs.getMetaDataByID(file1!.id); - assertEquals(ret, file1); - }); + await t.step('Get file metadata', async () => { + const ret = await fs.getMetaDataByID(file1!.id); + assertEquals(ret, file1); + }); - await t.step("List files", async () => { - const expected = [file1, file2]; - const ret = await fs.list(); - assertEquals(await asyncIterableToArray(ret), expected); - }); + await t.step('List files', async () => { + const expected = [file1, file2]; + const ret = await fs.list(); + assertEquals(await asyncIterableToArray(ret), expected); + }); - await t.step("Get File by path", async () => { - const ret = await fs.getRawContentByPath(fileName1); - assertEquals(new TextDecoder().decode(ret!), "bob le chef"); - }); + await t.step('Get File by path', async () => { + const ret = await fs.getRawContentByPath(fileName1); + assertEquals(new TextDecoder().decode(ret!), 'bob le chef'); + }); - await t.step("Get File by Id", async () => { - const ret = await fs.getRawContentByPath(fileName1); - assertEquals(new TextDecoder().decode(ret!), "bob le chef"); - }); + await t.step('Get File by Id', async () => { + const ret = await fs.getRawContentByPath(fileName1); + assertEquals(new TextDecoder().decode(ret!), 'bob le chef'); + }); - await t.step("Delete File", async () => { - await fs.deleteByPath(fileName1); - await fs.deleteByPath("404 not found"); - }); + await t.step('Delete File', async () => { + await fs.deleteByPath(fileName1); + await fs.deleteByPath('404 not found'); + }); - await t.step("List after delete", async () => { - const expected = [file2]; - const ret = await fs.list(); - assertEquals(await asyncIterableToArray(ret), expected); - }); + await t.step('List after delete', async () => { + const expected = [file2]; + const ret = await fs.list(); + assertEquals(await asyncIterableToArray(ret), expected); + }); - await t.step("Create File from reader", async () => { - const newPath = "boris/streamed-file.txt"; + await t.step('Create File from reader', async () => { + const newPath = 'boris/streamed-file.txt'; - const ret = await fs.createFromReader(newPath, new StringReader("hello world 18")); - assertEquals(ret.contentSize, 40); + const ret = await fs.createFromReader(newPath, new StringReader('hello world 18')); + assertEquals(ret.contentSize, 40); - const s = await fs.getRawContentByPath(newPath); - assertEquals(new TextDecoder().decode(s!), "hello world 18") - }); + const s = await fs.getRawContentByPath(newPath); + assertEquals(new TextDecoder().decode(s!), 'hello world 18'); + }); - await t.step("Rename file", async () => { - const newPath = "boris/file2.txt"; - const ret = await fs.renameById(file2!.id, newPath); - assertEquals(ret?.path, newPath); + await t.step('Rename file', async () => { + const newPath = 'boris/file2.txt'; + const ret = await fs.renameById(file2!.id, newPath); + assertEquals(ret?.path, newPath); - assertEquals( - new TextDecoder().decode((await fs.getRawContentByPath(newPath))!), - 'le geant vert ☀️', - ); - }); + assertEquals( + new TextDecoder().decode((await fs.getRawContentByPath(newPath))!), + 'le geant vert ☀️', + ); + }); - kv.close(); + kv.close(); }); -Deno.test("DenoKVFileService happy path with byte array", async (t) => { - const kv = await Deno.openKv(":memory:"); - const fs: FileService = new DenoKVFileService(kv); - await t.step("Create and read 1 binary file", async () => { - const inputContent = new Uint8Array([240]); - const file = await fs.createFromFullContent("hello", inputContent); +Deno.test('DenoKVFileService happy path with byte array', async (t) => { + const kv = await Deno.openKv(':memory:'); + const fs: FileService = new DenoKVFileService(kv); + await t.step('Create and read 1 binary file', async () => { + const inputContent = new Uint8Array([240]); + const file = await fs.createFromFullContent('hello', inputContent); - assertEquals(file.path, "hello"); - assertEquals(file.contentSize, 27); + assertEquals(file.path, 'hello'); + assertEquals(file.contentSize, 27); - const persistedContent = await fs.getRawContentByPath("hello"); - assertEquals(persistedContent, inputContent); - }); + const persistedContent = await fs.getRawContentByPath('hello'); + assertEquals(persistedContent, inputContent); + }); - kv.close(); + kv.close(); }); diff --git a/tests/utils.ts b/tests/utils.ts index bfa07bb..6191b55 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,22 +1,25 @@ - -export function* randomNumbers(seed: number, max: number, maxResult: number|undefined = undefined) : Generator { +export function* randomNumbers( + seed: number, + max: number, + maxResult: number | undefined = undefined, +): Generator { const a = 48271; - const m = 2147483647; + const m = 2147483647; max = Math.floor(max); let i = 0; while (true) { - seed = (a * seed) % m; + seed = (a * seed) % m; yield seed % max; - i ++; - if(maxResult !== undefined && i >= maxResult){ - return; + i++; + if (maxResult !== undefined && i >= maxResult) { + return; } } } -export function range(min: number, max: number): number[]{ - const ret = [] - for(let i = min; i < max; i ++){ +export function range(min: number, max: number): number[] { + const ret = []; + for (let i = min; i < max; i++) { ret.push(i); } return ret; @@ -24,37 +27,36 @@ export function range(min: number, max: number): number[]{ export function generateLargeText(seed: number, size: number): string { const words = [ - "Norway", - "Germany", - "Ecuador", - "Chile", - "Colombia", - "Sri Lanka", - "Guinea", - "United Kingdom", - "Egypt", - "Palau", - "Latvia", - "Congo, Democratic Republic of the", - "Malawi", - "Senegal", - "Trinidad and Tobago", - "Guatemala", - "Chad", - "New Zealand", - "Malaysia", - "Lithuania", - "Taiwan", - "Solomon Islands", - "Grenada", - "Hungary", - "Korea, South", + 'Norway', + 'Germany', + 'Ecuador', + 'Chile', + 'Colombia', + 'Sri Lanka', + 'Guinea', + 'United Kingdom', + 'Egypt', + 'Palau', + 'Latvia', + 'Congo, Democratic Republic of the', + 'Malawi', + 'Senegal', + 'Trinidad and Tobago', + 'Guatemala', + 'Chad', + 'New Zealand', + 'Malaysia', + 'Lithuania', + 'Taiwan', + 'Solomon Islands', + 'Grenada', + 'Hungary', + 'Korea, South', ]; let ret: string[] = []; for (let i = 0; i < size; i++) { - seed = (seed * 4931 + 2753) % words.length; - ret.push(words[seed]); + seed = (seed * 4931 + 2753) % words.length; + ret.push(words[seed]); } - return ret.join(" "); - } - + return ret.join(' '); +}