diff --git a/deno.json b/deno.json new file mode 100644 index 0000000..abca7a6 --- /dev/null +++ b/deno.json @@ -0,0 +1,10 @@ +{ + "fmt": { + "lineWidth": 120, + "indentWidth": 4, + "semiColons": true, + "singleQuote": true, + "proseWrap": "preserve", + "include": ["src/", "test/"] + } + } \ No newline at end of file diff --git a/src/fileservice.ts b/src/fileservice.ts index 949dd53..d2e87c0 100644 --- a/src/fileservice.ts +++ b/src/fileservice.ts @@ -1,58 +1,67 @@ -import { sha1 } from "https://denopkg.com/chiefbiiko/sha1/mod.ts"; - +import { sha1 } from 'https://denopkg.com/chiefbiiko/sha1/mod.ts'; export interface FileService { - createFromFullBody(path: string, body: string|Uint8Array): Promise; - list(): AsyncIterable; - deleteByPath(path: string): Promise; - getFullBodyByPath(path: string): Promise; - getRawBodyByPath(path: string): Promise; - getMetaDataByID(id: string): Promise; - renameById(id: string, newPath: string): Promise; - renameByPath(originalPath: string, newPath: string): Promise; + createFromFullBody( + path: string, + body: string | Uint8Array, + ): Promise; + list(): AsyncIterable; + deleteByPath(path: string): Promise; + getFullBodyByPath(path: string): Promise; + getRawBodyByPath(path: string): Promise; + getMetaDataByID(id: string): Promise; + renameById(id: string, newPath: string): Promise; + renameByPath( + originalPath: string, + newPath: string, + ): Promise; } interface SerializedFile { - id: string, - path: string, - bodySize: number, + id: string; + path: string; + bodySize: number; } export interface FileMetaData { - path: string, - id: string, - bodySize: number, + path: string; + id: string; + bodySize: number; } interface SerializedFilePathIndex { fileId: string; } - export class DenoKVBlobService { private readonly kv: Deno.Kv; private readonly blobSize: number; - constructor(kv: Deno.Kv, blobSize: number){ + constructor(kv: Deno.Kv, blobSize: number) { this.kv = kv; this.blobSize = blobSize; } - private *splitText(body: Uint8Array): Generator{ + private *splitText(body: Uint8Array): Generator { let blobCount = 0; - let txt = body.slice(blobCount * this.blobSize, (blobCount + 1) * this.blobSize); - while(txt.length > 0 ){ + let txt = body.slice( + blobCount * this.blobSize, + (blobCount + 1) * this.blobSize, + ); + while (txt.length > 0) { yield txt; blobCount += 1; - txt = body.slice(blobCount * this.blobSize, (blobCount + 1) * this.blobSize); + txt = body.slice( + blobCount * this.blobSize, + (blobCount + 1) * this.blobSize, + ); } - } - async persistBlob(index: string, body: Uint8Array): Promise{ + async persistBlob(index: string, body: Uint8Array): Promise { let blobCount = 0; - for(const txt of this.splitText(body)){ - const blob_id = sha1(txt, "utf8", "hex"); + for (const txt of this.splitText(body)) { + const blob_id = sha1(txt, 'utf8', 'hex'); await this.kv.set(['blob_index', index, blobCount], blob_id); await this.kv.set(['blob', blob_id], txt); blobCount += 1; @@ -67,88 +76,115 @@ export class DenoKVBlobService { return mergedArray; } - async getBlob(index: string): Promise{ + async getBlob(index: string): Promise { let ret = new Uint8Array(); - for await(const itm of this.kv.list({prefix: ['blob_index', index]})){ - const foo = await this.kv.get(['blob', itm.value]) - if(foo.value === null){ - throw new Error(`Inconsistant blob index for key "${foo.value}", this is very bad.`); + for await ( + const itm of this.kv.list({ prefix: ['blob_index', index] }) + ) { + const foo = await this.kv.get(['blob', itm.value]); + if (foo.value === null) { + throw new Error( + `Inconsistant blob index for key "${foo.value}", this is very bad.`, + ); } ret = this.concatUint8Array(ret, foo.value); } return ret; } - async getNumberOfBlobs(index: string): Promise{ + async getNumberOfBlobs(index: string): Promise { let ret = 0; - for await(const _itm of this.kv.list({prefix: ['blob_index', index]})){ + for await ( + const _itm of this.kv.list({ prefix: ['blob_index', index] }) + ) { ret += 1; } return ret; - } - - } export class DenoKVFileService implements FileService { private readonly kv: Deno.Kv; private readonly blobService: DenoKVBlobService; - constructor(kv: Deno.Kv){ + constructor(kv: Deno.Kv) { this.kv = kv; this.blobService = new DenoKVBlobService(kv, 500); } - async createFromFullBody(path: string, body: string|Uint8Array): Promise { + async createFromFullBody( + path: string, + body: string | Uint8Array, + ): Promise { const newId = crypto.randomUUID(); - const serializedFile: SerializedFile = {id: newId, path, bodySize: body.length}; - const idx: SerializedFilePathIndex = {fileId: newId}; + const serializedFile: SerializedFile = { + id: newId, + path, + bodySize: body.length, + }; + const idx: SerializedFilePathIndex = { fileId: newId }; await this.kv.set(['file', newId], serializedFile); await this.kv.set(['file_index_path', path], idx); - const bodyBytes: Uint8Array = body instanceof Uint8Array ? body: new TextEncoder().encode(body); + const bodyBytes: Uint8Array = body instanceof Uint8Array ? body : new TextEncoder().encode(body); await this.blobService.persistBlob(newId, bodyBytes); - return {id: newId, path, bodySize: body.length}; + return { id: newId, path, bodySize: body.length }; } - async* list(): AsyncIterable { - for await (const entry of this.kv.list({ prefix: ["file_index_path"] })) { - const ret = await this.kv.get(['file', entry.value.fileId]); - if(ret.value !== null) { - yield {path: ret.value.path, id: ret.value.id, bodySize: ret.value.bodySize}; + async *list(): AsyncIterable { + for await ( + const entry of this.kv.list({ + prefix: ['file_index_path'], + }) + ) { + const ret = await this.kv.get([ + 'file', + entry.value.fileId, + ]); + if (ret.value !== null) { + yield { + path: ret.value.path, + id: ret.value.id, + bodySize: ret.value.bodySize, + }; } } } async deleteByPath(path: string): Promise { - const idx = await this.kv.get(["file_index_path", path]); - if(idx.value === null){ + const idx = await this.kv.get([ + 'file_index_path', + path, + ]); + if (idx.value === null) { return; } // todo, this will fail silently if transaction fails. we might want to put that into a loop. await this.kv.atomic() .check(idx) - .delete(["file_index_path", path]) + .delete(['file_index_path', path]) .delete(idx.key) .commit(); } - async getFullBodyByPath(path: string): Promise { + async getFullBodyByPath(path: string): Promise { const ret = await this.getRawBodyByPath(path); return ret == null ? null : new TextDecoder().decode(ret); } - async getRawBodyByPath(path: string): Promise { - const idx = await this.kv.get(["file_index_path", path]); - if(idx.value === null){ + async getRawBodyByPath(path: string): Promise { + const idx = await this.kv.get([ + 'file_index_path', + path, + ]); + if (idx.value === null) { return null; } return this.blobService.getBlob(idx.value.fileId); } - async getMetaDataByID(id: string): Promise{ - const fileObject = await this.kv.get(['file', id]) - if(fileObject.value === null){ + async getMetaDataByID(id: string): Promise { + const fileObject = await this.kv.get(['file', id]); + if (fileObject.value === null) { return null; } return { @@ -158,27 +194,39 @@ export class DenoKVFileService implements FileService { }; } - async renameById(id: string, newPath: string): Promise { + async renameById(id: string, newPath: string): Promise { const fileObject = await this.kv.get(['file', id]); - if(fileObject.value === null){ + if (fileObject.value === null) { return null; } - const idx = await this.kv.get(["file_index_path", fileObject.value.path]); - const newFile: FileMetaData = {path: newPath, id: fileObject.value.id, bodySize: fileObject.value.bodySize}; + const idx = await this.kv.get([ + 'file_index_path', + fileObject.value.path, + ]); + const newFile: FileMetaData = { + path: newPath, + id: fileObject.value.id, + bodySize: fileObject.value.bodySize, + }; await this.kv.atomic() .check(fileObject) .check(idx) .delete(idx.key) - .set(["file_index_path", newPath], idx.value) + .set(['file_index_path', newPath], idx.value) .set(fileObject.key, newFile) - .commit() + .commit(); return this.getMetaDataByID(id); } - async renameByPath(originalPath: string, newPath: string): Promise { - - const idx = await this.kv.get(["file_index_path", originalPath]); - if(idx.value == null){ + async renameByPath( + originalPath: string, + newPath: string, + ): Promise { + const idx = await this.kv.get([ + 'file_index_path', + originalPath, + ]); + if (idx.value == null) { return null; } return this.renameById(idx.value.fileId, newPath); diff --git a/src/server.ts b/src/server.ts index d3624c7..5645be3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,9 +1,9 @@ -import { Application } from "https://deno.land/x/oak/mod.ts"; +import { Application } from 'https://deno.land/x/oak/mod.ts'; const app = new Application(); app.use((ctx) => { - ctx.response.body = "Hello World!"; + ctx.response.body = 'Hello World!'; }); await app.listen({ port: 9000 }); diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts index f4c06f4..2412944 100644 --- a/tests/fileservice_test.ts +++ b/tests/fileservice_test.ts @@ -1,6 +1,11 @@ import { assertEquals } from "https://deno.land/std/testing/asserts.ts"; -import {DenoKVFileService, FileMetaData, DenoKVBlobService, FileService} from "../src/fileservice.ts"; +import { + DenoKVBlobService, + DenoKVFileService, + FileMetaData, + FileService, +} from "../src/fileservice.ts"; function generateLargeText(seed: number, size: number): string { const words = [ @@ -28,128 +33,125 @@ function generateLargeText(seed: number, size: number): string { "Solomon Islands", "Grenada", "Hungary", - "Korea, South" - ] - let ret: string[] = []; - for(let i = 0; i < size; i ++){ - seed = (seed * 4931 + 2753) % words.length; - ret.push(words[seed]); - } - return ret.join(' '); + "Korea, South", + ]; + let ret: string[] = []; + for (let i = 0; i < size; i++) { + seed = (seed * 4931 + 2753) % words.length; + ret.push(words[seed]); + } + return ret.join(" "); } -async function asyncIterableToArray(ite: AsyncIterable): Promise{ - const ret: T[] = []; - for await(const e of ite){ - ret.push(e); - } - return ret; - +async function asyncIterableToArray(ite: AsyncIterable): Promise { + const ret: T[] = []; + for await (const e of ite) { + ret.push(e); + } + return ret; } Deno.test("DenoKVBlobService happy path", async (t) => { - localStorage.clear(); - const kv = await Deno.openKv(":memory:"); - - const bs = new DenoKVBlobService(kv, 50); + localStorage.clear(); + const kv = await Deno.openKv(":memory:"); - function toUint8Array(s: string): Uint8Array{ - return new TextEncoder().encode(s); - } + const bs = new DenoKVBlobService(kv, 50); - await t.step("read write small text", async () => { - await bs.persistBlob('0002', toUint8Array('hello world')); - const body = await bs.getBlob('0002'); - assertEquals(body, toUint8Array('hello world')); - assertEquals(await bs.getNumberOfBlobs('0002'), 1); - }); + function toUint8Array(s: string): Uint8Array { + return new TextEncoder().encode(s); + } - await t.step("read write large text", async () => { - const largeText = toUint8Array(generateLargeText(33, 100)); - await bs.persistBlob('0001', largeText); - const body = await bs.getBlob('0001'); - assertEquals(body, largeText); - assertEquals(await bs.getNumberOfBlobs('0001'), 21); - }); + await t.step("read write small text", async () => { + await bs.persistBlob("0002", toUint8Array("hello world")); + const body = await bs.getBlob("0002"); + assertEquals(body, toUint8Array("hello world")); + assertEquals(await bs.getNumberOfBlobs("0002"), 1); + }); - kv.close(); + await t.step("read write large text", async () => { + const largeText = toUint8Array(generateLargeText(33, 100)); + await bs.persistBlob("0001", largeText); + const body = await bs.getBlob("0001"); + assertEquals(body, largeText); + assertEquals(await bs.getNumberOfBlobs("0001"), 21); + }); + kv.close(); }); Deno.test("DenoKVFileService happy path", async (t) => { - const kv = await Deno.openKv(":memory:"); - const fs: FileService = new DenoKVFileService(kv); - const fileName1 = "didier/file1.txt"; - const fileName2 = "didier/file2.txt"; - let file1 : FileMetaData|undefined; - let file2: FileMetaData|undefined; + const kv = await Deno.openKv(":memory:"); + const fs: FileService = new DenoKVFileService(kv); + const fileName1 = "didier/file1.txt"; + const fileName2 = "didier/file2.txt"; + let file1: FileMetaData | undefined; + let file2: FileMetaData | undefined; + await t.step("Create 2 files", async () => { + file1 = await fs.createFromFullBody(fileName1, "bob le chef"); + file2 = await fs.createFromFullBody(fileName2, "le geant vert ☀️"); - await t.step("Create 2 files", async () => { - file1 = await fs.createFromFullBody(fileName1, "bob le chef"); - file2 = await fs.createFromFullBody(fileName2, "le geant vert ☀️"); + assertEquals(file1.path, fileName1); + assertEquals(file2.bodySize, "le geant vert ".length + 2); + }); - assertEquals(file1.path, fileName1); - assertEquals(file2.bodySize, "le geant vert ".length + 2); - }); + await t.step("Get file metadata", async () => { + const ret = await fs.getMetaDataByID(file1?.id ?? "404"); + assertEquals(ret, file1); + }); - await t.step("Get file metadata", async () => { - const ret = await fs.getMetaDataByID(file1?.id ?? '404'); - assertEquals(ret, file1) - }); + await t.step("List files", async () => { + const expected = [file1, file2]; + const ret = await fs.list(); + assertEquals(await asyncIterableToArray(ret), expected); + }); - await t.step("List files", async () => { - const expected = [file1, file2]; - const ret = await fs.list(); - assertEquals(await asyncIterableToArray(ret), expected); - }); + await t.step("Get File by path", async () => { + assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef"); + }); - await t.step("Get File by path", async () => { - assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef"); - }); + await t.step("Get File by Id", async () => { + assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef"); + }); - await t.step("Get File by Id", async () => { - assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef"); - }); + await t.step("Delete File", async () => { + await fs.deleteByPath(fileName1); + await fs.deleteByPath("404 not found"); + }); - await t.step("Delete File", async () => { - await fs.deleteByPath(fileName1); - await fs.deleteByPath("404 not found"); - }); + await t.step("List after delete", async () => { + const expected = [file2]; + const ret = await fs.list(); + assertEquals(await asyncIterableToArray(ret), expected); + }); - await t.step("List after delete", async () => { - const expected = [file2]; - const ret = await fs.list(); - assertEquals(await asyncIterableToArray(ret), expected); - }); + await t.step("Rename file", async () => { + const newPath = "boris/file2.txt"; + const ret = await fs.renameById(file2?.id ?? "404", newPath); + assertEquals(ret?.path, newPath); - await t.step("Rename file", async () => { - const newPath = "boris/file2.txt"; - const ret = await fs.renameById(file2?.id ?? '404', newPath); - assertEquals(ret?.path, newPath); + assertEquals( + (await fs.getFullBodyByPath(newPath))?.length, + file2?.bodySize ?? -1, + ); + }); - assertEquals((await fs.getFullBodyByPath(newPath))?.length, file2?.bodySize?? -1); - }); - - kv.close(); + kv.close(); }); - Deno.test("DenoKVFileService happy path with byte array", async (t) => { - const kv = await Deno.openKv(":memory:"); - const fs: FileService = new DenoKVFileService(kv); - await t.step("Create and read 1 binary file", async () => { - const inputBody = new Uint8Array([240]); - const file = await fs.createFromFullBody('hello', inputBody); + const kv = await Deno.openKv(":memory:"); + const fs: FileService = new DenoKVFileService(kv); + await t.step("Create and read 1 binary file", async () => { + const inputBody = new Uint8Array([240]); + const file = await fs.createFromFullBody("hello", inputBody); - assertEquals(file.path, 'hello'); - assertEquals(file.bodySize, inputBody.length); + assertEquals(file.path, "hello"); + assertEquals(file.bodySize, inputBody.length); - const bodyString = await fs.getFullBodyByPath('hello'); - assertEquals(bodyString, new TextDecoder().decode(inputBody)); - }); + const bodyString = await fs.getFullBodyByPath("hello"); + assertEquals(bodyString, new TextDecoder().decode(inputBody)); + }); - - kv.close(); - -}); \ No newline at end of file + kv.close(); +});