didier/3dverse-testtask
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored a bit the code; the original code was too experimental in style.

Browse Source
This commit is contained in:
didier amyot committed 2023-06-13 08:00:01 -04:00
1 parent f7a8d494f5
commit bbd705ee3b
3 files changed
+55 -60
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/fileservice.ts
+33 -40
View File
@@ -14,18 +14,16 @@ export interface FileService {
list(): AsyncIterable<FileMetaData>;
createFromFullBody(
createFromFullContent(
path: string,
body: string | Uint8Array | Deno.Reader,
content: string | Uint8Array,
): Promise<FileMetaData>;
createFromReader(path: string, body: Deno.Reader): Promise<FileMetaData>;
createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>;
getFullBodyByPath(path: string): Promise<string | null>;
getRawContentByPath(path: string): Promise<Uint8Array | null>;
getRawBodyByPath(path: string): Promise<Uint8Array | null>;
getBodyAsReader(path: string): Promise<Deno.Reader | null>;
getContentAsReader(path: string): Promise<Deno.Reader | null>;
getMetaDataByID(id: string): Promise<FileMetaData | null>;
@@ -42,14 +40,14 @@ export interface FileService {
interface SerializedFile {
id: string;
path: string;
bodySize: number;
contentSize: number;
compression: string;
}
export interface FileMetaData {
path: string;
id: string;
bodySize: number;
contentSize: number;
}
interface SerializedFilePathIndex {
@@ -73,36 +71,36 @@ export class DenoKVBlobService {
.join(""); // convert bytes to hex string
}
private *splitText(body: Uint8Array): Generator<Uint8Array> {
private *splitText(content: Uint8Array): Generator<Uint8Array> {
let blobCount = 0;
let txt = body.slice(
let txt = content.slice(
blobCount * this.blobSize,
(blobCount + 1) * this.blobSize,
);
while (txt.length > 0) {
yield txt;
blobCount += 1;
txt = body.slice(
txt = content.slice(
blobCount * this.blobSize,
(blobCount + 1) * this.blobSize,
);
}
}
private async unsafePersistBlob(mainIndex: string, index: number, body: Uint8Array): Promise<void> {
if(body.length > this.blobSize){
private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise<void> {
if(content.length > this.blobSize){
throw new Error('Persisted a too large blob');
}
const blob_id = await this.sha1(body);
const blob_id = await this.sha1(content);
await this.kv.set(['blob_index', mainIndex, index], blob_id);
await this.kv.set(['blob', blob_id], body);
await this.kv.set(['blob', blob_id], content);
}
async persistBlob(index: string, body: Uint8Array): Promise<number> {
async persistBlob(index: string, content: Uint8Array): Promise<number> {
let blobCount = 0;
let size = 0;
for (const txt of this.splitText(body)) {
for (const txt of this.splitText(content)) {
await this.unsafePersistBlob(index, blobCount, txt);
size += txt.length;
blobCount += 1;
@@ -110,10 +108,10 @@ export class DenoKVBlobService {
return size;
}
async persistReader(index: string, body: Deno.Reader): Promise<number>{
async persistReader(index: string, content: Deno.Reader): Promise<number>{
let i = 0;
let size = 0;
for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
for await (const chunk of iterateReader(content, {bufSize: this.blobSize})){
await this.unsafePersistBlob(index, i, chunk);
size += chunk.length;
i++;
@@ -181,14 +179,14 @@ export class DenoKVFileService implements FileService {
this.blobService = new DenoKVBlobService(kv, 32 * 1024);
}
createFromFullBody(
createFromFullContent(
path: string,
body: string | Uint8Array,
content: string | Uint8Array,
): Promise<FileMetaData> {
if (body instanceof Uint8Array){
return this.createFromReader(path, new Buffer(body));
if (content instanceof Uint8Array){
return this.createFromReader(path, new Buffer(content));
}
return this.createFromReader(path, new StringReader(body));
return this.createFromReader(path, new StringReader(content));
}
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>{
@@ -199,14 +197,14 @@ export class DenoKVFileService implements FileService {
const serializedFile: SerializedFile = {
id: newId,
path,
bodySize: size,
contentSize: size,
compression: 'gzip'
};
const idx: SerializedFilePathIndex = { fileId: newId };
await this.kv.set(['file', newId], serializedFile);
await this.kv.set(['file_index_path', path], idx);
return { id: newId, path, bodySize: size };
return { id: newId, path, contentSize: size };
}
@@ -224,7 +222,7 @@ export class DenoKVFileService implements FileService {
yield {
path: ret.value.path,
id: ret.value.id,
bodySize: ret.value.bodySize,
contentSize: ret.value.contentSize,
};
}
}
@@ -252,7 +250,7 @@ export class DenoKVFileService implements FileService {
return ret.value
}
private decompressBody(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
if(serializedFile.compression == 'gzip'){
const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader();
return readerFromStreamReader(ret);
@@ -262,18 +260,13 @@ export class DenoKVFileService implements FileService {
}
async getFullBodyByPath(path: string): Promise<string | null> {
const ret = await this.getRawBodyByPath(path);
return ret == null ? null : new TextDecoder().decode(ret);
}
async getRawBodyByPath(path: string): Promise<Uint8Array | null> {
const ret = await this.getBodyAsReader(path);
async getRawContentByPath(path: string): Promise<Uint8Array | null> {
const ret = await this.getContentAsReader(path);
return ret === null ? null : readAll(ret);
}
async getBodyAsReader(path: string): Promise<Deno.Reader | null> {
async getContentAsReader(path: string): Promise<Deno.Reader | null> {
const idx = await this.kv.get<SerializedFilePathIndex>([
'file_index_path',
path,
@@ -285,7 +278,7 @@ export class DenoKVFileService implements FileService {
if(ser.value === null){
return null;
}
return this.decompressBody(ser.value, this.blobService.getBlob(idx.value.fileId));
return this.decompressContent(ser.value, this.blobService.getBlob(idx.value.fileId));
}
@@ -297,7 +290,7 @@ export class DenoKVFileService implements FileService {
return {
path: fileObject.path,
id: fileObject.id,
bodySize: fileObject.bodySize,
contentSize: fileObject.contentSize,
};
}
@@ -313,7 +306,7 @@ export class DenoKVFileService implements FileService {
const newFile: SerializedFile = {
path: newPath,
id: fileObject.value.id,
bodySize: fileObject.value.bodySize,
contentSize: fileObject.value.contentSize,
compression: fileObject.value.compression,
};
await this.kv.atomic()
src/restapi.ts
+1 -1
View File
@@ -30,7 +30,7 @@ function getFileHandlerFactory(fileService: FileService): Handler {
return async (ctx: Context) => {
const untypedCtx: any = ctx;
const fileId = untypedCtx.params.path;
const reader = await fileService.getBodyAsReader(fileId);
const reader = await fileService.getContentAsReader(fileId);
if(reader == null){
ctx.response.status = 404;
return;
tests/fileservice_test.ts
+21 -19
View File
@@ -24,8 +24,8 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
await t.step("read write small text", async () => {
const size = await bs.persistBlob("0002", toUint8Array("hello world"));
const body = await bs.readAllBlob("0002");
assertEquals(body, toUint8Array("hello world"));
const content = await bs.readAllBlob("0002");
assertEquals(content, toUint8Array("hello world"));
assertEquals(await bs.getNumberOfBlobs("0002"), 1);
assertEquals(size, "hello world".length);
});
@@ -33,8 +33,8 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
await t.step("read write large text", async () => {
const largeText = toUint8Array(generateLargeText(33, 100));
const size = await bs.persistBlob("0001", largeText);
const body = await bs.readAllBlob("0001");
assertEquals(body, largeText);
const content = await bs.readAllBlob("0001");
assertEquals(content, largeText);
assertEquals(await bs.getNumberOfBlobs("0001"), 21);
assertEquals(size, largeText.length);
});
@@ -61,15 +61,15 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => {
let file2: FileMetaData | undefined;
await t.step("Create 2 files xyzabc", async () => {
file1 = await fs.createFromFullBody(fileName1, "bob le chef");
file2 = await fs.createFromFullBody(fileName2, "le geant vert ☀️");
file1 = await fs.createFromFullContent(fileName1, "bob le chef");
file2 = await fs.createFromFullContent(fileName2, "le geant vert ☀️");
assertEquals(file1.path, fileName1);
assertEquals(file2.bodySize, 47);
assertEquals(file2.contentSize, 47);
});
await t.step("Get file metadata", async () => {
const ret = await fs.getMetaDataByID(file1?.id ?? "404");
const ret = await fs.getMetaDataByID(file1!.id);
assertEquals(ret, file1);
});
@@ -80,11 +80,13 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => {
});
await t.step("Get File by path", async () => {
assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef");
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), "bob le chef");
});
await t.step("Get File by Id", async () => {
assertEquals(await fs.getFullBodyByPath(fileName1), "bob le chef");
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), "bob le chef");
});
await t.step("Delete File", async () => {
@@ -102,19 +104,19 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => {
const newPath = "boris/streamed-file.txt";
const ret = await fs.createFromReader(newPath, new StringReader("hello world 18"));
assertEquals(ret.bodySize, 40);
assertEquals(ret.contentSize, 40);
const s = await fs.getRawBodyByPath(newPath);
const s = await fs.getRawContentByPath(newPath);
assertEquals(new TextDecoder().decode(s!), "hello world 18")
});
await t.step("Rename file", async () => {
const newPath = "boris/file2.txt";
const ret = await fs.renameById(file2?.id ?? "404", newPath);
const ret = await fs.renameById(file2!.id, newPath);
assertEquals(ret?.path, newPath);
assertEquals(
await fs.getFullBodyByPath(newPath),
new TextDecoder().decode((await fs.getRawContentByPath(newPath))!),
'le geant vert ☀️',
);
});
@@ -126,14 +128,14 @@ Deno.test("DenoKVFileService happy path with byte array", async (t) => {
const kv = await Deno.openKv(":memory:");
const fs: FileService = new DenoKVFileService(kv);
await t.step("Create and read 1 binary file", async () => {
const inputBody = new Uint8Array([240]);
const file = await fs.createFromFullBody("hello", inputBody);
const inputContent = new Uint8Array([240]);
const file = await fs.createFromFullContent("hello", inputContent);
assertEquals(file.path, "hello");
assertEquals(file.bodySize, 27);
assertEquals(file.contentSize, 27);
const bodyString = await fs.getFullBodyByPath("hello");
assertEquals(bodyString, new TextDecoder().decode(inputBody));
const persistedContent = await fs.getRawContentByPath("hello");
assertEquals(persistedContent, inputContent);
});
kv.close();