First workable server.

src/fileservice.ts

@@ -1,7 +1,14 @@
-import { sha1 } from 'https://denopkg.com/chiefbiiko/sha1/mod.ts';
-import { iterateReader } from "https://deno.land/std@0.191.0/streams/iterate_reader.ts";
 import { readAll } from "https://deno.land/std@0.191.0/streams/read_all.ts";
-import { readerFromIterable } from "https://deno.land/std@0.144.0/streams/conversion.ts";
+import { readerFromIterable, iterateReader } from "https://deno.land/std@0.191.0/streams/mod.ts";
+import { Buffer } from "https://deno.land/std@0.191.0/io/buffer.ts";
+
+import { StringReader } from "https://deno.land/std@0.191.0/io/string_reader.ts";
+
+
+import { readerFromStreamReader, readableStreamFromReader } from "https://deno.land/std@0.191.0/streams/mod.ts";
+
+
+
 
 export interface FileService {
 
@@ -29,13 +36,14 @@ export interface FileService {
         newPath: string,
     ): Promise<FileMetaData | null>;
 
-    deleteByPath(path: string): Promise<void>;
+    deleteByPath(path: string): Promise<boolean>;
 }
 
 interface SerializedFile {
     id: string;
     path: string;
     bodySize: number;
+    compression: string;
 }
 
 export interface FileMetaData {
@@ -57,6 +65,14 @@ export class DenoKVBlobService {
         this.blobSize = blobSize;
     }
 
+    private async sha1(msgUint8: Uint8Array): Promise<string> {
+        const hashBuffer = await crypto.subtle.digest("SHA-1", msgUint8);
+        const hashArray = Array.from(new Uint8Array(hashBuffer));
+        return hashArray
+          .map((b) => b.toString(16).padStart(2, "0"))
+          .join(""); // convert bytes to hex string
+      }
+
     private *splitText(body: Uint8Array): Generator<Uint8Array> {
         let blobCount = 0;
         let txt = body.slice(
@@ -77,10 +93,10 @@ export class DenoKVBlobService {
         if(body.length > this.blobSize){
             throw new Error('Persisted a too large blob');
         }
-        const blob_id = sha1(body, 'utf8', 'hex'); // todo, should not be utf8
+        
+        const blob_id = await this.sha1(body);
         await this.kv.set(['blob_index', mainIndex, index], blob_id);
         await this.kv.set(['blob', blob_id], body);
-
     }
 
     async persistBlob(index: string, body: Uint8Array): Promise<number> {
@@ -98,7 +114,7 @@ export class DenoKVBlobService {
         let i = 0;
         let size = 0;
         for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
-            this.unsafePersistBlob(index, i, chunk);
+            await this.unsafePersistBlob(index, i, chunk);
             size += chunk.length;
             i++;
         }
@@ -162,36 +178,29 @@ export class DenoKVFileService implements FileService {
 
     constructor(kv: Deno.Kv) {
         this.kv = kv;
-        this.blobService = new DenoKVBlobService(kv, 500);
+        this.blobService = new DenoKVBlobService(kv, 32 * 1024);
     }
 
-    async createFromFullBody(
+    createFromFullBody(
         path: string,
         body: string | Uint8Array,
     ): Promise<FileMetaData> {
-        const newId = crypto.randomUUID();
-        const bodyBytes: Uint8Array = body instanceof Uint8Array ? body : new TextEncoder().encode(body);
-        const size = await this.blobService.persistBlob(newId, bodyBytes);
-
-        const serializedFile: SerializedFile = {
-            id: newId,
-            path,
-            bodySize: size,
-        };
-        const idx: SerializedFilePathIndex = { fileId: newId };
-        await this.kv.set(['file', newId], serializedFile);
-        await this.kv.set(['file_index_path', path], idx);
-        
-        return { id: newId, path, bodySize: size };
+        if (body instanceof Uint8Array){
+            return this.createFromReader(path, new Buffer(body));
+        }
+        return this.createFromReader(path, new StringReader(body));
     }
 
-    async createFromReader(path: string, body: Deno.Reader): Promise<FileMetaData>{
+    async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>{
         const newId = crypto.randomUUID();
-        const size = await this.blobService.persistReader(newId, body);
+
+        const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')).getReader()
+        const size = await this.blobService.persistReader(newId, readerFromStreamReader(compressedContent));
         const serializedFile: SerializedFile = {
             id: newId,
             path,
             bodySize: size,
+            compression: 'gzip'
         };
         const idx: SerializedFilePathIndex = { fileId: newId };
         await this.kv.set(['file', newId], serializedFile);
@@ -221,13 +230,13 @@ export class DenoKVFileService implements FileService {
         }
     }
 
-    async deleteByPath(path: string): Promise<void> {
+    async deleteByPath(path: string): Promise<boolean> {
         const idx = await this.kv.get<SerializedFilePathIndex>([
             'file_index_path',
             path,
         ]);
         if (idx.value === null) {
-            return;
+            return false;
         }
         // todo, this will fail silently if transaction fails.  we might want to put that into a loop.
         await this.kv.atomic()
@@ -235,6 +244,22 @@ export class DenoKVFileService implements FileService {
             .delete(['file_index_path', path])
             .delete(idx.key)
             .commit();
+        return true;
+    }    
+
+    private async getSerializedFile(fileId: string): Promise<SerializedFile | null>{
+        const ret = await this.kv.get<SerializedFile>(['file', fileId]);
+        return ret.value
+    }
+
+    private decompressBody(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
+        if(serializedFile.compression == 'gzip'){
+            const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader();
+            return readerFromStreamReader(ret);
+            
+        }
+        return content; 
+        
     }
 
     async getFullBodyByPath(path: string): Promise<string | null> {
@@ -243,14 +268,8 @@ export class DenoKVFileService implements FileService {
     }
 
     async getRawBodyByPath(path: string): Promise<Uint8Array | null> {
-        const idx = await this.kv.get<SerializedFilePathIndex>([
-            'file_index_path',
-            path,
-        ]);
-        if (idx.value === null) {
-            return null;
-        }
-        return this.blobService.readAllBlob(idx.value.fileId);
+        const ret = await this.getBodyAsReader(path);
+        return ret === null ? null : readAll(ret);
     }
 
 
@@ -262,19 +281,23 @@ export class DenoKVFileService implements FileService {
         if (idx.value === null) {
             return null;
         }
-        return this.blobService.getBlob(idx.value.fileId);
+        const ser = await this.kv.get<SerializedFile>(['file', idx.value.fileId]);
+        if(ser.value === null){
+            return null;
+        }
+        return this.decompressBody(ser.value, this.blobService.getBlob(idx.value.fileId));
 
     }
 
     async getMetaDataByID(id: string): Promise<FileMetaData | null> {
-        const fileObject = await this.kv.get<SerializedFile>(['file', id]);
-        if (fileObject.value === null) {
+        const fileObject = await this.getSerializedFile(id);
+        if (fileObject === null) {
             return null;
         }
         return {
-            path: fileObject.value.path,
-            id: fileObject.value.id,
-            bodySize: fileObject.value.bodySize,
+            path: fileObject.path,
+            id: fileObject.id,
+            bodySize: fileObject.bodySize,
         };
     }
 
@@ -287,10 +310,11 @@ export class DenoKVFileService implements FileService {
             'file_index_path',
             fileObject.value.path,
         ]);
-        const newFile: FileMetaData = {
+        const newFile: SerializedFile = {
             path: newPath,
             id: fileObject.value.id,
             bodySize: fileObject.value.bodySize,
+            compression: fileObject.value.compression,
         };
         await this.kv.atomic()
             .check(fileObject)

src/restapi.ts

@@ -1,10 +1,7 @@
 import { Context, Router } from "https://deno.land/x/oak/mod.ts";
 import { FileService } from "./fileservice.ts";
-import { readerFromStreamReader } from "https://deno.land/std@0.189.0/streams/mod.ts";
+import { readerFromStreamReader, readableStreamFromReader, TextLineStream } from "https://deno.land/std@0.191.0/streams/mod.ts";
 import { asyncIterableToArray } from "./utils.ts";
-import { StringReader } from "https://deno.land/std@0.178.0/io/string_reader.ts";
-import { isReader } from "https://deno.land/x/oak@v12.5.0/util.ts";
-import { createMockNext } from "https://deno.land/x/oak@v12.1.0/testing.ts";
 
 
 type Handler = (ctx: Context) => void;
@@ -17,7 +14,6 @@ function listFileHandlerFactory(fileService: FileService): Handler {
     };
 }
 
-
 function createFileHandlerFactory(fileService: FileService): Handler {
     return async (ctx: Context) => {
         const untypedCtx: any = ctx;
@@ -26,33 +22,43 @@ function createFileHandlerFactory(fileService: FileService): Handler {
 
         const reqBody = await ctx.request.body({ type: "stream" }).value;
         const reader = readerFromStreamReader(reqBody.getReader())
-        fileService.createFromReader(path, reader);
+        await fileService.createFromReader(path, reader);
     };
 }
 
-
 function getFileHandlerFactory(fileService: FileService): Handler {
     return async (ctx: Context) => {
         const untypedCtx: any = ctx;
-        const fileId = untypedCtx.params.file_id;
+        const fileId = untypedCtx.params.path;
         const reader = await fileService.getBodyAsReader(fileId);
         if(reader == null){
             ctx.response.status = 404;
             return;
         }
         ctx.response.status = 200;
-        ctx.response.body = reader;
-        ctx.response.type = "text/plain";
-        // ctx.send(reader)
+        ctx.response.body = reader; 
+    };
+}
 
+
+function deleteFileHandlerFactory(fileService: FileService): Handler {
+    return async (ctx: Context) => {
+        const untypedCtx: any = ctx;
+        const path = untypedCtx.params.path;
+        const isDeleted = await fileService.deleteByPath(path);
+        if(!isDeleted){
+            ctx.response.status = 404;
+            return;
+        }
+        ctx.response.status = 200;
     };
 }
 
 
 export function buildAPI(router: Router, fileService: FileService): Router {
-
     router.get('/files', listFileHandlerFactory(fileService));
-    router.get('/files/:file_id', getFileHandlerFactory(fileService));
+    router.get('/files/:path', getFileHandlerFactory(fileService));
+    router.delete('/file/:path', deleteFileHandlerFactory(fileService));
     router.post('/files', createFileHandlerFactory(fileService));
 
     return router;

src/utils.ts

@@ -18,7 +18,6 @@ export class AsyncIterableReader implements Deno.Reader {
     private readonly source: AsyncIterator<Uint8Array>;
     private interReadBuffer = new Uint8Array();
     private isSourceEmpty = false;
-    private shit = 0;
     constructor(source: AsyncIterable<Uint8Array>) {
         this.source = source[Symbol.asyncIterator]();
     }
@@ -58,8 +57,6 @@ export class AsyncIterableReader implements Deno.Reader {
     }
 
     async read(buffer: Uint8Array): Promise<number | null> {
-        this.shit += 1;
-        console.log("reading \> " + this.shit);
         if(this.isSourceEmpty && this.interReadBuffer.length < 1){
             return null;
         }