Started to implement server

src/fileservice.ts

@@ -1,22 +1,31 @@
 import { sha1 } from 'https://denopkg.com/chiefbiiko/sha1/mod.ts';
 import { iterateReader } from "https://deno.land/std@0.191.0/streams/iterate_reader.ts";
+import { AsyncIterableReader, concatUint8Array } from "./utils.ts";
 
 
 export interface FileService {
+
+    list(): AsyncIterable<FileMetaData>;
+
     createFromFullBody(
         path: string,
-        body: string | Uint8Array,
+        body: string | Uint8Array | Deno.Reader,
     ): Promise<FileMetaData>;
-    list(): AsyncIterable<FileMetaData>;
-    deleteByPath(path: string): Promise<void>;
+    createFromReader(path: string, body: Deno.Reader): Promise<FileMetaData>;
+    
+    
     getFullBodyByPath(path: string): Promise<string | null>;
     getRawBodyByPath(path: string): Promise<Uint8Array | null>;
+
     getMetaDataByID(id: string): Promise<FileMetaData | null>;
+    
     renameById(id: string, newPath: string): Promise<FileMetaData | null>;
     renameByPath(
         originalPath: string,
         newPath: string,
     ): Promise<FileMetaData | null>;
+
+    deleteByPath(path: string): Promise<void>;
 }
 
 interface SerializedFile {
@@ -70,28 +79,26 @@ export class DenoKVBlobService {
 
     }
 
-    async persistBlob(index: string, body: Uint8Array): Promise<void> {
+    async persistBlob(index: string, body: Uint8Array): Promise<number> {
         let blobCount = 0;
+        let size = 0;
         for (const txt of this.splitText(body)) {
             await this.unsafePersistBlob(index, blobCount, txt);
+            size += txt.length;
             blobCount += 1;
         }
+        return size;
     }
 
-    async persistReader(index: string, body: Deno.Reader): Promise<void>{
+    async persistReader(index: string, body: Deno.Reader): Promise<number>{
         let i = 0;
+        let size = 0;
         for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
             this.unsafePersistBlob(index, i, chunk);
+            size += chunk.length;
             i++;
         }
-    }
-
-    // todo: remove
-    private concatUint8Array(arr1: Uint8Array, arr2: Uint8Array): Uint8Array {
-        const mergedArray = new Uint8Array(arr1.length + arr2.length);
-        mergedArray.set(arr1);
-        mergedArray.set(arr2, arr1.length);
-        return mergedArray;
+        return size;
     }
 
     async getBlob(index: string): Promise<Uint8Array> {
@@ -105,11 +112,25 @@ export class DenoKVBlobService {
                     `Inconsistant blob index for key "${foo.value}", this is very bad.`,
                 );
             }
-            ret = this.concatUint8Array(ret, foo.value);
+            ret = concatUint8Array(ret, foo.value);
         }
         return ret;
     }
 
+    private async *iterRawBlobs(index: string): AsyncIterable<Uint8Array>{
+        const prefix = ['blob_index', index];
+        for await (const itm of this.kv.list<string>({ prefix })) {
+            const foo = await this.kv.get<Uint8Array>(['blob', itm.value]);
+            if (foo.value === null) {
+                throw new Error(
+                    `Inconsistant blob index for key "${foo.value}", this is very bad.`,
+                );
+            }
+            yield foo.value;
+
+        }
+
+}
     async getNumberOfBlobs(index: string): Promise<number> {
         let ret = 0;
         for await (
@@ -119,6 +140,10 @@ export class DenoKVBlobService {
         }
         return ret;
     }
+
+    streamBlob(index: string): Deno.Reader{
+        return new AsyncIterableReader(this.iterRawBlobs(index));
+    }
 }
 
 export class DenoKVFileService implements FileService {
@@ -135,17 +160,35 @@ export class DenoKVFileService implements FileService {
         body: string | Uint8Array,
     ): Promise<FileMetaData> {
         const newId = crypto.randomUUID();
+        const bodyBytes: Uint8Array = body instanceof Uint8Array ? body : new TextEncoder().encode(body);
+        const size = await this.blobService.persistBlob(newId, bodyBytes);
+
         const serializedFile: SerializedFile = {
             id: newId,
             path,
-            bodySize: body.length,
+            bodySize: size,
         };
         const idx: SerializedFilePathIndex = { fileId: newId };
         await this.kv.set(['file', newId], serializedFile);
         await this.kv.set(['file_index_path', path], idx);
-        const bodyBytes: Uint8Array = body instanceof Uint8Array ? body : new TextEncoder().encode(body);
-        await this.blobService.persistBlob(newId, bodyBytes);
-        return { id: newId, path, bodySize: body.length };
+        
+        return { id: newId, path, bodySize: size };
+    }
+
+    async createFromReader(path: string, body: Deno.Reader): Promise<FileMetaData>{
+        const newId = crypto.randomUUID();
+        const size = await this.blobService.persistReader(newId, body);
+        const serializedFile: SerializedFile = {
+            id: newId,
+            path,
+            bodySize: size,
+        };
+        const idx: SerializedFilePathIndex = { fileId: newId };
+        await this.kv.set(['file', newId], serializedFile);
+        await this.kv.set(['file_index_path', path], idx);
+        
+        return { id: newId, path, bodySize: size };
+
     }
 
     async *list(): AsyncIterable<FileMetaData> {

src/restapi.ts

@@ -0,0 +1,48 @@
+import { Context, Router } from "https://deno.land/x/oak/mod.ts";
+import { FileService } from "./fileservice.ts";
+import { readerFromStreamReader } from "https://deno.land/std@0.189.0/streams/mod.ts";
+import { asyncIterableToArray } from "./utils.ts";
+
+
+
+type Handler = (ctx: Context) => void;
+
+
+function listFileHandlerFactory(fileService: FileService): Handler {
+    return async (ctx: Context) => {
+        ctx.response.type = "application/json";
+        ctx.response.body = await asyncIterableToArray(fileService.list());
+    };
+}
+
+
+function createFileHandlerFactory(fileService: FileService): Handler {
+    return async (ctx: Context) => {
+        const untypedCtx: any = ctx;
+        ctx.response.body = 'This is a post';
+        const path = untypedCtx.request.url.searchParams.get('path');
+
+        const reqBody = await ctx.request.body({ type: "stream" }).value;
+        const reader = readerFromStreamReader(reqBody.getReader())
+        fileService.createFromReader(path, reader);
+    };
+}
+
+
+function getFileHandlerFactory(fileService: FileService): Handler {
+    return (ctx: Context) => {
+        const untypedCtx: any = ctx;
+        const fileId = untypedCtx.params.file_id;
+        // ctx.response.body = await fileService.getRawBodyByPath()
+    };
+}
+
+
+export function buildAPI(router: Router, fileService: FileService): Router {
+
+    router.get('/files', listFileHandlerFactory(fileService));
+    router.get('/files/:file_id', getFileHandlerFactory(fileService));
+    router.post('/files', createFileHandlerFactory(fileService));
+
+    return router;
+}

src/server.ts

@@ -1,9 +1,18 @@
-import { Application } from 'https://deno.land/x/oak/mod.ts';
+import { Application, Router } from 'https://deno.land/x/oak/mod.ts';
+import { buildAPI } from './restapi.ts';
+import { DenoKVFileService } from "./fileservice.ts";
 
-const app = new Application();
+async function main(){
+    const app = new Application();
+    const kv = await Deno.openKv();
+    const fileService = new DenoKVFileService(kv);
+    const router = new Router({ prefix: '/v1' });
 
-app.use((ctx) => {
-    ctx.response.body = 'Hello World!';
-});
+    app.use(
+        buildAPI(router, fileService).routes(),
+    );
 
-await app.listen({ port: 9000 });
+    await app.listen({ port: 9000 });
+}
+
+await main();

src/utils.ts

@@ -0,0 +1,69 @@
+export async function asyncIterableToArray<T>(ite: AsyncIterable<T>): Promise<T[]> {
+    const ret: T[] = [];
+    for await (const e of ite) {
+        ret.push(e);
+    }
+    return ret;
+}
+
+
+export function concatUint8Array(arr1: Uint8Array, arr2: Uint8Array): Uint8Array {
+    const mergedArray = new Uint8Array(arr1.length + arr2.length);
+    mergedArray.set(arr1);
+    mergedArray.set(arr2, arr1.length);
+    return mergedArray;
+}
+
+export class AsyncIterableReader implements Deno.Reader {
+    private readonly source: AsyncIterable<Uint8Array>;
+    private interReadBuffer = new Uint8Array();
+    private isSourceEmpty = false;
+    constructor(source: AsyncIterable<Uint8Array>) {
+        this.source = source;
+    }
+
+    private async readMinimumOf(n: number): Promise<Uint8Array> {
+        let ret = new Uint8Array();
+        console.log('))) ' + n )
+        
+        for await(const piece of this.source){
+            console.log(piece.length)
+            ret = concatUint8Array(ret, piece);
+            if(ret.length >= n){
+                console.log('*******************************')
+                return ret;
+            }
+        }
+        console.log('---------------');
+        console.log(this.source);
+        this.isSourceEmpty = true;
+        return ret;
+    }
+
+    private transferData(source: Uint8Array, dest: Uint8Array): number {
+        const limit = Math.min(source.length, dest.length);
+        for(let i = 0; i < limit; i ++){
+            dest[i] = source[i];
+        }
+        return limit;
+    }
+
+    private readFromInternalReadBuffer(buffer: Uint8Array): number {
+        const n = this.transferData(this.interReadBuffer, buffer);
+        this.interReadBuffer = this.interReadBuffer.subarray(n);
+        return n;
+    }
+
+    async read(buffer: Uint8Array): Promise<number | null> {
+        if(this.isSourceEmpty && this.interReadBuffer.length < 1){
+            console.log("done mother fucker");
+            return null;
+        }
+        const internalReadBufferTransferdSize = this.readFromInternalReadBuffer(buffer);
+        
+        const data = await this.readMinimumOf(buffer.length - internalReadBufferTransferdSize);
+        const transferedSize = this.transferData(data.subarray(internalReadBufferTransferdSize), buffer);
+        this.interReadBuffer = data.subarray(transferedSize);
+        return internalReadBufferTransferdSize + transferedSize;
+    }
+}