Added support for Reader in blob store.

.gitignore

@@ -1,2 +1,3 @@
 .vscode
 .direnv
+result

deno.json

@@ -6,5 +6,6 @@
         "singleQuote": true,
         "proseWrap": "preserve",
         "include": ["src/", "test/"]
-    }
+    },
+    "importMap": "./vendor/import_map.json"
 }

deno.lock

@@ -0,0 +1,18 @@
+{
+  "version": "2",
+  "remote": {
+    "https://deno.land/std@0.191.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e",
+    "https://deno.land/std@0.191.0/streams/_common.ts": "f45cba84f0d813de3326466095539602364a9ba521f804cc758f7a475cda692d",
+    "https://deno.land/std@0.191.0/streams/iterate_reader.ts": "bbec1d45c2df2c0c5920bad0549351446fdc8e0886d99e95959b259dbcdb6072",
+    "https://deno.land/std@0.191.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea",
+    "https://deno.land/std@0.191.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7",
+    "https://deno.land/std@0.191.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f",
+    "https://deno.land/std@0.191.0/types.d.ts": "dbaeb2c4d7c526db9828fc8df89d8aecf53b9ced72e0c4568f97ddd8cda616a4",
+    "https://deno.land/x/base64@v0.2.1/base.ts": "47dc8d68f07dc91524bdd6db36eccbe59cf4d935b5fc09f27357a3944bb3ff7b",
+    "https://deno.land/x/base64@v0.2.1/base64url.ts": "18bbf879b31f1f32cca8adaa2b6885ae325c2cec6a66c5817b684ca12c46ad5e",
+    "https://deno.land/x/base64@v0.2.1/mod.ts": "1cbdc4ba7229d3c6d1763fecdb9d46844777c7e153abb6dabea8b0dd01448db4",
+    "https://denopkg.com/chiefbiiko/sha1@master/deps.ts": "434b70f37ed145ca5259353c96fdee1e7d1cd8c6a32baad9dce2aedc79d0a3bc",
+    "https://denopkg.com/chiefbiiko/sha1@master/mod.ts": "146a101c9776cc9c807053c93f23e4b321ade5251f65745df418f4a75d5fd27b",
+    "https://denopkg.com/chiefbiiko/std-encoding@v1.1.1/mod.ts": "e61017da4cef146863d614a961b4e61f1df14b72c906092bd02161f913e76522"
+  }
+}

src/fileservice.ts

@@ -1,4 +1,6 @@
 import { sha1 } from 'https://denopkg.com/chiefbiiko/sha1/mod.ts';
+import { iterateReader } from "https://deno.land/std@0.191.0/streams/iterate_reader.ts";
+
 
 export interface FileService {
     createFromFullBody(
@@ -58,16 +60,32 @@ export class DenoKVBlobService {
         }
     }
 
+    private async unsafePersistBlob(mainIndex: string, index: number, body: Uint8Array): Promise<void> {
+        if(body.length > this.blobSize){
+            throw new Error('Persisted a too large blob');
+        }
+        const blob_id = sha1(body, 'utf8', 'hex'); // todo, should not be utf8
+        await this.kv.set(['blob_index', mainIndex, index], blob_id);
+        await this.kv.set(['blob', blob_id], body);
+
+    }
+
     async persistBlob(index: string, body: Uint8Array): Promise<void> {
         let blobCount = 0;
         for (const txt of this.splitText(body)) {
-            const blob_id = sha1(txt, 'utf8', 'hex');
+            await this.unsafePersistBlob(index, blobCount, txt);
-            await this.kv.set(['blob_index', index, blobCount], blob_id);
-            await this.kv.set(['blob', blob_id], txt);
             blobCount += 1;
         }
     }
 
+    async persistReader(index: string, body: Deno.Reader): Promise<void>{
+        let i = 0;
+        for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
+            this.unsafePersistBlob(index, i, chunk);
+            i++;
+        }
+    }
+
     // todo: remove
     private concatUint8Array(arr1: Uint8Array, arr2: Uint8Array): Uint8Array {
         const mergedArray = new Uint8Array(arr1.length + arr2.length);

tests/fileservice_test.ts

@@ -76,6 +76,14 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
     assertEquals(await bs.getNumberOfBlobs("0001"), 21);
   });
 
+  await t.step("read write large file", async () => {
+    const f = await Deno.open('tests/examples/teapot.obj');
+    await bs.persistReader("large-stream", f);
+   
+    assertEquals(await bs.getNumberOfBlobs("large-stream"), 1030);
+    f.close();
+  });
+
   kv.close();
 });