From 459dc26b4f096bfcdf24cc2d60eda28ebe60eb56 Mon Sep 17 00:00:00 2001
From: didier amyot <didier@nimblerx.com>
Date: Sat, 10 Jun 2023 09:30:58 -0400
Subject: [PATCH] Added support for Reader in blob store.

---
 .gitignore                |  1 +
 deno.json                 | 17 +++++++++--------
 deno.lock                 | 18 ++++++++++++++++++
 src/fileservice.ts        | 24 +++++++++++++++++++++---
 tests/fileservice_test.ts |  8 ++++++++
 5 files changed, 57 insertions(+), 11 deletions(-)
 create mode 100644 deno.lock

diff --git a/.gitignore b/.gitignore
index 9a1bdb4..45dae1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .vscode
 .direnv
+result
diff --git a/deno.json b/deno.json
index abca7a6..824137b 100644
--- a/deno.json
+++ b/deno.json
@@ -1,10 +1,11 @@
 {
     "fmt": {
-      "lineWidth": 120,
-      "indentWidth": 4,
-      "semiColons": true,
-      "singleQuote": true,
-      "proseWrap": "preserve",
-      "include": ["src/", "test/"]
-    }
-  }
\ No newline at end of file
+        "lineWidth": 120,
+        "indentWidth": 4,
+        "semiColons": true,
+        "singleQuote": true,
+        "proseWrap": "preserve",
+        "include": ["src/", "test/"]
+    },
+    "importMap": "./vendor/import_map.json"
+}
diff --git a/deno.lock b/deno.lock
new file mode 100644
index 0000000..b89fc87
--- /dev/null
+++ b/deno.lock
@@ -0,0 +1,18 @@
+{
+  "version": "2",
+  "remote": {
+    "https://deno.land/std@0.191.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e",
+    "https://deno.land/std@0.191.0/streams/_common.ts": "f45cba84f0d813de3326466095539602364a9ba521f804cc758f7a475cda692d",
+    "https://deno.land/std@0.191.0/streams/iterate_reader.ts": "bbec1d45c2df2c0c5920bad0549351446fdc8e0886d99e95959b259dbcdb6072",
+    "https://deno.land/std@0.191.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea",
+    "https://deno.land/std@0.191.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7",
+    "https://deno.land/std@0.191.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f",
+    "https://deno.land/std@0.191.0/types.d.ts": "dbaeb2c4d7c526db9828fc8df89d8aecf53b9ced72e0c4568f97ddd8cda616a4",
+    "https://deno.land/x/base64@v0.2.1/base.ts": "47dc8d68f07dc91524bdd6db36eccbe59cf4d935b5fc09f27357a3944bb3ff7b",
+    "https://deno.land/x/base64@v0.2.1/base64url.ts": "18bbf879b31f1f32cca8adaa2b6885ae325c2cec6a66c5817b684ca12c46ad5e",
+    "https://deno.land/x/base64@v0.2.1/mod.ts": "1cbdc4ba7229d3c6d1763fecdb9d46844777c7e153abb6dabea8b0dd01448db4",
+    "https://denopkg.com/chiefbiiko/sha1@master/deps.ts": "434b70f37ed145ca5259353c96fdee1e7d1cd8c6a32baad9dce2aedc79d0a3bc",
+    "https://denopkg.com/chiefbiiko/sha1@master/mod.ts": "146a101c9776cc9c807053c93f23e4b321ade5251f65745df418f4a75d5fd27b",
+    "https://denopkg.com/chiefbiiko/std-encoding@v1.1.1/mod.ts": "e61017da4cef146863d614a961b4e61f1df14b72c906092bd02161f913e76522"
+  }
+}
diff --git a/src/fileservice.ts b/src/fileservice.ts
index d2e87c0..26336f0 100644
--- a/src/fileservice.ts
+++ b/src/fileservice.ts
@@ -1,4 +1,6 @@
 import { sha1 } from 'https://denopkg.com/chiefbiiko/sha1/mod.ts';
+import { iterateReader } from "https://deno.land/std@0.191.0/streams/iterate_reader.ts";
+
 
 export interface FileService {
     createFromFullBody(
@@ -58,16 +60,32 @@ export class DenoKVBlobService {
         }
     }
 
+    private async unsafePersistBlob(mainIndex: string, index: number, body: Uint8Array): Promise<void> {
+        if(body.length > this.blobSize){
+            throw new Error('Persisted a too large blob');
+        }
+        const blob_id = sha1(body, 'utf8', 'hex'); // todo, should not be utf8
+        await this.kv.set(['blob_index', mainIndex, index], blob_id);
+        await this.kv.set(['blob', blob_id], body);
+
+    }
+
     async persistBlob(index: string, body: Uint8Array): Promise<void> {
         let blobCount = 0;
         for (const txt of this.splitText(body)) {
-            const blob_id = sha1(txt, 'utf8', 'hex');
-            await this.kv.set(['blob_index', index, blobCount], blob_id);
-            await this.kv.set(['blob', blob_id], txt);
+            await this.unsafePersistBlob(index, blobCount, txt);
             blobCount += 1;
         }
     }
 
+    async persistReader(index: string, body: Deno.Reader): Promise<void>{
+        let i = 0;
+        for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
+            this.unsafePersistBlob(index, i, chunk);
+            i++;
+        }
+    }
+
     // todo: remove
     private concatUint8Array(arr1: Uint8Array, arr2: Uint8Array): Uint8Array {
         const mergedArray = new Uint8Array(arr1.length + arr2.length);
diff --git a/tests/fileservice_test.ts b/tests/fileservice_test.ts
index 2412944..12b0b5f 100644
--- a/tests/fileservice_test.ts
+++ b/tests/fileservice_test.ts
@@ -76,6 +76,14 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
     assertEquals(await bs.getNumberOfBlobs("0001"), 21);
   });
 
+  await t.step("read write large file", async () => {
+    const f = await Deno.open('tests/examples/teapot.obj');
+    await bs.persistReader("large-stream", f);
+   
+    assertEquals(await bs.getNumberOfBlobs("large-stream"), 1030);
+    f.close();
+  });
+
   kv.close();
 });