Started to implement server

tests/fileservice_test.ts

@@ -6,50 +6,10 @@ import {
   FileMetaData,
   FileService,
 } from "../src/fileservice.ts";
+import { asyncIterableToArray } from "../src/utils.ts";
+import { generateLargeText } from "./utils.ts";
 
-function generateLargeText(seed: number, size: number): string {
-  const words = [
-    "Norway",
-    "Germany",
-    "Ecuador",
-    "Chile",
-    "Colombia",
-    "Sri Lanka",
-    "Guinea",
-    "United Kingdom",
-    "Egypt",
-    "Palau",
-    "Latvia",
-    "Congo, Democratic Republic of the",
-    "Malawi",
-    "Senegal",
-    "Trinidad and Tobago",
-    "Guatemala",
-    "Chad",
-    "New Zealand",
-    "Malaysia",
-    "Lithuania",
-    "Taiwan",
-    "Solomon Islands",
-    "Grenada",
-    "Hungary",
-    "Korea, South",
-  ];
-  let ret: string[] = [];
-  for (let i = 0; i < size; i++) {
-    seed = (seed * 4931 + 2753) % words.length;
-    ret.push(words[seed]);
-  }
-  return ret.join(" ");
-}
 
-async function asyncIterableToArray<T>(ite: AsyncIterable<T>): Promise<T[]> {
-  const ret: T[] = [];
-  for await (const e of ite) {
-    ret.push(e);
-  }
-  return ret;
-}
 
 Deno.test("DenoKVBlobService happy path", async (t) => {
   localStorage.clear();
@@ -62,25 +22,29 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
   }
 
   await t.step("read write small text", async () => {
-    await bs.persistBlob("0002", toUint8Array("hello world"));
+    const size = await bs.persistBlob("0002", toUint8Array("hello world"));
     const body = await bs.getBlob("0002");
     assertEquals(body, toUint8Array("hello world"));
     assertEquals(await bs.getNumberOfBlobs("0002"), 1);
+    assertEquals(size, "hello world".length);
   });
 
   await t.step("read write large text", async () => {
     const largeText = toUint8Array(generateLargeText(33, 100));
-    await bs.persistBlob("0001", largeText);
+    const size = await bs.persistBlob("0001", largeText);
     const body = await bs.getBlob("0001");
     assertEquals(body, largeText);
     assertEquals(await bs.getNumberOfBlobs("0001"), 21);
+    assertEquals(size, largeText.length);
   });
 
   await t.step("read write large file", async () => {
-    const f = await Deno.open('tests/examples/teapot.obj');
-    await bs.persistReader("large-stream", f);
+    const fileName = 'tests/examples/teapot.obj';
+    const f = await Deno.open(fileName);
+    const size = await bs.persistReader("large-stream", f);
    
     assertEquals(await bs.getNumberOfBlobs("large-stream"), 1030);
+    assertEquals((await Deno.readFile(fileName)).length, size);
     f.close();
   });
 
@@ -95,12 +59,12 @@ Deno.test("DenoKVFileService happy path", async (t) => {
   let file1: FileMetaData | undefined;
   let file2: FileMetaData | undefined;
 
-  await t.step("Create 2 files", async () => {
+  await t.step("Create 2 files xyzabc", async () => {
     file1 = await fs.createFromFullBody(fileName1, "bob le chef");
     file2 = await fs.createFromFullBody(fileName2, "le geant vert ☀️");
 
     assertEquals(file1.path, fileName1);
-    assertEquals(file2.bodySize, "le geant vert ".length + 2);
+    assertEquals(file2.bodySize, 20);
   });
 
   await t.step("Get file metadata", async () => {
@@ -133,14 +97,23 @@ Deno.test("DenoKVFileService happy path", async (t) => {
     assertEquals(await asyncIterableToArray(ret), expected);
   });
 
+  await t.step("Create File from reader", async () => {
+    const newPath = "boris/streamed-file.txt";
+    const file = await Deno.open('tests/examples/teapot.obj');
+
+    const ret = await fs.createFromReader(newPath, file);
+    assertEquals(ret.bodySize, 51493);
+    file.close();
+  });
+
   await t.step("Rename file", async () => {
     const newPath = "boris/file2.txt";
     const ret = await fs.renameById(file2?.id ?? "404", newPath);
     assertEquals(ret?.path, newPath);
 
     assertEquals(
-      (await fs.getFullBodyByPath(newPath))?.length,
-      file2?.bodySize ?? -1,
+      await fs.getFullBodyByPath(newPath),
+      'le geant vert ☀️',
     );
   });
 

tests/utils.ts

@@ -0,0 +1,56 @@
+
+
+export function* randomNumbers(seed: number, max: number) : Generator<number> {
+    const a = 48271;
+    const m =  2147483647;
+    max = Math.floor(max);
+    while (true) {
+        seed = (a * seed) % m; 
+        yield seed % max;
+    }
+}
+
+export function range(min: number, max: number): number[]{
+    const ret = []
+    for(let i = min; i < max; i ++){
+        ret.push(i);
+    }
+    return ret;
+}
+
+export function generateLargeText(seed: number, size: number): string {
+    const words = [
+      "Norway",
+      "Germany",
+      "Ecuador",
+      "Chile",
+      "Colombia",
+      "Sri Lanka",
+      "Guinea",
+      "United Kingdom",
+      "Egypt",
+      "Palau",
+      "Latvia",
+      "Congo, Democratic Republic of the",
+      "Malawi",
+      "Senegal",
+      "Trinidad and Tobago",
+      "Guatemala",
+      "Chad",
+      "New Zealand",
+      "Malaysia",
+      "Lithuania",
+      "Taiwan",
+      "Solomon Islands",
+      "Grenada",
+      "Hungary",
+      "Korea, South",
+    ];
+    let ret: string[] = [];
+    for (let i = 0; i < size; i++) {
+      seed = (seed * 4931 + 2753) % words.length;
+      ret.push(words[seed]);
+    }
+    return ret.join(" ");
+  }
+

tests/utils_test.ts

@@ -0,0 +1,83 @@
+import { readAll } from "https://deno.land/std@0.191.0/streams/read_all.ts";
+import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
+import { AsyncIterableReader } from "../src/utils.ts";
+import { randomNumbers, range } from "./utils.ts";
+
+
+Deno.test("AsyncIterableReader", async (t) => {
+
+    async function* miniSource(){
+        yield Promise.resolve(new Uint8Array([1, 2, 3]));
+        yield Promise.resolve(new Uint8Array([4, 5, 6, 7]));
+        yield Promise.resolve(new Uint8Array([8, 9]));
+    }
+
+    async function* largeSource(size: number):  AsyncIterable<Uint8Array>{
+        let val = 0
+        const random = randomNumbers(77, size / 4);
+        while(val < size){
+            const newVal = Math.min(val + random.next().value, size);
+            // console.log(range(val, newVal));
+            console.log("<<<<<<<<<<<<<<<<< " + newVal + "/" + size + " ( " + (newVal - val) )
+            yield Promise.resolve(new Uint8Array(range(val, newVal)));
+            
+            val = newVal;
+            
+
+        }
+        yield Promise.resolve(new Uint8Array([9]));
+        console.log(">>>>>>>>>>>>>>>>>>>>>>> ");
+        console.log(">>>>>>>>>>>>>>>>>>>>>>> ");
+        console.log(">>>>>>>>>>>>>>>>>>>>>>> ");
+        console.log(">>>>>>>>>>>>>>>>>>>>>>> ");
+    }
+
+    // await t.step('read all mini source', async () => {
+    //     const reader = new AsyncIterableReader(miniSource());
+    //     const expt = new Uint8Array([1, 2 ,3 ,4 ,5, 6,7,8,9])
+    //     assertEquals(await readAll(reader), expt);
+    // });
+
+    await t.step('largeSource', async () =>{
+        const source = largeSource(32920);
+        for await(const _piece of source){
+           //
+        }
+    });
+
+
+    await t.step('read all large source', async () => {
+        // for(let i = 101; i < (32 * 1024 * 50 + 101); i += 1){
+        let n = 32 * 1024 * 50 + 101 // 32920; //32 * 1024 * 2;
+        n = 32919
+        const reader = new AsyncIterableReader(largeSource(n));
+        const ret = await readAll(reader);
+        assertEquals(ret.length, n);
+        // }
+    });
+
+});
+
+
+// <<<<<<<<<<<<<<<<< 5137/32920
+// <<<<<<<<<<<<<<<<< 11593/32920
+// <<<<<<<<<<<<<<<<< 12563/32920
+// <<<<<<<<<<<<<<<<< 15976/32920
+// <<<<<<<<<<<<<<<<< 19155/32920
+// <<<<<<<<<<<<<<<<< 20038/32920
+// <<<<<<<<<<<<<<<<< 26452/32920
+// <<<<<<<<<<<<<<<<< 31797/32920
+// <<<<<<<<<<<<<<<<< 31994/32920
+// <<<<<<<<<<<<<<<<< 32912/32920
+// <<<<<<<<<<<<<<<<< 32920/32920
+
+
+// <<<<<<<<<<<<<<<<< 5137/32920
+// <<<<<<<<<<<<<<<<< 11593/32920
+// <<<<<<<<<<<<<<<<< 12563/32920
+// <<<<<<<<<<<<<<<<< 15976/32920
+// <<<<<<<<<<<<<<<<< 19155/32920
+// <<<<<<<<<<<<<<<<< 20038/32920
+// <<<<<<<<<<<<<<<<< 26452/32920
+// <<<<<<<<<<<<<<<<< 31797/32920
+// <<<<<<<<<<<<<<<<< 31994/32920