Linted code.

This commit is contained in:
didier amyot committed 2023-06-13 08:26:30 -04:00
1 parent d12600348f
commit eae420a5a4
8 files changed
+200 -214

No files matched your search

+9 -4
View File
@@ -1,5 +1,10 @@
export { Buffer, StringReader } from "https://deno.land/std@0.191.0/io/mod.ts";
export { readAll, readerFromIterable, readableStreamFromReader, readerFromStreamReader, iterateReader, TextLineStream } from "https://deno.land/std@0.191.0/streams/mod.ts";
export { Buffer, StringReader } from 'https://deno.land/std@0.191.0/io/mod.ts';
export {
iterateReader,
readableStreamFromReader,
readAll,
readerFromIterable,
readerFromStreamReader,
TextLineStream,
} from 'https://deno.land/std@0.191.0/streams/mod.ts';
export { Application, Context, Router } from 'https://deno.land/x/oak/mod.ts';
+11 -15
View File
@@ -1,15 +1,13 @@
import { Context, Router } from "./deps.ts";
import { FileService } from "./services/mod.ts";
import { readerFromStreamReader } from "./deps.ts";
import { asyncIterableToArray } from "./utils.ts";
import { Context, Router } from './deps.ts';
import { FileService } from './services/mod.ts';
import { readerFromStreamReader } from './deps.ts';
import { asyncIterableToArray } from './utils.ts';
type Handler = (ctx: Context) => void;
function listFileHandlerFactory(fileService: FileService): Handler {
return async (ctx: Context) => {
ctx.response.type = "application/json";
ctx.response.type = 'application/json';
ctx.response.body = await asyncIterableToArray(fileService.list());
};
}
@@ -20,8 +18,8 @@ function createFileHandlerFactory(fileService: FileService): Handler {
ctx.response.body = 'This is a post';
const path = untypedCtx.request.url.searchParams.get('path');
const reqBody = await ctx.request.body({ type: "stream" }).value;
const reader = readerFromStreamReader(reqBody.getReader())
const reqBody = await ctx.request.body({ type: 'stream' }).value;
const reader = readerFromStreamReader(reqBody.getReader());
await fileService.createFromReader(path, reader);
};
}
@@ -31,22 +29,21 @@ function getFileHandlerFactory(fileService: FileService): Handler {
const untypedCtx: any = ctx;
const fileId = untypedCtx.params.path;
const reader = await fileService.getContentAsReader(fileId);
if(reader == null){
if (reader == null) {
ctx.response.status = 404;
return;
}
ctx.response.status = 200;
ctx.response.body = reader;
ctx.response.body = reader;
};
}
function deleteFileHandlerFactory(fileService: FileService): Handler {
return async (ctx: Context) => {
const untypedCtx: any = ctx;
const path = untypedCtx.params.path;
const isDeleted = await fileService.deleteByPath(path);
if(!isDeleted){
if (!isDeleted) {
ctx.response.status = 404;
return;
}
@@ -54,7 +51,6 @@ function deleteFileHandlerFactory(fileService: FileService): Handler {
};
}
export function buildAPI(router: Router, fileService: FileService): Router {
router.get('/files', listFileHandlerFactory(fileService));
router.get('/files/:path', getFileHandlerFactory(fileService));
@@ -62,4 +58,4 @@ export function buildAPI(router: Router, fileService: FileService): Router {
router.post('/files', createFileHandlerFactory(fileService));
return router;
}
}
+2 -2
View File
@@ -1,8 +1,8 @@
import { Application, Router } from './deps.ts';
import { buildAPI } from './restapi.ts';
import { DenoKVFileService } from "./services/mod.ts";
import { DenoKVFileService } from './services/mod.ts';
async function main(){
async function main() {
const app = new Application();
const kv = await Deno.openKv();
const fileService = new DenoKVFileService(kv);
+27 -34
View File
@@ -1,9 +1,8 @@
import { readAll, readerFromIterable, iterateReader } from "../deps.ts";
import { Buffer, StringReader } from "../deps.ts";
import {FileService, FileMetaData, BlobService} from './interfaces.ts';
import { readerFromStreamReader, readableStreamFromReader } from "https://deno.land/std@0.191.0/streams/mod.ts";
import { iterateReader, readAll, readerFromIterable } from '../deps.ts';
import { Buffer, StringReader } from '../deps.ts';
import { BlobService, FileMetaData, FileService } from './interfaces.ts';
import { readableStreamFromReader, readerFromStreamReader } from 'https://deno.land/std@0.191.0/streams/mod.ts';
interface SerializedFile {
id: string;
@@ -12,7 +11,6 @@ interface SerializedFile {
compression: string;
}
interface SerializedFilePathIndex {
fileId: string;
}
@@ -27,12 +25,12 @@ export class DenoKVBlobService implements BlobService {
}
private async sha1(msgUint8: Uint8Array): Promise<string> {
const hashBuffer = await crypto.subtle.digest("SHA-1", msgUint8);
const hashBuffer = await crypto.subtle.digest('SHA-1', msgUint8);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray
.map((b) => b.toString(16).padStart(2, "0"))
.join(""); // convert bytes to hex string
}
.map((b) => b.toString(16).padStart(2, '0'))
.join(''); // convert bytes to hex string
}
private *splitText(content: Uint8Array): Generator<Uint8Array> {
let blobCount = 0;
@@ -51,10 +49,10 @@ export class DenoKVBlobService implements BlobService {
}
private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise<void> {
if(content.length > this.blobSize){
if (content.length > this.blobSize) {
throw new Error('Persisted a too large blob');
}
const blob_id = await this.sha1(content);
await this.kv.set(['blob_index', mainIndex, index], blob_id);
await this.kv.set(['blob', blob_id], content);
@@ -71,10 +69,10 @@ export class DenoKVBlobService implements BlobService {
return size;
}
async persistReader(index: string, content: Deno.Reader): Promise<number>{
async persistReader(index: string, content: Deno.Reader): Promise<number> {
let i = 0;
let size = 0;
for await (const chunk of iterateReader(content, {bufSize: this.blobSize})){
for await (const chunk of iterateReader(content, { bufSize: this.blobSize })) {
await this.unsafePersistBlob(index, i, chunk);
size += chunk.length;
i++;
@@ -100,11 +98,11 @@ export class DenoKVBlobService implements BlobService {
return await readAll(this.getBlob(index));
}
getBlob(index: string): Deno.Reader{
getBlob(index: string): Deno.Reader {
return readerFromIterable(this.iterBlob(index));
}
private async *iterRawBlobs(index: string): AsyncIterable<Uint8Array>{
private async *iterRawBlobs(index: string): AsyncIterable<Uint8Array> {
const prefix = ['blob_index', index];
for await (const itm of this.kv.list<string>({ prefix })) {
const foo = await this.kv.get<Uint8Array>(['blob', itm.value]);
@@ -127,8 +125,7 @@ export class DenoKVBlobService implements BlobService {
return ret;
}
streamBlob(index: string): Deno.Reader{
streamBlob(index: string): Deno.Reader {
return readerFromIterable(this.iterRawBlobs(index));
}
}
@@ -146,29 +143,29 @@ export class DenoKVFileService implements FileService {
path: string,
content: string | Uint8Array,
): Promise<FileMetaData> {
if (content instanceof Uint8Array){
if (content instanceof Uint8Array) {
return this.createFromReader(path, new Buffer(content));
}
return this.createFromReader(path, new StringReader(content));
}
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>{
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData> {
const newId = crypto.randomUUID();
const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')).getReader()
const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip'))
.getReader();
const size = await this.blobService.persistReader(newId, readerFromStreamReader(compressedContent));
const serializedFile: SerializedFile = {
id: newId,
path,
contentSize: size,
compression: 'gzip'
compression: 'gzip',
};
const idx: SerializedFilePathIndex = { fileId: newId };
await this.kv.set(['file', newId], serializedFile);
await this.kv.set(['file_index_path', path], idx);
return { id: newId, path, contentSize: size };
return { id: newId, path, contentSize: size };
}
async *list(): AsyncIterable<FileMetaData> {
@@ -206,21 +203,19 @@ export class DenoKVFileService implements FileService {
.delete(idx.key)
.commit();
return true;
}
}
private async getSerializedFile(fileId: string): Promise<SerializedFile | null>{
private async getSerializedFile(fileId: string): Promise<SerializedFile | null> {
const ret = await this.kv.get<SerializedFile>(['file', fileId]);
return ret.value
return ret.value;
}
private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
if(serializedFile.compression == 'gzip'){
if (serializedFile.compression == 'gzip') {
const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader();
return readerFromStreamReader(ret);
}
return content;
return content;
}
async getRawContentByPath(path: string): Promise<Uint8Array | null> {
@@ -228,7 +223,6 @@ export class DenoKVFileService implements FileService {
return ret === null ? null : readAll(ret);
}
async getContentAsReader(path: string): Promise<Deno.Reader | null> {
const idx = await this.kv.get<SerializedFilePathIndex>([
'file_index_path',
@@ -238,11 +232,10 @@ export class DenoKVFileService implements FileService {
return null;
}
const ser = await this.kv.get<SerializedFile>(['file', idx.value.fileId]);
if(ser.value === null){
if (ser.value === null) {
return null;
}
return this.decompressContent(ser.value, this.blobService.getBlob(idx.value.fileId));
}
async getMetaDataByID(id: string): Promise<FileMetaData | null> {
+4 -7
View File
@@ -4,24 +4,22 @@ export interface FileMetaData {
contentSize: number;
}
export interface FileService {
list(): AsyncIterable<FileMetaData>;
createFromFullContent(
path: string,
content: string | Uint8Array,
): Promise<FileMetaData>;
createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>;
getRawContentByPath(path: string): Promise<Uint8Array | null>;
getContentAsReader(path: string): Promise<Deno.Reader | null>;
getMetaDataByID(id: string): Promise<FileMetaData | null>;
renameById(id: string, newPath: string): Promise<FileMetaData | null>;
renameByPath(
@@ -32,7 +30,6 @@ export interface FileService {
deleteByPath(path: string): Promise<boolean>;
}
export interface BlobService {
persistBlob(index: string, content: Uint8Array): Promise<number>;
persistReader(index: string, content: Deno.Reader): Promise<number>;
@@ -40,4 +37,4 @@ export interface BlobService {
getBlob(index: string): Deno.Reader;
getNumberOfBlobs(index: string): Promise<number>;
streamBlob(index: string): Deno.Reader;
}
}
+1 -1
View File
@@ -1,2 +1,2 @@
export * from './interfaces.ts';
export {DenoKVBlobService, DenoKVFileService} from './denokv.ts';
export { DenoKVBlobService, DenoKVFileService } from './denokv.ts';
+104 -111
View File
@@ -1,142 +1,135 @@
import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
import { assertEquals } from 'https://deno.land/std/testing/asserts.ts';
import {
DenoKVBlobService,
DenoKVFileService,
FileMetaData,
FileService,
} from "../src/services/mod.ts";
import { asyncIterableToArray } from "../src/utils.ts";
import { generateLargeText } from "./utils.ts";
import { StringReader } from "https://deno.land/std@0.191.0/io/mod.ts";
import { DenoKVBlobService, DenoKVFileService, FileMetaData, FileService } from '../src/services/mod.ts';
import { asyncIterableToArray } from '../src/utils.ts';
import { generateLargeText } from './utils.ts';
import { StringReader } from 'https://deno.land/std@0.191.0/io/mod.ts';
Deno.test('DenoKVBlobService happy path', async (t) => {
localStorage.clear();
const kv = await Deno.openKv(':memory:');
const bs = new DenoKVBlobService(kv, 50);
Deno.test("DenoKVBlobService happy path", async (t) => {
localStorage.clear();
const kv = await Deno.openKv(":memory:");
function toUint8Array(s: string): Uint8Array {
return new TextEncoder().encode(s);
}
const bs = new DenoKVBlobService(kv, 50);
await t.step('read write small text', async () => {
const size = await bs.persistBlob('0002', toUint8Array('hello world'));
const content = await bs.readAllBlob('0002');
assertEquals(content, toUint8Array('hello world'));
assertEquals(await bs.getNumberOfBlobs('0002'), 1);
assertEquals(size, 'hello world'.length);
});
function toUint8Array(s: string): Uint8Array {
return new TextEncoder().encode(s);
}
await t.step('read write large text', async () => {
const largeText = toUint8Array(generateLargeText(33, 100));
const size = await bs.persistBlob('0001', largeText);
const content = await bs.readAllBlob('0001');
assertEquals(content, largeText);
assertEquals(await bs.getNumberOfBlobs('0001'), 21);
assertEquals(size, largeText.length);
});
await t.step("read write small text", async () => {
const size = await bs.persistBlob("0002", toUint8Array("hello world"));
const content = await bs.readAllBlob("0002");
assertEquals(content, toUint8Array("hello world"));
assertEquals(await bs.getNumberOfBlobs("0002"), 1);
assertEquals(size, "hello world".length);
});
await t.step('read write large file', async () => {
const fileName = 'tests/examples/teapot.obj';
const f = await Deno.open(fileName);
const size = await bs.persistReader('large-stream', f);
await t.step("read write large text", async () => {
const largeText = toUint8Array(generateLargeText(33, 100));
const size = await bs.persistBlob("0001", largeText);
const content = await bs.readAllBlob("0001");
assertEquals(content, largeText);
assertEquals(await bs.getNumberOfBlobs("0001"), 21);
assertEquals(size, largeText.length);
});
assertEquals(await bs.getNumberOfBlobs('large-stream'), 1030);
assertEquals((await Deno.readFile(fileName)).length, size);
f.close();
});
await t.step("read write large file", async () => {
const fileName = 'tests/examples/teapot.obj';
const f = await Deno.open(fileName);
const size = await bs.persistReader("large-stream", f);
assertEquals(await bs.getNumberOfBlobs("large-stream"), 1030);
assertEquals((await Deno.readFile(fileName)).length, size);
f.close();
});
kv.close();
kv.close();
});
Deno.test("DenoKVFileService happy path xxx ", async (t) => {
const kv = await Deno.openKv(":memory:");
const fs: FileService = new DenoKVFileService(kv);
const fileName1 = "didier/file1.txt";
const fileName2 = "didier/file2.txt";
let file1: FileMetaData | undefined;
let file2: FileMetaData | undefined;
Deno.test('DenoKVFileService happy path xxx ', async (t) => {
const kv = await Deno.openKv(':memory:');
const fs: FileService = new DenoKVFileService(kv);
const fileName1 = 'didier/file1.txt';
const fileName2 = 'didier/file2.txt';
let file1: FileMetaData | undefined;
let file2: FileMetaData | undefined;
await t.step("Create 2 files xyzabc", async () => {
file1 = await fs.createFromFullContent(fileName1, "bob le chef");
file2 = await fs.createFromFullContent(fileName2, "le geant vert ☀️");
await t.step('Create 2 files xyzabc', async () => {
file1 = await fs.createFromFullContent(fileName1, 'bob le chef');
file2 = await fs.createFromFullContent(fileName2, 'le geant vert ☀️');
assertEquals(file1.path, fileName1);
assertEquals(file2.contentSize, 47);
});
assertEquals(file1.path, fileName1);
assertEquals(file2.contentSize, 47);
});
await t.step("Get file metadata", async () => {
const ret = await fs.getMetaDataByID(file1!.id);
assertEquals(ret, file1);
});
await t.step('Get file metadata', async () => {
const ret = await fs.getMetaDataByID(file1!.id);
assertEquals(ret, file1);
});
await t.step("List files", async () => {
const expected = [file1, file2];
const ret = await fs.list();
assertEquals(await asyncIterableToArray(ret), expected);
});
await t.step('List files', async () => {
const expected = [file1, file2];
const ret = await fs.list();
assertEquals(await asyncIterableToArray(ret), expected);
});
await t.step("Get File by path", async () => {
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), "bob le chef");
});
await t.step('Get File by path', async () => {
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), 'bob le chef');
});
await t.step("Get File by Id", async () => {
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), "bob le chef");
});
await t.step('Get File by Id', async () => {
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), 'bob le chef');
});
await t.step("Delete File", async () => {
await fs.deleteByPath(fileName1);
await fs.deleteByPath("404 not found");
});
await t.step('Delete File', async () => {
await fs.deleteByPath(fileName1);
await fs.deleteByPath('404 not found');
});
await t.step("List after delete", async () => {
const expected = [file2];
const ret = await fs.list();
assertEquals(await asyncIterableToArray(ret), expected);
});
await t.step('List after delete', async () => {
const expected = [file2];
const ret = await fs.list();
assertEquals(await asyncIterableToArray(ret), expected);
});
await t.step("Create File from reader", async () => {
const newPath = "boris/streamed-file.txt";
await t.step('Create File from reader', async () => {
const newPath = 'boris/streamed-file.txt';
const ret = await fs.createFromReader(newPath, new StringReader("hello world 18"));
assertEquals(ret.contentSize, 40);
const ret = await fs.createFromReader(newPath, new StringReader('hello world 18'));
assertEquals(ret.contentSize, 40);
const s = await fs.getRawContentByPath(newPath);
assertEquals(new TextDecoder().decode(s!), "hello world 18")
});
const s = await fs.getRawContentByPath(newPath);
assertEquals(new TextDecoder().decode(s!), 'hello world 18');
});
await t.step("Rename file", async () => {
const newPath = "boris/file2.txt";
const ret = await fs.renameById(file2!.id, newPath);
assertEquals(ret?.path, newPath);
await t.step('Rename file', async () => {
const newPath = 'boris/file2.txt';
const ret = await fs.renameById(file2!.id, newPath);
assertEquals(ret?.path, newPath);
assertEquals(
new TextDecoder().decode((await fs.getRawContentByPath(newPath))!),
'le geant vert ☀️',
);
});
assertEquals(
new TextDecoder().decode((await fs.getRawContentByPath(newPath))!),
'le geant vert ☀️',
);
});
kv.close();
kv.close();
});
Deno.test("DenoKVFileService happy path with byte array", async (t) => {
const kv = await Deno.openKv(":memory:");
const fs: FileService = new DenoKVFileService(kv);
await t.step("Create and read 1 binary file", async () => {
const inputContent = new Uint8Array([240]);
const file = await fs.createFromFullContent("hello", inputContent);
Deno.test('DenoKVFileService happy path with byte array', async (t) => {
const kv = await Deno.openKv(':memory:');
const fs: FileService = new DenoKVFileService(kv);
await t.step('Create and read 1 binary file', async () => {
const inputContent = new Uint8Array([240]);
const file = await fs.createFromFullContent('hello', inputContent);
assertEquals(file.path, "hello");
assertEquals(file.contentSize, 27);
assertEquals(file.path, 'hello');
assertEquals(file.contentSize, 27);
const persistedContent = await fs.getRawContentByPath("hello");
assertEquals(persistedContent, inputContent);
});
const persistedContent = await fs.getRawContentByPath('hello');
assertEquals(persistedContent, inputContent);
});
kv.close();
kv.close();
});
+42 -40
View File
@@ -1,22 +1,25 @@
export function* randomNumbers(seed: number, max: number, maxResult: number|undefined = undefined) : Generator<number> {
export function* randomNumbers(
seed: number,
max: number,
maxResult: number | undefined = undefined,
): Generator<number> {
const a = 48271;
const m = 2147483647;
const m = 2147483647;
max = Math.floor(max);
let i = 0;
while (true) {
seed = (a * seed) % m;
seed = (a * seed) % m;
yield seed % max;
i ++;
if(maxResult !== undefined && i >= maxResult){
return;
i++;
if (maxResult !== undefined && i >= maxResult) {
return;
}
}
}
export function range(min: number, max: number): number[]{
const ret = []
for(let i = min; i < max; i ++){
export function range(min: number, max: number): number[] {
const ret = [];
for (let i = min; i < max; i++) {
ret.push(i);
}
return ret;
@@ -24,37 +27,36 @@ export function range(min: number, max: number): number[]{
export function generateLargeText(seed: number, size: number): string {
const words = [
"Norway",
"Germany",
"Ecuador",
"Chile",
"Colombia",
"Sri Lanka",
"Guinea",
"United Kingdom",
"Egypt",
"Palau",
"Latvia",
"Congo, Democratic Republic of the",
"Malawi",
"Senegal",
"Trinidad and Tobago",
"Guatemala",
"Chad",
"New Zealand",
"Malaysia",
"Lithuania",
"Taiwan",
"Solomon Islands",
"Grenada",
"Hungary",
"Korea, South",
'Norway',
'Germany',
'Ecuador',
'Chile',
'Colombia',
'Sri Lanka',
'Guinea',
'United Kingdom',
'Egypt',
'Palau',
'Latvia',
'Congo, Democratic Republic of the',
'Malawi',
'Senegal',
'Trinidad and Tobago',
'Guatemala',
'Chad',
'New Zealand',
'Malaysia',
'Lithuania',
'Taiwan',
'Solomon Islands',
'Grenada',
'Hungary',
'Korea, South',
];
let ret: string[] = [];
for (let i = 0; i < size; i++) {
seed = (seed * 4931 + 2753) % words.length;
ret.push(words[seed]);
seed = (seed * 4931 + 2753) % words.length;
ret.push(words[seed]);
}
return ret.join(" ");
}
return ret.join(' ');
}