Linted code.

This commit is contained in:
didier amyot committed 2023-06-13 08:26:30 -04:00
1 parent d12600348f
commit eae420a5a4
8 files changed
+128 -142

No files matched your search

+9 -4
View File
@@ -1,5 +1,10 @@
export { Buffer, StringReader } from "https://deno.land/std@0.191.0/io/mod.ts";
export { readAll, readerFromIterable, readableStreamFromReader, readerFromStreamReader, iterateReader, TextLineStream } from "https://deno.land/std@0.191.0/streams/mod.ts";
export { Buffer, StringReader } from 'https://deno.land/std@0.191.0/io/mod.ts';
export {
iterateReader,
readableStreamFromReader,
readAll,
readerFromIterable,
readerFromStreamReader,
TextLineStream,
} from 'https://deno.land/std@0.191.0/streams/mod.ts';
export { Application, Context, Router } from 'https://deno.land/x/oak/mod.ts';
+9 -13
View File
@@ -1,15 +1,13 @@
import { Context, Router } from "./deps.ts";
import { FileService } from "./services/mod.ts";
import { readerFromStreamReader } from "./deps.ts";
import { asyncIterableToArray } from "./utils.ts";
import { Context, Router } from './deps.ts';
import { FileService } from './services/mod.ts';
import { readerFromStreamReader } from './deps.ts';
import { asyncIterableToArray } from './utils.ts';
type Handler = (ctx: Context) => void;
function listFileHandlerFactory(fileService: FileService): Handler {
return async (ctx: Context) => {
ctx.response.type = "application/json";
ctx.response.type = 'application/json';
ctx.response.body = await asyncIterableToArray(fileService.list());
};
}
@@ -20,8 +18,8 @@ function createFileHandlerFactory(fileService: FileService): Handler {
ctx.response.body = 'This is a post';
const path = untypedCtx.request.url.searchParams.get('path');
const reqBody = await ctx.request.body({ type: "stream" }).value;
const reader = readerFromStreamReader(reqBody.getReader())
const reqBody = await ctx.request.body({ type: 'stream' }).value;
const reader = readerFromStreamReader(reqBody.getReader());
await fileService.createFromReader(path, reader);
};
}
@@ -31,7 +29,7 @@ function getFileHandlerFactory(fileService: FileService): Handler {
const untypedCtx: any = ctx;
const fileId = untypedCtx.params.path;
const reader = await fileService.getContentAsReader(fileId);
if(reader == null){
if (reader == null) {
ctx.response.status = 404;
return;
}
@@ -40,13 +38,12 @@ function getFileHandlerFactory(fileService: FileService): Handler {
};
}
function deleteFileHandlerFactory(fileService: FileService): Handler {
return async (ctx: Context) => {
const untypedCtx: any = ctx;
const path = untypedCtx.params.path;
const isDeleted = await fileService.deleteByPath(path);
if(!isDeleted){
if (!isDeleted) {
ctx.response.status = 404;
return;
}
@@ -54,7 +51,6 @@ function deleteFileHandlerFactory(fileService: FileService): Handler {
};
}
export function buildAPI(router: Router, fileService: FileService): Router {
router.get('/files', listFileHandlerFactory(fileService));
router.get('/files/:path', getFileHandlerFactory(fileService));
+2 -2
View File
@@ -1,8 +1,8 @@
import { Application, Router } from './deps.ts';
import { buildAPI } from './restapi.ts';
import { DenoKVFileService } from "./services/mod.ts";
import { DenoKVFileService } from './services/mod.ts';
async function main(){
async function main() {
const app = new Application();
const kv = await Deno.openKv();
const fileService = new DenoKVFileService(kv);
+22 -29
View File
@@ -1,9 +1,8 @@
import { readAll, readerFromIterable, iterateReader } from "../deps.ts";
import { Buffer, StringReader } from "../deps.ts";
import {FileService, FileMetaData, BlobService} from './interfaces.ts';
import { readerFromStreamReader, readableStreamFromReader } from "https://deno.land/std@0.191.0/streams/mod.ts";
import { iterateReader, readAll, readerFromIterable } from '../deps.ts';
import { Buffer, StringReader } from '../deps.ts';
import { BlobService, FileMetaData, FileService } from './interfaces.ts';
import { readableStreamFromReader, readerFromStreamReader } from 'https://deno.land/std@0.191.0/streams/mod.ts';
interface SerializedFile {
id: string;
@@ -12,7 +11,6 @@ interface SerializedFile {
compression: string;
}
interface SerializedFilePathIndex {
fileId: string;
}
@@ -27,11 +25,11 @@ export class DenoKVBlobService implements BlobService {
}
private async sha1(msgUint8: Uint8Array): Promise<string> {
const hashBuffer = await crypto.subtle.digest("SHA-1", msgUint8);
const hashBuffer = await crypto.subtle.digest('SHA-1', msgUint8);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray
.map((b) => b.toString(16).padStart(2, "0"))
.join(""); // convert bytes to hex string
.map((b) => b.toString(16).padStart(2, '0'))
.join(''); // convert bytes to hex string
}
private *splitText(content: Uint8Array): Generator<Uint8Array> {
@@ -51,7 +49,7 @@ export class DenoKVBlobService implements BlobService {
}
private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise<void> {
if(content.length > this.blobSize){
if (content.length > this.blobSize) {
throw new Error('Persisted a too large blob');
}
@@ -71,10 +69,10 @@ export class DenoKVBlobService implements BlobService {
return size;
}
async persistReader(index: string, content: Deno.Reader): Promise<number>{
async persistReader(index: string, content: Deno.Reader): Promise<number> {
let i = 0;
let size = 0;
for await (const chunk of iterateReader(content, {bufSize: this.blobSize})){
for await (const chunk of iterateReader(content, { bufSize: this.blobSize })) {
await this.unsafePersistBlob(index, i, chunk);
size += chunk.length;
i++;
@@ -100,11 +98,11 @@ export class DenoKVBlobService implements BlobService {
return await readAll(this.getBlob(index));
}
getBlob(index: string): Deno.Reader{
getBlob(index: string): Deno.Reader {
return readerFromIterable(this.iterBlob(index));
}
private async *iterRawBlobs(index: string): AsyncIterable<Uint8Array>{
private async *iterRawBlobs(index: string): AsyncIterable<Uint8Array> {
const prefix = ['blob_index', index];
for await (const itm of this.kv.list<string>({ prefix })) {
const foo = await this.kv.get<Uint8Array>(['blob', itm.value]);
@@ -127,8 +125,7 @@ export class DenoKVBlobService implements BlobService {
return ret;
}
streamBlob(index: string): Deno.Reader{
streamBlob(index: string): Deno.Reader {
return readerFromIterable(this.iterRawBlobs(index));
}
}
@@ -146,29 +143,29 @@ export class DenoKVFileService implements FileService {
path: string,
content: string | Uint8Array,
): Promise<FileMetaData> {
if (content instanceof Uint8Array){
if (content instanceof Uint8Array) {
return this.createFromReader(path, new Buffer(content));
}
return this.createFromReader(path, new StringReader(content));
}
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>{
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData> {
const newId = crypto.randomUUID();
const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')).getReader()
const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip'))
.getReader();
const size = await this.blobService.persistReader(newId, readerFromStreamReader(compressedContent));
const serializedFile: SerializedFile = {
id: newId,
path,
contentSize: size,
compression: 'gzip'
compression: 'gzip',
};
const idx: SerializedFilePathIndex = { fileId: newId };
await this.kv.set(['file', newId], serializedFile);
await this.kv.set(['file_index_path', path], idx);
return { id: newId, path, contentSize: size };
}
async *list(): AsyncIterable<FileMetaData> {
@@ -208,19 +205,17 @@ export class DenoKVFileService implements FileService {
return true;
}
private async getSerializedFile(fileId: string): Promise<SerializedFile | null>{
private async getSerializedFile(fileId: string): Promise<SerializedFile | null> {
const ret = await this.kv.get<SerializedFile>(['file', fileId]);
return ret.value
return ret.value;
}
private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
if(serializedFile.compression == 'gzip'){
if (serializedFile.compression == 'gzip') {
const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader();
return readerFromStreamReader(ret);
}
return content;
}
async getRawContentByPath(path: string): Promise<Uint8Array | null> {
@@ -228,7 +223,6 @@ export class DenoKVFileService implements FileService {
return ret === null ? null : readAll(ret);
}
async getContentAsReader(path: string): Promise<Deno.Reader | null> {
const idx = await this.kv.get<SerializedFilePathIndex>([
'file_index_path',
@@ -238,11 +232,10 @@ export class DenoKVFileService implements FileService {
return null;
}
const ser = await this.kv.get<SerializedFile>(['file', idx.value.fileId]);
if(ser.value === null){
if (ser.value === null) {
return null;
}
return this.decompressContent(ser.value, this.blobService.getBlob(idx.value.fileId));
}
async getMetaDataByID(id: string): Promise<FileMetaData | null> {
-3
View File
@@ -4,9 +4,7 @@ export interface FileMetaData {
contentSize: number;
}
export interface FileService {
list(): AsyncIterable<FileMetaData>;
createFromFullContent(
@@ -32,7 +30,6 @@ export interface FileService {
deleteByPath(path: string): Promise<boolean>;
}
export interface BlobService {
persistBlob(index: string, content: Uint8Array): Promise<number>;
persistReader(index: string, content: Deno.Reader): Promise<number>;
+1 -1
View File
@@ -1,2 +1,2 @@
export * from './interfaces.ts';
export {DenoKVBlobService, DenoKVFileService} from './denokv.ts';
export { DenoKVBlobService, DenoKVFileService } from './denokv.ts';
+48 -55
View File
@@ -1,20 +1,13 @@
import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
import { assertEquals } from 'https://deno.land/std/testing/asserts.ts';
import {
DenoKVBlobService,
DenoKVFileService,
FileMetaData,
FileService,
} from "../src/services/mod.ts";
import { asyncIterableToArray } from "../src/utils.ts";
import { generateLargeText } from "./utils.ts";
import { StringReader } from "https://deno.land/std@0.191.0/io/mod.ts";
import { DenoKVBlobService, DenoKVFileService, FileMetaData, FileService } from '../src/services/mod.ts';
import { asyncIterableToArray } from '../src/utils.ts';
import { generateLargeText } from './utils.ts';
import { StringReader } from 'https://deno.land/std@0.191.0/io/mod.ts';
Deno.test("DenoKVBlobService happy path", async (t) => {
Deno.test('DenoKVBlobService happy path', async (t) => {
localStorage.clear();
const kv = await Deno.openKv(":memory:");
const kv = await Deno.openKv(':memory:');
const bs = new DenoKVBlobService(kv, 50);
@@ -22,29 +15,29 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
return new TextEncoder().encode(s);
}
await t.step("read write small text", async () => {
const size = await bs.persistBlob("0002", toUint8Array("hello world"));
const content = await bs.readAllBlob("0002");
assertEquals(content, toUint8Array("hello world"));
assertEquals(await bs.getNumberOfBlobs("0002"), 1);
assertEquals(size, "hello world".length);
await t.step('read write small text', async () => {
const size = await bs.persistBlob('0002', toUint8Array('hello world'));
const content = await bs.readAllBlob('0002');
assertEquals(content, toUint8Array('hello world'));
assertEquals(await bs.getNumberOfBlobs('0002'), 1);
assertEquals(size, 'hello world'.length);
});
await t.step("read write large text", async () => {
await t.step('read write large text', async () => {
const largeText = toUint8Array(generateLargeText(33, 100));
const size = await bs.persistBlob("0001", largeText);
const content = await bs.readAllBlob("0001");
const size = await bs.persistBlob('0001', largeText);
const content = await bs.readAllBlob('0001');
assertEquals(content, largeText);
assertEquals(await bs.getNumberOfBlobs("0001"), 21);
assertEquals(await bs.getNumberOfBlobs('0001'), 21);
assertEquals(size, largeText.length);
});
await t.step("read write large file", async () => {
await t.step('read write large file', async () => {
const fileName = 'tests/examples/teapot.obj';
const f = await Deno.open(fileName);
const size = await bs.persistReader("large-stream", f);
const size = await bs.persistReader('large-stream', f);
assertEquals(await bs.getNumberOfBlobs("large-stream"), 1030);
assertEquals(await bs.getNumberOfBlobs('large-stream'), 1030);
assertEquals((await Deno.readFile(fileName)).length, size);
f.close();
});
@@ -52,66 +45,66 @@ Deno.test("DenoKVBlobService happy path", async (t) => {
kv.close();
});
Deno.test("DenoKVFileService happy path xxx ", async (t) => {
const kv = await Deno.openKv(":memory:");
Deno.test('DenoKVFileService happy path xxx ', async (t) => {
const kv = await Deno.openKv(':memory:');
const fs: FileService = new DenoKVFileService(kv);
const fileName1 = "didier/file1.txt";
const fileName2 = "didier/file2.txt";
const fileName1 = 'didier/file1.txt';
const fileName2 = 'didier/file2.txt';
let file1: FileMetaData | undefined;
let file2: FileMetaData | undefined;
await t.step("Create 2 files xyzabc", async () => {
file1 = await fs.createFromFullContent(fileName1, "bob le chef");
file2 = await fs.createFromFullContent(fileName2, "le geant vert ☀️");
await t.step('Create 2 files xyzabc', async () => {
file1 = await fs.createFromFullContent(fileName1, 'bob le chef');
file2 = await fs.createFromFullContent(fileName2, 'le geant vert ☀️');
assertEquals(file1.path, fileName1);
assertEquals(file2.contentSize, 47);
});
await t.step("Get file metadata", async () => {
await t.step('Get file metadata', async () => {
const ret = await fs.getMetaDataByID(file1!.id);
assertEquals(ret, file1);
});
await t.step("List files", async () => {
await t.step('List files', async () => {
const expected = [file1, file2];
const ret = await fs.list();
assertEquals(await asyncIterableToArray(ret), expected);
});
await t.step("Get File by path", async () => {
await t.step('Get File by path', async () => {
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), "bob le chef");
assertEquals(new TextDecoder().decode(ret!), 'bob le chef');
});
await t.step("Get File by Id", async () => {
await t.step('Get File by Id', async () => {
const ret = await fs.getRawContentByPath(fileName1);
assertEquals(new TextDecoder().decode(ret!), "bob le chef");
assertEquals(new TextDecoder().decode(ret!), 'bob le chef');
});
await t.step("Delete File", async () => {
await t.step('Delete File', async () => {
await fs.deleteByPath(fileName1);
await fs.deleteByPath("404 not found");
await fs.deleteByPath('404 not found');
});
await t.step("List after delete", async () => {
await t.step('List after delete', async () => {
const expected = [file2];
const ret = await fs.list();
assertEquals(await asyncIterableToArray(ret), expected);
});
await t.step("Create File from reader", async () => {
const newPath = "boris/streamed-file.txt";
await t.step('Create File from reader', async () => {
const newPath = 'boris/streamed-file.txt';
const ret = await fs.createFromReader(newPath, new StringReader("hello world 18"));
const ret = await fs.createFromReader(newPath, new StringReader('hello world 18'));
assertEquals(ret.contentSize, 40);
const s = await fs.getRawContentByPath(newPath);
assertEquals(new TextDecoder().decode(s!), "hello world 18")
assertEquals(new TextDecoder().decode(s!), 'hello world 18');
});
await t.step("Rename file", async () => {
const newPath = "boris/file2.txt";
await t.step('Rename file', async () => {
const newPath = 'boris/file2.txt';
const ret = await fs.renameById(file2!.id, newPath);
assertEquals(ret?.path, newPath);
@@ -124,17 +117,17 @@ Deno.test("DenoKVFileService happy path xxx ", async (t) => {
kv.close();
});
Deno.test("DenoKVFileService happy path with byte array", async (t) => {
const kv = await Deno.openKv(":memory:");
Deno.test('DenoKVFileService happy path with byte array', async (t) => {
const kv = await Deno.openKv(':memory:');
const fs: FileService = new DenoKVFileService(kv);
await t.step("Create and read 1 binary file", async () => {
await t.step('Create and read 1 binary file', async () => {
const inputContent = new Uint8Array([240]);
const file = await fs.createFromFullContent("hello", inputContent);
const file = await fs.createFromFullContent('hello', inputContent);
assertEquals(file.path, "hello");
assertEquals(file.path, 'hello');
assertEquals(file.contentSize, 27);
const persistedContent = await fs.getRawContentByPath("hello");
const persistedContent = await fs.getRawContentByPath('hello');
assertEquals(persistedContent, inputContent);
});
+37 -35
View File
@@ -1,5 +1,8 @@
export function* randomNumbers(seed: number, max: number, maxResult: number|undefined = undefined) : Generator<number> {
export function* randomNumbers(
seed: number,
max: number,
maxResult: number | undefined = undefined,
): Generator<number> {
const a = 48271;
const m = 2147483647;
max = Math.floor(max);
@@ -7,16 +10,16 @@ export function* randomNumbers(seed: number, max: number, maxResult: number|unde
while (true) {
seed = (a * seed) % m;
yield seed % max;
i ++;
if(maxResult !== undefined && i >= maxResult){
i++;
if (maxResult !== undefined && i >= maxResult) {
return;
}
}
}
export function range(min: number, max: number): number[]{
const ret = []
for(let i = min; i < max; i ++){
export function range(min: number, max: number): number[] {
const ret = [];
for (let i = min; i < max; i++) {
ret.push(i);
}
return ret;
@@ -24,37 +27,36 @@ export function range(min: number, max: number): number[]{
export function generateLargeText(seed: number, size: number): string {
const words = [
"Norway",
"Germany",
"Ecuador",
"Chile",
"Colombia",
"Sri Lanka",
"Guinea",
"United Kingdom",
"Egypt",
"Palau",
"Latvia",
"Congo, Democratic Republic of the",
"Malawi",
"Senegal",
"Trinidad and Tobago",
"Guatemala",
"Chad",
"New Zealand",
"Malaysia",
"Lithuania",
"Taiwan",
"Solomon Islands",
"Grenada",
"Hungary",
"Korea, South",
'Norway',
'Germany',
'Ecuador',
'Chile',
'Colombia',
'Sri Lanka',
'Guinea',
'United Kingdom',
'Egypt',
'Palau',
'Latvia',
'Congo, Democratic Republic of the',
'Malawi',
'Senegal',
'Trinidad and Tobago',
'Guatemala',
'Chad',
'New Zealand',
'Malaysia',
'Lithuania',
'Taiwan',
'Solomon Islands',
'Grenada',
'Hungary',
'Korea, South',
];
let ret: string[] = [];
for (let i = 0; i < size; i++) {
seed = (seed * 4931 + 2753) % words.length;
ret.push(words[seed]);
}
return ret.join(" ");
}
return ret.join(' ');
}