343 lines
10 KiB
TypeScript
343 lines
10 KiB
TypeScript
import { readAll } from "https://deno.land/std@0.191.0/streams/read_all.ts";
|
|
import { readerFromIterable, iterateReader } from "https://deno.land/std@0.191.0/streams/mod.ts";
|
|
import { Buffer } from "https://deno.land/std@0.191.0/io/buffer.ts";
|
|
|
|
import { StringReader } from "https://deno.land/std@0.191.0/io/string_reader.ts";
|
|
|
|
|
|
import { readerFromStreamReader, readableStreamFromReader } from "https://deno.land/std@0.191.0/streams/mod.ts";
|
|
|
|
|
|
|
|
|
|
export interface FileService {
|
|
|
|
list(): AsyncIterable<FileMetaData>;
|
|
|
|
createFromFullBody(
|
|
path: string,
|
|
body: string | Uint8Array | Deno.Reader,
|
|
): Promise<FileMetaData>;
|
|
createFromReader(path: string, body: Deno.Reader): Promise<FileMetaData>;
|
|
|
|
|
|
getFullBodyByPath(path: string): Promise<string | null>;
|
|
|
|
getRawBodyByPath(path: string): Promise<Uint8Array | null>;
|
|
|
|
getBodyAsReader(path: string): Promise<Deno.Reader | null>;
|
|
|
|
getMetaDataByID(id: string): Promise<FileMetaData | null>;
|
|
|
|
renameById(id: string, newPath: string): Promise<FileMetaData | null>;
|
|
|
|
renameByPath(
|
|
originalPath: string,
|
|
newPath: string,
|
|
): Promise<FileMetaData | null>;
|
|
|
|
deleteByPath(path: string): Promise<boolean>;
|
|
}
|
|
|
|
interface SerializedFile {
|
|
id: string;
|
|
path: string;
|
|
bodySize: number;
|
|
compression: string;
|
|
}
|
|
|
|
export interface FileMetaData {
|
|
path: string;
|
|
id: string;
|
|
bodySize: number;
|
|
}
|
|
|
|
interface SerializedFilePathIndex {
|
|
fileId: string;
|
|
}
|
|
|
|
export class DenoKVBlobService {
|
|
private readonly kv: Deno.Kv;
|
|
private readonly blobSize: number;
|
|
|
|
constructor(kv: Deno.Kv, blobSize: number) {
|
|
this.kv = kv;
|
|
this.blobSize = blobSize;
|
|
}
|
|
|
|
private async sha1(msgUint8: Uint8Array): Promise<string> {
|
|
const hashBuffer = await crypto.subtle.digest("SHA-1", msgUint8);
|
|
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
return hashArray
|
|
.map((b) => b.toString(16).padStart(2, "0"))
|
|
.join(""); // convert bytes to hex string
|
|
}
|
|
|
|
private *splitText(body: Uint8Array): Generator<Uint8Array> {
|
|
let blobCount = 0;
|
|
let txt = body.slice(
|
|
blobCount * this.blobSize,
|
|
(blobCount + 1) * this.blobSize,
|
|
);
|
|
while (txt.length > 0) {
|
|
yield txt;
|
|
blobCount += 1;
|
|
txt = body.slice(
|
|
blobCount * this.blobSize,
|
|
(blobCount + 1) * this.blobSize,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async unsafePersistBlob(mainIndex: string, index: number, body: Uint8Array): Promise<void> {
|
|
if(body.length > this.blobSize){
|
|
throw new Error('Persisted a too large blob');
|
|
}
|
|
|
|
const blob_id = await this.sha1(body);
|
|
await this.kv.set(['blob_index', mainIndex, index], blob_id);
|
|
await this.kv.set(['blob', blob_id], body);
|
|
}
|
|
|
|
async persistBlob(index: string, body: Uint8Array): Promise<number> {
|
|
let blobCount = 0;
|
|
let size = 0;
|
|
for (const txt of this.splitText(body)) {
|
|
await this.unsafePersistBlob(index, blobCount, txt);
|
|
size += txt.length;
|
|
blobCount += 1;
|
|
}
|
|
return size;
|
|
}
|
|
|
|
async persistReader(index: string, body: Deno.Reader): Promise<number>{
|
|
let i = 0;
|
|
let size = 0;
|
|
for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
|
|
await this.unsafePersistBlob(index, i, chunk);
|
|
size += chunk.length;
|
|
i++;
|
|
}
|
|
return size;
|
|
}
|
|
|
|
private async *iterBlob(index: string): AsyncIterable<Uint8Array> {
|
|
for await (
|
|
const itm of this.kv.list<string>({ prefix: ['blob_index', index] })
|
|
) {
|
|
const foo = await this.kv.get<Uint8Array>(['blob', itm.value]);
|
|
if (foo.value === null) {
|
|
throw new Error(
|
|
`Inconsistant blob index for key "${foo.value}", this is very bad.`,
|
|
);
|
|
}
|
|
yield foo.value;
|
|
}
|
|
}
|
|
|
|
async readAllBlob(index: string): Promise<Uint8Array> {
|
|
return await readAll(this.getBlob(index));
|
|
}
|
|
|
|
getBlob(index: string): Deno.Reader{
|
|
return readerFromIterable(this.iterBlob(index));
|
|
}
|
|
|
|
private async *iterRawBlobs(index: string): AsyncIterable<Uint8Array>{
|
|
const prefix = ['blob_index', index];
|
|
for await (const itm of this.kv.list<string>({ prefix })) {
|
|
const foo = await this.kv.get<Uint8Array>(['blob', itm.value]);
|
|
if (foo.value === null) {
|
|
throw new Error(
|
|
`Inconsistant blob index for key "${foo.value}", this is very bad.`,
|
|
);
|
|
}
|
|
yield foo.value;
|
|
}
|
|
}
|
|
|
|
async getNumberOfBlobs(index: string): Promise<number> {
|
|
let ret = 0;
|
|
for await (
|
|
const _itm of this.kv.list<string>({ prefix: ['blob_index', index] })
|
|
) {
|
|
ret += 1;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
streamBlob(index: string): Deno.Reader{
|
|
|
|
return readerFromIterable(this.iterRawBlobs(index));
|
|
}
|
|
}
|
|
|
|
export class DenoKVFileService implements FileService {
|
|
private readonly kv: Deno.Kv;
|
|
private readonly blobService: DenoKVBlobService;
|
|
|
|
constructor(kv: Deno.Kv) {
|
|
this.kv = kv;
|
|
this.blobService = new DenoKVBlobService(kv, 32 * 1024);
|
|
}
|
|
|
|
createFromFullBody(
|
|
path: string,
|
|
body: string | Uint8Array,
|
|
): Promise<FileMetaData> {
|
|
if (body instanceof Uint8Array){
|
|
return this.createFromReader(path, new Buffer(body));
|
|
}
|
|
return this.createFromReader(path, new StringReader(body));
|
|
}
|
|
|
|
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>{
|
|
const newId = crypto.randomUUID();
|
|
|
|
const compressedContent = readableStreamFromReader(content).pipeThrough(new CompressionStream('gzip')).getReader()
|
|
const size = await this.blobService.persistReader(newId, readerFromStreamReader(compressedContent));
|
|
const serializedFile: SerializedFile = {
|
|
id: newId,
|
|
path,
|
|
bodySize: size,
|
|
compression: 'gzip'
|
|
};
|
|
const idx: SerializedFilePathIndex = { fileId: newId };
|
|
await this.kv.set(['file', newId], serializedFile);
|
|
await this.kv.set(['file_index_path', path], idx);
|
|
|
|
return { id: newId, path, bodySize: size };
|
|
|
|
}
|
|
|
|
async *list(): AsyncIterable<FileMetaData> {
|
|
for await (
|
|
const entry of this.kv.list<SerializedFilePathIndex>({
|
|
prefix: ['file_index_path'],
|
|
})
|
|
) {
|
|
const ret = await this.kv.get<SerializedFile>([
|
|
'file',
|
|
entry.value.fileId,
|
|
]);
|
|
if (ret.value !== null) {
|
|
yield {
|
|
path: ret.value.path,
|
|
id: ret.value.id,
|
|
bodySize: ret.value.bodySize,
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
async deleteByPath(path: string): Promise<boolean> {
|
|
const idx = await this.kv.get<SerializedFilePathIndex>([
|
|
'file_index_path',
|
|
path,
|
|
]);
|
|
if (idx.value === null) {
|
|
return false;
|
|
}
|
|
// todo, this will fail silently if transaction fails. we might want to put that into a loop.
|
|
await this.kv.atomic()
|
|
.check(idx)
|
|
.delete(['file_index_path', path])
|
|
.delete(idx.key)
|
|
.commit();
|
|
return true;
|
|
}
|
|
|
|
private async getSerializedFile(fileId: string): Promise<SerializedFile | null>{
|
|
const ret = await this.kv.get<SerializedFile>(['file', fileId]);
|
|
return ret.value
|
|
}
|
|
|
|
private decompressBody(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
|
|
if(serializedFile.compression == 'gzip'){
|
|
const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader();
|
|
return readerFromStreamReader(ret);
|
|
|
|
}
|
|
return content;
|
|
|
|
}
|
|
|
|
async getFullBodyByPath(path: string): Promise<string | null> {
|
|
const ret = await this.getRawBodyByPath(path);
|
|
return ret == null ? null : new TextDecoder().decode(ret);
|
|
}
|
|
|
|
async getRawBodyByPath(path: string): Promise<Uint8Array | null> {
|
|
const ret = await this.getBodyAsReader(path);
|
|
return ret === null ? null : readAll(ret);
|
|
}
|
|
|
|
|
|
async getBodyAsReader(path: string): Promise<Deno.Reader | null> {
|
|
const idx = await this.kv.get<SerializedFilePathIndex>([
|
|
'file_index_path',
|
|
path,
|
|
]);
|
|
if (idx.value === null) {
|
|
return null;
|
|
}
|
|
const ser = await this.kv.get<SerializedFile>(['file', idx.value.fileId]);
|
|
if(ser.value === null){
|
|
return null;
|
|
}
|
|
return this.decompressBody(ser.value, this.blobService.getBlob(idx.value.fileId));
|
|
|
|
}
|
|
|
|
async getMetaDataByID(id: string): Promise<FileMetaData | null> {
|
|
const fileObject = await this.getSerializedFile(id);
|
|
if (fileObject === null) {
|
|
return null;
|
|
}
|
|
return {
|
|
path: fileObject.path,
|
|
id: fileObject.id,
|
|
bodySize: fileObject.bodySize,
|
|
};
|
|
}
|
|
|
|
async renameById(id: string, newPath: string): Promise<FileMetaData | null> {
|
|
const fileObject = await this.kv.get<SerializedFile>(['file', id]);
|
|
if (fileObject.value === null) {
|
|
return null;
|
|
}
|
|
const idx = await this.kv.get<SerializedFilePathIndex>([
|
|
'file_index_path',
|
|
fileObject.value.path,
|
|
]);
|
|
const newFile: SerializedFile = {
|
|
path: newPath,
|
|
id: fileObject.value.id,
|
|
bodySize: fileObject.value.bodySize,
|
|
compression: fileObject.value.compression,
|
|
};
|
|
await this.kv.atomic()
|
|
.check(fileObject)
|
|
.check(idx)
|
|
.delete(idx.key)
|
|
.set(['file_index_path', newPath], idx.value)
|
|
.set(fileObject.key, newFile)
|
|
.commit();
|
|
return this.getMetaDataByID(id);
|
|
}
|
|
|
|
async renameByPath(
|
|
originalPath: string,
|
|
newPath: string,
|
|
): Promise<FileMetaData | null> {
|
|
const idx = await this.kv.get<SerializedFilePathIndex>([
|
|
'file_index_path',
|
|
originalPath,
|
|
]);
|
|
if (idx.value == null) {
|
|
return null;
|
|
}
|
|
return this.renameById(idx.value.fileId, newPath);
|
|
}
|
|
}
|