Refactored a bit the code; the original code was too experimental in style.
This commit is contained in:
1 parent
f7a8d494f5
commit
bbd705ee3b
3 files changed
+55
-60
No files matched your search
+33
-40
@@ -14,18 +14,16 @@ export interface FileService {
|
||||
|
||||
list(): AsyncIterable<FileMetaData>;
|
||||
|
||||
createFromFullBody(
|
||||
createFromFullContent(
|
||||
path: string,
|
||||
body: string | Uint8Array | Deno.Reader,
|
||||
content: string | Uint8Array,
|
||||
): Promise<FileMetaData>;
|
||||
createFromReader(path: string, body: Deno.Reader): Promise<FileMetaData>;
|
||||
|
||||
createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>;
|
||||
|
||||
getFullBodyByPath(path: string): Promise<string | null>;
|
||||
getRawContentByPath(path: string): Promise<Uint8Array | null>;
|
||||
|
||||
getRawBodyByPath(path: string): Promise<Uint8Array | null>;
|
||||
|
||||
getBodyAsReader(path: string): Promise<Deno.Reader | null>;
|
||||
getContentAsReader(path: string): Promise<Deno.Reader | null>;
|
||||
|
||||
getMetaDataByID(id: string): Promise<FileMetaData | null>;
|
||||
|
||||
@@ -42,14 +40,14 @@ export interface FileService {
|
||||
interface SerializedFile {
|
||||
id: string;
|
||||
path: string;
|
||||
bodySize: number;
|
||||
contentSize: number;
|
||||
compression: string;
|
||||
}
|
||||
|
||||
export interface FileMetaData {
|
||||
path: string;
|
||||
id: string;
|
||||
bodySize: number;
|
||||
contentSize: number;
|
||||
}
|
||||
|
||||
interface SerializedFilePathIndex {
|
||||
@@ -73,36 +71,36 @@ export class DenoKVBlobService {
|
||||
.join(""); // convert bytes to hex string
|
||||
}
|
||||
|
||||
private *splitText(body: Uint8Array): Generator<Uint8Array> {
|
||||
private *splitText(content: Uint8Array): Generator<Uint8Array> {
|
||||
let blobCount = 0;
|
||||
let txt = body.slice(
|
||||
let txt = content.slice(
|
||||
blobCount * this.blobSize,
|
||||
(blobCount + 1) * this.blobSize,
|
||||
);
|
||||
while (txt.length > 0) {
|
||||
yield txt;
|
||||
blobCount += 1;
|
||||
txt = body.slice(
|
||||
txt = content.slice(
|
||||
blobCount * this.blobSize,
|
||||
(blobCount + 1) * this.blobSize,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async unsafePersistBlob(mainIndex: string, index: number, body: Uint8Array): Promise<void> {
|
||||
if(body.length > this.blobSize){
|
||||
private async unsafePersistBlob(mainIndex: string, index: number, content: Uint8Array): Promise<void> {
|
||||
if(content.length > this.blobSize){
|
||||
throw new Error('Persisted a too large blob');
|
||||
}
|
||||
|
||||
const blob_id = await this.sha1(body);
|
||||
const blob_id = await this.sha1(content);
|
||||
await this.kv.set(['blob_index', mainIndex, index], blob_id);
|
||||
await this.kv.set(['blob', blob_id], body);
|
||||
await this.kv.set(['blob', blob_id], content);
|
||||
}
|
||||
|
||||
async persistBlob(index: string, body: Uint8Array): Promise<number> {
|
||||
async persistBlob(index: string, content: Uint8Array): Promise<number> {
|
||||
let blobCount = 0;
|
||||
let size = 0;
|
||||
for (const txt of this.splitText(body)) {
|
||||
for (const txt of this.splitText(content)) {
|
||||
await this.unsafePersistBlob(index, blobCount, txt);
|
||||
size += txt.length;
|
||||
blobCount += 1;
|
||||
@@ -110,10 +108,10 @@ export class DenoKVBlobService {
|
||||
return size;
|
||||
}
|
||||
|
||||
async persistReader(index: string, body: Deno.Reader): Promise<number>{
|
||||
async persistReader(index: string, content: Deno.Reader): Promise<number>{
|
||||
let i = 0;
|
||||
let size = 0;
|
||||
for await (const chunk of iterateReader(body, {bufSize: this.blobSize})){
|
||||
for await (const chunk of iterateReader(content, {bufSize: this.blobSize})){
|
||||
await this.unsafePersistBlob(index, i, chunk);
|
||||
size += chunk.length;
|
||||
i++;
|
||||
@@ -181,14 +179,14 @@ export class DenoKVFileService implements FileService {
|
||||
this.blobService = new DenoKVBlobService(kv, 32 * 1024);
|
||||
}
|
||||
|
||||
createFromFullBody(
|
||||
createFromFullContent(
|
||||
path: string,
|
||||
body: string | Uint8Array,
|
||||
content: string | Uint8Array,
|
||||
): Promise<FileMetaData> {
|
||||
if (body instanceof Uint8Array){
|
||||
return this.createFromReader(path, new Buffer(body));
|
||||
if (content instanceof Uint8Array){
|
||||
return this.createFromReader(path, new Buffer(content));
|
||||
}
|
||||
return this.createFromReader(path, new StringReader(body));
|
||||
return this.createFromReader(path, new StringReader(content));
|
||||
}
|
||||
|
||||
async createFromReader(path: string, content: Deno.Reader): Promise<FileMetaData>{
|
||||
@@ -199,14 +197,14 @@ export class DenoKVFileService implements FileService {
|
||||
const serializedFile: SerializedFile = {
|
||||
id: newId,
|
||||
path,
|
||||
bodySize: size,
|
||||
contentSize: size,
|
||||
compression: 'gzip'
|
||||
};
|
||||
const idx: SerializedFilePathIndex = { fileId: newId };
|
||||
await this.kv.set(['file', newId], serializedFile);
|
||||
await this.kv.set(['file_index_path', path], idx);
|
||||
|
||||
return { id: newId, path, bodySize: size };
|
||||
return { id: newId, path, contentSize: size };
|
||||
|
||||
}
|
||||
|
||||
@@ -224,7 +222,7 @@ export class DenoKVFileService implements FileService {
|
||||
yield {
|
||||
path: ret.value.path,
|
||||
id: ret.value.id,
|
||||
bodySize: ret.value.bodySize,
|
||||
contentSize: ret.value.contentSize,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -252,7 +250,7 @@ export class DenoKVFileService implements FileService {
|
||||
return ret.value
|
||||
}
|
||||
|
||||
private decompressBody(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
|
||||
private decompressContent(serializedFile: SerializedFile, content: Deno.Reader): Deno.Reader {
|
||||
if(serializedFile.compression == 'gzip'){
|
||||
const ret = readableStreamFromReader(content).pipeThrough(new DecompressionStream('gzip')).getReader();
|
||||
return readerFromStreamReader(ret);
|
||||
@@ -262,18 +260,13 @@ export class DenoKVFileService implements FileService {
|
||||
|
||||
}
|
||||
|
||||
async getFullBodyByPath(path: string): Promise<string | null> {
|
||||
const ret = await this.getRawBodyByPath(path);
|
||||
return ret == null ? null : new TextDecoder().decode(ret);
|
||||
}
|
||||
|
||||
async getRawBodyByPath(path: string): Promise<Uint8Array | null> {
|
||||
const ret = await this.getBodyAsReader(path);
|
||||
async getRawContentByPath(path: string): Promise<Uint8Array | null> {
|
||||
const ret = await this.getContentAsReader(path);
|
||||
return ret === null ? null : readAll(ret);
|
||||
}
|
||||
|
||||
|
||||
async getBodyAsReader(path: string): Promise<Deno.Reader | null> {
|
||||
async getContentAsReader(path: string): Promise<Deno.Reader | null> {
|
||||
const idx = await this.kv.get<SerializedFilePathIndex>([
|
||||
'file_index_path',
|
||||
path,
|
||||
@@ -285,7 +278,7 @@ export class DenoKVFileService implements FileService {
|
||||
if(ser.value === null){
|
||||
return null;
|
||||
}
|
||||
return this.decompressBody(ser.value, this.blobService.getBlob(idx.value.fileId));
|
||||
return this.decompressContent(ser.value, this.blobService.getBlob(idx.value.fileId));
|
||||
|
||||
}
|
||||
|
||||
@@ -297,7 +290,7 @@ export class DenoKVFileService implements FileService {
|
||||
return {
|
||||
path: fileObject.path,
|
||||
id: fileObject.id,
|
||||
bodySize: fileObject.bodySize,
|
||||
contentSize: fileObject.contentSize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -313,7 +306,7 @@ export class DenoKVFileService implements FileService {
|
||||
const newFile: SerializedFile = {
|
||||
path: newPath,
|
||||
id: fileObject.value.id,
|
||||
bodySize: fileObject.value.bodySize,
|
||||
contentSize: fileObject.value.contentSize,
|
||||
compression: fileObject.value.compression,
|
||||
};
|
||||
await this.kv.atomic()
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ function getFileHandlerFactory(fileService: FileService): Handler {
|
||||
return async (ctx: Context) => {
|
||||
const untypedCtx: any = ctx;
|
||||
const fileId = untypedCtx.params.path;
|
||||
const reader = await fileService.getBodyAsReader(fileId);
|
||||
const reader = await fileService.getContentAsReader(fileId);
|
||||
if(reader == null){
|
||||
ctx.response.status = 404;
|
||||
return;
|
||||
|
||||
Reference in new issue
Block a user