Added a very complicated AsyncIterableReader class.
The class was already implemented in "https://deno.land/std@0.144.0/streams/conversion.ts". So, I litterally lost 1 day of work! I hate myself.
This commit is contained in:
didier amyot committed
1 parent
17a1b4bed3
commit
0d627c9051
125 files changed
+26943
-83
No files matched your search
+16
@@ -0,0 +1,16 @@
|
||||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
// This module is browser compatible.
|
||||
|
||||
export class DenoStdInternalError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "DenoStdInternalError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Make an assertion, if not `true`, then throw. */
|
||||
export function assert(expr: unknown, msg = ""): asserts expr {
|
||||
if (!expr) {
|
||||
throw new DenoStdInternalError(msg);
|
||||
}
|
||||
}
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
// This module is browser compatible.
|
||||
|
||||
/**
|
||||
* An abstraction of multiple Uint8Arrays
|
||||
*/
|
||||
export class BytesList {
|
||||
#len = 0;
|
||||
#chunks: {
|
||||
value: Uint8Array;
|
||||
start: number; // start offset from head of chunk
|
||||
end: number; // end offset from head of chunk
|
||||
offset: number; // offset of head in all bytes
|
||||
}[] = [];
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* Total size of bytes
|
||||
*/
|
||||
size() {
|
||||
return this.#len;
|
||||
}
|
||||
/**
|
||||
* Push bytes with given offset infos
|
||||
*/
|
||||
add(value: Uint8Array, start = 0, end = value.byteLength) {
|
||||
if (value.byteLength === 0 || end - start === 0) {
|
||||
return;
|
||||
}
|
||||
checkRange(start, end, value.byteLength);
|
||||
this.#chunks.push({
|
||||
value,
|
||||
end,
|
||||
start,
|
||||
offset: this.#len,
|
||||
});
|
||||
this.#len += end - start;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop head `n` bytes.
|
||||
*/
|
||||
shift(n: number) {
|
||||
if (n === 0) {
|
||||
return;
|
||||
}
|
||||
if (this.#len <= n) {
|
||||
this.#chunks = [];
|
||||
this.#len = 0;
|
||||
return;
|
||||
}
|
||||
const idx = this.getChunkIndex(n);
|
||||
this.#chunks.splice(0, idx);
|
||||
const [chunk] = this.#chunks;
|
||||
if (chunk) {
|
||||
const diff = n - chunk.offset;
|
||||
chunk.start += diff;
|
||||
}
|
||||
let offset = 0;
|
||||
for (const chunk of this.#chunks) {
|
||||
chunk.offset = offset;
|
||||
offset += chunk.end - chunk.start;
|
||||
}
|
||||
this.#len = offset;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find chunk index in which `pos` locates by binary-search
|
||||
* returns -1 if out of range
|
||||
*/
|
||||
getChunkIndex(pos: number): number {
|
||||
let max = this.#chunks.length;
|
||||
let min = 0;
|
||||
while (true) {
|
||||
const i = min + Math.floor((max - min) / 2);
|
||||
if (i < 0 || this.#chunks.length <= i) {
|
||||
return -1;
|
||||
}
|
||||
const { offset, start, end } = this.#chunks[i];
|
||||
const len = end - start;
|
||||
if (offset <= pos && pos < offset + len) {
|
||||
return i;
|
||||
} else if (offset + len <= pos) {
|
||||
min = i + 1;
|
||||
} else {
|
||||
max = i - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get indexed byte from chunks
|
||||
*/
|
||||
get(i: number): number {
|
||||
if (i < 0 || this.#len <= i) {
|
||||
throw new Error("out of range");
|
||||
}
|
||||
const idx = this.getChunkIndex(i);
|
||||
const { value, offset, start } = this.#chunks[idx];
|
||||
return value[start + i - offset];
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterator of bytes from given position
|
||||
*/
|
||||
*iterator(start = 0): IterableIterator<number> {
|
||||
const startIdx = this.getChunkIndex(start);
|
||||
if (startIdx < 0) return;
|
||||
const first = this.#chunks[startIdx];
|
||||
let firstOffset = start - first.offset;
|
||||
for (let i = startIdx; i < this.#chunks.length; i++) {
|
||||
const chunk = this.#chunks[i];
|
||||
for (let j = chunk.start + firstOffset; j < chunk.end; j++) {
|
||||
yield chunk.value[j];
|
||||
}
|
||||
firstOffset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns subset of bytes copied
|
||||
*/
|
||||
slice(start: number, end: number = this.#len): Uint8Array {
|
||||
if (end === start) {
|
||||
return new Uint8Array();
|
||||
}
|
||||
checkRange(start, end, this.#len);
|
||||
const result = new Uint8Array(end - start);
|
||||
const startIdx = this.getChunkIndex(start);
|
||||
const endIdx = this.getChunkIndex(end - 1);
|
||||
let written = 0;
|
||||
for (let i = startIdx; i < endIdx; i++) {
|
||||
const chunk = this.#chunks[i];
|
||||
const len = chunk.end - chunk.start;
|
||||
result.set(chunk.value.subarray(chunk.start, chunk.end), written);
|
||||
written += len;
|
||||
}
|
||||
const last = this.#chunks[endIdx];
|
||||
const rest = end - start - written;
|
||||
result.set(last.value.subarray(last.start, last.start + rest), written);
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Concatenate chunks into single Uint8Array copied.
|
||||
*/
|
||||
concat(): Uint8Array {
|
||||
const result = new Uint8Array(this.#len);
|
||||
let sum = 0;
|
||||
for (const { value, start, end } of this.#chunks) {
|
||||
result.set(value.subarray(start, end), sum);
|
||||
sum += end - start;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
function checkRange(start: number, end: number, len: number) {
|
||||
if (start < 0 || len < start || end < 0 || len < end || end < start) {
|
||||
throw new Error("invalid range");
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
// This module is browser compatible.
|
||||
|
||||
/** Check whether binary arrays are equal to each other using 8-bit comparisons.
|
||||
* @private
|
||||
* @param a first array to check equality
|
||||
* @param b second array to check equality
|
||||
*/
|
||||
export function equalsNaive(a: Uint8Array, b: Uint8Array): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
for (let i = 0; i < b.length; i++) {
|
||||
if (a[i] !== b[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Check whether binary arrays are equal to each other using 32-bit comparisons.
|
||||
* @private
|
||||
* @param a first array to check equality
|
||||
* @param b second array to check equality
|
||||
*/
|
||||
export function equals32Bit(a: Uint8Array, b: Uint8Array): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
const len = a.length;
|
||||
const compressable = Math.floor(len / 4);
|
||||
const compressedA = new Uint32Array(a.buffer, 0, compressable);
|
||||
const compressedB = new Uint32Array(b.buffer, 0, compressable);
|
||||
for (let i = compressable * 4; i < len; i++) {
|
||||
if (a[i] !== b[i]) return false;
|
||||
}
|
||||
for (let i = 0; i < compressedA.length; i++) {
|
||||
if (compressedA[i] !== compressedB[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Check whether binary arrays are equal to each other.
|
||||
* @param a first array to check equality
|
||||
* @param b second array to check equality
|
||||
*/
|
||||
export function equals(a: Uint8Array, b: Uint8Array): boolean {
|
||||
if (a.length < 1000) return equalsNaive(a, b);
|
||||
return equals32Bit(a, b);
|
||||
}
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
// This module is browser compatible.
|
||||
|
||||
/**
|
||||
* Provides helper functions to manipulate `Uint8Array` byte slices that are not
|
||||
* included on the `Uint8Array` prototype.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
|
||||
/** Returns the index of the first occurrence of the needle array in the source
|
||||
* array, or -1 if it is not present.
|
||||
*
|
||||
* A start index can be specified as the third argument that begins the search
|
||||
* at that given index. The start index defaults to the start of the array.
|
||||
*
|
||||
* The complexity of this function is O(source.lenth * needle.length).
|
||||
*
|
||||
* ```ts
|
||||
* import { indexOfNeedle } from "./mod.ts";
|
||||
* const source = new Uint8Array([0, 1, 2, 1, 2, 1, 2, 3]);
|
||||
* const needle = new Uint8Array([1, 2]);
|
||||
* console.log(indexOfNeedle(source, needle)); // 1
|
||||
* console.log(indexOfNeedle(source, needle, 2)); // 3
|
||||
* ```
|
||||
*/
|
||||
export function indexOfNeedle(
|
||||
source: Uint8Array,
|
||||
needle: Uint8Array,
|
||||
start = 0,
|
||||
): number {
|
||||
if (start >= source.length) {
|
||||
return -1;
|
||||
}
|
||||
if (start < 0) {
|
||||
start = Math.max(0, source.length + start);
|
||||
}
|
||||
const s = needle[0];
|
||||
for (let i = start; i < source.length; i++) {
|
||||
if (source[i] !== s) continue;
|
||||
const pin = i;
|
||||
let matched = 1;
|
||||
let j = i;
|
||||
while (matched < needle.length) {
|
||||
j++;
|
||||
if (source[j] !== needle[j - pin]) {
|
||||
break;
|
||||
}
|
||||
matched++;
|
||||
}
|
||||
if (matched === needle.length) {
|
||||
return pin;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** Returns the index of the last occurrence of the needle array in the source
|
||||
* array, or -1 if it is not present.
|
||||
*
|
||||
* A start index can be specified as the third argument that begins the search
|
||||
* at that given index. The start index defaults to the end of the array.
|
||||
*
|
||||
* The complexity of this function is O(source.lenth * needle.length).
|
||||
*
|
||||
* ```ts
|
||||
* import { lastIndexOfNeedle } from "./mod.ts";
|
||||
* const source = new Uint8Array([0, 1, 2, 1, 2, 1, 2, 3]);
|
||||
* const needle = new Uint8Array([1, 2]);
|
||||
* console.log(lastIndexOfNeedle(source, needle)); // 5
|
||||
* console.log(lastIndexOfNeedle(source, needle, 4)); // 3
|
||||
* ```
|
||||
*/
|
||||
export function lastIndexOfNeedle(
|
||||
source: Uint8Array,
|
||||
needle: Uint8Array,
|
||||
start = source.length - 1,
|
||||
): number {
|
||||
if (start < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (start >= source.length) {
|
||||
start = source.length - 1;
|
||||
}
|
||||
const e = needle[needle.length - 1];
|
||||
for (let i = start; i >= 0; i--) {
|
||||
if (source[i] !== e) continue;
|
||||
const pin = i;
|
||||
let matched = 1;
|
||||
let j = i;
|
||||
while (matched < needle.length) {
|
||||
j--;
|
||||
if (source[j] !== needle[needle.length - 1 - (pin - j)]) {
|
||||
break;
|
||||
}
|
||||
matched++;
|
||||
}
|
||||
if (matched === needle.length) {
|
||||
return pin - needle.length + 1;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** Returns true if the prefix array appears at the start of the source array,
|
||||
* false otherwise.
|
||||
*
|
||||
* The complexity of this function is O(prefix.length).
|
||||
*
|
||||
* ```ts
|
||||
* import { startsWith } from "./mod.ts";
|
||||
* const source = new Uint8Array([0, 1, 2, 1, 2, 1, 2, 3]);
|
||||
* const prefix = new Uint8Array([0, 1, 2]);
|
||||
* console.log(startsWith(source, prefix)); // true
|
||||
* ```
|
||||
*/
|
||||
export function startsWith(source: Uint8Array, prefix: Uint8Array): boolean {
|
||||
for (let i = 0, max = prefix.length; i < max; i++) {
|
||||
if (source[i] !== prefix[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Returns true if the suffix array appears at the end of the source array,
|
||||
* false otherwise.
|
||||
*
|
||||
* The complexity of this function is O(suffix.length).
|
||||
*
|
||||
* ```ts
|
||||
* import { endsWith } from "./mod.ts";
|
||||
* const source = new Uint8Array([0, 1, 2, 1, 2, 1, 2, 3]);
|
||||
* const suffix = new Uint8Array([1, 2, 3]);
|
||||
* console.log(endsWith(source, suffix)); // true
|
||||
* ```
|
||||
*/
|
||||
export function endsWith(source: Uint8Array, suffix: Uint8Array): boolean {
|
||||
for (
|
||||
let srci = source.length - 1, sfxi = suffix.length - 1;
|
||||
sfxi >= 0;
|
||||
srci--, sfxi--
|
||||
) {
|
||||
if (source[srci] !== suffix[sfxi]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Returns a new Uint8Array composed of `count` repetitions of the `source`
|
||||
* array.
|
||||
*
|
||||
* If `count` is negative, a `RangeError` is thrown.
|
||||
*
|
||||
* ```ts
|
||||
* import { repeat } from "./mod.ts";
|
||||
* const source = new Uint8Array([0, 1, 2]);
|
||||
* console.log(repeat(source, 3)); // [0, 1, 2, 0, 1, 2, 0, 1, 2]
|
||||
* console.log(repeat(source, 0)); // []
|
||||
* console.log(repeat(source, -1)); // RangeError
|
||||
* ```
|
||||
*/
|
||||
export function repeat(source: Uint8Array, count: number): Uint8Array {
|
||||
if (count === 0) {
|
||||
return new Uint8Array();
|
||||
}
|
||||
|
||||
if (count < 0) {
|
||||
throw new RangeError("bytes: negative repeat count");
|
||||
} else if ((source.length * count) / count !== source.length) {
|
||||
throw new Error("bytes: repeat count causes overflow");
|
||||
}
|
||||
|
||||
const int = Math.floor(count);
|
||||
|
||||
if (int !== count) {
|
||||
throw new Error("bytes: repeat count must be an integer");
|
||||
}
|
||||
|
||||
const nb = new Uint8Array(source.length * count);
|
||||
|
||||
let bp = copy(source, nb);
|
||||
|
||||
for (; bp < nb.length; bp *= 2) {
|
||||
copy(nb.slice(0, bp), nb, bp);
|
||||
}
|
||||
|
||||
return nb;
|
||||
}
|
||||
|
||||
/** Concatenate the given arrays into a new Uint8Array.
|
||||
*
|
||||
* ```ts
|
||||
* import { concat } from "./mod.ts";
|
||||
* const a = new Uint8Array([0, 1, 2]);
|
||||
* const b = new Uint8Array([3, 4, 5]);
|
||||
* console.log(concat(a, b)); // [0, 1, 2, 3, 4, 5]
|
||||
*/
|
||||
export function concat(...buf: Uint8Array[]): Uint8Array {
|
||||
let length = 0;
|
||||
for (const b of buf) {
|
||||
length += b.length;
|
||||
}
|
||||
|
||||
const output = new Uint8Array(length);
|
||||
let index = 0;
|
||||
for (const b of buf) {
|
||||
output.set(b, index);
|
||||
index += b.length;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/** Returns true if the source array contains the needle array, false otherwise.
|
||||
*
|
||||
* A start index can be specified as the third argument that begins the search
|
||||
* at that given index. The start index defaults to the beginning of the array.
|
||||
*
|
||||
* The complexity of this function is O(source.length * needle.length).
|
||||
*
|
||||
* ```ts
|
||||
* import { includesNeedle } from "./mod.ts";
|
||||
* const source = new Uint8Array([0, 1, 2, 1, 2, 1, 2, 3]);
|
||||
* const needle = new Uint8Array([1, 2]);
|
||||
* console.log(includesNeedle(source, needle)); // true
|
||||
* console.log(includesNeedle(source, needle, 6)); // false
|
||||
* ```
|
||||
*/
|
||||
export function includesNeedle(
|
||||
source: Uint8Array,
|
||||
needle: Uint8Array,
|
||||
start = 0,
|
||||
): boolean {
|
||||
return indexOfNeedle(source, needle, start) !== -1;
|
||||
}
|
||||
|
||||
/** Copy bytes from the `src` array to the `dst` array. Returns the number of
|
||||
* bytes copied.
|
||||
*
|
||||
* If the `src` array is larger than what the `dst` array can hold, only the
|
||||
* amount of bytes that fit in the `dst` array are copied.
|
||||
*
|
||||
* An offset can be specified as the third argument that begins the copy at
|
||||
* that given index in the `dst` array. The offset defaults to the beginning of
|
||||
* the array.
|
||||
*
|
||||
* ```ts
|
||||
* import { copy } from "./mod.ts";
|
||||
* const src = new Uint8Array([9, 8, 7]);
|
||||
* const dst = new Uint8Array([0, 1, 2, 3, 4, 5]);
|
||||
* console.log(copy(src, dst)); // 3
|
||||
* console.log(dst); // [9, 8, 7, 3, 4, 5]
|
||||
* ```
|
||||
*
|
||||
* ```ts
|
||||
* import { copy } from "./mod.ts";
|
||||
* const src = new Uint8Array([1, 1, 1, 1]);
|
||||
* const dst = new Uint8Array([0, 0, 0, 0]);
|
||||
* console.log(copy(src, dst, 1)); // 3
|
||||
* console.log(dst); // [0, 1, 1, 1]
|
||||
* ```
|
||||
*/
|
||||
export function copy(src: Uint8Array, dst: Uint8Array, off = 0): number {
|
||||
off = Math.max(0, Math.min(off, dst.byteLength));
|
||||
const dstBytesAvailable = dst.byteLength - off;
|
||||
if (src.byteLength > dstBytesAvailable) {
|
||||
src = src.subarray(0, dstBytesAvailable);
|
||||
}
|
||||
dst.set(src, off);
|
||||
return src.byteLength;
|
||||
}
|
||||
|
||||
export { equals } from "./equals.ts";
|
||||
+1017
File diff suppressed because it is too large.
Load diff
+82
@@ -0,0 +1,82 @@
|
||||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
export interface Reader {
|
||||
/** Reads up to `p.byteLength` bytes into `p`. It resolves to the number of
|
||||
* bytes read (`0` < `n` <= `p.byteLength`) and rejects if any error
|
||||
* encountered. Even if `read()` resolves to `n` < `p.byteLength`, it may
|
||||
* use all of `p` as scratch space during the call. If some data is
|
||||
* available but not `p.byteLength` bytes, `read()` conventionally resolves
|
||||
* to what is available instead of waiting for more.
|
||||
*
|
||||
* When `read()` encounters end-of-file condition, it resolves to EOF
|
||||
* (`null`).
|
||||
*
|
||||
* When `read()` encounters an error, it rejects with an error.
|
||||
*
|
||||
* Callers should always process the `n` > `0` bytes returned before
|
||||
* considering the EOF (`null`). Doing so correctly handles I/O errors that
|
||||
* happen after reading some bytes and also both of the allowed EOF
|
||||
* behaviors.
|
||||
*
|
||||
* Implementations should not retain a reference to `p`.
|
||||
*
|
||||
* Use iterateReader() from https://deno.land/std/streams/conversion.ts to turn a Reader into an
|
||||
* AsyncIterator.
|
||||
*/
|
||||
read(p: Uint8Array): Promise<number | null>;
|
||||
}
|
||||
|
||||
export interface ReaderSync {
|
||||
/** Reads up to `p.byteLength` bytes into `p`. It resolves to the number
|
||||
* of bytes read (`0` < `n` <= `p.byteLength`) and rejects if any error
|
||||
* encountered. Even if `read()` returns `n` < `p.byteLength`, it may use
|
||||
* all of `p` as scratch space during the call. If some data is available
|
||||
* but not `p.byteLength` bytes, `read()` conventionally returns what is
|
||||
* available instead of waiting for more.
|
||||
*
|
||||
* When `readSync()` encounters end-of-file condition, it returns EOF
|
||||
* (`null`).
|
||||
*
|
||||
* When `readSync()` encounters an error, it throws with an error.
|
||||
*
|
||||
* Callers should always process the `n` > `0` bytes returned before
|
||||
* considering the EOF (`null`). Doing so correctly handles I/O errors that happen
|
||||
* after reading some bytes and also both of the allowed EOF behaviors.
|
||||
*
|
||||
* Implementations should not retain a reference to `p`.
|
||||
*
|
||||
* Use iterateReaderSync() from https://deno.land/std/streams/conversion.ts to turn a ReaderSync
|
||||
* into an Iterator.
|
||||
*/
|
||||
readSync(p: Uint8Array): number | null;
|
||||
}
|
||||
|
||||
export interface Writer {
|
||||
/** Writes `p.byteLength` bytes from `p` to the underlying data stream. It
|
||||
* resolves to the number of bytes written from `p` (`0` <= `n` <=
|
||||
* `p.byteLength`) or reject with the error encountered that caused the
|
||||
* write to stop early. `write()` must reject with a non-null error if
|
||||
* would resolve to `n` < `p.byteLength`. `write()` must not modify the
|
||||
* slice data, even temporarily.
|
||||
*
|
||||
* Implementations should not retain a reference to `p`.
|
||||
*/
|
||||
write(p: Uint8Array): Promise<number>;
|
||||
}
|
||||
|
||||
export interface WriterSync {
|
||||
/** Writes `p.byteLength` bytes from `p` to the underlying data
|
||||
* stream. It returns the number of bytes written from `p` (`0` <= `n`
|
||||
* <= `p.byteLength`) and any error encountered that caused the write to
|
||||
* stop early. `writeSync()` must throw a non-null error if it returns `n` <
|
||||
* `p.byteLength`. `writeSync()` must not modify the slice data, even
|
||||
* temporarily.
|
||||
*
|
||||
* Implementations should not retain a reference to `p`.
|
||||
*/
|
||||
writeSync(p: Uint8Array): number;
|
||||
}
|
||||
|
||||
export interface Closer {
|
||||
close(): void;
|
||||
}
|
||||
+600
@@ -0,0 +1,600 @@
|
||||
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
|
||||
|
||||
import { Buffer } from "../io/buffer.ts";
|
||||
|
||||
const DEFAULT_CHUNK_SIZE = 16_640;
|
||||
const DEFAULT_BUFFER_SIZE = 32 * 1024;
|
||||
|
||||
function isCloser(value: unknown): value is Deno.Closer {
|
||||
return typeof value === "object" && value != null && "close" in value &&
|
||||
// deno-lint-ignore no-explicit-any
|
||||
typeof (value as Record<string, any>)["close"] === "function";
|
||||
}
|
||||
|
||||
/** Create a `Deno.Reader` from an iterable of `Uint8Array`s.
|
||||
*
|
||||
* ```ts
|
||||
* import { readerFromIterable, copy } from "./conversion.ts";
|
||||
*
|
||||
* const file = await Deno.open("metrics.txt", { write: true });
|
||||
* const reader = readerFromIterable((async function* () {
|
||||
* while (true) {
|
||||
* await new Promise((r) => setTimeout(r, 1000));
|
||||
* const message = `data: ${JSON.stringify(Deno.metrics())}\n\n`;
|
||||
* yield new TextEncoder().encode(message);
|
||||
* }
|
||||
* })());
|
||||
* await copy(reader, file);
|
||||
* ```
|
||||
*/
|
||||
export function readerFromIterable(
|
||||
iterable: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
||||
): Deno.Reader {
|
||||
const iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array> =
|
||||
(iterable as AsyncIterable<Uint8Array>)[Symbol.asyncIterator]?.() ??
|
||||
(iterable as Iterable<Uint8Array>)[Symbol.iterator]?.();
|
||||
const buffer = new Buffer();
|
||||
return {
|
||||
async read(p: Uint8Array): Promise<number | null> {
|
||||
if (buffer.length == 0) {
|
||||
const result = await iterator.next();
|
||||
if (result.done) {
|
||||
return null;
|
||||
} else {
|
||||
if (result.value.byteLength <= p.byteLength) {
|
||||
p.set(result.value);
|
||||
return result.value.byteLength;
|
||||
}
|
||||
p.set(result.value.subarray(0, p.byteLength));
|
||||
await writeAll(buffer, result.value.subarray(p.byteLength));
|
||||
return p.byteLength;
|
||||
}
|
||||
} else {
|
||||
const n = await buffer.read(p);
|
||||
if (n == null) {
|
||||
return this.read(p);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Create a `Writer` from a `WritableStreamDefaultWriter`. */
|
||||
export function writerFromStreamWriter(
|
||||
streamWriter: WritableStreamDefaultWriter<Uint8Array>,
|
||||
): Deno.Writer {
|
||||
return {
|
||||
async write(p: Uint8Array): Promise<number> {
|
||||
await streamWriter.ready;
|
||||
await streamWriter.write(p);
|
||||
return p.length;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Create a `Reader` from a `ReadableStreamDefaultReader`. */
|
||||
export function readerFromStreamReader(
|
||||
streamReader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
): Deno.Reader {
|
||||
const buffer = new Buffer();
|
||||
|
||||
return {
|
||||
async read(p: Uint8Array): Promise<number | null> {
|
||||
if (buffer.empty()) {
|
||||
const res = await streamReader.read();
|
||||
if (res.done) {
|
||||
return null; // EOF
|
||||
}
|
||||
|
||||
await writeAll(buffer, res.value);
|
||||
}
|
||||
|
||||
return buffer.read(p);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface WritableStreamFromWriterOptions {
|
||||
/**
|
||||
* If the `writer` is also a `Deno.Closer`, automatically close the `writer`
|
||||
* when the stream is closed, aborted, or a write error occurs.
|
||||
*
|
||||
* Defaults to `true`. */
|
||||
autoClose?: boolean;
|
||||
}
|
||||
|
||||
/** Create a `WritableStream` from a `Writer`. */
|
||||
export function writableStreamFromWriter(
|
||||
writer: Deno.Writer,
|
||||
options: WritableStreamFromWriterOptions = {},
|
||||
): WritableStream<Uint8Array> {
|
||||
const { autoClose = true } = options;
|
||||
|
||||
return new WritableStream({
|
||||
async write(chunk, controller) {
|
||||
try {
|
||||
await writeAll(writer, chunk);
|
||||
} catch (e) {
|
||||
controller.error(e);
|
||||
if (isCloser(writer) && autoClose) {
|
||||
writer.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
close() {
|
||||
if (isCloser(writer) && autoClose) {
|
||||
writer.close();
|
||||
}
|
||||
},
|
||||
abort() {
|
||||
if (isCloser(writer) && autoClose) {
|
||||
writer.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Create a `ReadableStream` from any kind of iterable.
|
||||
*
|
||||
* ```ts
|
||||
* import { readableStreamFromIterable } from "./conversion.ts";
|
||||
*
|
||||
* const r1 = readableStreamFromIterable(["foo, bar, baz"]);
|
||||
* const r2 = readableStreamFromIterable(async function* () {
|
||||
* await new Promise(((r) => setTimeout(r, 1000)));
|
||||
* yield "foo";
|
||||
* await new Promise(((r) => setTimeout(r, 1000)));
|
||||
* yield "bar";
|
||||
* await new Promise(((r) => setTimeout(r, 1000)));
|
||||
* yield "baz";
|
||||
* }());
|
||||
* ```
|
||||
*
|
||||
* If the produced iterator (`iterable[Symbol.asyncIterator]()` or
|
||||
* `iterable[Symbol.iterator]()`) is a generator, or more specifically is found
|
||||
* to have a `.throw()` method on it, that will be called upon
|
||||
* `readableStream.cancel()`. This is the case for the second input type above:
|
||||
*
|
||||
* ```ts
|
||||
* import { readableStreamFromIterable } from "./conversion.ts";
|
||||
*
|
||||
* const r3 = readableStreamFromIterable(async function* () {
|
||||
* try {
|
||||
* yield "foo";
|
||||
* } catch (error) {
|
||||
* console.log(error); // Error: Cancelled by consumer.
|
||||
* }
|
||||
* }());
|
||||
* const reader = r3.getReader();
|
||||
* console.log(await reader.read()); // { value: "foo", done: false }
|
||||
* await reader.cancel(new Error("Cancelled by consumer."));
|
||||
* ```
|
||||
*/
|
||||
export function readableStreamFromIterable<T>(
|
||||
iterable: Iterable<T> | AsyncIterable<T>,
|
||||
): ReadableStream<T> {
|
||||
const iterator: Iterator<T> | AsyncIterator<T> =
|
||||
(iterable as AsyncIterable<T>)[Symbol.asyncIterator]?.() ??
|
||||
(iterable as Iterable<T>)[Symbol.iterator]?.();
|
||||
return new ReadableStream({
|
||||
async pull(controller) {
|
||||
const { value, done } = await iterator.next();
|
||||
if (done) {
|
||||
controller.close();
|
||||
} else {
|
||||
controller.enqueue(value);
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
if (typeof iterator.throw == "function") {
|
||||
try {
|
||||
await iterator.throw(reason);
|
||||
} catch { /* `iterator.throw()` always throws on site. We catch it. */ }
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the generator function into a TransformStream.
|
||||
*
|
||||
* ```ts
|
||||
* import { readableStreamFromIterable, toTransformStream } from "./conversion.ts";
|
||||
*
|
||||
* const readable = readableStreamFromIterable([0, 1, 2])
|
||||
* .pipeThrough(toTransformStream(async function* (src) {
|
||||
* for await (const chunk of src) {
|
||||
* yield chunk * 100;
|
||||
* }
|
||||
* }));
|
||||
*
|
||||
* for await (const chunk of readable) {
|
||||
* console.log(chunk);
|
||||
* }
|
||||
* // output: 0, 100, 200
|
||||
* ```
|
||||
*
|
||||
* @param transformer A function to transform.
|
||||
* @param writableStrategy An object that optionally defines a queuing strategy for the stream.
|
||||
* @param readableStrategy An object that optionally defines a queuing strategy for the stream.
|
||||
*/
|
||||
export function toTransformStream<I, O>(
|
||||
transformer: (src: ReadableStream<I>) => Iterable<O> | AsyncIterable<O>,
|
||||
writableStrategy?: QueuingStrategy<I>,
|
||||
readableStrategy?: QueuingStrategy<O>,
|
||||
): TransformStream<I, O> {
|
||||
const {
|
||||
writable,
|
||||
readable,
|
||||
} = new TransformStream<I, I>(undefined, writableStrategy);
|
||||
|
||||
const iterable = transformer(readable);
|
||||
const iterator: Iterator<O> | AsyncIterator<O> =
|
||||
(iterable as AsyncIterable<O>)[Symbol.asyncIterator]?.() ??
|
||||
(iterable as Iterable<O>)[Symbol.iterator]?.();
|
||||
return {
|
||||
writable,
|
||||
readable: new ReadableStream<O>({
|
||||
async pull(controller) {
|
||||
let result: IteratorResult<O>;
|
||||
try {
|
||||
result = await iterator.next();
|
||||
} catch (error) {
|
||||
// Propagate error to stream from iterator
|
||||
// If the stream status is "errored", it will be thrown, but ignore.
|
||||
await readable.cancel(error).catch(() => {});
|
||||
controller.error(error);
|
||||
return;
|
||||
}
|
||||
if (result.done) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(result.value);
|
||||
},
|
||||
async cancel(reason) {
|
||||
// Propagate cancellation to readable and iterator
|
||||
if (typeof iterator.throw == "function") {
|
||||
try {
|
||||
await iterator.throw(reason);
|
||||
} catch {
|
||||
/* `iterator.throw()` always throws on site. We catch it. */
|
||||
}
|
||||
}
|
||||
await readable.cancel(reason);
|
||||
},
|
||||
}, readableStrategy),
|
||||
};
|
||||
}
|
||||
|
||||
export interface ReadableStreamFromReaderOptions {
|
||||
/** If the `reader` is also a `Deno.Closer`, automatically close the `reader`
|
||||
* when `EOF` is encountered, or a read error occurs.
|
||||
*
|
||||
* Defaults to `true`. */
|
||||
autoClose?: boolean;
|
||||
|
||||
/** The size of chunks to allocate to read, the default is ~16KiB, which is
|
||||
* the maximum size that Deno operations can currently support. */
|
||||
chunkSize?: number;
|
||||
|
||||
/** The queuing strategy to create the `ReadableStream` with. */
|
||||
strategy?: { highWaterMark?: number | undefined; size?: undefined };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a `ReadableStream<Uint8Array>` from from a `Deno.Reader`.
|
||||
*
|
||||
* When the pull algorithm is called on the stream, a chunk from the reader
|
||||
* will be read. When `null` is returned from the reader, the stream will be
|
||||
* closed along with the reader (if it is also a `Deno.Closer`).
|
||||
*
|
||||
* An example converting a `Deno.FsFile` into a readable stream:
|
||||
*
|
||||
* ```ts
|
||||
* import { readableStreamFromReader } from "./mod.ts";
|
||||
*
|
||||
* const file = await Deno.open("./file.txt", { read: true });
|
||||
* const fileStream = readableStreamFromReader(file);
|
||||
* ```
|
||||
*/
|
||||
export function readableStreamFromReader(
|
||||
reader: Deno.Reader | (Deno.Reader & Deno.Closer),
|
||||
options: ReadableStreamFromReaderOptions = {},
|
||||
): ReadableStream<Uint8Array> {
|
||||
const {
|
||||
autoClose = true,
|
||||
chunkSize = DEFAULT_CHUNK_SIZE,
|
||||
strategy,
|
||||
} = options;
|
||||
|
||||
return new ReadableStream({
|
||||
async pull(controller) {
|
||||
const chunk = new Uint8Array(chunkSize);
|
||||
try {
|
||||
const read = await reader.read(chunk);
|
||||
if (read === null) {
|
||||
if (isCloser(reader) && autoClose) {
|
||||
reader.close();
|
||||
}
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(chunk.subarray(0, read));
|
||||
} catch (e) {
|
||||
controller.error(e);
|
||||
if (isCloser(reader)) {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
if (isCloser(reader) && autoClose) {
|
||||
reader.close();
|
||||
}
|
||||
},
|
||||
}, strategy);
|
||||
}
|
||||
|
||||
/** Read Reader `r` until EOF (`null`) and resolve to the content as
|
||||
* Uint8Array`.
|
||||
*
|
||||
* ```ts
|
||||
* import { Buffer } from "../io/buffer.ts";
|
||||
* import { readAll } from "./conversion.ts";
|
||||
*
|
||||
* // Example from stdin
|
||||
* const stdinContent = await readAll(Deno.stdin);
|
||||
*
|
||||
* // Example from file
|
||||
* const file = await Deno.open("my_file.txt", {read: true});
|
||||
* const myFileContent = await readAll(file);
|
||||
* Deno.close(file.rid);
|
||||
*
|
||||
* // Example from buffer
|
||||
* const myData = new Uint8Array(100);
|
||||
* // ... fill myData array with data
|
||||
* const reader = new Buffer(myData.buffer);
|
||||
* const bufferContent = await readAll(reader);
|
||||
* ```
|
||||
*/
|
||||
export async function readAll(r: Deno.Reader): Promise<Uint8Array> {
|
||||
const buf = new Buffer();
|
||||
await buf.readFrom(r);
|
||||
return buf.bytes();
|
||||
}
|
||||
|
||||
/** Synchronously reads Reader `r` until EOF (`null`) and returns the content
|
||||
* as `Uint8Array`.
|
||||
*
|
||||
* ```ts
|
||||
* import { Buffer } from "../io/buffer.ts";
|
||||
* import { readAllSync } from "./conversion.ts";
|
||||
*
|
||||
* // Example from stdin
|
||||
* const stdinContent = readAllSync(Deno.stdin);
|
||||
*
|
||||
* // Example from file
|
||||
* const file = Deno.openSync("my_file.txt", {read: true});
|
||||
* const myFileContent = readAllSync(file);
|
||||
* Deno.close(file.rid);
|
||||
*
|
||||
* // Example from buffer
|
||||
* const myData = new Uint8Array(100);
|
||||
* // ... fill myData array with data
|
||||
* const reader = new Buffer(myData.buffer);
|
||||
* const bufferContent = readAllSync(reader);
|
||||
* ```
|
||||
*/
|
||||
export function readAllSync(r: Deno.ReaderSync): Uint8Array {
|
||||
const buf = new Buffer();
|
||||
buf.readFromSync(r);
|
||||
return buf.bytes();
|
||||
}
|
||||
|
||||
/** Write all the content of the array buffer (`arr`) to the writer (`w`).
|
||||
*
|
||||
* ```ts
|
||||
* import { Buffer } from "../io/buffer.ts";
|
||||
* import { writeAll } from "./conversion.ts";
|
||||
|
||||
* // Example writing to stdout
|
||||
* let contentBytes = new TextEncoder().encode("Hello World");
|
||||
* await writeAll(Deno.stdout, contentBytes);
|
||||
*
|
||||
* // Example writing to file
|
||||
* contentBytes = new TextEncoder().encode("Hello World");
|
||||
* const file = await Deno.open('test.file', {write: true});
|
||||
* await writeAll(file, contentBytes);
|
||||
* Deno.close(file.rid);
|
||||
*
|
||||
* // Example writing to buffer
|
||||
* contentBytes = new TextEncoder().encode("Hello World");
|
||||
* const writer = new Buffer();
|
||||
* await writeAll(writer, contentBytes);
|
||||
* console.log(writer.bytes().length); // 11
|
||||
* ```
|
||||
*/
|
||||
export async function writeAll(w: Deno.Writer, arr: Uint8Array) {
|
||||
let nwritten = 0;
|
||||
while (nwritten < arr.length) {
|
||||
nwritten += await w.write(arr.subarray(nwritten));
|
||||
}
|
||||
}
|
||||
|
||||
/** Synchronously write all the content of the array buffer (`arr`) to the
|
||||
* writer (`w`).
|
||||
*
|
||||
* ```ts
|
||||
* import { Buffer } from "../io/buffer.ts";
|
||||
* import { writeAllSync } from "./conversion.ts";
|
||||
*
|
||||
* // Example writing to stdout
|
||||
* let contentBytes = new TextEncoder().encode("Hello World");
|
||||
* writeAllSync(Deno.stdout, contentBytes);
|
||||
*
|
||||
* // Example writing to file
|
||||
* contentBytes = new TextEncoder().encode("Hello World");
|
||||
* const file = Deno.openSync('test.file', {write: true});
|
||||
* writeAllSync(file, contentBytes);
|
||||
* Deno.close(file.rid);
|
||||
*
|
||||
* // Example writing to buffer
|
||||
* contentBytes = new TextEncoder().encode("Hello World");
|
||||
* const writer = new Buffer();
|
||||
* writeAllSync(writer, contentBytes);
|
||||
* console.log(writer.bytes().length); // 11
|
||||
* ```
|
||||
*/
|
||||
export function writeAllSync(w: Deno.WriterSync, arr: Uint8Array): void {
|
||||
let nwritten = 0;
|
||||
while (nwritten < arr.length) {
|
||||
nwritten += w.writeSync(arr.subarray(nwritten));
|
||||
}
|
||||
}
|
||||
|
||||
/** Turns a Reader, `r`, into an async iterator.
|
||||
*
|
||||
* ```ts
|
||||
* import { iterateReader } from "./conversion.ts";
|
||||
*
|
||||
* let f = await Deno.open("/etc/passwd");
|
||||
* for await (const chunk of iterateReader(f)) {
|
||||
* console.log(chunk);
|
||||
* }
|
||||
* f.close();
|
||||
* ```
|
||||
*
|
||||
* Second argument can be used to tune size of a buffer.
|
||||
* Default size of the buffer is 32kB.
|
||||
*
|
||||
* ```ts
|
||||
* import { iterateReader } from "./conversion.ts";
|
||||
*
|
||||
* let f = await Deno.open("/etc/passwd");
|
||||
* const it = iterateReader(f, {
|
||||
* bufSize: 1024 * 1024
|
||||
* });
|
||||
* for await (const chunk of it) {
|
||||
* console.log(chunk);
|
||||
* }
|
||||
* f.close();
|
||||
* ```
|
||||
*
|
||||
* Iterator uses an internal buffer of fixed size for efficiency; it returns
|
||||
* a view on that buffer on each iteration. It is therefore caller's
|
||||
* responsibility to copy contents of the buffer if needed; otherwise the
|
||||
* next iteration will overwrite contents of previously returned chunk.
|
||||
*/
|
||||
export async function* iterateReader(
|
||||
r: Deno.Reader,
|
||||
options?: {
|
||||
bufSize?: number;
|
||||
},
|
||||
): AsyncIterableIterator<Uint8Array> {
|
||||
const bufSize = options?.bufSize ?? DEFAULT_BUFFER_SIZE;
|
||||
const b = new Uint8Array(bufSize);
|
||||
while (true) {
|
||||
const result = await r.read(b);
|
||||
if (result === null) {
|
||||
break;
|
||||
}
|
||||
|
||||
yield b.subarray(0, result);
|
||||
}
|
||||
}
|
||||
|
||||
/** Turns a ReaderSync, `r`, into an iterator.
|
||||
*
|
||||
* ```ts
|
||||
* import { iterateReaderSync } from "./conversion.ts";
|
||||
*
|
||||
* let f = Deno.openSync("/etc/passwd");
|
||||
* for (const chunk of iterateReaderSync(f)) {
|
||||
* console.log(chunk);
|
||||
* }
|
||||
* f.close();
|
||||
* ```
|
||||
*
|
||||
* Second argument can be used to tune size of a buffer.
|
||||
* Default size of the buffer is 32kB.
|
||||
*
|
||||
* ```ts
|
||||
* import { iterateReaderSync } from "./conversion.ts";
|
||||
|
||||
* let f = await Deno.open("/etc/passwd");
|
||||
* const iter = iterateReaderSync(f, {
|
||||
* bufSize: 1024 * 1024
|
||||
* });
|
||||
* for (const chunk of iter) {
|
||||
* console.log(chunk);
|
||||
* }
|
||||
* f.close();
|
||||
* ```
|
||||
*
|
||||
* Iterator uses an internal buffer of fixed size for efficiency; it returns
|
||||
* a view on that buffer on each iteration. It is therefore caller's
|
||||
* responsibility to copy contents of the buffer if needed; otherwise the
|
||||
* next iteration will overwrite contents of previously returned chunk.
|
||||
*/
|
||||
export function* iterateReaderSync(
|
||||
r: Deno.ReaderSync,
|
||||
options?: {
|
||||
bufSize?: number;
|
||||
},
|
||||
): IterableIterator<Uint8Array> {
|
||||
const bufSize = options?.bufSize ?? DEFAULT_BUFFER_SIZE;
|
||||
const b = new Uint8Array(bufSize);
|
||||
while (true) {
|
||||
const result = r.readSync(b);
|
||||
if (result === null) {
|
||||
break;
|
||||
}
|
||||
|
||||
yield b.subarray(0, result);
|
||||
}
|
||||
}
|
||||
|
||||
/** Copies from `src` to `dst` until either EOF (`null`) is read from `src` or
|
||||
* an error occurs. It resolves to the number of bytes copied or rejects with
|
||||
* the first error encountered while copying.
|
||||
*
|
||||
* ```ts
|
||||
* import { copy } from "./conversion.ts";
|
||||
*
|
||||
* const source = await Deno.open("my_file.txt");
|
||||
* const bytesCopied1 = await copy(source, Deno.stdout);
|
||||
* const destination = await Deno.create("my_file_2.txt");
|
||||
* const bytesCopied2 = await copy(source, destination);
|
||||
* ```
|
||||
*
|
||||
* @param src The source to copy from
|
||||
* @param dst The destination to copy to
|
||||
* @param options Can be used to tune size of the buffer. Default size is 32kB
|
||||
*/
|
||||
export async function copy(
|
||||
src: Deno.Reader,
|
||||
dst: Deno.Writer,
|
||||
options?: {
|
||||
bufSize?: number;
|
||||
},
|
||||
): Promise<number> {
|
||||
let n = 0;
|
||||
const bufSize = options?.bufSize ?? DEFAULT_BUFFER_SIZE;
|
||||
const b = new Uint8Array(bufSize);
|
||||
let gotEOF = false;
|
||||
while (gotEOF === false) {
|
||||
const result = await src.read(b);
|
||||
if (result === null) {
|
||||
gotEOF = true;
|
||||
} else {
|
||||
let nwritten = 0;
|
||||
while (nwritten < result) {
|
||||
nwritten += await dst.write(b.subarray(nwritten, result));
|
||||
}
|
||||
n += nwritten;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
Reference in new issue
Block a user