feat(encoding/unstable): adds streaming versions for hex, base32, base32hex, base64 and base64url (#4915)

* feat(encoding): adds streaming versions of several encoding methods.

Adds streaming methods for:
- Hex
- Base32
- Base64
- Base64Url

* move(encoding): streaming versions to their own modules.

* fix(encoding): jsdoc examples

* fix(encoding): error when chunk's are smaller than the minimum size.

* add(encoding): Base32Hex Encoder/Decoder Streams

* fix(encoding): mod.ts not exporting ./base32hex_stream.ts

* hex stream

* RandomSliceStream

* polish

* base32

* base64hex

* base64

* base64url

* tweak

* fix

---------

Co-authored-by: Asher Gomez <ashersaupingomez@gmail.com>
This commit is contained in:
Doctor 2024-08-22 15:02:36 +10:00 committed by GitHub
parent b0708f49aa
commit 0f7a71f96b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 691 additions and 0 deletions

View File

@ -0,0 +1,20 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
type Sliceable = {
slice(start?: number, end?: number): Sliceable;
length: number;
};
export class RandomSliceStream<T extends Sliceable>
extends TransformStream<T, T> {
constructor() {
super({
transform(chunk, controller) {
const i = Math.floor(Math.random() * chunk.length);
controller.enqueue(chunk.slice(0, i) as T);
controller.enqueue(chunk.slice(i) as T);
},
});
}
}

99
encoding/base32_stream.ts Normal file
View File

@ -0,0 +1,99 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
/**
* Utilities for encoding and decoding to and from base32 in a streaming manner.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @module
*/
import { decodeBase32, encodeBase32 } from "./base32.ts";
/**
* Converts a Uint8Array stream into a base32-encoded stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-6}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeBase32 } from "@std/encoding/base32";
* import { Base32EncoderStream } from "@std/encoding/base32-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["Hello,", " world!"])
* .pipeThrough(new TextEncoderStream())
* .pipeThrough(new Base32EncoderStream());
*
* assertEquals(await toText(stream), encodeBase32(new TextEncoder().encode("Hello, world!")));
* ```
*/
export class Base32EncoderStream extends TransformStream<Uint8Array, string> {
constructor() {
let push = new Uint8Array(0);
super({
transform(chunk, controller) {
const concat = new Uint8Array(push.length + chunk.length);
concat.set(push);
concat.set(chunk, push.length);
const remainder = -concat.length % 5;
controller.enqueue(
encodeBase32(concat.slice(0, remainder || undefined)),
);
push = remainder ? concat.slice(remainder) : new Uint8Array(0);
},
flush(controller) {
if (push.length) {
controller.enqueue(encodeBase32(push));
}
},
});
}
}
/**
* Decodes a base32-encoded stream into a Uint8Array stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-6}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { Base32DecoderStream } from "@std/encoding/base32-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["JBSWY3DPEBLW64TMMQQQ===="])
* .pipeThrough(new Base32DecoderStream())
* .pipeThrough(new TextDecoderStream());
*
* assertEquals(await toText(stream), "Hello World!");
* ```
*/
export class Base32DecoderStream extends TransformStream<string, Uint8Array> {
constructor() {
let push = "";
super({
transform(chunk, controller) {
push += chunk;
if (push.length < 8) {
return;
}
const remainder = -push.length % 8;
controller.enqueue(decodeBase32(push.slice(0, remainder || undefined)));
push = remainder ? chunk.slice(remainder) : "";
},
flush(controller) {
if (push.length) {
controller.enqueue(decodeBase32(push));
}
},
});
}
}

View File

@ -0,0 +1,33 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { assertEquals } from "@std/assert";
import { encodeBase32 } from "./base32.ts";
import { Base32DecoderStream, Base32EncoderStream } from "./base32_stream.ts";
import { RandomSliceStream } from "./_random_slice_stream.ts";
import { toText } from "../streams/to_text.ts";
import { concat } from "@std/bytes/concat";
Deno.test("Base32EncoderStream() encodes stream", async () => {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base32EncoderStream());
assertEquals(
await toText(readable),
encodeBase32(await Deno.readFile("./deno.lock")),
);
});
Deno.test("Base32DecoderStream() decodes stream", async () => {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base32EncoderStream())
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base32DecoderStream());
assertEquals(
concat(await Array.fromAsync(readable)),
await Deno.readFile("./deno.lock"),
);
});

View File

@ -0,0 +1,103 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
/**
* Utilities for encoding and decoding to and from base32hex in a streaming manner.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @module
*/
import { decodeBase32Hex, encodeBase32Hex } from "./base32hex.ts";
/**
* Converts a Uint8Array stream into a base32hex-encoded stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-6}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeBase32Hex } from "@std/encoding/base32hex";
* import { Base32HexEncoderStream } from "@std/encoding/base32hex-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["Hello,", " world!"])
* .pipeThrough(new TextEncoderStream())
* .pipeThrough(new Base32HexEncoderStream());
*
* assertEquals(await toText(stream), encodeBase32Hex(new TextEncoder().encode("Hello, world!")));
* ```
*/
export class Base32HexEncoderStream
extends TransformStream<Uint8Array, string> {
constructor() {
let push = new Uint8Array(0);
super({
transform(chunk, controller) {
const concat = new Uint8Array(push.length + chunk.length);
concat.set(push);
concat.set(chunk, push.length);
const remainder = -concat.length % 5;
controller.enqueue(
encodeBase32Hex(concat.slice(0, remainder || undefined)),
);
push = remainder ? concat.slice(remainder) : new Uint8Array(0);
},
flush(controller) {
if (push.length) {
controller.enqueue(encodeBase32Hex(push));
}
},
});
}
}
/**
* Decodes a base32hex-encoded stream into a Uint8Array stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-6}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { Base32HexDecoderStream } from "@std/encoding/base32hex-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["91IMOR3F5GG7ERRI", "DHI22==="])
* .pipeThrough(new Base32HexDecoderStream())
* .pipeThrough(new TextDecoderStream());
*
* assertEquals(await toText(stream), "Hello, world!");
* ```
*/
export class Base32HexDecoderStream
extends TransformStream<string, Uint8Array> {
constructor() {
let push = "";
super({
transform(chunk, controller) {
push += chunk;
if (push.length < 8) {
return;
}
const remainder = -push.length % 8;
controller.enqueue(
decodeBase32Hex(push.slice(0, remainder || undefined)),
);
push = remainder ? chunk.slice(remainder) : "";
},
flush(controller) {
if (push.length) {
controller.enqueue(decodeBase32Hex(push));
}
},
});
}
}

View File

@ -0,0 +1,36 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { assertEquals } from "@std/assert";
import { encodeBase32Hex } from "./base32hex.ts";
import {
Base32HexDecoderStream,
Base32HexEncoderStream,
} from "./base32hex_stream.ts";
import { RandomSliceStream } from "./_random_slice_stream.ts";
import { toText } from "@std/streams/to-text";
import { concat } from "@std/bytes/concat";
Deno.test("Base32EncoderStream() encodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base32HexEncoderStream());
assertEquals(
await toText(stream),
encodeBase32Hex(await Deno.readFile("./deno.lock")),
);
});
Deno.test("Base32DecoderStream() decodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base32HexEncoderStream())
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base32HexDecoderStream());
assertEquals(
concat(await Array.fromAsync(stream)),
await Deno.readFile("./deno.lock"),
);
});

99
encoding/base64_stream.ts Normal file
View File

@ -0,0 +1,99 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
/**
* Utilities for encoding and decoding to and from base64 in a streaming manner.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @module
*/
import { decodeBase64, encodeBase64 } from "./base64.ts";
/**
* Converts a Uint8Array stream into a base64-encoded stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-4}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeBase64 } from "@std/encoding/base64";
* import { Base64EncoderStream } from "@std/encoding/base64-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["Hello,", " world!"])
* .pipeThrough(new TextEncoderStream())
* .pipeThrough(new Base64EncoderStream());
*
* assertEquals(await toText(stream), encodeBase64(new TextEncoder().encode("Hello, world!")));
* ```
*/
export class Base64EncoderStream extends TransformStream<Uint8Array, string> {
constructor() {
let push = new Uint8Array(0);
super({
transform(chunk, controller) {
const concat = new Uint8Array(push.length + chunk.length);
concat.set(push);
concat.set(chunk, push.length);
const remainder = -concat.length % 3;
controller.enqueue(
encodeBase64(concat.slice(0, remainder || undefined)),
);
push = remainder ? concat.slice(remainder) : new Uint8Array(0);
},
flush(controller) {
if (push.length) {
controller.enqueue(encodeBase64(push));
}
},
});
}
}
/**
* Decodes a base64-encoded stream into a Uint8Array stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-4}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { Base64DecoderStream } from "@std/encoding/base64-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["SGVsbG8s", "IHdvcmxkIQ=="])
* .pipeThrough(new Base64DecoderStream())
* .pipeThrough(new TextDecoderStream());
*
* assertEquals(await toText(stream), "Hello, world!");
* ```
*/
export class Base64DecoderStream extends TransformStream<string, Uint8Array> {
constructor() {
let push = "";
super({
transform(chunk, controller) {
push += chunk;
if (push.length < 4) {
return;
}
const remainder = -push.length % 4;
controller.enqueue(decodeBase64(push.slice(0, remainder || undefined)));
push = remainder ? push.slice(remainder) : "";
},
flush(controller) {
if (push.length) {
controller.enqueue(decodeBase64(push));
}
},
});
}
}

View File

@ -0,0 +1,33 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { assertEquals } from "@std/assert";
import { encodeBase64 } from "./base64.ts";
import { Base64DecoderStream, Base64EncoderStream } from "./base64_stream.ts";
import { RandomSliceStream } from "./_random_slice_stream.ts";
import { toText } from "@std/streams/to-text";
import { concat } from "@std/bytes/concat";
Deno.test("Base64EncoderStream() encodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base64EncoderStream());
assertEquals(
await toText(stream),
encodeBase64(await Deno.readFile("./deno.lock")),
);
});
Deno.test("Base64DecoderStream() decodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base64EncoderStream())
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base64DecoderStream());
assertEquals(
concat(await Array.fromAsync(stream)),
await Deno.readFile("./deno.lock"),
);
});

View File

@ -0,0 +1,104 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
/**
* Utilities for encoding and decoding to and from base64url in a streaming manner.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @module
*/
import { decodeBase64Url, encodeBase64Url } from "./base64url.ts";
/**
* Converts a Uint8Array stream into a base64url-encoded stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-5}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeBase64Url } from "@std/encoding/base64url";
* import { Base64UrlEncoderStream } from "@std/encoding/base64url-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["Hello,", " world!"])
* .pipeThrough(new TextEncoderStream())
* .pipeThrough(new Base64UrlEncoderStream());
*
* assertEquals(await toText(stream), encodeBase64Url(new TextEncoder().encode("Hello, world!")));
* ```
*/
export class Base64UrlEncoderStream
extends TransformStream<Uint8Array, string> {
constructor() {
let push = new Uint8Array(0);
super({
transform(chunk, controller) {
const concat = new Uint8Array(push.length + chunk.length);
concat.set(push);
concat.set(chunk, push.length);
const remainder = -concat.length % 3;
controller.enqueue(
encodeBase64Url(concat.slice(0, remainder || undefined)),
);
push = remainder ? concat.slice(remainder) : new Uint8Array(0);
},
flush(controller) {
if (push.length) {
controller.enqueue(encodeBase64Url(push));
}
},
});
}
}
/**
* Decodes a base64url-encoded stream into a Uint8Array stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-5}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeBase64Url } from "@std/encoding/base64url";
* import { Base64UrlDecoderStream } from "@std/encoding/base64url-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["SGVsbG8s", "IHdvcmxkIQ"])
* .pipeThrough(new Base64UrlDecoderStream())
* .pipeThrough(new TextDecoderStream());
*
* assertEquals(await toText(stream), "Hello, world!");
* ```
*/
export class Base64UrlDecoderStream
extends TransformStream<string, Uint8Array> {
constructor() {
let push = "";
super({
transform(chunk, controller) {
push += chunk;
if (push.length < 4) {
return;
}
const remainder = -push.length % 4;
controller.enqueue(
decodeBase64Url(push.slice(0, remainder || undefined)),
);
push = remainder ? push.slice(remainder) : "";
},
flush(controller) {
if (push.length) {
controller.enqueue(decodeBase64Url(push));
}
},
});
}
}

View File

@ -0,0 +1,36 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { assertEquals } from "@std/assert";
import { encodeBase64Url } from "./base64url.ts";
import {
Base64UrlDecoderStream,
Base64UrlEncoderStream,
} from "./base64url_stream.ts";
import { RandomSliceStream } from "./_random_slice_stream.ts";
import { toText } from "@std/streams/to-text";
import { concat } from "@std/bytes/concat";
Deno.test("Base64UrlEncoderStream() encodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base64UrlEncoderStream());
assertEquals(
await toText(stream),
encodeBase64Url(await Deno.readFile("./deno.lock")),
);
});
Deno.test("Base64UrlDecoderStream() decodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base64UrlEncoderStream())
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base64UrlDecoderStream());
assertEquals(
concat(await Array.fromAsync(stream)),
await Deno.readFile("./deno.lock"),
);
});

View File

@ -5,11 +5,16 @@
".": "./mod.ts", ".": "./mod.ts",
"./ascii85": "./ascii85.ts", "./ascii85": "./ascii85.ts",
"./base32": "./base32.ts", "./base32": "./base32.ts",
"./base32-stream": "./base32_stream.ts",
"./base32hex": "./base32hex.ts", "./base32hex": "./base32hex.ts",
"./base32hex-stream": "./base32hex_stream.ts",
"./base58": "./base58.ts", "./base58": "./base58.ts",
"./base64": "./base64.ts", "./base64": "./base64.ts",
"./base64-stream": "./base64_stream.ts",
"./base64url": "./base64url.ts", "./base64url": "./base64url.ts",
"./base64url-stream": "./base64url_stream.ts",
"./hex": "./hex.ts", "./hex": "./hex.ts",
"./hex-stream": "./hex_stream.ts",
"./varint": "./varint.ts" "./varint": "./varint.ts"
} }
} }

85
encoding/hex_stream.ts Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
/**
* Utilities for encoding and decoding to and from hex in a streaming manner.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @module
*/
import { decodeHex, encodeHex } from "./hex.ts";
/**
* Converts a Uint8Array stream into a hex-encoded stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-8}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeHex } from "@std/encoding/hex";
* import { HexEncoderStream } from "@std/encoding/hex-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["Hello,", " world!"])
* .pipeThrough(new TextEncoderStream())
* .pipeThrough(new HexEncoderStream());
*
* assertEquals(await toText(stream), encodeHex(new TextEncoder().encode("Hello, world!")));
* ```
*/
export class HexEncoderStream extends TransformStream<Uint8Array, string> {
constructor() {
super({
transform(chunk, controller) {
controller.enqueue(encodeHex(chunk));
},
});
}
}
/**
* Decodes a hex-encoded stream into a Uint8Array stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-8}
*
* @example Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { HexDecoderStream } from "@std/encoding/hex-stream";
* import { toText } from "@std/streams/to-text";
*
* const stream = ReadableStream.from(["48656c6c6f2c", "20776f726c6421"])
* .pipeThrough(new HexDecoderStream())
* .pipeThrough(new TextDecoderStream());
*
* assertEquals(await toText(stream), "Hello, world!");
* ```
*/
export class HexDecoderStream extends TransformStream<string, Uint8Array> {
constructor() {
let push = "";
super({
transform(chunk, controller) {
push += chunk;
if (push.length < 2) {
return;
}
const remainder = -push.length % 2;
controller.enqueue(decodeHex(push.slice(0, remainder || undefined)));
push = remainder ? push.slice(remainder) : "";
},
flush(controller) {
if (push.length) {
controller.enqueue(decodeHex(push));
}
},
});
}
}

View File

@ -0,0 +1,33 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { assertEquals } from "@std/assert";
import { encodeHex } from "./hex.ts";
import { HexDecoderStream, HexEncoderStream } from "./hex_stream.ts";
import { toText } from "@std/streams/to-text";
import { concat } from "@std/bytes/concat";
import { RandomSliceStream } from "./_random_slice_stream.ts";
Deno.test("HexEncoderStream() encodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new RandomSliceStream())
.pipeThrough(new HexEncoderStream());
assertEquals(
await toText(stream),
encodeHex(await Deno.readFile("./deno.lock")),
);
});
Deno.test("HexDecoderStream() decodes stream", async () => {
const stream = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new HexEncoderStream())
.pipeThrough(new RandomSliceStream())
.pipeThrough(new HexDecoderStream());
assertEquals(
concat(await Array.fromAsync(stream)),
await Deno.readFile("./deno.lock"),
);
});

View File

@ -17,9 +17,14 @@
export * from "./ascii85.ts"; export * from "./ascii85.ts";
export * from "./base32.ts"; export * from "./base32.ts";
export * from "./base32_stream.ts";
export * from "./base32hex.ts"; export * from "./base32hex.ts";
export * from "./base32hex_stream.ts";
export * from "./base58.ts"; export * from "./base58.ts";
export * from "./base64.ts"; export * from "./base64.ts";
export * from "./base64_stream.ts";
export * from "./base64url.ts"; export * from "./base64url.ts";
export * from "./base64url_stream.ts";
export * from "./hex.ts"; export * from "./hex.ts";
export * from "./hex_stream.ts";
export * from "./varint.ts"; export * from "./varint.ts";