BREAKING(encoding): remove module utf8.ts (#728)

This commit is contained in:
Zhangyuan Nie 2021-02-16 07:22:07 -05:00 committed by GitHub
parent 06c2c8f37b
commit 5bc18f5d86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 116 additions and 111 deletions

View File

@ -12,7 +12,6 @@ import {
startsWith, startsWith,
} from "./mod.ts"; } from "./mod.ts";
import { assert, assertEquals, assertThrows } from "../testing/asserts.ts"; import { assert, assertEquals, assertThrows } from "../testing/asserts.ts";
import { decode, encode } from "../encoding/utf8.ts";
Deno.test("[bytes] indexOf1", () => { Deno.test("[bytes] indexOf1", () => {
const i = indexOf( const i = indexOf(
@ -28,7 +27,8 @@ Deno.test("[bytes] indexOf2", () => {
}); });
Deno.test("[bytes] indexOf3", () => { Deno.test("[bytes] indexOf3", () => {
const i = indexOf(encode("Deno"), encode("D")); const encoder = new TextEncoder();
const i = indexOf(encoder.encode("Deno"), encoder.encode("D"));
assertEquals(i, 0); assertEquals(i, 0);
}); });
@ -140,10 +140,11 @@ Deno.test("[bytes] repeat", () => {
}); });
Deno.test("[bytes] concat", () => { Deno.test("[bytes] concat", () => {
const u1 = encode("Hello "); const encoder = new TextEncoder();
const u2 = encode("World"); const u1 = encoder.encode("Hello ");
const u2 = encoder.encode("World");
const joined = concat(u1, u2); const joined = concat(u1, u2);
assertEquals(decode(joined), "Hello World"); assertEquals(new TextDecoder().decode(joined), "Hello World");
assert(u1 !== joined); assert(u1 !== joined);
assert(u2 !== joined); assert(u2 !== joined);
}); });
@ -158,21 +159,23 @@ Deno.test("[bytes] concat empty arrays", () => {
}); });
Deno.test("[bytes] concat multiple arrays", () => { Deno.test("[bytes] concat multiple arrays", () => {
const u1 = encode("Hello "); const encoder = new TextEncoder();
const u2 = encode("W"); const u1 = encoder.encode("Hello ");
const u3 = encode("o"); const u2 = encoder.encode("W");
const u4 = encode("r"); const u3 = encoder.encode("o");
const u5 = encode("l"); const u4 = encoder.encode("r");
const u6 = encode("d"); const u5 = encoder.encode("l");
const u6 = encoder.encode("d");
const joined = concat(u1, u2, u3, u4, u5, u6); const joined = concat(u1, u2, u3, u4, u5, u6);
assertEquals(decode(joined), "Hello World"); assertEquals(new TextDecoder().decode(joined), "Hello World");
assert(u1 !== joined); assert(u1 !== joined);
assert(u2 !== joined); assert(u2 !== joined);
}); });
Deno.test("[bytes] contains", () => { Deno.test("[bytes] contains", () => {
const source = encode("deno.land"); const encoder = new TextEncoder();
const pattern = encode("deno"); const source = encoder.encode("deno.land");
const pattern = encoder.encode("deno");
assert(contains(source, pattern)); assert(contains(source, pattern));
assert(contains(new Uint8Array([0, 1, 2, 3]), new Uint8Array([2, 3]))); assert(contains(new Uint8Array([0, 1, 2, 3]), new Uint8Array([2, 3])));

View File

@ -1,17 +0,0 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
/** A default TextEncoder instance */
export const encoder = new TextEncoder();
/** Shorthand for new TextEncoder().encode() */
export function encode(input?: string): Uint8Array {
return encoder.encode(input);
}
/** A default TextDecoder instance */
export const decoder = new TextDecoder();
/** Shorthand for new TextDecoder().decode() */
export function decode(input?: Uint8Array): string {
return decoder.decode(input);
}

View File

@ -1,7 +1,6 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import { xeval } from "./xeval.ts"; import { xeval } from "./xeval.ts";
import { StringReader } from "../io/readers.ts"; import { StringReader } from "../io/readers.ts";
import { decode, encode } from "../encoding/utf8.ts";
import { import {
assert, assert,
assertEquals, assertEquals,
@ -49,10 +48,10 @@ Deno.test({
stderr: "null", stderr: "null",
}); });
assert(p.stdin != null); assert(p.stdin != null);
await p.stdin.write(encode("hello")); await p.stdin.write(new TextEncoder().encode("hello"));
p.stdin.close(); p.stdin.close();
assertEquals(await p.status(), { code: 0, success: true }); assertEquals(await p.status(), { code: 0, success: true });
assertEquals(decode(await p.output()).trimEnd(), "hello"); assertEquals(new TextDecoder().decode(await p.output()).trimEnd(), "hello");
p.close(); p.close();
}, },
}); });
@ -65,8 +64,9 @@ Deno.test("xevalCliSyntaxError", async function (): Promise<void> {
stdout: "piped", stdout: "piped",
stderr: "piped", stderr: "piped",
}); });
const decoder = new TextDecoder();
assertEquals(await p.status(), { code: 1, success: false }); assertEquals(await p.status(), { code: 1, success: false });
assertEquals(decode(await p.output()), ""); assertEquals(decoder.decode(await p.output()), "");
assertStringIncludes(decode(await p.stderrOutput()), "SyntaxError"); assertStringIncludes(decoder.decode(await p.stderrOutput()), "SyntaxError");
p.close(); p.close();
}); });

View File

@ -1,5 +1,4 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import { decode } from "../encoding/utf8.ts";
import { import {
assert, assert,
assertEquals, assertEquals,
@ -130,10 +129,11 @@ Deno.test("expandGlobPermError", async function (): Promise<void> {
stdout: "piped", stdout: "piped",
stderr: "piped", stderr: "piped",
}); });
const decoder = new TextDecoder();
assertEquals(await p.status(), { code: 1, success: false }); assertEquals(await p.status(), { code: 1, success: false });
assertEquals(decode(await p.output()), ""); assertEquals(decoder.decode(await p.output()), "");
assertStringIncludes( assertStringIncludes(
decode(await p.stderrOutput()), decoder.decode(await p.stderrOutput()),
"Uncaught PermissionDenied", "Uncaught PermissionDenied",
); );
p.close(); p.close();

View File

@ -2,7 +2,6 @@
import { BufReader, BufWriter } from "../io/bufio.ts"; import { BufReader, BufWriter } from "../io/bufio.ts";
import { TextProtoReader } from "../textproto/mod.ts"; import { TextProtoReader } from "../textproto/mod.ts";
import { assert } from "../_util/assert.ts"; import { assert } from "../_util/assert.ts";
import { encoder } from "../encoding/utf8.ts";
import { Response, ServerRequest } from "./server.ts"; import { Response, ServerRequest } from "./server.ts";
import { STATUS_TEXT } from "./http_status.ts"; import { STATUS_TEXT } from "./http_status.ts";
@ -174,6 +173,7 @@ export async function writeChunkedBody(
w: BufWriter, w: BufWriter,
r: Deno.Reader, r: Deno.Reader,
): Promise<void> { ): Promise<void> {
const encoder = new TextEncoder();
for await (const chunk of Deno.iter(r)) { for await (const chunk of Deno.iter(r)) {
if (chunk.byteLength <= 0) continue; if (chunk.byteLength <= 0) continue;
const start = encoder.encode(`${chunk.byteLength.toString(16)}\r\n`); const start = encoder.encode(`${chunk.byteLength.toString(16)}\r\n`);
@ -221,6 +221,7 @@ export async function writeTrailers(
if (undeclared.length > 0) { if (undeclared.length > 0) {
throw new TypeError(`Undeclared trailers: ${Deno.inspect(undeclared)}.`); throw new TypeError(`Undeclared trailers: ${Deno.inspect(undeclared)}.`);
} }
const encoder = new TextEncoder();
for (const [key, value] of trailers) { for (const [key, value] of trailers) {
await writer.write(encoder.encode(`${key}: ${value}\r\n`)); await writer.write(encoder.encode(`${key}: ${value}\r\n`));
} }
@ -237,6 +238,7 @@ export async function writeResponse(
const statusCode = r.status || 200; const statusCode = r.status || 200;
const statusText = STATUS_TEXT.get(statusCode); const statusText = STATUS_TEXT.get(statusCode);
const writer = BufWriter.create(w); const writer = BufWriter.create(w);
const encoder = new TextEncoder();
if (!statusText) { if (!statusText) {
throw new Deno.errors.InvalidData("Bad status code"); throw new Deno.errors.InvalidData("Bad status code");
} }

View File

@ -14,7 +14,6 @@ import {
writeResponse, writeResponse,
writeTrailers, writeTrailers,
} from "./_io.ts"; } from "./_io.ts";
import { decode, encode } from "../encoding/utf8.ts";
import { BufReader, ReadLineResult } from "../io/bufio.ts"; import { BufReader, ReadLineResult } from "../io/bufio.ts";
import { Response, ServerRequest } from "./server.ts"; import { Response, ServerRequest } from "./server.ts";
import { StringReader } from "../io/readers.ts"; import { StringReader } from "../io/readers.ts";
@ -24,16 +23,18 @@ Deno.test("bodyReader", async () => {
const text = "Hello, Deno"; const text = "Hello, Deno";
const r = bodyReader( const r = bodyReader(
text.length, text.length,
new BufReader(new Deno.Buffer(encode(text))), new BufReader(new Deno.Buffer(new TextEncoder().encode(text))),
); );
assertEquals(decode(await Deno.readAll(r)), text); assertEquals(new TextDecoder().decode(await Deno.readAll(r)), text);
}); });
function chunkify(n: number, char: string): string { function chunkify(n: number, char: string): string {
const v = Array.from({ length: n }) const v = Array.from({ length: n })
.map(() => `${char}`) .map(() => `${char}`)
.join(""); .join("");
return `${n.toString(16)}\r\n${v}\r\n`; return `${n.toString(16)}\r\n${v}\r\n`;
} }
Deno.test("chunkedBodyReader", async () => { Deno.test("chunkedBodyReader", async () => {
const body = [ const body = [
chunkify(3, "a"), chunkify(3, "a"),
@ -43,7 +44,10 @@ Deno.test("chunkedBodyReader", async () => {
chunkify(0, ""), chunkify(0, ""),
].join(""); ].join("");
const h = new Headers(); const h = new Headers();
const r = chunkedBodyReader(h, new BufReader(new Deno.Buffer(encode(body)))); const r = chunkedBodyReader(
h,
new BufReader(new Deno.Buffer(new TextEncoder().encode(body))),
);
let result: number | null; let result: number | null;
// Use small buffer as some chunks exceed buffer size // Use small buffer as some chunks exceed buffer size
const buf = new Uint8Array(5); const buf = new Uint8Array(5);
@ -70,11 +74,14 @@ Deno.test("chunkedBodyReader with trailers", async () => {
const h = new Headers({ const h = new Headers({
trailer: "deno,node", trailer: "deno,node",
}); });
const r = chunkedBodyReader(h, new BufReader(new Deno.Buffer(encode(body)))); const r = chunkedBodyReader(
h,
new BufReader(new Deno.Buffer(new TextEncoder().encode(body))),
);
assertEquals(h.has("trailer"), true); assertEquals(h.has("trailer"), true);
assertEquals(h.has("deno"), false); assertEquals(h.has("deno"), false);
assertEquals(h.has("node"), false); assertEquals(h.has("node"), false);
const act = decode(await Deno.readAll(r)); const act = new TextDecoder().decode(await Deno.readAll(r));
const exp = "aaabbbbbcccccccccccdddddddddddddddddddddd"; const exp = "aaabbbbbcccccccccccdddddddddddddddddddddd";
assertEquals(act, exp); assertEquals(act, exp);
assertEquals(h.has("trailer"), false); assertEquals(h.has("trailer"), false);
@ -87,7 +94,10 @@ Deno.test("readTrailers", async () => {
trailer: "Deno, Node", trailer: "Deno, Node",
}); });
const trailer = ["deno: land", "node: js", "", ""].join("\r\n"); const trailer = ["deno: land", "node: js", "", ""].join("\r\n");
await readTrailers(h, new BufReader(new Deno.Buffer(encode(trailer)))); await readTrailers(
h,
new BufReader(new Deno.Buffer(new TextEncoder().encode(trailer))),
);
assertEquals(h.has("trailer"), false); assertEquals(h.has("trailer"), false);
assertEquals(h.get("deno"), "land"); assertEquals(h.get("deno"), "land");
assertEquals(h.get("node"), "js"); assertEquals(h.get("node"), "js");
@ -109,7 +119,7 @@ Deno.test(
async () => { async () => {
await readTrailers( await readTrailers(
h, h,
new BufReader(new Deno.Buffer(encode(trailer))), new BufReader(new Deno.Buffer(new TextEncoder().encode(trailer))),
); );
}, },
Deno.errors.InvalidData, Deno.errors.InvalidData,
@ -371,12 +381,13 @@ Deno.test("writeResponse with trailer", async () => {
Deno.test("writeResponseShouldNotModifyOriginHeaders", async () => { Deno.test("writeResponseShouldNotModifyOriginHeaders", async () => {
const headers = new Headers(); const headers = new Headers();
const buf = new Deno.Buffer(); const buf = new Deno.Buffer();
const decoder = new TextDecoder();
await writeResponse(buf, { body: "foo", headers }); await writeResponse(buf, { body: "foo", headers });
assert(decode(await Deno.readAll(buf)).includes("content-length: 3")); assert(decoder.decode(await Deno.readAll(buf)).includes("content-length: 3"));
await writeResponse(buf, { body: "hello", headers }); await writeResponse(buf, { body: "hello", headers });
assert(decode(await Deno.readAll(buf)).includes("content-length: 5")); assert(decoder.decode(await Deno.readAll(buf)).includes("content-length: 5"));
}); });
Deno.test("readRequestError", async function (): Promise<void> { Deno.test("readRequestError", async function (): Promise<void> {

View File

@ -1,5 +1,4 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import { encode } from "../encoding/utf8.ts";
import { BufReader, BufWriter } from "../io/bufio.ts"; import { BufReader, BufWriter } from "../io/bufio.ts";
import { assert } from "../_util/assert.ts"; import { assert } from "../_util/assert.ts";
import { Deferred, deferred, MuxAsyncIterator } from "../async/mod.ts"; import { Deferred, deferred, MuxAsyncIterator } from "../async/mod.ts";
@ -159,7 +158,7 @@ export class Server implements AsyncIterable<ServerRequest> {
try { try {
await writeResponse(writer, { await writeResponse(writer, {
status: 400, status: 400,
body: encode(`${error.message}\r\n\r\n`), body: new TextEncoder().encode(`${error.message}\r\n\r\n`),
}); });
} catch (error) { } catch (error) {
// The connection is broken. // The connection is broken.

View File

@ -23,7 +23,6 @@ import {
} from "./server.ts"; } from "./server.ts";
import { BufReader, BufWriter } from "../io/bufio.ts"; import { BufReader, BufWriter } from "../io/bufio.ts";
import { delay } from "../async/delay.ts"; import { delay } from "../async/delay.ts";
import { decode, encode } from "../encoding/utf8.ts";
import { mockConn } from "./_mock_conn.ts"; import { mockConn } from "./_mock_conn.ts";
import { dirname, fromFileUrl, join, resolve } from "../path/mod.ts"; import { dirname, fromFileUrl, join, resolve } from "../path/mod.ts";
@ -82,7 +81,7 @@ Deno.test("requestContentLength", function (): void {
const req = new ServerRequest(); const req = new ServerRequest();
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("content-length", "5"); req.headers.set("content-length", "5");
const buf = new Deno.Buffer(encode("Hello")); const buf = new Deno.Buffer(new TextEncoder().encode("Hello"));
req.r = new BufReader(buf); req.r = new BufReader(buf);
assertEquals(req.contentLength, 5); assertEquals(req.contentLength, 5);
} }
@ -103,7 +102,7 @@ Deno.test("requestContentLength", function (): void {
chunkOffset += chunkSize; chunkOffset += chunkSize;
} }
chunksData += "0\r\n\r\n"; chunksData += "0\r\n\r\n";
const buf = new Deno.Buffer(encode(chunksData)); const buf = new Deno.Buffer(new TextEncoder().encode(chunksData));
req.r = new BufReader(buf); req.r = new BufReader(buf);
assertEquals(req.contentLength, null); assertEquals(req.contentLength, null);
} }
@ -133,9 +132,9 @@ Deno.test("requestBodyWithContentLength", async function (): Promise<void> {
const req = new ServerRequest(); const req = new ServerRequest();
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("content-length", "5"); req.headers.set("content-length", "5");
const buf = new Deno.Buffer(encode("Hello")); const buf = new Deno.Buffer(new TextEncoder().encode("Hello"));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const body = decode(await Deno.readAll(req.body)); const body = new TextDecoder().decode(await Deno.readAll(req.body));
assertEquals(body, "Hello"); assertEquals(body, "Hello");
} }
@ -145,9 +144,9 @@ Deno.test("requestBodyWithContentLength", async function (): Promise<void> {
const req = new ServerRequest(); const req = new ServerRequest();
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("Content-Length", "5000"); req.headers.set("Content-Length", "5000");
const buf = new Deno.Buffer(encode(longText)); const buf = new Deno.Buffer(new TextEncoder().encode(longText));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const body = decode(await Deno.readAll(req.body)); const body = new TextDecoder().decode(await Deno.readAll(req.body));
assertEquals(body, longText); assertEquals(body, longText);
} }
// Handler ignored to consume body // Handler ignored to consume body
@ -159,7 +158,7 @@ Deno.test(
const req = new ServerRequest(); const req = new ServerRequest();
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("content-length", "" + text.length); req.headers.set("content-length", "" + text.length);
const tr = totalReader(new Deno.Buffer(encode(text))); const tr = totalReader(new Deno.Buffer(new TextEncoder().encode(text)));
req.r = new BufReader(tr); req.r = new BufReader(tr);
req.w = new BufWriter(new Deno.Buffer()); req.w = new BufWriter(new Deno.Buffer());
await req.respond({ status: 200, body: "ok" }); await req.respond({ status: 200, body: "ok" });
@ -187,7 +186,7 @@ Deno.test(
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("transfer-encoding", "chunked"); req.headers.set("transfer-encoding", "chunked");
req.headers.set("trailer", "deno,node"); req.headers.set("trailer", "deno,node");
const body = encode(text); const body = new TextEncoder().encode(text);
const tr = totalReader(new Deno.Buffer(body)); const tr = totalReader(new Deno.Buffer(body));
req.r = new BufReader(tr); req.r = new BufReader(tr);
req.w = new BufWriter(new Deno.Buffer()); req.w = new BufWriter(new Deno.Buffer());
@ -220,9 +219,9 @@ Deno.test("requestBodyWithTransferEncoding", async function (): Promise<void> {
chunkOffset += chunkSize; chunkOffset += chunkSize;
} }
chunksData += "0\r\n\r\n"; chunksData += "0\r\n\r\n";
const buf = new Deno.Buffer(encode(chunksData)); const buf = new Deno.Buffer(new TextEncoder().encode(chunksData));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const body = decode(await Deno.readAll(req.body)); const body = new TextDecoder().decode(await Deno.readAll(req.body));
assertEquals(body, shortText); assertEquals(body, shortText);
} }
@ -243,9 +242,9 @@ Deno.test("requestBodyWithTransferEncoding", async function (): Promise<void> {
chunkOffset += chunkSize; chunkOffset += chunkSize;
} }
chunksData += "0\r\n\r\n"; chunksData += "0\r\n\r\n";
const buf = new Deno.Buffer(encode(chunksData)); const buf = new Deno.Buffer(new TextEncoder().encode(chunksData));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const body = decode(await Deno.readAll(req.body)); const body = new TextDecoder().decode(await Deno.readAll(req.body));
assertEquals(body, longText); assertEquals(body, longText);
} }
}); });
@ -258,14 +257,14 @@ Deno.test("requestBodyReaderWithContentLength", async function (): Promise<
const req = new ServerRequest(); const req = new ServerRequest();
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("content-length", "" + shortText.length); req.headers.set("content-length", "" + shortText.length);
const buf = new Deno.Buffer(encode(shortText)); const buf = new Deno.Buffer(new TextEncoder().encode(shortText));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const readBuf = new Uint8Array(6); const readBuf = new Uint8Array(6);
let offset = 0; let offset = 0;
while (offset < shortText.length) { while (offset < shortText.length) {
const nread = await req.body.read(readBuf); const nread = await req.body.read(readBuf);
assert(nread !== null); assert(nread !== null);
const s = decode(readBuf.subarray(0, nread as number)); const s = new TextDecoder().decode(readBuf.subarray(0, nread as number));
assertEquals(shortText.substr(offset, nread as number), s); assertEquals(shortText.substr(offset, nread as number), s);
offset += nread as number; offset += nread as number;
} }
@ -279,14 +278,14 @@ Deno.test("requestBodyReaderWithContentLength", async function (): Promise<
const req = new ServerRequest(); const req = new ServerRequest();
req.headers = new Headers(); req.headers = new Headers();
req.headers.set("Content-Length", "5000"); req.headers.set("Content-Length", "5000");
const buf = new Deno.Buffer(encode(longText)); const buf = new Deno.Buffer(new TextEncoder().encode(longText));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const readBuf = new Uint8Array(1000); const readBuf = new Uint8Array(1000);
let offset = 0; let offset = 0;
while (offset < longText.length) { while (offset < longText.length) {
const nread = await req.body.read(readBuf); const nread = await req.body.read(readBuf);
assert(nread !== null); assert(nread !== null);
const s = decode(readBuf.subarray(0, nread as number)); const s = new TextDecoder().decode(readBuf.subarray(0, nread as number));
assertEquals(longText.substr(offset, nread as number), s); assertEquals(longText.substr(offset, nread as number), s);
offset += nread as number; offset += nread as number;
} }
@ -314,14 +313,14 @@ Deno.test("requestBodyReaderWithTransferEncoding", async function (): Promise<
chunkOffset += chunkSize; chunkOffset += chunkSize;
} }
chunksData += "0\r\n\r\n"; chunksData += "0\r\n\r\n";
const buf = new Deno.Buffer(encode(chunksData)); const buf = new Deno.Buffer(new TextEncoder().encode(chunksData));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const readBuf = new Uint8Array(6); const readBuf = new Uint8Array(6);
let offset = 0; let offset = 0;
while (offset < shortText.length) { while (offset < shortText.length) {
const nread = await req.body.read(readBuf); const nread = await req.body.read(readBuf);
assert(nread !== null); assert(nread !== null);
const s = decode(readBuf.subarray(0, nread as number)); const s = new TextDecoder().decode(readBuf.subarray(0, nread as number));
assertEquals(shortText.substr(offset, nread as number), s); assertEquals(shortText.substr(offset, nread as number), s);
offset += nread as number; offset += nread as number;
} }
@ -346,14 +345,14 @@ Deno.test("requestBodyReaderWithTransferEncoding", async function (): Promise<
chunkOffset += chunkSize; chunkOffset += chunkSize;
} }
chunksData += "0\r\n\r\n"; chunksData += "0\r\n\r\n";
const buf = new Deno.Buffer(encode(chunksData)); const buf = new Deno.Buffer(new TextEncoder().encode(chunksData));
req.r = new BufReader(buf); req.r = new BufReader(buf);
const readBuf = new Uint8Array(1000); const readBuf = new Uint8Array(1000);
let offset = 0; let offset = 0;
while (offset < longText.length) { while (offset < longText.length) {
const nread = await req.body.read(readBuf); const nread = await req.body.read(readBuf);
assert(nread !== null); assert(nread !== null);
const s = decode(readBuf.subarray(0, nread as number)); const s = new TextDecoder().decode(readBuf.subarray(0, nread as number));
assertEquals(longText.substr(offset, nread as number), s); assertEquals(longText.substr(offset, nread as number), s);
offset += nread as number; offset += nread as number;
} }
@ -553,9 +552,11 @@ Deno.test({
}); });
await Deno.writeAll( await Deno.writeAll(
conn, conn,
encode("GET / HTTP/1.1\r\nmalformedHeader\r\n\r\n\r\n\r\n"), new TextEncoder().encode(
"GET / HTTP/1.1\r\nmalformedHeader\r\n\r\n\r\n\r\n",
),
); );
const responseString = decode(await Deno.readAll(conn)); const responseString = new TextDecoder().decode(await Deno.readAll(conn));
assertMatch( assertMatch(
responseString, responseString,
/^HTTP\/1\.1 400 Bad Request\r\ncontent-length: \d+\r\n\r\n.*\r\n\r\n$/ms, /^HTTP\/1\.1 400 Bad Request\r\ncontent-length: \d+\r\n\r\n.*\r\n\r\n$/ms,
@ -584,12 +585,12 @@ Deno.test({
}); });
await Deno.writeAll( await Deno.writeAll(
conn, conn,
encode( new TextEncoder().encode(
"PUT / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\nzzzzzzz\r\nhello", "PUT / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\nzzzzzzz\r\nhello",
), ),
); );
await conn.closeWrite(); await conn.closeWrite();
const responseString = decode(await Deno.readAll(conn)); const responseString = new TextDecoder().decode(await Deno.readAll(conn));
assertEquals( assertEquals(
responseString, responseString,
"HTTP/1.1 200 OK\r\ncontent-length: 13\r\n\r\nHello, world!", "HTTP/1.1 200 OK\r\ncontent-length: 13\r\n\r\nHello, world!",
@ -617,10 +618,12 @@ Deno.test({
}); });
await Deno.writeAll( await Deno.writeAll(
conn, conn,
encode("PUT / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nHello"), new TextEncoder().encode(
"PUT / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nHello",
),
); );
conn.closeWrite(); conn.closeWrite();
const responseString = decode(await Deno.readAll(conn)); const responseString = new TextDecoder().decode(await Deno.readAll(conn));
assertEquals( assertEquals(
responseString, responseString,
"HTTP/1.1 200 OK\r\ncontent-length: 13\r\n\r\nHello, world!", "HTTP/1.1 200 OK\r\ncontent-length: 13\r\n\r\nHello, world!",
@ -647,7 +650,7 @@ Deno.test({
}); });
await Deno.writeAll( await Deno.writeAll(
conn, conn,
encode([ new TextEncoder().encode([
// A normal request is required: // A normal request is required:
"GET / HTTP/1.1", "GET / HTTP/1.1",
"Host: localhost", "Host: localhost",

View File

@ -3,12 +3,11 @@
// Copyright 2009 The Go Authors. All rights reserved. // Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
import { encode } from "../encoding/utf8.ts";
/** Reader utility for strings */ /** Reader utility for strings */
export class StringReader extends Deno.Buffer { export class StringReader extends Deno.Buffer {
constructor(s: string) { constructor(s: string) {
super(encode(s).buffer); super(new TextEncoder().encode(s).buffer);
} }
} }

View File

@ -3,7 +3,6 @@ import { assertEquals } from "../testing/asserts.ts";
import { LimitedReader, MultiReader, StringReader } from "./readers.ts"; import { LimitedReader, MultiReader, StringReader } from "./readers.ts";
import { StringWriter } from "./writers.ts"; import { StringWriter } from "./writers.ts";
import { copyN } from "./ioutil.ts"; import { copyN } from "./ioutil.ts";
import { decode } from "../encoding/utf8.ts";
Deno.test("ioStringReader", async function (): Promise<void> { Deno.test("ioStringReader", async function (): Promise<void> {
const r = new StringReader("abcdef"); const r = new StringReader("abcdef");
@ -14,17 +13,18 @@ Deno.test("ioStringReader", async function (): Promise<void> {
}); });
Deno.test("ioStringReader", async function (): Promise<void> { Deno.test("ioStringReader", async function (): Promise<void> {
const decoder = new TextDecoder();
const r = new StringReader("abcdef"); const r = new StringReader("abcdef");
const buf = new Uint8Array(3); const buf = new Uint8Array(3);
const res1 = await r.read(buf); const res1 = await r.read(buf);
assertEquals(res1, 3); assertEquals(res1, 3);
assertEquals(decode(buf), "abc"); assertEquals(decoder.decode(buf), "abc");
const res2 = await r.read(buf); const res2 = await r.read(buf);
assertEquals(res2, 3); assertEquals(res2, 3);
assertEquals(decode(buf), "def"); assertEquals(decoder.decode(buf), "def");
const res3 = await r.read(buf); const res3 = await r.read(buf);
assertEquals(res3, null); assertEquals(res3, null);
assertEquals(decode(buf), "def"); assertEquals(decoder.decode(buf), "def");
}); });
Deno.test("ioMultiReader", async function (): Promise<void> { Deno.test("ioMultiReader", async function (): Promise<void> {
@ -38,20 +38,21 @@ Deno.test("ioMultiReader", async function (): Promise<void> {
}); });
Deno.test("ioLimitedReader", async function (): Promise<void> { Deno.test("ioLimitedReader", async function (): Promise<void> {
const decoder = new TextDecoder();
let sr = new StringReader("abc"); let sr = new StringReader("abc");
let r = new LimitedReader(sr, 2); let r = new LimitedReader(sr, 2);
let buffer = await Deno.readAll(r); let buffer = await Deno.readAll(r);
assertEquals(decode(buffer), "ab"); assertEquals(decoder.decode(buffer), "ab");
assertEquals(decode(await Deno.readAll(sr)), "c"); assertEquals(decoder.decode(await Deno.readAll(sr)), "c");
sr = new StringReader("abc"); sr = new StringReader("abc");
r = new LimitedReader(sr, 3); r = new LimitedReader(sr, 3);
buffer = await Deno.readAll(r); buffer = await Deno.readAll(r);
assertEquals(decode(buffer), "abc"); assertEquals(decoder.decode(buffer), "abc");
assertEquals((await Deno.readAll(r)).length, 0); assertEquals((await Deno.readAll(r)).length, 0);
sr = new StringReader("abc"); sr = new StringReader("abc");
r = new LimitedReader(sr, 4); r = new LimitedReader(sr, 4);
buffer = await Deno.readAll(r); buffer = await Deno.readAll(r);
assertEquals(decode(buffer), "abc"); assertEquals(decoder.decode(buffer), "abc");
assertEquals((await Deno.readAll(r)).length, 0); assertEquals((await Deno.readAll(r)).length, 0);
}); });

View File

@ -1,7 +1,6 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
type Writer = Deno.Writer; type Writer = Deno.Writer;
type WriterSync = Deno.WriterSync; type WriterSync = Deno.WriterSync;
import { decode, encode } from "../encoding/utf8.ts";
/** Writer utility for buffering string chunks */ /** Writer utility for buffering string chunks */
export class StringWriter implements Writer, WriterSync { export class StringWriter implements Writer, WriterSync {
@ -10,7 +9,7 @@ export class StringWriter implements Writer, WriterSync {
private cache: string | undefined; private cache: string | undefined;
constructor(private base: string = "") { constructor(private base: string = "") {
const c = encode(base); const c = new TextEncoder().encode(base);
this.chunks.push(c); this.chunks.push(c);
this.byteLength += c.byteLength; this.byteLength += c.byteLength;
} }
@ -36,7 +35,7 @@ export class StringWriter implements Writer, WriterSync {
buf.set(chunk, offs); buf.set(chunk, offs);
offs += chunk.byteLength; offs += chunk.byteLength;
} }
this.cache = decode(buf); this.cache = new TextDecoder().decode(buf);
return this.cache; return this.cache;
} }
} }

View File

@ -4,7 +4,6 @@ import { copyN } from "../io/ioutil.ts";
import { MultiReader } from "../io/readers.ts"; import { MultiReader } from "../io/readers.ts";
import { extname } from "../path/mod.ts"; import { extname } from "../path/mod.ts";
import { BufReader, BufWriter } from "../io/bufio.ts"; import { BufReader, BufWriter } from "../io/bufio.ts";
import { encoder } from "../encoding/utf8.ts";
import { assert } from "../_util/assert.ts"; import { assert } from "../_util/assert.ts";
import { TextProtoReader } from "../textproto/mod.ts"; import { TextProtoReader } from "../textproto/mod.ts";
import { hasOwnProperty } from "../_util/has_own_property.ts"; import { hasOwnProperty } from "../_util/has_own_property.ts";
@ -39,6 +38,8 @@ function randomBoundary(): string {
return boundary; return boundary;
} }
const encoder = new TextEncoder();
/** /**
* Checks whether `buf` should be considered to match the boundary. * Checks whether `buf` should be considered to match the boundary.
* *

View File

@ -6,7 +6,6 @@
import type { BufReader } from "../io/bufio.ts"; import type { BufReader } from "../io/bufio.ts";
import { concat } from "../bytes/mod.ts"; import { concat } from "../bytes/mod.ts";
import { decode } from "../encoding/utf8.ts";
// FROM https://github.com/denoland/deno/blob/b34628a26ab0187a827aa4ebe256e23178e25d39/cli/js/web/headers.ts#L9 // FROM https://github.com/denoland/deno/blob/b34628a26ab0187a827aa4ebe256e23178e25d39/cli/js/web/headers.ts#L9
const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/g; const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/g;
@ -15,7 +14,7 @@ function str(buf: Uint8Array | null | undefined): string {
if (buf == null) { if (buf == null) {
return ""; return "";
} else { } else {
return decode(buf); return new TextDecoder().decode(buf);
} }
} }

View File

@ -1,5 +1,4 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import { decode, encode } from "../encoding/utf8.ts";
import { hasOwnProperty } from "../_util/has_own_property.ts"; import { hasOwnProperty } from "../_util/has_own_property.ts";
import { BufReader, BufWriter } from "../io/bufio.ts"; import { BufReader, BufWriter } from "../io/bufio.ts";
import { readLong, readShort, sliceLongToBytes } from "../io/ioutil.ts"; import { readLong, readShort, sliceLongToBytes } from "../io/ioutil.ts";
@ -222,6 +221,7 @@ class WebSocketImpl implements WebSocket {
} }
async *[Symbol.asyncIterator](): AsyncIterableIterator<WebSocketEvent> { async *[Symbol.asyncIterator](): AsyncIterableIterator<WebSocketEvent> {
const decoder = new TextDecoder();
let frames: WebSocketFrame[] = []; let frames: WebSocketFrame[] = [];
let payloadsLength = 0; let payloadsLength = 0;
while (!this._isClosed) { while (!this._isClosed) {
@ -248,7 +248,7 @@ class WebSocketImpl implements WebSocket {
} }
if (frames[0].opcode === OpCode.TextFrame) { if (frames[0].opcode === OpCode.TextFrame) {
// text // text
yield decode(concat); yield decoder.decode(concat);
} else { } else {
// binary // binary
yield concat; yield concat;
@ -260,7 +260,7 @@ class WebSocketImpl implements WebSocket {
case OpCode.Close: { case OpCode.Close: {
// [0x12, 0x34] -> 0x1234 // [0x12, 0x34] -> 0x1234
const code = (frame.payload[0] << 8) | frame.payload[1]; const code = (frame.payload[0] << 8) | frame.payload[1];
const reason = decode( const reason = decoder.decode(
frame.payload.subarray(2, frame.payload.length), frame.payload.subarray(2, frame.payload.length),
); );
await this.close(code, reason); await this.close(code, reason);
@ -313,7 +313,9 @@ class WebSocketImpl implements WebSocket {
const opcode = typeof data === "string" const opcode = typeof data === "string"
? OpCode.TextFrame ? OpCode.TextFrame
: OpCode.BinaryFrame; : OpCode.BinaryFrame;
const payload = typeof data === "string" ? encode(data) : data; const payload = typeof data === "string"
? new TextEncoder().encode(data)
: data;
const isLastFrame = true; const isLastFrame = true;
const frame = { const frame = {
isLastFrame, isLastFrame,
@ -325,7 +327,9 @@ class WebSocketImpl implements WebSocket {
} }
ping(data: WebSocketMessage = ""): Promise<void> { ping(data: WebSocketMessage = ""): Promise<void> {
const payload = typeof data === "string" ? encode(data) : data; const payload = typeof data === "string"
? new TextEncoder().encode(data)
: data;
const frame = { const frame = {
isLastFrame: true, isLastFrame: true,
opcode: OpCode.Ping, opcode: OpCode.Ping,
@ -345,7 +349,7 @@ class WebSocketImpl implements WebSocket {
const header = [code >>> 8, code & 0x00ff]; const header = [code >>> 8, code & 0x00ff];
let payload: Uint8Array; let payload: Uint8Array;
if (reason) { if (reason) {
const reasonBytes = encode(reason); const reasonBytes = new TextEncoder().encode(reason);
payload = new Uint8Array(2 + reasonBytes.byteLength); payload = new Uint8Array(2 + reasonBytes.byteLength);
payload.set(header); payload.set(header);
payload.set(reasonBytes, 2); payload.set(reasonBytes, 2);
@ -484,7 +488,7 @@ export async function handshake(
} }
headerStr += "\r\n"; headerStr += "\r\n";
await bufWriter.write(encode(headerStr)); await bufWriter.write(new TextEncoder().encode(headerStr));
await bufWriter.flush(); await bufWriter.flush();
const tpReader = new TextProtoReader(bufReader); const tpReader = new TextProtoReader(bufReader);

View File

@ -20,7 +20,6 @@ import {
unmask, unmask,
writeFrame, writeFrame,
} from "./mod.ts"; } from "./mod.ts";
import { decode, encode } from "../encoding/utf8.ts";
import { delay } from "../async/delay.ts"; import { delay } from "../async/delay.ts";
import { serve } from "../http/server.ts"; import { serve } from "../http/server.ts";
import { deferred } from "../async/deferred.ts"; import { deferred } from "../async/deferred.ts";
@ -211,7 +210,7 @@ Deno.test("[ws] write and read masked frame", async () => {
isLastFrame: true, isLastFrame: true,
mask, mask,
opcode: OpCode.TextFrame, opcode: OpCode.TextFrame,
payload: encode(msg), payload: new TextEncoder().encode(msg),
}, },
buf, buf,
); );
@ -220,12 +219,12 @@ Deno.test("[ws] write and read masked frame", async () => {
assertEquals(frame.isLastFrame, true); assertEquals(frame.isLastFrame, true);
assertEquals(frame.mask, mask); assertEquals(frame.mask, mask);
unmask(frame.payload, frame.mask); unmask(frame.payload, frame.mask);
assertEquals(frame.payload, encode(msg)); assertEquals(frame.payload, new TextEncoder().encode(msg));
}); });
Deno.test("[ws] handshake should not send search when it's empty", async () => { Deno.test("[ws] handshake should not send search when it's empty", async () => {
const writer = new Deno.Buffer(); const writer = new Deno.Buffer();
const reader = new Deno.Buffer(encode("HTTP/1.1 400\r\n")); const reader = new Deno.Buffer(new TextEncoder().encode("HTTP/1.1 400\r\n"));
await assertThrowsAsync( await assertThrowsAsync(
async (): Promise<void> => { async (): Promise<void> => {
@ -248,7 +247,9 @@ Deno.test(
"[ws] handshake should send search correctly", "[ws] handshake should send search correctly",
async function wsHandshakeWithSearch(): Promise<void> { async function wsHandshakeWithSearch(): Promise<void> {
const writer = new Deno.Buffer(); const writer = new Deno.Buffer();
const reader = new Deno.Buffer(encode("HTTP/1.1 400\r\n")); const reader = new Deno.Buffer(
new TextEncoder().encode("HTTP/1.1 400\r\n"),
);
await assertThrowsAsync( await assertThrowsAsync(
async (): Promise<void> => { async (): Promise<void> => {
@ -322,9 +323,9 @@ Deno.test({
const ping = await readFrame(bufr); const ping = await readFrame(bufr);
const third = await readFrame(bufr); const third = await readFrame(bufr);
assertEquals(first.opcode, OpCode.TextFrame); assertEquals(first.opcode, OpCode.TextFrame);
assertEquals(decode(first.payload), "first"); assertEquals(new TextDecoder().decode(first.payload), "first");
assertEquals(first.opcode, OpCode.TextFrame); assertEquals(first.opcode, OpCode.TextFrame);
assertEquals(decode(second.payload), "second"); assertEquals(new TextDecoder().decode(second.payload), "second");
assertEquals(ping.opcode, OpCode.Ping); assertEquals(ping.opcode, OpCode.Ping);
assertEquals(third.opcode, OpCode.BinaryFrame); assertEquals(third.opcode, OpCode.BinaryFrame);
assertEquals(bytes.equals(third.payload, new Uint8Array([3])), true); assertEquals(bytes.equals(third.payload, new Uint8Array([3])), true);
@ -458,7 +459,7 @@ Deno.test("[ws] WebSocket should act as asyncIterator", async () => {
} }
assertEquals(events.length, 3); assertEquals(events.length, 3);
assertEquals(events[0], ["ping", encode("Hello")]); assertEquals(events[0], ["ping", new TextEncoder().encode("Hello")]);
assertEquals(events[1], "Hello"); assertEquals(events[1], "Hello");
assertEquals(events[2], { code: 1011, reason: "42" }); assertEquals(events[2], { code: 1011, reason: "42" });
}); });