From b8cf2599242a9d85d03b57d3649ccdf8bce1530e Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Fri, 15 Nov 2024 15:54:28 +0100 Subject: [PATCH] feat(fetch): accept async iterables for body (#26882) Reland of #24623, but with a fix for `String` objects. Co-authored-by: crowlkats --- ext/fetch/22_body.js | 14 +++ ext/fetch/lib.deno_fetch.d.ts | 2 + ext/web/06_streams.js | 63 ++++---------- ext/webidl/00_webidl.js | 126 +++++++++++++++++++++++++++ ext/webidl/internal.d.ts | 26 ++++++ tests/integration/node_unit_tests.rs | 1 + tests/unit/fetch_test.ts | 27 ++++++ tests/unit/streams_test.ts | 15 +++- tests/unit_node/fetch_test.ts | 18 ++++ tests/wpt/runner/expectation.json | 10 ++- 10 files changed, 251 insertions(+), 51 deletions(-) create mode 100644 tests/unit_node/fetch_test.ts diff --git a/ext/fetch/22_body.js b/ext/fetch/22_body.js index 61a06b4afd..c7e977c0b4 100644 --- a/ext/fetch/22_body.js +++ b/ext/fetch/22_body.js @@ -15,6 +15,7 @@ import { core, primordials } from "ext:core/mod.js"; const { isAnyArrayBuffer, isArrayBuffer, + isStringObject, } = core; const { ArrayBufferIsView, @@ -466,6 +467,8 @@ function extractBody(object) { if (object.locked || isReadableStreamDisturbed(object)) { throw new TypeError("ReadableStream is locked or disturbed"); } + } else if (object[webidl.AsyncIterable] === webidl.AsyncIterable) { + stream = ReadableStream.from(object.open()); } if (typeof source === "string") { // WARNING: this deviates from spec (expects length to be set) @@ -483,6 +486,9 @@ function extractBody(object) { return { body, contentType }; } +webidl.converters["async iterable"] = webidl + .createAsyncIterableConverter(webidl.converters.Uint8Array); + webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => { // Union for (ReadableStream or Blob or ArrayBufferView or ArrayBuffer or FormData or URLSearchParams or USVString) if (ObjectPrototypeIsPrototypeOf(ReadableStreamPrototype, V)) { @@ -501,6 +507,14 @@ webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => { if (ArrayBufferIsView(V)) { return webidl.converters["ArrayBufferView"](V, prefix, context, opts); } + if (webidl.isAsyncIterable(V) && !isStringObject(V)) { + return webidl.converters["async iterable"]( + V, + prefix, + context, + opts, + ); + } } // BodyInit conversion is passed to extractBody(), which calls core.encode(). // core.encode() will UTF-8 encode strings with replacement, being equivalent to the USV normalization. diff --git a/ext/fetch/lib.deno_fetch.d.ts b/ext/fetch/lib.deno_fetch.d.ts index d219a38592..8614dec899 100644 --- a/ext/fetch/lib.deno_fetch.d.ts +++ b/ext/fetch/lib.deno_fetch.d.ts @@ -163,6 +163,8 @@ type BodyInit = | FormData | URLSearchParams | ReadableStream + | Iterable + | AsyncIterable | string; /** @category Fetch */ type RequestDestination = diff --git a/ext/web/06_streams.js b/ext/web/06_streams.js index a4f2275c52..f29e5f2040 100644 --- a/ext/web/06_streams.js +++ b/ext/web/06_streams.js @@ -70,7 +70,6 @@ const { String, Symbol, SymbolAsyncIterator, - SymbolIterator, SymbolFor, TypeError, TypedArrayPrototypeGetBuffer, @@ -5084,34 +5083,6 @@ function initializeCountSizeFunction(globalObject) { WeakMapPrototypeSet(countSizeFunctionWeakMap, globalObject, size); } -// Ref: https://tc39.es/ecma262/#sec-getiterator -function getAsyncOrSyncIterator(obj) { - let iterator; - if (obj[SymbolAsyncIterator] != null) { - iterator = obj[SymbolAsyncIterator](); - if (!isObject(iterator)) { - throw new TypeError( - "[Symbol.asyncIterator] returned a non-object value", - ); - } - } else if (obj[SymbolIterator] != null) { - iterator = obj[SymbolIterator](); - if (!isObject(iterator)) { - throw new TypeError("[Symbol.iterator] returned a non-object value"); - } - } else { - throw new TypeError("No iterator found"); - } - if (typeof iterator.next !== "function") { - throw new TypeError("iterator.next is not a function"); - } - return iterator; -} - -function isObject(x) { - return (typeof x === "object" && x != null) || typeof x === "function"; -} - const _resourceBacking = Symbol("[[resourceBacking]]"); // This distinction exists to prevent unrefable streams being used in // regular fast streams that are unaware of refability @@ -5197,21 +5168,22 @@ class ReadableStream { } static from(asyncIterable) { + const prefix = "Failed to execute 'ReadableStream.from'"; webidl.requiredArguments( arguments.length, 1, - "Failed to execute 'ReadableStream.from'", + prefix, ); - asyncIterable = webidl.converters.any(asyncIterable); - - const iterator = getAsyncOrSyncIterator(asyncIterable); + asyncIterable = webidl.converters["async iterable"]( + asyncIterable, + prefix, + "Argument 1", + ); + const iter = asyncIterable.open(); const stream = createReadableStream(noop, async () => { // deno-lint-ignore prefer-primordials - const res = await iterator.next(); - if (!isObject(res)) { - throw new TypeError("iterator.next value is not an object"); - } + const res = await iter.next(); if (res.done) { readableStreamDefaultControllerClose(stream[_controller]); } else { @@ -5221,17 +5193,8 @@ class ReadableStream { ); } }, async (reason) => { - if (iterator.return == null) { - return undefined; - } else { - // deno-lint-ignore prefer-primordials - const res = await iterator.return(reason); - if (!isObject(res)) { - throw new TypeError("iterator.return value is not an object"); - } else { - return undefined; - } - } + // deno-lint-ignore prefer-primordials + await iter.return(reason); }, 0); return stream; } @@ -6892,6 +6855,10 @@ webidl.converters.StreamPipeOptions = webidl { key: "signal", converter: webidl.converters.AbortSignal }, ]); +webidl.converters["async iterable"] = webidl.createAsyncIterableConverter( + webidl.converters.any, +); + internals.resourceForReadableStream = resourceForReadableStream; export { diff --git a/ext/webidl/00_webidl.js b/ext/webidl/00_webidl.js index 1d05aae5fa..eb18cbcc3e 100644 --- a/ext/webidl/00_webidl.js +++ b/ext/webidl/00_webidl.js @@ -26,6 +26,7 @@ const { Float32Array, Float64Array, FunctionPrototypeBind, + FunctionPrototypeCall, Int16Array, Int32Array, Int8Array, @@ -77,6 +78,7 @@ const { StringPrototypeToWellFormed, Symbol, SymbolIterator, + SymbolAsyncIterator, SymbolToStringTag, TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetSymbolToStringTag, @@ -920,6 +922,127 @@ function createSequenceConverter(converter) { }; } +function isAsyncIterable(obj) { + if (obj[SymbolAsyncIterator] === undefined) { + if (obj[SymbolIterator] === undefined) { + return false; + } + } + + return true; +} + +const AsyncIterable = Symbol("[[asyncIterable]]"); + +function createAsyncIterableConverter(converter) { + return function ( + V, + prefix = undefined, + context = undefined, + opts = { __proto__: null }, + ) { + if (type(V) !== "Object") { + throw makeException( + TypeError, + "can not be converted to async iterable.", + prefix, + context, + ); + } + + let isAsync = true; + let method = V[SymbolAsyncIterator]; + if (method === undefined) { + method = V[SymbolIterator]; + + if (method === undefined) { + throw makeException( + TypeError, + "is not iterable.", + prefix, + context, + ); + } + + isAsync = false; + } + + return { + value: V, + [AsyncIterable]: AsyncIterable, + open(context) { + const iter = FunctionPrototypeCall(method, V); + if (type(iter) !== "Object") { + throw new TypeError( + `${context} could not be iterated because iterator method did not return object, but ${ + type(iter) + }.`, + ); + } + + let asyncIterator = iter; + + if (!isAsync) { + asyncIterator = { + // deno-lint-ignore require-await + async next() { + // deno-lint-ignore prefer-primordials + return iter.next(); + }, + }; + } + + return { + async next() { + // deno-lint-ignore prefer-primordials + const iterResult = await asyncIterator.next(); + if (type(iterResult) !== "Object") { + throw TypeError( + `${context} failed to iterate next value because the next() method did not return an object, but ${ + type(iterResult) + }.`, + ); + } + + if (iterResult.done) { + return { done: true }; + } + + const iterValue = converter( + iterResult.value, + `${context} failed to iterate next value`, + `The value returned from the next() method`, + opts, + ); + + return { done: false, value: iterValue }; + }, + async return(reason) { + if (asyncIterator.return === undefined) { + return undefined; + } + + // deno-lint-ignore prefer-primordials + const returnPromiseResult = await asyncIterator.return(reason); + if (type(returnPromiseResult) !== "Object") { + throw TypeError( + `${context} failed to close iterator because the return() method did not return an object, but ${ + type(returnPromiseResult) + }.`, + ); + } + + return undefined; + }, + [SymbolAsyncIterator]() { + return this; + }, + }; + }, + }; + }; +} + function createRecordConverter(keyConverter, valueConverter) { return (V, prefix, context, opts) => { if (type(V) !== "Object") { @@ -1302,9 +1425,11 @@ function setlike(obj, objPrototype, readonly) { export { assertBranded, + AsyncIterable, brand, configureInterface, converters, + createAsyncIterableConverter, createBranded, createDictionaryConverter, createEnumConverter, @@ -1315,6 +1440,7 @@ export { createSequenceConverter, illegalConstructor, invokeCallbackFunction, + isAsyncIterable, makeException, mixinPairIterable, requiredArguments, diff --git a/ext/webidl/internal.d.ts b/ext/webidl/internal.d.ts index 1ce45463ec..375d548d32 100644 --- a/ext/webidl/internal.d.ts +++ b/ext/webidl/internal.d.ts @@ -438,6 +438,27 @@ declare module "ext:deno_webidl/00_webidl.js" { opts?: any, ) => T[]; + /** + * Create a converter that converts an async iterable of the inner type. + */ + function createAsyncIterableConverter( + converter: ( + v: V, + prefix?: string, + context?: string, + opts?: any, + ) => T, + ): ( + v: any, + prefix?: string, + context?: string, + opts?: any, + ) => ConvertedAsyncIterable; + + interface ConvertedAsyncIterable extends AsyncIterableIterator { + value: V; + } + /** * Create a converter that converts a Promise of the inner type. */ @@ -559,4 +580,9 @@ declare module "ext:deno_webidl/00_webidl.js" { | "Symbol" | "BigInt" | "Object"; + + /** + * Check whether a value is an async iterable. + */ + function isAsyncIterable(v: any): boolean; } diff --git a/tests/integration/node_unit_tests.rs b/tests/integration/node_unit_tests.rs index 40bd7b2fbb..9cb1af9496 100644 --- a/tests/integration/node_unit_tests.rs +++ b/tests/integration/node_unit_tests.rs @@ -72,6 +72,7 @@ util::unit_test_factory!( dgram_test, domain_test, fs_test, + fetch_test, http_test, http2_test, inspector_test, diff --git a/tests/unit/fetch_test.ts b/tests/unit/fetch_test.ts index 3ae96746a7..6d3fd8cc1d 100644 --- a/tests/unit/fetch_test.ts +++ b/tests/unit/fetch_test.ts @@ -2119,3 +2119,30 @@ Deno.test( await server; }, ); + +Deno.test("fetch async iterable", async () => { + const iterable = (async function* () { + yield new Uint8Array([1, 2, 3, 4, 5]); + yield new Uint8Array([6, 7, 8, 9, 10]); + })(); + const res = new Response(iterable); + const actual = await res.bytes(); + const expected = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assertEquals(actual, expected); +}); + +Deno.test("fetch iterable", async () => { + const iterable = (function* () { + yield new Uint8Array([1, 2, 3, 4, 5]); + yield new Uint8Array([6, 7, 8, 9, 10]); + })(); + const res = new Response(iterable); + const actual = await res.bytes(); + const expected = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assertEquals(actual, expected); +}); + +Deno.test("fetch string object", async () => { + const res = new Response(Object("hello")); + assertEquals(await res.text(), "hello"); +}); diff --git a/tests/unit/streams_test.ts b/tests/unit/streams_test.ts index b866fa7d51..73f9a60953 100644 --- a/tests/unit/streams_test.ts +++ b/tests/unit/streams_test.ts @@ -1,5 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assertEquals, assertRejects, fail } from "./test_util.ts"; +import { + assertEquals, + assertRejects, + assertThrows, + fail, +} from "./test_util.ts"; const { core, @@ -533,3 +538,11 @@ Deno.test(async function decompressionStreamInvalidGzipStillReported() { "corrupt gzip stream does not have a matching checksum", ); }); + +Deno.test(function readableStreamFromWithStringThrows() { + assertThrows( + () => ReadableStream.from("string"), + TypeError, + "Failed to execute 'ReadableStream.from': Argument 1 can not be converted to async iterable.", + ); +}); diff --git a/tests/unit_node/fetch_test.ts b/tests/unit_node/fetch_test.ts new file mode 100644 index 0000000000..399d6052a5 --- /dev/null +++ b/tests/unit_node/fetch_test.ts @@ -0,0 +1,18 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "@std/assert"; +import { createReadStream } from "node:fs"; + +Deno.test("fetch node stream", async () => { + const file = createReadStream("tests/testdata/assets/fixture.json"); + + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: file, + }); + + assertEquals( + await response.text(), + await Deno.readTextFile("tests/testdata/assets/fixture.json"), + ); +}); diff --git a/tests/wpt/runner/expectation.json b/tests/wpt/runner/expectation.json index 3477d8d9bf..23cc85b498 100644 --- a/tests/wpt/runner/expectation.json +++ b/tests/wpt/runner/expectation.json @@ -3257,8 +3257,14 @@ "owning-type-message-port.any.worker.html": false, "owning-type.any.html": false, "owning-type.any.worker.html": false, - "from.any.html": true, - "from.any.worker.html": true + "from.any.html": [ + "ReadableStream.from ignores a null @@asyncIterator", + "ReadableStream.from accepts a string" + ], + "from.any.worker.html": [ + "ReadableStream.from ignores a null @@asyncIterator", + "ReadableStream.from accepts a string" + ] }, "transform-streams": { "backpressure.any.html": true,