From 2e2d7cd696329dbe804d8b3e1345c74580a2d03f Mon Sep 17 00:00:00 2001 From: James M Snell <jasnell@gmail.com> Date: Sat, 10 Jul 2021 19:26:31 -0700 Subject: [PATCH 1/3] streams: implement TextEncoderStream and TextDecoderStream Experimental as part of the web streams implementation Signed-off-by: James M Snell <jasnell@gmail.com> --- doc/api/webstreams.md | 98 ++++++++ lib/internal/webstreams/encoding.js | 236 ++++++++++++++++++ lib/stream/web.js | 7 + .../test-whatwg-webstreams-encoding.js | 102 ++++++++ tools/doc/type-parser.mjs | 4 + 5 files changed, 447 insertions(+) create mode 100644 lib/internal/webstreams/encoding.js create mode 100644 test/parallel/test-whatwg-webstreams-encoding.js diff --git a/doc/api/webstreams.md b/doc/api/webstreams.md index e40850ef8531bb..458945146ccb03 100644 --- a/doc/api/webstreams.md +++ b/doc/api/webstreams.md @@ -1118,5 +1118,103 @@ added: REPLACEME * `chunk` {any} * Returns: {number} +### Class: `TextEncoderStream` +<!-- YAML +added: REPLACEME +--> + +#### `new TextEncoderStream()` +<!-- YAML +added: REPLACEME +--> + +Creates a new `TextEncoderStream` instance. +#### `textEncoderStream.encoding` +<!-- YAML +added: REPLACEME +--> + +* Type: {string} + +The encoding supported by the `TextEncoderStream` instance. + +#### `textEncoderStream.readable` +<!-- YAML +added: REPLACEME +--> + +* Type: {ReadableStream} + +#### `textEncoderStream.writable` +<!-- YAML +added: REPLACEME +--> + +* Type: {WritableStream} + +### Class: `TextDecoderStream` +<!-- YAML +added: REPLACEME +--> + +#### `new TextDecoderStream([encoding[, options]])` +<!-- YAML +added: REPLACEME +--> + +* `encoding` {string} Identifies the `encoding` that this `TextDecoder` instance + supports. **Default:** `'utf-8'`. +* `options` {Object} + * `fatal` {boolean} `true` if decoding failures are fatal. + * `ignoreBOM` {boolean} When `true`, the `TextDecoderStream` will include the + byte order mark in the decoded result. When `false`, the byte order mark + will be removed from the output. This option is only used when `encoding` is + `'utf-8'`, `'utf-16be'` or `'utf-16le'`. **Default:** `false`. + +Creates a new `TextDecoderStream` instance. + +#### `textDecoderStream.encoding` +<!-- YAML +added: REPLACEME +--> + +* Type: {string} + +The encoding supported by the `TextDecoderStream` instance. + +#### `textDecoderStream.fatal` +<!-- YAML +added: REPLACEME +--> + +* Type: {boolean} + +The value will be `true` if decoding errors result in a `TypeError` being +thrown. + +#### `textDecoderStream.ignoreBOM` +<!-- YAML +added: REPLACEME +--> + +* Type: {boolean} + +The value will be `true` if the decoding result will include the byte order +mark. + +#### `textDecoderStream.readable` +<!-- YAML +added: REPLACEME +--> + +* Type: {ReadableStream} + +#### `textDecoderStream.writable` +<!-- YAML +added: REPLACEME +--> + +* Type: {WritableStream} + [Streams]: stream.md [WHATWG Streams Standard]: https://streams.spec.whatwg.org/ diff --git a/lib/internal/webstreams/encoding.js b/lib/internal/webstreams/encoding.js new file mode 100644 index 00000000000000..37e8e526926230 --- /dev/null +++ b/lib/internal/webstreams/encoding.js @@ -0,0 +1,236 @@ +'use strict'; + +const { + ObjectDefineProperties, + Symbol, +} = primordials; + +const { + TextDecoder, + TextEncoder, +} = require('internal/encoding'); + +const { + TransformStream, +} = require('internal/webstreams/transformstream'); + +const { + kEnumerableProperty, +} = require('internal/webstreams/util'); + +const { + codes: { + ERR_INVALID_THIS, + }, +} = require('internal/errors'); + +const { + inspect, +} = require('internal/util/inspect'); + +const { + customInspectSymbol: kInspect +} = require('internal/util'); + +const kHandle = Symbol('kHandle'); +const kTransform = Symbol('kTransform'); +const kType = Symbol('kType'); + +/** + * @typedef {import('./readablestream').ReadableStream} ReadableStream + * @typedef {import('./writablestream').WritableStream} WritableStream + */ + +function isTextEncoderStream(value) { + return typeof value?.[kHandle] === 'object' && + value?.[kType] === 'TextEncoderStream'; +} + +function isTextDecoderStream(value) { + return typeof value?.[kHandle] === 'object' && + value?.[kType] === 'TextDecoderStream'; +} + +class TextEncoderStream { + constructor() { + this[kType] = 'TextEncoderStream'; + this[kHandle] = new TextEncoder(); + this[kTransform] = new TransformStream({ + transform: (chunk, controller) => { + const value = this[kHandle].encode(chunk); + if (value) + controller.enqueue(value); + }, + flush: (controller) => { + const value = this[kHandle].encode(); + if (value.byteLength > 0) + controller.enqueue(value); + controller.terminate(); + }, + }); + } + + /** + * @readonly + * @type {string} + */ + get encoding() { + if (!isTextEncoderStream(this)) + throw new ERR_INVALID_THIS('TextEncoderStream'); + return this[kHandle].encoding; + } + + /** + * @readonly + * @type {ReadableStream} + */ + get readable() { + if (!isTextEncoderStream(this)) + throw new ERR_INVALID_THIS('TextEncoderStream'); + return this[kTransform].readable; + } + + /** + * @readonly + * @type {WritableStream} + */ + get writable() { + if (!isTextEncoderStream(this)) + throw new ERR_INVALID_THIS('TextEncoderStream'); + return this[kTransform].writable; + } + + [kInspect](depth, options) { + if (!isTextEncoderStream(this)) + throw new ERR_INVALID_THIS('TextEncoderStream'); + if (depth < 0) + return this; + + const opts = { + ...options, + depth: options.depth == null ? null : options.depth - 1 + }; + + return `${this[kType]} ${inspect({ + encoding: this[kHandle].encoding, + readable: this[kTransform].readable, + writable: this[kTransform].writable, + }, opts)}`; + } +} + +class TextDecoderStream { + /** + * @param {string} [encoding] + * @param {{ + * fatal? : boolean, + * ignoreBOM? : boolean, + * }} [options] + */ + constructor(encoding = 'utf-8', options = {}) { + this[kType] = 'TextDecoderStream'; + this[kHandle] = new TextDecoder(encoding, options); + this[kTransform] = new TransformStream({ + transform: (chunk, controller) => { + const value = this[kHandle].decode(chunk, { stream: true }); + if (value) + controller.enqueue(value); + }, + flush: (controller) => { + const value = this[kHandle].decode(); + if (value) + controller.enqueue(value); + controller.terminate(); + }, + }); + } + + /** + * @readonly + * @type {string} + */ + get encoding() { + if (!isTextDecoderStream(this)) + throw new ERR_INVALID_THIS('TextDecoderStream'); + return this[kHandle].encoding; + } + + /** + * @readonly + * @type {boolean} + */ + get fatal() { + if (!isTextDecoderStream(this)) + throw new ERR_INVALID_THIS('TextDecoderStream'); + return this[kHandle].fatal; + } + + /** + * @readonly + * @type {boolean} + */ + get ignoreBOM() { + if (!isTextDecoderStream(this)) + throw new ERR_INVALID_THIS('TextDecoderStream'); + return this[kHandle].ignoreBOM; + } + + /** + * @readonly + * @type {ReadableStream} + */ + get readable() { + if (!isTextDecoderStream(this)) + throw new ERR_INVALID_THIS('TextDecoderStream'); + return this[kTransform].readable; + } + + /** + * @readonly + * @type {WritableStream} + */ + get writable() { + if (!isTextDecoderStream(this)) + throw new ERR_INVALID_THIS('TextDecoderStream'); + return this[kTransform].writable; + } + + [kInspect](depth, options) { + if (!isTextDecoderStream(this)) + throw new ERR_INVALID_THIS('TextDecoderStream'); + if (depth < 0) + return this; + + const opts = { + ...options, + depth: options.depth == null ? null : options.depth - 1 + }; + + return `${this[kType]} ${inspect({ + encoding: this[kHandle].encoding, + fatal: this[kHandle].fatal, + ignoreBOM: this[kHandle].ignoreBOM, + readable: this[kTransform].readable, + writable: this[kTransform].writable, + }, opts)}`; + } +} + +ObjectDefineProperties(TextEncoderStream.prototype, { + encoding: kEnumerableProperty, + readable: kEnumerableProperty, + writable: kEnumerableProperty, +}); + +ObjectDefineProperties(TextDecoderStream.prototype, { + encoding: kEnumerableProperty, + fatal: kEnumerableProperty, + ignoreBOM: kEnumerableProperty, + readable: kEnumerableProperty, + writable: kEnumerableProperty, +}); + +module.exports = { + TextEncoderStream, + TextDecoderStream, +}; diff --git a/lib/stream/web.js b/lib/stream/web.js index 929abd19044458..06b320f001a646 100644 --- a/lib/stream/web.js +++ b/lib/stream/web.js @@ -31,6 +31,11 @@ const { CountQueuingStrategy, } = require('internal/webstreams/queuingstrategies'); +const { + TextEncoderStream, + TextDecoderStream, +} = require('internal/webstreams/encoding'); + module.exports = { ReadableStream, ReadableStreamDefaultReader, @@ -45,4 +50,6 @@ module.exports = { WritableStreamDefaultController, ByteLengthQueuingStrategy, CountQueuingStrategy, + TextEncoderStream, + TextDecoderStream, }; diff --git a/test/parallel/test-whatwg-webstreams-encoding.js b/test/parallel/test-whatwg-webstreams-encoding.js new file mode 100644 index 00000000000000..97061650496c0d --- /dev/null +++ b/test/parallel/test-whatwg-webstreams-encoding.js @@ -0,0 +1,102 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); + +const { + TextEncoderStream, + TextDecoderStream, +} = require('stream/web'); + +const kEuroBytes = Buffer.from([0xe2, 0x82, 0xac]); +const kEuro = Buffer.from([0xe2, 0x82, 0xac]).toString(); + +[1, false, [], {}, 'hello'].forEach((i) => { + assert.throws(() => new TextDecoderStream(i), { + code: 'ERR_ENCODING_NOT_SUPPORTED', + }); +}); + +[1, false, 'hello'].forEach((i) => { + assert.throws(() => new TextDecoderStream(undefined, i), { + code: 'ERR_INVALID_ARG_TYPE', + }); +}); + +{ + const tds = new TextDecoderStream(); + const writer = tds.writable.getWriter(); + const reader = tds.readable.getReader(); + reader.read().then(common.mustCall(({ value, done }) => { + assert(!done); + assert.strictEqual(kEuro, value); + reader.read().then(common.mustCall(({ done }) => { + assert(done); + })); + })); + Promise.all([ + writer.write(kEuroBytes.slice(0, 1)), + writer.write(kEuroBytes.slice(1, 2)), + writer.write(kEuroBytes.slice(2, 3)), + writer.close(), + ]).then(common.mustCall()); + + assert.strictEqual(tds.encoding, 'utf-8'); + assert.strictEqual(tds.fatal, false); + assert.strictEqual(tds.ignoreBOM, false); + + assert.throws( + () => Reflect.get(TextDecoderStream.prototype, 'encoding', {}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws( + () => Reflect.get(TextDecoderStream.prototype, 'fatal', {}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws( + () => Reflect.get(TextDecoderStream.prototype, 'ignoreBOM', {}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws( + () => Reflect.get(TextDecoderStream.prototype, 'readable', {}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws( + () => Reflect.get(TextDecoderStream.prototype, 'writable', {}), { + code: 'ERR_INVALID_THIS', + }); +} + +{ + const tds = new TextEncoderStream(); + const writer = tds.writable.getWriter(); + const reader = tds.readable.getReader(); + reader.read().then(common.mustCall(({ value, done }) => { + assert(!done); + const buf = Buffer.from(value.buffer, value.byteOffset, value.byteLength); + assert.deepStrictEqual(kEuroBytes, buf); + reader.read().then(common.mustCall(({ done }) => { + assert(done); + })); + })); + Promise.all([ + writer.write(kEuro), + writer.close(), + ]).then(common.mustCall()); + + assert.strictEqual(tds.encoding, 'utf-8'); + + assert.throws( + () => Reflect.get(TextEncoderStream.prototype, 'encoding', {}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws( + () => Reflect.get(TextEncoderStream.prototype, 'readable', {}), { + code: 'ERR_INVALID_THIS', + }); + assert.throws( + () => Reflect.get(TextEncoderStream.prototype, 'writable', {}), { + code: 'ERR_INVALID_THIS', + }); +} diff --git a/tools/doc/type-parser.mjs b/tools/doc/type-parser.mjs index fff8f5afbd06cb..660a33e840466d 100644 --- a/tools/doc/type-parser.mjs +++ b/tools/doc/type-parser.mjs @@ -253,6 +253,10 @@ const customTypesMap = { 'webstreams.md#webstreamsapi_class_bytelengthqueuingstrategy', 'CountQueuingStrategy': 'webstreams.md#webstreamsapi_class_countqueuingstrategy', + 'TextEncoderStream': + 'webstreams.md#webstreamsapi_class_textencoderstream', + 'TextDecoderStream': + 'webstreams.md#webstreamsapi_class_textdecoderstream', }; const arrayPart = /(?:\[])+$/; From fd90b113325b23cf377d06137b0fc8cad4f7c222 Mon Sep 17 00:00:00 2001 From: James M Snell <jasnell@gmail.com> Date: Tue, 13 Jul 2021 13:22:19 -0700 Subject: [PATCH 2/3] fixup! streams: implement TextEncoderStream and TextDecoderStream --- lib/internal/webstreams/encoding.js | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/lib/internal/webstreams/encoding.js b/lib/internal/webstreams/encoding.js index 37e8e526926230..5af59bc9f4a502 100644 --- a/lib/internal/webstreams/encoding.js +++ b/lib/internal/webstreams/encoding.js @@ -15,6 +15,7 @@ const { } = require('internal/webstreams/transformstream'); const { + customInspect, kEnumerableProperty, } = require('internal/webstreams/util'); @@ -24,10 +25,6 @@ const { }, } = require('internal/errors'); -const { - inspect, -} = require('internal/util/inspect'); - const { customInspectSymbol: kInspect } = require('internal/util'); @@ -103,19 +100,11 @@ class TextEncoderStream { [kInspect](depth, options) { if (!isTextEncoderStream(this)) throw new ERR_INVALID_THIS('TextEncoderStream'); - if (depth < 0) - return this; - - const opts = { - ...options, - depth: options.depth == null ? null : options.depth - 1 - }; - - return `${this[kType]} ${inspect({ + return customInspect(depth, options, 'TextEncoderStream', { encoding: this[kHandle].encoding, readable: this[kTransform].readable, writable: this[kTransform].writable, - }, opts)}`; + }); } } @@ -198,21 +187,13 @@ class TextDecoderStream { [kInspect](depth, options) { if (!isTextDecoderStream(this)) throw new ERR_INVALID_THIS('TextDecoderStream'); - if (depth < 0) - return this; - - const opts = { - ...options, - depth: options.depth == null ? null : options.depth - 1 - }; - - return `${this[kType]} ${inspect({ + return customInspect(depth, options, 'TextDecoderStream', { encoding: this[kHandle].encoding, fatal: this[kHandle].fatal, ignoreBOM: this[kHandle].ignoreBOM, readable: this[kTransform].readable, writable: this[kTransform].writable, - }, opts)}`; + }); } } From e7f2b115c3e61960a4584002eb79765d3947abe1 Mon Sep 17 00:00:00 2001 From: James M Snell <jasnell@gmail.com> Date: Wed, 14 Jul 2021 08:05:19 -0700 Subject: [PATCH 3/3] [Squash] nit Co-authored-by: Antoine du Hamel <duhamelantoine1995@gmail.com> --- doc/api/webstreams.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api/webstreams.md b/doc/api/webstreams.md index 458945146ccb03..2c3f6fcb8eb22f 100644 --- a/doc/api/webstreams.md +++ b/doc/api/webstreams.md @@ -1129,6 +1129,7 @@ added: REPLACEME --> Creates a new `TextEncoderStream` instance. + #### `textEncoderStream.encoding` <!-- YAML added: REPLACEME