From 01698e6bf33de2e7b4fe222badde16514e87a813 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 11 Sep 2019 01:13:01 +0200 Subject: [PATCH] util: add encodeInto to TextEncoder Add function encodeInto to TextEncoder, and add MessageChannel to the encodeInto.any.js test. Fixes: https://github.com/nodejs/node/issues/28851 Fixes: https://github.com/nodejs/node/issues/26904 Refs: https://github.com/nodejs/node/pull/28862 Co-authored-by: AtticusYang --- doc/api/util.md | 18 +++++++++ lib/internal/encoding.js | 18 ++++++++- src/node_buffer.cc | 39 ++++++++++++++++++++ test/fixtures/wpt/LICENSE.md | 34 +++-------------- test/fixtures/wpt/README.md | 2 +- test/fixtures/wpt/encoding/encodeInto.any.js | 2 +- test/fixtures/wpt/versions.json | 2 +- test/wpt/status/encoding.json | 1 - test/wpt/status/url.json | 2 +- test/wpt/test-encoding.js | 7 ++++ 10 files changed, 91 insertions(+), 34 deletions(-) diff --git a/doc/api/util.md b/doc/api/util.md index 8fab6baaaf8d80..51d0bc9df90709 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -1076,6 +1076,24 @@ The `TextEncoder` class is also available on the global object. UTF-8 encodes the `input` string and returns a `Uint8Array` containing the encoded bytes. +### textEncoder.encodeInto(src, dest) + +* `src` {string} The text to encode. +* `dest` {Uint8Array} The array to hold the encode result. +* Returns: {Object} + * `read` {number} The read Unicode code units of src. + * `written` {number} The written UTF-8 bytes of dest. + +UTF-8 encodes the `src` string to the `dest` Unit8Array and returns an object +containing the read Unicode code units and written UTF-8 bytes. + +```js +const encoder = new TextEncoder(); +const src = 'this is some data'; +const dest = new Uint8Array(10); +const { read, written } = encoder.encodeInto(src, dest); +``` + ### textEncoder.encoding * {string} diff --git a/lib/internal/encoding.js b/lib/internal/encoding.js index 2de2cc565ca3c2..dabcd5eaccebe0 100644 --- a/lib/internal/encoding.js +++ b/lib/internal/encoding.js @@ -25,10 +25,14 @@ const { const { isArrayBuffer, - isArrayBufferView + isArrayBufferView, + isUint8Array } = require('internal/util/types'); +const { validateString } = require('internal/validators'); + const { + encodeInto, encodeUtf8String } = internalBinding('buffer'); @@ -304,6 +308,8 @@ function getEncodingFromLabel(label) { return encodings.get(trimAsciiWhitespace(label.toLowerCase())); } +const encodeIntoResults = new Uint32Array(2); + class TextEncoder { constructor() { this[kEncoder] = true; @@ -319,6 +325,15 @@ class TextEncoder { return encodeUtf8String(`${input}`); } + encodeInto(src, dest) { + validateEncoder(this); + validateString(src, 'src'); + if (!dest || !isUint8Array(dest)) + throw new ERR_INVALID_ARG_TYPE('dest', 'Uint8Array', dest); + encodeInto(src, dest, encodeIntoResults); + return { read: encodeIntoResults[0], written: encodeIntoResults[1] }; + } + [inspect](depth, opts) { validateEncoder(this); if (typeof depth === 'number' && depth < 0) @@ -336,6 +351,7 @@ class TextEncoder { Object.defineProperties( TextEncoder.prototype, { 'encode': { enumerable: true }, + 'encodeInto': { enumerable: true }, 'encoding': { enumerable: true }, [Symbol.toStringTag]: { configurable: true, diff --git a/src/node_buffer.cc b/src/node_buffer.cc index 220b9cf42a7887..7b05172e7dcae9 100644 --- a/src/node_buffer.cc +++ b/src/node_buffer.cc @@ -1047,6 +1047,44 @@ static void EncodeUtf8String(const FunctionCallbackInfo& args) { } +static void EncodeInto(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + Isolate* isolate = env->isolate(); + CHECK_GE(args.Length(), 3); + CHECK(args[0]->IsString()); + CHECK(args[1]->IsUint8Array()); + CHECK(args[2]->IsUint32Array()); + + Local source = args[0].As(); + int source_utf8len = source->Utf8Length(isolate); + + Local dest = args[1].As(); + Local buf = dest->Buffer(); + char* write_result = + static_cast(buf->GetContents().Data()) + dest->ByteOffset(); + size_t dest_length = dest->ByteLength(); + + if (dest_length > static_cast(source_utf8len)) + dest_length = source_utf8len; + + // results = [ read, written ] + Local result_arr = args[2].As(); + uint32_t* results = reinterpret_cast( + static_cast(result_arr->Buffer()->GetContents().Data()) + + result_arr->ByteOffset()); + + int nchars; + int written = source->WriteUtf8( + isolate, + write_result, + dest_length, + &nchars, + String::NO_NULL_TERMINATION | String::REPLACE_INVALID_UTF8); + results[0] = nchars; + results[1] = written; +} + + void SetBufferPrototype(const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); @@ -1078,6 +1116,7 @@ void Initialize(Local target, env->SetMethod(target, "swap32", Swap32); env->SetMethod(target, "swap64", Swap64); + env->SetMethod(target, "encodeInto", EncodeInto); env->SetMethodNoSideEffect(target, "encodeUtf8String", EncodeUtf8String); target->Set(env->context(), diff --git a/test/fixtures/wpt/LICENSE.md b/test/fixtures/wpt/LICENSE.md index 6b346a528cd7bd..ad4858c8745cfa 100644 --- a/test/fixtures/wpt/LICENSE.md +++ b/test/fixtures/wpt/LICENSE.md @@ -1,33 +1,11 @@ -# Dual-License for W3C Test Suites +# The 3-Clause BSD License -All documents in this Repository are licensed by contributors to be distributed under both the [W3C Test Suite License](#w3c-test-suite-license) and the [W3C 3-clause BSD License](#w3c-3-clause-bsd-license), reproduced below. The choice of license is up to the licensee. For more information, see [Licenses for W3C Test Suites](https://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html) - -# W3C Test Suite License - -This document, Test Suites and other documents that link to this statement are provided by the copyright holders under the following license: By using and/or copying this document, or the W3C document from which this statement is linked, you (the licensee) agree that you have read, understood, and will comply with the following terms and conditions: - -Permission to copy, and distribute the contents of this document, or the W3C document from which this statement is linked, in any medium for any purpose and without fee or royalty is hereby granted, provided that you include the following on ALL copies of the document, or portions thereof, that you use: - -* A link or URL to the original W3C document. -* The pre-existing copyright notice of the original author, or if it doesn't exist, a notice (hypertext is preferred, but a textual representation is permitted) of the form: "Copyright © [$date-of-document] World Wide Web Consortium, (MIT, ERCIM, Keio, Beihang) and others. All Rights Reserved. http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html" -* If it exists, the STATUS of the W3C document. - -When space permits, inclusion of the full text of this NOTICE should be provided. We request that authorship attribution be provided in any software, documents, or other items or products that you create pursuant to the implementation of the contents of this document, or any portion thereof. - -No right to create modifications or derivatives of W3C documents is granted pursuant to this license. However, if additional requirements (documented in the Copyright FAQ) are satisfied, the right to create modifications or derivatives is sometimes granted by the W3C to individuals complying with those requirements. - -If a Test Suite distinguishes the test harness (or, framework for navigation) and the actual tests, permission is given to remove or alter the harness or navigation if the Test Suite in question allows to do so. The tests themselves shall NOT be changed in any way. - -The name and trademarks of W3C and other copyright holders may NOT be used in advertising or publicity pertaining to this document or other documents that link to this statement without specific, written prior permission. Title to copyright in this document will at all times remain with copyright holders. Permission is given to use the trademarked string "W3C" within claims of performance concerning W3C Specifications or features described therein, and there only, if the test suite so authorizes. - -THIS WORK IS PROVIDED BY W3C, MIT, ERCIM, KEIO, BEIHANG, THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL W3C, MIT, ERCIM, KEIO, BEIHANG, THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# W3C 3-clause BSD License +Copyright 2019 web-platform-tests contributors Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -* Redistributions of works must retain the original copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the original copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* Neither the name of the W3C nor the names of its contributors may be used to endorse or promote products derived from this work without specific prior written permission. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/test/fixtures/wpt/README.md b/test/fixtures/wpt/README.md index 802e20d816711c..dcf42abe7925f7 100644 --- a/test/fixtures/wpt/README.md +++ b/test/fixtures/wpt/README.md @@ -11,7 +11,7 @@ See [test/wpt](../../wpt/README.md) for information on how these tests are run. Last update: - console: https://github.com/web-platform-tests/wpt/tree/9786a4b131/console -- encoding: https://github.com/web-platform-tests/wpt/tree/7287608f90/encoding +- encoding: https://github.com/web-platform-tests/wpt/tree/5059d2c777/encoding - url: https://github.com/web-platform-tests/wpt/tree/418f7fabeb/url - resources: https://github.com/web-platform-tests/wpt/tree/e1fddfbf80/resources - interfaces: https://github.com/web-platform-tests/wpt/tree/712c9f275e/interfaces diff --git a/test/fixtures/wpt/encoding/encodeInto.any.js b/test/fixtures/wpt/encoding/encodeInto.any.js index fda0d1b72ce787..8b3e743f71b9ce 100644 --- a/test/fixtures/wpt/encoding/encodeInto.any.js +++ b/test/fixtures/wpt/encoding/encodeInto.any.js @@ -126,7 +126,7 @@ Float64Array].forEach(view => { test(() => { assert_throws(new TypeError(), () => new TextEncoder().encodeInto("", new view(new ArrayBuffer(0)))); - }, "Invalid encodeInto() destination: " + view); + }, "Invalid encodeInto() destination: " + view.name); }); test(() => { diff --git a/test/fixtures/wpt/versions.json b/test/fixtures/wpt/versions.json index 0ca53d9103651f..c560f6844e9e3d 100644 --- a/test/fixtures/wpt/versions.json +++ b/test/fixtures/wpt/versions.json @@ -4,7 +4,7 @@ "path": "console" }, "encoding": { - "commit": "7287608f90f6b9530635d10086fd2ab386faab38", + "commit": "5059d2c77703d67d2f76931b44e6d2437526b6e9", "path": "encoding" }, "url": { diff --git a/test/wpt/status/encoding.json b/test/wpt/status/encoding.json index a81ba605c1e72f..585b17820170d5 100644 --- a/test/wpt/status/encoding.json +++ b/test/wpt/status/encoding.json @@ -52,6 +52,5 @@ "fail": "No implementation of TextDecoderStream and TextEncoderStream" }, "encodeInto.any.js": { - "fail": "TextEncoder.prototype.encodeInto not implemented" } } diff --git a/test/wpt/status/url.json b/test/wpt/status/url.json index fa27b25abdb54c..afd5acdcbfc455 100644 --- a/test/wpt/status/url.json +++ b/test/wpt/status/url.json @@ -12,4 +12,4 @@ "idlharness.any.js": { "fail": "getter/setter names are wrong, etc." } -} \ No newline at end of file +} diff --git a/test/wpt/test-encoding.js b/test/wpt/test-encoding.js index f868a7bac39987..8145debd66a11a 100644 --- a/test/wpt/test-encoding.js +++ b/test/wpt/test-encoding.js @@ -3,10 +3,17 @@ // Flags: --expose-internals require('../common'); +const { MessageChannel } = require('worker_threads'); const { WPTRunner } = require('../common/wpt'); const runner = new WPTRunner('encoding'); // Copy global descriptors from the global object runner.copyGlobalsFromObject(global, ['TextDecoder', 'TextEncoder']); +runner.defineGlobal('MessageChannel', { + get() { + return MessageChannel; + } +}); + runner.runJsTests();