From 9eab0e4775a0c798ed327576126f0620cd8a395f Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 8 Jun 2020 15:46:16 -0700 Subject: [PATCH 1/6] feat: support Uint8Array input in place of Buffer --- src/compile/encodings.js | 51 +++++++++++++++++++++++++++++++++++++--- test/bytes.js | 2 +- test/corrupted.js | 2 +- test/custom-types.js | 2 +- test/nested.js | 16 ++++++------- test/repeated.js | 16 ++++++------- 6 files changed, 67 insertions(+), 22 deletions(-) diff --git a/src/compile/encodings.js b/src/compile/encodings.js index 97f98ba..a18fb1c 100644 --- a/src/compile/encodings.js +++ b/src/compile/encodings.js @@ -4,6 +4,32 @@ var varint = require('varint') var svarint = require('signed-varint') const { Buffer } = require('buffer') +/** + * @template T + * @typedef {{bytes?:number} & function(T, Buffer, number):Buffer} Encoder + */ +/** + * @template T + * @typedef {{bytes?:number} & function(Buffer, number):T} Decoder + */ + +/** + * @template T + * @typedef {Object} Codec + * @property {number} type + * @property {Encoder} encode + * @property {Decoder} decode + * @property {function(T):number} encodingLength + */ + +/** + * @template T + * @param {number} type + * @param {Encoder} encode + * @param {Decoder} decode + * @param {function(T):number} encodingLength + * @returns {Codec} + */ var encoder = function (type, encode, decode, encodingLength) { encode.bytes = decode.bytes = 0 @@ -17,7 +43,12 @@ var encoder = function (type, encode, decode, encodingLength) { exports.make = encoder -exports.bytes = (function (tag) { +/** @type {Codec} */ +exports.bytes = (function () { + /** + * @param {Uint8Array|string} val + * @returns {number} + */ var bufferLength = function (val) { return Buffer.isBuffer(val) ? val.length : Buffer.byteLength(val) } @@ -34,7 +65,7 @@ exports.bytes = (function (tag) { varint.encode(len, buffer, offset) offset += varint.encode.bytes - if (Buffer.isBuffer(val)) val.copy(buffer, offset) + if (val instanceof Uint8Array) buffer.set(val, offset) else buffer.write(val, offset, len) offset += len @@ -58,6 +89,7 @@ exports.bytes = (function (tag) { return encoder(2, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.string = (function () { var encodingLength = function (val) { var len = Buffer.byteLength(val) @@ -94,7 +126,12 @@ exports.string = (function () { return encoder(2, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.bool = (function () { + /** + * @param {boolean} val + * @returns {number} + */ var encodingLength = function (val) { return 1 } @@ -114,6 +151,7 @@ exports.bool = (function () { return encoder(0, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.int32 = (function () { var decode = function (buffer, offset) { var val = varint.decode(buffer, offset) @@ -134,6 +172,7 @@ exports.int32 = (function () { return encoder(0, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.int64 = (function () { var decode = function (buffer, offset) { var val = varint.decode(buffer, offset) @@ -178,11 +217,13 @@ exports.int64 = (function () { return encoder(0, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.sint32 = exports.sint64 = (function () { return encoder(0, svarint.encode, svarint.decode, svarint.encodingLength) })() +/** @type {Codec} */ exports.uint32 = exports.uint64 = exports.enum = @@ -191,6 +232,7 @@ exports.varint = (function () { })() // we cannot represent these in javascript so we just use buffers +/** @type {Codec} */ exports.fixed64 = exports.sfixed64 = (function () { var encodingLength = function (val) { @@ -198,7 +240,7 @@ exports.sfixed64 = (function () { } var encode = function (val, buffer, offset) { - val.copy(buffer, offset) + buffer.set(val, offset) encode.bytes = 8 return buffer } @@ -232,6 +274,7 @@ exports.double = (function () { return encoder(1, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.fixed32 = (function () { var encodingLength = function (val) { return 4 @@ -252,6 +295,7 @@ exports.fixed32 = (function () { return encoder(5, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.sfixed32 = (function () { var encodingLength = function (val) { return 4 @@ -272,6 +316,7 @@ exports.sfixed32 = (function () { return encoder(5, encode, decode, encodingLength) })() +/** @type {Codec} */ exports.float = (function () { var encodingLength = function (val) { return 4 diff --git a/test/bytes.js b/test/bytes.js index f0301c0..30ecfed 100644 --- a/test/bytes.js +++ b/test/bytes.js @@ -6,7 +6,7 @@ var Bytes = protobuf(require('./test.proto')).Bytes tape('bytes encode + decode', function (t) { var b1 = Bytes.encode({ - req: Buffer.from([0, 1, 2, 3]) + req: Uint8Array.from([0, 1, 2, 3]) }) var o1 = Bytes.decode(b1) diff --git a/test/corrupted.js b/test/corrupted.js index f6e6ad2..ceae87a 100644 --- a/test/corrupted.js +++ b/test/corrupted.js @@ -27,7 +27,7 @@ var messages = protobuf(protoStr) tape('invalid message decode', function (t) { var didFail = false try { - messages.ABC.decode(Buffer.from([8, 182, 168, 235, 144, 178, 41])) + messages.ABC.decode(Uint8Array.from([8, 182, 168, 235, 144, 178, 41])) } catch (e) { didFail = true } diff --git a/test/custom-types.js b/test/custom-types.js index fb0498b..0e2cda3 100644 --- a/test/custom-types.js +++ b/test/custom-types.js @@ -8,7 +8,7 @@ tape('custom types encode + decode', function (t) { var b1 = CustomType.encode({ req: { num: 5, - payload: Buffer.from([]) + payload: Uint8Array.from([]) } }) diff --git a/test/nested.js b/test/nested.js index 5c98666..45e016b 100644 --- a/test/nested.js +++ b/test/nested.js @@ -7,20 +7,20 @@ var Nested = protobuf(require('./test.proto')).Nested tape('nested encode', function (t) { var b1 = Nested.encode({ num: 1, - payload: Buffer.from('lol'), + payload: new TextEncoder().encode('lol'), meh: { num: 2, - payload: Buffer.from('bar') + payload: new TextEncoder().encode('bar') } }) var b2 = Nested.encode({ num: 1, - payload: Buffer.from('lol'), + payload: new TextEncoder().encode('lol'), meeeh: 42, meh: { num: 2, - payload: Buffer.from('bar') + payload: new TextEncoder().encode('bar') } }) @@ -31,10 +31,10 @@ tape('nested encode', function (t) { tape('nested encode + decode', function (t) { var b1 = Nested.encode({ num: 1, - payload: Buffer.from('lol'), + payload: new TextEncoder().encode('lol'), meh: { num: 2, - payload: Buffer.from('bar') + payload: new TextEncoder().encode('bar') } }) @@ -48,11 +48,11 @@ tape('nested encode + decode', function (t) { var b2 = Nested.encode({ num: 1, - payload: Buffer.from('lol'), + payload: new TextEncoder().encode('lol'), meeeh: 42, meh: { num: 2, - payload: Buffer.from('bar') + payload: new TextEncoder().encode('bar') } }) diff --git a/test/repeated.js b/test/repeated.js index fc97030..1852318 100644 --- a/test/repeated.js +++ b/test/repeated.js @@ -8,20 +8,20 @@ tape('repeated encode', function (t) { var b1 = Repeated.encode({ list: [{ num: 1, - payload: Buffer.from('lol') + payload: new TextEncoder().encode('lol') }, { num: 2, - payload: Buffer.from('lol1') + payload: new TextEncoder().encode('lol1') }] }) var b2 = Repeated.encode({ list: [{ num: 1, - payload: Buffer.from('lol') + payload: new TextEncoder().encode('lol') }, { num: 2, - payload: Buffer.from('lol1'), + payload: new TextEncoder().encode('lol1'), meeeeh: 100 }], meeh: 42 @@ -35,10 +35,10 @@ tape('repeated encode + decode', function (t) { var b1 = Repeated.encode({ list: [{ num: 1, - payload: Buffer.from('lol') + payload: new TextEncoder().encode('lol') }, { num: 2, - payload: Buffer.from('lol1') + payload: new TextEncoder().encode('lol1') }] }) @@ -53,10 +53,10 @@ tape('repeated encode + decode', function (t) { var b2 = Repeated.encode({ list: [{ num: 1, - payload: Buffer.from('lol') + payload: new TextEncoder().encode('lol') }, { num: 2, - payload: Buffer.from('lol1'), + payload: new TextEncoder().encode('lol1'), meeeeh: 100 }], meeh: 42 From ef69a42ea530c8878f021de9a928874685f6d32e Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 9 Jun 2020 10:01:51 -0700 Subject: [PATCH 2/6] chore: remove api comments --- src/compile/encodings.js | 45 ---------------------------------------- 1 file changed, 45 deletions(-) diff --git a/src/compile/encodings.js b/src/compile/encodings.js index a18fb1c..c742f57 100644 --- a/src/compile/encodings.js +++ b/src/compile/encodings.js @@ -4,32 +4,6 @@ var varint = require('varint') var svarint = require('signed-varint') const { Buffer } = require('buffer') -/** - * @template T - * @typedef {{bytes?:number} & function(T, Buffer, number):Buffer} Encoder - */ -/** - * @template T - * @typedef {{bytes?:number} & function(Buffer, number):T} Decoder - */ - -/** - * @template T - * @typedef {Object} Codec - * @property {number} type - * @property {Encoder} encode - * @property {Decoder} decode - * @property {function(T):number} encodingLength - */ - -/** - * @template T - * @param {number} type - * @param {Encoder} encode - * @param {Decoder} decode - * @param {function(T):number} encodingLength - * @returns {Codec} - */ var encoder = function (type, encode, decode, encodingLength) { encode.bytes = decode.bytes = 0 @@ -43,12 +17,7 @@ var encoder = function (type, encode, decode, encodingLength) { exports.make = encoder -/** @type {Codec} */ exports.bytes = (function () { - /** - * @param {Uint8Array|string} val - * @returns {number} - */ var bufferLength = function (val) { return Buffer.isBuffer(val) ? val.length : Buffer.byteLength(val) } @@ -89,7 +58,6 @@ exports.bytes = (function () { return encoder(2, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.string = (function () { var encodingLength = function (val) { var len = Buffer.byteLength(val) @@ -126,12 +94,7 @@ exports.string = (function () { return encoder(2, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.bool = (function () { - /** - * @param {boolean} val - * @returns {number} - */ var encodingLength = function (val) { return 1 } @@ -151,7 +114,6 @@ exports.bool = (function () { return encoder(0, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.int32 = (function () { var decode = function (buffer, offset) { var val = varint.decode(buffer, offset) @@ -172,7 +134,6 @@ exports.int32 = (function () { return encoder(0, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.int64 = (function () { var decode = function (buffer, offset) { var val = varint.decode(buffer, offset) @@ -217,13 +178,11 @@ exports.int64 = (function () { return encoder(0, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.sint32 = exports.sint64 = (function () { return encoder(0, svarint.encode, svarint.decode, svarint.encodingLength) })() -/** @type {Codec} */ exports.uint32 = exports.uint64 = exports.enum = @@ -232,7 +191,6 @@ exports.varint = (function () { })() // we cannot represent these in javascript so we just use buffers -/** @type {Codec} */ exports.fixed64 = exports.sfixed64 = (function () { var encodingLength = function (val) { @@ -274,7 +232,6 @@ exports.double = (function () { return encoder(1, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.fixed32 = (function () { var encodingLength = function (val) { return 4 @@ -295,7 +252,6 @@ exports.fixed32 = (function () { return encoder(5, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.sfixed32 = (function () { var encodingLength = function (val) { return 4 @@ -316,7 +272,6 @@ exports.sfixed32 = (function () { return encoder(5, encode, decode, encodingLength) })() -/** @type {Codec} */ exports.float = (function () { var encodingLength = function (val) { return 4 From c4efe49f91232e06fddbfdfd8458037f336c9156 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 9 Jun 2020 13:40:02 -0700 Subject: [PATCH 3/6] chore: pull TextEncoder from ipfs-utils --- package.json | 1 + test/nested.js | 2 ++ test/repeated.js | 1 + 3 files changed, 4 insertions(+) diff --git a/package.json b/package.json index a93f689..36dffb1 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "varint": "^5.0.0" }, "devDependencies": { + "ipfs-utils": "^2.3.0", "aegir": "^21.4.0", "benchmark": "^2.1.4", "protocol-buffers": "^4.1.0", diff --git a/test/nested.js b/test/nested.js index 45e016b..bbd77fd 100644 --- a/test/nested.js +++ b/test/nested.js @@ -2,6 +2,8 @@ var tape = require('tape') var protobuf = require('../') +const TextEncoder = require('ipfs-utils/src/text-encoder') + var Nested = protobuf(require('./test.proto')).Nested tape('nested encode', function (t) { diff --git a/test/repeated.js b/test/repeated.js index 1852318..4929c3b 100644 --- a/test/repeated.js +++ b/test/repeated.js @@ -3,6 +3,7 @@ var tape = require('tape') var protobuf = require('../') var Repeated = protobuf(require('./test.proto')).Repeated +const TextEncoder = require('ipfs-utils/src/text-encoder') tape('repeated encode', function (t) { var b1 = Repeated.encode({ From 8ee9c9febd7c8e1e0d348a53a15c03721d81ce25 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 10 Jun 2020 13:36:29 -0700 Subject: [PATCH 4/6] chore: add node 10 as a test target --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0f91c03..5a82517 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ stages: - cov node_js: + - '10' - '12' os: From eade4bed750835c0ca8fc37359b043a4a873cbce Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 15 Jun 2020 14:28:30 -0700 Subject: [PATCH 5/6] chore: enable testing on node 14 and current --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5a82517..902eb41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,8 @@ stages: node_js: - '10' - '12' + - '14' + - 'current' os: - linux From b2a6745b17364650e4296372500f44d012f388ad Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 15 Jun 2020 14:37:41 -0700 Subject: [PATCH 6/6] chore: remove current node from test target --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 902eb41..00fadd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ node_js: - '10' - '12' - '14' - - 'current' os: - linux