From e9e40a2cceb6392065a4f217d17fc4211aa1bbd7 Mon Sep 17 00:00:00 2001 From: Warren James <28974128+W-A-James@users.noreply.github.com> Date: Wed, 4 Jan 2023 09:57:35 -0500 Subject: [PATCH] feat(NODE-4870): Support BigInt serialization (#541) --- src/parser/serializer.ts | 32 +++++--- test/node/bigint.test.ts | 166 ++++++++++++++++++++++++++++++++++++++ test/node/bigint_tests.js | 72 ----------------- test/node/long.test.ts | 24 ++++++ test/node/long_tests.js | 18 ----- 5 files changed, 209 insertions(+), 103 deletions(-) create mode 100644 test/node/bigint.test.ts delete mode 100644 test/node/bigint_tests.js create mode 100644 test/node/long.test.ts delete mode 100644 test/node/long_tests.js diff --git a/src/parser/serializer.ts b/src/parser/serializer.ts index 91e901da..baf3fe9b 100644 --- a/src/parser/serializer.ts +++ b/src/parser/serializer.ts @@ -12,15 +12,7 @@ import type { MinKey } from '../min_key'; import type { ObjectId } from '../objectid'; import type { BSONRegExp } from '../regexp'; import { ByteUtils } from '../utils/byte_utils'; -import { - isAnyArrayBuffer, - isBigInt64Array, - isBigUInt64Array, - isDate, - isMap, - isRegExp, - isUint8Array -} from './utils'; +import { isAnyArrayBuffer, isDate, isMap, isRegExp, isUint8Array } from './utils'; /** @public */ export interface SerializeOptions { @@ -103,6 +95,20 @@ function serializeNumber(buffer: Uint8Array, key: string, value: number, index: return index; } +function serializeBigInt(buffer: Uint8Array, key: string, value: bigint, index: number) { + buffer[index++] = constants.BSON_DATA_LONG; + // Number of written bytes + const numberOfWrittenBytes = ByteUtils.encodeUTF8Into(buffer, key, index); + // Encode the name + index += numberOfWrittenBytes; + buffer[index++] = 0; + NUMBER_SPACE.setBigInt64(0, value, true); + // Write BigInt value + buffer.set(EIGHT_BYTE_VIEW_ON_NUMBER, index); + index += EIGHT_BYTE_VIEW_ON_NUMBER.byteLength; + return index; +} + function serializeNull(buffer: Uint8Array, key: string, _: unknown, index: number) { // Set long type buffer[index++] = constants.BSON_DATA_NULL; @@ -675,7 +681,7 @@ export function serializeInto( } else if (typeof value === 'number') { index = serializeNumber(buffer, key, value, index); } else if (typeof value === 'bigint') { - throw new BSONError('Unsupported type BigInt, please use Decimal128'); + index = serializeBigInt(buffer, key, value, index); } else if (typeof value === 'boolean') { index = serializeBoolean(buffer, key, value, index); } else if (value instanceof Date || isDate(value)) { @@ -777,8 +783,8 @@ export function serializeInto( index = serializeString(buffer, key, value, index); } else if (type === 'number') { index = serializeNumber(buffer, key, value, index); - } else if (type === 'bigint' || isBigInt64Array(value) || isBigUInt64Array(value)) { - throw new BSONError('Unsupported type BigInt, please use Decimal128'); + } else if (type === 'bigint') { + index = serializeBigInt(buffer, key, value, index); } else if (type === 'boolean') { index = serializeBoolean(buffer, key, value, index); } else if (value instanceof Date || isDate(value)) { @@ -881,7 +887,7 @@ export function serializeInto( } else if (type === 'number') { index = serializeNumber(buffer, key, value, index); } else if (type === 'bigint') { - throw new BSONError('Unsupported type BigInt, please use Decimal128'); + index = serializeBigInt(buffer, key, value, index); } else if (type === 'boolean') { index = serializeBoolean(buffer, key, value, index); } else if (value instanceof Date || isDate(value)) { diff --git a/test/node/bigint.test.ts b/test/node/bigint.test.ts new file mode 100644 index 00000000..fe771ce3 --- /dev/null +++ b/test/node/bigint.test.ts @@ -0,0 +1,166 @@ +import { BSON } from '../register-bson'; +import { bufferFromHexArray } from './tools/utils'; +import { BSON_DATA_LONG } from '../../src/constants'; +import { BSONDataView } from '../../src/utils/byte_utils'; + +describe('BSON BigInt serialization Support', function () { + // Index for the data type byte of a BSON document with a + // NOTE: These offsets only apply for documents with the shape {a : } + // where n is a BigInt + type SerializedDocParts = { + dataType: number; + key: string; + value: bigint; + }; + /** + * NOTE: this function operates on serialized BSON documents with the shape { : } + * where n is some int64. This function assumes that keys are properly encoded + * with the necessary null byte at the end and only at the end of the key string + */ + function getSerializedDocParts(serializedDoc: Uint8Array): SerializedDocParts { + const DATA_TYPE_OFFSET = 4; + const KEY_OFFSET = 5; + + const dataView = BSONDataView.fromUint8Array(serializedDoc); + const keySlice = serializedDoc.slice(KEY_OFFSET); + + let keyLength = 0; + while (keySlice[keyLength++] !== 0); + + const valueOffset = KEY_OFFSET + keyLength; + const key = Buffer.from(serializedDoc.slice(KEY_OFFSET, KEY_OFFSET + keyLength)).toString( + 'utf8' + ); + + return { + dataType: dataView.getInt8(DATA_TYPE_OFFSET), + key: key.slice(0, keyLength - 1), + value: dataView.getBigInt64(valueOffset, true) + }; + } + + it('serializes bigints with the correct BSON type', function () { + const testDoc = { a: 0n }; + const serializedDoc = getSerializedDocParts(BSON.serialize(testDoc)); + expect(serializedDoc.dataType).to.equal(BSON_DATA_LONG); + }); + + it('serializes bigints into little-endian byte order', function () { + const testDoc = { a: 0x1234567812345678n }; + const serializedDoc = getSerializedDocParts(BSON.serialize(testDoc)); + const expectedResult = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 type + '6100', // 'a' key with null terminator + '7856341278563412' + ]) + ); + + expect(expectedResult.value).to.equal(serializedDoc.value); + }); + + it('serializes a BigInt that can be safely represented as a Number', function () { + const testDoc = { a: 0x23n }; + const serializedDoc = getSerializedDocParts(BSON.serialize(testDoc)); + const expectedResult = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 type + '6100', // 'a' key with null terminator + '2300000000000000' // little endian int64 + ]) + ); + expect(serializedDoc).to.deep.equal(expectedResult); + }); + + it('serializes a BigInt in the valid range [-2^63, 2^63 - 1]', function () { + const testDoc = { a: 0xfffffffffffffff1n }; + const serializedDoc = getSerializedDocParts(BSON.serialize(testDoc)); + const expectedResult = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 + '6100', // 'a' key with null terminator + 'f1ffffffffffffff' + ]) + ); + expect(serializedDoc).to.deep.equal(expectedResult); + }); + + it('wraps to negative on a BigInt that is larger than (2^63 -1)', function () { + const maxIntPlusOne = { a: 2n ** 63n }; + const serializedMaxIntPlusOne = getSerializedDocParts(BSON.serialize(maxIntPlusOne)); + const expectedResultForMaxIntPlusOne = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 + '6100', // 'a' key with null terminator + '0000000000000080' + ]) + ); + expect(serializedMaxIntPlusOne).to.deep.equal(expectedResultForMaxIntPlusOne); + }); + + it('serializes BigInts at the edges of the valid range [-2^63, 2^63 - 1]', function () { + const maxPositiveInt64 = { a: 2n ** 63n - 1n }; + const serializedMaxPositiveInt64 = getSerializedDocParts(BSON.serialize(maxPositiveInt64)); + const expectedSerializationForMaxPositiveInt64 = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 + '6100', // 'a' key with null terminator + 'ffffffffffffff7f' + ]) + ); + expect(serializedMaxPositiveInt64).to.deep.equal(expectedSerializationForMaxPositiveInt64); + + const minPositiveInt64 = { a: -(2n ** 63n) }; + const serializedMinPositiveInt64 = getSerializedDocParts(BSON.serialize(minPositiveInt64)); + const expectedSerializationForMinPositiveInt64 = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 + '6100', // 'a' key with null terminator + '0000000000000080' + ]) + ); + expect(serializedMinPositiveInt64).to.deep.equal(expectedSerializationForMinPositiveInt64); + }); + + it('truncates a BigInt that is larger than a 64-bit int', function () { + const testDoc = { a: 2n ** 64n + 1n }; + const serializedDoc = getSerializedDocParts(BSON.serialize(testDoc)); + const expectedSerialization = getSerializedDocParts( + bufferFromHexArray([ + '12', //int64 + '6100', // 'a' key with null terminator + '0100000000000000' + ]) + ); + expect(serializedDoc).to.deep.equal(expectedSerialization); + }); + + it('serializes array of BigInts', function () { + const testArr = { a: [1n] }; + const serializedArr = BSON.serialize(testArr); + const expectedSerialization = bufferFromHexArray([ + '04', // array + '6100', // 'a' key with null terminator + bufferFromHexArray([ + '12', // int64 + '3000', // '0' key with null terminator + '0100000000000000' // 1n (little-endian) + ]).toString('hex') + ]); + expect(serializedArr).to.deep.equal(expectedSerialization); + }); + + it('serializes Map with BigInt values', function () { + const testMap = new Map(); + testMap.set('a', 1n); + const serializedMap = getSerializedDocParts(BSON.serialize(testMap)); + const expectedSerialization = getSerializedDocParts( + bufferFromHexArray([ + '12', // int64 + '6100', // 'a' key with null terminator + '0100000000000000' + ]) + ); + expect(serializedMap).to.deep.equal(expectedSerialization); + }); +}); diff --git a/test/node/bigint_tests.js b/test/node/bigint_tests.js deleted file mode 100644 index d20f8def..00000000 --- a/test/node/bigint_tests.js +++ /dev/null @@ -1,72 +0,0 @@ -'use strict'; - -const BSON = require('../register-bson'); -const BSONTypeError = BSON.BSONTypeError; - -describe('BSON BigInt Support', function () { - before(function () { - try { - BigInt(0); - } catch (_) { - this.skip('JS VM does not support BigInt'); - } - }); - it('Should serialize an int that fits in int32', function () { - const testDoc = { b: BigInt(32) }; - expect(() => BSON.serialize(testDoc)).to.throw(BSONTypeError); - - // const serializedDoc = BSON.serialize(testDoc); - // // prettier-ignore - // const resultBuffer = Buffer.from([0x0C, 0x00, 0x00, 0x00, 0x10, 0x62, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00]); - // const resultDoc = BSON.deserialize(serializedDoc); - // expect(Array.from(serializedDoc)).to.have.members(Array.from(resultBuffer)); - // expect(BigInt(resultDoc.b)).to.equal(testDoc.b); - }); - - it('Should serialize an int that fits in int64', function () { - const testDoc = { b: BigInt(0x1ffffffff) }; - expect(() => BSON.serialize(testDoc)).to.throw(BSONTypeError); - - // const serializedDoc = BSON.serialize(testDoc); - // // prettier-ignore - // const resultBuffer = Buffer.from([0x10, 0x00, 0x00, 0x00, 0x12, 0x62, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x00]); - // const resultDoc = BSON.deserialize(serializedDoc); - // expect(Array.from(serializedDoc)).to.have.members(Array.from(resultBuffer)); - // expect(BigInt(resultDoc.b)).to.equal(testDoc.b); - }); - - it('Should serialize an int that fits in decimal128', function () { - const testDoc = { b: BigInt('9223372036854776001') }; // int64 max + 1 - expect(() => BSON.serialize(testDoc)).to.throw(BSONTypeError); - - // const serializedDoc = BSON.serialize(testDoc); - // // prettier-ignore - // const resultBuffer = Buffer.from([0x18, 0x00, 0x00, 0x00, 0x13, 0x62, 0x00, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x30, 0x00]); - // const resultDoc = BSON.deserialize(serializedDoc); - // expect(Array.from(serializedDoc)).to.have.members(Array.from(resultBuffer)); - // expect(resultDoc.b._bsontype).to.equal('Decimal128'); - // expect(BigInt(resultDoc.b.toString())).to.equal(testDoc.b); - }); - - it('Should throw if BigInt is too large to serialize', function () { - const testDoc = { - b: BigInt('9'.repeat(35)) - }; // decimal 128 can only encode 34 digits of precision - expect(() => BSON.serialize(testDoc)).to.throw(BSONTypeError); - // expect(() => BSON.serialize(testDoc)).to.throw(); - }); - - it('Should accept BigInts in Long constructor', function (done) { - const Long = BSON.Long; - expect(new Long(BigInt('0')).toString()).to.equal('0'); - expect(new Long(BigInt('-1')).toString()).to.equal('-1'); - expect(new Long(BigInt('-1'), true).toString()).to.equal('18446744073709551615'); - expect(new Long(BigInt('123456789123456789')).toString()).to.equal('123456789123456789'); - expect(new Long(BigInt('123456789123456789'), true).toString()).to.equal('123456789123456789'); - expect(new Long(BigInt('13835058055282163712')).toString()).to.equal('-4611686018427387904'); - expect(new Long(BigInt('13835058055282163712'), true).toString()).to.equal( - '13835058055282163712' - ); - done(); - }); -}); diff --git a/test/node/long.test.ts b/test/node/long.test.ts new file mode 100644 index 00000000..9a73aedc --- /dev/null +++ b/test/node/long.test.ts @@ -0,0 +1,24 @@ +import { Long } from '../register-bson'; + +describe('Long', function () { + it('accepts strings in the constructor', function () { + expect(new Long('0').toString()).to.equal('0'); + expect(new Long('00').toString()).to.equal('0'); + expect(new Long('-1').toString()).to.equal('-1'); + expect(new Long('-1', true).toString()).to.equal('18446744073709551615'); + expect(new Long('123456789123456789').toString()).to.equal('123456789123456789'); + expect(new Long('123456789123456789', true).toString()).to.equal('123456789123456789'); + expect(new Long('13835058055282163712').toString()).to.equal('-4611686018427387904'); + expect(new Long('13835058055282163712', true).toString()).to.equal('13835058055282163712'); + }); + + it('accepts BigInts in Long constructor', function () { + expect(new Long(0n).toString()).to.equal('0'); + expect(new Long(-1n).toString()).to.equal('-1'); + expect(new Long(-1n, true).toString()).to.equal('18446744073709551615'); + expect(new Long(123456789123456789n).toString()).to.equal('123456789123456789'); + expect(new Long(123456789123456789n, true).toString()).to.equal('123456789123456789'); + expect(new Long(13835058055282163712n).toString()).to.equal('-4611686018427387904'); + expect(new Long(13835058055282163712n, true).toString()).to.equal('13835058055282163712'); + }); +}); diff --git a/test/node/long_tests.js b/test/node/long_tests.js deleted file mode 100644 index a1ea2e4c..00000000 --- a/test/node/long_tests.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -const BSON = require('../register-bson'); -const Long = BSON.Long; - -describe('Long', function () { - it('accepts strings in the constructor', function (done) { - expect(new Long('0').toString()).to.equal('0'); - expect(new Long('00').toString()).to.equal('0'); - expect(new Long('-1').toString()).to.equal('-1'); - expect(new Long('-1', true).toString()).to.equal('18446744073709551615'); - expect(new Long('123456789123456789').toString()).to.equal('123456789123456789'); - expect(new Long('123456789123456789', true).toString()).to.equal('123456789123456789'); - expect(new Long('13835058055282163712').toString()).to.equal('-4611686018427387904'); - expect(new Long('13835058055282163712', true).toString()).to.equal('13835058055282163712'); - done(); - }); -});