diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index a4f6f0536fa..23c1fcda420 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -13,7 +13,6 @@ require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors require,dc-polyfill,MIT,Copyright 2023 Datadog Inc. require,ignore,MIT,Copyright 2013 Kael Zhang and contributors require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc. -require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc. require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates. require,koalas,MIT,Copyright 2013-2017 Brian Woodward @@ -21,7 +20,6 @@ require,limiter,MIT,Copyright 2011 John Hurliman require,lodash.sortby,MIT,Copyright JS Foundation and other contributors require,lru-cache,ISC,Copyright (c) 2010-2022 Isaac Z. Schlueter and Contributors require,module-details-from-path,MIT,Copyright 2016 Thomas Watson Steen -require,msgpack-lite,MIT,Copyright 2015 Yusuke Kawasaki require,opentracing,MIT,Copyright 2016 Resonance Labs Inc require,path-to-regexp,MIT,Copyright 2014 Blake Embrey require,pprof-format,MIT,Copyright 2022 Stephen Belanger @@ -59,10 +57,12 @@ dev,get-port,MIT,Copyright Sindre Sorhus dev,glob,ISC,Copyright Isaac Z. Schlueter and Contributors dev,globals,MIT,Copyright (c) Sindre Sorhus (https://sindresorhus.com) dev,graphql,MIT,Copyright 2015 Facebook Inc. +dev,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki dev,jszip,MIT,Copyright 2015-2016 Stuart Knightley and contributors dev,knex,MIT,Copyright (c) 2013-present Tim Griesser dev,mkdirp,MIT,Copyright 2010 James Halliday dev,mocha,MIT,Copyright 2011-2018 JS Foundation and contributors https://js.foundation +dev,msgpack-lite,MIT,Copyright 2015 Yusuke Kawasaki dev,multer,MIT,Copyright 2014 Hage Yaapa dev,nock,MIT,Copyright 2017 Pedro Teixeira and other contributors dev,nyc,ISC,Copyright 2015 Contributors diff --git a/package.json b/package.json index dd90ee51661..008fd1f17d3 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,6 @@ "dc-polyfill": "^0.1.4", "ignore": "^5.2.4", "import-in-the-middle": "1.11.2", - "int64-buffer": "^0.1.9", "istanbul-lib-coverage": "3.2.0", "jest-docblock": "^29.7.0", "koalas": "^1.0.2", @@ -103,7 +102,6 @@ "lodash.sortby": "^4.7.0", "lru-cache": "^7.14.0", "module-details-from-path": "^1.0.3", - "msgpack-lite": "^0.1.26", "opentracing": ">=0.12.1", "path-to-regexp": "^0.1.12", "pprof-format": "^2.1.0", @@ -143,10 +141,12 @@ "glob": "^7.1.6", "globals": "^15.10.0", "graphql": "0.13.2", + "int64-buffer": "^0.1.9", "jszip": "^3.5.0", "knex": "^2.4.2", "mkdirp": "^3.0.1", "mocha": "^9", + "msgpack-lite": "^0.1.26", "multer": "^1.4.5-lts.1", "nock": "^11.3.3", "nyc": "^15.1.0", diff --git a/packages/dd-trace/src/datastreams/processor.js b/packages/dd-trace/src/datastreams/processor.js index d036af805a7..d997ba098ae 100644 --- a/packages/dd-trace/src/datastreams/processor.js +++ b/packages/dd-trace/src/datastreams/processor.js @@ -1,7 +1,5 @@ const os = require('os') const pkg = require('../../../../package.json') -// Message pack int encoding is done in big endian, but data streams uses little endian -const Uint64 = require('int64-buffer').Uint64BE const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js') const { DsmPathwayCodec } = require('./pathway') @@ -19,8 +17,8 @@ const HIGH_ACCURACY_DISTRIBUTION = 0.0075 class StatsPoint { constructor (hash, parentHash, edgeTags) { - this.hash = new Uint64(hash) - this.parentHash = new Uint64(parentHash) + this.hash = hash.readBigUInt64BE() + this.parentHash = parentHash.readBigUInt64BE() this.edgeTags = edgeTags this.edgeLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION) this.pathwayLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION) @@ -344,8 +342,8 @@ class DataStreamsProcessor { backlogs.push(backlog.encode()) } serializedBuckets.push({ - Start: new Uint64(timeNs), - Duration: new Uint64(this.bucketSizeNs), + Start: BigInt(timeNs), + Duration: BigInt(this.bucketSizeNs), Stats: points, Backlogs: backlogs }) diff --git a/packages/dd-trace/src/datastreams/writer.js b/packages/dd-trace/src/datastreams/writer.js index 5f789f2e056..220b3dfecf7 100644 --- a/packages/dd-trace/src/datastreams/writer.js +++ b/packages/dd-trace/src/datastreams/writer.js @@ -2,9 +2,10 @@ const pkg = require('../../../../package.json') const log = require('../log') const request = require('../exporters/common/request') const { URL, format } = require('url') -const msgpack = require('msgpack-lite') +const { MsgpackEncoder } = require('../msgpack') const zlib = require('zlib') -const codec = msgpack.createCodec({ int64: true }) + +const msgpack = new MsgpackEncoder() function makeRequest (data, url, cb) { const options = { @@ -41,7 +42,7 @@ class DataStreamsWriter { log.debug(() => `Maximum number of active requests reached. Payload discarded: ${JSON.stringify(payload)}`) return } - const encodedPayload = msgpack.encode(payload, { codec }) + const encodedPayload = msgpack.encode(payload) zlib.gzip(encodedPayload, { level: 1 }, (err, compressedData) => { if (err) { diff --git a/packages/dd-trace/src/encode/0.4.js b/packages/dd-trace/src/encode/0.4.js index 02d96cb8a26..d5c72bdb575 100644 --- a/packages/dd-trace/src/encode/0.4.js +++ b/packages/dd-trace/src/encode/0.4.js @@ -1,26 +1,20 @@ 'use strict' const { truncateSpan, normalizeSpan } = require('./tags-processors') -const Chunk = require('./chunk') +const { Chunk, MsgpackEncoder } = require('../msgpack') const log = require('../log') const { isTrue } = require('../util') const coalesce = require('koalas') const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB -const float64Array = new Float64Array(1) -const uInt8Float64Array = new Uint8Array(float64Array.buffer) - -float64Array[0] = -1 - -const bigEndian = uInt8Float64Array[7] === 0 - function formatSpan (span) { return normalizeSpan(truncateSpan(span, false)) } class AgentEncoder { constructor (writer, limit = SOFT_LIMIT) { + this._msgpack = new MsgpackEncoder() this._limit = limit this._traceBytes = new Chunk() this._stringBytes = new Chunk() @@ -84,11 +78,11 @@ class AgentEncoder { bytes.reserve(1) if (span.type && span.meta_struct) { - bytes.buffer[bytes.length++] = 0x8d + bytes.buffer[bytes.length - 1] = 0x8d } else if (span.type || span.meta_struct) { - bytes.buffer[bytes.length++] = 0x8c + bytes.buffer[bytes.length - 1] = 0x8c } else { - bytes.buffer[bytes.length++] = 0x8b + bytes.buffer[bytes.length - 1] = 0x8b } if (span.type) { @@ -135,43 +129,31 @@ class AgentEncoder { this._cacheString('') } - _encodeArrayPrefix (bytes, value) { - const length = value.length - const offset = bytes.length + _encodeBuffer (bytes, buffer) { + this._msgpack.encodeBin(bytes, buffer) + } - bytes.reserve(5) - bytes.length += 5 + _encodeBool (bytes, value) { + this._msgpack.encodeBoolean(bytes, value) + } - bytes.buffer[offset] = 0xdd - bytes.buffer[offset + 1] = length >> 24 - bytes.buffer[offset + 2] = length >> 16 - bytes.buffer[offset + 3] = length >> 8 - bytes.buffer[offset + 4] = length + _encodeArrayPrefix (bytes, value) { + this._msgpack.encodeArrayPrefix(bytes, value) } _encodeMapPrefix (bytes, keysLength) { - const offset = bytes.length - - bytes.reserve(5) - bytes.length += 5 - bytes.buffer[offset] = 0xdf - bytes.buffer[offset + 1] = keysLength >> 24 - bytes.buffer[offset + 2] = keysLength >> 16 - bytes.buffer[offset + 3] = keysLength >> 8 - bytes.buffer[offset + 4] = keysLength + this._msgpack.encodeMapPrefix(bytes, keysLength) } _encodeByte (bytes, value) { - bytes.reserve(1) - - bytes.buffer[bytes.length++] = value + this._msgpack.encodeByte(bytes, value) } + // TODO: Use BigInt instead. _encodeId (bytes, id) { const offset = bytes.length bytes.reserve(9) - bytes.length += 9 id = id.toArray() @@ -186,36 +168,16 @@ class AgentEncoder { bytes.buffer[offset + 8] = id[7] } - _encodeInteger (bytes, value) { - const offset = bytes.length - - bytes.reserve(5) - bytes.length += 5 + _encodeNumber (bytes, value) { + this._msgpack.encodeNumber(bytes, value) + } - bytes.buffer[offset] = 0xce - bytes.buffer[offset + 1] = value >> 24 - bytes.buffer[offset + 2] = value >> 16 - bytes.buffer[offset + 3] = value >> 8 - bytes.buffer[offset + 4] = value + _encodeInteger (bytes, value) { + this._msgpack.encodeInteger(bytes, value) } _encodeLong (bytes, value) { - const offset = bytes.length - const hi = (value / Math.pow(2, 32)) >> 0 - const lo = value >>> 0 - - bytes.reserve(9) - bytes.length += 9 - - bytes.buffer[offset] = 0xcf - bytes.buffer[offset + 1] = hi >> 24 - bytes.buffer[offset + 2] = hi >> 16 - bytes.buffer[offset + 3] = hi >> 8 - bytes.buffer[offset + 4] = hi - bytes.buffer[offset + 5] = lo >> 24 - bytes.buffer[offset + 6] = lo >> 16 - bytes.buffer[offset + 7] = lo >> 8 - bytes.buffer[offset + 8] = lo + this._msgpack.encodeLong(bytes, value) } _encodeMap (bytes, value) { @@ -252,23 +214,7 @@ class AgentEncoder { } _encodeFloat (bytes, value) { - float64Array[0] = value - - const offset = bytes.length - bytes.reserve(9) - bytes.length += 9 - - bytes.buffer[offset] = 0xcb - - if (bigEndian) { - for (let i = 0; i <= 7; i++) { - bytes.buffer[offset + i + 1] = uInt8Float64Array[i] - } - } else { - for (let i = 7; i >= 0; i--) { - bytes.buffer[bytes.length - i - 1] = uInt8Float64Array[i] - } - } + this._msgpack.encodeFloat(bytes, value) } _encodeMetaStruct (bytes, value) { @@ -294,7 +240,6 @@ class AgentEncoder { const offset = bytes.length bytes.reserve(prefixLength) - bytes.length += prefixLength this._encodeObject(bytes, value) diff --git a/packages/dd-trace/src/encode/agentless-ci-visibility.js b/packages/dd-trace/src/encode/agentless-ci-visibility.js index dea15182323..bc5d9fc42b6 100644 --- a/packages/dd-trace/src/encode/agentless-ci-visibility.js +++ b/packages/dd-trace/src/encode/agentless-ci-visibility.js @@ -251,37 +251,6 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { } } - _encodeNumber (bytes, value) { - if (Math.floor(value) !== value) { // float 64 - return this._encodeFloat(bytes, value) - } - return this._encodeLong(bytes, value) - } - - _encodeLong (bytes, value) { - const isPositive = value >= 0 - - const hi = isPositive ? (value / Math.pow(2, 32)) >> 0 : Math.floor(value / Math.pow(2, 32)) - const lo = value >>> 0 - const flag = isPositive ? 0xcf : 0xd3 - - const offset = bytes.length - - // int 64 - bytes.reserve(9) - bytes.length += 9 - - bytes.buffer[offset] = flag - bytes.buffer[offset + 1] = hi >> 24 - bytes.buffer[offset + 2] = hi >> 16 - bytes.buffer[offset + 3] = hi >> 8 - bytes.buffer[offset + 4] = hi - bytes.buffer[offset + 5] = lo >> 24 - bytes.buffer[offset + 6] = lo >> 16 - bytes.buffer[offset + 7] = lo >> 8 - bytes.buffer[offset + 8] = lo - } - _encode (bytes, trace) { if (this._isReset) { this._encodePayloadStart(bytes) @@ -380,7 +349,6 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { // Get offset of the events list to update the length of the array when calling `makePayload` this._eventsOffset = bytes.length bytes.reserve(5) - bytes.length += 5 } reset () { diff --git a/packages/dd-trace/src/encode/coverage-ci-visibility.js b/packages/dd-trace/src/encode/coverage-ci-visibility.js index bdf4b17a3cc..5b31d83cb12 100644 --- a/packages/dd-trace/src/encode/coverage-ci-visibility.js +++ b/packages/dd-trace/src/encode/coverage-ci-visibility.js @@ -1,6 +1,6 @@ 'use strict' const { AgentEncoder } = require('./0.4') -const Chunk = require('./chunk') +const { Chunk } = require('../msgpack') const { distributionMetric, @@ -82,7 +82,6 @@ class CoverageCIVisibilityEncoder extends AgentEncoder { // Get offset of the coverages list to update the length of the array when calling `makePayload` this._coveragesOffset = bytes.length bytes.reserve(5) - bytes.length += 5 } makePayload () { diff --git a/packages/dd-trace/src/encode/span-stats.js b/packages/dd-trace/src/encode/span-stats.js index 15410cec203..43215756c7c 100644 --- a/packages/dd-trace/src/encode/span-stats.js +++ b/packages/dd-trace/src/encode/span-stats.js @@ -22,10 +22,6 @@ function truncate (value, maxLength, suffix = '') { } class SpanStatsEncoder extends AgentEncoder { - _encodeBool (bytes, value) { - this._encodeByte(bytes, value ? 0xc3 : 0xc2) - } - makePayload () { const traceSize = this._traceBytes.length const buffer = Buffer.allocUnsafe(traceSize) @@ -34,32 +30,6 @@ class SpanStatsEncoder extends AgentEncoder { return buffer } - _encodeMapPrefix (bytes, length) { - const offset = bytes.length - - bytes.reserve(1) - bytes.length += 1 - - bytes.buffer[offset] = 0x80 + length - } - - _encodeBuffer (bytes, buffer) { - const length = buffer.length - const offset = bytes.length - - bytes.reserve(5) - bytes.length += 5 - - bytes.buffer[offset] = 0xc6 - bytes.buffer[offset + 1] = length >> 24 - bytes.buffer[offset + 2] = length >> 16 - bytes.buffer[offset + 3] = length >> 8 - bytes.buffer[offset + 4] = length - - buffer.copy(bytes.buffer, offset + 5) - bytes.length += length - } - _encodeStat (bytes, stat) { this._encodeMapPrefix(bytes, 12) diff --git a/packages/dd-trace/src/encode/chunk.js b/packages/dd-trace/src/msgpack/chunk.js similarity index 85% rename from packages/dd-trace/src/encode/chunk.js rename to packages/dd-trace/src/msgpack/chunk.js index 8a17b45f430..02999086c55 100644 --- a/packages/dd-trace/src/encode/chunk.js +++ b/packages/dd-trace/src/msgpack/chunk.js @@ -10,6 +10,7 @@ const DEFAULT_MIN_SIZE = 2 * 1024 * 1024 // 2MB class Chunk { constructor (minSize = DEFAULT_MIN_SIZE) { this.buffer = Buffer.allocUnsafe(minSize) + this.view = new DataView(this.buffer.buffer) this.length = 0 this._minSize = minSize } @@ -20,11 +21,9 @@ class Chunk { if (length < 0x20) { // fixstr this.reserve(length + 1) - this.length += 1 this.buffer[offset] = length | 0xa0 } else if (length < 0x100000000) { // str 32 this.reserve(length + 5) - this.length += 5 this.buffer[offset] = 0xdb this.buffer[offset + 1] = length >> 24 this.buffer[offset + 2] = length >> 16 @@ -32,7 +31,7 @@ class Chunk { this.buffer[offset + 4] = length } - this.length += this.buffer.utf8Write(value, this.length, length) + this.buffer.utf8Write(value, this.length - length, length) return this.length - offset } @@ -42,22 +41,26 @@ class Chunk { } set (array) { + const length = this.length + this.reserve(array.length) - this.buffer.set(array, this.length) - this.length += array.length + this.buffer.set(array, length) } reserve (size) { if (this.length + size > this.buffer.length) { this._resize(this._minSize * Math.ceil((this.length + size) / this._minSize)) } + + this.length += size } _resize (size) { const oldBuffer = this.buffer this.buffer = Buffer.allocUnsafe(size) + this.view = new DataView(this.buffer.buffer) oldBuffer.copy(this.buffer, 0, 0, this.length) } diff --git a/packages/dd-trace/src/msgpack/encoder.js b/packages/dd-trace/src/msgpack/encoder.js new file mode 100644 index 00000000000..6fa39d82148 --- /dev/null +++ b/packages/dd-trace/src/msgpack/encoder.js @@ -0,0 +1,309 @@ +'use strict' + +const Chunk = require('./chunk') + +class MsgpackEncoder { + encode (value) { + const bytes = new Chunk() + + this.encodeValue(bytes, value) + + return bytes.buffer.subarray(0, bytes.length) + } + + encodeValue (bytes, value) { + switch (typeof value) { + case 'bigint': + this.encodeBigInt(bytes, value) + break + case 'boolean': + this.encodeBoolean(bytes, value) + break + case 'number': + this.encodeNumber(bytes, value) + break + case 'object': + if (value === null) { + this.encodeNull(bytes, value) + } else if (Array.isArray(value)) { + this.encodeArray(bytes, value) + } else if (Buffer.isBuffer(value) || ArrayBuffer.isView(value)) { + this.encodeBin(bytes, value) + } else { + this.encodeMap(bytes, value) + } + break + case 'string': + this.encodeString(bytes, value) + break + case 'symbol': + this.encodeString(bytes, value.toString()) + break + default: // function, symbol, undefined + this.encodeNull(bytes, value) + break + } + } + + encodeNull (bytes) { + const offset = bytes.length + + bytes.reserve(1) + bytes.buffer[offset] = 0xc0 + } + + encodeBoolean (bytes, value) { + const offset = bytes.length + + bytes.reserve(1) + bytes.buffer[offset] = value ? 0xc3 : 0xc2 + } + + encodeString (bytes, value) { + bytes.write(value) + } + + encodeFixArray (bytes, size = 0) { + const offset = bytes.length + + bytes.reserve(1) + bytes.buffer[offset] = 0x90 + size + } + + encodeArrayPrefix (bytes, value) { + const length = value.length + const offset = bytes.length + + bytes.reserve(5) + bytes.buffer[offset] = 0xdd + bytes.buffer[offset + 1] = length >> 24 + bytes.buffer[offset + 2] = length >> 16 + bytes.buffer[offset + 3] = length >> 8 + bytes.buffer[offset + 4] = length + } + + encodeArray (bytes, value) { + if (value.length < 16) { + this.encodeFixArray(bytes, value.length) + } else { + this.encodeArrayPrefix(bytes, value) + } + + for (const item of value) { + this.encodeValue(bytes, item) + } + } + + encodeFixMap (bytes, size = 0) { + const offset = bytes.length + + bytes.reserve(1) + bytes.buffer[offset] = 0x80 + size + } + + encodeMapPrefix (bytes, keysLength) { + const offset = bytes.length + + bytes.reserve(5) + bytes.buffer[offset] = 0xdf + bytes.buffer[offset + 1] = keysLength >> 24 + bytes.buffer[offset + 2] = keysLength >> 16 + bytes.buffer[offset + 3] = keysLength >> 8 + bytes.buffer[offset + 4] = keysLength + } + + encodeByte (bytes, value) { + bytes.reserve(1) + bytes.buffer[bytes.length - 1] = value + } + + encodeBin (bytes, value) { + const offset = bytes.length + + if (value.byteLength < 256) { + bytes.reserve(2) + bytes.buffer[offset] = 0xc4 + bytes.buffer[offset + 1] = value.byteLength + } else if (value.byteLength < 65536) { + bytes.reserve(3) + bytes.buffer[offset] = 0xc5 + bytes.buffer[offset + 1] = value.byteLength >> 8 + bytes.buffer[offset + 2] = value.byteLength + } else { + bytes.reserve(5) + bytes.buffer[offset] = 0xc6 + bytes.buffer[offset + 1] = value.byteLength >> 24 + bytes.buffer[offset + 2] = value.byteLength >> 16 + bytes.buffer[offset + 3] = value.byteLength >> 8 + bytes.buffer[offset + 4] = value.byteLength + } + + bytes.set(value) + } + + encodeInteger (bytes, value) { + const offset = bytes.length + + bytes.reserve(5) + bytes.buffer[offset] = 0xce + bytes.buffer[offset + 1] = value >> 24 + bytes.buffer[offset + 2] = value >> 16 + bytes.buffer[offset + 3] = value >> 8 + bytes.buffer[offset + 4] = value + } + + encodeShort (bytes, value) { + const offset = bytes.length + + bytes.reserve(3) + bytes.buffer[offset] = 0xcd + bytes.buffer[offset + 1] = value >> 8 + bytes.buffer[offset + 2] = value + } + + encodeLong (bytes, value) { + const offset = bytes.length + const hi = (value / Math.pow(2, 32)) >> 0 + const lo = value >>> 0 + + bytes.reserve(9) + bytes.buffer[offset] = 0xcf + bytes.buffer[offset + 1] = hi >> 24 + bytes.buffer[offset + 2] = hi >> 16 + bytes.buffer[offset + 3] = hi >> 8 + bytes.buffer[offset + 4] = hi + bytes.buffer[offset + 5] = lo >> 24 + bytes.buffer[offset + 6] = lo >> 16 + bytes.buffer[offset + 7] = lo >> 8 + bytes.buffer[offset + 8] = lo + } + + encodeNumber (bytes, value) { + if (Number.isNaN(value)) { + value = 0 + } + if (Number.isInteger(value)) { + if (value >= 0) { + this.encodeUnsigned(bytes, value) + } else { + this.encodeSigned(bytes, value) + } + } else { + this.encodeFloat(bytes, value) + } + } + + encodeSigned (bytes, value) { + const offset = bytes.length + + if (value >= -0x20) { + bytes.reserve(1) + bytes.buffer[offset] = value + } else if (value >= -0x80) { + bytes.reserve(2) + bytes.buffer[offset] = 0xd0 + bytes.buffer[offset + 1] = value + } else if (value >= -0x8000) { + bytes.reserve(3) + bytes.buffer[offset] = 0xd1 + bytes.buffer[offset + 1] = value >> 8 + bytes.buffer[offset + 2] = value + } else if (value >= -0x80000000) { + bytes.reserve(5) + bytes.buffer[offset] = 0xd2 + bytes.buffer[offset + 1] = value >> 24 + bytes.buffer[offset + 2] = value >> 16 + bytes.buffer[offset + 3] = value >> 8 + bytes.buffer[offset + 4] = value + } else { + const hi = Math.floor(value / Math.pow(2, 32)) + const lo = value >>> 0 + + bytes.reserve(9) + bytes.buffer[offset] = 0xd3 + bytes.buffer[offset + 1] = hi >> 24 + bytes.buffer[offset + 2] = hi >> 16 + bytes.buffer[offset + 3] = hi >> 8 + bytes.buffer[offset + 4] = hi + bytes.buffer[offset + 5] = lo >> 24 + bytes.buffer[offset + 6] = lo >> 16 + bytes.buffer[offset + 7] = lo >> 8 + bytes.buffer[offset + 8] = lo + } + } + + encodeUnsigned (bytes, value) { + const offset = bytes.length + + if (value <= 0x7f) { + bytes.reserve(1) + bytes.buffer[offset] = value + } else if (value <= 0xff) { + bytes.reserve(2) + bytes.buffer[offset] = 0xcc + bytes.buffer[offset + 1] = value + } else if (value <= 0xffff) { + bytes.reserve(3) + bytes.buffer[offset] = 0xcd + bytes.buffer[offset + 1] = value >> 8 + bytes.buffer[offset + 2] = value + } else if (value <= 0xffffffff) { + bytes.reserve(5) + bytes.buffer[offset] = 0xce + bytes.buffer[offset + 1] = value >> 24 + bytes.buffer[offset + 2] = value >> 16 + bytes.buffer[offset + 3] = value >> 8 + bytes.buffer[offset + 4] = value + } else { + const hi = (value / Math.pow(2, 32)) >> 0 + const lo = value >>> 0 + + bytes.reserve(9) + bytes.buffer[offset] = 0xcf + bytes.buffer[offset + 1] = hi >> 24 + bytes.buffer[offset + 2] = hi >> 16 + bytes.buffer[offset + 3] = hi >> 8 + bytes.buffer[offset + 4] = hi + bytes.buffer[offset + 5] = lo >> 24 + bytes.buffer[offset + 6] = lo >> 16 + bytes.buffer[offset + 7] = lo >> 8 + bytes.buffer[offset + 8] = lo + } + } + + // TODO: Support BigInt larger than 64bit. + encodeBigInt (bytes, value) { + const offset = bytes.length + + bytes.reserve(9) + + if (value >= 0n) { + bytes.buffer[offset] = 0xcf + bytes.view.setBigUint64(offset + 1, value) + } else { + bytes.buffer[offset] = 0xd3 + bytes.view.setBigInt64(offset + 1, value) + } + } + + encodeMap (bytes, value) { + const keys = Object.keys(value) + + this.encodeMapPrefix(bytes, keys.length) + + for (const key of keys) { + this.encodeValue(bytes, key) + this.encodeValue(bytes, value[key]) + } + } + + encodeFloat (bytes, value) { + const offset = bytes.length + + bytes.reserve(9) + bytes.buffer[offset] = 0xcb + bytes.view.setFloat64(offset + 1, value) + } +} + +module.exports = { MsgpackEncoder } diff --git a/packages/dd-trace/src/msgpack/index.js b/packages/dd-trace/src/msgpack/index.js new file mode 100644 index 00000000000..03228d27044 --- /dev/null +++ b/packages/dd-trace/src/msgpack/index.js @@ -0,0 +1,6 @@ +'use strict' + +const Chunk = require('./chunk') +const { MsgpackEncoder } = require('./encoder') + +module.exports = { Chunk, MsgpackEncoder } diff --git a/packages/dd-trace/test/datastreams/processor.spec.js b/packages/dd-trace/test/datastreams/processor.spec.js index 0c30bc77947..110d9ff6c35 100644 --- a/packages/dd-trace/test/datastreams/processor.spec.js +++ b/packages/dd-trace/test/datastreams/processor.spec.js @@ -294,11 +294,11 @@ describe('DataStreamsProcessor', () => { Service: 'service1', Version: 'v1', Stats: [{ - Start: new Uint64(1680000000000), - Duration: new Uint64(10000000000), + Start: 1680000000000n, + Duration: 10000000000n, Stats: [{ - Hash: new Uint64(DEFAULT_CURRENT_HASH), - ParentHash: new Uint64(DEFAULT_PARENT_HASH), + Hash: DEFAULT_CURRENT_HASH.readBigUInt64BE(), + ParentHash: DEFAULT_PARENT_HASH.readBigUInt64BE(), EdgeTags: mockCheckpoint.edgeTags, EdgeLatency: edgeLatency.toProto(), PathwayLatency: pathwayLatency.toProto(), diff --git a/packages/dd-trace/test/encode/agentless-ci-visibility.spec.js b/packages/dd-trace/test/encode/agentless-ci-visibility.spec.js index 54ddab1a2a6..259ff78df2e 100644 --- a/packages/dd-trace/test/encode/agentless-ci-visibility.spec.js +++ b/packages/dd-trace/test/encode/agentless-ci-visibility.spec.js @@ -67,14 +67,14 @@ describe('agentless-ci-visibility-encode', () => { const buffer = encoder.makePayload() const decodedTrace = msgpack.decode(buffer, { codec }) - expect(decodedTrace.version.toNumber()).to.equal(1) + expect(decodedTrace.version).to.equal(1) expect(decodedTrace.metadata['*']).to.contain({ language: 'javascript', library_version: ddTraceVersion }) const spanEvent = decodedTrace.events[0] expect(spanEvent.type).to.equal('span') - expect(spanEvent.version.toNumber()).to.equal(1) + expect(spanEvent.version).to.equal(1) expect(spanEvent.content.trace_id.toString(10)).to.equal(trace[0].trace_id.toString(10)) expect(spanEvent.content.span_id.toString(10)).to.equal(trace[0].span_id.toString(10)) expect(spanEvent.content.parent_id.toString(10)).to.equal(trace[0].parent_id.toString(10)) @@ -84,9 +84,9 @@ describe('agentless-ci-visibility-encode', () => { service: 'test-s', type: 'foo' }) - expect(spanEvent.content.error.toNumber()).to.equal(0) - expect(spanEvent.content.start.toNumber()).to.equal(123) - expect(spanEvent.content.duration.toNumber()).to.equal(456) + expect(spanEvent.content.error).to.equal(0) + expect(spanEvent.content.start).to.equal(123) + expect(spanEvent.content.duration).to.equal(456) expect(spanEvent.content.meta).to.eql({ bar: 'baz' @@ -276,6 +276,6 @@ describe('agentless-ci-visibility-encode', () => { const decodedTrace = msgpack.decode(buffer, { codec }) const spanEvent = decodedTrace.events[0] expect(spanEvent.type).to.equal('span') - expect(spanEvent.version.toNumber()).to.equal(1) + expect(spanEvent.version).to.equal(1) }) }) diff --git a/packages/dd-trace/test/msgpack/encoder.spec.js b/packages/dd-trace/test/msgpack/encoder.spec.js new file mode 100644 index 00000000000..cfda0a9e7d7 --- /dev/null +++ b/packages/dd-trace/test/msgpack/encoder.spec.js @@ -0,0 +1,88 @@ +'use strict' + +require('../setup/tap') + +const { expect } = require('chai') +const msgpack = require('msgpack-lite') +const codec = msgpack.createCodec({ int64: true }) +const { MsgpackEncoder } = require('../../src/msgpack/encoder') + +function randString (length) { + return Array.from({ length }, () => { + return String.fromCharCode(Math.floor(Math.random() * 256)) + }).join('') +} + +describe('msgpack/encoder', () => { + let encoder + + beforeEach(() => { + encoder = new MsgpackEncoder() + }) + + it('should encode to msgpack', () => { + const data = [ + { first: 'test' }, + { + fixstr: 'foo', + str: randString(1000), + fixuint: 127, + fixint: -31, + uint8: 255, + uint16: 65535, + uint32: 4294967295, + uint53: 9007199254740991, + int8: -15, + int16: -32767, + int32: -2147483647, + int53: -9007199254740991, + float: 12345.6789, + biguint: BigInt('9223372036854775807'), + bigint: BigInt('-9223372036854775807'), + buffer: Buffer.from('test'), + uint8array: new Uint8Array([1, 2, 3, 4]), + uint32array: new Uint32Array([1, 2]) + } + ] + + const buffer = encoder.encode(data) + const decoded = msgpack.decode(buffer, { codec }) + + expect(decoded).to.be.an('array') + expect(decoded[0]).to.be.an('object') + expect(decoded[0]).to.have.property('first', 'test') + expect(decoded[1]).to.be.an('object') + expect(decoded[1]).to.have.property('fixstr', 'foo') + expect(decoded[1]).to.have.property('str') + expect(decoded[1].str).to.have.length(1000) + expect(decoded[1]).to.have.property('fixuint', 127) + expect(decoded[1]).to.have.property('fixint', -31) + expect(decoded[1]).to.have.property('uint8', 255) + expect(decoded[1]).to.have.property('uint16', 65535) + expect(decoded[1]).to.have.property('uint32', 4294967295) + expect(decoded[1]).to.have.property('uint53') + expect(decoded[1].uint53.toString()).to.equal('9007199254740991') + expect(decoded[1]).to.have.property('int8', -15) + expect(decoded[1]).to.have.property('int16', -32767) + expect(decoded[1]).to.have.property('int32', -2147483647) + expect(decoded[1]).to.have.property('int53') + expect(decoded[1].int53.toString()).to.equal('-9007199254740991') + expect(decoded[1]).to.have.property('float', 12345.6789) + expect(decoded[1]).to.have.property('biguint') + expect(decoded[1].biguint.toString()).to.equal('9223372036854775807') + expect(decoded[1]).to.have.property('bigint') + expect(decoded[1].bigint.toString()).to.equal('-9223372036854775807') + expect(decoded[1]).to.have.property('buffer') + expect(decoded[1].buffer.toString('utf8')).to.equal('test') + expect(decoded[1]).to.have.property('buffer') + expect(decoded[1].buffer.toString('utf8')).to.equal('test') + expect(decoded[1]).to.have.property('uint8array') + expect(decoded[1].uint8array[0]).to.equal(1) + expect(decoded[1].uint8array[1]).to.equal(2) + expect(decoded[1].uint8array[2]).to.equal(3) + expect(decoded[1].uint8array[3]).to.equal(4) + expect(decoded[1]).to.have.property('uint32array') + expect(decoded[1].uint32array[0]).to.equal(1) + expect(decoded[1].uint32array[4]).to.equal(2) + }) +})