remove dependency on msgpack-lite (#4969)

* remove dependency on msgpack-lite * remove int64-buffer as well * fix datastreams processor * fix span stats * fix ci visibility * add support for encoding buffers and typedarrays * fix ci visibility agentless encoder * make everything faster and fix negative 53bit ints * remove debug log * remove more usage of dataview * optimize chunk write * stop handling typedarrays in set
DataDog · Dec 12, 2024 · 329bdf9 · 329bdf9
1 parent 594ca4c
commit 329bdf9
Show file tree

Hide file tree

Showing 14 changed files with 457 additions and 170 deletions.
diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv
@@ -13,15 +13,13 @@ require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors
 require,dc-polyfill,MIT,Copyright 2023 Datadog Inc.
 require,ignore,MIT,Copyright 2013 Kael Zhang and contributors
 require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
-require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki
 require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
 require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,limiter,MIT,Copyright 2011 John Hurliman
 require,lodash.sortby,MIT,Copyright JS Foundation and other contributors
 require,lru-cache,ISC,Copyright (c) 2010-2022 Isaac Z. Schlueter and Contributors
 require,module-details-from-path,MIT,Copyright 2016 Thomas Watson Steen
-require,msgpack-lite,MIT,Copyright 2015 Yusuke Kawasaki
 require,opentracing,MIT,Copyright 2016 Resonance Labs Inc
 require,path-to-regexp,MIT,Copyright 2014 Blake Embrey
 require,pprof-format,MIT,Copyright 2022 Stephen Belanger
@@ -59,10 +57,12 @@ dev,get-port,MIT,Copyright Sindre Sorhus
 dev,glob,ISC,Copyright Isaac Z. Schlueter and Contributors
 dev,globals,MIT,Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com)
 dev,graphql,MIT,Copyright 2015 Facebook Inc.
+dev,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki
 dev,jszip,MIT,Copyright 2015-2016 Stuart Knightley and contributors
 dev,knex,MIT,Copyright (c) 2013-present Tim Griesser
 dev,mkdirp,MIT,Copyright 2010 James Halliday
 dev,mocha,MIT,Copyright 2011-2018 JS Foundation and contributors https://js.foundation
+dev,msgpack-lite,MIT,Copyright 2015 Yusuke Kawasaki
 dev,multer,MIT,Copyright 2014 Hage Yaapa
 dev,nock,MIT,Copyright 2017 Pedro Teixeira and other contributors
 dev,nyc,ISC,Copyright 2015 Contributors

diff --git a/package.json b/package.json
@@ -95,15 +95,13 @@
     "dc-polyfill": "^0.1.4",
     "ignore": "^5.2.4",
     "import-in-the-middle": "1.11.2",
-    "int64-buffer": "^0.1.9",
     "istanbul-lib-coverage": "3.2.0",
     "jest-docblock": "^29.7.0",
     "koalas": "^1.0.2",
     "limiter": "1.1.5",
     "lodash.sortby": "^4.7.0",
     "lru-cache": "^7.14.0",
     "module-details-from-path": "^1.0.3",
-    "msgpack-lite": "^0.1.26",
     "opentracing": ">=0.12.1",
     "path-to-regexp": "^0.1.12",
     "pprof-format": "^2.1.0",
@@ -143,10 +141,12 @@
     "glob": "^7.1.6",
     "globals": "^15.10.0",
     "graphql": "0.13.2",
+    "int64-buffer": "^0.1.9",
     "jszip": "^3.5.0",
     "knex": "^2.4.2",
     "mkdirp": "^3.0.1",
     "mocha": "^9",
+    "msgpack-lite": "^0.1.26",
     "multer": "^1.4.5-lts.1",
     "nock": "^11.3.3",
     "nyc": "^15.1.0",

diff --git a/packages/dd-trace/src/datastreams/processor.js b/packages/dd-trace/src/datastreams/processor.js
@@ -1,7 +1,5 @@
 const os = require('os')
 const pkg = require('../../../../package.json')
-// Message pack int encoding is done in big endian, but data streams uses little endian
-const Uint64 = require('int64-buffer').Uint64BE
 
 const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js')
 const { DsmPathwayCodec } = require('./pathway')
@@ -19,8 +17,8 @@ const HIGH_ACCURACY_DISTRIBUTION = 0.0075
 
 class StatsPoint {
   constructor (hash, parentHash, edgeTags) {
-    this.hash = new Uint64(hash)
-    this.parentHash = new Uint64(parentHash)
+    this.hash = hash.readBigUInt64BE()
+    this.parentHash = parentHash.readBigUInt64BE()
     this.edgeTags = edgeTags
     this.edgeLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
     this.pathwayLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
@@ -344,8 +342,8 @@ class DataStreamsProcessor {
         backlogs.push(backlog.encode())
       }
       serializedBuckets.push({
-        Start: new Uint64(timeNs),
-        Duration: new Uint64(this.bucketSizeNs),
+        Start: BigInt(timeNs),
+        Duration: BigInt(this.bucketSizeNs),
         Stats: points,
         Backlogs: backlogs
       })

diff --git a/packages/dd-trace/src/datastreams/writer.js b/packages/dd-trace/src/datastreams/writer.js
@@ -2,9 +2,10 @@ const pkg = require('../../../../package.json')
 const log = require('../log')
 const request = require('../exporters/common/request')
 const { URL, format } = require('url')
-const msgpack = require('msgpack-lite')
+const { MsgpackEncoder } = require('../msgpack')
 const zlib = require('zlib')
-const codec = msgpack.createCodec({ int64: true })
+
+const msgpack = new MsgpackEncoder()
 
 function makeRequest (data, url, cb) {
   const options = {
@@ -41,7 +42,7 @@ class DataStreamsWriter {
       log.debug(() => `Maximum number of active requests reached. Payload discarded: ${JSON.stringify(payload)}`)
       return
     }
-    const encodedPayload = msgpack.encode(payload, { codec })
+    const encodedPayload = msgpack.encode(payload)
 
     zlib.gzip(encodedPayload, { level: 1 }, (err, compressedData) => {
       if (err) {

diff --git a/packages/dd-trace/src/encode/0.4.js b/packages/dd-trace/src/encode/0.4.js
@@ -1,26 +1,20 @@
 'use strict'
 
 const { truncateSpan, normalizeSpan } = require('./tags-processors')
-const Chunk = require('./chunk')
+const { Chunk, MsgpackEncoder } = require('../msgpack')
 const log = require('../log')
 const { isTrue } = require('../util')
 const coalesce = require('koalas')
 
 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB
 
-const float64Array = new Float64Array(1)
-const uInt8Float64Array = new Uint8Array(float64Array.buffer)
-
-float64Array[0] = -1
-
-const bigEndian = uInt8Float64Array[7] === 0
-
 function formatSpan (span) {
   return normalizeSpan(truncateSpan(span, false))
 }
 
 class AgentEncoder {
   constructor (writer, limit = SOFT_LIMIT) {
+    this._msgpack = new MsgpackEncoder()
     this._limit = limit
     this._traceBytes = new Chunk()
     this._stringBytes = new Chunk()
@@ -84,11 +78,11 @@ class AgentEncoder {
       bytes.reserve(1)
 
       if (span.type && span.meta_struct) {
-        bytes.buffer[bytes.length++] = 0x8d
+        bytes.buffer[bytes.length - 1] = 0x8d
       } else if (span.type || span.meta_struct) {
-        bytes.buffer[bytes.length++] = 0x8c
+        bytes.buffer[bytes.length - 1] = 0x8c
       } else {
-        bytes.buffer[bytes.length++] = 0x8b
+        bytes.buffer[bytes.length - 1] = 0x8b
       }
 
       if (span.type) {
@@ -135,43 +129,31 @@ class AgentEncoder {
     this._cacheString('')
   }
 
-  _encodeArrayPrefix (bytes, value) {
-    const length = value.length
-    const offset = bytes.length
+  _encodeBuffer (bytes, buffer) {
+    this._msgpack.encodeBin(bytes, buffer)
+  }
 
-    bytes.reserve(5)
-    bytes.length += 5
+  _encodeBool (bytes, value) {
+    this._msgpack.encodeBoolean(bytes, value)
+  }
 
-    bytes.buffer[offset] = 0xdd
-    bytes.buffer[offset + 1] = length >> 24
-    bytes.buffer[offset + 2] = length >> 16
-    bytes.buffer[offset + 3] = length >> 8
-    bytes.buffer[offset + 4] = length
+  _encodeArrayPrefix (bytes, value) {
+    this._msgpack.encodeArrayPrefix(bytes, value)
   }
 
   _encodeMapPrefix (bytes, keysLength) {
-    const offset = bytes.length
-
-    bytes.reserve(5)
-    bytes.length += 5
-    bytes.buffer[offset] = 0xdf
-    bytes.buffer[offset + 1] = keysLength >> 24
-    bytes.buffer[offset + 2] = keysLength >> 16
-    bytes.buffer[offset + 3] = keysLength >> 8
-    bytes.buffer[offset + 4] = keysLength
+    this._msgpack.encodeMapPrefix(bytes, keysLength)
   }
 
   _encodeByte (bytes, value) {
-    bytes.reserve(1)
-
-    bytes.buffer[bytes.length++] = value
+    this._msgpack.encodeByte(bytes, value)
   }
 
+  // TODO: Use BigInt instead.
   _encodeId (bytes, id) {
     const offset = bytes.length
 
     bytes.reserve(9)
-    bytes.length += 9
 
     id = id.toArray()
 
@@ -186,36 +168,16 @@ class AgentEncoder {
     bytes.buffer[offset + 8] = id[7]
   }
 
-  _encodeInteger (bytes, value) {
-    const offset = bytes.length
-
-    bytes.reserve(5)
-    bytes.length += 5
+  _encodeNumber (bytes, value) {
+    this._msgpack.encodeNumber(bytes, value)
+  }
 
-    bytes.buffer[offset] = 0xce
-    bytes.buffer[offset + 1] = value >> 24
-    bytes.buffer[offset + 2] = value >> 16
-    bytes.buffer[offset + 3] = value >> 8
-    bytes.buffer[offset + 4] = value
+  _encodeInteger (bytes, value) {
+    this._msgpack.encodeInteger(bytes, value)
   }
 
   _encodeLong (bytes, value) {
-    const offset = bytes.length
-    const hi = (value / Math.pow(2, 32)) >> 0
-    const lo = value >>> 0
-
-    bytes.reserve(9)
-    bytes.length += 9
-
-    bytes.buffer[offset] = 0xcf
-    bytes.buffer[offset + 1] = hi >> 24
-    bytes.buffer[offset + 2] = hi >> 16
-    bytes.buffer[offset + 3] = hi >> 8
-    bytes.buffer[offset + 4] = hi
-    bytes.buffer[offset + 5] = lo >> 24
-    bytes.buffer[offset + 6] = lo >> 16
-    bytes.buffer[offset + 7] = lo >> 8
-    bytes.buffer[offset + 8] = lo
+    this._msgpack.encodeLong(bytes, value)
   }
 
   _encodeMap (bytes, value) {
@@ -252,23 +214,7 @@ class AgentEncoder {
   }
 
   _encodeFloat (bytes, value) {
-    float64Array[0] = value
-
-    const offset = bytes.length
-    bytes.reserve(9)
-    bytes.length += 9
-
-    bytes.buffer[offset] = 0xcb
-
-    if (bigEndian) {
-      for (let i = 0; i <= 7; i++) {
-        bytes.buffer[offset + i + 1] = uInt8Float64Array[i]
-      }
-    } else {
-      for (let i = 7; i >= 0; i--) {
-        bytes.buffer[bytes.length - i - 1] = uInt8Float64Array[i]
-      }
-    }
+    this._msgpack.encodeFloat(bytes, value)
   }
 
   _encodeMetaStruct (bytes, value) {
@@ -294,7 +240,6 @@ class AgentEncoder {
     const offset = bytes.length
 
     bytes.reserve(prefixLength)
-    bytes.length += prefixLength
 
     this._encodeObject(bytes, value)
 

diff --git a/packages/dd-trace/src/encode/agentless-ci-visibility.js b/packages/dd-trace/src/encode/agentless-ci-visibility.js
@@ -251,37 +251,6 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     }
   }
 
-  _encodeNumber (bytes, value) {
-    if (Math.floor(value) !== value) { // float 64
-      return this._encodeFloat(bytes, value)
-    }
-    return this._encodeLong(bytes, value)
-  }
-
-  _encodeLong (bytes, value) {
-    const isPositive = value >= 0
-
-    const hi = isPositive ? (value / Math.pow(2, 32)) >> 0 : Math.floor(value / Math.pow(2, 32))
-    const lo = value >>> 0
-    const flag = isPositive ? 0xcf : 0xd3
-
-    const offset = bytes.length
-
-    // int 64
-    bytes.reserve(9)
-    bytes.length += 9
-
-    bytes.buffer[offset] = flag
-    bytes.buffer[offset + 1] = hi >> 24
-    bytes.buffer[offset + 2] = hi >> 16
-    bytes.buffer[offset + 3] = hi >> 8
-    bytes.buffer[offset + 4] = hi
-    bytes.buffer[offset + 5] = lo >> 24
-    bytes.buffer[offset + 6] = lo >> 16
-    bytes.buffer[offset + 7] = lo >> 8
-    bytes.buffer[offset + 8] = lo
-  }
-
   _encode (bytes, trace) {
     if (this._isReset) {
       this._encodePayloadStart(bytes)
@@ -380,7 +349,6 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     // Get offset of the events list to update the length of the array when calling `makePayload`
     this._eventsOffset = bytes.length
     bytes.reserve(5)
-    bytes.length += 5
   }
 
   reset () {

diff --git a/packages/dd-trace/src/encode/coverage-ci-visibility.js b/packages/dd-trace/src/encode/coverage-ci-visibility.js
@@ -1,6 +1,6 @@
 'use strict'
 const { AgentEncoder } = require('./0.4')
-const Chunk = require('./chunk')
+const { Chunk } = require('../msgpack')
 
 const {
   distributionMetric,
@@ -82,7 +82,6 @@ class CoverageCIVisibilityEncoder extends AgentEncoder {
     // Get offset of the coverages list to update the length of the array when calling `makePayload`
     this._coveragesOffset = bytes.length
     bytes.reserve(5)
-    bytes.length += 5
   }
 
   makePayload () {

diff --git a/packages/dd-trace/src/encode/span-stats.js b/packages/dd-trace/src/encode/span-stats.js
@@ -22,10 +22,6 @@ function truncate (value, maxLength, suffix = '') {
 }
 
 class SpanStatsEncoder extends AgentEncoder {
-  _encodeBool (bytes, value) {
-    this._encodeByte(bytes, value ? 0xc3 : 0xc2)
-  }
-
   makePayload () {
     const traceSize = this._traceBytes.length
     const buffer = Buffer.allocUnsafe(traceSize)
@@ -34,32 +30,6 @@ class SpanStatsEncoder extends AgentEncoder {
     return buffer
   }
 
-  _encodeMapPrefix (bytes, length) {
-    const offset = bytes.length
-
-    bytes.reserve(1)
-    bytes.length += 1
-
-    bytes.buffer[offset] = 0x80 + length
-  }
-
-  _encodeBuffer (bytes, buffer) {
-    const length = buffer.length
-    const offset = bytes.length
-
-    bytes.reserve(5)
-    bytes.length += 5
-
-    bytes.buffer[offset] = 0xc6
-    bytes.buffer[offset + 1] = length >> 24
-    bytes.buffer[offset + 2] = length >> 16
-    bytes.buffer[offset + 3] = length >> 8
-    bytes.buffer[offset + 4] = length
-
-    buffer.copy(bytes.buffer, offset + 5)
-    bytes.length += length
-  }
-
   _encodeStat (bytes, stat) {
     this._encodeMapPrefix(bytes, 12)