V5.14.0 proposal

.github/workflows/profiling.yml

@@ -34,7 +34,7 @@ jobs:
       - uses: ./.github/actions/node/20
       - run: yarn test:profiler:ci
       - run: yarn test:integration:profiler
-      - uses: ./.github/actions/node/21
+      - uses: ./.github/actions/node/latest
       - run: yarn test:profiler:ci
       - run: yarn test:integration:profiler
       - uses: codecov/codecov-action@v3

.github/workflows/tracing.yml

@@ -31,7 +31,7 @@ jobs:
       - run: yarn test:trace:core:ci
       - uses: ./.github/actions/node/20
       - run: yarn test:trace:core:ci
-      - uses: ./.github/actions/node/21
+      - uses: ./.github/actions/node/latest
       - run: yarn test:trace:core:ci
       - uses: codecov/codecov-action@v3
 

README.md

@@ -108,4 +108,11 @@ If you would like to trace your bundled application then please read this page o
 
 ## Security Vulnerabilities
 
-Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue.
+Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue.
+
+## Datadog With OpenTelemetery
+
+Please refer to the [Node.js Custom Instrumentation using OpenTelemetry API](https://docs.datadoghq.com/tracing/trace_collection/custom_instrumentation/nodejs/otel/) document. It includes information on how to use the OpenTelemetry API with dd-trace-js
+
+Note that our internal implementation of the OpenTelemetry API is currently set within the version range `>=1.0.0 <1.9.0`. This range will be updated at a regular cadence therefore, we recommend updating your tracer to the latest release to ensure up to date support.
+

docs/test.ts

@@ -260,6 +260,7 @@ tracer.use('aws-sdk', awsSdkOptions);
 tracer.use('bunyan');
 tracer.use('couchbase');
 tracer.use('cassandra-driver');
+tracer.use('child_process');
 tracer.use('connect');
 tracer.use('connect', httpServerOptions);
 tracer.use('cypress');

index.d.ts

@@ -144,6 +144,7 @@ interface Plugins {
   "aws-sdk": tracer.plugins.aws_sdk;
   "bunyan": tracer.plugins.bunyan;
   "cassandra-driver": tracer.plugins.cassandra_driver;
+  "child_process": tracer.plugins.child_process;
   "connect": tracer.plugins.connect;
   "couchbase": tracer.plugins.couchbase;
   "cucumber": tracer.plugins.cucumber;
@@ -1207,6 +1208,12 @@ declare namespace tracer {
      */
     interface cassandra_driver extends Instrumentation {}
 
+    /**
+     * This plugin automatically instruments the
+     * [child_process](https://nodejs.org/api/child_process.html) module.
+     */
+    interface child_process extends Instrumentation {}
+
     /**
      * This plugin automatically instruments the
      * [connect](https://github.com/senchalabs/connect) module.

init.js

@@ -1,7 +1,31 @@
 'use strict'
 
-const tracer = require('.')
+const path = require('path')
+const Module = require('module')
 
-tracer.init()
+let initBailout = false
 
-module.exports = tracer
+if (process.env.DD_INJECTION_ENABLED) {
+  // If we're running via single-step install, and we're not in the app's
+  // node_modules, then we should not initialize the tracer. This prevents
+  // single-step-installed tracer from clobbering the manually-installed tracer.
+  let resolvedInApp
+  const entrypoint = process.argv[1]
+  try {
+    resolvedInApp = Module.createRequire(entrypoint).resolve('dd-trace')
+  } catch (e) {
+    // Ignore. If we can't resolve the module, we assume it's not in the app.
+  }
+  if (resolvedInApp) {
+    const ourselves = path.join(__dirname, 'index.js')
+    if (ourselves !== resolvedInApp) {
+      initBailout = true
+    }
+  }
+}
+
+if (!initBailout) {
+  const tracer = require('.')
+  tracer.init()
+  module.exports = tracer
+}

integration-tests/init.spec.js

@@ -0,0 +1,57 @@
+const {
+  createSandbox,
+  spawnProc
+} = require('./helpers')
+const { assert } = require('chai')
+const path = require('path')
+
+const DD_INJECTION_ENABLED = 'tracing'
+
+describe('init.js', () => {
+  let cwd, proc, sandbox
+
+  async function runTest (cwd, env, expected) {
+    return new Promise((resolve, reject) => {
+      spawnProc(path.join(cwd, 'init/index.js'), { cwd, env }, data => {
+        try {
+          assert.strictEqual(data.toString(), expected)
+          resolve()
+        } catch (e) {
+          reject(e)
+        }
+      }).then(subproc => {
+        proc = subproc
+      })
+    })
+  }
+
+  before(async () => {
+    sandbox = await createSandbox()
+    cwd = sandbox.folder
+  })
+  afterEach(() => {
+    proc && proc.kill()
+  })
+  after(() => {
+    return sandbox.remove()
+  })
+
+  context('when dd-trace is not in the app dir', () => {
+    const NODE_OPTIONS = `--require ${path.join(__dirname, '..', 'init.js')}`
+    it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS }, 'true\n')
+    })
+    it('should not initialize the tracer, if DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n')
+    })
+  })
+  context('when dd-trace in the app dir', () => {
+    const NODE_OPTIONS = '--require dd-trace/init.js'
+    it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS }, 'true\n')
+    })
+    it('should initialize the tracer, if DD_INJECTION_ENABLED', () => {
+      return runTest(cwd, { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'true\n')
+    })
+  })
+})

integration-tests/init/index.js

@@ -0,0 +1,3 @@
+// eslint-disable-next-line no-console
+console.log(!!global._ddtrace)
+process.exit()

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.13.0",
+  "version": "5.14.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -74,9 +74,9 @@
     "@datadog/native-iast-rewriter": "2.3.1",
     "@datadog/native-iast-taint-tracking": "2.1.0",
     "@datadog/native-metrics": "^2.0.0",
-    "@datadog/pprof": "5.2.0",
+    "@datadog/pprof": "5.3.0",
     "@datadog/sketches-js": "^2.1.0",
-    "@opentelemetry/api": "^1.0.0",
+    "@opentelemetry/api": ">=1.0.0 <1.9.0",
     "@opentelemetry/core": "^1.14.0",
     "crypto-randomuuid": "^1.0.0",
     "dc-polyfill": "^0.1.4",

packages/dd-trace/src/appsec/iast/iast-log.js

@@ -2,35 +2,9 @@
 
 const dc = require('dc-polyfill')
 const log = require('../../log')
-const { calculateDDBasePath } = require('../../util')
 
 const telemetryLog = dc.channel('datadog:telemetry:log')
 
-const ddBasePath = calculateDDBasePath(__dirname)
-const EOL = '\n'
-const STACK_FRAME_LINE_REGEX = /^\s*at\s/gm
-
-function sanitize (logEntry, stack) {
-  if (!stack) return logEntry
-
-  let stackLines = stack.split(EOL)
-
-  const firstIndex = stackLines.findIndex(l => l.match(STACK_FRAME_LINE_REGEX))
-
-  const isDDCode = firstIndex > -1 && stackLines[firstIndex].includes(ddBasePath)
-  stackLines = stackLines
-    .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath))
-    .map(line => line.replace(ddBasePath, ''))
-
-  logEntry.stack_trace = stackLines.join(EOL)
-
-  if (!isDDCode) {
-    logEntry.message = 'omitted'
-  }
-
-  return logEntry
-}
-
 function getTelemetryLog (data, level) {
   try {
     data = typeof data === 'function' ? data() : data
@@ -42,18 +16,13 @@ function getTelemetryLog (data, level) {
       message = String(data.message || data)
     }
 
-    let logEntry = {
+    const logEntry = {
       message,
       level
     }
-
     if (data.stack) {
-      logEntry = sanitize(logEntry, data.stack)
-      if (logEntry.stack_trace === '') {
-        return
-      }
+      logEntry.stack_trace = data.stack
     }
-
     return logEntry
   } catch (e) {
     log.error(e)

packages/dd-trace/src/config.js

@@ -16,7 +16,7 @@ const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('./plugins/util/tags')
 const { getGitMetadataFromGitProperties, removeUserSensitiveInfo } = require('./git_properties')
 const { updateConfig } = require('./telemetry')
 const telemetryMetrics = require('./telemetry/metrics')
-const { getIsGCPFunction, getIsAzureFunctionConsumptionPlan } = require('./serverless')
+const { getIsGCPFunction, getIsAzureFunction } = require('./serverless')
 const { ORIGIN_KEY } = require('./constants')
 
 const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
@@ -339,7 +339,7 @@ class Config {
 
     // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent
     this.memcachedCommandEnabled = isTrue(DD_TRACE_MEMCACHED_COMMAND_ENABLED)
-    this.isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan()
+    this.isAzureFunction = getIsAzureFunction()
     this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG)
     this.installSignature = {
       id: DD_INSTRUMENTATION_INSTALL_ID,
@@ -417,8 +417,8 @@ class Config {
   _isInServerlessEnvironment () {
     const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
     const isGCPFunction = getIsGCPFunction()
-    const isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan()
-    return inAWSLambda || isGCPFunction || isAzureFunctionConsumptionPlan
+    const isAzureFunction = getIsAzureFunction()
+    return inAWSLambda || isGCPFunction || isAzureFunction
   }
 
   // for _merge to work, every config value must have a default value
@@ -549,6 +549,7 @@ class Config {
       DD_IAST_REDACTION_VALUE_PATTERN,
       DD_IAST_REQUEST_SAMPLING,
       DD_IAST_TELEMETRY_VERBOSITY,
+      DD_INJECTION_ENABLED,
       DD_INSTRUMENTATION_TELEMETRY_ENABLED,
       DD_INSTRUMENTATION_CONFIG_ID,
       DD_LOGS_INJECTION,
@@ -705,7 +706,14 @@ class Config {
     this._setBoolean(env, 'telemetry.debug', DD_TELEMETRY_DEBUG)
     this._setBoolean(env, 'telemetry.dependencyCollection', DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED)
     this._setValue(env, 'telemetry.heartbeatInterval', maybeInt(Math.floor(DD_TELEMETRY_HEARTBEAT_INTERVAL * 1000)))
-    this._setBoolean(env, 'telemetry.logCollection', coalesce(DD_TELEMETRY_LOG_COLLECTION_ENABLED, DD_IAST_ENABLED))
+    const hasTelemetryLogsUsingFeatures =
+      isTrue(DD_IAST_ENABLED) ||
+      isTrue(DD_PROFILING_ENABLED) ||
+      (typeof DD_INJECTION_ENABLED === 'string' && DD_INJECTION_ENABLED.split(',').includes('profiling'))
+        ? true
+        : undefined
+    this._setBoolean(env, 'telemetry.logCollection', coalesce(DD_TELEMETRY_LOG_COLLECTION_ENABLED,
+      hasTelemetryLogsUsingFeatures))
     this._setBoolean(env, 'telemetry.metrics', DD_TELEMETRY_METRICS_ENABLED)
     this._setBoolean(env, 'traceId128BitGenerationEnabled', DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)
     this._setBoolean(env, 'traceId128BitLoggingEnabled', DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED)
@@ -786,8 +794,10 @@ class Config {
     this._setBoolean(opts, 'spanRemoveIntegrationFromService', options.spanRemoveIntegrationFromService)
     this._setBoolean(opts, 'startupLogs', options.startupLogs)
     this._setTags(opts, 'tags', tags)
-    this._setBoolean(opts, 'telemetry.logCollection', options.iastOptions &&
-    (options.iastOptions === true || options.iastOptions.enabled === true))
+    const hasTelemetryLogsUsingFeatures =
+      (options.iastOptions && (options.iastOptions === true || options.iastOptions?.enabled === true)) ||
+      (options.profiling && options.profiling === true)
+    this._setBoolean(opts, 'telemetry.logCollection', hasTelemetryLogsUsingFeatures)
     this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled)
     this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled)
     this._setString(opts, 'version', options.version || tags.version)
@@ -867,7 +877,7 @@ class Config {
     return coalesce(
       this.options.stats,
       process.env.DD_TRACE_STATS_COMPUTATION_ENABLED,
-      getIsGCPFunction() || getIsAzureFunctionConsumptionPlan()
+      getIsGCPFunction() || getIsAzureFunction()
     )
   }
 

packages/dd-trace/src/encode/0.4.js

@@ -83,13 +83,17 @@ class AgentEncoder {
       span = formatSpan(span)
       bytes.reserve(1)
 
-      if (span.type) {
+      if (span.type && span.meta_struct) {
+        bytes.buffer[bytes.length++] = 0x8d
+      } else if (span.type || span.meta_struct) {
         bytes.buffer[bytes.length++] = 0x8c
+      } else {
+        bytes.buffer[bytes.length++] = 0x8b
+      }
 
+      if (span.type) {
         this._encodeString(bytes, 'type')
         this._encodeString(bytes, span.type)
-      } else {
-        bytes.buffer[bytes.length++] = 0x8b
       }
 
       this._encodeString(bytes, 'trace_id')
@@ -114,6 +118,10 @@ class AgentEncoder {
       this._encodeMap(bytes, span.meta)
       this._encodeString(bytes, 'metrics')
       this._encodeMap(bytes, span.metrics)
+      if (span.meta_struct) {
+        this._encodeString(bytes, 'meta_struct')
+        this._encodeObject(bytes, span.meta_struct)
+      }
     }
   }
 
@@ -263,6 +271,45 @@ class AgentEncoder {
     }
   }
 
+  _encodeObject (bytes, value, circularReferencesDetector = new Set()) {
+    circularReferencesDetector.add(value)
+    if (Array.isArray(value)) {
+      return this._encodeObjectAsArray(bytes, value, circularReferencesDetector)
+    } else if (value !== null && typeof value === 'object') {
+      return this._encodeObjectAsMap(bytes, value, circularReferencesDetector)
+    } else if (typeof value === 'string' || typeof value === 'number') {
+      this._encodeValue(bytes, value)
+    }
+  }
+
+  _encodeObjectAsMap (bytes, value, circularReferencesDetector) {
+    const keys = Object.keys(value)
+    const validKeys = keys.filter(key =>
+      typeof value[key] === 'string' ||
+      typeof value[key] === 'number' ||
+      (value[key] !== null && typeof value[key] === 'object' && !circularReferencesDetector.has(value[key])))
+
+    this._encodeMapPrefix(bytes, validKeys.length)
+
+    for (const key of validKeys) {
+      this._encodeString(bytes, key)
+      this._encodeObject(bytes, value[key], circularReferencesDetector)
+    }
+  }
+
+  _encodeObjectAsArray (bytes, value, circularReferencesDetector) {
+    const validValue = value.filter(item =>
+      typeof item === 'string' ||
+      typeof item === 'number' ||
+      (item !== null && typeof item === 'object' && !circularReferencesDetector.has(item)))
+
+    this._encodeArrayPrefix(bytes, validValue)
+
+    for (const item of validValue) {
+      this._encodeObject(bytes, item, circularReferencesDetector)
+    }
+  }
+
   _cacheString (value) {
     if (!(value in this._stringMap)) {
       this._stringCount++