V5.1.0 proposal

.github/workflows/plugins.yml

@@ -20,7 +20,7 @@ env:
 
 
 jobs:
-  aerospike-3:
+  aerospike-node-14:
     runs-on: ubuntu-latest
     container:
       image: ubuntu:18.04
@@ -74,7 +74,7 @@ jobs:
       - if: always()
         uses: codecov/codecov-action@v3
 
-  aerospike-4:
+  aerospike-node-16:
     runs-on: ubuntu-latest
     services:
       aerospike:
@@ -84,7 +84,7 @@ jobs:
     env:
       PLUGINS: aerospike
       SERVICES: aerospike
-      PACKAGE_VERSION_RANGE: '4.0.0 - 5.4.0'
+      PACKAGE_VERSION_RANGE: '>=4.0.0 <5.2.0'
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/testagent/start
@@ -101,15 +101,15 @@ jobs:
           echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV
       - uses: ./.github/actions/node/oldest
       - name: Install dependencies and run tests 
-        if: env.MAJOR_VERSION != '5'
+        if: env.MAJOR_VERSION == '4'
         run: |
           yarn install --ignore-engines
           yarn test:plugins:ci
       - if: always()
         uses: ./.github/actions/testagent/logs
       - uses: codecov/codecov-action@v3
 
-  aerospike-5:
+  aerospike-node-18-20:
     strategy:
       matrix:
         node-version: [18]
@@ -145,7 +145,7 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - name: Install dependencies and run tests 
-        if: env.MAJOR_VERSION == '5'
+        if: env.MAJOR_VERSION >= '5'
         run: |
           yarn install --ignore-engines
           yarn test:plugins:ci

.github/workflows/profiling.yml

@@ -19,6 +19,7 @@ jobs:
       - uses: ./.github/actions/node/setup
       - run: yarn install
       - run: yarn test:profiler:ci
+      - run: yarn test:integration:profiler
       - uses: codecov/codecov-action@v3
 
   ubuntu:
@@ -29,10 +30,13 @@ jobs:
       - run: yarn install
       - uses: ./.github/actions/node/18
       - run: yarn test:profiler:ci
+      - run: yarn test:integration:profiler
       - uses: ./.github/actions/node/20
       - run: yarn test:profiler:ci
+      - run: yarn test:integration:profiler
       - uses: ./.github/actions/node/latest
       - run: yarn test:profiler:ci
+      - run: yarn test:integration:profiler
       - uses: codecov/codecov-action@v3
 
   windows:
@@ -42,4 +46,5 @@ jobs:
       - uses: ./.github/actions/node/setup
       - run: yarn install
       - run: yarn test:profiler:ci
+      - run: yarn test:integration:profiler
       - uses: codecov/codecov-action@v3

MIGRATING.md

@@ -4,6 +4,21 @@ This guide describes the steps to upgrade dd-trace from a major version to the
 next. If you are having any issues related to migrating, please feel free to
 open an issue or contact our [support](https://www.datadoghq.com/support/) team.
 
+## 4.0 to 5.0
+
+### Node 16 is no longer supported
+
+Node.js 16 has reached EOL in September 2023 and is no longer supported. Generally
+speaking, we highly recommend always keeping Node.js up to date regardless of
+our support policy.
+
+### Update `trace<T>` TypeScript declaration
+
+The TypeScript declaration for `trace<T>` has been updated to enforce 
+that calls to `tracer.trace(name, fn)` must receive a function which takes at least 
+the span object. Previously the span was technically optional when it should not have 
+been as the span must be handled.
+
 ## 3.0 to 4.0
 
 ### Node 14 is no longer supported

README.md

@@ -1,7 +1,8 @@
 # `dd-trace`: Node.js APM Tracer Library
 
-[![npm v4](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v4&logo=npm)](https://www.npmjs.com/package/dd-trace)
-[![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=blue&label=dd-trace%40v3&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node12)
+[![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v5&logo=npm)](https://www.npmjs.com/package/dd-trace)
+[![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=blue&label=dd-trace%40v4&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node16)
+[![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=blue&label=dd-trace%40v3&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node14)
 [![codecov](https://codecov.io/gh/DataDog/dd-trace-js/branch/master/graph/badge.svg)](https://codecov.io/gh/DataDog/dd-trace-js)
 
 <img align="right" src="https://user-images.githubusercontent.com/551402/208212084-1d0c07e2-4135-4c61-b2da-8f2fddbc66ed.png" alt="Bits the dog  JavaScript" width="200px"/>
@@ -28,24 +29,25 @@ Most of the documentation for `dd-trace` is available on these webpages:
 | [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square)     | `>= v12` | **End of Life** | 2021-07-13     | 2022-02-25  |
 | [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | **End of Life** | 2022-01-28     | 2023-08-15  |
 | [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | **Maintenance** | 2022-08-15     | 2024-05-15  |
-| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square)        | `>= v16` | **Current**     | 2023-05-12     | Unknown     |
+| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=white&label=%20&style=flat-square)        | `>= v16` | **Maintenance**     | 2023-05-12     | 2025-01-11     |
+| [`v5`](https://github.com/DataDog/dd-trace-js/tree/v5.x) | ![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square)        | `>= v18` | **Current**     | 2024-01-11     | Unknown     |
 
-We currently maintain two release lines, namely `v3` and `v4`.
-Features and bug fixes that are merged are released to the `v4` line and, if appropriate, also the `v3` line.
+We currently maintain three release lines, namely `v5`, `v4` and `v3`.
+Features and bug fixes that are merged are released to the `v5` line and, if appropriate, also the `v4` & `v3` line.
 
-For any new projects it is recommended to use the `v4` release line:
+For any new projects it is recommended to use the `v5` release line:
 
 ```sh
 $ npm install dd-trace
 $ yarn add dd-trace
 ```
 
-However, existing projects that already use the `v3` release line, or projects that need to support EOL versions of Node.js, may continue to use these release lines.
+However, existing projects that already use the `v4` & `v3` release line, or projects that need to support EOL versions of Node.js, may continue to use these release lines.
 This is done by specifying the version when installing the package.
 
 ```sh
-$ npm install dd-trace@3
-$ yarn add dd-trace@3
+$ npm install dd-trace@4
+$ yarn add dd-trace@4
 ```
 
 Any backwards-breaking functionality that is introduced into the library will result in an increase of the major version of the library and therefore a new release line.

integration-tests/ci-visibility-intake.js

@@ -19,11 +19,13 @@ const DEFAULT_GIT_UPLOAD_STATUS = 200
 const DEFAULT_INFO_RESPONSE = {
   endpoints: ['/evp_proxy/v2']
 }
+const DEFAULT_CORRELATION_ID = '1234'
 
 let settings = DEFAULT_SETTINGS
 let suitesToSkip = DEFAULT_SUITES_TO_SKIP
 let gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS
 let infoResponse = DEFAULT_INFO_RESPONSE
+let correlationId = DEFAULT_CORRELATION_ID
 
 class FakeCiVisIntake extends FakeAgent {
   setInfoResponse (newInfoResponse) {
@@ -38,6 +40,10 @@ class FakeCiVisIntake extends FakeAgent {
     suitesToSkip = newSuitesToSkip
   }
 
+  setItrCorrelationId (newCorrelationId) {
+    correlationId = newCorrelationId
+  }
+
   setSettings (newSettings) {
     settings = newSettings
   }
@@ -140,7 +146,10 @@ class FakeCiVisIntake extends FakeAgent {
       '/evp_proxy/v2/api/v2/ci/tests/skippable'
     ], (req, res) => {
       res.status(200).send(JSON.stringify({
-        data: suitesToSkip
+        data: suitesToSkip,
+        meta: {
+          correlation_id: correlationId
+        }
       }))
       this.emit('message', {
         headers: req.headers,

integration-tests/ci-visibility.spec.js

@@ -1151,6 +1151,31 @@ testFrameworks.forEach(({
           }).catch(done)
         })
       })
+      it('reports itr_correlation_id in test suites', (done) => {
+        const itrCorrelationId = '4321'
+        receiver.setItrCorrelationId(itrCorrelationId)
+        const eventsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)
+            testSuites.forEach(testSuite => {
+              assert.equal(testSuite.itr_correlation_id, itrCorrelationId)
+            })
+          }, 25000)
+        childProcess = exec(
+          runTestsWithCoverageCommand,
+          {
+            cwd,
+            env: getCiVisAgentlessConfig(receiver.port),
+            stdio: 'inherit'
+          }
+        )
+        childProcess.on('exit', () => {
+          eventsPromise.then(() => {
+            done()
+          }).catch(done)
+        })
+      })
     })
 
     describe('evp proxy', () => {
@@ -1602,6 +1627,31 @@ testFrameworks.forEach(({
           }).catch(done)
         })
       })
+      it('reports itr_correlation_id in test suites', (done) => {
+        const itrCorrelationId = '4321'
+        receiver.setItrCorrelationId(itrCorrelationId)
+        const eventsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)
+            testSuites.forEach(testSuite => {
+              assert.equal(testSuite.itr_correlation_id, itrCorrelationId)
+            })
+          }, 25000)
+        childProcess = exec(
+          runTestsWithCoverageCommand,
+          {
+            cwd,
+            env: getCiVisEvpProxyConfig(receiver.port),
+            stdio: 'inherit'
+          }
+        )
+        childProcess.on('exit', () => {
+          eventsPromise.then(() => {
+            done()
+          }).catch(done)
+        })
+      })
     })
   })
 })

integration-tests/cucumber/cucumber.spec.js

@@ -766,6 +766,32 @@ versions.forEach(version => {
                 })
               })
             }
+            it('reports itr_correlation_id in test suites', (done) => {
+              const itrCorrelationId = '4321'
+              receiver.setItrCorrelationId(itrCorrelationId)
+              const eventsPromise = receiver
+                .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+                  const events = payloads.flatMap(({ payload }) => payload.events)
+                  const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)
+                  testSuites.forEach(testSuite => {
+                    assert.equal(testSuite.itr_correlation_id, itrCorrelationId)
+                  })
+                }, 25000)
+
+              childProcess = exec(
+                runTestsWithCoverageCommand,
+                {
+                  cwd,
+                  env: envVars,
+                  stdio: 'inherit'
+                }
+              )
+              childProcess.on('exit', () => {
+                eventsPromise.then(() => {
+                  done()
+                }).catch(done)
+              })
+            })
           })
         })
       })

integration-tests/cypress/cypress.spec.js

@@ -738,6 +738,40 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+      it('reports itr_correlation_id in tests', (done) => {
+        const itrCorrelationId = '4321'
+        receiver.setItrCorrelationId(itrCorrelationId)
+        const eventsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            tests.forEach(test => {
+              assert.equal(test.itr_correlation_id, itrCorrelationId)
+            })
+          }, 25000)
+
+        const {
+          NODE_OPTIONS,
+          ...restEnvVars
+        } = getCiVisAgentlessConfig(receiver.port)
+
+        childProcess = exec(
+          testCommand,
+          {
+            cwd,
+            env: {
+              ...restEnvVars,
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+            },
+            stdio: 'pipe'
+          }
+        )
+        childProcess.on('exit', () => {
+          eventsPromise.then(() => {
+            done()
+          }).catch(done)
+        })
+      })
     })
   })
 })

integration-tests/helpers.js

@@ -10,8 +10,7 @@ const childProcess = require('child_process')
 const { fork } = childProcess
 const exec = promisify(childProcess.exec)
 const http = require('http')
-const fs = require('fs')
-const mkdir = promisify(fs.mkdir)
+const fs = require('fs/promises')
 const os = require('os')
 const path = require('path')
 const rimraf = promisify(require('rimraf'))
@@ -87,7 +86,8 @@ class FakeAgent extends EventEmitter {
     const errors = []
 
     const timeoutObj = setTimeout(() => {
-      resultReject([...errors, new Error('timeout')])
+      const errorsMsg = errors.length === 0 ? '' : `, additionally:\n${errors.map(e => e.stack).join('\n')}\n===\n`
+      resultReject(new Error(`timeout${errorsMsg}`, { cause: { errors } }))
     }, timeout)
 
     const resultPromise = new Promise((resolve, reject) => {
@@ -126,7 +126,8 @@ class FakeAgent extends EventEmitter {
     const errors = []
 
     const timeoutObj = setTimeout(() => {
-      resultReject([...errors, new Error('timeout')])
+      const errorsMsg = errors.length === 0 ? '' : `, additionally:\n${errors.map(e => e.stack).join('\n')}\n===\n`
+      resultReject(new Error(`timeout${errorsMsg}`, { cause: { errors } }))
     }, timeout)
 
     const resultPromise = new Promise((resolve, reject) => {
@@ -204,12 +205,21 @@ async function createSandbox (dependencies = [], isGitRepo = false,
   // We might use NODE_OPTIONS to init the tracer. We don't want this to affect this operations
   const { NODE_OPTIONS, ...restOfEnv } = process.env
 
-  await mkdir(folder)
+  await fs.mkdir(folder)
   await exec(`yarn pack --filename ${out}`) // TODO: cache this
   await exec(`yarn add ${allDependencies.join(' ')}`, { cwd: folder, env: restOfEnv })
 
   for (const path of integrationTestsPaths) {
-    await exec(`cp -R ${path} ${folder}`)
+    if (process.platform === 'win32') {
+      await exec(`Copy-Item -Recurse -Path "${path}" -Destination "${folder}"`, { shell: 'powershell.exe' })
+    } else {
+      await exec(`cp -R ${path} ${folder}`)
+    }
+  }
+  if (process.platform === 'win32') {
+    // On Windows, we can only sync entire filesystem volume caches.
+    await exec(`Write-VolumeCache ${folder[0]}`, { shell: 'powershell.exe' })
+  } else {
     await exec(`sync ${folder}`)
   }
 
@@ -219,7 +229,7 @@ async function createSandbox (dependencies = [], isGitRepo = false,
 
   if (isGitRepo) {
     await exec('git init', { cwd: folder })
-    await exec('echo "node_modules/" > .gitignore', { cwd: folder })
+    await fs.writeFile(path.join(folder, '.gitignore'), 'node_modules/', { flush: true })
     await exec('git config user.email "john@doe.com"', { cwd: folder })
     await exec('git config user.name "John Doe"', { cwd: folder })
     await exec('git config commit.gpgsign false', { cwd: folder })