[ci-visibility] Adapt early flake detection to new format (#4137)

DataDog · Mar 7, 2024 · ff980c6 · ff980c6
1 parent 38745b9
commit ff980c6
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 44 deletions.
diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js
@@ -646,9 +646,11 @@ testFrameworks.forEach(({
             receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
           }
           // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new
-          receiver.setKnownTests([
-            `${name}.ci-visibility/test/ci-visibility-test.js.ci visibility can report tests`
-          ])
+          receiver.setKnownTests({
+            [name]: {
+              'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests']
+            }
+          })
           const NUM_RETRIES_EFD = 3
           receiver.setSettings({
             itr_enabled: false,
@@ -728,9 +730,11 @@ testFrameworks.forEach(({
             receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
           }
           // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new
-          receiver.setKnownTests([
-            `${name}.ci-visibility/test-early-flake-detection/test.js.ci visibility can report tests`
-          ])
+          receiver.setKnownTests({
+            [name]: {
+              'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests']
+            }
+          })
           receiver.setSettings({
             itr_enabled: false,
             code_coverage: false,
@@ -811,9 +815,11 @@ testFrameworks.forEach(({
             receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
           }
           // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new
-          receiver.setKnownTests([
-            `${name}.ci-visibility/test/ci-visibility-test.js.ci visibility can report tests`
-          ])
+          receiver.setKnownTests({
+            [name]: {
+              'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests']
+            }
+          })
           receiver.setSettings({
             itr_enabled: false,
             code_coverage: false,
@@ -873,7 +879,7 @@ testFrameworks.forEach(({
             receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
           }
           // Tests from ci-visibility/test/occasionally-failing-test will be considered new
-          receiver.setKnownTests([])
+          receiver.setKnownTests({})
 
           const NUM_RETRIES_EFD = 5
           receiver.setSettings({
@@ -945,7 +951,7 @@ testFrameworks.forEach(({
             receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
           }
           // Tests from ci-visibility/test/skipped-and-todo-test will be considered new
-          receiver.setKnownTests([])
+          receiver.setKnownTests({})
 
           const NUM_RETRIES_EFD = 5
           receiver.setSettings({

diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js
@@ -796,13 +796,14 @@ versions.forEach(version => {
                 }
               })
               // "cucumber.ci-visibility/features/farewell.feature.Say" whatever will be considered new
-              receiver.setKnownTests([
-                'cucumber.ci-visibility/features/farewell.feature.Say farewell',
-                'cucumber.ci-visibility/features/greetings.feature.Say greetings',
-                'cucumber.ci-visibility/features/greetings.feature.Say yeah',
-                'cucumber.ci-visibility/features/greetings.feature.Say yo',
-                'cucumber.ci-visibility/features/greetings.feature.Say skip'
-              ])
+              receiver.setKnownTests(
+                {
+                  cucumber: {
+                    'ci-visibility/features/farewell.feature': ['Say farewell'],
+                    'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip']
+                  }
+                }
+              )
               const eventsPromise = receiver
                 .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
                   const events = payloads.flatMap(({ payload }) => payload.events)
@@ -871,13 +872,12 @@ versions.forEach(version => {
                   assert.equal(newTests.length, 0)
                 })
               // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new
-              receiver.setKnownTests([
-                'cucumber.ci-visibility/features/farewell.feature.Say farewell',
-                'cucumber.ci-visibility/features/greetings.feature.Say greetings',
-                'cucumber.ci-visibility/features/greetings.feature.Say yeah',
-                'cucumber.ci-visibility/features/greetings.feature.Say yo',
-                'cucumber.ci-visibility/features/greetings.feature.Say skip'
-              ])
+              receiver.setKnownTests({
+                cucumber: {
+                  'ci-visibility/features/farewell.feature': ['Say farewell'],
+                  'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip']
+                }
+              })
 
               childProcess = exec(
                 runTestsCommand,
@@ -907,7 +907,7 @@ versions.forEach(version => {
                 }
               })
               // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new
-              receiver.setKnownTests([])
+              receiver.setKnownTests({})
 
               const eventsPromise = receiver
                 .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
@@ -959,12 +959,12 @@ versions.forEach(version => {
               })
               // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new
               // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new
-              receiver.setKnownTests([
-                'cucumber.ci-visibility/features/farewell.feature.Say farewell',
-                'cucumber.ci-visibility/features/greetings.feature.Say greetings',
-                'cucumber.ci-visibility/features/greetings.feature.Say yeah',
-                'cucumber.ci-visibility/features/greetings.feature.Say yo'
-              ])
+              receiver.setKnownTests({
+                cucumber: {
+                  'ci-visibility/features/farewell.feature': ['Say farewell'],
+                  'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo']
+                }
+              })
 
               const eventsPromise = receiver
                 .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {

diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js
@@ -900,9 +900,6 @@ moduleType.forEach(({
         }
       )
 
-      childProcess.stdout.pipe(process.stdout)
-      childProcess.stderr.pipe(process.stderr)
-
       childProcess.on('exit', () => {
         receiverPromise.then(() => {
           done()

diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js
@@ -92,7 +92,8 @@ function getStatusFromResultLatest (result) {
 }
 
 function isNewTest (testSuite, testName) {
-  return !knownTests.includes(`cucumber.${testSuite}.${testName}`)
+  const testsForSuite = knownTests.cucumber?.[testSuite] || []
+  return !testsForSuite.includes(testName)
 }
 
 function getTestStatusFromRetries (testStatuses) {

diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js
@@ -140,14 +140,16 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
     // Function that receives a list of known tests for a test service and
     // returns the ones that belong to the current suite
     getKnownTestsForSuite (knownTests) {
+      if (this.knownTestsForThisSuite) {
+        return this.knownTestsForThisSuite
+      }
       let knownTestsForSuite = knownTests
-      // If jest runs in band, the known tests are not serialized, so they're an array.
-      if (!Array.isArray(knownTests)) {
+      // If jest is using workers, known tests are serialized to json.
+      // If jest runs in band, they are not.
+      if (typeof knownTestsForSuite === 'string') {
         knownTestsForSuite = JSON.parse(knownTestsForSuite)
       }
-      return knownTestsForSuite
-        .filter(test => test.includes(this.testSuite))
-        .map(test => test.replace(`jest.${this.testSuite}.`, '').trim())
+      return knownTestsForSuite.jest?.[this.testSuite] || []
     }
 
     // Add the `add_test` event we don't have the test object yet, so

diff --git a/packages/datadog-instrumentations/src/mocha.js b/packages/datadog-instrumentations/src/mocha.js
@@ -106,7 +106,10 @@ function getTestFullName (test) {
 }
 
 function isNewTest (test) {
-  return !knownTests.includes(getTestFullName(test))
+  const testSuite = getTestSuitePath(test.file, process.cwd())
+  const testName = removeEfdStringFromTestName(test.fullTitle())
+  const testsForSuite = knownTests.mocha?.[testSuite] || []
+  return !testsForSuite.includes(testName)
 }
 
 function retryTest (test) {

diff --git a/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js b/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js
@@ -1,6 +1,5 @@
 const request = require('../../exporters/common/request')
 const id = require('../../id')
-const log = require('../../log')
 
 function getKnownTests ({
   url,
@@ -71,7 +70,6 @@ function getKnownTests ({
     } else {
       try {
         const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
-        log.debug(() => `Number of received known tests: ${Object.keys(knownTests).length}`)
         done(null, knownTests)
       } catch (err) {
         done(err)