[test visibility] Add test source file to test suites (#4677)

DataDog · Oct 1, 2024 · 97ea7df · 97ea7df
1 parent d3f1235
commit 97ea7df
Show file tree

Hide file tree

Showing 16 changed files with 148 additions and 20 deletions.
diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js
@@ -27,6 +27,7 @@ const {
   TEST_ITR_FORCED_RUN,
   TEST_ITR_UNSKIPPABLE,
   TEST_SOURCE_FILE,
+  TEST_SOURCE_START,
   TEST_EARLY_FLAKE_ENABLED,
   TEST_IS_NEW,
   TEST_IS_RETRY,
@@ -161,6 +162,7 @@ versions.forEach(version => {
                   testSuiteEvents.forEach(({
                     content: {
                       meta,
+                      metrics,
                       test_suite_id: testSuiteId,
                       test_module_id: testModuleId,
                       test_session_id: testSessionId
@@ -172,6 +174,8 @@ versions.forEach(version => {
                     assert.exists(testSuiteId)
                     assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
                     assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
+                    assert.isTrue(meta[TEST_SOURCE_FILE].startsWith(featuresPath))
+                    assert.equal(metrics[TEST_SOURCE_START], 1)
                   })
 
                   assert.includeMembers(testEvents.map(test => test.content.resource), [
@@ -1193,9 +1197,11 @@ versions.forEach(version => {
             const events = payloads.flatMap(({ payload }) => payload.events)
 
             const test = events.find(event => event.type === 'test').content
+            const testSuite = events.find(event => event.type === 'test_suite_end').content
             // The test is in a subproject
             assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE])
             assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+            assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
           })
 
         childProcess = exec(

diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js
@@ -28,6 +28,7 @@ const {
   TEST_ITR_UNSKIPPABLE,
   TEST_ITR_FORCED_RUN,
   TEST_SOURCE_FILE,
+  TEST_SOURCE_START,
   TEST_IS_NEW,
   TEST_IS_RETRY,
   TEST_EARLY_FLAKE_ENABLED,
@@ -269,6 +270,7 @@ moduleTypes.forEach(({
           testSuiteEvents.forEach(({
             content: {
               meta,
+              metrics,
               test_suite_id: testSuiteId,
               test_module_id: testModuleId,
               test_session_id: testSessionId
@@ -280,6 +282,8 @@ moduleTypes.forEach(({
             assert.exists(testSuiteId)
             assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
             assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
+            assert.isTrue(meta[TEST_SOURCE_FILE].startsWith('cypress/e2e/'))
+            assert.equal(metrics[TEST_SOURCE_START], 1)
           })
 
           assert.includeMembers(testEvents.map(test => test.content.resource), [
@@ -1417,9 +1421,11 @@ moduleTypes.forEach(({
           const events = payloads.flatMap(({ payload }) => payload.events)
 
           const test = events.find(event => event.type === 'test').content
+          const testSuite = events.find(event => event.type === 'test_suite_end').content
           // The test is in a subproject
           assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE])
           assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+          assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
         }, 25000)
 
       childProcess = exec(

diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js
@@ -167,6 +167,8 @@ describe('jest CommonJS', () => {
 
         suites.forEach(testSuite => {
           assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
+          assert.isTrue(testSuite.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'))
+          assert.equal(testSuite.metrics[TEST_SOURCE_START], 1)
         })
 
         done()
@@ -253,9 +255,11 @@ describe('jest CommonJS', () => {
         const events = payloads.flatMap(({ payload }) => payload.events)
 
         const test = events.find(event => event.type === 'test').content
+        const testSuite = events.find(event => event.type === 'test_suite_end').content
         // The test is in a subproject
         assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE])
         assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+        assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
       })
 
     childProcess = exec(

diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js
@@ -167,6 +167,8 @@ describe('mocha CommonJS', function () {
 
         suites.forEach(testSuite => {
           assert.equal(testSuite.meta[TEST_SESSION_NAME], 'my-test-session')
+          assert.isTrue(testSuite.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'))
+          assert.equal(testSuite.metrics[TEST_SOURCE_START], 1)
         })
 
         done()
@@ -253,9 +255,11 @@ describe('mocha CommonJS', function () {
         const events = payloads.flatMap(({ payload }) => payload.events)
 
         const test = events.find(event => event.type === 'test').content
+        const testSuite = events.find(event => event.type === 'test_suite_end').content
         // The test is in a subproject
         assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE])
         assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+        assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
       })
 
     childProcess = exec(

diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js
@@ -110,6 +110,8 @@ versions.forEach((version) => {
               if (testSuiteEvent.content.meta[TEST_STATUS] === 'fail') {
                 assert.exists(testSuiteEvent.content.meta[ERROR_MESSAGE])
               }
+              assert.isTrue(testSuiteEvent.content.meta[TEST_SOURCE_FILE].endsWith('-test.js'))
+              assert.equal(testSuiteEvent.content.metrics[TEST_SOURCE_START], 1)
             })
 
             assert.includeMembers(testEvents.map(test => test.content.resource), [
@@ -674,9 +676,11 @@ versions.forEach((version) => {
           const events = payloads.flatMap(({ payload }) => payload.events)
 
           const test = events.find(event => event.type === 'test').content
+          const testSuite = events.find(event => event.type === 'test_suite_end').content
           // The test is in a subproject
           assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE])
           assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+          assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
         })
 
       childProcess = exec(

diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js
@@ -16,7 +16,9 @@ const {
   TEST_CODE_OWNERS,
   TEST_CODE_COVERAGE_LINES_PCT,
   TEST_SESSION_NAME,
-  TEST_COMMAND
+  TEST_COMMAND,
+  TEST_SOURCE_FILE,
+  TEST_SOURCE_START
 } = require('../../packages/dd-trace/src/plugins/util/test')
 
 const versions = ['1.6.0', 'latest']
@@ -142,6 +144,10 @@ versions.forEach((version) => {
         testSuiteEvents.forEach(testSuite => {
           assert.equal(testSuite.content.meta[TEST_SESSION_NAME], 'my-test-session')
           assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run')
+          assert.isTrue(
+            testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility')
+          )
+          assert.equal(testSuite.content.metrics[TEST_SOURCE_START], 1)
         })
         // TODO: check error messages
       }).then(() => done()).catch(done)
@@ -313,7 +319,9 @@ versions.forEach((version) => {
           const events = payloads.flatMap(({ payload }) => payload.events)
 
           const test = events.find(event => event.type === 'test').content
+          const testSuite = events.find(event => event.type === 'test_suite_end').content
           assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
+          assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js']))
         })
 
       childProcess = exec(

diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js
@@ -467,7 +467,12 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) {
       isUnskippable = isMarkedAsUnskippable(pickle)
       isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath)
 
-      testSuiteStartCh.publish({ testSuitePath, isUnskippable, isForcedToRun, itrCorrelationId })
+      testSuiteStartCh.publish({
+        testFileAbsolutePath,
+        isUnskippable,
+        isForcedToRun,
+        itrCorrelationId
+      })
     }
 
     let isNew = false
@@ -593,7 +598,7 @@ function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion)
       if (!pickleResultByFile[testFileAbsolutePath]) {
         pickleResultByFile[testFileAbsolutePath] = []
         testSuiteStartCh.publish({
-          testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd())
+          testFileAbsolutePath
         })
       }
     }

diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js
@@ -648,6 +648,7 @@ function jestAdapterWrapper (jestAdapter, jestVersion) {
       testSuiteStartCh.publish({
         testSuite: environment.testSuite,
         testEnvironmentOptions: environment.testEnvironmentOptions,
+        testSourceFile: environment.testSourceFile,
         displayName: environment.displayName,
         frameworkVersion: jestVersion
       })

diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js
@@ -118,7 +118,15 @@ class CucumberPlugin extends CiPlugin {
       this.tracer._exporter.flush()
     })
 
-    this.addSub('ci:cucumber:test-suite:start', ({ testSuitePath, isUnskippable, isForcedToRun, itrCorrelationId }) => {
+    this.addSub('ci:cucumber:test-suite:start', ({
+      testFileAbsolutePath,
+      isUnskippable,
+      isForcedToRun,
+      itrCorrelationId
+    }) => {
+      const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd())
+      const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot)
+
       const testSuiteMetadata = getTestSuiteCommonTags(
         this.command,
         this.frameworkVersion,
@@ -139,6 +147,16 @@ class CucumberPlugin extends CiPlugin {
       if (this.testSessionName) {
         testSuiteMetadata[TEST_SESSION_NAME] = this.testSessionName
       }
+      if (testSourceFile) {
+        testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile
+        testSuiteMetadata[TEST_SOURCE_START] = 1
+      }
+
+      const codeOwners = this.getCodeOwners(testSuiteMetadata)
+      if (codeOwners) {
+        testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners
+      }
+
       const testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', {
         childOf: this.testModuleSpan,
         tags: {

diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js
@@ -247,11 +247,22 @@ class CypressPlugin {
     return this.libraryConfigurationPromise
   }
 
-  getTestSuiteSpan (suite) {
+  getTestSuiteSpan ({ testSuite, testSuiteAbsolutePath }) {
     const testSuiteSpanMetadata =
-      getTestSuiteCommonTags(this.command, this.frameworkVersion, suite, TEST_FRAMEWORK_NAME)
+      getTestSuiteCommonTags(this.command, this.frameworkVersion, testSuite, TEST_FRAMEWORK_NAME)
+
     this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
 
+    if (testSuiteAbsolutePath) {
+      const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
+      testSuiteSpanMetadata[TEST_SOURCE_FILE] = testSourceFile
+      testSuiteSpanMetadata[TEST_SOURCE_START] = 1
+      const codeOwners = this.getTestCodeOwners({ testSuite, testSourceFile })
+      if (codeOwners) {
+        testSuiteSpanMetadata[TEST_CODE_OWNERS] = codeOwners
+      }
+    }
+
     return this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_suite`, {
       childOf: this.testModuleSpan,
       tags: {
@@ -482,7 +493,10 @@ class CypressPlugin {
       // dd:testSuiteStart hasn't been triggered for whatever reason
       // We will create the test suite span on the spot if that's the case
       log.warn('There was an error creating the test suite event.')
-      this.testSuiteSpan = this.getTestSuiteSpan(spec.relative)
+      this.testSuiteSpan = this.getTestSuiteSpan({
+        testSuite: spec.relative,
+        testSuiteAbsolutePath: spec.absolute
+      })
     }
 
     // Get tests that didn't go through `dd:afterEach`
@@ -593,7 +607,7 @@ class CypressPlugin {
 
   getTasks () {
     return {
-      'dd:testSuiteStart': (testSuite) => {
+      'dd:testSuiteStart': ({ testSuite, testSuiteAbsolutePath }) => {
         const suitePayload = {
           isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled,
           knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [],
@@ -603,7 +617,7 @@ class CypressPlugin {
         if (this.testSuiteSpan) {
           return suitePayload
         }
-        this.testSuiteSpan = this.getTestSuiteSpan(testSuite)
+        this.testSuiteSpan = this.getTestSuiteSpan({ testSuite, testSuiteAbsolutePath })
         return suitePayload
       },
       'dd:beforeEach': (test) => {

diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js
@@ -61,7 +61,10 @@ beforeEach(function () {
 })
 
 before(function () {
-  cy.task('dd:testSuiteStart', Cypress.mocha.getRootSuite().file).then((suiteConfig) => {
+  cy.task('dd:testSuiteStart', {
+    testSuite: Cypress.mocha.getRootSuite().file,
+    testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute
+  }).then((suiteConfig) => {
     if (suiteConfig) {
       isEarlyFlakeDetectionEnabled = suiteConfig.isEarlyFlakeDetectionEnabled
       knownTestsForSuite = suiteConfig.knownTestsForSuite

diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js
@@ -160,7 +160,13 @@ class JestPlugin extends CiPlugin {
       })
     })
 
-    this.addSub('ci:jest:test-suite:start', ({ testSuite, testEnvironmentOptions, frameworkVersion, displayName }) => {
+    this.addSub('ci:jest:test-suite:start', ({
+      testSuite,
+      testSourceFile,
+      testEnvironmentOptions,
+      frameworkVersion,
+      displayName
+    }) => {
       const {
         _ddTestSessionId: testSessionId,
         _ddTestCommand: testCommand,
@@ -202,6 +208,16 @@ class JestPlugin extends CiPlugin {
       if (testSessionName) {
         testSuiteMetadata[TEST_SESSION_NAME] = testSessionName
       }
+      if (testSourceFile) {
+        testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile
+        // Test suite is the whole test file, so we can use the first line as the start
+        testSuiteMetadata[TEST_SOURCE_START] = 1
+      }
+
+      const codeOwners = this.getCodeOwners(testSuiteMetadata)
+      if (codeOwners) {
+        testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners
+      }
 
       this.testSuiteSpan = this.tracer.startSpan('jest.test_suite', {
         childOf: testSessionSpanContext,

diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js
@@ -129,6 +129,19 @@ class MochaPlugin extends CiPlugin {
       if (this.testSessionName) {
         testSuiteMetadata[TEST_SESSION_NAME] = this.testSessionName
       }
+      if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) {
+        testSuiteMetadata[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
+      } else {
+        testSuiteMetadata[TEST_SOURCE_FILE] = testSuite
+      }
+      if (testSuiteMetadata[TEST_SOURCE_FILE]) {
+        testSuiteMetadata[TEST_SOURCE_START] = 1
+      }
+
+      const codeOwners = this.getCodeOwners(testSuiteMetadata)
+      if (codeOwners) {
+        testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners
+      }
 
       const testSuiteSpan = this.tracer.startSpan('mocha.test_suite', {
         childOf: this.testModuleSpan,

diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js
@@ -70,6 +70,7 @@ class PlaywrightPlugin extends CiPlugin {
     this.addSub('ci:playwright:test-suite:start', (testSuiteAbsolutePath) => {
       const store = storage.getStore()
       const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.rootDir)
+      const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
 
       const testSuiteMetadata = getTestSuiteCommonTags(
         this.command,
@@ -80,6 +81,14 @@ class PlaywrightPlugin extends CiPlugin {
       if (this.testSessionName) {
         testSuiteMetadata[TEST_SESSION_NAME] = this.testSessionName
       }
+      if (testSourceFile) {
+        testSuiteMetadata[TEST_SOURCE_FILE] = testSourceFile
+        testSuiteMetadata[TEST_SOURCE_START] = 1
+      }
+      const codeOwners = this.getCodeOwners(testSuiteMetadata)
+      if (codeOwners) {
+        testSuiteMetadata[TEST_CODE_OWNERS] = codeOwners
+      }
 
       const testSuiteSpan = this.tracer.startSpan('playwright.test_suite', {
         childOf: this.testModuleSpan,