chore: Add open telemetry to cypress to allow us to monitor the perfo…

…rmance of the app overtime (#26305) * initial commit, a kinda working prototype * Ready to test in CI * "SyntaxError: Cannot use import statement outside a module" I blame VS code for always inserting the wrong dependency * try using built js instead of the ts file * typescript fixes? * get version correctly, don't use optional chaining in child process. * trying this, idk * try running telemetry for driver-integration-tests-chrome * fix missing spans, add more attributes for some spans * fix missing spans and add suite spans * types * Remove un-used require * remove spans for describe blocks in favor of the full title for tests * migrate to sync resource discovery, start new custom exporters for spans * encrypted * localhost * don't do things on child process * latest changes * update server start span time / add v8 snapshot span & update command span names * prepare for sync * don't send blank key * make telemetry work again for sending directly to honeycomb * web-socket exporter * Add in IPC exporter and message the child process before disconnecting * Use the cloud api by default * move cloud span exporter into telemetry package * shutdown fixes * fix enabled * improve types * run in ci * yml is the worst * type! * add spans for timing insights for visible areas of improvement * type errors * lets try sending data to staging * types * types again * remove problematic attributes * clean up exporters * i like this better even though it doesn't seem to matter much * some self review cleanup * Update comment * add debug messages * mocha tests * actually exit with the right code... oops * simple mistake... have to look into how to fix with ts... * try this i guess * don't return undefined * read me diagram * color? * no rect * moar diagram * docs! * formatting * build more binaries * Supposedly fix cypress in cypress test failures * test 'fixes' * try this instead * do not transpile cypress packages dir * lets try escaping our regex string * Add some diagnostics to help test the built binary.... * try a more complex solution * fix tests probably * just ignore the specific file * fix unit tests * cr updates * Apply suggestions from code review Co-authored-by: Matt Schile <mschile@cypress.io> Co-authored-by: Bill Glesias <bglesias@gmail.com> * Pr updates * don't change the command queue * move encoding and decoding telemetry context for ipc to the telemetry package * build darn it * plead for mercy from the testing gods, i merely wished to have named test reports, but clearly i have overreached. * pr updates, send record key * pr review * optional chaining fails tests * Apply suggestions from code review Co-authored-by: Bill Glesias <bglesias@gmail.com> * tests for cloud-span-exporter * bad merge * adding tests for the remaining exporters * note * docs * Correctly set test under the current run span for component testing * gate sending the message. * pr updates * finally, fingers crossed. --------- Co-authored-by: Emily Rohrbough <emilyrohrbough@yahoo.com> Co-authored-by: Matt Schile <mschile@cypress.io> Co-authored-by: Bill Glesias <bglesias@gmail.com>
cypress-io · Apr 9, 2023 · 62f58e0 · 62f58e0 · cypress-bot · Apr 9, 2023
1 parent 3aa1c48
commit 62f58e0
Show file tree

Hide file tree

Showing 54 changed files with 3,947 additions and 69 deletions.
diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml
@@ -30,7 +30,7 @@ mainBuildFilters: &mainBuildFilters
         - /^release\/\d+\.\d+\.\d+$/
         # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
         - 'update-v8-snapshot-cache-on-develop'
-        - 'issue-25951-next-app'
+        - 'matth/misc/telemetry'
 
 # usually we don't build Mac app - it takes a long time
 # but sometimes we want to really confirm we are doing the right thing
@@ -41,7 +41,6 @@ macWorkflowFilters: &darwin-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'issue-25951-next-app', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -52,7 +51,6 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'fix/preflight', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -73,7 +71,6 @@ windowsWorkflowFilters: &windows-workflow-filters
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'lmiller/fixing-vite-windows', << pipeline.git.branch >> ]
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'fix/preflight', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -139,7 +136,7 @@ commands:
       - run:
           name: Check current branch to persist artifacts
           command: |
-            if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "issue-25951-next-app" && "$CIRCLE_BRANCH" != "update-v8-snapshot-cache-on-develop" ]]; then
+            if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "update-v8-snapshot-cache-on-develop" ]]; then
               echo "Not uploading artifacts or posting install comment for this branch."
               circleci-agent step halt
             fi
@@ -495,6 +492,8 @@ commands:
             if [[ -v MAIN_RECORD_KEY ]]; then
               # internal PR
               CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \
+              CYPRESS_INTERNAL_ENABLE_TELEMETRY="true" \
+              OTEL_RESOURCE_ATTRIBUTES="ci.branch=$CIRCLE_BRANCH,ci.job=$CIRCLE_JOB,ci.node-index=$CIRCLE_NODE_INDEX,ci.circle=$CIRCLECI,ci.build-url=$CIRCLE_BUILD_URL,ci.build-number=$CIRCLE_BUILD_NUM" \
               yarn cypress:run --record --parallel --group 5x-driver-<<parameters.browser>> --browser <<parameters.browser>>
             else
               # external PR
@@ -567,6 +566,8 @@ commands:
               PERCY_PARALLEL_NONCE=$CIRCLE_WORKFLOW_WORKSPACE_ID \
               PERCY_ENABLE=${PERCY_TOKEN:-0} \
               PERCY_PARALLEL_TOTAL=-1 \
+              CYPRESS_INTERNAL_ENABLE_TELEMETRY="true" \
+              OTEL_RESOURCE_ATTRIBUTES="ci.branch=$CIRCLE_BRANCH,ci.job=$CIRCLE_JOB,ci.node-index=$CIRCLE_NODE_INDEX,ci.circle=$CIRCLECI,ci.build-url=$CIRCLE_BUILD_URL,ci.build-number=$CIRCLE_BUILD_NUM" \
               $cmd yarn workspace @packages/<<parameters.package>> cypress:run:<<parameters.type>> --browser <<parameters.browser>> --record --parallel --group <<parameters.package>>-<<parameters.type>>
             else
               # external PR
@@ -1307,7 +1308,7 @@ jobs:
       <<: *defaultsParameters
     steps:
       - restore_cached_workspace
-      - run: 
+      - run:
           name: 'Determine if Release Workflow should be triggered'
           command: |
             if [[ "$CIRCLE_BRANCH" != "develop" ]]; then
@@ -1455,7 +1456,7 @@ jobs:
             # run type checking for each individual package
             - run: yarn lerna run types
             - verify-mocha-results:
-                expectedResultCount: 18
+                expectedResultCount: 19
       - store_test_results:
           path: /tmp/cypress
       # CLI tests generate HTML files with sample CLI command output

diff --git a/package.json b/package.json
@@ -49,7 +49,7 @@
     "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src,__snapshots__ --exclude cypress-tests.ts,*only.cy.js",
     "stop-only-all": "yarn stop-only --folder packages",
     "pretest": "yarn ensure-deps",
-    "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{config,data-context,electron,errors,extension,https-proxy,launcher,net-stubbing,network,packherd-require,proxy,rewriter,scaffold-config,socket,v8-snapshot-require}'\" --scope \"'@tooling/{electron-mksnapshot,v8-snapshot}'\"",
+    "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{config,data-context,electron,errors,extension,https-proxy,launcher,net-stubbing,network,packherd-require,proxy,rewriter,scaffold-config,socket,v8-snapshot-require,telemetry}'\" --scope \"'@tooling/{electron-mksnapshot,v8-snapshot}'\"",
     "test-debug": "lerna exec yarn test-debug --ignore \"'@packages/{driver,root,static,web-config}'\"",
     "pretest-e2e": "yarn ensure-deps",
     "test-integration": "lerna exec yarn test-integration --ignore \"'@packages/{driver,root,static,web-config}'\"",
@@ -275,4 +275,4 @@
     "sharp": "0.29.3",
     "vue-template-compiler": "2.6.12"
   }
-}
+}
diff --git a/packages/app/index.d.ts b/packages/app/index.d.ts
@@ -13,9 +13,9 @@ export {}
  * To work around this, we build the driver, eventManager
  * and some other dependencies using webpack, and consumed the dist'd
  * source code.
- * 
+ *
  * This is attached to `window` under the `UnifiedRunner` namespace.
- * 
+ *
  * For now, just declare the types that we need to give us type safety where possible.
  * Eventually, we should decouple the event manager and import it directly.
  */
@@ -37,7 +37,7 @@ declare global {
        * We get a reference to the copy of React (and React DOM)
        * that is used in the Reporter and Driver, which are bundled with
        * webpack.
-       * 
+       *
        * Unfortunately, attempting to have React in a project
        * using Vue causes mad conflicts because React'S JSX type
        * is ambient, so we cannot actually type it.
@@ -54,7 +54,7 @@ declare global {
        * Any React components or general code needed from
        * runner, reporter or driver are also bundled with
        * webpack and made available via the window.UnifedRunner namespace.
-       * 
+       *
        * We cannot import the correct types, because this causes the linter and type
        * checker to run on runner and reporter, and it blows up.
        */
@@ -64,4 +64,4 @@ declare global {
       }
     }
   }
-}
+}
diff --git a/packages/app/package.json b/packages/app/package.json
@@ -29,6 +29,7 @@
     "@iconify/vue": "3.0.0-beta.1",
     "@intlify/vite-plugin-vue-i18n": "2.4.0",
     "@packages/frontend-shared": "0.0.0-development",
+    "@packages/telemetry": "0.0.0-development",
     "@percy/cypress": "^3.1.0",
     "@popperjs/core": "2.11.6",
     "@testing-library/cypress": "9.0.0",

diff --git a/packages/app/src/main.ts b/packages/app/src/main.ts
@@ -11,15 +11,23 @@ import { createPinia } from './store'
 import Toast, { POSITION } from 'vue-toastification'
 import 'vue-toastification/dist/index.css'
 import { createWebsocket, getRunnerConfigFromWindow } from './runner'
+import { telemetry } from '@packages/telemetry/src/browser'
 
-const app = createApp(App)
-
+// Grab the time just before loading config to include that in the cypress:app span
+const now = performance.now()
 const config = getRunnerConfigFromWindow()
 
+telemetry.init({ namespace: 'cypress:app', config })
+telemetry.startSpan({ name: 'cypress:app', attachType: 'root', opts: { startTime: now } })
+
+const app = createApp(App)
+
 const ws = createWebsocket(config)
 
 window.ws = ws
 
+telemetry.attachWebSocket(ws)
+
 // This injects the Cypress driver and Reporter, which are bundled with Webpack.
 // No need to wait for it to finish - it's resolved async with a deferred promise,
 // So it'll be ready when we need to run a spec. If not, we will wait for it.

diff --git a/packages/app/src/runner/event-manager.ts b/packages/app/src/runner/event-manager.ts
@@ -13,6 +13,8 @@ import { useScreenshotStore } from '../store/screenshot-store'
 import { useStudioStore } from '../store/studio-store'
 import { getAutIframeModel } from '.'
 import { handlePausing } from './events/pausing'
+import { addTelemetryListeners } from './events/telemetry'
+import { telemetry } from '@packages/telemetry/src/browser'
 
 export type CypressInCypressMochaEvent = Array<Array<string | Record<string, any>>>
 
@@ -129,6 +131,12 @@ export class EventManager {
       window.location.href = url
     })
 
+    this.ws.on('update:telemetry:context', (contextString) => {
+      const context = JSON.parse(contextString)
+
+      telemetry.setRootContext(context)
+    })
+
     this.ws.on('automation:push:message', (msg, data = {}) => {
       if (!Cypress) return
 
@@ -346,6 +354,7 @@ export class EventManager {
     // that Cypress knows not to set any more
     // cookies
     $window.on('beforeunload', () => {
+      telemetry.getSpan('cypress:app')?.end()
       this.reporterBus.emit('reporter:restart:test:run')
 
       this._clearAllCookies()
@@ -452,6 +461,8 @@ export class EventManager {
   }
 
   _addListeners () {
+    addTelemetryListeners(Cypress)
+
     Cypress.on('message', (msg, data, cb) => {
       this.ws.emit('client:request', msg, data, cb)
     })

diff --git a/packages/app/src/runner/events/telemetry.ts b/packages/app/src/runner/events/telemetry.ts
@@ -0,0 +1,39 @@
+import { telemetry } from '@packages/telemetry/src/browser'
+
+export const addTelemetryListeners = (Cypress) => {
+  Cypress.on('test:before:run', (attributes, test) => {
+    // we emit the 'test:before:run' events within various driver tests
+    try {
+      // If a span for a previous test hasn't been ended, end it before starting the new test span
+      const previousTestSpan = telemetry.findActiveSpan((span) => {
+        return span.name.startsWith('test:')
+      })
+
+      if (previousTestSpan) {
+        telemetry.endActiveSpanAndChildren(previousTestSpan)
+      }
+
+      const span = telemetry.startSpan({ name: `test:${test.fullTitle()}`, active: true })
+
+      span?.setAttributes({
+        currentRetry: attributes.currentRetry,
+      })
+    } catch (error) {
+      // TODO: log error when client side debug logging is available
+    }
+  })
+
+  Cypress.on('test:after:run', (attributes, test) => {
+    try {
+      const span = telemetry.getSpan(`test:${test.fullTitle()}`)
+
+      span?.setAttributes({
+        timings: JSON.stringify(attributes.timings),
+      })
+
+      span?.end()
+    } catch (error) {
+      // TODO: log error when client side debug logging is available
+    }
+  })
+}
diff --git a/packages/data-context/package.json b/packages/data-context/package.json
@@ -19,6 +19,7 @@
     "@babel/generator": "7.17.9",
     "@babel/parser": "7.13.0",
     "@graphql-tools/batch-execute": "^8.4.6",
+    "@packages/telemetry": "0.0.0-development",
     "@urql/core": "2.4.4",
     "@urql/exchange-execute": "1.1.0",
     "@urql/exchange-graphcache": "4.3.6",

diff --git a/packages/data-context/src/data/ProjectConfigIpc.ts b/packages/data-context/src/data/ProjectConfigIpc.ts
@@ -11,6 +11,8 @@ import { autoBindDebug, hasTypeScriptInstalled, toPosix } from '../util'
 import _ from 'lodash'
 import { pathToFileURL } from 'url'
 import os from 'os'
+import type { OTLPTraceExporterCloud } from '@packages/telemetry'
+import { telemetry, encodeTelemetryContext } from '@packages/telemetry'
 
 const pkg = require('@packages/root')
 const debug = debugLib(`cypress:lifecycle:ProjectConfigIpc`)
@@ -76,6 +78,14 @@ export class ProjectConfigIpc extends EventEmitter {
       this.emit('disconnect')
     })
 
+    // This forwards telemetry requests from the child process to the server
+    this.on('export:telemetry', (data) => {
+      // Not too worried about tracking successes
+      (telemetry.exporter() as OTLPTraceExporterCloud)?.send(data, () => {}, (err) => {
+        debug('error exporting telemetry data from child process %s', err)
+      })
+    })
+
     return autoBindDebug(this)
   }
 
@@ -87,6 +97,7 @@ export class ProjectConfigIpc extends EventEmitter {
   send(event: 'execute:plugins', evt: string, ids: {eventId: string, invocationId: string}, args: any[]): boolean
   send(event: 'setupTestingType', testingType: TestingType, options: Cypress.PluginConfigOptions): boolean
   send(event: 'loadConfig'): boolean
+  send(event: 'main:process:will:disconnect'): void
   send (event: string, ...args: any[]) {
     if (this._childProcess.killed || !this._childProcess.connected) {
       return false
@@ -96,7 +107,8 @@ export class ProjectConfigIpc extends EventEmitter {
   }
 
   on(evt: 'childProcess:unhandledError', listener: (err: CypressError) => void): this
-
+  on(evt: 'export:telemetry', listener: (data: string) => void): void
+  on(evt: 'main:process:will:disconnect:ack', listener: () => void): void
   on(evt: 'warning', listener: (warningErr: CypressError) => void): this
   on (evt: string, listener: (...args: any[]) => void) {
     return super.on(evt, listener)
@@ -319,6 +331,11 @@ export class ProjectConfigIpc extends EventEmitter {
       debug(`no typescript found, just use regular Node.js`)
     }
 
+    const telemetryCtx = encodeTelemetryContext({ context: telemetry.getActiveContextObject(), version: pkg.version })
+
+    // Pass the active context from the main process to the child process as the --telemetryCtx flag.
+    configProcessArgs.push('--telemetryCtx', telemetryCtx)
+
     if (process.env.CYPRESS_INTERNAL_E2E_TESTING_SELF_PARENT_PROJECT) {
       if (isSandboxNeeded()) {
         configProcessArgs.push('--no-sandbox')

diff --git a/packages/data-context/src/data/ProjectConfigManager.ts b/packages/data-context/src/data/ProjectConfigManager.ts
@@ -19,6 +19,8 @@ import { autoBindDebug } from '../util/autoBindDebug'
 import type { EventRegistrar } from './EventRegistrar'
 import type { DataContext } from '../DataContext'
 import { isDependencyInstalled, WIZARD_BUNDLERS } from '@packages/scaffold-config'
+import type { OTLPTraceExporterCloud } from '@packages/telemetry'
+import { telemetry } from '@packages/telemetry'
 
 const debug = debugLib(`cypress:lifecycle:ProjectConfigManager`)
 
@@ -248,13 +250,17 @@ export class ProjectConfigManager {
   }
 
   private async setupNodeEvents (loadConfigReply: LoadConfigReply): Promise<void> {
+    const nodeEventsSpan = telemetry.startSpan({ name: 'dataContext:setupNodeEvents' })
+
     assert(this._eventsIpc, 'Expected _eventsIpc to be defined at this point')
     this._state = 'loadingNodeEvents'
 
     try {
       assert(this._testingType, 'Cannot setup node events without a testing type')
       this._registeredEventsTarget = this._testingType
-      const config = await this.getFullInitialConfig()
+      const config = await this.getFullInitialConfig();
+
+      (telemetry.exporter() as OTLPTraceExporterCloud)?.attachProjectId(config.projectId)
 
       const setupNodeEventsReply = await this._eventsIpc.callSetupNodeEventsWithConfig(this._testingType, config, this.options.handlers)
 
@@ -271,6 +277,7 @@ export class ProjectConfigManager {
 
       throw error
     } finally {
+      nodeEventsSpan?.end()
       this.options.ctx.emitter.toLaunchpad()
       this.options.ctx.emitter.toApp()
     }
@@ -606,6 +613,34 @@ export class ProjectConfigManager {
     return true
   }
 
+  /**
+   * Informs the child process if the main process will soon disconnect.
+   * @returns promise
+   */
+  mainProcessWillDisconnect (): Promise<void> {
+    return new Promise((resolve, reject) => {
+      if (!this._eventsIpc) {
+        debug(`mainProcessWillDisconnect message not set, no IPC available`)
+        reject()
+
+        return
+      }
+
+      this._eventsIpc.send('main:process:will:disconnect')
+
+      // If for whatever reason we don't get an ack in 5s, bail.
+      const timeoutId = setTimeout(() => {
+        debug(`mainProcessWillDisconnect message timed out`)
+        reject()
+      }, 5000)
+
+      this._eventsIpc.on('main:process:will:disconnect:ack', () => {
+        clearTimeout(timeoutId)
+        resolve()
+      })
+    })
+  }
+
   private async closeWatchers () {
     await Promise.all(Array.from(this._watchers).map((watcher) => {
       return watcher.close().catch((error) => {