From d89b497c3c8e296d9f707f2e6b368cf23a33f0ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Fri, 6 Dec 2024 14:22:27 +0100 Subject: [PATCH] Performance analysis workflow (#10625) * Performance analsysis workflow * add `replayio install` step * use proper preview deployment for pull_request runs * enable metadata params * parseInt pullRequest --- .github/workflows/perf_analysis.yml | 50 +++ package.json | 7 +- packages/e2e-tests/package.json | 4 +- packages/e2e-tests/scripts/record-node.ts | 6 +- .../e2e-tests/scripts/record-playwright.ts | 102 ------ packages/e2e-tests/scripts/save-examples.ts | 204 ++++------- packages/e2e-tests/tsconfig.json | 6 +- packages/playwright-recorder/package.json | 13 + packages/playwright-recorder/src/config.ts | 7 + packages/playwright-recorder/src/index.ts | 228 ++++++++++++ packages/playwright-recorder/tsconfig.json | 14 + scripts/perf-analysis/main.ts | 120 +++++++ .../tests/devtools-object-preview.ts | 25 ++ scripts/perf-analysis/tsconfig.json | 14 + tsconfig.eslint.json | 2 +- tsconfig.json | 3 +- yarn.lock | 330 +++++++++++++++++- 17 files changed, 881 insertions(+), 254 deletions(-) create mode 100644 .github/workflows/perf_analysis.yml delete mode 100644 packages/e2e-tests/scripts/record-playwright.ts create mode 100644 packages/playwright-recorder/package.json create mode 100644 packages/playwright-recorder/src/config.ts create mode 100644 packages/playwright-recorder/src/index.ts create mode 100644 packages/playwright-recorder/tsconfig.json create mode 100644 scripts/perf-analysis/main.ts create mode 100644 scripts/perf-analysis/tests/devtools-object-preview.ts create mode 100644 scripts/perf-analysis/tsconfig.json diff --git a/.github/workflows/perf_analysis.yml b/.github/workflows/perf_analysis.yml new file mode 100644 index 00000000000..a204378b585 --- /dev/null +++ b/.github/workflows/perf_analysis.yml @@ -0,0 +1,50 @@ +name: Performance analysis + +on: + pull_request: + workflow_dispatch: + schedule: + - cron: "0 */2 * * *" # every 2 hours + +jobs: + perf-analysis: + name: Performance analysis + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Get the yarn cache path. + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "{dir}={$(yarn config get cacheFolder)}" >> $GITHUB_OUTPUT + - name: Restore yarn cache + uses: actions/cache@v3 + id: yarn-cache + with: + path: | + ${{ steps.yarn-cache-dir-path.outputs.dir }} + **/node_modules + key: yarn-cache-folder-${{ hashFiles('**/yarn.lock', '.yarnrc.yml') }} + restore-keys: "yarn-cache-folder-" + # Actually install packages with Yarn + - name: Install packages + run: yarn install + env: + YARN_CHECKSUM_BEHAVIOR: "update" + - name: Install Replay Chromium + run: npx replayio@latest install + - name: Wait for Vercel preview deployment to be ready + uses: patrickedqvist/wait-for-vercel-preview@v1.3.1 + if: github.event_name == 'pull_request' + id: wait-for-vercel-preview + with: + token: ${{ secrets.GITHUB_TOKEN }} + max_timeout: 240 + - name: Run performance analysis + run: yarn run perf-analysis + env: + REPLAY_API_KEY: ${{ secrets.PERFORMANCE_ANALYSIS_REPLAY_API_KEY }} + DEVTOOLS_URL: ${{ steps.wait-for-vercel-preview.outputs.url }} + GITHUB_REPOSITORY: ${{ env.GITHUB_REPOSITORY }} + GITHUB_REF_NAME: ${{ env.GITHUB_REF_NAME }} + GITHUB_PR: ${{ github.event.pull_request && github.event.pull_request.number }} + GITHUB_SHA: ${{ env.GITHUB_SHA }} diff --git a/package.json b/package.json index fa7a3e88b1a..d49c83b4819 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "typecheck:watch": "yarn typecheck --watch", "gql:schema": "gq https://graphql.replay.io/v1/graphql -H \"X-Hasura-Admin-Secret: $HASURA_KEY\" --introspect > schema.graphql", "gql:gen": "apollo codegen:generate --localSchemaFile=schema.graphql --includes='{packages/shared,src}/**/*.{ts,tsx}' --target=typescript --tagName=gql --outputFlat packages/shared/graphql/generated", - "gql": "yarn gql:schema && yarn gql:gen" + "gql": "yarn gql:schema && yarn gql:gen", + "perf-analysis": "cd scripts/perf-analysis && tsx main.ts" }, "dependencies": { "@apollo/client": "^3.5.10", @@ -109,6 +110,7 @@ "@babel/preset-typescript": "^7.15.0", "@babel/types": "^7.14.8", "@bcoe/v8-coverage": "^0.2.3", + "@devtools-repo/playwright-recorder": "workspace:*", "@jest/console": "^28.0.2", "@jridgewell/trace-mapping": "^0.3.14", "@next/bundle-analyzer": "^12.2.0", @@ -190,8 +192,9 @@ "stylelint": "^14.16.0", "stylelint-config-prettier": "^9.0.5", "tailwindcss": "^3.2.4", - "ts-node": "^10.7.0", + "ts-node": "^10.9.2", "tsconfig-paths": "^3.14.1", + "tsx": "^4.19.2", "typescript": "^5.4.2", "utf-8-validate": "^5.0.8", "uuid": "^7.0.3", diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index 8342ac749f4..f24872c4c9f 100644 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -20,11 +20,13 @@ "author": "", "license": "ISC", "devDependencies": { + "@devtools-repo/playwright-recorder": "*", "@playwright/test": "^1.37.0", "@replayio/playwright": "3.0.0-alpha.14", + "axios": "^0.21.1", "cli-spinners": "^2.7.0", "cypress": "^12.5.1", - "ts-node": "^10.7.0", + "ts-node": "^10.9.2", "ws": "^7.4.6", "yargs": "^17.6.0" }, diff --git a/packages/e2e-tests/scripts/record-node.ts b/packages/e2e-tests/scripts/record-node.ts index 3884ef3526a..18fb5f6ffe2 100644 --- a/packages/e2e-tests/scripts/record-node.ts +++ b/packages/e2e-tests/scripts/record-node.ts @@ -15,13 +15,13 @@ function getRecordingId(file: string) { } return contents; } - return null; + return; } catch (e) { - return null; + return; } } -export async function recordNodeExample(scriptPath: string): Promise { +export async function recordNodeExample(scriptPath: string): Promise { const nodePath = config.nodePath || execSync("which replay-node").toString().trim(); if (!nodePath) { console.warn("\x1b[1m\x1b[31m" + "Node e2e tests require @replayio/node" + "\x1b[0m"); diff --git a/packages/e2e-tests/scripts/record-playwright.ts b/packages/e2e-tests/scripts/record-playwright.ts deleted file mode 100644 index b1b2cc747ff..00000000000 --- a/packages/e2e-tests/scripts/record-playwright.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { readFileSync } from "fs"; -import path from "path"; -import type { Page, expect as expectFunction } from "@playwright/test"; -import { chromium, expect } from "@playwright/test"; -import { getExecutablePath } from "@replayio/playwright"; -import * as cli from "@replayio/replay"; -import findLast from "lodash/findLast"; - -import config from "../config"; - -export async function recordPlaywright( - script: (page: Page, expect?: typeof expectFunction) => Promise -) { - let executablePath: string | undefined = undefined; - if (config.shouldRecordTest) { - executablePath = config.browserPath || getExecutablePath(); - } - - const browserServer = await chromium.launchServer({ - env: { - ...process.env, - // @ts-ignore - RECORD_REPLAY_DRIVER: config.driverPath, - RECORD_ALL_CONTENT: "1", - }, - executablePath, //: config.browserPath, - headless: config.headless, - }); - - if (process.env.RECORD_REPLAY_VERBOSE) { - // TODO: Always keep logs, and make them available if the recording failed. - const stderr = browserServer.process().stderr; - stderr?.addListener("data", data => { - console.debug(`[RUNTIME] ${data}`); - }); - } - - const browser = await chromium.connect(browserServer.wsEndpoint()); - const context = await browser.newContext({ - ignoreHTTPSErrors: true, - }); - const page = await context.newPage(); - try { - return await script(page, expect); - } finally { - await page.close(); - await context.close(); - await browser.close(); - await browserServer.close(); - } -} - -export async function uploadLastRecording(url: string): Promise { - const list = cli.listAllRecordings(); - const id = findLast(list, rec => rec.metadata.uri === url)?.id; - - if (id) { - // When running the Replay backend tests, we run against a selfcontained backend and we don't - // want to force it to run Recording.processRecording on every test fixture because it would be - // really slow and not do anything useful. By hardcoding this metadata, we can convince the Replay - // upload library that since this was a "passed" result, it does not need to process the recording. - cli.addLocalRecordingMetadata(id, { - test: { - file: "fake.html", - path: ["fake.html"], - result: "passed", - runner: { - name: "fake", - version: "", - }, - run: { - id: "00000000-0000-4000-8000-000000000000", - title: "fake", - }, - title: "", - version: 1, - }, - }); - - return await cli.uploadRecording(id, { - apiKey: config.replayApiKey, - server: config.backendUrl, - verbose: true, - strict: true, - }); - } else { - const recordingsLog = readRecordingsLog().split("\n").slice(-11).join("\n"); - throw Error( - `No recording found matching url "${url}" in list:\n${list - .map(rec => rec.metadata.uri) - .join("\n")}\nLast 10 lines of recordings.log:\n${recordingsLog}` - ); - } -} - -function readRecordingsLog() { - const dir = - process.env.RECORD_REPLAY_DIRECTORY || - path.join(process.env.HOME || process.env.USERPROFILE, ".replay"); - const file = path.join(dir, "recordings.log"); - return readFileSync(file, "utf8"); -} diff --git a/packages/e2e-tests/scripts/save-examples.ts b/packages/e2e-tests/scripts/save-examples.ts index 2af2d77571e..43b077eda13 100755 --- a/packages/e2e-tests/scripts/save-examples.ts +++ b/packages/e2e-tests/scripts/save-examples.ts @@ -6,24 +6,26 @@ import { existsSync, writeFileSync } from "fs"; import assert from "node:assert/strict"; -import { join } from "path"; +import path from "path"; import type { Page, expect as expectFunction } from "@playwright/test"; -import { removeRecording, uploadRecording } from "@replayio/replay"; -import axios from "axios"; import chalk from "chalk"; import difference from "lodash/difference"; import { v4 as uuidv4 } from "uuid"; import yargs from "yargs"; -import { SetRecordingIsPrivateVariables } from "../../shared/graphql/generated/SetRecordingIsPrivate"; -import { UpdateRecordingTitleVariables } from "../../shared/graphql/generated/UpdateRecordingTitle"; import config from "../config"; import examplesJson from "../examples.json"; import { TestRecordingIntersectionValue } from "../helpers"; import { getStats } from "./get-stats"; import { loadRecording } from "./loadRecording"; import { recordNodeExample } from "./record-node"; -import { recordPlaywright, uploadLastRecording } from "./record-playwright"; + +import { + saveRecording, + recordPlaywright, + findLastRecordingId, + removeRecording, +} from "@devtools-repo/playwright-recorder"; type Target = "all" | "browser" | "node"; @@ -71,55 +73,15 @@ type TestExampleFile = { runtime: "chromium" | "node"; playwrightScript?: PlaywrightScript; }; -const examplesJsonPath = join(__dirname, "..", "examples.json"); - -let mutableExamplesJSON = { ...examplesJson }; - -const exampleToNewRecordingId: { [example: string]: string } = {}; - -async function saveRecording( - example: string, - apiKey: string, - recordingId: string, - skipUpload?: boolean -) { - console.log( - `Saving ${chalk.grey.bold(example)} with recording id ${chalk.yellow.bold(recordingId)}` - ); - - if (!skipUpload) { - await uploadRecording(recordingId, { - apiKey, - server: config.backendUrl, - strict: true, - }); - } - - const response = await axios({ - url: config.graphqlUrl, - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - }, - data: { - query: ` - query GetRecordingBuildId($recordingId: UUID!) { - recording(uuid: $recordingId) { - buildId - } - } - `, - variables: { - recordingId, - }, - }, - }); +const examplesJsonPath = path.join(__dirname, "..", "examples.json"); - const buildId = response.data.data.recording.buildId; +let mutableExamplesJSON: Record = { + ...examplesJson, +}; - await makeReplayPublic(apiKey, recordingId); - await updateRecordingTitle(apiKey, recordingId, `E2E Example: ${example}`); +const exampleToNewRecordingId: Record = {}; +function updateExamplesJSONFile(example: string, recordingId: string, buildId: string) { mutableExamplesJSON = { ...mutableExamplesJSON, [example]: { @@ -132,6 +94,28 @@ async function saveRecording( writeFileSync(examplesJsonPath, JSON.stringify(mutableExamplesJSON, null, 2)); } +// When running the Replay backend tests, we run against a selfcontained backend and we don't +// want to force it to run Recording.processRecording on every test fixture because it would be +// really slow and not do anything useful. By hardcoding this metadata, we can convince the Replay +// upload library that since this was a "passed" result, it does not need to process the recording. +const fakeTestPassedRecordingMetadata = { + test: { + file: "fake.html", + path: ["fake.html"], + result: "passed", + runner: { + name: "fake", + version: "", + }, + run: { + id: "00000000-0000-4000-8000-000000000000", + title: "fake", + }, + title: "", + version: 1, + }, +}; + interface TestRunCallbackArgs { example: TestExampleFile; examplePath: string; @@ -143,7 +127,8 @@ async function saveExamples( ) { let examplesToRun: TestExampleFile[] = []; - for (const key in examplesJson) { + let key: keyof typeof examplesJson; + for (key in examplesJson) { const { buildId, playwrightScript, @@ -158,8 +143,8 @@ async function saveExamples( const [_, runtime] = buildId.split("-"); - let category: TestExampleFile["category"]; - let folder: TestExampleFile["folder"]; + let category: TestExampleFile["category"] | undefined; + let folder: TestExampleFile["folder"] | undefined; switch (runtime) { case "chromium": @@ -175,10 +160,13 @@ async function saveExamples( } } + assert(category, "Unknown category for runtime: " + runtime); + assert(folder, "Unknown folder for runtime: " + runtime); + if (category === examplesTarget) { let resolvedPlaywrightScript: PlaywrightScript | undefined; if (playwrightScript) { - const playwrightScriptModule = require(join("..", playwrightScript)); + const playwrightScriptModule = require(path.join("..", playwrightScript)); assert( typeof playwrightScriptModule.default === "function", `Expected default export to be a function in ${playwrightScript}` @@ -218,7 +206,7 @@ async function saveExamples( } for (const example of examplesToRun) { - const examplePath = join(example.folder, example.filename); + const examplePath = path.join(example.folder, example.filename); if (existsSync(examplePath)) { await callback({ example, examplePath }); } else { @@ -252,19 +240,23 @@ async function saveBrowserExample({ example }: TestRunCallbackArgs) { }) ); - const recordingId = await raceForTime(CONFIG.uploadTimeout, uploadLastRecording(exampleUrl)); - if (recordingId == null) { - throw new Error(`Recording "${example.filename}" not uploaded`); - } + const recordingId = findLastRecordingId(exampleUrl); exampleToNewRecordingId[example.filename] = recordingId; - if (config.useExampleFile && recordingId) { - await saveRecording(example.filename, config.replayApiKey, recordingId, true); - } + const { buildId } = await saveRecording( + { + recordingId, + title: `E2E Example: ${example.filename}`, + metadata: fakeTestPassedRecordingMetadata, + }, + config.replayApiKey + ); - if (recordingId) { - removeRecording(recordingId); + if (config.useExampleFile) { + updateExamplesJSONFile(example.filename, recordingId, buildId); } + + removeRecording(recordingId); } async function saveNodeExamples() { @@ -275,7 +267,14 @@ async function saveNodeExamples() { const recordingId = await recordNodeExample(examplePath); if (recordingId) { - await saveRecording(example.filename, config.replayApiKey, recordingId); + const { buildId } = await saveRecording( + { + recordingId, + title: `E2E Example: ${example.filename}`, + }, + config.replayApiKey + ); + updateExamplesJSONFile(example.filename, recordingId, buildId); removeRecording(recordingId); exampleToNewRecordingId[example.filename] = recordingId; @@ -287,75 +286,6 @@ async function saveNodeExamples() { }); } -function logError(e: any, variables: any) { - if (e.response) { - console.log("Parameters"); - console.log(JSON.stringify(variables, undefined, 2)); - console.log("Response"); - console.log(JSON.stringify(e.response.data, undefined, 2)); - } - - throw e.message; -} - -async function makeReplayPublic(apiKey: string, recordingId: string) { - const variables: SetRecordingIsPrivateVariables = { - recordingId: recordingId, - isPrivate: false, - }; - - return axios({ - url: config.graphqlUrl, - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - }, - data: { - query: ` - mutation MakeReplayPublic($recordingId: ID!, $isPrivate: Boolean!) { - updateRecordingPrivacy(input: { id: $recordingId, private: $isPrivate }) { - success - } - } - `, - variables, - }, - }).catch(e => { - logError(e, variables); - }); -} - -async function updateRecordingTitle(apiKey: string, recordingId: string, title: string) { - const variables: UpdateRecordingTitleVariables = { - recordingId: recordingId, - title, - }; - - return axios({ - url: config.graphqlUrl, - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - }, - data: { - query: ` - mutation UpdateRecordingTitle($recordingId: ID!, $title: String!) { - updateRecordingTitle(input: { id: $recordingId, title: $title }) { - success - recording { - uuid - title - } - } - } - `, - variables, - }, - }).catch(e => { - logError(e, variables); - }); -} - async function sleep(timeoutMs: number) { return new Promise(r => setTimeout(() => r(), timeoutMs)); } @@ -427,7 +357,7 @@ async function waitUntilMessage( for (const recordingId of newRecordingIds) { try { await loadRecording(recordingId); - } catch (e) { + } catch (e: any) { console.error(`Ignored error during processing: ${e?.stack || e}`); } } diff --git a/packages/e2e-tests/tsconfig.json b/packages/e2e-tests/tsconfig.json index d0dc35acafe..a5fb257bc07 100644 --- a/packages/e2e-tests/tsconfig.json +++ b/packages/e2e-tests/tsconfig.json @@ -1,10 +1,14 @@ { "compilerOptions": { + "strict": true, "target": "es2018", "moduleResolution": "Node", "downlevelIteration": true, "esModuleInterop": true, "resolveJsonModule": true, - "types": ["cypress", "node"] + "types": ["cypress", "node"], + "paths": { + "@devtools-repo/playwright-recorder": ["../playwright-recorder/src/index.ts"], + } } } diff --git a/packages/playwright-recorder/package.json b/packages/playwright-recorder/package.json new file mode 100644 index 00000000000..84e3664d45d --- /dev/null +++ b/packages/playwright-recorder/package.json @@ -0,0 +1,13 @@ +{ + "name": "@devtools-repo/playwright-recorder", + "private": true, + "version": "0.0.0", + "main": "./dist/index.js", + "dependencies": { + "@playwright/test": "^1.37.0", + "@replayio/replay": "^0.22.4", + "axios": "^0.21.1", + "chalk": "^4", + "strip-ansi": "^6.0.0" + } +} diff --git a/packages/playwright-recorder/src/config.ts b/packages/playwright-recorder/src/config.ts new file mode 100644 index 00000000000..93c207ba9b1 --- /dev/null +++ b/packages/playwright-recorder/src/config.ts @@ -0,0 +1,7 @@ +export default { + backendUrl: process.env.DISPATCH_ADDRESS || "wss://dispatch.replay.io", + graphqlUrl: process.env.GRAPHQL_ADDRESS || "https://api.replay.io/v1/graphql", + browserPath: process.env.RECORD_REPLAY_PATH, + driverPath: process.env.RECORD_REPLAY_DRIVER, + headless: process.env.RECORD_REPLAY_PLAYWRIGHT_HEADLESS != "false", +}; diff --git a/packages/playwright-recorder/src/index.ts b/packages/playwright-recorder/src/index.ts new file mode 100644 index 00000000000..51145e116d0 --- /dev/null +++ b/packages/playwright-recorder/src/index.ts @@ -0,0 +1,228 @@ +import assert from "assert"; +import fs from "fs"; +import path from "path"; +import type { Page, expect as expectFunction } from "@playwright/test"; +import { chromium, expect } from "@playwright/test"; +import { getExecutablePath } from "@replayio/playwright"; +import * as cli from "@replayio/replay"; +import axios from "axios"; +import chalk from "chalk"; + +import config from "./config"; + +// those are copied from shared/graphql/generated +interface SetRecordingIsPrivateVariables { + recordingId: string; + isPrivate: boolean; +} +interface UpdateRecordingTitleVariables { + recordingId: string; + title: string; +} + +function logError(e: any, variables: any) { + if (e.response) { + console.log("Parameters"); + console.log(JSON.stringify(variables, undefined, 2)); + console.log("Response"); + console.log(JSON.stringify(e.response.data, undefined, 2)); + } + + throw e.message; +} + +async function makeReplayPublic(apiKey: string, recordingId: string) { + const variables: SetRecordingIsPrivateVariables = { + recordingId: recordingId, + isPrivate: false, + }; + + return axios({ + url: config.graphqlUrl, + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + data: { + query: ` + mutation MakeReplayPublic($recordingId: ID!, $isPrivate: Boolean!) { + updateRecordingPrivacy(input: { id: $recordingId, private: $isPrivate }) { + success + } + } + `, + variables, + }, + }).catch(e => { + logError(e, variables); + }); +} + +async function updateRecordingTitle(apiKey: string, recordingId: string, title: string) { + const variables: UpdateRecordingTitleVariables = { + recordingId: recordingId, + title, + }; + + return axios({ + url: config.graphqlUrl, + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + data: { + query: ` + mutation UpdateRecordingTitle($recordingId: ID!, $title: String!) { + updateRecordingTitle(input: { id: $recordingId, title: $title }) { + success + recording { + uuid + title + } + } + } + `, + variables, + }, + }).catch(e => { + logError(e, variables); + }); +} + +export async function saveRecording( + { + recordingId, + title, + metadata, + }: { + recordingId: string; + title: string; + metadata?: Record; + }, + apiKey: string +) { + console.log( + `Saving ${chalk.grey.bold(title)} with recording id ${chalk.yellow.bold(recordingId)}` + ); + + if (metadata) { + cli.addLocalRecordingMetadata(recordingId, metadata); + } + + await cli.uploadRecording(recordingId, { + apiKey, + server: config.backendUrl, + strict: true, + }); + + const response = await axios({ + url: config.graphqlUrl, + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + data: { + query: ` + query GetRecordingBuildId($recordingId: UUID!) { + recording(uuid: $recordingId) { + buildId + } + } + `, + variables: { + recordingId, + }, + }, + }); + + const buildId = response.data.data.recording.buildId; + assert(typeof buildId === "string", "Expected buildId to be a string"); + + await makeReplayPublic(apiKey, recordingId); + await updateRecordingTitle(apiKey, recordingId, title); + + return { buildId }; +} + +export async function recordPlaywright( + script: (page: Page, expect: typeof expectFunction) => Promise +) { + let executablePath = config.browserPath ?? getExecutablePath() ?? undefined; + + const browserServer = await chromium.launchServer({ + env: { + ...process.env, + RECORD_ALL_CONTENT: "1", + ...(config.driverPath && { + RECORD_REPLAY_DRIVER: config.driverPath, + }), + }, + executablePath, + headless: config.headless, + }); + + if (process.env.RECORD_REPLAY_VERBOSE) { + // TODO: Always keep logs, and make them available if the recording failed. + const stderr = browserServer.process().stderr; + stderr?.addListener("data", data => { + console.debug(`[RUNTIME] ${data}`); + }); + } + + const browser = await chromium.connect(browserServer.wsEndpoint()); + const context = await browser.newContext({ + ignoreHTTPSErrors: true, + }); + const page = await context.newPage(); + try { + return await script(page, expect); + } finally { + await page.close(); + await context.close(); + await browser.close(); + await browserServer.close(); + } +} + +function findLast(arr: T[], predicate: (el: T) => boolean): T | undefined { + for (let i = arr.length - 1; i >= 0; i--) { + if (predicate(arr[i])) { + return arr[i]; + } + } +} + +export function findLastRecordingId(url?: string) { + const list = cli.listAllRecordings(); + if (!url) { + const recordingId = list[list.length - 1].id; + assert(recordingId, "No recordings found"); + return recordingId; + } + const recordingId = findLast(list, rec => rec.metadata.uri === url)?.id; + if (!recordingId) { + const recordingsLog = readRecordingsLog().split("\n").slice(-11).join("\n"); + throw Error( + `No recording found matching url "${url}" in list:\n${list + .map(rec => rec.metadata.uri) + .join("\n")}\nLast 10 lines of recordings.log:\n${recordingsLog}` + ); + } + return recordingId; +} + +function readRecordingsLog() { + let replayDir: string; + if (process.env.RECORD_REPLAY_DIRECTORY) { + replayDir = process.env.RECORD_REPLAY_DIRECTORY; + } else { + const dir = process.env.HOME || process.env.USERPROFILE; + assert(dir, "HOME or USERPROFILE environment variable must be set"); + replayDir = path.join(dir, ".replay"); + } + + const file = path.join(replayDir, "recordings.log"); + return fs.readFileSync(file, "utf8"); +} + +export const removeRecording = (recordingId: string) => cli.removeRecording(recordingId); diff --git a/packages/playwright-recorder/tsconfig.json b/packages/playwright-recorder/tsconfig.json new file mode 100644 index 00000000000..e27b616ebe4 --- /dev/null +++ b/packages/playwright-recorder/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "composite": true, + "strict": true, + "target": "ES2022", + "lib": ["es2023"], + "moduleResolution": "Node", + "downlevelIteration": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "rootDir": "./src", + "outDir": "./dist", + } +} diff --git a/scripts/perf-analysis/main.ts b/scripts/perf-analysis/main.ts new file mode 100644 index 00000000000..de3ba86940a --- /dev/null +++ b/scripts/perf-analysis/main.ts @@ -0,0 +1,120 @@ +#!/usr/bin/env ts-node + +import assert from "node:assert/strict"; +import { SimpleProtocolClient } from "@replayio/protocol"; +import WebSocket from "ws"; + +import { devtoolsObjectPreview } from "./tests/devtools-object-preview"; + +import { + findLastRecordingId, + recordPlaywright, + saveRecording, +} from "@devtools-repo/playwright-recorder"; + +const PLAYWRIGHT_TIMEOUT = 60_000; +const DISPATCH_URL = process.env.DISPATCH_ADDRESS ?? "wss://dispatch.replay.io"; +const DEVTOOL_URL = process.env.DEVTOOLS_URL ?? "https://app.replay.io"; + +const tests = [ + { + title: "devtools-object-preview", + url: `${DEVTOOL_URL}/recording/appreplayio--87db126a-82cf-4477-b244-b57a118d0b1b`, + script: devtoolsObjectPreview, + }, +]; + +async function sleep(timeoutMs: number) { + return new Promise(r => setTimeout(() => r(), timeoutMs)); +} + +async function raceForTime(timeoutMs: number, promise: Promise) { + return Promise.race([ + promise, + sleep(timeoutMs).then(() => Promise.reject(new Error(`Race timeout after ${timeoutMs}ms`))), + ]); +} + +async function main() { + const apiKey = process.env.REPLAY_API_KEY; + assert(apiKey, "REPLAY_API_KEY env var is required"); + + for (const test of tests) { + console.log(`Recording test: ${test.title}`); + await raceForTime( + PLAYWRIGHT_TIMEOUT, + recordPlaywright(async page => { + const waitForLogPromise = test.script(page); + const goToPagePromise = page.goto(test.url); + + await Promise.all([goToPagePromise, waitForLogPromise]); + }) + ); + + console.log(`Looking for recording ID...`); + const recordingId = findLastRecordingId(); + console.log(`Found Recording ID: ${recordingId}`); + + await saveRecording( + { + recordingId, + title: test.title, + }, + apiKey + ); + + console.log(`Sending performance analysis request...`); + const socket = new WebSocket(DISPATCH_URL); + const client = new SimpleProtocolClient( + socket, + { + onClose: (code, reason) => { + if (code !== 1000) { + console.log("WS closed", code, reason); + } + }, + onError: err => console.log("WS error", err), + }, + console.log + ); + try { + await client.sendCommand("Authentication.setAccessToken", { + accessToken: apiKey, + }); + const { sessionId } = await client.sendCommand("Recording.createSession", { + recordingId, + }); + await client.sendCommand( + "Session.experimentalCommand", + { + name: "runPerformanceAnalysis", + params: { + metadata: { + testTitle: test.title, + repo: process.env.GITHUB_REPOSITORY, + branch: process.env.GITHUB_REF_NAME, + pullRequest: process.env.GITHUB_PR ? parseInt(process.env.GITHUB_PR, 0) : undefined, + commit: process.env.GITHUB_SHA, + }, + }, + }, + sessionId + ); + console.log("Performance analysis requested."); + } finally { + socket.close(1000, "Done"); + } + } +} + +main().catch(err => { + console.error("Failed with error:"); + if (err instanceof Error) { + console.error(err); + } else if (err && typeof err === "object") { + console.error(JSON.stringify(err)); + } else { + console.error(err); + } + process.exit(1); +}); diff --git a/scripts/perf-analysis/tests/devtools-object-preview.ts b/scripts/perf-analysis/tests/devtools-object-preview.ts new file mode 100644 index 00000000000..6f14779e5c0 --- /dev/null +++ b/scripts/perf-analysis/tests/devtools-object-preview.ts @@ -0,0 +1,25 @@ +/* Copyright 2020-2024 Record Replay Inc. */ + +import type { Page } from "@playwright/test"; + +const MsPerSecond = 1000; + +function waitForTime(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export async function devtoolsObjectPreview(page: Page) { + await page.getByRole("button", { name: "Viewer DevTools" }).click(); + + await page.locator('[data-test-name="Message"]').hover(); + await waitForTime(MsPerSecond); + + await page.locator('[data-test-id="ConsoleMessageHoverButton"]').click(); + await waitForTime(MsPerSecond); + + await page.locator("div:nth-child(5) > .toolbar-panel-button > button").click(); + await waitForTime(MsPerSecond); + + await page.getByRole("button", { name: ": console{debug: ƒ(...r" }).click(); + await waitForTime(2 * MsPerSecond); +} diff --git a/scripts/perf-analysis/tsconfig.json b/scripts/perf-analysis/tsconfig.json new file mode 100644 index 00000000000..2734fbe041f --- /dev/null +++ b/scripts/perf-analysis/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "strict": true, + "target": "ES2022", + "lib": ["es2023"], + "moduleResolution": "Node", + "downlevelIteration": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "paths": { + "@devtools-repo/playwright-recorder": ["../../packages/playwright-recorder/src/index.ts"], + } + } +} diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json index d92e0d9e14a..0f89dc971e4 100644 --- a/tsconfig.eslint.json +++ b/tsconfig.eslint.json @@ -1,4 +1,4 @@ { "extends": "./tsconfig.json", - "include": ["src", "packages", "pages"] + "include": ["src", "packages", "pages", "scripts"] } diff --git a/tsconfig.json b/tsconfig.json index 416b10aae7a..7cc25d2d1f0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -53,6 +53,7 @@ ], "replay-next": ["packages/replay-next/*"], "@recordreplay/accordion": ["packages/accordion/index.tsx"], + "@devtools-repo/playwright-recorder": ["packages/playwright-recorder/src/index.ts"], "components": ["packages/design/index.ts"], "components/*": ["packages/design/*"], "icons": ["packages/icons/index.tsx"], @@ -94,4 +95,4 @@ "packages/e2e-tests/scripts/*.ts", ], "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"] -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index 739595f050d..5868daec623 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1785,6 +1785,18 @@ __metadata: languageName: node linkType: hard +"@devtools-repo/playwright-recorder@*, @devtools-repo/playwright-recorder@workspace:*, @devtools-repo/playwright-recorder@workspace:packages/playwright-recorder": + version: 0.0.0-use.local + resolution: "@devtools-repo/playwright-recorder@workspace:packages/playwright-recorder" + dependencies: + "@playwright/test": ^1.37.0 + "@replayio/replay": ^0.22.4 + axios: ^0.21.1 + chalk: ^4 + strip-ansi: ^6.0.0 + languageName: unknown + linkType: soft + "@discoveryjs/json-ext@npm:^0.5.0": version: 0.5.7 resolution: "@discoveryjs/json-ext@npm:0.5.7" @@ -1822,6 +1834,174 @@ __metadata: languageName: node linkType: hard +"@esbuild/aix-ppc64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/aix-ppc64@npm:0.23.1" + conditions: os=aix & cpu=ppc64 + languageName: node + linkType: hard + +"@esbuild/android-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/android-arm64@npm:0.23.1" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/android-arm@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/android-arm@npm:0.23.1" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + +"@esbuild/android-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/android-x64@npm:0.23.1" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/darwin-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/darwin-arm64@npm:0.23.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/darwin-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/darwin-x64@npm:0.23.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/freebsd-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/freebsd-arm64@npm:0.23.1" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/freebsd-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/freebsd-x64@npm:0.23.1" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/linux-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-arm64@npm:0.23.1" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/linux-arm@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-arm@npm:0.23.1" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@esbuild/linux-ia32@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-ia32@npm:0.23.1" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + +"@esbuild/linux-loong64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-loong64@npm:0.23.1" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + +"@esbuild/linux-mips64el@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-mips64el@npm:0.23.1" + conditions: os=linux & cpu=mips64el + languageName: node + linkType: hard + +"@esbuild/linux-ppc64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-ppc64@npm:0.23.1" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@esbuild/linux-riscv64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-riscv64@npm:0.23.1" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + +"@esbuild/linux-s390x@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-s390x@npm:0.23.1" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + +"@esbuild/linux-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/linux-x64@npm:0.23.1" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/netbsd-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/netbsd-x64@npm:0.23.1" + conditions: os=netbsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/openbsd-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/openbsd-arm64@npm:0.23.1" + conditions: os=openbsd & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/openbsd-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/openbsd-x64@npm:0.23.1" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/sunos-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/sunos-x64@npm:0.23.1" + conditions: os=sunos & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/win32-arm64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/win32-arm64@npm:0.23.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/win32-ia32@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/win32-ia32@npm:0.23.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@esbuild/win32-x64@npm:0.23.1": + version: 0.23.1 + resolution: "@esbuild/win32-x64@npm:0.23.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@eslint/eslintrc@npm:^1.3.0": version: 1.3.0 resolution: "@eslint/eslintrc@npm:1.3.0" @@ -8238,6 +8418,89 @@ __metadata: languageName: node linkType: hard +"esbuild@npm:~0.23.0": + version: 0.23.1 + resolution: "esbuild@npm:0.23.1" + dependencies: + "@esbuild/aix-ppc64": 0.23.1 + "@esbuild/android-arm": 0.23.1 + "@esbuild/android-arm64": 0.23.1 + "@esbuild/android-x64": 0.23.1 + "@esbuild/darwin-arm64": 0.23.1 + "@esbuild/darwin-x64": 0.23.1 + "@esbuild/freebsd-arm64": 0.23.1 + "@esbuild/freebsd-x64": 0.23.1 + "@esbuild/linux-arm": 0.23.1 + "@esbuild/linux-arm64": 0.23.1 + "@esbuild/linux-ia32": 0.23.1 + "@esbuild/linux-loong64": 0.23.1 + "@esbuild/linux-mips64el": 0.23.1 + "@esbuild/linux-ppc64": 0.23.1 + "@esbuild/linux-riscv64": 0.23.1 + "@esbuild/linux-s390x": 0.23.1 + "@esbuild/linux-x64": 0.23.1 + "@esbuild/netbsd-x64": 0.23.1 + "@esbuild/openbsd-arm64": 0.23.1 + "@esbuild/openbsd-x64": 0.23.1 + "@esbuild/sunos-x64": 0.23.1 + "@esbuild/win32-arm64": 0.23.1 + "@esbuild/win32-ia32": 0.23.1 + "@esbuild/win32-x64": 0.23.1 + dependenciesMeta: + "@esbuild/aix-ppc64": + optional: true + "@esbuild/android-arm": + optional: true + "@esbuild/android-arm64": + optional: true + "@esbuild/android-x64": + optional: true + "@esbuild/darwin-arm64": + optional: true + "@esbuild/darwin-x64": + optional: true + "@esbuild/freebsd-arm64": + optional: true + "@esbuild/freebsd-x64": + optional: true + "@esbuild/linux-arm": + optional: true + "@esbuild/linux-arm64": + optional: true + "@esbuild/linux-ia32": + optional: true + "@esbuild/linux-loong64": + optional: true + "@esbuild/linux-mips64el": + optional: true + "@esbuild/linux-ppc64": + optional: true + "@esbuild/linux-riscv64": + optional: true + "@esbuild/linux-s390x": + optional: true + "@esbuild/linux-x64": + optional: true + "@esbuild/netbsd-x64": + optional: true + "@esbuild/openbsd-arm64": + optional: true + "@esbuild/openbsd-x64": + optional: true + "@esbuild/sunos-x64": + optional: true + "@esbuild/win32-arm64": + optional: true + "@esbuild/win32-ia32": + optional: true + "@esbuild/win32-x64": + optional: true + bin: + esbuild: bin/esbuild + checksum: 0413c3b9257327fb598427688b7186ea335bf1693746fe5713cc93c95854d6388b8ed4ad643fddf5b5ace093f7dcd9038dd58e087bf2da1f04dfb4c5571660af + languageName: node + linkType: hard + "escalade@npm:^3.1.1": version: 3.1.1 resolution: "escalade@npm:3.1.1" @@ -9411,6 +9674,16 @@ __metadata: languageName: node linkType: hard +"fsevents@npm:~2.3.3": + version: 2.3.3 + resolution: "fsevents@npm:2.3.3" + dependencies: + node-gyp: latest + checksum: 11e6ea6fea15e42461fc55b4b0e4a0a3c654faa567f1877dbd353f39156f69def97a69936d1746619d656c4b93de2238bf731f6085a03a50cabf287c9d024317 + conditions: os=darwin + languageName: node + linkType: hard + "fsevents@patch:fsevents@2.3.2#~builtin, fsevents@patch:fsevents@^2.3.2#~builtin, fsevents@patch:fsevents@~2.3.2#~builtin": version: 2.3.2 resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=18f3a7" @@ -9420,6 +9693,15 @@ __metadata: languageName: node linkType: hard +"fsevents@patch:fsevents@~2.3.3#~builtin": + version: 2.3.3 + resolution: "fsevents@patch:fsevents@npm%3A2.3.3#~builtin::version=2.3.3&hash=18f3a7" + dependencies: + node-gyp: latest + conditions: os=darwin + languageName: node + linkType: hard + "function-bind@npm:^1.1.1": version: 1.1.1 resolution: "function-bind@npm:1.1.1" @@ -9450,13 +9732,15 @@ __metadata: version: 0.0.0-use.local resolution: "functional-tests@workspace:packages/e2e-tests" dependencies: + "@devtools-repo/playwright-recorder": "*" "@playwright/test": ^1.37.0 "@replayio/playwright": 3.0.0-alpha.14 + axios: ^0.21.1 chalk: ^4 cli-spinners: ^2.7.0 cypress: ^12.5.1 strip-ansi: ^6.0.0 - ts-node: ^10.7.0 + ts-node: ^10.9.2 ws: ^7.4.6 yargs: ^17.6.0 languageName: unknown @@ -9560,6 +9844,15 @@ __metadata: languageName: node linkType: hard +"get-tsconfig@npm:^4.7.5": + version: 4.8.1 + resolution: "get-tsconfig@npm:4.8.1" + dependencies: + resolve-pkg-maps: ^1.0.0 + checksum: 12df01672e691d2ff6db8cf7fed1ddfef90ed94a5f3d822c63c147a26742026d582acd86afcd6f65db67d809625d17dd7f9d34f4d3f38f69bc2f48e19b2bdd5b + languageName: node + linkType: hard + "get-value@npm:^2.0.3": version: 2.0.6 resolution: "get-value@npm:2.0.6" @@ -14885,6 +15178,7 @@ __metadata: "@babel/preset-typescript": ^7.15.0 "@babel/types": ^7.14.8 "@bcoe/v8-coverage": ^0.2.3 + "@devtools-repo/playwright-recorder": "workspace:*" "@ffmpeg-installer/ffmpeg": ^1.1.0 "@headlessui/react": ^1.4.3 "@heroicons/react": ^1.0.6 @@ -15034,8 +15328,9 @@ __metadata: stylelint-config-prettier: ^9.0.5 suspense: ^0.0.53 tailwindcss: ^3.2.4 - ts-node: ^10.7.0 + ts-node: ^10.9.2 tsconfig-paths: ^3.14.1 + tsx: ^4.19.2 typescript: ^5.4.2 use-context-menu: 0.4.13 utf-8-validate: ^5.0.8 @@ -15293,6 +15588,13 @@ __metadata: languageName: node linkType: hard +"resolve-pkg-maps@npm:^1.0.0": + version: 1.0.0 + resolution: "resolve-pkg-maps@npm:1.0.0" + checksum: 1012afc566b3fdb190a6309cc37ef3b2dcc35dff5fa6683a9d00cd25c3247edfbc4691b91078c97adc82a29b77a2660c30d791d65dab4fc78bfc473f60289977 + languageName: node + linkType: hard + "resolve.exports@npm:^1.1.0": version: 1.1.0 resolution: "resolve.exports@npm:1.1.0" @@ -16977,9 +17279,9 @@ __metadata: languageName: node linkType: hard -"ts-node@npm:^10.7.0": - version: 10.8.1 - resolution: "ts-node@npm:10.8.1" +"ts-node@npm:^10.9.2": + version: 10.9.2 + resolution: "ts-node@npm:10.9.2" dependencies: "@cspotcode/source-map-support": ^0.8.0 "@tsconfig/node10": ^1.0.7 @@ -17011,7 +17313,7 @@ __metadata: ts-node-script: dist/bin-script.js ts-node-transpile-only: dist/bin-transpile.js ts-script: dist/bin-script-deprecated.js - checksum: 7d1aa7aa3ae1c0459c4922ed0dbfbade442cfe0c25aebaf620cdf1774f112c8d7a9b14934cb6719274917f35b2c503ba87bcaf5e16a0d39ba0f68ce3e7728363 + checksum: fde256c9073969e234526e2cfead42591b9a2aec5222bac154b0de2fa9e4ceb30efcd717ee8bc785a56f3a119bdd5aa27b333d9dbec94ed254bd26f8944c67ac languageName: node linkType: hard @@ -17062,6 +17364,22 @@ __metadata: languageName: node linkType: hard +"tsx@npm:^4.19.2": + version: 4.19.2 + resolution: "tsx@npm:4.19.2" + dependencies: + esbuild: ~0.23.0 + fsevents: ~2.3.3 + get-tsconfig: ^4.7.5 + dependenciesMeta: + fsevents: + optional: true + bin: + tsx: dist/cli.mjs + checksum: 7f9f1b338a73297725a9217cedaaad862f7c81d5264093c74b98a71491ad5413b11248d604c0e650f4f7da6f365249f1426fdb58a1325ab9e15448156b1edff6 + languageName: node + linkType: hard + "tty@npm:1.0.1": version: 1.0.1 resolution: "tty@npm:1.0.1"