diff --git a/.buildkite/runtime-tests.yml b/.buildkite/runtime-tests.yml index c6696a9cf65..fb5c2c180da 100644 --- a/.buildkite/runtime-tests.yml +++ b/.buildkite/runtime-tests.yml @@ -1,11 +1,10 @@ steps: - label: "Runtime e2e tests (linux-x86_64)" + if: (build.env("OS") == "linux" && build.env("ARCH") == "x86_64") || (build.env("OS") == null && build.env("ARCH") == null) timeout_in_minutes: 60 - key: "runtime-fe-e2e-tests-linux_x86_64" commands: - "mkdir -p /mnt/data/replay" - - "be_cmd install_yarn -- npm i -g yarn" - - "be_cmd node_deps -- yarn install" + - "be_cmd node_deps -- npx yarn install" - "be_cmd run_tests -- ts-node ./packages/e2e-tests/scripts/buildkite_run_fe_tests_from_artifact-linux-x86_64.ts" agents: - "deploy=true" @@ -29,7 +28,7 @@ steps: RUNTIME_TEAM_API_KEY: runtime-replay-api-key BUILDEVENT_APIKEY: honeycomb-api-key BUILDEVENT_BUILDKITE_API_TOKEN: buildkite-api-token-honeycomb-build-events - - replayio/buildevents#76b6f57: ~ + - replayio/buildevents#0c2adb2: ~ - "ssh://git@github.com/replayio/fly-buildkite-plugin.git#v0.77": # This image's source code is here https://github.com/replayio/backend-e2e-base-image image: "registry.fly.io/buildkite-backend-e2e-tests:v14" @@ -48,9 +47,37 @@ steps: AUTOMATED_TEST_SECRET: AUTOMATED_TEST_SECRET RECORD_REPLAY_API_KEY: RECORD_REPLAY_API_KEY RUNTIME_TEAM_API_KEY: RUNTIME_TEAM_API_KEY - env: DISPLAY: ":1" PLAYWRIGHT_HEADLESS: "true" PLAYWRIGHT_CHROMIUM: "true" RECORD_REPLAY_DIRECTORY: "/mnt/data/replay" + + - label: "Runtime e2e tests (macos-arm64)" + if: build.env("OS") == "macos" && build.env("ARCH") == "arm64" + timeout_in_minutes: 60 + commands: + - "be_cmd node_deps -- npx yarn install" + - "be_cmd run_tests -- ts-node ./packages/e2e-tests/scripts/buildkite_run_fe_tests_from_artifact-macos-arm64.ts" + agents: + - "os=macos" + - "arch=arm64" + - "queue=runtime" + - "runtimeType=test" + plugins: + - seek-oss/aws-sm#v2.3.1: + region: us-east-2 + env: + BUILDKITE_AGENT_TOKEN: "prod/buildkite-agent-token" + AUTOMATED_TEST_SECRET: "graphql-automated-test-secret" + RECORD_REPLAY_API_KEY: "runtime-record-replay-api-key" + HASURA_ADMIN_SECRET: "prod/hasura-admin-secret" + RUNTIME_TEAM_API_KEY: runtime-replay-api-key + BUILDEVENT_APIKEY: honeycomb-api-key + BUILDEVENT_BUILDKITE_API_TOKEN: buildkite-api-token-honeycomb-build-events + - replayio/buildevents#0c2adb2: ~ + env: + DISPLAY: ":1" + PLAYWRIGHT_HEADLESS: "true" + PLAYWRIGHT_CHROMIUM: "true" + ARCHITECTURE: "arm64" diff --git a/packages/e2e-tests/scripts/build_products.ts b/packages/e2e-tests/scripts/build_products.ts index 852cf646ce8..4b035bfadfc 100644 --- a/packages/e2e-tests/scripts/build_products.ts +++ b/packages/e2e-tests/scripts/build_products.ts @@ -45,9 +45,9 @@ export function install_build_products(RUNTIME_BUILD_ID, PLATFORM, ARCHITECTURE) // Set Chrome binary path based on OS let CHROME_BINARY; - if (PLATFORM === "linux") { + if (os.platform() === "linux") { CHROME_BINARY = path.join(TMP_DIR, "build", "replay-chromium", "chrome"); - } else if (PLATFORM === "darwin") { + } else if (os.platform() === "darwin") { CHROME_BINARY = path.join( TMP_DIR, "build", @@ -56,10 +56,10 @@ export function install_build_products(RUNTIME_BUILD_ID, PLATFORM, ARCHITECTURE) "MacOS", "Chromium" ); - } else if (PLATFORM === "win32") { + } else if (os.platform() === "win32") { CHROME_BINARY = path.join(TMP_DIR, "build", "replay-chromium", "chrome.exe"); } else { - throw new Error(`Unsupported platform: ${PLATFORM}`); + throw new Error(`Unsupported platform: ${os.platform()}`); } return CHROME_BINARY; } diff --git a/packages/e2e-tests/scripts/buildkite_run_fe_tests.ts b/packages/e2e-tests/scripts/buildkite_run_fe_tests.ts index 929f81d9a60..208274f0871 100644 --- a/packages/e2e-tests/scripts/buildkite_run_fe_tests.ts +++ b/packages/e2e-tests/scripts/buildkite_run_fe_tests.ts @@ -1,6 +1,8 @@ /* Copyright 2024 Record Replay Inc. */ import { exec, execSync } from "child_process"; +import http from "http"; +import os from "os"; import path from "path"; import chalk from "chalk"; import difference from "lodash/difference"; @@ -138,7 +140,55 @@ function githubUrlToRepository(url) { return url?.replace(/.*github.com[:\/](.*)\.git/, "$1"); } -export default function run_fe_tests( +function waitForHTTPStatus( + url: string, + statusCode: number = 200, // by default we wait for OK + timeoutMs = 15000, // keep waiting for 15s total + retryTime = 1000 // retry after 1s +): Promise { + const startTime = Date.now(); + return new Promise((resolve, reject) => { + function attemptConnection() { + const request = http.get(url, res => { + if (res.statusCode === statusCode) { + resolve(); + return; + } + + console.error(`Server is up, but responded with status code ${res.statusCode}`); + + if (Date.now() - startTime >= timeoutMs) { + reject( + new Error(`Server did not respond with 200 OK within ${timeoutMs / 1000} seconds`) + ); + return; + } + + setTimeout(attemptConnection, 1000); // Retry after 1 second + }); + + request.on("error", (err: Error) => { + if (err["code"] === "ECONNREFUSED") { + if (Date.now() - startTime >= timeoutMs) { + reject(new Error(`Failed to connect to the server within ${timeoutMs / 1000} seconds`)); + return; + } + + setTimeout(attemptConnection, 1000); // Retry after 1 second if connection was refused + return; + } + + reject(new Error(`Error while trying to connect to the server: ${err.message}`)); + }); + + request.end(); + } + + attemptConnection(); + }); +} + +export default async function run_fe_tests( CHROME_BINARY_PATH, runInCI = true, nWorkers = 4, @@ -150,7 +200,7 @@ export default function run_fe_tests( console.group("START"); console.time("START time"); - const envWrapper = runInCI ? "xvfb-run" : ""; + const envWrapper = runInCI && os.platform() === "linux" ? "xvfb-run" : ""; const TestRootPath = path.join( process.env.REPLAY_DIR ? path.join(process.env.REPLAY_DIR, "devtools") : ".", "packages/e2e-tests" @@ -191,7 +241,7 @@ export default function run_fe_tests( }); // Start the webserver. - webProc = exec("yarn dev", (error, stdout, stderr) => { + webProc = exec("npx yarn dev", (error, stdout, stderr) => { if (error) { console.error(`yarn dev exec ERROR: ${error}`); } @@ -205,8 +255,10 @@ export default function run_fe_tests( stdio: "inherit", }); - // Wait a little, to let the yarn dev server start up. - execSync("sleep 2"); + // make sure the dev server is up and running. + console.log("waiting for dev server to start up"); + await waitForHTTPStatus("http://localhost:8080/"); + console.log("dev server up, continuing with test"); } try { diff --git a/packages/e2e-tests/scripts/buildkite_run_fe_tests_from_artifact-macos-arm64.ts b/packages/e2e-tests/scripts/buildkite_run_fe_tests_from_artifact-macos-arm64.ts new file mode 100644 index 00000000000..9270baf28bf --- /dev/null +++ b/packages/e2e-tests/scripts/buildkite_run_fe_tests_from_artifact-macos-arm64.ts @@ -0,0 +1,5 @@ +/* Copyright 2024 Record Replay Inc. */ + +import run_fe_tests_from_artifact from "./buildkite_run_fe_tests_from_artifact"; + +run_fe_tests_from_artifact("macOS", "arm64");