From 07c4f1cfad1de03c505843168da0057c84ae48e3 Mon Sep 17 00:00:00 2001 From: OJ Kwon <1210596+kwonoj@users.noreply.github.com> Date: Wed, 29 Mar 2023 09:42:02 -0700 Subject: [PATCH] ci(workflow): run integration subset daily (#4367) ### Description - closes WEB-803 This PR expands current daily integration to have one more job, that runs only known set of files. We'll use this to measure if Turbopack starts to flaky or not over time. Currently it contains just a few set of files. --- .../actions/next-integration-stat/index.js | 80 +++++++++++++++---- .../next-integration-stat/src/index.ts | 67 +++++++++++++--- .github/workflows/nextjs-integration-test.yml | 48 ++++++++++- 3 files changed, 167 insertions(+), 28 deletions(-) diff --git a/.github/actions/next-integration-stat/index.js b/.github/actions/next-integration-stat/index.js index 9e13dface1e3c..9aa940d004e8d 100644 --- a/.github/actions/next-integration-stat/index.js +++ b/.github/actions/next-integration-stat/index.js @@ -16356,21 +16356,71 @@ nextjsVersion, ref: sha, }; - const failedJobResults = fullJobLogsFromWorkflow.reduce( - (acc, { logs, job }) => { - // Split logs per each test suites, exclude if it's arbitrary log does not contain test data - const splittedLogs = logs - .split("NEXT_INTEGRATION_TEST: true") - .filter((log) => log.includes("--test output start--")); - // Iterate each chunk of logs, find out test name and corresponding test data - const failedTestResultsData = collectFailedTestResults( - splittedLogs, - job - ); - return acc.concat(failedTestResultsData); - }, - [] - ); + const [failedJobResults, flakyMonitorJobResults] = + fullJobLogsFromWorkflow.reduce( + (acc, { logs, job }) => { + var _a, _b, _c; + // Split logs per each test suites, exclude if it's arbitrary log does not contain test data + const splittedLogs = logs + .split("NEXT_INTEGRATION_TEST: true") + .filter((log) => log.includes("--test output start--")); + // There is a job named `Next.js integration test (FLAKY_SUBSET)`, which we runs known subset of the tests + // that are flaky. If given job is flaky subset monitoring, we are interested in to grab test results only. + // [NOTE]: this is similar to `collectFailedTestResults`, but not identical: collectFailedTestResults intentionally + // skips if test success, while in here we want to collect all the test results. + if (job.name.includes("FLAKY_SUBSET")) { + const splittedLogs = logs.split("--test output start--"); + const ret = []; + let logLine = splittedLogs.shift(); + while (logLine) { + try { + const testData = + (_c = + (_b = + (_a = + logLine === null || logLine === void 0 + ? void 0 + : logLine + .split("--test output start--") + .pop()) === null || _a === void 0 + ? void 0 + : _a.split("--test output end--")) === null || + _b === void 0 + ? void 0 + : _b.shift()) === null || _c === void 0 + ? void 0 + : _c.trim(); + ret.push({ + job: job.name, + // We may able to parse test suite name, but skipping for now + name: "empty", + data: JSON.parse(testData), + }); + } catch (_) { + console.log("Failed to parse flaky subset test results", { + logs, + }); + } finally { + logLine = splittedLogs.shift(); + } + } + acc[1] = acc[1].concat(ret); + } else { + // Iterate each chunk of logs, find out test name and corresponding test data + const failedTestResultsData = collectFailedTestResults( + splittedLogs, + job + ); + acc[0] = acc[0].concat(failedTestResultsData); + } + return acc; + }, + [[], []] + ); + console.log(`Flakyness test subset results`, { + flakyMonitorJobResults, + }); + testResultManifest.flakyMonitorJobResults = flakyMonitorJobResults; testResultManifest.result = failedJobResults; // Collect all test results into single manifest to store into file. This'll allow to upload / compare test results // across different runs. diff --git a/.github/actions/next-integration-stat/src/index.ts b/.github/actions/next-integration-stat/src/index.ts index ac677eb0560d0..4cd05721ef10d 100644 --- a/.github/actions/next-integration-stat/src/index.ts +++ b/.github/actions/next-integration-stat/src/index.ts @@ -65,6 +65,7 @@ interface TestResultManifest { nextjsVersion: string; ref: string; result: Array; + flakyMonitorJobResults: Array; } // A comment marker to identify the comment created by this action. @@ -470,21 +471,63 @@ async function getFailedJobResults( ref: sha, } as any; - const failedJobResults = fullJobLogsFromWorkflow.reduce( - (acc, { logs, job }) => { - // Split logs per each test suites, exclude if it's arbitrary log does not contain test data - const splittedLogs = logs - .split("NEXT_INTEGRATION_TEST: true") - .filter((log) => log.includes("--test output start--")); + const [failedJobResults, flakyMonitorJobResults] = + fullJobLogsFromWorkflow.reduce( + (acc, { logs, job }) => { + // Split logs per each test suites, exclude if it's arbitrary log does not contain test data + const splittedLogs = logs + .split("NEXT_INTEGRATION_TEST: true") + .filter((log) => log.includes("--test output start--")); + + // There is a job named `Next.js integration test (FLAKY_SUBSET)`, which we runs known subset of the tests + // that are flaky. If given job is flaky subset monitoring, we are interested in to grab test results only. + // [NOTE]: this is similar to `collectFailedTestResults`, but not identical: collectFailedTestResults intentionally + // skips if test success, while in here we want to collect all the test results. + if (job.name.includes("FLAKY_SUBSET")) { + const splittedLogs = logs.split("--test output start--"); + const ret = []; + let logLine = splittedLogs.shift(); + while (logLine) { + try { + const testData = logLine + ?.split("--test output start--") + .pop() + ?.split("--test output end--") + ?.shift() + ?.trim()!; + + ret.push({ + job: job.name, + // We may able to parse test suite name, but skipping for now + name: "empty", + data: JSON.parse(testData), + }); + } catch (_) { + console.log("Failed to parse flaky subset test results", { + logs, + }); + } finally { + logLine = splittedLogs.shift(); + } + } + acc[1] = acc[1].concat(ret); + } else { + // Iterate each chunk of logs, find out test name and corresponding test data + const failedTestResultsData = collectFailedTestResults( + splittedLogs, + job + ); + acc[0] = acc[0].concat(failedTestResultsData); + } - // Iterate each chunk of logs, find out test name and corresponding test data - const failedTestResultsData = collectFailedTestResults(splittedLogs, job); + return acc; + }, + [[], []] as [Array, Array] + ); - return acc.concat(failedTestResultsData); - }, - [] as Array - ); + console.log(`Flakyness test subset results`, { flakyMonitorJobResults }); + testResultManifest.flakyMonitorJobResults = flakyMonitorJobResults; testResultManifest.result = failedJobResults; // Collect all test results into single manifest to store into file. This'll allow to upload / compare test results diff --git a/.github/workflows/nextjs-integration-test.yml b/.github/workflows/nextjs-integration-test.yml index d86aacfeeeb8c..69c95ec531cf1 100644 --- a/.github/workflows/nextjs-integration-test.yml +++ b/.github/workflows/nextjs-integration-test.yml @@ -225,10 +225,56 @@ jobs: RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testIntegration / Group ${{ matrix.group }} NEXT_INTEGRATION_TEST: true + test_flaky_subset: + # This job name is being used in github action to collect test results. Do not change it, or should update + # ./.github/actions/next-integration-stat to match the new name. + name: Next.js integration test (FLAKY_SUBSET) + runs-on: ubuntu-latest + needs: [setup_nextjs] + env: + # Enabling backtrace will makes snapshot tests fail + RUST_BACKTRACE: 0 + NEXT_TELEMETRY_DISABLED: 1 + # Path to the next-dev binary located in **docker container** image. + NEXT_DEV_BIN: /work/next-dev + FAILED_TEST_LIST_PATH: /work/integration-test-data/test-results/main/failed-test-path-list.json + # Glob pattern to run specific tests with --turbo. + NEXT_DEV_TEST_GLOB: "*" + # pnpm version should match to what upstream next.js uses + PNPM_VERSION: 7.24.3 + # List of test files to run that expected to pass always. Whole test suite should pass. + TEST_FILES_LIST: | + test/development/acceptance-app/dynamic-error.test.ts \ + test/development/acceptance-app/unsupported-app-features.test.ts \ + test/development/acceptance-app/ReactRefresh.test.ts + + strategy: + matrix: + node: [16] + + steps: + - uses: actions/cache/restore@v3 + id: restore-build + with: + path: ./* + key: ${{ inputs.version }}-${{ github.sha }} + fail-on-cache-miss: true + + - run: | + docker run -i -v nextjs-test-volume:/volume --rm loomchild/volume-backup restore < volume.tar.bz2 + docker run --rm --mount src=nextjs-test-volume,dst=/work mcr.microsoft.com/playwright:v1.28.1-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && __INTERNAL_CUSTOM_TURBOPACK_BINARY=${NEXT_DEV_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_TEST_CONTINUE_ON_ERROR=TRUE NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev xvfb-run node run-tests.js --type development -c 1 $TEST_FILES_LIST >> /proc/1/fd/1" + name: Run test/development + # This should not fail, but if fails collect result to update dashboard. + continue-on-error: true + env: + # marker to parse log output, do not delete / change. + NEXT_INTEGRATION_TEST: true + # Collect integration test results from execute_tests, # Store it as github artifact for next step to consume. collect_nextjs_integration_stat: - needs: [test_dev, test_dev_e2e, test_cna, test_integration] + needs: + [test_dev, test_dev_e2e, test_cna, test_integration, test_flaky_subset] name: Next.js integration test status report runs-on: ubuntu-latest permissions: