From 191ec234f580497796c046983e944f22b7d1f319 Mon Sep 17 00:00:00 2001 From: Citrine Date: Sun, 21 Jan 2024 12:43:25 -0800 Subject: [PATCH 1/8] feat(test_framework): Add workflow scripts to show test results in PRs The trigger for running tests is disabled until we want to enable them. In order to run, these scripts (like other workflow scripts) must be in the main branch (master). --- .github/workflows/process_test_results.yml | 41 ++++++++++++++++++++++ .github/workflows/run_tests.yml | 34 ++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 .github/workflows/process_test_results.yml create mode 100644 .github/workflows/run_tests.yml diff --git a/.github/workflows/process_test_results.yml b/.github/workflows/process_test_results.yml new file mode 100644 index 00000000000..55a7604eff5 --- /dev/null +++ b/.github/workflows/process_test_results.yml @@ -0,0 +1,41 @@ +name: Process Test Results + +on: + workflow_run: + workflows: ["Run Tests"] + types: + - completed + +permissions: {} + +jobs: + process-test-results: + name: Process Test Results + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + + permissions: + checks: write + + # needed unless run with comment_mode: off + pull-requests: write + + # required by download step to access artifacts API + actions: read + + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + time_unit: milliseconds + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + files: "artifacts/Test Results/*.json" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 00000000000..19a7d0a603b --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,34 @@ +name: Run Tests + +on: +# pull_request + workflow_dispatch: + +jobs: + upload-event_file: + name: Upload Event File + runs-on: ubuntu-latest + steps: + - name: Upload Event File + uses: actions/upload-artifact@v4 + with: + name: Event File + path: ${{ github.event_path }} + + run-tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Run Tests + run: docker-compose -f tools/headless_testing/docker-compose.yml up + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: Test Results + path: | + tools/headless_testing/testlog/results.json From 39c29caa6c3a2d21e150afcd13178eb40921e6cf Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:27:04 +0100 Subject: [PATCH 2/8] Remove globallos, set Supreme Isthmus to 1.8. --- tools/headless_testing/download-maps.sh | 2 +- tools/headless_testing/startscript.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/headless_testing/download-maps.sh b/tools/headless_testing/download-maps.sh index 8901c70e6f9..131eb233f66 100644 --- a/tools/headless_testing/download-maps.sh +++ b/tools/headless_testing/download-maps.sh @@ -1,3 +1,3 @@ #!/bin/bash -engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.6.4" +engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.8" diff --git a/tools/headless_testing/startscript.txt b/tools/headless_testing/startscript.txt index e41b9cf9f84..de123d076d9 100644 --- a/tools/headless_testing/startscript.txt +++ b/tools/headless_testing/startscript.txt @@ -1,6 +1,6 @@ [GAME] { - MapName=Supreme Isthmus v1.6.4; + MapName=Supreme Isthmus v1.8; GameType=Beyond All Reason $VERSION; GameStartDelay=0; StartPosType=0; @@ -10,7 +10,7 @@ FixedRNGSeed = 1; [MODOPTIONS] { - debugcommands=1:cheat|2:godmode|3:globallos|30:runtestsheadless; + debugcommands=1:cheat|2:godmode|30:runtestsheadless; deathmode=neverend; } [ALLYTEAM0] From 2b109051270c2e9f8882e1ecc179552ffcaf5b2a Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:32:20 +0100 Subject: [PATCH 3/8] Add skipped tests, cleanup error message. --- common/testing/mocha_json_reporter.lua | 33 ++++++++++++++++++++++---- luaui/Widgets/dbg_test_runner.lua | 1 + 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/testing/mocha_json_reporter.lua b/common/testing/mocha_json_reporter.lua index 56b4570fd4e..a4d9ad40a47 100644 --- a/common/testing/mocha_json_reporter.lua +++ b/common/testing/mocha_json_reporter.lua @@ -8,11 +8,13 @@ function MochaJSONReporter:new() local obj = { totalTests = 0, totalPasses = 0, + totalSkipped = 0, totalFailures = 0, startTime = nil, endTime = nil, duration = nil, - tests = {} + tests = {}, + skipped = {} } setmetatable(obj, self) self.__index = self @@ -28,21 +30,37 @@ function MochaJSONReporter:endTests(duration) self.duration = duration end -function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage) +function MochaJSONReporter:extractError(text) + local errorIndex = text:match'^%[string "[%p%a%s]*%"]:[%d]+:().*' + if errorIndex and errorIndex > 0 then + text = text:sub(errorIndex + 1) + return text + end + errorIndex = text:match'^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*' + if errorIndex and errorIndex > 0 then + text = text:sub(errorIndex) + end + return text +end + +function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage) local result = { title = label, fullTitle = label, file = filePath, duration = duration, } - if success then + if skipped then + self.totalSkipped = self.totalSkipped + 1 + result.err = {} + elseif success then self.totalPasses = self.totalPasses + 1 result.err = {} else self.totalFailures = self.totalFailures + 1 if errorMessage ~= nil then result.err = { - message = errorMessage, + message = self:extractError(errorMessage), stack = errorMessage } else @@ -54,6 +72,9 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM self.totalTests = self.totalTests + 1 self.tests[#(self.tests) + 1] = result + if skipped then + self.skipped[#(self.skipped) + 1] = {fullTitle = label} + end end function MochaJSONReporter:report(filePath) @@ -63,12 +84,14 @@ function MochaJSONReporter:report(filePath) ["tests"] = self.totalTests, ["passes"] = self.totalPasses, ["pending"] = 0, + ["skipped"] = self.totalSkipped, ["failures"] = self.totalFailures, ["start"] = formatTimestamp(self.startTime), ["end"] = formatTimestamp(self.endTime), ["duration"] = self.duration }, - ["tests"] = self.tests + ["tests"] = self.tests, + ["pending"] = self.skipped } local encoded = Json.encode(output) diff --git a/luaui/Widgets/dbg_test_runner.lua b/luaui/Widgets/dbg_test_runner.lua index 2c4bada6f2d..10b6fd8eddf 100644 --- a/luaui/Widgets/dbg_test_runner.lua +++ b/luaui/Widgets/dbg_test_runner.lua @@ -82,6 +82,7 @@ local function logTestResult(testResult) testResult.label, testResult.filename, (testResult.result == TestResults.TEST_RESULT.PASS), + (testResult.result == TestResults.TEST_RESULT.SKIP), testResult.milliseconds, testResult.error ) From 4b560e65c0b6d49eddb6f8d088930fb53915a553 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:33:55 +0100 Subject: [PATCH 4/8] Run an extra test on headless checking for errors at infolog.txt. --- common/testing/infologtest.lua | 34 +++++++++++++++++++++++++++++++ luaui/Widgets/dbg_test_runner.lua | 6 ++++++ 2 files changed, 40 insertions(+) create mode 100644 common/testing/infologtest.lua diff --git a/common/testing/infologtest.lua b/common/testing/infologtest.lua new file mode 100644 index 00000000000..f9ccae903eb --- /dev/null +++ b/common/testing/infologtest.lua @@ -0,0 +1,34 @@ +-- 'hidden' test checking infolog.txt for errors, used by headless runs. + +local maxErrors = 10 + +local function skipErrors(line) + if string.find(line, 'Could not finalize projectile-texture atlas', nil, true) then + return true + end +end + +local function infologTest() + local errors = {} + local infolog = VFS.LoadFile("infolog.txt") + if infolog then + local fileLines = string.lines(infolog) + for i, line in ipairs(fileLines) do + if string.find(line, 'Error:', nil, true) and not skipErrors(line) then + errors[#errors+1] = line + if #errors > maxErrors then + return errors + end + end + end + end + return errors +end + + +function test() + local errors = infologTest() + if #errors > 0 then + error(table.concat(errors, "\n"), 0) + end +end diff --git a/luaui/Widgets/dbg_test_runner.lua b/luaui/Widgets/dbg_test_runner.lua index 10b6fd8eddf..c86317031bb 100644 --- a/luaui/Widgets/dbg_test_runner.lua +++ b/luaui/Widgets/dbg_test_runner.lua @@ -41,6 +41,7 @@ local config = { } local testReporter = nil +local headless = false -- utils -- ===== @@ -71,6 +72,7 @@ local function logEndTests(duration) testReporter:endTests(duration) testReporter:report(config.testResultsFilePath) + headless = false end local function logTestResult(testResult) @@ -142,6 +144,9 @@ local function findAllTestFiles(patterns) result[#result + 1] = testFileInfo end end + if headless then + result[#result+1] = {label="infolog", filename="common/testing/infologtest.lua"} + end return result end @@ -1258,6 +1263,7 @@ function widget:Initialize() self, "runtestsheadless", function(cmd, optLine, optWords, data, isRepeat, release, actions) + headless = true config.noColorOutput = true config.quitWhenDone = true config.gameStartTestPatterns = Util.splitPhrases(optLine) From 11676060c363922ade0f81e8824dfb105acd0a8a Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:38:37 +0100 Subject: [PATCH 5/8] Polish and merge github workflows. - Deploy now depends on successful tests. - Action processing tests now fails on inconclusive or failure. --- .github/workflows/process_test_results.yml | 41 ------------------ .github/workflows/run_tests.yml | 34 --------------- .../{quick_deploy.yml => test_and_deploy.yml} | 43 ++++++++++++++++--- 3 files changed, 38 insertions(+), 80 deletions(-) delete mode 100644 .github/workflows/process_test_results.yml delete mode 100644 .github/workflows/run_tests.yml rename .github/workflows/{quick_deploy.yml => test_and_deploy.yml} (58%) diff --git a/.github/workflows/process_test_results.yml b/.github/workflows/process_test_results.yml deleted file mode 100644 index 55a7604eff5..00000000000 --- a/.github/workflows/process_test_results.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Process Test Results - -on: - workflow_run: - workflows: ["Run Tests"] - types: - - completed - -permissions: {} - -jobs: - process-test-results: - name: Process Test Results - runs-on: ubuntu-latest - if: github.event.workflow_run.conclusion != 'skipped' - - permissions: - checks: write - - # needed unless run with comment_mode: off - pull-requests: write - - # required by download step to access artifacts API - actions: read - - steps: - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - github-token: ${{ github.token }} - run-id: ${{ github.event.workflow_run.id }} - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - time_unit: milliseconds - commit: ${{ github.event.workflow_run.head_sha }} - event_file: artifacts/Event File/event.json - event_name: ${{ github.event.workflow_run.event }} - files: "artifacts/Test Results/*.json" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml deleted file mode 100644 index 19a7d0a603b..00000000000 --- a/.github/workflows/run_tests.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Run Tests - -on: -# pull_request - workflow_dispatch: - -jobs: - upload-event_file: - name: Upload Event File - runs-on: ubuntu-latest - steps: - - name: Upload Event File - uses: actions/upload-artifact@v4 - with: - name: Event File - path: ${{ github.event_path }} - - run-tests: - name: Run Tests - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Run Tests - run: docker-compose -f tools/headless_testing/docker-compose.yml up - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: Test Results - path: | - tools/headless_testing/testlog/results.json diff --git a/.github/workflows/quick_deploy.yml b/.github/workflows/test_and_deploy.yml similarity index 58% rename from .github/workflows/quick_deploy.yml rename to .github/workflows/test_and_deploy.yml index d6de550837e..16d00fa8e28 100644 --- a/.github/workflows/quick_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -1,14 +1,44 @@ -# Workflow to push the change to deploy the change to players -# on push to master much quicker without waiting for cron jobs. -name: Deploy +name: Run Tests + on: + workflow_dispatch: + pull_request: push: branches: - - master + - 'master' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + jobs: + run-tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Run Tests + run: docker compose -f tools/headless_testing/docker-compose.yml up + timeout-minutes: 30 + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + action_fail: true + action_fail_on_inconclusive: true + check_name: "Test Results" + time_unit: milliseconds + files: "tools/headless_testing/testlog/results.json" + deploy: + name: Deploy runs-on: ubuntu-latest - if: github.repository == 'beyond-all-reason/Beyond-All-Reason' + needs: run-tests + if: | + github.repository == 'beyond-all-reason/Beyond-All-Reason' && + github.event_name == 'push' && + github.ref == 'refs/heads/master' permissions: id-token: write steps: @@ -19,6 +49,7 @@ jobs: ssh -i id.key -o StrictHostKeyChecking=no debian@repos.beyondallreason.dev byar env: SSH_KEY: ${{ secrets.SSH_REPOS_DEPLOY_KEY }} + - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@v2 @@ -28,10 +59,12 @@ jobs: token_format: id_token id_token_audience: cdnupdater id_token_include_email: true + - name: Sync files to CDN run: | curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \ -X POST -d '["byar"]' https://rapidsyncer-ssd-7xiouooxaa-ey.a.run.app/sync + - name: Update CDN pointer run: | curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \ From 89fc451d8afdcab0c571535156cb5f97c254a420 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 19:01:34 +0100 Subject: [PATCH 6/8] Set LogFlushLevel to 0. --- tools/headless_testing/springsettings.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/headless_testing/springsettings.cfg b/tools/headless_testing/springsettings.cfg index 4cfae692517..5f8ac198062 100644 --- a/tools/headless_testing/springsettings.cfg +++ b/tools/headless_testing/springsettings.cfg @@ -1 +1,2 @@ RapidTagResolutionOrder = repos-cdn.beyondallreason.dev;repos.beyondallreason.dev +LogFlushLevel = 0 From e7925ffaa8c8d747bee7370af8adaf4d4f07309b Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 19:08:26 +0100 Subject: [PATCH 7/8] Add missing parenthesis. --- common/testing/mocha_json_reporter.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/testing/mocha_json_reporter.lua b/common/testing/mocha_json_reporter.lua index a4d9ad40a47..f10414e3646 100644 --- a/common/testing/mocha_json_reporter.lua +++ b/common/testing/mocha_json_reporter.lua @@ -31,12 +31,12 @@ function MochaJSONReporter:endTests(duration) end function MochaJSONReporter:extractError(text) - local errorIndex = text:match'^%[string "[%p%a%s]*%"]:[%d]+:().*' + local errorIndex = text:match('^%[string "[%p%a%s]*%"]:[%d]+:().*') if errorIndex and errorIndex > 0 then text = text:sub(errorIndex + 1) return text end - errorIndex = text:match'^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*' + errorIndex = text:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*') if errorIndex and errorIndex > 0 then text = text:sub(errorIndex) end From 78a7c5a190d277e41de607759c190b0dde2ee0e8 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 19:10:53 +0100 Subject: [PATCH 8/8] Match all infolog.txt errors starting with Error after the timestamp part. --- common/testing/infologtest.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/testing/infologtest.lua b/common/testing/infologtest.lua index f9ccae903eb..8722f6b2114 100644 --- a/common/testing/infologtest.lua +++ b/common/testing/infologtest.lua @@ -14,7 +14,8 @@ local function infologTest() if infolog then local fileLines = string.lines(infolog) for i, line in ipairs(fileLines) do - if string.find(line, 'Error:', nil, true) and not skipErrors(line) then + local errorIndex = line:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] Error().*') + if errorIndex and errorIndex > 0 and not skipErrors(line) then errors[#errors+1] = line if #errors > maxErrors then return errors