Automatic tests

.github/workflows/quick_deploy.yml → .github/workflows/test_and_deploy.yml

@@ -1,14 +1,44 @@
-# Workflow to push the change to deploy the change to players
-# on push to master much quicker without waiting for cron jobs.
-name: Deploy
+name: Run Tests
+
 on:
+  workflow_dispatch:
+  pull_request:
   push:
     branches:
-      - master
+      - 'master'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 jobs:
+  run-tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Run Tests
+        run: docker compose -f tools/headless_testing/docker-compose.yml up
+        timeout-minutes: 30
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          action_fail: true
+          action_fail_on_inconclusive: true
+          check_name: "Test Results"
+          time_unit: milliseconds
+          files: "tools/headless_testing/testlog/results.json"
+
   deploy:
+    name: Deploy
     runs-on: ubuntu-latest
-    if: github.repository == 'beyond-all-reason/Beyond-All-Reason'
+    needs: run-tests
+    if: |
+      github.repository == 'beyond-all-reason/Beyond-All-Reason' &&
+      github.event_name == 'push' &&
+      github.ref == 'refs/heads/master'
     permissions:
       id-token: write
     steps:
@@ -19,6 +49,7 @@ jobs:
           ssh -i id.key -o StrictHostKeyChecking=no debian@repos.beyondallreason.dev byar
         env:
           SSH_KEY: ${{ secrets.SSH_REPOS_DEPLOY_KEY }}
+
       - name: Authenticate to Google Cloud
         id: auth
         uses: google-github-actions/auth@v2
@@ -28,10 +59,12 @@ jobs:
           token_format: id_token
           id_token_audience: cdnupdater
           id_token_include_email: true
+
       - name: Sync files to CDN
         run: |
           curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \
             -X POST -d '["byar"]' https://rapidsyncer-ssd-7xiouooxaa-ey.a.run.app/sync
+
       - name: Update CDN pointer
         run: |
           curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \

common/testing/infologtest.lua

@@ -0,0 +1,35 @@
+-- 'hidden' test checking infolog.txt for errors, used by headless runs.
+
+local maxErrors = 10
+
+local function skipErrors(line)
+	if string.find(line, 'Could not finalize projectile-texture atlas', nil, true) then
+		return true
+	end
+end
+
+local function infologTest()
+	local errors = {}
+	local infolog = VFS.LoadFile("infolog.txt")
+	if infolog then
+		local fileLines = string.lines(infolog)
+		for i, line in ipairs(fileLines) do
+			local errorIndex = line:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] Error().*')
+			if errorIndex and errorIndex > 0 and not skipErrors(line) then
+				errors[#errors+1] = line
+				if #errors > maxErrors then
+					return errors
+				end
+			end
+		end
+	end
+	return errors
+end
+
+
+function test()
+	local errors = infologTest()
+	if #errors > 0 then
+		error(table.concat(errors, "\n"), 0)
+	end
+end

common/testing/mocha_json_reporter.lua

@@ -8,11 +8,13 @@ function MochaJSONReporter:new()
 	local obj = {
 		totalTests = 0,
 		totalPasses = 0,
+		totalSkipped = 0,
 		totalFailures = 0,
 		startTime = nil,
 		endTime = nil,
 		duration = nil,
-		tests = {}
+		tests = {},
+		skipped = {}
 	}
 	setmetatable(obj, self)
 	self.__index = self
@@ -28,21 +30,37 @@ function MochaJSONReporter:endTests(duration)
 	self.duration = duration
 end
 
-function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage)
+function MochaJSONReporter:extractError(text)
+	local errorIndex = text:match('^%[string "[%p%a%s]*%"]:[%d]+:().*')
+	if errorIndex and errorIndex > 0 then
+		text = text:sub(errorIndex + 1)
+		return text
+	end
+	errorIndex = text:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*')
+	if errorIndex and errorIndex > 0 then
+		text = text:sub(errorIndex)
+	end
+	return text
+end
+
+function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage)
 	local result = {
 		title = label,
 		fullTitle = label,
 		file = filePath,
 		duration = duration,
 	}
-	if success then
+	if skipped then
+		self.totalSkipped = self.totalSkipped + 1
+		result.err = {}
+	elseif success then
 		self.totalPasses = self.totalPasses + 1
 		result.err = {}
 	else
 		self.totalFailures = self.totalFailures + 1
 		if errorMessage ~= nil then
 			result.err = {
-				message = errorMessage,
+				message = self:extractError(errorMessage),
 				stack = errorMessage
 			}
 		else
@@ -54,6 +72,9 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM
 
 	self.totalTests = self.totalTests + 1
 	self.tests[#(self.tests) + 1] = result
+	if skipped then
+		self.skipped[#(self.skipped) + 1] = {fullTitle = label}
+	end
 end
 
 function MochaJSONReporter:report(filePath)
@@ -63,12 +84,14 @@ function MochaJSONReporter:report(filePath)
 			["tests"] = self.totalTests,
 			["passes"] = self.totalPasses,
 			["pending"] = 0,
+			["skipped"] = self.totalSkipped,
 			["failures"] = self.totalFailures,
 			["start"] = formatTimestamp(self.startTime),
 			["end"] = formatTimestamp(self.endTime),
 			["duration"] = self.duration
 		},
-		["tests"] = self.tests
+		["tests"] = self.tests,
+		["pending"] = self.skipped
 	}
 
 	local encoded = Json.encode(output)

luaui/Widgets/dbg_test_runner.lua

@@ -41,6 +41,7 @@ local config = {
 }
 
 local testReporter = nil
+local headless = false
 
 -- utils
 -- =====
@@ -71,6 +72,7 @@ local function logEndTests(duration)
 	testReporter:endTests(duration)
 
 	testReporter:report(config.testResultsFilePath)
+	headless = false
 end
 
 local function logTestResult(testResult)
@@ -82,6 +84,7 @@ local function logTestResult(testResult)
 		testResult.label,
 		testResult.filename,
 		(testResult.result == TestResults.TEST_RESULT.PASS),
+		(testResult.result == TestResults.TEST_RESULT.SKIP),
 		testResult.milliseconds,
 		testResult.error
 	)
@@ -141,6 +144,9 @@ local function findAllTestFiles(patterns)
 			result[#result + 1] = testFileInfo
 		end
 	end
+	if headless then
+		result[#result+1] = {label="infolog", filename="common/testing/infologtest.lua"}
+	end
 	return result
 end
 
@@ -1257,6 +1263,7 @@ function widget:Initialize()
 		self,
 		"runtestsheadless",
 		function(cmd, optLine, optWords, data, isRepeat, release, actions)
+			headless = true
 			config.noColorOutput = true
 			config.quitWhenDone = true
 			config.gameStartTestPatterns = Util.splitPhrases(optLine)

tools/headless_testing/download-maps.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.6.4"
+engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.8"

tools/headless_testing/springsettings.cfg

@@ -1 +1,2 @@
 RapidTagResolutionOrder = repos-cdn.beyondallreason.dev;repos.beyondallreason.dev
+LogFlushLevel = 0

tools/headless_testing/startscript.txt

@@ -1,6 +1,6 @@
 [GAME]
 {
-	MapName=Supreme Isthmus v1.6.4;
+	MapName=Supreme Isthmus v1.8;
 	GameType=Beyond All Reason $VERSION;
 	GameStartDelay=0;
 	StartPosType=0;
@@ -10,7 +10,7 @@
 	FixedRNGSeed = 1;
   [MODOPTIONS]
   {
-    debugcommands=1:cheat|2:godmode|3:globallos|30:runtestsheadless;
+    debugcommands=1:cheat|2:godmode|30:runtestsheadless;
 		deathmode=neverend;
   }
   [ALLYTEAM0]