Testing framework with depends

.github/workflows/process_test_results.yml

@@ -0,0 +1,56 @@
+name: Process Test Results
+
+on:
+  workflow_run:
+    workflows: ["Run Tests"]
+    types:
+      - completed
+
+permissions: {}
+
+jobs:
+  process-test-results:
+    name: Process Test Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    permissions:
+      checks: write
+
+      # needed unless run with comment_mode: off
+      pull-requests: write
+
+      # required by download step to access artifacts API
+      actions: read
+
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          github-token: ${{ github.token }}
+          run-id: ${{ github.event.workflow_run.id }}
+
+      - name: Create context file
+        run: |
+            printf '{
+              "event": "${{github.event.workflow_run.event}}",
+              "branch": "${{github.event.workflow_run.head_branch}}"
+            }' >> artifacts/context.json
+
+      - name: Upload Context File
+        uses: actions/upload-artifact@v4
+        with:
+          name: Context File
+          path: artifacts/context.json
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          action_fail_on_inconclusive: true
+          check_name: "Test Results (${{ github.event.workflow_run.event || github.event_name }})"
+          time_unit: milliseconds
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/Test Results/*.json"

.github/workflows/quick_deploy.yml

@@ -2,13 +2,35 @@
 # on push to master much quicker without waiting for cron jobs.
 name: Deploy
 on:
-  push:
-    branches:
-      - master
+  workflow_run:
+    workflows: ["Process Test Results"]
+    types:
+      - completed
 jobs:
+  get_context:
+    runs-on: ubuntu-latest
+    if: |
+      github.repository == 'beyond-all-reason/Beyond-All-Reason' &&
+      github.event.workflow_run.conclusion == 'success'
+    outputs:
+      event: ${{ fromJson(steps.context.outputs.data).event }}
+      branch: ${{ fromJson(steps.context.outputs.data).branch }}
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          github-token: ${{ github.token }}
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Get context json
+        id: context
+        run: echo "data=$(jq -c . < 'artifacts/Context File/context.json')" >> $GITHUB_OUTPUT
   deploy:
+    needs: [get_context]
     runs-on: ubuntu-latest
-    if: github.repository == 'beyond-all-reason/Beyond-All-Reason'
+    if: |
+      needs.get_context.outputs.event == 'push' &&
+      needs.get_context.outputs.branch == 'master'
     permissions:
       id-token: write
     steps:

.github/workflows/run_tests.yml

@@ -0,0 +1,35 @@
+name: Run Tests
+
+on:
+#  pull_request
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+jobs:
+  run-tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload Event File
+        uses: actions/upload-artifact@v4
+        with:
+          name: Event File
+          path: ${{ github.event_path }}
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Run Tests
+        run: docker compose -f tools/headless_testing/docker-compose.yml up
+        timeout-minutes: 30
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Test Results
+          path: |
+            tools/headless_testing/testlog/results.json

common/testing/infologtest.lua

@@ -0,0 +1,32 @@
+local maxErrors = 10
+
+local function skipErrors(line)
+	if string.find(line, 'Could not finalize projectile-texture atlas', nil, true) then
+		return true
+	end
+end
+
+local function infologTest()
+	local errors = {}
+	local infolog = VFS.LoadFile("infolog.txt")
+	if infolog then
+		local fileLines = string.lines(infolog)
+		for i, line in ipairs(fileLines) do
+			if string.find(line, 'Error:', nil, true) and not skipErrors(line) then
+				errors[#errors+1] = line
+				if #errors > maxErrors then
+					return errors
+				end
+			end
+		end
+	end
+	return errors
+end
+
+
+function test()
+	local errors = infologTest()
+	if #errors > 0 then
+		error(table.concat(errors, "\n"), 0)
+	end
+end

common/testing/mocha_json_reporter.lua

@@ -8,11 +8,13 @@ function MochaJSONReporter:new()
 	local obj = {
 		totalTests = 0,
 		totalPasses = 0,
+		totalSkipped = 0,
 		totalFailures = 0,
 		startTime = nil,
 		endTime = nil,
 		duration = nil,
-		tests = {}
+		tests = {},
+		skipped = {}
 	}
 	setmetatable(obj, self)
 	self.__index = self
@@ -28,21 +30,37 @@ function MochaJSONReporter:endTests(duration)
 	self.duration = duration
 end
 
-function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage)
+function MochaJSONReporter:extractError(text)
+	local errorIndex = text:match'^%[string "[%p%a%s]*%"]:[%d]+:().*'
+	if errorIndex and errorIndex > 0 then
+		text = text:sub(errorIndex + 1)
+		return text
+	end
+	errorIndex = text:match'^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*'
+	if errorIndex and errorIndex > 0 then
+		text = text:sub(errorIndex)
+	end
+	return text
+end
+
+function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage)
 	local result = {
 		title = label,
 		fullTitle = label,
 		file = filePath,
 		duration = duration,
 	}
-	if success then
+	if skipped then
+		self.totalSkipped = self.totalSkipped + 1
+		result.err = {}
+	elseif success then
 		self.totalPasses = self.totalPasses + 1
 		result.err = {}
 	else
 		self.totalFailures = self.totalFailures + 1
 		if errorMessage ~= nil then
 			result.err = {
-				message = errorMessage,
+				message = self:extractError(errorMessage),
 				stack = errorMessage
 			}
 		else
@@ -54,6 +72,9 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM
 
 	self.totalTests = self.totalTests + 1
 	self.tests[#(self.tests) + 1] = result
+	if skipped then
+		self.skipped[#(self.skipped) + 1] = {fullTitle = label}
+	end
 end
 
 function MochaJSONReporter:report(filePath)
@@ -63,12 +84,14 @@ function MochaJSONReporter:report(filePath)
 			["tests"] = self.totalTests,
 			["passes"] = self.totalPasses,
 			["pending"] = 0,
+			["skipped"] = self.totalSkipped,
 			["failures"] = self.totalFailures,
 			["start"] = formatTimestamp(self.startTime),
 			["end"] = formatTimestamp(self.endTime),
 			["duration"] = self.duration
 		},
-		["tests"] = self.tests
+		["tests"] = self.tests,
+		["pending"] = self.skipped
 	}
 
 	local encoded = Json.encode(output)

luaui/Widgets/Tests/balance/test_arm_vs_cor_fighters.lua

@@ -0,0 +1,97 @@
+function setup()
+	Test.clearMap()
+end
+
+function cleanup()
+	Test.clearMap()
+
+	Spring.SendCommands("setspeed " .. 1)
+end
+
+function test()
+	local units = {
+		[0] = "armfig",
+		[1] = "corveng"
+	}
+	local n = 200
+
+	local midX, midZ = Game.mapSizeX / 2, Game.mapSizeZ / 2
+	local xOffset = 1000
+	local zStep = 10
+	local startZ = midZ - zStep * n / 2
+
+	-- make two lines of units facing each other
+	SyncedRun(function(locals)
+		do
+			local x = locals.midX - locals.xOffset
+			for i = 1, locals.n do
+				local z = locals.startZ + locals.zStep * i
+				local y = Spring.GetGroundHeight(x, z)
+				local unitID = Spring.CreateUnit(locals.units[0], x, y, z, "east", 0)
+			end
+		end
+
+		do
+			local x = locals.midX + locals.xOffset
+			for i = 1, locals.n do
+				local z = locals.startZ + locals.zStep * i
+				local y = Spring.GetGroundHeight(x, z)
+				local unitID = Spring.CreateUnit(locals.units[1], x, y, z, "west", 1)
+			end
+
+		end
+	end)
+
+	Test.waitFrames(1)
+
+	if false then
+		Spring.GiveOrderToUnitArray(Spring.GetTeamUnits(0), CMD.FIGHT, { midX, 0, midZ }, 0)
+		Spring.GiveOrderToUnitArray(Spring.GetTeamUnits(1), CMD.FIGHT, { midX, 0, midZ }, 0)
+	else
+		SyncedRun(function(locals)
+			local midX = locals.midX
+			local midZ = locals.midZ
+			for _, unitID in ipairs(Spring.GetAllUnits()) do
+				local ux, uy, uz = Spring.GetUnitPosition(unitID)
+
+				Spring.GiveOrderToUnit(unitID, CMD.FIGHT, { 2 * midX - ux, 0, uz }, 0)
+				Spring.GiveOrderToUnit(unitID, CMD.FIGHT, { midX, 0, midZ }, { "shift" })
+			end
+		end)
+	end
+
+	Spring.SendCommands("setspeed " .. 5)
+
+	-- wait until one team has no units left
+	Test.waitUntil(function()
+		return #(Spring.GetTeamUnits(0)) == 0 or #(Spring.GetTeamUnits(1)) == 0
+	end, 60 * 30)
+
+	Spring.SendCommands("setspeed " .. 1)
+
+	if #(Spring.GetTeamUnits(0)) > #(Spring.GetTeamUnits(1)) then
+		winner = 0
+	elseif #(Spring.GetTeamUnits(1)) > #(Spring.GetTeamUnits(0)) then
+		winner = 1
+	end
+
+	resultStr = "RESULT: "
+	if winner ~= nil then
+		unitName = units[winner]
+		if UnitDefNames and units[winner] and UnitDefNames[units[winner]] then
+			unitName = UnitDefNames[units[winner]].translatedHumanName or units[winner]
+		end
+		unitsLeft = #(Spring.GetAllUnits())
+		resultStr = resultStr .. "team " .. winner .. " wins"
+		resultStr = resultStr .. " with " .. unitsLeft
+		resultStr = resultStr .. " (" .. string.format("%.f%%", 100 * unitsLeft / n) .. ")"
+		resultStr = resultStr .. " " .. unitName .. " left"
+	else
+		resultStr = resultStr .. "tie"
+	end
+
+	Spring.Echo(resultStr)
+
+	-- cor fighters should win
+	assert(winner == 1)
+end