Render pretty summaries (#49)

* harness: render-summary script

Signed-off-by: William Woodruff <william@trailofbits.com>

* test-harness: render-summary

Signed-off-by: William Woodruff <william@trailofbits.com>

* render-summary: tweaks

Signed-off-by: William Woodruff <william@trailofbits.com>

---------

Signed-off-by: William Woodruff <william@trailofbits.com>

.github/workflows/test-harness.yml

@@ -12,16 +12,36 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ">=3.12"
+
       - uses: actions/setup-go@v4
         with:
           go-version: ">=1.20.5"
 
       - name: run tests
         run: make test-go
+
+      - name: render summary
+        run: |
+          python ./harness/render-summary.py \
+            limbo.json \
+            harness/gocryptox509/results.json
   openssl:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ">=3.12"
+
       - name: run tests
         run: make CXX=clang++ test-openssl
+
+      - name: render summary
+        run: |
+          python ./harness/render-summary.py \
+            limbo.json \
+            harness/openssl/results.json

harness/render-summary.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+
+# render-summary.py: take a `results.json` from a harness run and render
+# it as a GitHub Actions step summary, collating against `limbo.json`
+
+if summary := os.getenv("GITHUB_STEP_SUMMARY"):
+    _OUT = Path(summary).open("wt+")
+else:
+    _OUT = sys.stdout
+
+_FAILED_RESULT_TEMPLATE = """
+### ❌ `{testcase_id}`
+
+* Expected result: {expected_result}
+* Actual result: {actual_result}
+
+{description}
+
+Additional context: {context}
+"""
+
+def _render(s: str) -> None:
+    print(f"{s}\n", file=_OUT)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("limbo", type=Path)
+parser.add_argument("results", type=Path)
+args = parser.parse_args()
+
+limbo = json.loads(args.limbo.read_text())
+results = json.loads(args.results.read_text())
+
+_render(f"## Limbo results for `{results["harness"]}`")
+
+for result in results["results"]:
+    testcase_id = result["id"]
+    actual_result = result["actual_result"]
+    context = result["context"]
+    if not context:
+        # Normalize missing context into an empty string.
+        context = ""
+
+    testcase = next(t for t in limbo["testcases"] if t["id"] == testcase_id)
+    expected_result = testcase["expected_result"]
+    description = testcase["description"]
+
+    # TODO: Render success cases as well?
+    if actual_result == expected_result:
+        continue
+
+    summary = _FAILED_RESULT_TEMPLATE.format(
+        testcase_id=testcase_id,
+        expected_result=expected_result,
+        actual_result=actual_result,
+        description=description,
+        context=context,
+    )
+    _render(summary)