From 7070e7bcc321951552dab76022bc76cbe8a02f79 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Thu, 2 May 2024 10:27:21 +0300
Subject: [PATCH] ci: Prepare to show failure differences of QNS run relative
 to main (#1856)

* WIP

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Again

* Make less noisy

* Restore condition

---------

Signed-off-by: Lars Eggert <lars@eggert.org>
---
 .github/workflows/qns.yml | 65 ++++++++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml
index 45ea31ffc5..604a28c93a 100644
--- a/.github/workflows/qns.yml
+++ b/.github/workflows/qns.yml
@@ -1,8 +1,8 @@
 name: QUIC Network Simulator
-
 on:
   schedule:
-    - cron: '42 3 * * 2,5' # Runs at 03:42 UTC (m and h chosen arbitrarily) twice a week.
+    # Run at 1 AM each day, so there is a `main`-branch baseline in the cache.
+    - cron: '0 1 * * *'
   workflow_dispatch:
   pull_request:
     branches: ["main"]
@@ -78,7 +78,6 @@ jobs:
           path: /tmp/${{ env.LATEST }}.tar
 
   implementations:
-    if: ${{ github.event_name == 'pull_request' }}
     name: Determine interop pairs
     needs: docker-image
     runs-on: ubuntu-latest
@@ -120,7 +119,6 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
   run-qns:
-    if: ${{ github.event_name == 'pull_request' }}
     name: Run QNS
     needs: implementations
     strategy:
@@ -150,13 +148,20 @@ jobs:
           client: ${{ steps.depair.outputs.client }}
           server: ${{ steps.depair.outputs.server }}
           implementations: ${{ needs.implementations.outputs.implementations }}
+          test: handshake,transfer
 
   report:
-    if: ${{ always() && github.event_name == 'pull_request' }}
     name: Report results
     needs: run-qns
     runs-on: ubuntu-latest
     steps:
+      - name: Download cached main-branch results
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: results-main
+          key: qns-${{ runner.os }}-${{ github.sha }}
+          restore-keys: qns-${{ runner.os }}-
+
       - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
       - uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
         with:
@@ -164,6 +169,9 @@ jobs:
           path: results
 
       - run: |
+          ls -l results-main || true
+          cat results-main/result.json || true
+          echo "[]" > result.json
           for RUN in results/*; do
             [ -e "$RUN/result.json" ] || continue
             CLIENT=$(jq -r < "$RUN/result.json" '.clients[0]')
@@ -193,22 +201,57 @@ jobs:
                 echo "**$RESULT**"
               } >> "$GROUP.md"
             done
+            jq < "$RUN/grouped.json" ". += { client: \"$CLIENT\", server: \"$SERVER\" }" > new.json && \
+              jq < result.json --argjson new "$(cat new.json)" '. += [$new]' > result.json.tmp && \
+              rm new.json && \
+              mv result.json.tmp result.json
           done
+          DIFFER='def post_recurse(f): def r: (f | select(. != null) | r), .; r; def post_recurse: post_recurse(.[]?); (. | (post_recurse | arrays) |= sort)'
+          diff <(jq -S "$DIFFER" results-main/result.json) <(jq -S "$DIFFER" result.json) || true
+          diff -Baur results-main/result.json result.json || true
           {
             echo "### Failed Interop Tests"
+            SHA=$(cat results-main/baseline-sha.txt || true)
+            if [ -n "$SHA" ]; then
+              {
+                echo "Interop failures relative to $SHA."
+                echo
+              } >> results.md
+            fi
+            if [ -e failed.md ]; then
+              echo "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*"
+              cat failed.md
+            else
+              echo "None :tada:"
+            fi
+            echo "<details><summary>All results</summary>"
+            echo
             echo "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*"
-            cat failed.md
-            echo "<details><summary>Succeeded and unsupported tests</summary>"
             for GROUP in succeeded unsupported; do
-              echo
               echo "### ${GROUP^} Interop Tests"
-              echo
-              cat "$GROUP.md"
-              echo
+              if [ -e "$GROUP.md" ]; then
+                cat "$GROUP.md"
+              else
+                echo "None :question:"
+              fi
             done
+            echo
             echo "</details>"
           } >> comment.md
 
+      - name: Remember main-branch push URL
+        if: github.ref == 'refs/heads/main'
+        run: echo "${{ github.sha }}" > baseline-sha.txt
+
+      - name: Cache main-branch results
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            result.json
+            baseline-sha.txt
+          key: qns-${{ runner.os }}-${{ github.sha }}
+
       - uses: ./.github/actions/pr-comment-data-export
         with:
           name: qns