From c840abfbff576d649de9a20b5337da288bb1a164 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 26 Feb 2024 13:53:42 +0100 Subject: [PATCH 01/17] feat(.github): run QUIC Interop Runner Run the QUIC Interop Runner testcases on pull requests when entering the merge queue. --- .github/workflows/qns-comment.yml | 56 +++++++++++++++++++++++++++++++ .github/workflows/qns.yml | 15 ++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/qns-comment.yml diff --git a/.github/workflows/qns-comment.yml b/.github/workflows/qns-comment.yml new file mode 100644 index 0000000000..a2cbd9cba5 --- /dev/null +++ b/.github/workflows/qns-comment.yml @@ -0,0 +1,56 @@ +# Post test results as pull request comment. +# +# This is done as a separate workflow as it requires write permissions. The +# tests itself might run off of a fork, i.e. an untrusted environment and should +# thus not be granted write permissions. + +name: Comment on the pull request + +on: + workflow_run: + workflows: ["QUIC Network Simulator"] + types: + - completed + +jobs: + upload: + permissions: + pull-requests: write + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: Download comment-data + uses: dawidd6/action-download-artifact@v2.11.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workflow: main.yml + run_id: ${{ github.event.workflow_run.id }} + name: comment-data + + - name: Format GitHub comment + run: | + pwd + ls -la + echo '[**QUIC Interop Runner**](https://github.com/quic-interop/quic-interop-runner)' >> comment + echo '' >> comment + echo '```' >> comment + cat summary >> comment + echo '```' >> comment + echo '' >> comment + echo 'Download artifacts [here](' >> comment + cat logs-url >> comment + echo ').' >> comment + if: always() + shell: bash + + - name: Read PR Number + id: pr-number + run: echo "::set-output name=number::$(cat pr-number)" + shell: bash + + - name: Comment PR + uses: thollander/actions-comment-pull-request@v2 + with: + filePath: comment + pr_number: ${{ steps.pr-number.outputs.number }} diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index edbd2824b9..9cbaef4bd4 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -11,11 +11,9 @@ on: - '.github/workflows/qns.yml' merge_group: branches: ["main"] - paths: - - 'qns/**' - - '.github/workflows/qns.yml' + jobs: - docker-image: + quic-network-simulator: runs-on: ubuntu-latest permissions: packages: write @@ -48,6 +46,7 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} - name: Build and push + id: docker_build_and_push uses: docker/build-push-action@v5 with: push: ${{ github.event_name != 'pull_request' }} @@ -58,3 +57,11 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64, linux/arm64 + + - name: Run QUIC Interop tests + # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. + uses: mxinden/quic-interop-runner@action + with: + name: 'neqo-latest' + image: ${{ steps.docker_build_and_push.outputs.imageID }} + url: https://github.com/mozilla/neqo From 33924409c56b19a21a038cd580fac8b5b5ff27a8 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 27 Feb 2024 11:18:30 +0100 Subject: [PATCH 02/17] Do not always construct comment --- .github/workflows/qns-comment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/qns-comment.yml b/.github/workflows/qns-comment.yml index a2cbd9cba5..d595637e70 100644 --- a/.github/workflows/qns-comment.yml +++ b/.github/workflows/qns-comment.yml @@ -41,7 +41,6 @@ jobs: echo 'Download artifacts [here](' >> comment cat logs-url >> comment echo ').' >> comment - if: always() shell: bash - name: Read PR Number From d2377aa1b3c6ed8b4378a5e60f07b14a0211addf Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 28 Feb 2024 12:02:10 +0100 Subject: [PATCH 03/17] Test for all pull requests to main --- .github/workflows/qns.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index 9cbaef4bd4..4b8ca93d21 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -6,9 +6,10 @@ on: workflow_dispatch: pull_request: branches: ["main"] - paths: - - 'qns/**' - - '.github/workflows/qns.yml' + # TODO: Re-introduce + # paths: + # - 'qns/**' + # - '.github/workflows/qns.yml' merge_group: branches: ["main"] From 69a1bcabf2b15e2edc83438b19504ac84f555426 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 28 Feb 2024 12:02:34 +0100 Subject: [PATCH 04/17] Don't build arm64 on pull requests --- .github/workflows/qns.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index 4b8ca93d21..06b00de077 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -57,7 +57,8 @@ jobs: RUST_VERSION=stable cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64, linux/arm64 + # On pull requests only build amd64 for the sake of CI time. + platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64, linux/arm64' }} - name: Run QUIC Interop tests # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. From b490228240b00fb5a072ebbf326ae62cef5c15be Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 1 Mar 2024 16:32:50 +0100 Subject: [PATCH 05/17] Use actions/download-artifact --- .github/workflows/qns-comment.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/qns-comment.yml b/.github/workflows/qns-comment.yml index d595637e70..7633d447b6 100644 --- a/.github/workflows/qns-comment.yml +++ b/.github/workflows/qns-comment.yml @@ -21,10 +21,8 @@ jobs: github.event.workflow_run.event == 'pull_request' steps: - name: Download comment-data - uses: dawidd6/action-download-artifact@v2.11.0 + uses: actions/download-artifact@v4 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - workflow: main.yml run_id: ${{ github.event.workflow_run.id }} name: comment-data From b055192622a961a14de5bfddd6cb73f9e2263706 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 1 Mar 2024 16:33:01 +0100 Subject: [PATCH 06/17] Provide client, server and test argument --- .github/workflows/qns.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index 06b00de077..b329ad91f3 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -67,3 +67,6 @@ jobs: name: 'neqo-latest' image: ${{ steps.docker_build_and_push.outputs.imageID }} url: https://github.com/mozilla/neqo + test: handshake + client: neqo-latest,quic-go,ngtcp2,neqo,msquic + server: neqo-latest,quic-go,ngtcp2,neqo,msquic From b9191aa40a1c73fbac316b9b6c1163157b3d93d2 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 1 Mar 2024 17:26:40 +0100 Subject: [PATCH 07/17] Fix action args --- .github/workflows/qns-comment.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qns-comment.yml b/.github/workflows/qns-comment.yml index 7633d447b6..83ec69f086 100644 --- a/.github/workflows/qns-comment.yml +++ b/.github/workflows/qns-comment.yml @@ -13,7 +13,7 @@ on: - completed jobs: - upload: + comment: permissions: pull-requests: write runs-on: ubuntu-latest @@ -23,8 +23,9 @@ jobs: - name: Download comment-data uses: actions/download-artifact@v4 with: - run_id: ${{ github.event.workflow_run.id }} + run-id: ${{ github.event.workflow_run.id }} name: comment-data + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Format GitHub comment run: | From c10ce08636c7e189cef24bb9f1df4028359dea22 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 11:16:52 +0100 Subject: [PATCH 08/17] Copy action into neqo repo For now until https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. --- .github/actions/quic-interop-runner.yml | 112 ++++++++++++++++++++++++ .github/workflows/qns.yml | 2 +- 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 .github/actions/quic-interop-runner.yml diff --git a/.github/actions/quic-interop-runner.yml b/.github/actions/quic-interop-runner.yml new file mode 100644 index 0000000000..7bb3874e72 --- /dev/null +++ b/.github/actions/quic-interop-runner.yml @@ -0,0 +1,112 @@ +name: 'QUIC Interop Runner Action' +description: 'Run the QUIC Interop Runner tests.' +author: 'mxinden' + +inputs: + name: + description: 'Name of the QUIC implementation' + required: true + image: + description: 'Docker image to be tested. Needs to reside either locally, or on some registry.' + required: true + url: + description: 'URL of the QUIC implementation' + required: true + role: + description: 'client/server/both' + required: false + default: 'both' + client: + description: 'client implementations (comma-separated)' + required: false + default: '' + server: + description: 'server implementations (comma-separated)' + required: false + default: '' + test: + description: 'test cases (comma-separatated)' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Install dependencies + run: | + sudo add-apt-repository ppa:wireshark-dev/stable + sudo apt-get update + sudo apt-get install -y wireshark tshark jq + shell: bash + + - uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Enable IPv6 support + run: sudo modprobe ip6table_filter + shell: bash + + - name: Checkout quic-interop/quic-interop-runner repository + uses: actions/checkout@v2 + with: + repository: 'quic-interop/quic-interop-runner' + path: 'quic-interop-runner' + + - name: Install Python packages + run: | + cd quic-interop-runner + pip install -U pip + pip install -r requirements.txt + shell: bash + + - run: docker image ls + shell: bash + + - name: Run tests + id: test-run + run: | + cd quic-interop-runner + jq --arg key "${{ inputs.name }}" --argjson newEntry '{"image": "${{ inputs.image }}", "url": "${{ inputs.url }}", "role": "${{ inputs.role }}"}' '.[$key] = $newEntry' implementations.json > temp.$$ && mv temp.$$ implementations.json + cat implementations.json + ARGS="--log-dir logs --must-include ${{ inputs.name }}" + if [ -n "${{ inputs.client }}" ]; then + ARGS="$ARGS --client ${{ inputs.client }}" + fi + if [ -n "${{ inputs.server }}" ]; then + ARGS="$ARGS --server ${{ inputs.server }}" + fi + if [ -n "${{ inputs.test }}" ]; then + ARGS="$ARGS --test ${{ inputs.test }}" + fi + python run.py $ARGS 2>&1 | tee summary + shell: bash + + - uses: actions/upload-artifact@v4 + id: artifact-upload-step + if: always() + with: + name: logs + path: quic-interop-runner/logs + + # This action might be running off of a fork and would thus not have write + # permissions on the origin repository. In order to allow a separate + # priviledged action to post a comment on a pull request, upload the + # necessary metadata. + - name: store comment-data + shell: bash + if: github.event_name == 'pull_request' + env: + PULL_REQUEST_NUMBER: ${{ github.event.number }} + run: | + mkdir comment-data + mv quic-interop-runner/summary comment-data/summary + echo $PULL_REQUEST_NUMBER > comment-data/pr-number + echo '${{ steps.artifact-upload-step.outputs.artifact-url }}' > comment-data/logs-url + + - name: Upload comment data + uses: actions/upload-artifact@v4 + if: github.event_name == 'pull_request' + with: + name: comment-data + path: ./comment-data diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index b329ad91f3..5cb827caf3 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -62,7 +62,7 @@ jobs: - name: Run QUIC Interop tests # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. - uses: mxinden/quic-interop-runner@action + uses: ./.github/actions/quic-interop-runner with: name: 'neqo-latest' image: ${{ steps.docker_build_and_push.outputs.imageID }} From a9d81121c098851b52aa85f33f7b30a2eff8d01d Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 11:25:07 +0100 Subject: [PATCH 09/17] checkout --- .github/workflows/qns.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index e31b96c71b..ea023da31d 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -59,6 +59,9 @@ jobs: # On pull requests only build amd64 for the sake of CI time. platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64, linux/arm64' }} + - name: Checkout + uses: actions/checkout@v4 + - name: Run QUIC Interop tests # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. uses: ./.github/actions/quic-interop-runner From 08d91ea149a845b5a0f9d36ae011637920389bc9 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 11:28:28 +0100 Subject: [PATCH 10/17] yml --- .github/workflows/qns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index ea023da31d..377780f6b8 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -64,7 +64,7 @@ jobs: - name: Run QUIC Interop tests # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. - uses: ./.github/actions/quic-interop-runner + uses: ./.github/actions/quic-interop-runner.yml with: name: 'neqo-latest' image: ${{ steps.docker_build_and_push.outputs.imageID }} From 36e629cbc874090bde6513128864bdea0631878a Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 11:33:51 +0100 Subject: [PATCH 11/17] introduce folder --- .../{quic-interop-runner.yml => quic-interop-runner/action.yml} | 0 .github/workflows/qns.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/actions/{quic-interop-runner.yml => quic-interop-runner/action.yml} (100%) diff --git a/.github/actions/quic-interop-runner.yml b/.github/actions/quic-interop-runner/action.yml similarity index 100% rename from .github/actions/quic-interop-runner.yml rename to .github/actions/quic-interop-runner/action.yml diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index 377780f6b8..ea023da31d 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -64,7 +64,7 @@ jobs: - name: Run QUIC Interop tests # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. - uses: ./.github/actions/quic-interop-runner.yml + uses: ./.github/actions/quic-interop-runner with: name: 'neqo-latest' image: ${{ steps.docker_build_and_push.outputs.imageID }} From a72e3a621916d3a13522b5f489c4f4f132316051 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 11:45:19 +0100 Subject: [PATCH 12/17] load image --- .github/workflows/qns.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index ea023da31d..d0256f9021 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -58,6 +58,7 @@ jobs: cache-to: type=gha,mode=max # On pull requests only build amd64 for the sake of CI time. platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64, linux/arm64' }} + load: true - name: Checkout uses: actions/checkout@v4 From 33ce044e41f4b3178137b15102c9d67bf24b312e Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 11:50:43 +0100 Subject: [PATCH 13/17] Only run interop on pull requests --- .github/workflows/qns.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index d0256f9021..76a27e7986 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -64,6 +64,7 @@ jobs: uses: actions/checkout@v4 - name: Run QUIC Interop tests + if: ${{ github.event_name == 'pull_request' }} # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. uses: ./.github/actions/quic-interop-runner with: From 6a8278836e29b7a4792760ed2826e6c249b9fa1d Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 13:31:17 +0100 Subject: [PATCH 14/17] Remove pull_request path filter --- .github/workflows/qns.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index 76a27e7986..2b1deb4be8 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -6,10 +6,6 @@ on: workflow_dispatch: pull_request: branches: ["main"] - # TODO: Re-introduce - # paths: - # - 'qns/**' - # - '.github/workflows/qns.yml' merge_group: jobs: @@ -58,7 +54,7 @@ jobs: cache-to: type=gha,mode=max # On pull requests only build amd64 for the sake of CI time. platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64, linux/arm64' }} - load: true + load: ${{ github.event_name == 'pull_request' }} - name: Checkout uses: actions/checkout@v4 From 0be3c49903d6c8314547dcc10e2e352d35c8ddf8 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 14:23:39 +0100 Subject: [PATCH 15/17] Trigger CI From 07fe7402e3f327f73cfc78b4e91d9464dc97bf23 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 19:20:37 +0100 Subject: [PATCH 16/17] Cache Python deps --- .../actions/quic-interop-runner/action.yml | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/actions/quic-interop-runner/action.yml b/.github/actions/quic-interop-runner/action.yml index 7bb3874e72..6e79b97cfe 100644 --- a/.github/actions/quic-interop-runner/action.yml +++ b/.github/actions/quic-interop-runner/action.yml @@ -32,6 +32,16 @@ inputs: runs: using: "composite" steps: + - name: Checkout quic-interop/quic-interop-runner repository + uses: actions/checkout@v4 + with: + repository: 'quic-interop/quic-interop-runner' + path: 'quic-interop-runner' + + - name: Enable IPv6 support + run: sudo modprobe ip6table_filter + shell: bash + - name: Install dependencies run: | sudo add-apt-repository ppa:wireshark-dev/stable @@ -39,19 +49,11 @@ runs: sudo apt-get install -y wireshark tshark jq shell: bash - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.8 - - - name: Enable IPv6 support - run: sudo modprobe ip6table_filter - shell: bash - - - name: Checkout quic-interop/quic-interop-runner repository - uses: actions/checkout@v2 - with: - repository: 'quic-interop/quic-interop-runner' - path: 'quic-interop-runner' + cache: 'pip' + cache-dependency-path: 'quic-interop-runner/requirements.txt' - name: Install Python packages run: | @@ -60,9 +62,6 @@ runs: pip install -r requirements.txt shell: bash - - run: docker image ls - shell: bash - - name: Run tests id: test-run run: | From d2e3efa8e7a4888400b19d7e294771907fc1c355 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 6 Mar 2024 19:26:45 +0100 Subject: [PATCH 17/17] Trigger CI to test python cache