nfr.yml

name: Non Functional Testing

on:
  workflow_dispatch:
    inputs:
      test_label:
        description: NFR test to run. Choose between performance, upgrade, scale, or all
        required: true
        default: all
        type: choice
        options: [performance, upgrade, scale, all]
      version:
        description: Version of NGF under test
        required: true
        default: edge
      image_tag:
        description: Tag of the NGF and NGINX Docker images
        required: true
        default: edge
      type:
        description: Type of NGINX image to test
        required: true
        default: both
        type: choice
        options: [oss, plus, both]
  schedule:
    - cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.ref_name }}-nfr
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  vars:
    name: Set up vars
    runs-on: ubuntu-22.04
    outputs:
      test_label: ${{ github.event.inputs.test_label || 'all' }}
      version: ${{ github.event.inputs.version || 'edge' }}
      image_tag: ${{ github.event.inputs.image_tag || 'edge' }}
      types: ${{ steps.var.outputs.types }}
    permissions:
      contents: read
    steps:
      - name: Set vars
        id: var
        run: |
          if ${{ github.event.inputs.type == 'both' || github.event_name == 'schedule' }}; then
          echo 'types=["oss","plus"]' >> $GITHUB_OUTPUT
          else
          echo 'types=["${{ github.event.inputs.type }}"]' >> $GITHUB_OUTPUT
          fi

  setup-and-run-tests:
    name: Setup and Run NFR Tests
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      id-token: write # needed for authenticating to GCP
    needs: vars
    strategy:
      fail-fast: false
      matrix:
        type: ${{ fromJson(needs.vars.outputs.types) }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
        with:
          token_format: access_token
          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
          install_components: kubectl

      - name: Setup dotenv file
        working-directory: ./tests/scripts
        run: |
          echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
          echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env
          echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
          echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
          echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
          echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
          echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env
          echo "GKE_CLUSTER_REGION=us-west1" >> vars.env
          echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
          echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
          echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
          echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
          echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
          echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
          echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
          echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env
          echo "GINKGO_LABEL=" >> vars.env
          echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env
          echo "GKE_NUM_NODES=12" >> vars.env
          echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env

      - name: Create GKE cluster
        working-directory: ./tests
        run: make create-gke-cluster CI=true

      - name: Create and setup VM
        working-directory: ./tests
        run: make create-and-setup-vm

      - name: Run Tests
        working-directory: ./tests
        run: |
          if ${{ needs.vars.outputs.test_label != 'all' }}; then
          sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true;
          else
          make nfr-test CI=true;
          fi

      - name: Upload Artifacts
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: results-${{ matrix.type }}
          path: tests/results/**/*-${{ matrix.type }}.*

      - name: Cleanup
        working-directory: ./tests
        if: always()
        run: |
          bash scripts/cleanup-vm.sh true
          make delete-gke-cluster
          rm -rf scripts/vars.env

  pr-results:
    name: Open PR with results
    runs-on: ubuntu-22.04
    permissions:
      contents: write # needed for opening PR with the results files
      pull-requests: write # needed for opening PR with the results files
    needs: [vars, setup-and-run-tests]
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Download Artifacts
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          path: tests/results/
          merge-multiple: true

      - name: Open a PR with the results
        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
        with:
          token: ${{ secrets.NGINX_PAT }}
          commit-message: NFR Test Results for NGF version ${{ needs.vars.outputs.version }}
          author: nginx-bot <integrations@nginx.com>
          committer: nginx-bot <integrations@nginx.com>
          branch: tests/nfr-tests-${{ needs.vars.outputs.version }}
          delete-branch: true
          title: NFR Test Results for NGF version ${{ needs.vars.outputs.version }}
          add-paths: |
            tests/results/
          body: |
            Update with NFR test results for NGF version ${{ needs.vars.outputs.version }} ${{ needs.vars.outputs.types }}
            - Auto-generated by the NFR tests workflow run ${{ github.run_id }}
            - Tests ran using Docker image tag ${{ needs.vars.outputs.image_tag }}
            - ${{ needs.vars.outputs.test_label }} test(s) ran
          assignees: ${{ github.actor }}
          draft: ${{ github.event_name != 'schedule' }}