Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add regression workflow #5809

Merged
merged 5 commits into from
Jun 21, 2024
Merged

add regression workflow #5809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cd8e797
add regression workflow
pdabelf5 Jun 20, 2024
8710772
only checkout specified branch
pdabelf5 Jun 20, 2024
ea0c1c6
Merge branch 'main' into regression-workflow
pdabelf5 Jun 21, 2024
e210ad7
Merge branch 'main' into regression-workflow
pdabelf5 Jun 21, 2024
0d7265d
remove test branch
pdabelf5 Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ jobs:
outputs:
docs_only: ${{ github.event.pull_request && steps.docs.outputs.docs_only == 'true' }}
k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
latest_kindest_node_versions: ${{ steps.vars.outputs.latest_kindest_node_versions }}
go_path: ${{ steps.vars.outputs.go_path }}
go_code_md5: ${{ steps.vars.outputs.go_code_md5 }}
binary_cache_hit: ${{ steps.binary-cache.outputs.cache-hit }}
Expand Down Expand Up @@ -83,19 +82,6 @@ jobs:
| sed 's/^.\{1\}//' \
| tr -d '\n')
echo "k8s_latest=$kindest_latest" >> $GITHUB_OUTPUT
kindest_versions=$(curl -s "https://hub.docker.com/v2/repositories/kindest/node/tags" \
| grep -o '"name": *"[^"]*' \
| grep -o '[^"]*$' \
| grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
| sort -rV \
| awk -F. '!seen[$1"."$2]++' \
| head -n 7 \
| sort -V \
| sed 's/v//g' \
| sed 's/$//' \
| sed 's/, $//' \
| jq -R -s -c 'split("\n")[:-1]')
echo "latest_kindest_node_versions=$kindest_versions" >> $GITHUB_OUTPUT
echo "go_path=$(go env GOPATH)" >> $GITHUB_OUTPUT
source .github/data/version.txt
echo "ic_version=${IC_VERSION}" >> $GITHUB_OUTPUT
Expand Down Expand Up @@ -163,7 +149,6 @@ jobs:
run: |
echo docs_only: ${{ github.event.pull_request && steps.docs.outputs.docs_only == 'true' }}
echo k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
echo latest_kindest_node_versions: ${{ steps.vars.outputs.latest_kindest_node_versions }}
echo go_path: ${{ steps.vars.outputs.go_path }}
echo go_code_md5: ${{ steps.vars.outputs.go_code_md5 }}
echo binary_cache_hit: ${{ steps.binary-cache.outputs.cache-hit }}
Expand Down
308 changes: 308 additions & 0 deletions .github/workflows/regression.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,308 @@
name: Run Regression tests
run-name: Run NIC Regression workflow, triggered from ${{ github.event_name }} by @${{ github.actor }}

on:
schedule:
- cron: 00 03 * * *
workflow_dispatch:
inputs:
branch:
type: string
description: "Branch to run regression workflow on"
default: main

defaults:
run:
shell: bash

concurrency:
group: ${{ github.ref_name }}-regression
cancel-in-progress: true

permissions:
contents: read

jobs:
checks:
name: Checks and variables
runs-on: ubuntu-22.04
permissions:
contents: read
id-token: write
outputs:
k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
latest_kindest_node_versions: ${{ steps.vars.outputs.latest_kindest_node_versions }}
stable_tag: ${{ steps.vars.outputs.stable_tag }}
branch: ${{ steps.vars.outputs.branch }}
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
ref: ${{ inputs.branch && inputs.branch || github.event.repository.default_branch }}

- name: Output Variables
id: vars
run: |
kindest_latest=$(curl -s "https://hub.docker.com/v2/repositories/kindest/node/tags" \
| grep -o '"name": *"[^"]*' \
| grep -o '[^"]*$' \
| grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
| sort -rV \
| head -n 1 \
| sed 's/^.\{1\}//' \
| tr -d '\n')
echo "k8s_latest=$kindest_latest" >> $GITHUB_OUTPUT
kindest_versions=$(curl -s "https://hub.docker.com/v2/repositories/kindest/node/tags" \
| grep -o '"name": *"[^"]*' \
| grep -o '[^"]*$' \
| grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
| sort -rV \
| awk -F. '!seen[$1"."$2]++' \
| head -n 7 \
| sort -V \
| sed 's/v//g' \
| sed 's/$//' \
| sed 's/, $//' \
| jq -R -s -c 'split("\n")[:-1]')
echo "latest_kindest_node_versions=$kindest_versions" >> $GITHUB_OUTPUT
source .github/data/version.txt
./.github/scripts/variables.sh stable_tag >> $GITHUB_OUTPUT
branch=${{ github.event.repository.default_branch }}
if [ -n "${{ inputs.branch }}" ]; then
branch=${{ inputs.branch }}
fi
echo "branch=${branch}" >> $GITHUB_OUTPUT

- name: Output variables
run: |
echo k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
echo latest_kindest_node_versions: ${{ steps.vars.outputs.latest_kindest_node_versions }}
echo stable_tag: ${{ steps.vars.outputs.stable_tag }}
echo branch: ${{ steps.vars.outputs.branch }}

helm-tests:
name: Helm Tests ${{ matrix.base-os }}
runs-on: ubuntu-22.04
needs: [checks]
strategy:
fail-fast: false
matrix:
include:
- base-os: debian
image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress
tag: ${{ needs.checks.outputs.stable_tag }}
type: oss
- base-os: debian-plus
image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress
tag: ${{ needs.checks.outputs.stable_tag }}
type: plus
permissions:
contents: read
id-token: write
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
ref: ${{ needs.checks.outputs.branch }}

- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
with:
token_format: access_token
workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}

- name: Login to GCR
uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

- name: Pull build image
run: |
docker pull ${{ matrix.image }}:${{ matrix.tag }}

- name: Deploy Kubernetes
id: k8s
run: |
kind create cluster --name ${{ github.run_id }} --image=kindest/node:v${{ needs.checks.outputs.k8s_latest }} --wait 75s
kind load docker-image "${{ matrix.image }}:${{ matrix.tag }}" --name ${{ github.run_id }}

- name: Install Chart
run: >
helm install
${{ matrix.type }}
.
--set controller.image.repository=${{ matrix.image }}
--set controller.image.tag=${{ matrix.tag }}
--set controller.service.type=NodePort
--set controller.nginxplus=${{ contains(matrix.type, 'plus') && 'true' || 'false' }}
--set controller.telemetryReporting.enable=false
--wait
working-directory: ${{ github.workspace }}/charts/nginx-ingress

- name: Expose Test Ingresses
run: |
kubectl port-forward service/${{ matrix.type }}-nginx-ingress-controller 8080:80 8443:443 &

- name: Test HTTP
run: |
counter=0
max_attempts=5
until [ $(curl --write-out %{http_code} -s --output /dev/null http://localhost:8080) -eq 404 ]; do
if [ ${counter} -eq ${max_attempts} ]; then
exit 1
fi
printf '.'; counter=$(($counter+1)); sleep 5;
done

- name: Test HTTPS
run: |
counter=0
max_attempts=5
until [ $(curl --write-out %{http_code} -ks --output /dev/null https://localhost:8443) -eq 000 ]; do
if [ ${counter} -eq ${max_attempts} ]; then
exit 1
fi
printf '.'; counter=$(($counter+1)); sleep 5;
done

setup-regression-matrix:
name: Setup Matrix for Smoke Tests
runs-on: ubuntu-22.04
needs: [checks]
permissions:
contents: read
id-token: write
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
ref: ${{ needs.checks.outputs.branch }}

- id: set-matrix
run: |
echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest '${{ needs.checks.outputs.latest_kindest_node_versions }}' '.k8s += $latest'))" >> $GITHUB_OUTPUT

regression-tests:
name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} regression tests
runs-on: ubuntu-22.04
needs: [checks,setup-regression-matrix]
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.setup-regression-matrix.outputs.matrix) }}
permissions:
contents: read
id-token: write
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
ref: ${{ needs.checks.outputs.branch }}

- name: Set image variables
id: image_details
run: |
echo "name=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic${{ contains(matrix.images.nap_modules, 'dos') && '-dos' || '' }}${{ contains(matrix.images.nap_modules, 'waf') && '-nap' || '' }}/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
echo "tag=${{ needs.checks.outputs.stable_tag }}${{ contains(matrix.images.image, 'ubi') && '-ubi' || '' }}${{ contains(matrix.images.image, 'alpine') && '-alpine' || '' }}${{ contains(matrix.images.target, 'aws') && '-mktpl' || '' }}${{ contains(matrix.images.image, 'fips') && '-fips' || ''}}" >> $GITHUB_OUTPUT

- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
with:
token_format: access_token
workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}

- name: Login to GCR
uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

- name: NAP modules
id: nap_modules
run: |
[[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || modules="${{ matrix.images.nap_modules }}"
echo "modules=${modules}" >> $GITHUB_OUTPUT
if: ${{ matrix.images.nap_modules }}

- name: Pull build image
run: |
docker pull ${{ steps.image_details.outputs.name }}:${{ steps.image_details.outputs.tag }}

- name: Run Regression Tests
id: regression-tests
uses: ./.github/actions/smoke-tests
with:
image-type: ${{ matrix.images.image }}
image-name: ${{ steps.image_details.outputs.name }}
tag: ${{ steps.image_details.outputs.tag }}
marker: ${{ matrix.images.marker != '' && matrix.images.marker || '' }}
k8s-version: ${{ matrix.k8s }}
azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"

- name: Upload Test Results
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: ${{ steps.regression-tests.outputs.test-results-name }}-${{ matrix.k8s }}
path: ${{ github.workspace }}/tests/${{ steps.regression-tests.outputs.test-results-name }}-${{ matrix.k8s }}.html
if: always()

tag-stable:
name: Tag tested image as nightly
needs: [checks, regression-tests]
permissions:
contents: read # To checkout repository
id-token: write # To sign into Google Container Registry
uses: ./.github/workflows/retag-images.yml
with:
source_tag: ${{ needs.checks.outputs.stable_tag }}
target_tag: nightly
dry_run: false
secrets: inherit

release-oss:
# pushes nightly images to docker hub
name: Release Docker OSS
needs: [checks, regression-tests]
uses: ./.github/workflows/oss-release.yml
with:
gcr_release_registry: false
ecr_public_registry: true
dockerhub_public_registry: true
quay_public_registry: true
github_public_registry: true
source_tag: ${{ needs.checks.outputs.stable_tag }}
target_tag: "nightly"
dry_run: false
permissions:
contents: read
id-token: write
packages: write
secrets: inherit

release-plus:
# pushes plus nightly images to nginx registry
name: Release Docker Plus
needs: [checks, regression-tests]
uses: ./.github/workflows/plus-release.yml
with:
nginx_registry: true
gcr_release_registry: false
gcr_mktpl_registry: false
ecr_mktpl_registry: false
az_mktpl_registry: false
source_tag: ${{ needs.checks.outputs.stable_tag }}
target_tag: "nightly"
dry_run: false
permissions:
contents: read
id-token: write
secrets: inherit
Loading