Add basic e2e tests for UBI image (#5764)

Run basic e2e tests on a Kind cluster for the UBI image to make sure
the UBI image can be deployed as expected. Add a new input parameter
`distro` to the conformance workflow to allow testing of the UBI image.

Fixes: #5729

Signed-off-by: Xu Liu <xliu2@vmware.com>

.github/workflows/build.yml

@@ -69,13 +69,21 @@ jobs:
     if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
     runs-on: [ubuntu-latest]
     steps:
+    - name: Free disk space
+      # https://github.com/actions/virtual-environments/issues/709
+      run: |
+        sudo apt-get clean
+        df -h
     - uses: actions/checkout@v4
       with:
         show-progress: false
     - name: Build Antrea UBI9 Docker image without pushing to registry
       if: ${{ github.repository != 'antrea-io/antrea' || github.event_name != 'push' || github.ref != 'refs/heads/main' }}
       run: |
         ./hack/build-antrea-linux-all.sh --pull --distro ubi
+    - name: Clean up docker build cache
+      # Clean up build cache to avoid running out of disk space in the following go tests.
+      run: docker builder prune -f
     - name: Build and push Antrea UBI9 Docker image to registry
       if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       env:
@@ -87,6 +95,29 @@ jobs:
         docker push antrea/antrea-ubi:latest
         docker push antrea/antrea-agent-ubi:latest
         docker push antrea/antrea-controller-ubi:latest
+    - name: Install Kind
+      run: |
+        KIND_VERSION=$(head -n1 ./ci/kind/version)
+        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+        chmod +x ./kind
+        sudo mv kind /usr/local/bin
+    - name: Run basic e2e tests
+      run: |
+        mkdir log
+        ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-e2e-kind.sh --encap-mode encap \
+        --antrea-controller-image antrea/antrea-controller-ubi \
+        --antrea-agent-image antrea/antrea-agent-ubi \
+        --run '^TestBasic$'
+    - name: Tar log files
+      if: ${{ failure() }}
+      run: tar -czf log.tar.gz log
+    - name: Upload test log
+      uses: actions/upload-artifact@v3
+      if: ${{ failure() }}
+      with:
+        name: e2e-kind-ubi-basic.tar.gz
+        path: log.tar.gz
+        retention-days: 30
 
   build-scale:
     needs: check-changes

.github/workflows/conformance.yml

@@ -10,6 +10,13 @@ on:
       antrea-values:
         description: The Antrea Chart values. Multiple values can be separated with commas (e.g. key1=val1,key2=val2). Default configuration will be tested if empty.
         required: false
+      antrea-image-distro:
+        description: The Antrea image distribution to test. It could be ubuntu or ubi.
+        type: choice
+        options:
+          - ubuntu
+          - ubi
+        default: ubuntu
       k8s-version:
         description: The K8s version (e.g. v1.27.1) to test. Kind's default K8s version will be used if empty.
         required: false
@@ -50,36 +57,49 @@ jobs:
         run: |
           if git show-ref --tags --verify --quiet refs/tags/${{ inputs.antrea-version }}; then
             echo "released=true" >> $GITHUB_OUTPUT
+            echo "image-tag=${{ inputs.antrea-version }}" >> $GITHUB_OUTPUT
           else
             echo "released=false" >> $GITHUB_OUTPUT
+            echo "image-tag=latest" >> $GITHUB_OUTPUT
           fi
       - name: Build Antrea image if required
         if: ${{ steps.check-release.outputs.released == 'false' }}
         run: |
-          ./hack/build-antrea-linux-all.sh --pull
+          ./hack/build-antrea-linux-all.sh --pull --distro ${{ inputs.antrea-image-distro }}
       - name: Install Kind
         run: |
-          KIND_VERSION=$(head -n1 ./ci/kind/version)
+          KIND_VERSION=$(head -n1 ./ci/kind/version || echo v0.20.0)
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
           chmod +x ./kind
           sudo mv kind /usr/local/bin
       - name: Create K8s cluster
         run: |
-          # The command also loads local antrea/antrea-agent-ubuntu:latest and antrea/antrea-controller-ubuntu:latest
-          # into Nodes if they exist.
+          images=()
+          images+=(antrea/antrea-controller-${{ inputs.antrea-image-distro }}:${{ steps.check-release.outputs.image-tag }})
+          images+=(antrea/antrea-agent-${{ inputs.antrea-image-distro }}:${{ steps.check-release.outputs.image-tag }})
+          images+=(antrea/antrea-${{ inputs.antrea-image-distro }}:${{ steps.check-release.outputs.image-tag }})
           ./ci/kind/kind-setup.sh create kind \
-            --k8s-version "${{ inputs.k8s-version }}"
+            --k8s-version "${{ inputs.k8s-version }}" \
+            --images "${images[*]}"
       - name: Install Antrea
         run: |
+          helm_args=()
+          helm_repo="./build/charts/antrea"
           if [ ${{ steps.check-release.outputs.released }} == 'true' ]; then
+            helm_repo="antrea/antrea"
+            helm_args+=(--version "${{ inputs.antrea-version }}")
             helm repo add antrea https://charts.antrea.io
             helm repo update
-            helm install --namespace kube-system antrea antrea/antrea --version "${{ inputs.antrea-version }}" \
-              --set "${{ inputs.antrea-values }}"
+          fi
+          if helm show values ${helm_repo} | grep -q '^controllerImage:'; then
+            helm_args+=(--set controllerImage.repository="antrea/antrea-controller-${{ inputs.antrea-image-distro }}")
+            helm_args+=(--set agentImage.repository="antrea/antrea-agent-${{ inputs.antrea-image-distro }}")
           else
-            helm install --namespace kube-system antrea ./build/charts/antrea \
-              --set "${{ inputs.antrea-values }}"
+            helm_args+=(--set image.repository="antrea/antrea-${{ inputs.antrea-image-distro }}")
           fi
+          helm install --namespace kube-system antrea ${helm_repo} \
+            --set "${{ inputs.antrea-values }}" \
+            "${helm_args[@]}"
           kubectl rollout status -n kube-system ds/antrea-agent --timeout=5m
       - name: Run e2e tests
         run: |

ci/kind/test-e2e-kind.sh

@@ -40,6 +40,9 @@ _usage="Usage: $0 [--encap-mode <mode>] [--ip-family <v4|v6|dual>] [--coverage]
         --setup-only                  Only perform setting up the cluster and run test.
         --cleanup-only                Only perform cleaning up the cluster.
         --test-only                   Only run test on current cluster. Not set up/clean up the cluster.
+        --antrea-controller-image     The Antrea controller image to use for the test. Default is antrea/antrea-controller-ubuntu.
+        --antrea-agent-image          The Antrea agent image to use for the test. Default is antrea/antrea-agent-ubuntu.
+        --antrea-image-tag            The Antrea image tag to use for the test. Default is latest.
         --help, -h                    Print this message and exit.
 "
 
@@ -82,6 +85,10 @@ setup_only=false
 cleanup_only=false
 test_only=false
 run=""
+antrea_controller_image="antrea/antrea-controller-ubuntu"
+antrea_agent_image="antrea/antrea-agent-ubuntu"
+use_non_default_images=false
+antrea_image_tag="latest"
 while [[ $# -gt 0 ]]
 do
 key="$1"
@@ -155,6 +162,20 @@ case $key in
     test_only=true
     shift
     ;;
+    --antrea-controller-image)
+    antrea_controller_image="$2"
+    use_non_default_images=true
+    shift 2
+    ;;
+    --antrea-agent-image)
+    antrea_agent_image="$2"
+    use_non_default_images=true
+    shift 2
+    ;;
+    --antrea-image-tag)
+    antrea_image_tag="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -186,6 +207,11 @@ if [[ $cleanup_only == "true" ]];then
   exit 0
 fi
 
+if $use_non_default_images && $coverage; then
+    echoerr "Cannot use non-default images when coverage is enabled"
+    exit 1
+fi
+
 trap "quit" INT EXIT
 
 manifest_args="$manifest_args --verbose-log"
@@ -234,11 +260,11 @@ done
 # The Antrea images should not be pulled, as we want to use the local build.
 if $coverage; then
     manifest_args="$manifest_args --coverage"
-    COMMON_IMAGES_LIST+=("antrea/antrea-agent-ubuntu-coverage:latest" \
-                         "antrea/antrea-controller-ubuntu-coverage:latest")
+    COMMON_IMAGES_LIST+=("${antrea_controller_image}-coverage:${antrea_image_tag}" \
+                         "${antrea_agent_image}-coverage:${antrea_image_tag}")
 else
-    COMMON_IMAGES_LIST+=("antrea/antrea-agent-ubuntu:latest" \
-                         "antrea/antrea-controller-ubuntu:latest")
+    COMMON_IMAGES_LIST+=("${antrea_controller_image}:${antrea_image_tag}" \
+                         "${antrea_agent_image}:${antrea_image_tag}")
 fi
 if $flow_visibility; then
     if $coverage; then