From 80a6f9440c3ee5717e2da5c10954832392aba9c4 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Sat, 4 Mar 2023 18:42:40 +0800 Subject: [PATCH] fix chart install/upgrade e2e --- .github/workflows/build-x86-image.yaml | 6 +-- .github/workflows/scheduled-e2e.yaml | 58 ++++++++++++-------------- Makefile | 41 ++++++++++-------- 3 files changed, 51 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 2985815d261..0d1ca3f7e5d 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -822,11 +822,7 @@ jobs: sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN - run: | - make kind-install-chart - kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s - kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s - kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s + run: make kind-install-chart - name: Cleanup run: sh dist/images/cleanup.sh diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index c0731fbe4ab..210f53c5f4f 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -967,15 +967,18 @@ jobs: - name: Cleanup run: sh dist/images/cleanup.sh - helm-direct-upgrade-e2e: - name: Helm Upgrade E2E + chart-upgrade-e2e: + name: Chart Upgrade E2E runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: fail-fast: false matrix: - branch: [release-1.9] - master: [master] + from: + - release-1.9 + to: + - release-1.11 + - master steps: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 @@ -1000,14 +1003,19 @@ jobs: restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- - name: Create branch directory - run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }} + run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.from }} - - name: Check out branch - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: - ref: ${{ matrix.branch }} + ref: ${{ matrix.from }} fetch-depth: 1 - path: test/e2e/kube-ovn/branches/${{ matrix.branch }} + path: test/e2e/kube-ovn/branches/${{ matrix.from }} + + - uses: actions/checkout@v3 + with: + ref: ${{ matrix.to }} + fetch-depth: 1 + path: test/e2e/kube-ovn/branches/${{ matrix.to }} - name: Install kind uses: helm/kind-action@v1 @@ -1019,7 +1027,6 @@ jobs: run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo - name: Create kind cluster - working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} run: | sudo pip3 install j2cli sudo pip3 install "j2cli[yaml]" @@ -1028,33 +1035,22 @@ jobs: sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN - working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + working-directory: test/e2e/kube-ovn/branches/${{ matrix.from }} run: | - docker pull kubeovn/kube-ovn:$(cat VERSION) - make kind-install-chart - kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s - kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s - kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s - - - name: Check out branch - uses: actions/checkout@v3 - with: - ref: ${{ matrix.master }} - fetch-depth: 1 - path: test/e2e/kube-ovn/branches/${{ matrix.master }} + version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"') + docker pull kubeovn/kube-ovn:$version + VERSION=$version make kind-install-chart - - name: Install Kube-OVN - working-directory: test/e2e/kube-ovn/branches/${{ matrix.master }} + - name: Upgrade Kube-OVN + working-directory: test/e2e/kube-ovn/branches/${{ matrix.to }} run: | - docker pull kubeovn/kube-ovn:$(cat VERSION) - make kind-upgrade-chart - kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s - kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s - kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s + version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"') + docker pull kubeovn/kube-ovn:$version + VERSION=$version make kind-upgrade-chart - name: Run E2E run: make k8s-conformance-e2e - name: Cleanup - working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }} + working-directory: test/e2e/kube-ovn/branches/${{ matrix.to }} run: sh dist/images/cleanup.sh diff --git a/Makefile b/Makefile index 09f4b9544cb..3b76c34a8ce 100644 --- a/Makefile +++ b/Makefile @@ -296,26 +296,31 @@ kind-untaint-control-plane: done; \ done -.PHONY: kind-upgrade-chart -kind-upgrade-chart: - $(eval MASTERNODES = $(shell docker exec -i kube-ovn-control-plane kubectl get nodes -l node-role.kubernetes.io/control-plane="" -o jsonpath='{.items[*].status.addresses[].address}')) - $(eval EMPTY := ) - $(eval SPACE := $(EMPTY)) - $(eval MASTERS = $(subst SPACE,,,$(strip $$(MASTERNODES)))) - helm upgrade --debug kubeovn ./kubeovn-helm --set MASTER_NODES=$(MASTERNODES) - kubectl -n kube-system get pods -o wide - .PHONY: kind-install-chart kind-install-chart: kind-load-image kind-untaint-control-plane - kubectl label no -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite - kubectl label no -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite - kubectl label no -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite - $(eval MASTERNODES = $(shell docker exec -i kube-ovn-control-plane kubectl get nodes -l node-role.kubernetes.io/control-plane="" -o jsonpath='{.items[*].status.addresses[].address}')) - $(eval EMPTY := ) - $(eval SPACE := $(EMPTY)) - $(eval MASTERS = $(subst SPACE,,,$(strip $$(MASTERNODES)))) - helm install kubeovn ./kubeovn-helm --set MASTER_NODES=$(MASTERNODES) - kubectl -n kube-system get pods -o wide + kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite + kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite + kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite + ips=$$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}') && \ + helm install kubeovn ./kubeovn-helm \ + --set global.images.kubeovn.tag=$(VERSION) \ + --set MASTER_NODES="$$(echo $$ips | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g')" + kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s + kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 120s + kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s + kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s + kubectl rollout status deployment/coredns -n kube-system --timeout 60s + +.PHONY: kind-upgrade-chart +kind-upgrade-chart: + $(eval OVN_DB_IPS = $(shell kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g')) + helm upgrade --debug kubeovn ./kubeovn-helm \ + --set global.images.kubeovn.tag=$(VERSION) \ + --set MASTER_NODES='$(OVN_DB_IPS)' + kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s + kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s + kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s + kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s .PHONY: kind-install kind-install: kind-load-image