fix chart install/upgrade e2e (#2426)

kubeovn · Mar 6, 2023 · 86c01e6 · 86c01e6
1 parent 322eab3
commit 86c01e6
Show file tree

Hide file tree

Showing 7 changed files with 570 additions and 261 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -822,11 +822,7 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
-        run: |
-          make kind-install-chart
-          kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
-          kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s
-          kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s
+        run: make kind-install-chart
 
       - name: Cleanup
         run: sh dist/images/cleanup.sh

diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml
@@ -969,15 +969,17 @@ jobs:
       - name: Cleanup
         run: sh dist/images/cleanup.sh
 
-  helm-direct-upgrade-e2e:
-    name: Helm Upgrade E2E
+  chart-upgrade-e2e:
+    name: Chart Upgrade E2E
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        branch: [release-1.9]
-        master: [master]
+        case:
+          - release-1.9 => release-1.11
+          - release-1.9 => master
+          - release-1.11 => master
     steps:
       - uses: actions/checkout@v3
       - uses: azure/setup-helm@v3
@@ -1001,15 +1003,27 @@ jobs:
           key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('**/go.sum') }}
           restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
 
+      - name: Parse versions
+        run: |
+          version_from=$(echo ${{ matrix.case }} | awk '{print $1}')
+          version_to=$(echo ${{ matrix.case }} | awk '{print $3}')
+          echo "VERSION_FROM=$version_from" >> "$GITHUB_ENV"
+          echo "VERSION_TO=$version_to" >> "$GITHUB_ENV"
+
       - name: Create branch directory
-        run: mkdir -p test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        run: mkdir -p test/e2e/kube-ovn/branches
 
-      - name: Check out branch
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v3
         with:
-          ref: ${{ matrix.branch }}
+          ref: ${{ env.VERSION_FROM }}
           fetch-depth: 1
-          path: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+          path: test/e2e/kube-ovn/branches/${{ env.VERSION_FROM }}
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.VERSION_TO }}
+          fetch-depth: 1
+          path: test/e2e/kube-ovn/branches/${{ env.VERSION_TO }}
 
       - name: Install kind
         uses: helm/kind-action@v1
@@ -1021,7 +1035,6 @@ jobs:
         run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
 
       - name: Create kind cluster
-        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
         run: |
           sudo pip3 install j2cli
           sudo pip3 install "j2cli[yaml]"
@@ -1030,33 +1043,24 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
-        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        working-directory: test/e2e/kube-ovn/branches/${{ env.VERSION_FROM }}
         run: |
-          docker pull kubeovn/kube-ovn:$(cat VERSION)
-          make kind-install-chart
-          kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
-          kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s
-          kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s
-
-      - name: Check out branch
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ matrix.master }}
-          fetch-depth: 1
-          path: test/e2e/kube-ovn/branches/${{ matrix.master }}
+          version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
+          docker pull kubeovn/kube-ovn:$version
+          VERSION=$version make kind-install-chart
 
-      - name: Install Kube-OVN
-        working-directory: test/e2e/kube-ovn/branches/${{ matrix.master }}
+      - name: Upgrade Kube-OVN
+        working-directory: test/e2e/kube-ovn/branches/${{ env.VERSION_TO }}
         run: |
-          docker pull kubeovn/kube-ovn:$(cat VERSION)
-          make kind-upgrade-chart
-          kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
-          kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s
-          kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s
+          version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
+          docker pull kubeovn/kube-ovn:$version
+          VERSION=$version make kind-upgrade-chart
 
       - name: Run E2E
+        env:
+          E2E_BRANCH: ${{ env.VERSION_TO }}
         run: make k8s-conformance-e2e
 
       - name: Cleanup
-        working-directory: test/e2e/kube-ovn/branches/${{ matrix.branch }}
+        working-directory: test/e2e/kube-ovn/branches/${{ env.VERSION_TO }}
         run: sh dist/images/cleanup.sh
diff --git a/Makefile b/Makefile
@@ -292,26 +292,33 @@ kind-untaint-control-plane:
 		done; \
 	done
 
-.PHONY: kind-upgrade-chart
-kind-upgrade-chart:
-	$(eval MASTERNODES = $(shell docker exec -i kube-ovn-control-plane kubectl get nodes -l node-role.kubernetes.io/control-plane=""  -o jsonpath='{.items[*].status.addresses[].address}'))
-	$(eval EMPTY := )
-	$(eval SPACE := $(EMPTY))
-	$(eval MASTERS = $(subst SPACE,,,$(strip $$(MASTERNODES))))
-	helm upgrade --debug kubeovn ./kubeovn-helm --set MASTER_NODES=$(MASTERNODES)
-	kubectl -n kube-system get pods -o wide
-
 .PHONY: kind-install-chart
 kind-install-chart: kind-load-image kind-untaint-control-plane
-	kubectl label no -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite
-	kubectl label no -lnode-role.kubernetes.io/control-plane  kube-ovn/role=master --overwrite
-	kubectl label no -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel  --overwrite
-	$(eval MASTERNODES = $(shell docker exec -i kube-ovn-control-plane kubectl get nodes -l node-role.kubernetes.io/control-plane=""  -o jsonpath='{.items[*].status.addresses[].address}'))
-	$(eval EMPTY := )
-	$(eval SPACE := $(EMPTY))
-	$(eval MASTERS = $(subst SPACE,,,$(strip $$(MASTERNODES))))
-	helm install kubeovn ./kubeovn-helm --set MASTER_NODES=$(MASTERNODES)
-	kubectl -n kube-system get pods -o wide
+	kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite
+	kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite
+	kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite
+	ips=$$(kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}') && \
+	helm install kubeovn ./kubeovn-helm \
+		--set global.images.kubeovn.tag=$(VERSION) \
+		--set replicaCount=$$(echo $$ips | awk '{print NF}') \
+		--set MASTER_NODES="$$(echo $$ips | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g')"
+	kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
+	kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 120s
+	kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s
+	kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s
+	kubectl rollout status deployment/coredns -n kube-system --timeout 60s
+
+.PHONY: kind-upgrade-chart
+kind-upgrade-chart: kind-load-image
+	$(eval OVN_DB_IPS = $(shell kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g'))
+	helm upgrade kubeovn ./kubeovn-helm \
+		--set global.images.kubeovn.tag=$(VERSION) \
+		--set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \
+		--set MASTER_NODES='$(OVN_DB_IPS)'
+	kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
+	kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s
+	kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s
+	kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s
 
 .PHONY: kind-install
 kind-install: kind-load-image

diff --git a/kubeovn-helm/templates/kube-ovn-crd.yaml b/kubeovn-helm/templates/kube-ovn-crd.yaml
@@ -557,24 +557,34 @@ spec:
       subresources:
         status: {}
       additionalPrinterColumns:
-      - jsonPath: .spec.v4ip
-        name: IP
+      - jsonPath: .status.v4Ip
+        name: V4IP
         type: string
-      - jsonPath: .spec.macAddress
+      - jsonPath: .status.v6Ip
+        name: V6IP
+        type: string
+      - jsonPath: .status.macAddress
         name: Mac
         type: string
       - jsonPath: .spec.type
         name: Type
         type: string
+      - jsonPath: .status.ready
+        name: Ready
+        type: boolean
       schema:
         openAPIV3Schema:
           type: object
           properties:
             status:
               type: object
               properties:
+                ready:
+                  type: boolean
                 v4Ip:
                   type: string
+                v6Ip:
+                  type: string
                 macAddress:
                   type: string
                 conditions:
@@ -601,7 +611,9 @@ spec:
                   type: string
                 type:
                   type: string
-                v4ip:
+                v4Ip:
+                  type: string
+                v6Ip:
                   type: string
                 macAddress:
                   type: string
@@ -639,6 +651,12 @@ spec:
       - jsonPath: .status.ready
         name: Ready
         type: boolean
+      - jsonPath: .spec.ipType
+        name: IpType
+        type: string
+      - jsonPath: .spec.ipName
+        name: IpName
+        type: string
       schema:
         openAPIV3Schema:
           type: object
@@ -678,6 +696,8 @@ spec:
               properties:
                 ovnEip:
                   type: string
+                ipType:
+                  type: string
                 ipName:
                   type: string
 ---
@@ -767,6 +787,9 @@ spec:
         - jsonPath: .status.enableExternal
           name: EnableExternal
           type: boolean
+        - jsonPath: .status.enableBfd
+          name: EnableBfd
+          type: boolean
         - jsonPath: .status.standby
           name: Standby
           type: boolean
@@ -784,6 +807,8 @@ spec:
               properties:
                 enableExternal:
                   type: boolean
+                enableBfd:
+                  type: boolean
                 namespaces:
                   items:
                     type: string
@@ -797,6 +822,10 @@ spec:
                         type: string
                       nextHopIP:
                         type: string
+                      ecmp:
+                        type: string
+                      bfdId:
+                        type: string
                     type: object
                   type: array
                 policyRoutes:
@@ -851,6 +880,8 @@ spec:
                   type: boolean
                 enableExternal:
                   type: boolean
+                enableBfd:
+                  type: boolean
                 subnets:
                   items:
                     type: string
@@ -867,6 +898,10 @@ spec:
                   type: string
                 udpSessionLoadBalancer:
                   type: string
+                sctpLoadBalancer:
+                  type: string
+                sctpSessionLoadBalancer:
+                  type: string
               type: object
           type: object
       served: true
@@ -1204,8 +1239,6 @@ spec:
                   type: string
                 natOutgoing:
                   type: boolean
-                u2oRouting:
-                  type: boolean
                 externalEgressGateway:
                   type: string
                 policyRoutingPriority:

diff --git a/kubeovn-helm/templates/ovn-CR.yaml b/kubeovn-helm/templates/ovn-CR.yaml
@@ -57,6 +57,7 @@ rules:
       - watch
       - patch
       - update
+      - delete
   - apiGroups:
       - ""
       - networking.k8s.io

diff --git a/kubeovn-helm/templates/post-upgrade.yaml b/kubeovn-helm/templates/post-upgrade.yaml
@@ -53,4 +53,14 @@ spec:
       containers:
         - name: post-upgrade-job
           image: "{{ .Values.global.registry.address}}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}"
-          command: ["/kube-ovn/upgrade-ovs.sh"]
+          command:
+            - sh
+            - -c
+            - /kube-ovn/upgrade-ovs.sh 2>&1 | tee -a /var/log/kube-ovn/upgrade-ovs.log
+          volumeMounts:
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
+      volumes:
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn