From f3b646053361002526e6ae4fb4ad3bf3f37cbab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Mon, 4 Sep 2023 14:00:55 +0800 Subject: [PATCH] chart: fix ovs-ovn upgrade (#3164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 张祖建 --- .github/workflows/scheduled-e2e.yaml | 28 ++----- Makefile | 35 ++++----- charts/README.md | 6 -- ...post-upgrade.yaml => upgrade-ovs-ovn.yaml} | 77 ++++++++++++++++++- charts/values.yaml | 2 - dist/images/start-ovs.sh | 9 ++- dist/images/upgrade-ovs.sh | 4 +- yamls/ovn.yaml | 2 +- 8 files changed, 106 insertions(+), 57 deletions(-) rename charts/templates/{post-upgrade.yaml => upgrade-ovs-ovn.yaml} (54%) diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index 786153763e2..e8b2d356b8a 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -25,7 +25,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 ip-family: @@ -115,7 +114,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 ip-family: @@ -198,7 +196,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 ip-family: @@ -281,7 +278,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 ip-family: @@ -341,7 +337,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 ip-family: @@ -425,7 +420,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 steps: - uses: actions/checkout@v3 @@ -547,7 +541,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 steps: - uses: actions/checkout@v3 @@ -598,7 +591,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 steps: @@ -652,7 +644,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 steps: @@ -918,7 +909,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 steps: @@ -969,7 +959,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 - release-1.8 steps: @@ -1057,7 +1046,6 @@ jobs: branch: - master - release-1.11 - - release-1.10 - release-1.9 ssl: - "true" @@ -1148,8 +1136,11 @@ jobs: matrix: case: - release-1.9 => release-1.11 + - release-1.9 => release-1.12 - release-1.9 => master + - release-1.11 => release-1.12 - release-1.11 => master + - release-1.12 => master steps: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 @@ -1211,7 +1202,7 @@ jobs: run: | sudo pip3 install j2cli sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init + sudo PATH=~/.local/bin:$PATH make kind-init-ha sudo cp -r /root/.kube/ ~/.kube/ sudo chown -R $(id -un). ~/.kube/ @@ -1227,16 +1218,7 @@ jobs: run: | version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"') docker pull kubeovn/kube-ovn:$version - - restart_ovs=false - v1=$(printf "$VERSION_FROM\\nrelease-1.11" | sort -Vr | head -n1) - v2=$(printf "$VERSION_TO\\nrelease-1.12" | sort -Vr | head -n1) - if [ $v1 = "release-1.11" ]; then - if [ $VERSION_TO = "master" -o $VERSION_TO = $v2 ]; then - restart_ovs=true - fi - fi - CHART_UPGRADE_RESTART_OVS=$restart_ovs VERSION=$version make kind-upgrade-chart + VERSION=$version make kind-upgrade-chart - name: Run E2E env: diff --git a/Makefile b/Makefile index 8c4f92bbaa5..374e8be08d7 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,6 @@ endif CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/control-plane -CHART_UPGRADE_RESTART_OVS=$(shell echo $${CHART_UPGRADE_RESTART_OVS:-false}) - MULTUS_VERSION = v4.0.2 MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:$(MULTUS_VERSION)-thick MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/$(MULTUS_VERSION)/deployments/multus-daemonset-thick.yml @@ -423,30 +421,31 @@ kind-install-chart: kind-load-image kind-untaint-control-plane kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite - ips=$$(kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}') && \ + ips=$$(kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr '\n' ',' | sed 's/,$$//') && \ helm install kubeovn ./charts \ --set global.images.kubeovn.tag=$(VERSION) \ - --set replicaCount=$$(echo $$ips | awk '{print NF}') \ - --set MASTER_NODES="$$(echo $$ips | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g')" - kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s - kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 120s - kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s - kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s - kubectl rollout status deployment/coredns -n kube-system --timeout 60s + --set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \ + --set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" + sleep 60 + kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central + kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn + kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller + kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-cni + kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-pinger .PHONY: kind-upgrade-chart kind-upgrade-chart: kind-load-image - $(eval OVN_DB_IPS = $(shell kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g')) + $(eval OVN_DB_IPS = $(shell kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr '\n' ',' | sed -e 's/,$$//' -e 's/,/\\,/g')) helm upgrade kubeovn ./charts \ --set global.images.kubeovn.tag=$(VERSION) \ --set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \ - --set MASTER_NODES='$(OVN_DB_IPS)' \ - --set restart_ovs=$(CHART_UPGRADE_RESTART_OVS) - kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s - kubectl rollout status daemonset/ovs-ovn -n kube-system --timeout 120s - kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 120s - kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s - kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s + --set MASTER_NODES='$(OVN_DB_IPS)' + sleep 90 + kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central + kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn + kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller + kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-cni + kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-pinger .PHONY: kind-install kind-install: kind-load-image diff --git a/charts/README.md b/charts/README.md index 0c6e622d0e5..72dbb577230 100644 --- a/charts/README.md +++ b/charts/README.md @@ -18,9 +18,3 @@ $ helm install --debug kubeovn ./kubeovn-helm --set MASTER_NODES=${Node0},${Node # upgrade to this version $ helm upgrade --debug kubeovn ./kubeovn-helm --set MASTER_NODES=${Node0},${Node1},${Node2}, --set replicaCount=3 ``` - -If you are upgrading Kube-OVN from versions prior to v1.12, you need to set `restart_ovs` to `true`: - -```shell -$ helm upgrade --debug kubeovn ./kubeovn-helm --set MASTER_NODES=${Node0},${Node1},${Node2}, --set replicaCount=3 --set restart_ovs=true -``` diff --git a/charts/templates/post-upgrade.yaml b/charts/templates/upgrade-ovs-ovn.yaml similarity index 54% rename from charts/templates/post-upgrade.yaml rename to charts/templates/upgrade-ovs-ovn.yaml index cbf98cafd64..b565bdff43e 100644 --- a/charts/templates/post-upgrade.yaml +++ b/charts/templates/upgrade-ovs-ovn.yaml @@ -1,4 +1,72 @@ -{{ if .Values.restart_ovs }} +{{ if (lookup "apps/v1" "DaemonSet" "kube-system" "ovs-ovn") }} +{{ if eq (lookup "apps/v1" "DaemonSet" "kube-system" "ovs-ovn").spec.updateStrategy.type "OnDelete" }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovs-ovn-upgrade + namespace: kube-system + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "2" + "helm.sh/hook-delete-policy": hook-succeeded + name: system:ovs-ovn-upgrade +rules: + - apiGroups: + - apps + resources: + - daemonsets + resourceNames: + - ovs-ovn + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovs-ovn-upgrade + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/hook-delete-policy": hook-succeeded +roleRef: + name: system:ovs-ovn-upgrade + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: ovs-ovn-upgrade + namespace: kube-system +--- apiVersion: batch/v1 kind: Job metadata: @@ -13,7 +81,7 @@ metadata: # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. "helm.sh/hook": post-upgrade - "helm.sh/hook-weight": "-5" + "helm.sh/hook-weight": "4" "helm.sh/hook-delete-policy": hook-succeeded spec: completions: 1 @@ -49,8 +117,8 @@ spec: hostNetwork: true nodeSelector: kubernetes.io/os: "linux" - serviceAccount: ovn - serviceAccountName: ovn + serviceAccount: ovs-ovn-upgrade + serviceAccountName: ovs-ovn-upgrade containers: - name: post-upgrade-job image: "{{ .Values.global.registry.address}}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}" @@ -71,3 +139,4 @@ spec: hostPath: path: /var/log/kube-ovn {{ end }} +{{ end }} diff --git a/charts/values.yaml b/charts/values.yaml index cfb63577436..3b51c4206bb 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -114,8 +114,6 @@ imagePullSecrets: [] nameOverride: "" fullnameOverride: "" -restart_ovs: false - # hybrid dpdk HYBRID_DPDK: false HUGEPAGE_SIZE_TYPE: hugepages-2Mi # Default diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh index 45b40cef869..997c12b8065 100755 --- a/dist/images/start-ovs.sh +++ b/dist/images/start-ovs.sh @@ -55,7 +55,14 @@ function quit { pid=$(/usr/share/ovn/scripts/ovn-ctl status_controller | awk '{print $NF}') if cgroup_match "${pid}" self; then /usr/share/ovn/scripts/grace_stop_ovn_controller - /usr/share/openvswitch/scripts/ovs-ctl stop + fi + pid=$(/usr/share/openvswitch/scripts/ovs-ctl status | grep ovsdb-server | awk '{print $NF}') + if cgroup_match "${pid}" self; then + /usr/share/openvswitch/scripts/ovs-ctl --no-ovs-vswitchd stop + fi + pid=$(/usr/share/openvswitch/scripts/ovs-ctl status | grep ovs-vswitchd | awk '{print $NF}') + if cgroup_match "${pid}" self; then + /usr/share/openvswitch/scripts/ovs-ctl --no-ovsdb-server stop fi fi diff --git a/dist/images/upgrade-ovs.sh b/dist/images/upgrade-ovs.sh index a5d6c9c2226..6ecabecae9b 100755 --- a/dist/images/upgrade-ovs.sh +++ b/dist/images/upgrade-ovs.sh @@ -5,9 +5,9 @@ set -e POD_NAMESPACE=${POD_NAMESPACE:-kube-system} dsGenVer=`kubectl -n $POD_NAMESPACE get ds ovs-ovn -o jsonpath={.metadata.generation}` +kubectl -n $POD_NAMESPACE delete pod -l app=ovs,pod-template-generation!=$dsGenVer + for node in `kubectl get node -o jsonpath='{.items[*].metadata.name}'`; do - # delete pod with old version - kubectl -n $POD_NAMESPACE delete pod -l app=ovs,pod-template-generation!=$dsGenVer --field-selector spec.nodeName=$node # wait the pod with new version to be created and delete it while true; do pod=`kubectl -n $POD_NAMESPACE get pod -l app=ovs,pod-template-generation=$dsGenVer --field-selector spec.nodeName=$node -o name` diff --git a/yamls/ovn.yaml b/yamls/ovn.yaml index 49da216a132..42fd8954b99 100644 --- a/yamls/ovn.yaml +++ b/yamls/ovn.yaml @@ -214,7 +214,7 @@ spec: updateStrategy: type: RollingUpdate rollingUpdate: - maxSurge: 100% + maxSurge: 1 maxUnavailable: 0 template: metadata: