From 3a590a37f64523c648b9fe250753973f48f6c994 Mon Sep 17 00:00:00 2001 From: zzm Date: Wed, 13 Dec 2023 17:57:55 +0800 Subject: [PATCH 1/5] symmetric deploy --- .../internal/controller/moduledeployment_controller.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index 9361768cb..c032b9781 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -81,6 +81,16 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, utils.Error(err, "Failed to get moduleDeployment", "moduleDeploymentName", moduleDeployment.Name) } + // update symmetric moduleDeployment replicas + if moduleDeployment.Spec.Replicas == -1 { + deployment := &v1.Deployment{} + err := r.Client.Get(ctx, types.NamespacedName{Namespace: moduleDeployment.Namespace, Name: moduleDeployment.Spec.BaseDeploymentName}, deployment) + if err != nil { + return ctrl.Result{}, utils.Error(err, "Failed to get deployment", "deploymentName", deployment.Name) + } + moduleDeployment.Spec.Replicas = deployment.Status.AvailableReplicas + } + if moduleDeployment.DeletionTimestamp != nil { event.PublishModuleDeploymentDeleteEvent(r.Client, ctx, moduleDeployment) if !utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer) && From f5c64e4ee50f0358c7b0a7a1abe33c0a7d5b2833 Mon Sep 17 00:00:00 2001 From: zzm Date: Fri, 15 Dec 2023 10:14:05 +0800 Subject: [PATCH 2/5] symmetric deploy --- ...controller_operation_strategy_suit_test.go | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go b/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go index 886127daf..e1af0f3ae 100644 --- a/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go +++ b/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go @@ -174,6 +174,176 @@ var _ = Describe("ModuleDeployment Controller OperationStrategy Test", func() { }) }) + Context("test symmetric deployment", func() { + namespace := "module-symmetric-deployment-namespace" + namespaceObj := prepareNamespace(namespace) + deployment := prepareDeployment(namespace) + moduleDeploymentName := "module-symmetric-deployment-test" + moduleDeployment := utils.PrepareModuleDeployment(namespace, moduleDeploymentName) + nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} + // personal params + moduleDeployment.Spec.Replicas = -1 + moduleDeployment.Spec.OperationStrategy.NeedConfirm = false + moduleDeployment.Spec.OperationStrategy.BatchCount = 1 + It("1 prepare namespace", func() { + Eventually(func() error { + err := k8sClient.Create(context.TODO(), &namespaceObj) + if err != nil { + return err + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("2 prepare deployment", func() { + Eventually(func() error { + derr := k8sClient.Create(context.TODO(), &deployment) + if derr != nil { + return derr + } + + // mock + i := int32(3) + deployment.Spec.Replicas = &i + umderr2 := k8sClient.Update(context.TODO(), &deployment) + if umderr2 != nil { + return umderr2 + } + + deployment.Status.Replicas = 3 + deployment.Status.ReadyReplicas = 3 + deployment.Status.AvailableReplicas = 3 + umderr := k8sClient.Status().Update(context.TODO(), &deployment) + if umderr != nil { + return umderr + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("3 prepare moduleDeployment", func() { + Eventually(func() error { + nn := types.NamespacedName{Namespace: namespace, Name: deployment.ObjectMeta.Name} + err2 := k8sClient.Get(context.TODO(), nn, &deployment) + if err2 != nil { + return err2 + } + + mderr := k8sClient.Create(context.TODO(), &moduleDeployment) + if mderr != nil { + return mderr + } + + return nil + }, timeout, interval).Should(Succeed()) + + }) + + It("4 prepare pod 1", func() { + Eventually(func() error { + pod1 := preparePod(namespace, "fake-pod-sym-1") + if err := k8sClient.Create(context.TODO(), &pod1); err != nil { + return err + } + + // when install module, the podIP is necessary + pod1.Status.PodIP = "127.0.0.1" + if perr := k8sClient.Status().Update(context.TODO(), &pod1); perr != nil { + return perr + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + It("5 prepare pod 2", func() { + Eventually(func() error { + pod2 := preparePod(namespace, "fake-pod-sym-2") + if err := k8sClient.Create(context.TODO(), &pod2); err != nil { + return err + } + + // when install module, the podIP is necessary + pod2.Status.PodIP = "127.0.0.1" + if perr := k8sClient.Status().Update(context.TODO(), &pod2); perr != nil { + return perr + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + It("6 prepare pod 3", func() { + Eventually(func() error { + pod3 := preparePod(namespace, "fake-pod-sym-3") + if err := k8sClient.Create(context.TODO(), &pod3); err != nil { + return err + } + + // when install module, the podIP is necessary + pod3.Status.PodIP = "127.0.0.1" + if perr := k8sClient.Status().Update(context.TODO(), &pod3); perr != nil { + return perr + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("7 wait replicaset created", func() { + Eventually(func() bool { + set := map[string]string{label.ModuleDeploymentLabel: moduleDeployment.Name} + replicaSetList := &v1alpha1.ModuleReplicaSetList{} + err := k8sClient.List(context.TODO(), replicaSetList, &client.ListOptions{LabelSelector: labels.SelectorFromSet(set)}, client.InNamespace(moduleDeployment.Namespace)) + if err != nil { + return false + } + + return len(replicaSetList.Items) == 1 + }, timeout, interval).Should(BeTrue()) + }) + + It("8 wait the moduleDeployment is completed", func() { + Eventually(func() bool { + if k8sClient.Get(context.TODO(), nn, &moduleDeployment) != nil { + return false + } + + status := moduleDeployment.Status.ReleaseStatus + if status == nil { + return false + } + + return status.Progress == v1alpha1.ModuleDeploymentReleaseProgressCompleted + }, timeout, interval).Should(BeTrue()) + }) + + It("9 check the moduleDeployment replicas", func() { + Eventually(func() bool { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return false + } + + return moduleDeployment.Spec.Replicas == 3 + }, timeout, interval).Should(BeTrue()) + }) + + It("10 check replicaSet replicas", func() { + Eventually(func() error { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return err + } + + return checkModuleDeploymentReplicas(types.NamespacedName{Namespace: moduleDeployment.Namespace, Name: moduleDeploymentName}, 3) + }, timeout, interval).Should(Succeed()) + }) + + It("11 delete moduleDeployment", func() { + Expect(k8sClient.Delete(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + }) + Context("test batchConfirm strategy", func() { moduleDeploymentName := "module-deployment-test-for-batch-confirm" nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} From 47b4ed7e8184c09b3900aa147aa7d5cb5e502eb1 Mon Sep 17 00:00:00 2001 From: zzm Date: Fri, 15 Dec 2023 13:41:34 +0800 Subject: [PATCH 3/5] symmetric deploy --- ...controller_operation_strategy_suit_test.go | 47 ++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go b/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go index e1af0f3ae..915547eb0 100644 --- a/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go +++ b/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go @@ -225,12 +225,6 @@ var _ = Describe("ModuleDeployment Controller OperationStrategy Test", func() { It("3 prepare moduleDeployment", func() { Eventually(func() error { - nn := types.NamespacedName{Namespace: namespace, Name: deployment.ObjectMeta.Name} - err2 := k8sClient.Get(context.TODO(), nn, &deployment) - if err2 != nil { - return err2 - } - mderr := k8sClient.Create(context.TODO(), &moduleDeployment) if mderr != nil { return mderr @@ -338,7 +332,46 @@ var _ = Describe("ModuleDeployment Controller OperationStrategy Test", func() { }, timeout, interval).Should(Succeed()) }) - It("11 delete moduleDeployment", func() { + }) + + Context("test symmetric deployment err", func() { + namespace := "module-symmetric-deployment-namespace" + moduleDeploymentName := "module-symmetric-deployment-test-2" + moduleDeployment := utils.PrepareModuleDeployment(namespace, moduleDeploymentName) + nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} + + It("0 prepare moduleDeployment", func() { + Eventually(func() error { + mderr := k8sClient.Create(context.TODO(), &moduleDeployment) + if mderr != nil { + return mderr + } + + return nil + }, timeout, interval).Should(Succeed()) + + }) + + It("1 test symmetric deployment err", func() { + Eventually(func() error { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return err + } + + moduleDeployment.Spec.BaseDeploymentName = "test-err" + // personal params + moduleDeployment.Spec.Replicas = -1 + moduleDeployment.Spec.OperationStrategy.NeedConfirm = false + moduleDeployment.Spec.OperationStrategy.BatchCount = 1 + if err := k8sClient.Update(context.TODO(), &moduleDeployment); err != nil { + return err + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("2 delete moduleDeployment", func() { Expect(k8sClient.Delete(context.TODO(), &moduleDeployment)).Should(Succeed()) }) From b8309aaeda92e71675645e428412331ff94fdc36 Mon Sep 17 00:00:00 2001 From: zzm Date: Fri, 15 Dec 2023 15:27:05 +0800 Subject: [PATCH 4/5] symmetric deploy ci --- ...build_batch_symmetric_deploy_to_aliyun.yml | 335 ++++++++++++++++++ ...yment_symmetric_batch_deploy_provider.yaml | 28 ++ 2 files changed, 363 insertions(+) create mode 100644 .github/workflows/module_controller_ci_build_batch_symmetric_deploy_to_aliyun.yml create mode 100644 module-controller/config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml diff --git a/.github/workflows/module_controller_ci_build_batch_symmetric_deploy_to_aliyun.yml b/.github/workflows/module_controller_ci_build_batch_symmetric_deploy_to_aliyun.yml new file mode 100644 index 000000000..3a831315f --- /dev/null +++ b/.github/workflows/module_controller_ci_build_batch_symmetric_deploy_to_aliyun.yml @@ -0,0 +1,335 @@ +name: Module Controller Integration Test Symmetric batch deploy +run-name: ${{ github.actor }} pushed module-controller code + +on: + push: + branches: + - master + paths: + - 'module-controller/**' + + pull_request: + branches: + - master + paths: + - 'module-controller/**' + + # enable manually running the workflow + workflow_dispatch: + +env: + CGO_ENABLED: 0 + GOOS: linux + WORK_DIR: module-controller + TAG: ci-test-master-latest + DOCKERHUB_REGISTRY: serverless-registry.cn-shanghai.cr.aliyuncs.com + MODULE_CONTROLLER_IMAGE_PATH: opensource/test/module-controller + INTEGRATION_TESTS_IMAGE_PATH: opensource/test/module-controller-integration-tests + POD_NAMESPACE: default + +defaults: + run: + working-directory: module-controller + +jobs: + unit-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Docker login + uses: docker/login-action@v2.2.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + logout: false + + - name: Set up Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('${{ env.WORK_DIR }}/*Dockerfile') }} + + - name: Build and push module-controller Docker images + uses: docker/build-push-action@v4.1.1 + with: + context: ${{ env.WORK_DIR }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + file: ${{ env.WORK_DIR }}/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.MODULE_CONTROLLER_IMAGE_PATH }}:${{ env.TAG }} + + - run: sleep 30 + + - name: Set up Minikube + run: | + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + + - name: Start Minikube + run: minikube start + + - name: Prepare development env + run: | + kubectl apply -f config/crd/bases/serverless.alipay.com_moduledeployments.yaml + kubectl apply -f config/crd/bases/serverless.alipay.com_modulereplicasets.yaml + kubectl apply -f config/crd/bases/serverless.alipay.com_modules.yaml + kubectl apply -f config/crd/bases/serverless.alipay.com_moduletemplates.yaml + kubectl apply -f config/rbac/role.yaml + kubectl apply -f config/rbac/role_binding.yaml + kubectl apply -f config/rbac/service_account.yaml + kubectl apply -f config/samples/ci/dynamic-stock-batch-deployment.yaml + kubectl apply -f config/samples/ci/module-deployment-controller.yaml + kubectl apply -f config/samples/ci/dynamic-stock-service.yaml + + - run: sleep 60 + + - name: minikube logs + run: minikube logs + + - name: get pod + run: | + kubectl get pod + + - name: wait base pod available + run: | + kubectl wait --for=condition=available deployment/dynamic-stock-deployment --timeout=300s + + - name: wait module controller pod available + run: | + kubectl wait --for=condition=available deployment/module-controller --timeout=300s + + - name: apply moduledeployment symmetric batch release + run: | + kubectl apply -f config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml + + - name: get moduledeployment + run: | + kubectl get moduledeployment + + - name: get modulereplicaset + run: | + kubectl get modulereplicaset + + - run: sleep 15 + + - name: get module + run: | + kubectl get module -oyaml + + - name: exist module + run: | + moduleCount=$(kubectl get module | wc -l) + if [[ $moduleCount -lt 1 ]]; then + echo "ERROR: 创建首批 module 资源失败" + exit 1 + fi + + - name: wait module available + run: | + # 定义要等待的资源名称和字段值 + modulename=$(kubectl get module -o name) + desired_field_value="Available" + + # 定义等待的超时时间（以秒为单位） + timeout_seconds=300 + + start_time=$(date +%s) + end_time=$((start_time + timeout_seconds)) + + while true; do + current_time=$(date +%s) + if [ $current_time -gt $end_time ]; then + echo "等待超时" + exit 1 + fi + + # 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值 + field_value=$(kubectl get $modulename -o custom-columns=STATUS:.status.status --no-headers) + + if [ "$field_value" == "$desired_field_value" ]; then + echo "字段值已满足条件" + exit 0 + else + echo "等待字段值满足条件..." + sleep 5 # 等待一段时间后再次检查 + fi + done + + - name: check moduledeployment pause + run: | + # 定义要等待的资源名称和字段值 + moduledeploymentname=$(kubectl get moduledeployment -o name) + desired_field_value=true + + # 定义等待的超时时间（以秒为单位） + timeout_seconds=300 + + start_time=$(date +%s) + end_time=$((start_time + timeout_seconds)) + + while true; do + current_time=$(date +%s) + if [ $current_time -gt $end_time ]; then + echo "等待超时" + exit 1 + fi + + # 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值 + field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers) + + if [ "$field_value" == "$desired_field_value" ]; then + echo "字段值已满足条件，执行分组确认" + kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge + exit 0 + else + echo "等待字段值满足条件..." + sleep 5 # 等待一段时间后再次检查 + fi + done + + - run: sleep 15 + + - name: get module + run: | + kubectl get module + + - name: exist module more then 1 + run: | + moduleCount=$(kubectl get module | wc -l) + if [[ $moduleCount -lt 2 ]]; then + echo "ERROR: 创建第二批 module 资源失败" + exit 1 + fi + + - name: check moduledeployment pause + run: | + # 定义要等待的资源名称和字段值 + moduledeploymentname=$(kubectl get moduledeployment -o name) + desired_field_value=true + + # 定义等待的超时时间（以秒为单位） + timeout_seconds=300 + + start_time=$(date +%s) + end_time=$((start_time + timeout_seconds)) + + while true; do + current_time=$(date +%s) + if [ $current_time -gt $end_time ]; then + echo "等待超时" + exit 1 + fi + + # 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值 + field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers) + + if [ "$field_value" == "$desired_field_value" ]; then + echo "字段值已满足条件，执行分组确认" + kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge + exit 0 + else + echo "等待字段值满足条件..." + sleep 5 # 等待一段时间后再次检查 + fi + done + + - name: get module + run: | + kubectl get module + + - name: exist module more then 2 + run: | + moduleCount=$(kubectl get module | wc -l) + if [[ $moduleCount -lt 3 ]]; then + echo "ERROR: 创建第三批 module 资源失败" + exit 1 + fi + - name: wait module available + run: | + # 定义要等待的资源类型和期望的字段值 + moduletype="module" + desired_field_value="Available" + + # 定义等待的超时时间（以秒为单位） + timeout_seconds=300 + + start_time=$(date +%s) + end_time=$((start_time + timeout_seconds)) + + while true; do + current_time=$(date +%s) + if [ $current_time -gt $end_time ]; then + echo "等待超时" + exit 1 + fi + + # 获取所有的资源对象名，并循环处理 + for modulename in $(kubectl get $moduletype -o name); do + # 使用 kubectl get 命令获取每个资源对象的详细信息，并提取自定义字段的值 + field_value=$(kubectl get $modulename -o custom-columns=STATUS:.status.status --no-headers) + + # 检查字段值是否满足期望 + if [ "$field_value" != "$desired_field_value" ]; then + echo "等待字段值满足条件..." + sleep 5 # 等待一段时间后再次检查 + continue 2 # 如果字段值未满足，则跳出循环，进入下一轮等待 + fi + done + + # 如果所有资源对象的字段值都满足期望，则结束脚本 + echo "字段值已满足条件" + exit 0 + done + + - name: batch release successfully then check module install + run: | + label_selector="app=dynamic-stock" + max_attempts=10 + timeout=300 + interval=30 + + for ((i=0; i<$max_attempts; i++)); do + # 获取满足标签选择器条件的所有Pod + podnames=($(kubectl get pods -l $label_selector -o jsonpath='{.items[*].metadata.name}')) + + # 初始化一个变量，用于检测是否所有Pod都满足条件 + all_pods_condition_met=true + + # 遍历所有Pod进行日志检索 + for podname in "${podnames[@]}"; do + log_entry=$(kubectl exec -it $podname -- sh -c 'grep "Install Biz: provider:1.0.2 success" ~/logs/sofa-ark/*.log || true' 2>/dev/null) + + # 如果没有找到日志条目，则将标志设置为false + if [ -z "$log_entry" ]; then + all_pods_condition_met=false + break + fi + done + + # 如果所有Pod都满足条件，则退出循环 + if $all_pods_condition_met; then + echo "所有Pod都满足条件。" + break + fi + + # 如果这不是最后一次尝试，则等待一段时间后继续 + if [ $i -lt $((max_attempts-1)) ]; then + echo "有些Pod未满足条件。等待 $interval 秒后进行下一次尝试。" + sleep $interval + else + # 如果是最后一次尝试，则输出超时消息 + echo "已达到最大尝试次数。有些Pod未满足条件。" + fi + done + + - name: delete deployment + run: | + kubectl delete -n default deployment dynamic-stock-deployment \ No newline at end of file diff --git a/module-controller/config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml b/module-controller/config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml new file mode 100644 index 000000000..37c6e9aec --- /dev/null +++ b/module-controller/config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml @@ -0,0 +1,28 @@ +apiVersion: serverless.alipay.com/v1alpha1 +kind: ModuleDeployment +metadata: + labels: + app.kubernetes.io/name: moduledeployment + app.kubernetes.io/instance: moduledeployment-sample + app.kubernetes.io/part-of: module-controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: module-controller + name: moduledeployment-sample-provider +spec: + baseDeploymentName: dynamic-stock-deployment + template: + spec: + module: + name: provider + version: '1.0.2' + url: http://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/dynamic-provider-1.0.2-ark-biz.jar + replicas: -1 + operationStrategy: + needConfirm: true + grayTimeBetweenBatchSeconds: 120 + useBeta: false + batchCount: 3 + upgradePolicy: install_then_uninstall + schedulingStrategy: + schedulingPolicy: scatter + From 91090ade4017a060815ab13c6e574cceca9a21ff Mon Sep 17 00:00:00 2001 From: zzm Date: Tue, 2 Jan 2024 10:07:19 +0800 Subject: [PATCH 5/5] symmetric deploy ut group and beta --- ...controller_operation_strategy_suit_test.go | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go b/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go index 915547eb0..42587249a 100644 --- a/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go +++ b/module-controller/internal/controller/moduledeployment_controller_operation_strategy_suit_test.go @@ -517,6 +517,155 @@ var _ = Describe("ModuleDeployment Controller OperationStrategy Test", func() { }) }) + Context("test symmetric batchConfirm strategy", func() { + moduleDeploymentName := "module-deployment-test-for-symmetric-batch-confirm" + nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} + moduleDeployment := utils.PrepareModuleDeployment(namespace, moduleDeploymentName) + moduleDeployment.Spec.Replicas = -1 + moduleDeployment.Spec.OperationStrategy.NeedConfirm = true + moduleDeployment.Spec.OperationStrategy.BatchCount = 2 + + It("0 prepare deployment", func() { + Eventually(func() error { + deployment.Status.Replicas = 2 + deployment.Status.ReadyReplicas = 2 + deployment.Status.AvailableReplicas = 2 + umderr := k8sClient.Status().Update(context.TODO(), &deployment) + if umderr != nil { + return umderr + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("0. prepare 2 pods", func() { + Eventually(func() bool { + pod := preparePod(namespace, "fake-pod-5") + if err := k8sClient.Create(context.TODO(), &pod); err != nil { + return false + } + pod.Status.PodIP = "127.0.0.1" + if k8sClient.Status().Update(context.TODO(), &pod) != nil { + return false + } + + pod2 := preparePod(namespace, "fake-pod-6") + if err := k8sClient.Create(context.TODO(), &pod2); err != nil { + return false + } + pod2.Status.PodIP = "127.0.0.1" + return k8sClient.Status().Update(context.TODO(), &pod2) == nil + }, timeout, interval).Should(BeTrue()) + }) + + It("1. create a new moduleDeployment", func() { + Expect(k8sClient.Create(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + It("2. check if the replicas is 1", func() { + Eventually(func() error { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return err + } + + if !moduleDeployment.Spec.Pause { + return fmt.Errorf("the deployment is not paused") + } + + return checkModuleDeploymentReplicas( + types.NamespacedName{ + Name: moduleDeploymentName, + Namespace: moduleDeployment.Namespace}, 1) + }, timeout, interval).Should(Succeed()) + }) + + It("3. resume", func() { + Eventually(func() bool { + Expect(k8sClient.Get(context.TODO(), nn, &moduleDeployment)).Should(Succeed()) + + moduleDeployment.Spec.Pause = false + return Expect(k8sClient.Update(context.TODO(), &moduleDeployment)).Should(Succeed()) + }, timeout, interval).Should(BeTrue()) + }) + + It("4. check if the moduleDeployment status is completed", func() { + Eventually(func() bool { + if k8sClient.Get(context.TODO(), nn, &moduleDeployment) != nil { + return false + } + + if moduleDeployment.Spec.Pause != false { + return false + } + + return moduleDeployment.Status.ReleaseStatus.Progress == v1alpha1.ModuleDeploymentReleaseProgressCompleted + }, timeout, interval).Should(BeTrue()) + }) + + It("5. add another finalizer to prevent module-deployment from being deleted ", func() { + utils.AddFinalizer(&moduleDeployment.ObjectMeta, "test") + Expect(k8sClient.Update(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + It("6. delete moduleDeployment", func() { + Expect(k8sClient.Delete(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + It("7. check if the replicas is 1", func() { + Eventually(func() error { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return err + } + + if !moduleDeployment.Spec.Pause { + return fmt.Errorf("the deployment is not paused") + } + + return checkModuleDeploymentReplicas( + types.NamespacedName{ + Name: moduleDeploymentName, + Namespace: moduleDeployment.Namespace}, 1) + }, timeout, interval).Should(Succeed()) + }) + + It("8. resume", func() { + Eventually(func() bool { + Expect(k8sClient.Get(context.TODO(), nn, &moduleDeployment)).Should(Succeed()) + + moduleDeployment.Spec.Pause = false + return Expect(k8sClient.Update(context.TODO(), &moduleDeployment)).Should(Succeed()) + }, timeout, interval).Should(BeTrue()) + }) + + It("9. check if the moduleDeployment status is Terminated", func() { + Eventually(func() error { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return err + } + + if moduleDeployment.Spec.Pause != false { + return fmt.Errorf("the module-deployment is paused") + } + + if moduleDeployment.Status.ReleaseStatus == nil { + return fmt.Errorf("release status is nil") + } + + if moduleDeployment.Status.ReleaseStatus.Progress != v1alpha1.ModuleDeploymentReleaseProgressTerminated { + return fmt.Errorf("expect status %v, but got %v", + v1alpha1.ModuleDeploymentReleaseProgressTerminated, moduleDeployment.Status.ReleaseStatus.Progress) + } + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("10. clean module-deployment", func() { + utils.RemoveFinalizer(&moduleDeployment.ObjectMeta, "test") + Expect(k8sClient.Update(context.TODO(), &moduleDeployment)) + }) + }) + Context("test useBeta strategy", func() { moduleDeploymentName := "module-deployment-test-for-use-beta" nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} @@ -554,6 +703,56 @@ var _ = Describe("ModuleDeployment Controller OperationStrategy Test", func() { }) }) + Context("test symmetric useBeta strategy", func() { + moduleDeploymentName := "module-deployment-test-for-symmetric-use-beta" + nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} + moduleDeployment := utils.PrepareModuleDeployment(namespace, moduleDeploymentName) + moduleDeployment.Spec.Replicas = -1 + moduleDeployment.Spec.OperationStrategy.UseBeta = true + moduleDeployment.Spec.OperationStrategy.NeedConfirm = true + moduleDeployment.Spec.OperationStrategy.BatchCount = 2 + + It("0 prepare deployment", func() { + Eventually(func() error { + deployment.Status.Replicas = 4 + deployment.Status.ReadyReplicas = 4 + deployment.Status.AvailableReplicas = 4 + umderr := k8sClient.Status().Update(context.TODO(), &deployment) + if umderr != nil { + return umderr + } + + return nil + }, timeout, interval).Should(Succeed()) + }) + + It("0. prepare pods", func() { + Eventually(func() bool { + pod := preparePod(namespace, "fake-pod-symmetric-use-beta") + pod.Labels[label.ModuleLabelPrefix+"dynamic-provider"] = "true" + if err := k8sClient.Create(context.TODO(), &pod); err != nil { + return false + } + pod.Status.PodIP = "127.0.0.1" + return k8sClient.Status().Update(context.TODO(), &pod) == nil + }, timeout, interval).Should(BeTrue()) + }) + + It("1. create a new moduleDeployment", func() { + Expect(k8sClient.Create(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + It("2. check if use Beta strategy", func() { + Eventually(func() error { + return checkModuleDeploymentReplicas(nn, 1) + }) + }) + + It("3. clean environment", func() { + Expect(k8sClient.Delete(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + }) + Context("delete module deployment", func() { It("clean module replicaset and deployment", func() { key := types.NamespacedName{