Merge pull request #401 from sofastack/zzm-231212-symmetric-deployment

symmetric deploy

.github/workflows/module_controller_ci_build_batch_symmetric_deploy_to_aliyun.yml

@@ -0,0 +1,335 @@
+name: Module Controller Integration Test Symmetric batch deploy
+run-name: ${{ github.actor }} pushed module-controller code
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'module-controller/**'
+
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'module-controller/**'
+
+  # enable manually running the workflow
+  workflow_dispatch:
+
+env:
+  CGO_ENABLED: 0
+  GOOS: linux
+  WORK_DIR: module-controller
+  TAG: ci-test-master-latest
+  DOCKERHUB_REGISTRY: serverless-registry.cn-shanghai.cr.aliyuncs.com
+  MODULE_CONTROLLER_IMAGE_PATH: opensource/test/module-controller
+  INTEGRATION_TESTS_IMAGE_PATH: opensource/test/module-controller-integration-tests
+  POD_NAMESPACE: default
+
+defaults:
+  run:
+    working-directory: module-controller
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker login
+        uses: docker/login-action@v2.2.0
+        with:
+          registry: ${{ env.DOCKERHUB_REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          logout: false
+
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ hashFiles('${{ env.WORK_DIR }}/*Dockerfile') }}
+
+      - name: Build and push module-controller Docker images
+        uses: docker/build-push-action@v4.1.1
+        with:
+          context: ${{ env.WORK_DIR }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          file: ${{ env.WORK_DIR }}/Dockerfile
+          platforms: linux/amd64
+          push: true
+          tags: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.MODULE_CONTROLLER_IMAGE_PATH }}:${{ env.TAG }}
+
+      - run: sleep 30
+
+      - name: Set up Minikube
+        run: |
+          curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+          sudo install minikube-linux-amd64 /usr/local/bin/minikube
+
+      - name: Start Minikube
+        run: minikube start
+
+      - name: Prepare development env
+        run: |
+          kubectl apply -f config/crd/bases/serverless.alipay.com_moduledeployments.yaml
+          kubectl apply -f config/crd/bases/serverless.alipay.com_modulereplicasets.yaml
+          kubectl apply -f config/crd/bases/serverless.alipay.com_modules.yaml
+          kubectl apply -f config/crd/bases/serverless.alipay.com_moduletemplates.yaml
+          kubectl apply -f config/rbac/role.yaml
+          kubectl apply -f config/rbac/role_binding.yaml
+          kubectl apply -f config/rbac/service_account.yaml
+          kubectl apply -f config/samples/ci/dynamic-stock-batch-deployment.yaml
+          kubectl apply -f config/samples/ci/module-deployment-controller.yaml
+          kubectl apply -f config/samples/ci/dynamic-stock-service.yaml
+
+      - run: sleep 60
+
+      - name: minikube logs
+        run: minikube logs
+
+      - name: get pod
+        run: |
+          kubectl get pod
+
+      - name: wait base pod available
+        run: |
+          kubectl wait --for=condition=available deployment/dynamic-stock-deployment --timeout=300s
+
+      - name: wait module controller pod available
+        run: |
+          kubectl wait --for=condition=available deployment/module-controller --timeout=300s
+
+      - name: apply moduledeployment symmetric batch release
+        run: |
+          kubectl apply -f config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml
+
+      - name: get moduledeployment
+        run: |
+          kubectl get moduledeployment
+
+      - name: get modulereplicaset
+        run: |
+          kubectl get modulereplicaset
+
+      - run: sleep 15
+
+      - name: get module
+        run: |
+          kubectl get module -oyaml
+
+      - name: exist module
+        run: |
+          moduleCount=$(kubectl get module | wc -l)
+          if [[ $moduleCount -lt 1 ]]; then
+            echo "ERROR: 创建首批 module 资源失败"
+            exit 1
+          fi
+
+      - name: wait module available
+        run: |
+          # 定义要等待的资源名称和字段值
+          modulename=$(kubectl get module -o name)
+          desired_field_value="Available"
+          
+          # 定义等待的超时时间（以秒为单位）
+          timeout_seconds=300
+          
+          start_time=$(date +%s)
+          end_time=$((start_time + timeout_seconds))
+          
+          while true; do
+            current_time=$(date +%s)
+            if [ $current_time -gt $end_time ]; then
+              echo "等待超时"
+              exit 1
+            fi
+          
+            # 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
+            field_value=$(kubectl get $modulename -o custom-columns=STATUS:.status.status --no-headers)
+          
+            if [ "$field_value" == "$desired_field_value" ]; then
+              echo "字段值已满足条件"
+              exit 0
+            else
+              echo "等待字段值满足条件..."
+              sleep 5  # 等待一段时间后再次检查
+            fi
+          done
+
+      - name: check moduledeployment pause
+        run: |
+          # 定义要等待的资源名称和字段值
+          moduledeploymentname=$(kubectl get moduledeployment -o name)
+          desired_field_value=true
+          
+          # 定义等待的超时时间（以秒为单位）
+          timeout_seconds=300
+          
+          start_time=$(date +%s)
+          end_time=$((start_time + timeout_seconds))
+          
+          while true; do
+            current_time=$(date +%s)
+            if [ $current_time -gt $end_time ]; then
+              echo "等待超时"
+              exit 1
+            fi
+          
+            # 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
+            field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
+          
+            if [ "$field_value" == "$desired_field_value" ]; then
+              echo "字段值已满足条件，执行分组确认"
+              kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
+              exit 0
+            else
+              echo "等待字段值满足条件..."
+              sleep 5  # 等待一段时间后再次检查
+            fi
+          done
+
+      - run: sleep 15
+
+      - name: get module
+        run: |
+          kubectl get module
+
+      - name: exist module more then 1
+        run: |
+          moduleCount=$(kubectl get module | wc -l)
+          if [[ $moduleCount -lt 2 ]]; then
+            echo "ERROR: 创建第二批 module 资源失败"
+            exit 1
+          fi
+
+      - name: check moduledeployment pause
+        run: |
+          # 定义要等待的资源名称和字段值
+          moduledeploymentname=$(kubectl get moduledeployment -o name)
+          desired_field_value=true
+          
+          # 定义等待的超时时间（以秒为单位）
+          timeout_seconds=300
+          
+          start_time=$(date +%s)
+          end_time=$((start_time + timeout_seconds))
+          
+          while true; do
+            current_time=$(date +%s)
+            if [ $current_time -gt $end_time ]; then
+              echo "等待超时"
+              exit 1
+            fi
+          
+            # 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
+            field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
+          
+            if [ "$field_value" == "$desired_field_value" ]; then
+              echo "字段值已满足条件，执行分组确认"
+              kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
+              exit 0
+            else
+              echo "等待字段值满足条件..."
+              sleep 5  # 等待一段时间后再次检查
+            fi
+          done
+
+      - name: get module
+        run: |
+          kubectl get module
+
+      - name: exist module more then 2
+        run: |
+          moduleCount=$(kubectl get module | wc -l)
+          if [[ $moduleCount -lt 3 ]]; then
+            echo "ERROR: 创建第三批 module 资源失败"
+            exit 1
+          fi
+      - name: wait module available
+        run: |
+          # 定义要等待的资源类型和期望的字段值
+          moduletype="module"
+          desired_field_value="Available"
+          
+          # 定义等待的超时时间（以秒为单位）
+          timeout_seconds=300
+          
+          start_time=$(date +%s)
+          end_time=$((start_time + timeout_seconds))
+          
+          while true; do
+            current_time=$(date +%s)
+            if [ $current_time -gt $end_time ]; then
+              echo "等待超时"
+              exit 1
+            fi
+          
+            # 获取所有的资源对象名，并循环处理
+            for modulename in $(kubectl get $moduletype -o name); do
+              # 使用 kubectl get 命令获取每个资源对象的详细信息，并提取自定义字段的值
+              field_value=$(kubectl get $modulename -o custom-columns=STATUS:.status.status --no-headers)
+          
+              # 检查字段值是否满足期望
+              if [ "$field_value" != "$desired_field_value" ]; then
+                echo "等待字段值满足条件..."
+                sleep 5  # 等待一段时间后再次检查
+                continue 2  # 如果字段值未满足，则跳出循环，进入下一轮等待
+              fi
+            done
+          
+            # 如果所有资源对象的字段值都满足期望，则结束脚本
+            echo "字段值已满足条件"
+            exit 0
+          done
+
+      - name: batch release successfully then check module install
+        run: |
+          label_selector="app=dynamic-stock"
+          max_attempts=10
+          timeout=300
+          interval=30
+          
+          for ((i=0; i<$max_attempts; i++)); do
+            # 获取满足标签选择器条件的所有Pod
+            podnames=($(kubectl get pods -l $label_selector -o jsonpath='{.items[*].metadata.name}'))
+          
+            # 初始化一个变量，用于检测是否所有Pod都满足条件
+            all_pods_condition_met=true
+          
+            # 遍历所有Pod进行日志检索
+            for podname in "${podnames[@]}"; do
+                log_entry=$(kubectl exec -it $podname -- sh -c 'grep "Install Biz: provider:1.0.2 success" ~/logs/sofa-ark/*.log || true' 2>/dev/null)
+          
+                # 如果没有找到日志条目，则将标志设置为false
+                if [ -z "$log_entry" ]; then
+                  all_pods_condition_met=false
+                  break
+                fi
+              done
+          
+              # 如果所有Pod都满足条件，则退出循环
+              if $all_pods_condition_met; then
+                echo "所有Pod都满足条件。"
+                break
+              fi
+          
+              # 如果这不是最后一次尝试，则等待一段时间后继续
+              if [ $i -lt $((max_attempts-1)) ]; then
+                echo "有些Pod未满足条件。等待 $interval 秒后进行下一次尝试。"
+              sleep $interval
+              else
+                # 如果是最后一次尝试，则输出超时消息
+                echo "已达到最大尝试次数。有些Pod未满足条件。"
+              fi
+          done
+
+      - name: delete deployment
+        run: |
+          kubectl delete -n default deployment dynamic-stock-deployment

module-controller/config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml

@@ -0,0 +1,28 @@
+apiVersion: serverless.alipay.com/v1alpha1
+kind: ModuleDeployment
+metadata:
+  labels:
+    app.kubernetes.io/name: moduledeployment
+    app.kubernetes.io/instance: moduledeployment-sample
+    app.kubernetes.io/part-of: module-controller
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: module-controller
+  name: moduledeployment-sample-provider
+spec:
+  baseDeploymentName: dynamic-stock-deployment
+  template:
+    spec:
+      module:
+        name: provider
+        version: '1.0.2'
+        url: http://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/dynamic-provider-1.0.2-ark-biz.jar
+  replicas: -1
+  operationStrategy:
+    needConfirm: true
+    grayTimeBetweenBatchSeconds: 120
+    useBeta: false
+    batchCount: 3
+    upgradePolicy: install_then_uninstall
+  schedulingStrategy:
+    schedulingPolicy: scatter
+

module-controller/internal/controller/moduledeployment_controller.go

@@ -81,6 +81,16 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{}, utils.Error(err, "Failed to get moduleDeployment", "moduleDeploymentName", moduleDeployment.Name)
 	}
 
+	// update symmetric moduleDeployment replicas
+	if moduleDeployment.Spec.Replicas == -1 {
+		deployment := &v1.Deployment{}
+		err := r.Client.Get(ctx, types.NamespacedName{Namespace: moduleDeployment.Namespace, Name: moduleDeployment.Spec.BaseDeploymentName}, deployment)
+		if err != nil {
+			return ctrl.Result{}, utils.Error(err, "Failed to get deployment", "deploymentName", deployment.Name)
+		}
+		moduleDeployment.Spec.Replicas = deployment.Status.AvailableReplicas
+	}
+
 	if moduleDeployment.DeletionTimestamp != nil {
 		event.PublishModuleDeploymentDeleteEvent(r.Client, ctx, moduleDeployment)
 		if !utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer) &&