Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Merge pull request #406 from sofastack/feat.current_group_confirmation
Browse files Browse the repository at this point in the history 
添加confirm确认显式指定分批数，保证幂等
  • Loading branch information
@gold300jin
gold300jin authored Jan 22, 2024
2 parents 1ec069f + 302fde1 commit 7e5c6f2
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 71 deletions.
42 changes: 18 additions & 24 deletions .github/workflows/module_controller_ci_build_batch_deploy_to_aliyun.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -183,11 +182,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand All @@ -213,7 +212,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -229,11 +227,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "1" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":2}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down Expand Up @@ -353,7 +351,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -369,11 +366,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand All @@ -400,7 +397,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -416,11 +412,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "1" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":2}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down Expand Up @@ -496,7 +492,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -512,11 +507,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down Expand Up @@ -579,7 +574,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -595,11 +589,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "1" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":2}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down
21 changes: 9 additions & 12 deletions ...ub/workflows/module_controller_ci_build_batch_scaleup_then_scaledown_deploy_to_aliyun.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -183,11 +182,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down Expand Up @@ -579,7 +578,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -595,11 +593,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down Expand Up @@ -738,7 +736,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -754,11 +751,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down
14 changes: 6 additions & 8 deletions .github/workflows/module_controller_ci_build_batch_symmetric_deploy_to_aliyun.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -183,11 +182,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "0" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":1}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand All @@ -213,7 +212,6 @@ jobs:
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间（以秒为单位）
timeout_seconds=300
Expand All @@ -229,11 +227,11 @@ jobs:
fi
# 使用 kubectl get 命令获取资源对象的详细信息，并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
field_value=$(kubectl get $moduledeploymentname -o custom-columns=CONFIRMBATCHNUM:.spec.confirmBatchNum --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
if [ "$field_value" == "1" ]; then
echo "字段值已满足条件，执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
kubectl patch $moduledeploymentname -p '{"spec":{"confirmBatchNum":2}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
Expand Down
7 changes: 3 additions & 4 deletions module-controller/api/v1alpha1/moduledeployment_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,10 +163,9 @@ type ModuleDeploymentSpec struct {

ProgressDeadlineSeconds int32 `json:"progressDeadlineSeconds,omitempty"`

// Indicates that the moduleDeployment is paused and will not be processed by the
// moduleDeployment controller.
// +optional
Pause bool `json:"pause,omitempty"`
// +kubebuilder:default:=0
// +kubebuilder:validation:Minimum=0
ConfirmBatchNum int32 `json:"confirmBatchNum,omitempty"`

OperationStrategy ModuleOperationStrategy `json:"operationStrategy,omitempty"`

Expand Down
9 changes: 5 additions & 4 deletions module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ spec:
Important: Run "make" to regenerate code after modifying this file'
minLength: 1
type: string
confirmBatchNum:
default: 0
format: int32
minimum: 0
type: integer
minReadySeconds:
format: int32
type: integer
Expand Down Expand Up @@ -75,10 +80,6 @@ spec:
useBeta:
type: boolean
type: object
pause:
description: Indicates that the moduleDeployment is paused and will
not be processed by the moduleDeployment controller.
type: boolean
progressDeadlineSeconds:
format: int32
type: integer
Expand Down
23 changes: 16 additions & 7 deletions module-controller/internal/controller/moduledeployment_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,10 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
event.PublishModuleDeploymentCreateEvent(r.Client, ctx, moduleDeployment)
}

if moduleDeployment.Spec.Pause {
if moduleDeployment.Status.ReleaseStatus != nil &&
moduleDeployment.Spec.ConfirmBatchNum < moduleDeployment.Status.ReleaseStatus.CurrentBatch &&
moduleDeployment.Status.ReleaseStatus.Progress == v1alpha1.ModuleDeploymentReleaseProgressPaused {

return ctrl.Result{}, nil
}

Expand Down Expand Up @@ -162,19 +165,26 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
return ctrl.Result{}, err
}
}
case v1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation:
moduleDeployment.Spec.Pause = true
if err := r.Update(ctx, moduleDeployment); err != nil {
return ctrl.Result{}, err

if moduleDeployment.Spec.ConfirmBatchNum > 0 {
moduleDeployment.Spec.ConfirmBatchNum = 0
if err := utils.UpdateResource(r.Client, ctx, moduleDeployment); err != nil {
return ctrl.Result{}, err
}
}

case v1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation:

moduleDeployment.Status.ReleaseStatus.Progress = v1alpha1.ModuleDeploymentReleaseProgressPaused
log.Log.Info("update moduleDeployment releaseStatus progress to paused", "moduleDeploymentName", moduleDeployment.Name)
if err := utils.UpdateStatus(r.Client, ctx, moduleDeployment); err != nil {
return ctrl.Result{}, utils.Error(err, "update moduleDeployment releaseStatus progress to paused failed")
}
case v1alpha1.ModuleDeploymentReleaseProgressPaused:
if !moduleDeployment.Spec.Pause && time.Since(moduleDeployment.Status.ReleaseStatus.NextReconcileTime.Time) >= 0 {
if time.Since(moduleDeployment.Status.ReleaseStatus.NextReconcileTime.Time) >= 0 &&
moduleDeployment.Spec.ConfirmBatchNum == moduleDeployment.Status.ReleaseStatus.CurrentBatch {

moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1
moduleDeployment.Status.ReleaseStatus.Progress = v1alpha1.ModuleDeploymentReleaseProgressExecuting
log.Log.Info("update moduleDeployment progress from paused to executing", "moduleDeploymentName", moduleDeployment.Name)
if err := utils.UpdateStatus(r.Client, ctx, moduleDeployment); err != nil {
Expand Down Expand Up @@ -465,7 +475,6 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context,
}
}
// TODO update current batch
moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1
moduleDeployment.Status.ReleaseStatus.BatchProgress = v1alpha1.ModuleDeploymentReleaseProgressExecuting
log.Log.Info("update moduleDeployment batch progress to executing", "moduleDeploymentName", moduleDeployment.Name)
err = utils.UpdateStatus(r.Client, ctx, moduleDeployment)
Expand Down
Loading

0 comments on commit 7e5c6f2

Please sign in to comment.