diff --git a/pkg/manager/member/utils.go b/pkg/manager/member/utils.go index b68197167b..ee251127b6 100644 --- a/pkg/manager/member/utils.go +++ b/pkg/manager/member/utils.go @@ -155,6 +155,10 @@ func imagePullFailed(pod *corev1.Pod) bool { return false } +func MemberPodName(tcName string, ordinal int32, memberType v1alpha1.MemberType) string { + return fmt.Sprintf("%s-%s-%d", tcName, memberType.String(), ordinal) +} + func TikvPodName(tcName string, ordinal int32) string { return fmt.Sprintf("%s-%d", controller.TiKVMemberName(tcName), ordinal) } diff --git a/pkg/webhook/pod/pd_deleter.go b/pkg/webhook/pod/pd_deleter.go index 6b1a3fc6be..56f5e00a02 100644 --- a/pkg/webhook/pod/pd_deleter.go +++ b/pkg/webhook/pod/pd_deleter.go @@ -15,6 +15,7 @@ package pod import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/label" pdutil "github.com/pingcap/tidb-operator/pkg/manager/member" operatorUtils "github.com/pingcap/tidb-operator/pkg/util" "github.com/pingcap/tidb-operator/pkg/webhook/util" @@ -27,11 +28,25 @@ func (pc *PodAdmissionControl) admitDeletePdPods(payload *admitPayload) *admissi name := payload.pod.Name namespace := payload.pod.Namespace - isInOrdinal, err := operatorUtils.IsPodOrdinalNotExceedReplicas(payload.pod, payload.ownerStatefulSet) + ordinal, err := operatorUtils.GetOrdinalFromPodName(name) if err != nil { return util.ARFail(err) } - ordinal, err := operatorUtils.GetOrdinalFromPodName(name) + + // If the pd pod is deleted by restarter, it is necessary to check former pd restart status + if _, exist := payload.pod.Annotations[label.AnnPodDeferDeleting]; exist { + existed, err := checkFormerPodRestartStatus(pc.kubeCli, v1alpha1.PDMemberType, payload.tc, namespace, ordinal, *payload.ownerStatefulSet.Spec.Replicas) + if err != nil { + return util.ARFail(err) + } + if existed { + return &admission.AdmissionResponse{ + Allowed: false, + } + } + } + + isInOrdinal, err := operatorUtils.IsPodOrdinalNotExceedReplicas(payload.pod, payload.ownerStatefulSet) if err != nil { return util.ARFail(err) } diff --git a/pkg/webhook/pod/tikv_deleter.go b/pkg/webhook/pod/tikv_deleter.go index c153af1ecc..00ef3771a2 100644 --- a/pkg/webhook/pod/tikv_deleter.go +++ b/pkg/webhook/pod/tikv_deleter.go @@ -50,6 +50,23 @@ func (pc *PodAdmissionControl) admitDeleteTiKVPods(payload *admitPayload) *admis name := pod.Name namespace := pod.Namespace tcName := tc.Name + ordinal, err := operatorUtils.GetOrdinalFromPodName(name) + if err != nil { + return util.ARFail(err) + } + + // If the tikv pod is deleted by restarter, it is necessary to check former tikv restart status + if _, exist := payload.pod.Annotations[label.AnnPodDeferDeleting]; exist { + existed, err := checkFormerPodRestartStatus(pc.kubeCli, v1alpha1.TiKVMemberType, payload.tc, namespace, ordinal, *payload.ownerStatefulSet.Spec.Replicas) + if err != nil { + return util.ARFail(err) + } + if existed { + return &admission.AdmissionResponse{ + Allowed: false, + } + } + } storesInfo, err := pdClient.GetStores() if err != nil { diff --git a/pkg/webhook/pod/util.go b/pkg/webhook/pod/util.go index a6c41686d2..55021535a6 100644 --- a/pkg/webhook/pod/util.go +++ b/pkg/webhook/pod/util.go @@ -138,3 +138,19 @@ func getOwnerStatefulSetForTiDBComponent(pod *core.Pod, kubeCli kubernetes.Inter } return kubeCli.AppsV1().StatefulSets(namespace).Get(ownerStatefulSetName, meta.GetOptions{}) } + +// checkFormerPodRestartStatus checks whether there are any former pod is going to be restarted +// return true if existed +func checkFormerPodRestartStatus(kubeCli kubernetes.Interface, memberType v1alpha1.MemberType, tc *v1alpha1.TidbCluster, namespace string, ordinal int32, replicas int32) (bool, error) { + for i := replicas - 1; i > ordinal; i-- { + podName := memberUtil.MemberPodName(tc.Name, i, memberType) + pod, err := kubeCli.CoreV1().Pods(namespace).Get(podName, meta.GetOptions{}) + if err != nil { + return false, err + } + if _, existed := pod.Annotations[label.AnnPodDeferDeleting]; existed { + return true, nil + } + } + return false, nil +}