sync pump before tidb (pingcap#2515)

sre-bot · May 21, 2020 · bb20aa5 · bb20aa5
1 parent ed779d5
commit bb20aa5
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 8 deletions.
diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go
@@ -463,6 +463,15 @@ func (tc *TidbCluster) TiKVIsAvailable() bool {
 	return true
 }
 
+func (tc *TidbCluster) PumpIsAvailable() bool {
+	var lowerLimit int32 = 1
+	if tc.Status.Pump.StatefulSet == nil || tc.Status.Pump.StatefulSet.ReadyReplicas < lowerLimit {
+		return false
+	}
+
+	return true
+}
+
 func (tc *TidbCluster) GetClusterID() string {
 	return tc.Status.ClusterID
 }

diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go
@@ -185,6 +185,11 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
 		return err
 	}
 
+	// syncing the pump cluster
+	if err := tcc.pumpMemberManager.Sync(tc); err != nil {
+		return err
+	}
+
 	// works that should do to making the tidb cluster current state match the desired state:
 	//   - waiting for the tikv cluster available(at least one peer works)
 	//   - create or update tidb headless service
@@ -230,11 +235,6 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
 		return err
 	}
 
-	// syncing the pump cluster
-	if err := tcc.pumpMemberManager.Sync(tc); err != nil {
-		return err
-	}
-
 	// syncing the some tidbcluster status attributes
 	// 	- sync tidbmonitor reference
 	return tcc.tidbClusterStatusManager.Sync(tc)

diff --git a/pkg/manager/member/pump_member_manager.go b/pkg/manager/member/pump_member_manager.go
@@ -116,6 +116,11 @@ func (pmm *pumpMemberManager) syncPumpStatefulSetForTidbCluster(tc *v1alpha1.Tid
 		return pmm.setControl.CreateStatefulSet(tc, newPumpSet)
 	}
 
+	// Wait for PD & TiKV upgrading done
+	if tc.Status.PD.Phase == v1alpha1.UpgradePhase || tc.Status.TiKV.Phase == v1alpha1.UpgradePhase {
+		return nil
+	}
+
 	return updateStatefulSet(pmm.setControl, tc, newPumpSet, oldPumpSet)
 }
 

diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go
@@ -106,7 +106,11 @@ func (tmm *tidbMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
 	if !tc.TiKVIsAvailable() {
 		return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName)
 	}
-
+	if tc.Spec.Pump != nil {
+		if !tc.PumpIsAvailable() {
+			return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for Pump cluster running", ns, tcName)
+		}
+	}
 	// Sync TiDB Headless Service
 	if err := tmm.syncTiDBHeadlessServiceForTidbCluster(tc); err != nil {
 		return err
@@ -763,7 +767,8 @@ func (tmm *tidbMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, se
 	if err != nil {
 		return err
 	}
-	if upgrading && tc.Status.TiKV.Phase != v1alpha1.UpgradePhase && tc.Status.PD.Phase != v1alpha1.UpgradePhase {
+	if upgrading && tc.Status.TiKV.Phase != v1alpha1.UpgradePhase &&
+		tc.Status.PD.Phase != v1alpha1.UpgradePhase && tc.Status.Pump.Phase != v1alpha1.UpgradePhase {
 		tc.Status.TiDB.Phase = v1alpha1.UpgradePhase
 	} else {
 		tc.Status.TiDB.Phase = v1alpha1.NormalPhase

diff --git a/pkg/manager/member/tidb_upgrader.go b/pkg/manager/member/tidb_upgrader.go
@@ -46,7 +46,8 @@ func (tdu *tidbUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stateful
 	ns := tc.GetNamespace()
 	tcName := tc.GetName()
 
-	if tc.Status.PD.Phase == v1alpha1.UpgradePhase || tc.Status.TiKV.Phase == v1alpha1.UpgradePhase {
+	if tc.Status.PD.Phase == v1alpha1.UpgradePhase || tc.Status.TiKV.Phase == v1alpha1.UpgradePhase ||
+		tc.Status.Pump.Phase == v1alpha1.UpgradePhase {
 		_, podSpec, err := GetLastAppliedConfig(oldSet)
 		if err != nil {
 			return err