do not limit failover count when maxFailoverCount = 0 (#978)

pingcap · Oct 9, 2019 · 5d1c178 · 5d1c178
1 parent cf41bdd
commit 5d1c178
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 2 deletions.
diff --git a/pkg/manager/member/tidb_failover.go b/pkg/manager/member/tidb_failover.go
@@ -45,7 +45,7 @@ func (tf *tidbFailover) Failover(tc *v1alpha1.TidbCluster) error {
 		}
 	}
 
-	if len(tc.Status.TiDB.FailureMembers) >= int(tc.Spec.TiDB.MaxFailoverCount) {
+	if tc.Spec.TiDB.MaxFailoverCount > 0 && len(tc.Status.TiDB.FailureMembers) >= int(tc.Spec.TiDB.MaxFailoverCount) {
 		glog.Warningf("the failure members count reached the limit:%d", tc.Spec.TiDB.MaxFailoverCount)
 		return nil
 	}

diff --git a/pkg/manager/member/tidb_failover_test.go b/pkg/manager/member/tidb_failover_test.go
@@ -157,6 +157,52 @@ func TestFakeTiDBFailoverFailover(t *testing.T) {
 				t.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(2))
 			},
 		},
+		{
+			name: "max failover count but maxFailoverCount = 0",
+			update: func(tc *v1alpha1.TidbCluster) {
+				tc.Spec.TiDB.MaxFailoverCount = 0
+				tc.Status.TiDB.Members = map[string]v1alpha1.TiDBMember{
+					"failover-tidb-0": {
+						Name:   "failover-tidb-0",
+						Health: false,
+					},
+					"failover-tidb-1": {
+						Name:   "failover-tidb-1",
+						Health: false,
+					},
+					"failover-tidb-2": {
+						Name:   "failover-tidb-2",
+						Health: false,
+					},
+					"failover-tidb-3": {
+						Name:   "failover-tidb-3",
+						Health: false,
+					},
+					"failover-tidb-4": {
+						Name:   "failover-tidb-4",
+						Health: false,
+					},
+				}
+				tc.Status.TiDB.FailureMembers = map[string]v1alpha1.TiDBFailureMember{
+					"failover-tidb-0": {
+						PodName: "failover-tidb-0",
+					},
+					"failover-tidb-1": {
+						PodName: "failover-tidb-1",
+					},
+					"failover-tidb-2": {
+						PodName: "failover-tidb-2",
+					},
+				}
+			},
+			errExpectFn: func(t *GomegaWithT, err error) {
+				t.Expect(err).NotTo(HaveOccurred())
+			},
+			expectFn: func(t *GomegaWithT, tc *v1alpha1.TidbCluster) {
+				t.Expect(len(tc.Status.TiDB.FailureMembers)).To(Equal(4))
+				t.Expect(int(tc.Spec.TiDB.Replicas)).To(Equal(2))
+			},
+		},
 	}
 
 	for i := range tests {

diff --git a/pkg/manager/member/tikv_failover.go b/pkg/manager/member/tikv_failover.go
@@ -51,7 +51,7 @@ func (tf *tikvFailover) Failover(tc *v1alpha1.TidbCluster) error {
 			if tc.Status.TiKV.FailureStores == nil {
 				tc.Status.TiKV.FailureStores = map[string]v1alpha1.TiKVFailureStore{}
 			}
-			if len(tc.Status.TiKV.FailureStores) >= int(tc.Spec.TiKV.MaxFailoverCount) {
+			if tc.Spec.TiKV.MaxFailoverCount > 0 && len(tc.Status.TiKV.FailureStores) >= int(tc.Spec.TiKV.MaxFailoverCount) {
 				glog.Warningf("%s/%s failure stores count reached the limit: %d", ns, tcName, tc.Spec.TiKV.MaxFailoverCount)
 				return nil
 			}

diff --git a/pkg/manager/member/tikv_failover_test.go b/pkg/manager/member/tikv_failover_test.go
@@ -254,6 +254,48 @@ func TestTiKVFailoverFailover(t *testing.T) {
 				g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(3))
 			},
 		},
+		{
+			name: "exceed max failover count2 but maxFailoverCount = 0",
+			update: func(tc *v1alpha1.TidbCluster) {
+				tc.Spec.TiKV.MaxFailoverCount = 0
+				tc.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{
+					"12": {
+						State:              v1alpha1.TiKVStateDown,
+						PodName:            "tikv-12",
+						LastTransitionTime: metav1.Time{Time: time.Now().Add(-70 * time.Minute)},
+					},
+					"13": {
+						State:              v1alpha1.TiKVStateDown,
+						PodName:            "tikv-13",
+						LastTransitionTime: metav1.Time{Time: time.Now().Add(-61 * time.Minute)},
+					},
+					"14": {
+						State:              v1alpha1.TiKVStateDown,
+						PodName:            "tikv-14",
+						LastTransitionTime: metav1.Time{Time: time.Now().Add(-70 * time.Minute)},
+					},
+				}
+				tc.Status.TiKV.FailureStores = map[string]v1alpha1.TiKVFailureStore{
+					"1": {
+						PodName: "tikv-1",
+						StoreID: "1",
+					},
+					"2": {
+						PodName: "tikv-2",
+						StoreID: "2",
+					},
+					"3": {
+						PodName: "tikv-3",
+						StoreID: "3",
+					},
+				}
+			},
+			err: false,
+			expectFn: func(tc *v1alpha1.TidbCluster) {
+				g.Expect(int(tc.Spec.TiKV.Replicas)).To(Equal(3))
+				g.Expect(len(tc.Status.TiKV.FailureStores)).To(Equal(6))
+			},
+		},
 	}
 	for i := range tests {
 		testFn(&tests[i], t)