etcdutil: return original enpoints when all are evicted (#7779)

ref #7499, ref #7730

Return the originally picked endpoints directly if all are evicted to gain better availability.

Signed-off-by: JmPotato <ghzpotato@gmail.com>

pkg/utils/etcdutil/health_checker.go

@@ -93,7 +93,7 @@ func (checker *healthChecker) syncer(ctx context.Context) {
 	for {
 		select {
 		case <-ctx.Done():
-			log.Info("etcd client is closed, exit update endpoint goroutine",
+			log.Info("etcd client is closed, exit the endpoint syncer goroutine",
 				zap.String("source", checker.source))
 			return
 		case <-ticker.C:
@@ -110,7 +110,7 @@ func (checker *healthChecker) inspector(ctx context.Context) {
 	for {
 		select {
 		case <-ctx.Done():
-			log.Info("etcd client is closed, exit health check goroutine",
+			log.Info("etcd client is closed, exit the health inspector goroutine",
 				zap.String("source", checker.source))
 			checker.close()
 			return
@@ -329,6 +329,14 @@ func (checker *healthChecker) filterEps(eps []string) []string {
 		}
 		pickedEps = append(pickedEps, ep)
 	}
+	// If the pickedEps is empty, it means all endpoints are evicted,
+	// to gain better availability, just use the original picked endpoints.
+	if len(pickedEps) == 0 {
+		log.Warn("all etcd endpoints are evicted, use the picked endpoints directly",
+			zap.Strings("endpoints", eps),
+			zap.String("source", checker.source))
+		return eps
+	}
 	return pickedEps
 }
 

pkg/utils/etcdutil/health_checker_test.go

@@ -201,7 +201,45 @@ func TestPickEps(t *testing.T) {
 				},
 			},
 			map[string]int{"A": 0, "B": 1, "C": 1, "D": 0},
-			[]string{},
+			[]string{"B", "C"},
+		},
+		// {B, C} -> {A, B, C}
+		{
+			[]healthProbe{
+				{
+					ep:   "A",
+					took: time.Millisecond,
+				},
+				{
+					ep:   "B",
+					took: time.Millisecond,
+				},
+				{
+					ep:   "C",
+					took: time.Millisecond,
+				},
+			},
+			map[string]int{"A": 1, "B": 2, "C": 2, "D": 0},
+			[]string{"A", "B", "C"},
+		},
+		// {A, B, C} -> {A, C, E}
+		{
+			[]healthProbe{
+				{
+					ep:   "A",
+					took: time.Millisecond,
+				},
+				{
+					ep:   "C",
+					took: time.Millisecond,
+				},
+				{
+					ep:   "E",
+					took: time.Millisecond,
+				},
+			},
+			map[string]int{"A": 2, "B": 0, "D": 0},
+			[]string{"C", "E"},
 		},
 	}
 	check(re, testCases)
@@ -318,7 +356,7 @@ func TestLatencyPick(t *testing.T) {
 				},
 			},
 			map[string]int{"A": 1, "B": 1, "C": 0},
-			[]string{},
+			[]string{"A", "B"},
 		},
 	}
 	check(re, testCases)