cluster: add envs to skip topology sanity check in scale-in process (#…

…1627)
pingcap · Nov 22, 2021 · a713f26 · a713f26
1 parent d4de166
commit a713f26
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 26 deletions.
diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go
@@ -458,7 +458,16 @@ func DestroyClusterTombstone(
 		}
 	}
 
-	pdEndpoints := cluster.GetPDList()
+	var pdEndpoints []string
+	forcePDEndpoints := os.Getenv(EnvNamePDEndpointOverwrite) // custom set PD endpoint list
+
+	if forcePDEndpoints != "" {
+		pdEndpoints = strings.Split(forcePDEndpoints, ",")
+		log.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints)
+	} else {
+		pdEndpoints = cluster.GetPDList()
+	}
+
 	var pdClient = api.NewPDClient(pdEndpoints, 10*time.Second, tlsCfg)
 
 	tcpProxy := proxy.GetTCPProxy()

diff --git a/pkg/cluster/operation/operation.go b/pkg/cluster/operation/operation.go
@@ -21,6 +21,12 @@ import (
 	"github.com/pingcap/tiup/pkg/set"
 )
 
+// environment variable names that used to interrupt operations
+const (
+	EnvNameSkipScaleInTopoCheck = "SKIP_SCALEIN_TOPO_CHECK"
+	EnvNamePDEndpointOverwrite  = "FORCE_PD_ENDPOINTS"
+)
+
 // Options represents the operation options
 type Options struct {
 	Roles               []string

diff --git a/pkg/cluster/operation/scale_in.go b/pkg/cluster/operation/scale_in.go
@@ -18,7 +18,9 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/fatih/color"
@@ -28,6 +30,7 @@ import (
 	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/proxy"
 	"github.com/pingcap/tiup/pkg/set"
+	"github.com/pingcap/tiup/pkg/tui"
 	"github.com/pingcap/tiup/pkg/utils"
 )
 
@@ -109,38 +112,57 @@ func ScaleInCluster(
 		deletedDiff[inst.ComponentName()] = append(deletedDiff[inst.ComponentName()], inst)
 	}
 
-	// Cannot delete all PD servers
-	if len(deletedDiff[spec.ComponentPD]) == len(cluster.PDServers) {
-		return errors.New("cannot delete all PD servers")
+	skipTopoCheck := false
+	if v := os.Getenv(EnvNameSkipScaleInTopoCheck); v != "" { // any value except empty will work as "true"
+		skipTopoCheck = true
 	}
 
-	// Cannot delete all TiKV servers
-	if len(deletedDiff[spec.ComponentTiKV]) == len(cluster.TiKVServers) {
-		return errors.New("cannot delete all TiKV servers")
-	}
+	if skipTopoCheck {
+		log.Warnf("%s is set, topology checks ignored, the cluster might be broken after the operations!", EnvNameSkipScaleInTopoCheck)
+		if ok, input := tui.PromptForConfirmYes("Are you sure to continue? [y/N]"); !ok {
+			return errors.Errorf("user aborted with '%s'", input)
+		}
+	} else {
+		// Cannot delete all PD servers
+		if len(deletedDiff[spec.ComponentPD]) == len(cluster.PDServers) {
+			return errors.New("cannot delete all PD servers")
+		}
 
-	// Cannot delete TiSpark master server if there's any TiSpark worker remains
-	if len(deletedDiff[spec.ComponentTiSpark]) > 0 {
-		var cntDiffTiSparkMaster int
-		var cntDiffTiSparkWorker int
-		for _, inst := range deletedDiff[spec.ComponentTiSpark] {
-			switch inst.Role() {
-			case spec.RoleTiSparkMaster:
-				cntDiffTiSparkMaster++
-			case spec.RoleTiSparkWorker:
-				cntDiffTiSparkWorker++
-			}
+		// Cannot delete all TiKV servers
+		if len(deletedDiff[spec.ComponentTiKV]) == len(cluster.TiKVServers) {
+			return errors.New("cannot delete all TiKV servers")
 		}
-		if cntDiffTiSparkMaster == len(cluster.TiSparkMasters) &&
-			cntDiffTiSparkWorker < len(cluster.TiSparkWorkers) {
-			return errors.New("cannot delete tispark master when there are workers left")
+
+		// Cannot delete TiSpark master server if there's any TiSpark worker remains
+		if len(deletedDiff[spec.ComponentTiSpark]) > 0 {
+			var cntDiffTiSparkMaster int
+			var cntDiffTiSparkWorker int
+			for _, inst := range deletedDiff[spec.ComponentTiSpark] {
+				switch inst.Role() {
+				case spec.RoleTiSparkMaster:
+					cntDiffTiSparkMaster++
+				case spec.RoleTiSparkWorker:
+					cntDiffTiSparkWorker++
+				}
+			}
+			if cntDiffTiSparkMaster == len(cluster.TiSparkMasters) &&
+				cntDiffTiSparkWorker < len(cluster.TiSparkWorkers) {
+				return errors.New("cannot delete tispark master when there are workers left")
+			}
 		}
 	}
 
 	var pdEndpoints []string
-	for _, instance := range (&spec.PDComponent{Topology: cluster}).Instances() {
-		if !deletedNodes.Exist(instance.ID()) {
-			pdEndpoints = append(pdEndpoints, Addr(instance))
+	forcePDEndpoints := os.Getenv(EnvNamePDEndpointOverwrite) // custom set PD endpoint list
+
+	if forcePDEndpoints != "" {
+		pdEndpoints = strings.Split(forcePDEndpoints, ",")
+		log.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints)
+	} else {
+		for _, instance := range (&spec.PDComponent{Topology: cluster}).Instances() {
+			if !deletedNodes.Exist(instance.ID()) {
+				pdEndpoints = append(pdEndpoints, Addr(instance))
+			}
 		}
 	}
 

diff --git a/pkg/cluster/operation/upgrade.go b/pkg/cluster/operation/upgrade.go
@@ -17,8 +17,10 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 
 	perrs "github.com/pingcap/errors"
@@ -60,9 +62,19 @@ func Upgrade(
 		var origLeaderScheduleLimit int
 		var origRegionScheduleLimit int
 		var err error
+
+		var pdEndpoints []string
+		forcePDEndpoints := os.Getenv(EnvNamePDEndpointOverwrite) // custom set PD endpoint list
+
 		switch component.Name() {
 		case spec.ComponentTiKV:
-			pdClient := api.NewPDClient(topo.(*spec.Specification).GetPDList(), 10*time.Second, tlsCfg)
+			if forcePDEndpoints != "" {
+				pdEndpoints = strings.Split(forcePDEndpoints, ",")
+				log.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints)
+			} else {
+				pdEndpoints = topo.(*spec.Specification).GetPDList()
+			}
+			pdClient := api.NewPDClient(pdEndpoints, 10*time.Second, tlsCfg)
 			origLeaderScheduleLimit, origRegionScheduleLimit, err = increaseScheduleLimit(ctx, pdClient)
 			if err != nil {
 				// the config modifing error should be able to be safely ignored, as it will