diff --git a/components/cluster/command/prune.go b/components/cluster/command/prune.go index 3faca3fe6f..7c9ae5682b 100644 --- a/components/cluster/command/prune.go +++ b/components/cluster/command/prune.go @@ -14,15 +14,6 @@ package command import ( - "fmt" - - "github.com/fatih/color" - perrs "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cliutil" - operator "github.com/pingcap/tiup/pkg/cluster/operation" - "github.com/pingcap/tiup/pkg/cluster/spec" - "github.com/pingcap/tiup/pkg/cluster/task" - "github.com/pingcap/tiup/pkg/logger/log" "github.com/spf13/cobra" ) @@ -37,68 +28,9 @@ func newPruneCmd() *cobra.Command { clusterName := args[0] - metadata, err := spec.ClusterMetadata(clusterName) - if err != nil { - return err - } - - return destroyTombstoneIfNeed(clusterName, metadata, gOpt, skipConfirm) + return manager.DestroyTombstone(clusterName, gOpt, skipConfirm) }, } return cmd } - -func destroyTombstoneIfNeed(clusterName string, metadata *spec.ClusterMeta, opt operator.Options, skipConfirm bool) error { - topo := metadata.Topology - - if !operator.NeedCheckTombstone(topo) { - return nil - } - - tlsCfg, err := topo.TLSConfig(tidbSpec.Path(clusterName, spec.TLSCertKeyDir)) - if err != nil { - return perrs.AddStack(err) - } - - ctx := task.NewContext() - err = ctx.SetSSHKeySet(spec.ClusterPath(clusterName, "ssh", "id_rsa"), - spec.ClusterPath(clusterName, "ssh", "id_rsa.pub")) - if err != nil { - return perrs.AddStack(err) - } - - err = ctx.SetClusterSSH(topo, metadata.User, gOpt.SSHTimeout, gOpt.SSHType, topo.BaseTopo().GlobalOptions.SSHType) - if err != nil { - return perrs.AddStack(err) - } - - nodes, err := operator.DestroyTombstone(ctx, topo, true /* returnNodesOnly */, opt, tlsCfg) - if err != nil { - return perrs.AddStack(err) - } - - if len(nodes) == 0 { - return nil - } - - if !skipConfirm { - err = cliutil.PromptForConfirmOrAbortError( - color.HiYellowString(fmt.Sprintf("Will destroy these nodes: %v\nDo you confirm this action? [y/N]:", nodes)), - ) - if err != nil { - return err - } - } - - log.Infof("Start destroy Tombstone nodes: %v ...", nodes) - - _, err = operator.DestroyTombstone(ctx, topo, false /* returnNodesOnly */, opt, tlsCfg) - if err != nil { - return perrs.AddStack(err) - } - - log.Infof("Destroy success") - - return spec.SaveClusterMeta(clusterName, metadata) -} diff --git a/components/dm/task/update_dm_meta.go b/components/dm/task/update_dm_meta.go index c2456c4d34..4395d7789a 100644 --- a/components/dm/task/update_dm_meta.go +++ b/components/dm/task/update_dm_meta.go @@ -41,50 +41,58 @@ func NewUpdateDMMeta(cluster string, metadata *dmspec.Metadata, deletedNodesID [ } // Execute implements the Task interface +// the metadata especially the topology is in wide use, +// the other callers point to this field by a pointer, +// so we should update the original topology directly, and don't make a copy func (u *UpdateDMMeta) Execute(ctx *task.Context) error { - // make a copy - newMeta := &dmspec.Metadata{} - *newMeta = *u.metadata - newMeta.Topology = &dmspec.Specification{ - GlobalOptions: u.metadata.Topology.GlobalOptions, - // MonitoredOptions: u.metadata.Topology.MonitoredOptions, - ServerConfigs: u.metadata.Topology.ServerConfigs, - } - deleted := set.NewStringSet(u.deletedNodesID...) topo := u.metadata.Topology + masters := make([]dmspec.MasterSpec, 0) for i, instance := range (&dmspec.DMMasterComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Masters = append(newMeta.Topology.Masters, topo.Masters[i]) + masters = append(masters, topo.Masters[i]) } + topo.Masters = masters + + workers := make([]dmspec.WorkerSpec, 0) for i, instance := range (&dmspec.DMWorkerComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Workers = append(newMeta.Topology.Workers, topo.Workers[i]) + workers = append(workers, topo.Workers[i]) } + topo.Workers = workers + + monitors := make([]spec.PrometheusSpec, 0) for i, instance := range (&spec.MonitorComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Monitors = append(newMeta.Topology.Monitors, topo.Monitors[i]) + monitors = append(monitors, topo.Monitors[i]) } + topo.Monitors = monitors + + grafanas := make([]spec.GrafanaSpec, 0) for i, instance := range (&spec.GrafanaComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Grafanas = append(newMeta.Topology.Grafanas, topo.Grafanas[i]) + grafanas = append(grafanas, topo.Grafanas[i]) } + topo.Grafanas = grafanas + + alertmanagers := make([]spec.AlertmanagerSpec, 0) for i, instance := range (&spec.AlertManagerComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Alertmanagers = append(newMeta.Topology.Alertmanagers, topo.Alertmanagers[i]) + alertmanagers = append(alertmanagers, topo.Alertmanagers[i]) } + topo.Alertmanagers = alertmanagers - return dmspec.GetSpecManager().SaveMeta(u.cluster, newMeta) + return dmspec.GetSpecManager().SaveMeta(u.cluster, u.metadata) } // Rollback implements the Task interface diff --git a/pkg/cluster/manager.go b/pkg/cluster/manager.go index 2afd9ee197..b48968f2ef 100644 --- a/pkg/cluster/manager.go +++ b/pkg/cluster/manager.go @@ -1395,53 +1395,51 @@ func (m *Manager) ScaleIn( var regenConfigTasks []task.Task hasImported := false deletedNodes := set.NewStringSet(nodes...) - for _, component := range topo.ComponentsByStartOrder() { - for _, instance := range component.Instances() { - if deletedNodes.Exist(instance.ID()) { - continue - } - deployDir := spec.Abs(base.User, instance.DeployDir()) - // data dir would be empty for components which don't need it - dataDirs := spec.MultiDirAbs(base.User, instance.DataDir()) - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(base.User, instance.LogDir()) + topo.IterInstance(func(instance spec.Instance) { + if deletedNodes.Exist(instance.ID()) { + return + } + deployDir := spec.Abs(base.User, instance.DeployDir()) + // data dir would be empty for components which don't need it + dataDirs := spec.MultiDirAbs(base.User, instance.DataDir()) + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(base.User, instance.LogDir()) - // Download and copy the latest component to remote if the cluster is imported from Ansible - tb := task.NewBuilder() - if instance.IsImported() { - switch compName := instance.ComponentName(); compName { - case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: - version := m.bindVersion(compName, base.Version) - tb.Download(compName, instance.OS(), instance.Arch(), version). - CopyComponent( - compName, - instance.OS(), - instance.Arch(), - version, - "", // use default srcPath - instance.GetHost(), - deployDir, - ) - } - hasImported = true + // Download and copy the latest component to remote if the cluster is imported from Ansible + tb := task.NewBuilder() + if instance.IsImported() { + switch compName := instance.ComponentName(); compName { + case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: + version := m.bindVersion(compName, base.Version) + tb.Download(compName, instance.OS(), instance.Arch(), version). + CopyComponent( + compName, + instance.OS(), + instance.Arch(), + version, + "", // use default srcPath + instance.GetHost(), + deployDir, + ) } - - t := tb.InitConfig(clusterName, - base.Version, - m.specManager, - instance, - base.User, - true, // always ignore config check result in scale in - meta.DirPaths{ - Deploy: deployDir, - Data: dataDirs, - Log: logDir, - Cache: m.specManager.Path(clusterName, spec.TempConfigPath), - }, - ).Build() - regenConfigTasks = append(regenConfigTasks, t) + hasImported = true } - } + + t := tb.InitConfig(clusterName, + base.Version, + m.specManager, + instance, + base.User, + true, // always ignore config check result in scale in + meta.DirPaths{ + Deploy: deployDir, + Data: dataDirs, + Log: logDir, + Cache: m.specManager.Path(clusterName, spec.TempConfigPath), + }, + ).Build() + regenConfigTasks = append(regenConfigTasks, t) + }) // handle dir scheme changes if hasImported { @@ -1461,7 +1459,6 @@ func (m *Manager) ScaleIn( m.specManager.Path(clusterName, "ssh", "id_rsa.pub")). ClusterSSH(topo, base.User, sshTimeout, sshType, metadata.GetTopology().BaseTopo().GlobalOptions.SSHType) - // TODO: support command scale in operation. scale(b, metadata, tlsCfg) t := b.Parallel(force, regenConfigTasks...).Parallel(force, buildDynReloadProm(metadata.GetTopology())...).Build() @@ -1595,6 +1592,126 @@ func (m *Manager) ScaleOut( return nil } +// DestroyTombstone destory and remove instances that is in tombstone state +func (m *Manager) DestroyTombstone( + clusterName string, + gOpt operator.Options, + skipConfirm bool, +) error { + var ( + sshTimeout = gOpt.SSHTimeout + sshType = gOpt.SSHType + ) + + metadata, err := m.meta(clusterName) + // allow specific validation errors so that user can recover a broken + // cluster if it is somehow in a bad state. + if err != nil && + !errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) { + return perrs.AddStack(err) + } + + topo := metadata.GetTopology() + base := metadata.GetBaseMeta() + + clusterMeta := metadata.(*spec.ClusterMeta) + cluster := clusterMeta.Topology + + if !operator.NeedCheckTombstone(cluster) { + return nil + } + + tlsCfg, err := topo.TLSConfig(m.specManager.Path(clusterName, spec.TLSCertKeyDir)) + if err != nil { + return err + } + + b := task.NewBuilder(). + SSHKeySet( + m.specManager.Path(clusterName, "ssh", "id_rsa"), + m.specManager.Path(clusterName, "ssh", "id_rsa.pub")). + ClusterSSH(topo, base.User, sshTimeout, sshType, metadata.GetTopology().BaseTopo().GlobalOptions.SSHType) + + var nodes []string + b. + Func("FindTomestoneNodes", func(ctx *task.Context) (err error) { + nodes, err = operator.DestroyTombstone(ctx, cluster, true /* returnNodesOnly */, gOpt, tlsCfg) + if !skipConfirm { + err = cliutil.PromptForConfirmOrAbortError( + color.HiYellowString(fmt.Sprintf("Will destroy these nodes: %v\nDo you confirm this action? [y/N]:", nodes)), + ) + if err != nil { + return err + } + } + log.Infof("Start destroy Tombstone nodes: %v ...", nodes) + return err + }). + ClusterOperate(cluster, operator.DestroyTombstoneOperation, gOpt, tlsCfg). + UpdateMeta(clusterName, clusterMeta, nodes). + UpdateTopology(clusterName, m.specManager.Path(clusterName), clusterMeta, nodes) + + var regenConfigTasks []task.Task + deletedNodes := set.NewStringSet(nodes...) + topo.IterInstance(func(instance spec.Instance) { + if deletedNodes.Exist(instance.ID()) { + return + } + deployDir := spec.Abs(base.User, instance.DeployDir()) + // data dir would be empty for components which don't need it + dataDirs := spec.MultiDirAbs(base.User, instance.DataDir()) + // log dir will always be with values, but might not used by the component + logDir := spec.Abs(base.User, instance.LogDir()) + + // Download and copy the latest component to remote if the cluster is imported from Ansible + tb := task.NewBuilder() + if instance.IsImported() { + switch compName := instance.ComponentName(); compName { + case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: + version := m.bindVersion(compName, base.Version) + tb.Download(compName, instance.OS(), instance.Arch(), version). + CopyComponent( + compName, + instance.OS(), + instance.Arch(), + version, + "", // use default srcPath + instance.GetHost(), + deployDir, + ) + } + } + + t := tb.InitConfig(clusterName, + base.Version, + m.specManager, + instance, + base.User, + true, // always ignore config check result in scale in + meta.DirPaths{ + Deploy: deployDir, + Data: dataDirs, + Log: logDir, + Cache: m.specManager.Path(clusterName, spec.TempConfigPath), + }, + ).Build() + regenConfigTasks = append(regenConfigTasks, t) + }) + + t := b.Parallel(true, regenConfigTasks...).Parallel(true, buildDynReloadProm(metadata.GetTopology())...).Build() + if err := t.Execute(task.NewContext()); err != nil { + if errorx.Cast(err) != nil { + // FIXME: Map possible task errors and give suggestions. + return err + } + return perrs.Trace(err) + } + + log.Infof("Destroy success") + + return nil +} + func (m *Manager) meta(name string) (metadata spec.Metadata, err error) { exist, err := m.specManager.Exist(name) if err != nil { diff --git a/pkg/cluster/operation/scale_in.go b/pkg/cluster/operation/scale_in.go index 4b14c0ca1a..398ad20a3d 100644 --- a/pkg/cluster/operation/scale_in.go +++ b/pkg/cluster/operation/scale_in.go @@ -252,16 +252,6 @@ func ScaleInCluster( } } - pdServers := make([]spec.PDSpec, 0, len(cluster.PDServers)) - for i := 0; i < len(cluster.PDServers); i++ { - s := cluster.PDServers[i] - id := s.Host + ":" + strconv.Itoa(s.ClientPort) - if !deletedNodes.Exist(id) { - pdServers = append(pdServers, s) - } - } - cluster.PDServers = pdServers - for i := 0; i < len(cluster.TiKVServers); i++ { s := cluster.TiKVServers[i] id := s.Host + ":" + strconv.Itoa(s.Port) diff --git a/pkg/cluster/task/update_meta.go b/pkg/cluster/task/update_meta.go index 8e6c1f5caa..8fe272d1c7 100644 --- a/pkg/cluster/task/update_meta.go +++ b/pkg/cluster/task/update_meta.go @@ -29,91 +29,122 @@ type UpdateMeta struct { } // Execute implements the Task interface +// the metadata especially the topology is in wide use, +// the other callers point to this field by a pointer, +// so we should update the original topology directly, and don't make a copy func (u *UpdateMeta) Execute(ctx *Context) error { - // make a copy - newMeta := &spec.ClusterMeta{} - *newMeta = *u.metadata - newMeta.Topology = &spec.Specification{ - GlobalOptions: u.metadata.Topology.GlobalOptions, - MonitoredOptions: u.metadata.Topology.MonitoredOptions, - ServerConfigs: u.metadata.Topology.ServerConfigs, - } - deleted := set.NewStringSet(u.deletedNodesID...) topo := u.metadata.Topology + + tidbServers := make([]spec.TiDBSpec, 0) for i, instance := range (&spec.TiDBComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.TiDBServers = append(newMeta.Topology.TiDBServers, topo.TiDBServers[i]) + tidbServers = append(tidbServers, topo.TiDBServers[i]) } + topo.TiDBServers = tidbServers + + tikvServers := make([]spec.TiKVSpec, 0) for i, instance := range (&spec.TiKVComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.TiKVServers = append(newMeta.Topology.TiKVServers, topo.TiKVServers[i]) + tikvServers = append(tikvServers, topo.TiKVServers[i]) } + topo.TiKVServers = tikvServers + + pdServers := make([]spec.PDSpec, 0) for i, instance := range (&spec.PDComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.PDServers = append(newMeta.Topology.PDServers, topo.PDServers[i]) + pdServers = append(pdServers, topo.PDServers[i]) } + topo.PDServers = pdServers + + tiflashServers := make([]spec.TiFlashSpec, 0) for i, instance := range (&spec.TiFlashComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.TiFlashServers = append(newMeta.Topology.TiFlashServers, topo.TiFlashServers[i]) + tiflashServers = append(tiflashServers, topo.TiFlashServers[i]) } + topo.TiFlashServers = tiflashServers + + pumpServers := make([]spec.PumpSpec, 0) for i, instance := range (&spec.PumpComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.PumpServers = append(newMeta.Topology.PumpServers, topo.PumpServers[i]) + pumpServers = append(pumpServers, topo.PumpServers[i]) } + topo.PumpServers = pumpServers + + drainerServers := make([]spec.DrainerSpec, 0) for i, instance := range (&spec.DrainerComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Drainers = append(newMeta.Topology.Drainers, topo.Drainers[i]) + drainerServers = append(drainerServers, topo.Drainers[i]) } + topo.Drainers = drainerServers + + cdcServers := make([]spec.CDCSpec, 0) for i, instance := range (&spec.CDCComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.CDCServers = append(newMeta.Topology.CDCServers, topo.CDCServers[i]) + cdcServers = append(cdcServers, topo.CDCServers[i]) } + topo.CDCServers = cdcServers + + tisparkWorkers := make([]spec.TiSparkWorkerSpec, 0) for i, instance := range (&spec.TiSparkWorkerComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.TiSparkWorkers = append(newMeta.Topology.TiSparkWorkers, topo.TiSparkWorkers[i]) + tisparkWorkers = append(tisparkWorkers, topo.TiSparkWorkers[i]) } + topo.TiSparkWorkers = tisparkWorkers + + tisparkMasters := make([]spec.TiSparkMasterSpec, 0) for i, instance := range (&spec.TiSparkMasterComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.TiSparkMasters = append(newMeta.Topology.TiSparkMasters, topo.TiSparkMasters[i]) + tisparkMasters = append(tisparkMasters, topo.TiSparkMasters[i]) } + topo.TiSparkMasters = tisparkMasters + + monitors := make([]spec.PrometheusSpec, 0) for i, instance := range (&spec.MonitorComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Monitors = append(newMeta.Topology.Monitors, topo.Monitors[i]) + monitors = append(monitors, topo.Monitors[i]) } + topo.Monitors = monitors + + grafanas := make([]spec.GrafanaSpec, 0) for i, instance := range (&spec.GrafanaComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Grafanas = append(newMeta.Topology.Grafanas, topo.Grafanas[i]) + grafanas = append(grafanas, topo.Grafanas[i]) } + topo.Grafanas = grafanas + + alertmanagers := make([]spec.AlertmanagerSpec, 0) for i, instance := range (&spec.AlertManagerComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { continue } - newMeta.Topology.Alertmanagers = append(newMeta.Topology.Alertmanagers, topo.Alertmanagers[i]) + alertmanagers = append(alertmanagers, topo.Alertmanagers[i]) } - return spec.SaveClusterMeta(u.cluster, newMeta) + topo.Alertmanagers = alertmanagers + + return spec.SaveClusterMeta(u.cluster, u.metadata) } // Rollback implements the Task interface diff --git a/tests/tiup-cluster/script/scale_core.sh b/tests/tiup-cluster/script/scale_core.sh index d3f18f7088..508b58a635 100755 --- a/tests/tiup-cluster/script/scale_core.sh +++ b/tests/tiup-cluster/script/scale_core.sh @@ -40,12 +40,17 @@ function scale_core() { echo "start scale in tidb" tiup-cluster $client --yes scale-in $name -N n1:4000 wait_instance_num_reach $name $total_sub_one $native_ssh + # ensure Prometheus's configuration is updated automatically + ! tiup-cluster $client exec $name -N n1 --command "grep -q n1:10080 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + echo "start scale out tidb" topo=./topo/full_scale_in_tidb.yaml tiup-cluster $client --yes scale-out $name $topo # after scale-out, ensure the service is enabled tiup-cluster $client exec $name -N n1 --command "systemctl status tidb-4000 | grep Loaded |grep 'enabled; vendor'" + tiup-cluster $client exec $name -N n1 --command "grep -q n1:10080 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + # scale in tikv maybe exists in several minutes or hours, and the GitHub CI is not guaranteed # echo "start scale in tikv" # tiup-cluster --yes scale-in $name -N n3:20160 # wait_instance_num_reach $name $total_sub_one $native_ssh @@ -53,6 +58,19 @@ function scale_core() { # topo=./topo/full_scale_in_tikv.yaml # tiup-cluster --yes scale-out $name $topo + echo "start scale in pump" + tiup-cluster $client --yes scale-in $name -N n3:8250 + wait_instance_num_reach $name $total_sub_one $native_ssh + + # ensure Prometheus's configuration is updated automatically + ! tiup-cluster $client exec $name -N n1 --command "grep -q n3:8250 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + + echo "start scale out pump" + topo=./topo/full_scale_in_pump.yaml + tiup-cluster $client --yes scale-out $name $topo + # after scale-out, ensure this instance come back + tiup-cluster $client exec $name -N n1 --command "grep -q n3:8250 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + echo "start scale in pd" tiup-cluster $client --yes scale-in $name -N n3:2379 wait_instance_num_reach $name $total_sub_one $native_ssh @@ -60,12 +78,15 @@ function scale_core() { # validate https://github.com/pingcap/tiup/issues/786 # ensure that this instance is removed from the startup scripts of other components that need to rely on PD ! tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/tidb-4000/scripts/run_tidb.sh" + # ensure Prometheus's configuration is updated automatically + ! tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale out pd" topo=./topo/full_scale_in_pd.yaml tiup-cluster $client --yes scale-out $name $topo # after scale-out, ensure this instance come back tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/tidb-4000/scripts/run_tidb.sh" + tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale in tidb" tiup-cluster $client --yes scale-in $name -N n2:4000 diff --git a/tests/tiup-dm/test_cmd.sh b/tests/tiup-dm/test_cmd.sh index d8bf0c9161..4912ea2da7 100755 --- a/tests/tiup-dm/test_cmd.sh +++ b/tests/tiup-dm/test_cmd.sh @@ -54,20 +54,26 @@ total_sub_one=12 echo "start scale in dm-master" tiup-dm --yes scale-in $name -N $ipprefix.101:8261 wait_instance_num_reach $name $total_sub_one false +# ensure Prometheus's configuration is updated automatically +! tiup-dm exec $name -N $ipprefix.101 --command "grep -q $ipprefix.101:8261 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale out dm-master" topo_master=./topo/full_scale_in_dm-master.yaml sed "s/__IPPREFIX__/$ipprefix/g" $topo_master.tpl > $topo_master tiup-dm --yes scale-out $name $topo_master +tiup-dm exec $name -N $ipprefix.101 --command "grep -q $ipprefix.101:8261 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale in dm-worker" yes | tiup-dm scale-in $name -N $ipprefix.102:8262 wait_instance_num_reach $name $total_sub_one +# ensure Prometheus's configuration is updated automatically +! tiup-dm exec $name -N $ipprefix.101 --command "grep -q $ipprefix.102:8262 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale out dm-worker" topo_worker=./topo/full_scale_in_dm-worker.yaml sed "s/__IPPREFIX__/$ipprefix/g" $topo_worker.tpl > $topo_worker yes | tiup-dm scale-out $name $topo_worker +tiup-dm exec $name -N $ipprefix.101 --command "grep -q $ipprefix.102:8262 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale in grafana" yes | tiup-dm scale-in $name -N $ipprefix.101:3000