cluster: updateTopology maybe hang forever (#1144)

* fix(cluster): UpdateTopology maybe hangs forever * tests: add prometheus,grafana start testcase
pingcap · Feb 22, 2021 · ffed4ff · ffed4ff
1 parent 30b7746
commit ffed4ff
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/pkg/cluster/task/update_topology.go b/pkg/cluster/task/update_topology.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
@@ -37,6 +38,11 @@ func (u *UpdateTopology) Execute(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	// fix https://github.com/pingcap/tiup/issues/333
+	// etcd client defaults to wait forever
+	// if all pd were down, don't hang forever
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
 	txn := client.Txn(ctx)
 
 	topo := u.metadata.Topology

diff --git a/tests/tiup-cluster/script/cmd_subtest.sh b/tests/tiup-cluster/script/cmd_subtest.sh
@@ -92,6 +92,10 @@ function cmd_subtest() {
 
     tiup-cluster $client --yes stop $name
 
+    # test start prometheus,grafana won't hang-forever(can't update topology)
+    # let the CI to stop the job if hang forever
+    ! tiup-cluster $client --yes start $name -R prometheus,grafana
+
     tiup-cluster $client --yes restart $name
 
     tiup-cluster $client _test $name writable