add spec.paused field to pause the tidb cluster syncing

pingcap · Mar 23, 2020 · 7b57a0c · 7b57a0c
1 parent 1f09144
commit 7b57a0c
Show file tree

Hide file tree

Showing 5 changed files with 109 additions and 0 deletions.
diff --git a/docs/api-references/docs.html b/docs/api-references/docs.html
@@ -739,6 +739,19 @@ <h3 id="pingcap.com/v1alpha1.TidbCluster">TidbCluster
 </tr>
 <tr>
 <td>
+<code>Paused</code></br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Indicates that the tidb cluster is paused and will not be processed by
+the controller.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>version</code></br>
 <em>
 string
@@ -11631,6 +11644,19 @@ <h3 id="pingcap.com/v1alpha1.TidbClusterSpec">TidbClusterSpec
 </tr>
 <tr>
 <td>
+<code>Paused</code></br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Indicates that the tidb cluster is paused and will not be processed by
+the controller.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>version</code></br>
 <em>
 string

diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go
@@ -119,6 +119,11 @@ type TidbClusterSpec struct {
 	// +optional
 	Helper *HelperSpec `json:"helper,omitempty"`
 
+	// Indicates that the tidb cluster is paused and will not be processed by
+	// the controller.
+	// +optional
+	Paused bool
+
 	// TODO: remove optional after defaulting logic introduced
 	// TiDB cluster version
 	// +optional

diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go
@@ -114,6 +114,13 @@ func (tcc *defaultTidbClusterControl) validate(tc *v1alpha1.TidbCluster) bool {
 }
 
 func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster) error {
+	// Skip syncing if the tidb cluster is paused.
+	if tc.Spec.Paused {
+		// TODO when we record condition of the tidb cluster, we can emit event on state change
+		klog.Infof("tidb cluster %s/%s is paused", tc.GetNamespace(), tc.GetName())
+		return nil
+	}
+
 	// syncing all PVs managed by operator's reclaim policy to Retain
 	if err := tcc.reclaimPolicyManager.Sync(tc); err != nil {
 		return err

diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go
@@ -38,6 +38,7 @@ import (
 	"github.com/pingcap/tidb-operator/tests/apiserver"
 	e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config"
 	utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image"
+	utilpod "github.com/pingcap/tidb-operator/tests/e2e/util/pod"
 	"github.com/pingcap/tidb-operator/tests/e2e/util/portforward"
 	"github.com/pingcap/tidb-operator/tests/pkg/apimachinery"
 	"github.com/pingcap/tidb-operator/tests/pkg/blockwriter"
@@ -46,8 +47,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -863,6 +866,67 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		framework.ExpectNoError(err)
 	})
 
+	ginkgo.It("TiDB cluster can be paused and unpaused", func() {
+		tcName := "paused"
+		tc := fixture.GetTidbCluster(ns, tcName, utilimage.TiDBV3Version)
+		tc.Spec.PD.Replicas = 1
+		tc.Spec.TiKV.Replicas = 1
+		tc.Spec.TiDB.Replicas = 1
+		err := genericCli.Create(context.TODO(), tc)
+		framework.ExpectNoError(err)
+		err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 15*time.Second)
+		framework.ExpectNoError(err)
+
+		podListBeforePaused, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Pause the tidb cluster")
+		err = controller.GuaranteedUpdate(genericCli, tc, func() error {
+			tc.Spec.Paused = true
+			return nil
+		})
+		framework.ExpectNoError(err)
+		ginkgo.By("Make a change")
+		err = controller.GuaranteedUpdate(genericCli, tc, func() error {
+			tc.Spec.Version = utilimage.TiDBV3UpgradeVersion
+			return nil
+		})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Check pods are not changed when the tidb cluster is paused")
+		err = utilpod.WaitForPodsAreChanged(c, podListBeforePaused.Items, time.Minute*5)
+		framework.ExpectEqual(err, wait.ErrWaitTimeout, "Pods are changed when the tidb cluster is paused")
+
+		ginkgo.By("Unpause the tidb cluster")
+		err = controller.GuaranteedUpdate(genericCli, tc, func() error {
+			tc.Spec.Paused = false
+			return nil
+		})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Check the tidb cluster will be upgraded now")
+		listOptions := metav1.ListOptions{
+			LabelSelector: labels.SelectorFromSet(label.New().Instance(tcName).Component(label.TiDBLabelVal).Labels()).String(),
+		}
+		err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
+			podList, err := c.CoreV1().Pods(ns).List(listOptions)
+			if err != nil && !apierrors.IsNotFound(err) {
+				return false, err
+			}
+			for _, pod := range podList.Items {
+				for _, c := range pod.Spec.Containers {
+					if c.Name == v1alpha1.TiDBMemberType.String() {
+						if c.Image == tc.TiDBImage() {
+							return true, nil
+						}
+					}
+				}
+			}
+			return false, nil
+		})
+		framework.ExpectNoError(err)
+	})
+
 	ginkgo.It("tidb-scale: clear TiDB failureMembers when scale TiDB to zero", func() {
 		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "tidb-scale", "admin", "")
 		cluster.Resources["pd.replicas"] = "3"