stability: add scale & upgrade case functions

tests/actions.go

@@ -23,11 +23,13 @@ import (
 
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/golang/glog"
+	"github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/label"
+	"k8s.io/api/apps/v1beta1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -101,6 +103,8 @@ type operatorActions struct {
 	pdControl controller.PDControlInterface
 }
 
+var _ = OperatorActions(&operatorActions{})
+
 type OperatorInfo struct {
 	Namespace      string
 	ReleaseName    string
@@ -120,7 +124,7 @@ type TidbClusterInfo struct {
 	StorageClassName string
 	Password         string
 	RecordCount      string
-	InsertBetchSize  string
+	InsertBatchSize  string
 	Resources        map[string]string
 	Args             map[string]string
 }
@@ -349,10 +353,44 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterInfo) error {
 	return nil
 }
 
-func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error   { return nil }
-func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error { return nil }
-func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error      { return nil }
-func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error       { return nil }
+func chartPath(name string, tag string) string {
+	return "/charts/" + tag + "/" + name
+}
+
+func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {
+	cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
+		info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString())
+	glog.Info("[SCALE] " + cmd)
+	res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "failed to scale tidb cluster: %s", string(res))
+	}
+	return nil
+}
+
+func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error {
+	cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
+		info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString())
+	glog.Info("[UPGRADE] " + cmd)
+	res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "failed to upgrade tidb cluster: %s", string(res))
+	}
+	return nil
+}
+
+func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error { return nil }
+func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error  { return nil }
+
+func getComponentContainer(set *v1beta1.StatefulSet) (corev1.Container, bool) {
+	name := set.Labels[label.ComponentLabelKey]
+	for _, c := range set.Spec.Template.Spec.Containers {
+		if c.Name == name {
+			return c, true
+		}
+	}
+	return corev1.Container{}, false
+}
 
 func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, error) {
 	tcName := tc.GetName()
@@ -391,6 +429,11 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err
 			ns, pdSetName, pdSet.Status.ReadyReplicas, pdSet.Status.Replicas)
 		return false, nil
 	}
+	if c, ok := getComponentContainer(pdSet); !ok || tc.Spec.PD.Image != c.Image {
+		glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=pd].image(%s) != %s",
+			ns, pdSetName, c.Image, tc.Spec.PD.Image)
+		return false, nil
+	}
 
 	for _, member := range tc.Status.PD.Members {
 		if !member.Health {
@@ -451,6 +494,11 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e
 			ns, tikvSetName, tikvSet.Status.ReadyReplicas, tikvSet.Status.Replicas)
 		return false, nil
 	}
+	if c, ok := getComponentContainer(tikvSet); !ok || tc.Spec.TiKV.Image != c.Image {
+		glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s",
+			ns, tikvSetName, c.Image, tc.Spec.TiKV.Image)
+		return false, nil
+	}
 
 	for _, store := range tc.Status.TiKV.Stores {
 		if store.State != v1alpha1.TiKVStateUp {
@@ -500,6 +548,11 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e
 			ns, tidbSetName, tidbSet.Status.ReadyReplicas, tidbSet.Status.Replicas)
 		return false, nil
 	}
+	if c, ok := getComponentContainer(tidbSet); !ok || tc.Spec.TiDB.Image != c.Image {
+		glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s",
+			ns, tidbSetName, c.Image, tc.Spec.TiDB.Image)
+		return false, nil
+	}
 
 	_, err = oa.kubeCli.CoreV1().Services(ns).Get(tidbSetName, metav1.GetOptions{})
 	if err != nil {

tests/cluster_info.go

@@ -0,0 +1,50 @@
+package tests
+
+import "strconv"
+
+func (tc *TidbClusterInfo) set(name string, value string) (string, bool) {
+	// NOTE: not thread-safe, maybe make info struct immutable
+	if tc.Args == nil {
+		tc.Args = make(map[string]string)
+	}
+	origVal, ok := tc.Args[name]
+	tc.Args[name] = value
+	return origVal, ok
+}
+
+func (tc *TidbClusterInfo) ScalePD(replicas uint) *TidbClusterInfo {
+	tc.set("pd.replicas", strconv.Itoa(int(replicas)))
+	return tc
+}
+
+func (tc *TidbClusterInfo) ScaleTiKV(replicas uint) *TidbClusterInfo {
+	tc.set("tikv.replicas", strconv.Itoa(int(replicas)))
+	return tc
+}
+
+func (tc *TidbClusterInfo) ScaleTiDB(replicas uint) *TidbClusterInfo {
+	tc.set("tidb.replicas", strconv.Itoa(int(replicas)))
+	return tc
+}
+
+func (tc *TidbClusterInfo) UpgradePD(image string) *TidbClusterInfo {
+	tc.PDImage = image
+	return tc
+}
+
+func (tc *TidbClusterInfo) UpgradeTiKV(image string) *TidbClusterInfo {
+	tc.TiKVImage = image
+	return tc
+}
+
+func (tc *TidbClusterInfo) UpgradeTiDB(image string) *TidbClusterInfo {
+	tc.TiDBImage = image
+	return tc
+}
+
+func (tc *TidbClusterInfo) UpgradeAll(tag string) *TidbClusterInfo {
+	return tc.
+		UpgradePD("pingcap/pd:" + tag).
+		UpgradeTiKV("pingcap/tikv:" + tag).
+		UpgradeTiDB("pingcap/tidb:" + tag)
+}