From b5ef238eadabd80a2bdee231f472d600a52e6204 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Fri, 5 Jul 2019 15:42:27 -0700 Subject: [PATCH 1/9] create pending pods immediately for autoscalers Create the statefulsets immediately. A blocking init container wait-for-pd is used to prevent TiKV and TiDB from immediately starting up. --- Makefile | 5 +- .../controller-manager-deployment.yaml | 2 + cmd/controller-manager/main.go | 6 +- cmd/wait-for-pd/main.go | 65 +++++++++++++++++++ images/tidb-operator/Dockerfile | 1 + .../tidbcluster/tidb_cluster_control.go | 26 ++++++-- .../tidbcluster/tidb_cluster_controller.go | 3 + .../tidb_cluster_controller_test.go | 4 ++ pkg/manager/member/member.go | 30 +++++++++ pkg/manager/member/pd_member_manager.go | 15 +++-- pkg/manager/member/pd_member_manager_test.go | 4 +- pkg/manager/member/tidb_member_manager.go | 35 ++++++---- .../member/tidb_member_manager_test.go | 4 +- pkg/manager/member/tikv_member_manager.go | 42 +++++++----- .../member/tikv_member_manager_test.go | 14 ++-- 15 files changed, 205 insertions(+), 51 deletions(-) create mode 100644 cmd/wait-for-pd/main.go create mode 100644 pkg/manager/member/member.go diff --git a/Makefile b/Makefile index b9d5b73c51..a1475d5e68 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ docker-push: docker docker: build docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator -build: controller-manager scheduler discovery admission-controller +build: controller-manager scheduler discovery admission-controller wait-for-pd controller-manager: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go @@ -44,6 +44,9 @@ discovery: admission-controller: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-admission-controller cmd/admission-controller/main.go +wait-for-pd: + $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/wait-for-pd cmd/wait-for-pd/main.go + e2e-setup: # ginkgo doesn't work with retool for Go 1.11 @GO111MODULE=on CGO_ENABLED=0 go get github.com/onsi/ginkgo@v1.6.0 diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml index e61bae0a0e..b16429411e 100644 --- a/charts/tidb-operator/templates/controller-manager-deployment.yaml +++ b/charts/tidb-operator/templates/controller-manager-deployment.yaml @@ -47,3 +47,5 @@ spec: fieldPath: metadata.namespace - name: TZ value: {{ .Values.timezone | default "UTC" }} + - name: MY_IMAGE + value: "{{ .Values.operatorImage }}" diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index 2309c25622..84c439bd57 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -99,6 +99,10 @@ func main() { if err != nil { glog.Fatalf("failed to get kubernetes Clientset: %v", err) } + operatorImage := os.Getenv("MY_IMAGE") + if operatorImage == "" { + glog.Fatal("MY_IMAGE env variable not set") + } var informerFactory informers.SharedInformerFactory var kubeInformerFactory kubeinformers.SharedInformerFactory @@ -129,7 +133,7 @@ func main() { }, } - tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod) + tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, operatorImage, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod) controllerCtx, cancel := context.WithCancel(context.Background()) defer cancel() go informerFactory.Start(controllerCtx.Done()) diff --git a/cmd/wait-for-pd/main.go b/cmd/wait-for-pd/main.go new file mode 100644 index 0000000000..b1e5cf46b4 --- /dev/null +++ b/cmd/wait-for-pd/main.go @@ -0,0 +1,65 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/pdapi" + "k8s.io/apiserver/pkg/util/logs" + + "flag" + "os" + "time" +) + +const ( + timeout = 5 * time.Second +) + +func init() { + flag.Parse() +} + +// wait-for-pd waits for 1 PD to be running +func main() { + logs.InitLogs() + defer logs.FlushLogs() + + oneSecond, err := time.ParseDuration("1s") + if err != nil { + glog.Fatalf("Error parsing time %v", err) + } + + tcName := os.Getenv("CLUSTER_NAME") + if tcName == "" { + glog.Fatalf("Expected CLUSTER_NAME env variable to be set") + } + namespace := os.Getenv("NAMESPACE") + if namespace == "" { + glog.Fatalf("Expected NAMESPACE env variable to be set") + } + + pdClient := pdapi.NewPDClient(pdapi.PdClientURL(pdapi.Namespace(namespace), tcName), timeout) + + for { + membersInfo, err := pdClient.GetMembers() + if err != nil { + glog.Errorf("Error using pdClient to get members %v", err) + } else if membersInfo.Leader != nil { + glog.Infof("Found a PD member. Exiting now.") + break + } + time.Sleep(oneSecond) + } +} diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index 2db8eeb21b..8e58ab2039 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -5,3 +5,4 @@ ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler ADD bin/tidb-discovery /usr/local/bin/tidb-discovery ADD bin/tidb-admission-controller /usr/local/bin/tidb-admission-controller +ADD bin/wait-for-pd /usr/local/bin/wait-for-pd diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go index 8f8da6be68..0168dfdd7b 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control.go @@ -14,6 +14,7 @@ package tidbcluster import ( + perrors "github.com/pingcap/errors" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/manager" @@ -84,6 +85,19 @@ func (tcc *defaultTidbClusterControl) UpdateTidbCluster(tc *v1alpha1.TidbCluster } func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster) error { + // When the error is a RequeError, do not immediately return + // This allows us to immediately create all the statefulsets + var requeueError error + checkRequeue := func(err error) error { + if perrors.Find(err, controller.IsRequeueError) != nil { + if requeueError == nil { + requeueError = err + } + return nil + } + return err + } + // syncing all PVs managed by operator's reclaim policy to Retain if err := tcc.reclaimPolicyManager.Sync(tc); err != nil { return err @@ -105,7 +119,7 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster // - upgrade the pd cluster // - scale out/in the pd cluster // - failover the pd cluster - if err := tcc.pdMemberManager.Sync(tc); err != nil { + if err := checkRequeue(tcc.pdMemberManager.Sync(tc)); err != nil { return err } @@ -118,7 +132,7 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster // - upgrade the tikv cluster // - scale out/in the tikv cluster // - failover the tikv cluster - if err := tcc.tikvMemberManager.Sync(tc); err != nil { + if err := checkRequeue(tcc.tikvMemberManager.Sync(tc)); err != nil { return err } @@ -130,7 +144,7 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster // - upgrade the tidb cluster // - scale out/in the tidb cluster // - failover the tidb cluster - if err := tcc.tidbMemberManager.Sync(tc); err != nil { + if err := checkRequeue(tcc.tidbMemberManager.Sync(tc)); err != nil { return err } @@ -138,5 +152,9 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster // - label.StoreIDLabelKey // - label.MemberIDLabelKey // - label.NamespaceLabelKey - return tcc.metaManager.Sync(tc) + if err := tcc.metaManager.Sync(tc); err != nil { + return err + } + + return requeueError } diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 3b971b4e0e..7d6092a4f0 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -73,6 +73,7 @@ func NewController( informerFactory informers.SharedInformerFactory, kubeInformerFactory kubeinformers.SharedInformerFactory, autoFailover bool, + operatorImage string, pdFailoverPeriod time.Duration, tikvFailoverPeriod time.Duration, tidbFailoverPeriod time.Duration, @@ -138,6 +139,7 @@ func NewController( podInformer.Lister(), nodeInformer.Lister(), autoFailover, + operatorImage, tikvFailover, tikvScaler, tikvUpgrader, @@ -151,6 +153,7 @@ func NewController( podInformer.Lister(), tidbUpgrader, autoFailover, + operatorImage, tidbFailover, ), meta.NewReclaimPolicyManager( diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller_test.go b/pkg/controller/tidbcluster/tidb_cluster_controller_test.go index 2f06163aea..243fc7238e 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller_test.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller_test.go @@ -226,6 +226,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer) nodeInformer := kubeInformerFactory.Core().V1().Nodes() epsInformer := kubeInformerFactory.Core().V1().Endpoints() autoFailover := true + operatorImage := "pingcap/tidb-operator:latest" tcc := NewController( kubeCli, @@ -233,6 +234,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer) informerFactory, kubeInformerFactory, autoFailover, + operatorImage, 5*time.Minute, 5*time.Minute, 5*time.Minute, @@ -291,6 +293,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer) podInformer.Lister(), nodeInformer.Lister(), autoFailover, + operatorImage, tikvFailover, tikvScaler, tikvUpgrader, @@ -308,6 +311,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer) podInformer.Lister(), tidbUpgrader, autoFailover, + operatorImage, tidbFailover, ), meta.NewReclaimPolicyManager( diff --git a/pkg/manager/member/member.go b/pkg/manager/member/member.go new file mode 100644 index 0000000000..a3100da3d6 --- /dev/null +++ b/pkg/manager/member/member.go @@ -0,0 +1,30 @@ +package member + +import ( + corev1 "k8s.io/api/core/v1" +) + +// WaitForPDContainer gives the container spec for the wait-for-pd init container +func WaitForPDContainer(tcName string, operatorImage string) corev1.Container { + initEnvs := []corev1.EnvVar{ + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "CLUSTER_NAME", + Value: tcName, + }, + } + + return corev1.Container{ + Name: "wait-for-pd", + Image: operatorImage, + Command: []string{"wait-for-pd"}, + Env: initEnvs, + } +} diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index ead59f4f54..ba6f377683 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -197,7 +197,7 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu oldPDSet := oldPDSetTmp.DeepCopy() if err := pmm.syncTidbClusterStatus(tc, oldPDSet); err != nil { - glog.Errorf("failed to sync TidbCluster: [%s/%s]'s status, error: %v", ns, tcName, err) + return err } if !templateEqual(newPDSet.Spec.Template, oldPDSet.Spec.Template) || tc.Status.PD.Phase == v1alpha1.UpgradePhase { @@ -268,13 +268,14 @@ func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set // get endpoints info eps, epErr := pmm.epsLister.Endpoints(ns).Get(controller.PDMemberName(tcName)) if epErr != nil { - return fmt.Errorf("%s, %s", err, epErr) - } - // pd service has no endpoints - if eps != nil && len(eps.Subsets) == 0 { - return fmt.Errorf("%s, service %s/%s has no endpoints", err, ns, controller.PDMemberName(tcName)) + err = fmt.Errorf("%s, %s", err, epErr) + } else { + // pd service has no endpoints + if eps != nil && len(eps.Subsets) == 0 { + err = fmt.Errorf("%s, service %s/%s has no endpoints", err, ns, controller.PDMemberName(tcName)) + } } - return err + return controller.RequeueErrorf("error getting PD health: %v", err) } cluster, err := pdClient.GetCluster() diff --git a/pkg/manager/member/pd_member_manager_test.go b/pkg/manager/member/pd_member_manager_test.go index dfd0b22b99..6a4c88ecb6 100644 --- a/pkg/manager/member/pd_member_manager_test.go +++ b/pkg/manager/member/pd_member_manager_test.go @@ -402,7 +402,7 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { errWhenUpdatePDService: false, errWhenUpdatePDPeerService: false, errWhenGetPDHealth: true, - err: false, + err: true, expectPDServiceFn: nil, expectPDPeerServiceFn: nil, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { @@ -423,7 +423,7 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { errWhenUpdatePDPeerService: false, errWhenGetCluster: true, errWhenGetPDHealth: false, - err: false, + err: true, expectPDServiceFn: nil, expectPDPeerServiceFn: nil, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index b40756c81a..35c1006604 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -47,6 +47,7 @@ type tidbMemberManager struct { podLister corelisters.PodLister tidbUpgrader Upgrader autoFailover bool + operatorImage string tidbFailover Failover tidbStatefulSetIsUpgradingFn func(corelisters.PodLister, *apps.StatefulSet, *v1alpha1.TidbCluster) (bool, error) } @@ -60,6 +61,7 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, podLister corelisters.PodLister, tidbUpgrader Upgrader, autoFailover bool, + operatorImage string, tidbFailover Failover) manager.Manager { return &tidbMemberManager{ setControl: setControl, @@ -70,6 +72,7 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, podLister: podLister, tidbUpgrader: tidbUpgrader, autoFailover: autoFailover, + operatorImage: operatorImage, tidbFailover: tidbFailover, tidbStatefulSetIsUpgradingFn: tidbStatefulSetIsUpgrading, } @@ -79,17 +82,17 @@ func (tmm *tidbMemberManager) Sync(tc *v1alpha1.TidbCluster) error { ns := tc.GetNamespace() tcName := tc.GetName() + // Sync Tidb StatefulSet + if err := tmm.syncTiDBStatefulSetForTidbCluster(tc); err != nil { + return err + } + if !tc.TiKVIsAvailable() { return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName) } // Sync TiDB Headless Service - if err := tmm.syncTiDBHeadlessServiceForTidbCluster(tc); err != nil { - return err - } - - // Sync Tidb StatefulSet - return tmm.syncTiDBStatefulSetForTidbCluster(tc) + return tmm.syncTiDBHeadlessServiceForTidbCluster(tc) } func (tmm *tidbMemberManager) syncTiDBHeadlessServiceForTidbCluster(tc *v1alpha1.TidbCluster) error { @@ -147,6 +150,11 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid tc.Status.TiDB.StatefulSet = &apps.StatefulSetStatus{} return nil } + + if !tc.TiKVIsAvailable() { + return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName) + } + oldTiDBSet := oldTiDBSetTemp.DeepCopy() if err != nil { return err @@ -347,13 +355,14 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust Annotations: podAnnotations, }, Spec: corev1.PodSpec{ - SchedulerName: tc.Spec.SchedulerName, - Affinity: tc.Spec.TiDB.Affinity, - NodeSelector: tc.Spec.TiDB.NodeSelector, - Containers: containers, - RestartPolicy: corev1.RestartPolicyAlways, - Tolerations: tc.Spec.TiDB.Tolerations, - Volumes: vols, + SchedulerName: tc.Spec.SchedulerName, + Affinity: tc.Spec.TiDB.Affinity, + NodeSelector: tc.Spec.TiDB.NodeSelector, + InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tmm.operatorImage)}, + Containers: containers, + RestartPolicy: corev1.RestartPolicyAlways, + Tolerations: tc.Spec.TiDB.Tolerations, + Volumes: vols, }, }, ServiceName: controller.TiDBPeerMemberName(tcName), diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index e91e2ebd45..e6d9022c08 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -101,7 +101,7 @@ func TestTiDBMemberManagerSyncCreate(t *testing.T) { }, errWhenCreateStatefulSet: false, err: true, - setCreated: false, + setCreated: true, }, { name: "error when create statefulset", @@ -534,6 +534,7 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet tidbUpgrader := NewFakeTiDBUpgrader() tidbFailover := NewFakeTiDBFailover() tidbControl := controller.NewFakeTiDBControl() + operatorImage := "pingcap/tidb-operator:latest" tmm := &tidbMemberManager{ setControl, @@ -544,6 +545,7 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet podInformer.Lister(), tidbUpgrader, true, + operatorImage, tidbFailover, tidbStatefulSetIsUpgrading, } diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 896e826aca..8863d548fc 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -47,6 +47,7 @@ type tikvMemberManager struct { podLister corelisters.PodLister nodeLister corelisters.NodeLister autoFailover bool + operatorImage string tikvFailover Failover tikvScaler Scaler tikvUpgrader Upgrader @@ -62,21 +63,23 @@ func NewTiKVMemberManager(pdControl pdapi.PDControlInterface, podLister corelisters.PodLister, nodeLister corelisters.NodeLister, autoFailover bool, + operatorImage string, tikvFailover Failover, tikvScaler Scaler, tikvUpgrader Upgrader) manager.Manager { kvmm := tikvMemberManager{ - pdControl: pdControl, - podLister: podLister, - nodeLister: nodeLister, - setControl: setControl, - svcControl: svcControl, - setLister: setLister, - svcLister: svcLister, - autoFailover: autoFailover, - tikvFailover: tikvFailover, - tikvScaler: tikvScaler, - tikvUpgrader: tikvUpgrader, + pdControl: pdControl, + podLister: podLister, + nodeLister: nodeLister, + setControl: setControl, + svcControl: svcControl, + setLister: setLister, + svcLister: svcLister, + autoFailover: autoFailover, + operatorImage: operatorImage, + tikvFailover: tikvFailover, + tikvScaler: tikvScaler, + tikvUpgrader: tikvUpgrader, } kvmm.tikvStatefulSetIsUpgradingFn = tikvStatefulSetIsUpgrading return &kvmm @@ -95,6 +98,9 @@ type SvcConfig struct { func (tkmm *tikvMemberManager) Sync(tc *v1alpha1.TidbCluster) error { ns := tc.GetNamespace() tcName := tc.GetName() + if err := tkmm.syncStatefulSetForTidbCluster(tc); err != nil { + return err + } if !tc.PDIsAvailable() { return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) @@ -114,7 +120,8 @@ func (tkmm *tikvMemberManager) Sync(tc *v1alpha1.TidbCluster) error { return err } } - return tkmm.syncStatefulSetForTidbCluster(tc) + + return nil } func (tkmm *tikvMemberManager) syncServiceForTidbCluster(tc *v1alpha1.TidbCluster, svcConfig SvcConfig) error { @@ -182,6 +189,10 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl return nil } + if !tc.PDIsAvailable() { + return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) + } + oldSet := oldSetTmp.DeepCopy() if err := tkmm.syncTidbClusterStatus(tc, oldSet); err != nil { @@ -336,9 +347,10 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) Annotations: podAnnotations, }, Spec: corev1.PodSpec{ - SchedulerName: tc.Spec.SchedulerName, - Affinity: tc.Spec.TiKV.Affinity, - NodeSelector: tc.Spec.TiKV.NodeSelector, + SchedulerName: tc.Spec.SchedulerName, + Affinity: tc.Spec.TiKV.Affinity, + NodeSelector: tc.Spec.TiKV.NodeSelector, + InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tkmm.operatorImage)}, Containers: []corev1.Container{ { Name: v1alpha1.TiKVMemberType.String(), diff --git a/pkg/manager/member/tikv_member_manager_test.go b/pkg/manager/member/tikv_member_manager_test.go index 0e2afa6d9d..8beccb1587 100644 --- a/pkg/manager/member/tikv_member_manager_test.go +++ b/pkg/manager/member/tikv_member_manager_test.go @@ -139,8 +139,8 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { prepare: func(tc *v1alpha1.TidbCluster) { tc.Status.PD.Members = map[string]v1alpha1.PDMember{} }, - errWhenCreateStatefulSet: false, - errWhenCreateTiKVPeerService: false, + errWhenCreateStatefulSet: true, + errWhenCreateTiKVPeerService: true, err: true, tikvPeerSvcCreated: false, setCreated: false, @@ -152,10 +152,10 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { prepare: func(tc *v1alpha1.TidbCluster) { tc.Spec.TiKV.Requests.Storage = "100xxxxi" }, - errWhenCreateStatefulSet: false, + errWhenCreateStatefulSet: true, errWhenCreateTiKVPeerService: false, err: true, - tikvPeerSvcCreated: true, + tikvPeerSvcCreated: false, setCreated: false, pdStores: &pdapi.StoresInfo{Count: 0, Stores: []*pdapi.StoreInfo{}}, tombstoneStores: &pdapi.StoresInfo{Count: 0, Stores: []*pdapi.StoreInfo{}}, @@ -164,9 +164,9 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { name: "error when create statefulset", prepare: nil, errWhenCreateStatefulSet: true, - errWhenCreateTiKVPeerService: false, + errWhenCreateTiKVPeerService: true, err: true, - tikvPeerSvcCreated: true, + tikvPeerSvcCreated: false, setCreated: false, pdStores: &pdapi.StoresInfo{Count: 0, Stores: []*pdapi.StoreInfo{}}, tombstoneStores: &pdapi.StoresInfo{Count: 0, Stores: []*pdapi.StoreInfo{}}, @@ -174,7 +174,7 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { { name: "error when create tikv peer service", prepare: nil, - errWhenCreateStatefulSet: false, + errWhenCreateStatefulSet: true, errWhenCreateTiKVPeerService: true, err: true, tikvPeerSvcCreated: false, From 55f49c08f5a5956881d37a3af437dc51f45146aa Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Tue, 9 Jul 2019 16:14:57 -0700 Subject: [PATCH 2/9] upgrade go.* for pd version 3 This gives us the ClusterVersion field for the Config API. --- go.mod | 42 ++---------------- go.sum | 131 +++++++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 113 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index 18e20d40b6..e10fbd6c4d 100644 --- a/go.mod +++ b/go.mod @@ -2,20 +2,15 @@ module github.com/pingcap/tidb-operator require ( github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect github.com/Microsoft/go-winio v0.4.12 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/coreos/bbolt v1.3.1-coreos.6 // indirect github.com/coreos/etcd v0.0.0-20180530235116-2b3aa7e1d49d // indirect - github.com/coreos/go-semver v0.2.0 // indirect - github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 // indirect github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea // indirect github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v0.7.3-0.20171023200535-7848b8beb9d3 @@ -23,38 +18,27 @@ require ( github.com/docker/go-units v0.3.3 // indirect github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect github.com/emicklei/go-restful v2.8.0+incompatible github.com/evanphx/json-patch v4.1.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/camelcase v1.0.0 // indirect - github.com/fsnotify/fsnotify v1.4.7 // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-openapi/spec v0.18.0 // indirect github.com/go-sql-driver/mysql v1.4.0 github.com/gogo/protobuf v1.1.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 // indirect github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 // indirect github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 // indirect github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e // indirect - github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/googleapis/gnostic v0.2.0 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 // indirect - github.com/gorilla/websocket v1.2.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.4.1 // indirect github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 // indirect - github.com/hpcloud/tail v1.0.0 // indirect github.com/imdario/mergo v0.3.7 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 - github.com/jonboulle/clockwork v0.1.0 // indirect github.com/json-iterator/go v1.1.5 // indirect github.com/juju/errors v0.0.0-20180806074554-22422dad46e1 github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect @@ -65,19 +49,16 @@ require ( github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/onsi/ginkgo v1.6.0 // indirect - github.com/onsi/gomega v1.4.1 + github.com/onsi/gomega v1.4.2 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pborman/uuid v1.2.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 // indirect github.com/pingcap/errors v0.11.0 - github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4 - github.com/pingcap/pd v2.1.0-beta+incompatible + github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c + github.com/pingcap/pd v0.0.0-20190628112929-bfbaa0662040 github.com/pingcap/tidb v2.1.0-beta+incompatible - github.com/pkg/errors v0.8.0 // indirect github.com/prometheus/client_golang v0.8.0 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e // indirect @@ -86,35 +67,20 @@ require ( github.com/robfig/cron v1.1.0 github.com/russross/blackfriday v1.5.2+incompatible // indirect github.com/sirupsen/logrus v1.0.6 - github.com/soheilhy/cmux v0.1.4 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 - github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect github.com/uber/jaeger-lib v2.0.0+incompatible // indirect - github.com/ugorji/go v1.1.1 // indirect github.com/unrolled/render v0.0.0-20180807193321-4206df6ff701 // indirect - github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect - go.uber.org/atomic v1.3.2 // indirect - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.9.1 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 // indirect - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect golang.org/x/tools v0.0.0-20190405180640-052fc3cfdbc2 // indirect - google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5 // indirect - google.golang.org/grpc v1.12.0 // indirect - gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect - gopkg.in/fsnotify.v1 v1.4.7 // indirect - gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3 // indirect gopkg.in/square/go-jose.v2 v2.3.0 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.2.1 + gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20181128191700-6db15a15d2d3 k8s.io/apiextensions-apiserver v0.0.0-20190118124337-a384d17938fe // indirect k8s.io/apimachinery v0.0.0-20181128191346-49ce2735e507 diff --git a/go.sum b/go.sum index b590cbe18f..804804bc72 100644 --- a/go.sum +++ b/go.sum @@ -14,10 +14,15 @@ github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVk github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 h1:HD4PLRzjuCVW79mQ0/pdsalOLHJ+FaEoqJLxfltpb2U= github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= @@ -28,8 +33,10 @@ github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazu github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 h1:d/cVoZOrJPJHKH1NdeUjyVAWKp4OpOT+Q+6T1sH7jeU= @@ -48,6 +55,8 @@ github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= @@ -64,10 +73,11 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonreference v0.17.0 h1:yJW3HCkTHg7NOA+gZ83IPHzUSnUzGXhGmsdiCcMexbA= @@ -77,17 +87,24 @@ github.com/go-openapi/spec v0.18.0 h1:aIjeyG5mo5/FrvDkpKKEGZPmF9MPHahS72mzfVqeQX github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE= -github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 h1:7xqw01UYS+KCI25bMrPxwNYkSns2Db1ziQPpVq99FpE= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 h1:f5gsjBiF9tRRVomCvrkGMMWI8W1f2OBFar2c5oakAP0= @@ -96,20 +113,28 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu4i+jD32SE9jQXyfnOvwhHqlT0= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.4.1 h1:pX7cnDwSSmG0dR9yNjCQSSpmsJOqFdT7SzVp5Yl9uVw= @@ -132,12 +157,25 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1 h1:wnhMXidtb70kDZCeLt/ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJm034bVoG30IshVm688T2hi8= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 h1:WQM1NildKThwdP7qWrNAFGzp4ijNLw8RlgENkaI4MJs= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= @@ -146,44 +184,62 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.1 h1:PZSj/UFNaVp3KxrzHOcS7oyuWA7LoOY/77yCTEFu21U= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ= -github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4 h1:fYmrrTx2rWqgvlvTLoHvSlZ78Ckd5BB+NIn0+PVZRTc= -github.com/pingcap/kvproto v0.0.0-20180606093822-b7ba8ea1c0b4/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= -github.com/pingcap/pd v2.1.0-beta+incompatible h1:DZrskt6POM+zhTZvUjUJKXV9OVWpV7cdcq1tXQGOiCY= -github.com/pingcap/pd v2.1.0-beta+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= +github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8= +github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= +github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c h1:pY/MQQ5UajEHfSnQS8rFAM9gw9bBKzqBl414cdfhpRQ= +github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7 h1:kOHAMalwF69bJrtWrOdVaCSvZjLucrJhP4NQKIu6uM4= +github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= +github.com/pingcap/pd v0.0.0-20190628112929-bfbaa0662040 h1:jYKdpf1GeALuS7IqQvWNat/Wy3I2P6vLokM7AJJlgew= +github.com/pingcap/pd v0.0.0-20190628112929-bfbaa0662040/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb v2.1.0-beta+incompatible h1:SQUmscnvvjHLjaIycQqtHujBahUnlKwTz6dQQhqgGSc= github.com/pingcap/tidb v2.1.0-beta+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= @@ -192,6 +248,8 @@ github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfm github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/russross/blackfriday v1.5.2+incompatible h1:/YIL6L1Deczl4O/cQ7ZVdrdKwuB6y7EWpw9LkD8xofE= github.com/russross/blackfriday v1.5.2+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= @@ -204,8 +262,14 @@ github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= @@ -214,20 +278,31 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQG github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w= -github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMVXab7UayvSlIpiyGwRvuX3EKYY7UA= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/unrolled/render v0.0.0-20180807193321-4206df6ff701 h1:BJ/T25enw0WcbWqV132hGXRQdqCqe9XBzqh4AWVH7Bc= github.com/unrolled/render v0.0.0-20180807193321-4206df6ff701/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf h1:rmttwKPEgG/l4UscTDYtaJgeUsedKPKSyFfNQLI6q+I= +go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -236,6 +311,7 @@ golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxT golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -243,8 +319,10 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJV golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= @@ -265,14 +343,21 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGB gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5 h1:2PjFmwzH/sxgW9CRJDlEiwMHO8rOk1eMDzVL14HC1e4= -google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.12.0 h1:Mm8atZtkT+P6R43n/dqNDWkPPu5BwRVu/1rJnJCeZH8= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f h1:FU37niK8AQ59mHcskRyQL7H0ErSeNh650vdcj8HqdSI= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0 h1:ArxJuB1NWfPY6r9Gp9gqwplT0Ge7nqv9msgu03lHLmo= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= +gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= @@ -281,14 +366,16 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3 h1:AFxeG48hTWHhDTQDk/m2gorfVHUEa9vo3tp3D7TzwjI= -gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.3.0 h1:nLzhkFyl5bkblqYBoiWJUt5JkWOzmiaBtCxdJAqJd3U= gopkg.in/square/go-jose.v2 v2.3.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/api v0.0.0-20181128191700-6db15a15d2d3 h1:yU+uHaWFaeWjRoVDuKI2qxcOP9PPFJ+665yJuHI5Ils= k8s.io/api v0.0.0-20181128191700-6db15a15d2d3/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apiextensions-apiserver v0.0.0-20190118124337-a384d17938fe h1:6ESMq1b8HbEOueImZYJ+rjo6X/sEgWloFy8cBrbemV4= From 07e3ee49be1c21e020e4704a3df765d822f3c4e5 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Tue, 9 Jul 2019 13:50:21 -0700 Subject: [PATCH 3/9] use the PD cluster status is initialized API --- cmd/wait-for-pd/main.go | 92 ++++++++++++++++++++++- pkg/manager/member/member.go | 4 +- pkg/manager/member/tidb_member_manager.go | 3 +- pkg/manager/member/tikv_member_manager.go | 2 +- pkg/pdapi/pdapi.go | 39 ++++++++++ 5 files changed, 132 insertions(+), 8 deletions(-) diff --git a/cmd/wait-for-pd/main.go b/cmd/wait-for-pd/main.go index b1e5cf46b4..1de06976a1 100644 --- a/cmd/wait-for-pd/main.go +++ b/cmd/wait-for-pd/main.go @@ -16,9 +16,11 @@ package main import ( "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/pdapi" + "github.com/pingcap/tidb-operator/version" "k8s.io/apiserver/pkg/util/logs" "flag" + "fmt" "os" "time" ) @@ -27,12 +29,55 @@ const ( timeout = 5 * time.Second ) +var ( + printVersion bool + waitForInitialization bool + waitForLeader bool +) + func init() { + flag.BoolVar(&printVersion, "V", false, "Show version and quit") + flag.BoolVar(&printVersion, "version", false, "Show version and quit") + flag.BoolVar(&waitForInitialization, pdapi.WaitForInitializationFlag, false, "Wait for initialization of the cluster. This means all replicas have come online") + flag.BoolVar(&waitForLeader, pdapi.WaitForLeaderFlag, false, "Wait for just the presence of a PD leader.") flag.Parse() } +func pdHasLeader(pdClient pdapi.PDClient) (bool, error) { + memberInfo, err := pdClient.GetPDLeader() + if err != nil { + return false, err + } + return memberInfo != nil, nil +} + +func pdIsInitialized(pdClient pdapi.PDClient) (*bool, error) { + status, err := pdClient.GetClusterStatus() + if err != nil { + f := false + return &f, err + } + return status.IsInitialized, nil +} + +// On older versions this will return the empty semver version 0.0.0 +// Most tidb-operator users will be using version 3.0+ which has the version field. +func pdVersion(pdClient pdapi.PDClient) (string, error) { + conf, err := pdClient.GetConfig() + if err != nil { + return "", err + } + return fmt.Sprintf("%s", conf.ClusterVersion), nil +} + // wait-for-pd waits for 1 PD to be running func main() { + if printVersion { + version.PrintVersionInfo() + os.Exit(0) + } + version.LogVersionInfo() + logs.InitLogs() defer logs.FlushLogs() @@ -52,12 +97,51 @@ func main() { pdClient := pdapi.NewPDClient(pdapi.PdClientURL(pdapi.Namespace(namespace), tcName), timeout) - for { - membersInfo, err := pdClient.GetMembers() + var waitFunction func() bool + + waitForLeaderFunc := func() bool { + hasLeader, err := pdHasLeader(pdClient) if err != nil { - glog.Errorf("Error using pdClient to get members %v", err) - } else if membersInfo.Leader != nil { + glog.Infof("Error using pdClient to get members %v", err) + } else if hasLeader { glog.Infof("Found a PD member. Exiting now.") + return true + } else { + glog.Infof("PD Leader not found") + } + return false + } + + waitForInitializationFunc := func() bool { + isInit, err := pdIsInitialized(pdClient) + if err != nil { + glog.Infof("Error using pdClient to get cluster status %v", err) + } else if isInit == nil { + version, verr := pdVersion(pdClient) + if verr != nil || version == "" { + glog.Errorf("Error using pdClient to get cluster version %v", verr) + } + glog.Warningf("For this PD version %s the cluster status API does not support is_initialized. Will now wait for just a PD leader", version) + waitFunction = waitForLeaderFunc + } else if *isInit { + glog.Infof("Cluster is inititialized. Exiting now.") + return true + } else { + glog.Infof("PD is not initialized") + } + return false + } + + if waitForInitialization { + waitFunction = waitForInitializationFunc + } else if waitForLeader { + waitFunction = waitForLeaderFunc + } else { + glog.Fatalf("Expected either the flag --initialization or --leader") + } + + for { + if waitFunction() { break } time.Sleep(oneSecond) diff --git a/pkg/manager/member/member.go b/pkg/manager/member/member.go index a3100da3d6..4304a7ec38 100644 --- a/pkg/manager/member/member.go +++ b/pkg/manager/member/member.go @@ -5,7 +5,7 @@ import ( ) // WaitForPDContainer gives the container spec for the wait-for-pd init container -func WaitForPDContainer(tcName string, operatorImage string) corev1.Container { +func WaitForPDContainer(tcName string, operatorImage string, arguments []string) corev1.Container { initEnvs := []corev1.EnvVar{ { Name: "NAMESPACE", @@ -24,7 +24,7 @@ func WaitForPDContainer(tcName string, operatorImage string) corev1.Container { return corev1.Container{ Name: "wait-for-pd", Image: operatorImage, - Command: []string{"wait-for-pd"}, + Command: append([]string{"wait-for-pd"}, arguments...), Env: initEnvs, } } diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 35c1006604..e05d0fd5b8 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/manager" + "github.com/pingcap/tidb-operator/pkg/pdapi" "github.com/pingcap/tidb-operator/pkg/util" apps "k8s.io/api/apps/v1beta1" corev1 "k8s.io/api/core/v1" @@ -358,7 +359,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.TiDB.Affinity, NodeSelector: tc.Spec.TiDB.NodeSelector, - InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tmm.operatorImage)}, + InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tmm.operatorImage, []string{"-" + pdapi.WaitForInitializationFlag})}, Containers: containers, RestartPolicy: corev1.RestartPolicyAlways, Tolerations: tc.Spec.TiDB.Tolerations, diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 8863d548fc..0a23fdc64d 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -350,7 +350,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.TiKV.Affinity, NodeSelector: tc.Spec.TiKV.NodeSelector, - InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tkmm.operatorImage)}, + InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tkmm.operatorImage, []string{"-" + pdapi.WaitForLeaderFlag})}, Containers: []corev1.Container{ { Name: v1alpha1.TiKVMemberType.String(), diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index 51b4fe7ee1..cf8a1f1bda 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -32,6 +32,11 @@ import ( const ( timeout = 5 * time.Second + + // WaitForInitializationFlag is located here only because it is convenient + WaitForInitializationFlag = "initialization" + // WaitForLeaderFlag is located here only because it is convenient + WaitForLeaderFlag = "leader" ) // Namespace is a newtype of a string @@ -83,6 +88,8 @@ type PDClient interface { GetConfig() (*server.Config, error) // GetCluster returns used when syncing pod labels. GetCluster() (*metapb.Cluster, error) + // GetClusterStatus is used to determine if the cluster is initialized + GetClusterStatus() (*ClusterStatus, error) // GetMembers returns all PD members from cluster GetMembers() (*MembersInfo, error) // GetStores lists all TiKV stores from cluster @@ -120,6 +127,7 @@ var ( storePrefix = "pd/api/v1/store" configPrefix = "pd/api/v1/config" clusterIDPrefix = "pd/api/v1/cluster" + clusterStatusPrefix = "pd/api/v1/cluster/status" schedulersPrefix = "pd/api/v1/schedulers" pdLeaderPrefix = "pd/api/v1/leader" pdLeaderTransferPrefix = "pd/api/v1/leader/transfer" @@ -139,6 +147,13 @@ func NewPDClient(url string, timeout time.Duration) PDClient { } } +// ClusterStatus is the same as server.ClusterStatus except that IsInitialized is nullable. +// This allows for backwards compatibility testing. +type ClusterStatus struct { + RaftBootstrapTime time.Time `json:"raft_bootstrap_time,omitempty"` + IsInitialized *bool `json:"is_initialized"` +} + // following struct definitions are copied from github.com/pingcap/pd/server/api/store // these are not exported by that package @@ -233,6 +248,20 @@ func (pc *pdClient) GetConfig() (*server.Config, error) { return config, nil } +func (pc *pdClient) GetClusterStatus() (*ClusterStatus, error) { + apiURL := fmt.Sprintf("%s/%s", pc.url, clusterStatusPrefix) + body, err := httputil.GetBodyOK(pc.httpClient, apiURL) + if err != nil { + return nil, err + } + clusterStatus := &ClusterStatus{} + err = json.Unmarshal(body, clusterStatus) + if err != nil { + return nil, err + } + return clusterStatus, nil +} + func (pc *pdClient) GetCluster() (*metapb.Cluster, error) { apiURL := fmt.Sprintf("%s/%s", pc.url, clusterIDPrefix) body, err := httputil.GetBodyOK(pc.httpClient, apiURL) @@ -594,6 +623,7 @@ const ( GetHealthActionType ActionType = "GetHealth" GetConfigActionType ActionType = "GetConfig" GetClusterActionType ActionType = "GetCluster" + GetClusterStatusActionType ActionType = "GetClusterStatus" GetMembersActionType ActionType = "GetMembers" GetStoresActionType ActionType = "GetStores" GetTombStoneStoresActionType ActionType = "GetTombStoneStores" @@ -676,6 +706,15 @@ func (pc *FakePDClient) GetCluster() (*metapb.Cluster, error) { return result.(*metapb.Cluster), nil } +func (pc *FakePDClient) GetClusterStatus() (*ClusterStatus, error) { + action := &Action{} + result, err := pc.fakeAPI(GetClusterStatusActionType, action) + if err != nil { + return nil, err + } + return result.(*ClusterStatus), nil +} + func (pc *FakePDClient) GetMembers() (*MembersInfo, error) { action := &Action{} result, err := pc.fakeAPI(GetMembersActionType, action) From 02e8b738660f9520f151e7c0be752c8374f06043 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 24 Jul 2019 12:50:41 -0700 Subject: [PATCH 4/9] don't move on to failover until the cluster is initialized This only works on newer versions of TiDB --- cmd/wait-for-pd/main.go | 11 +-------- .../tidbcluster/tidb_cluster_controller.go | 1 + pkg/manager/member/tidb_member_manager.go | 11 +++++++-- .../member/tidb_member_manager_test.go | 20 +++++++++++----- pkg/pdapi/pdapi.go | 23 +++++++++++++------ 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/cmd/wait-for-pd/main.go b/cmd/wait-for-pd/main.go index 1de06976a1..d1f18dd31d 100644 --- a/cmd/wait-for-pd/main.go +++ b/cmd/wait-for-pd/main.go @@ -51,15 +51,6 @@ func pdHasLeader(pdClient pdapi.PDClient) (bool, error) { return memberInfo != nil, nil } -func pdIsInitialized(pdClient pdapi.PDClient) (*bool, error) { - status, err := pdClient.GetClusterStatus() - if err != nil { - f := false - return &f, err - } - return status.IsInitialized, nil -} - // On older versions this will return the empty semver version 0.0.0 // Most tidb-operator users will be using version 3.0+ which has the version field. func pdVersion(pdClient pdapi.PDClient) (string, error) { @@ -113,7 +104,7 @@ func main() { } waitForInitializationFunc := func() bool { - isInit, err := pdIsInitialized(pdClient) + isInit, err := pdClient.GetClusterInitialized() if err != nil { glog.Infof("Error using pdClient to get cluster status %v", err) } else if isInit == nil { diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 7d6092a4f0..0829bec2c1 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -148,6 +148,7 @@ func NewController( setControl, svcControl, tidbControl, + pdControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 3a367bb38c..e5eef9e18a 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -43,6 +43,7 @@ type tidbMemberManager struct { setControl controller.StatefulSetControlInterface svcControl controller.ServiceControlInterface tidbControl controller.TiDBControlInterface + pdControl pdapi.PDControlInterface setLister v1beta1.StatefulSetLister svcLister corelisters.ServiceLister podLister corelisters.PodLister @@ -57,6 +58,7 @@ type tidbMemberManager struct { func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, svcControl controller.ServiceControlInterface, tidbControl controller.TiDBControlInterface, + pdControl pdapi.PDControlInterface, setLister v1beta1.StatefulSetLister, svcLister corelisters.ServiceLister, podLister corelisters.PodLister, @@ -68,6 +70,7 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, setControl: setControl, svcControl: svcControl, tidbControl: tidbControl, + pdControl: pdControl, setLister: setLister, svcLister: svcLister, podLister: podLister, @@ -151,8 +154,12 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid tc.Status.TiDB.StatefulSet = &apps.StatefulSetStatus{} return nil } - - if !tc.TiKVIsAvailable() { + initialized, err := controller.GetPDClient(tmm.pdControl, tc).GetClusterInitialized() + if err != nil { + return err + } + // initialized is nil in TiDB versions (< 3.0) + if (initialized != nil && !*initialized) || (initialized == nil && !tc.TiKVIsAvailable()) { return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName) } diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index e6d9022c08..b2a6a99898 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -25,6 +25,7 @@ import ( informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/pdapi" apps "k8s.io/api/apps/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -62,7 +63,7 @@ func TestTiDBMemberManagerSyncCreate(t *testing.T) { test.prepare(tc) } - tmm, fakeSetControl, _, _ := newFakeTiDBMemberManager() + tmm, fakeSetControl, _, _, _ := newFakeTiDBMemberManager() if test.errWhenCreateStatefulSet { fakeSetControl.SetCreateStatefulSetError(errors.NewInternalError(fmt.Errorf("API server failed")), 0) @@ -140,7 +141,7 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { ns := tc.GetNamespace() tcName := tc.GetName() - tmm, fakeSetControl, _, _ := newFakeTiDBMemberManager() + tmm, fakeSetControl, _, _, fakePDControl := newFakeTiDBMemberManager() if test.statusChange == nil { fakeSetControl.SetStatusChange(func(set *apps.StatefulSet) { @@ -168,6 +169,11 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { fakeSetControl.SetUpdateStatefulSetError(errors.NewInternalError(fmt.Errorf("API server failed")), 0) } + pdClient := controller.NewFakePDClient(fakePDControl, tc) + pdClient.AddReaction(pdapi.GetClusterInitializedActionType, func(action *pdapi.Action) (interface{}, error) { + return func() *bool { return nil }(), nil + }) + err = tmm.Sync(tc1) if test.err { g.Expect(err).To(HaveOccurred()) @@ -239,7 +245,7 @@ func TestTiDBMemberManagerTiDBStatefulSetIsUpgrading(t *testing.T) { expectUpgrading bool } testFn := func(test *testcase, t *testing.T) { - pmm, _, podIndexer, _ := newFakeTiDBMemberManager() + pmm, _, podIndexer, _, _ := newFakeTiDBMemberManager() tc := newTidbClusterForTiDB() tc.Status.TiDB.StatefulSet = &apps.StatefulSetStatus{ UpdateRevision: "v3", @@ -352,7 +358,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { if test.updateTC != nil { test.updateTC(tc) } - pmm, _, _, tidbControl := newFakeTiDBMemberManager() + pmm, _, _, tidbControl, _ := newFakeTiDBMemberManager() if test.upgradingFn != nil { pmm.tidbStatefulSetIsUpgradingFn = test.upgradingFn @@ -521,7 +527,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { } } -func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSetControl, cache.Indexer, *controller.FakeTiDBControl) { +func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSetControl, cache.Indexer, *controller.FakeTiDBControl, *pdapi.FakePDControl) { cli := fake.NewSimpleClientset() kubeCli := kubefake.NewSimpleClientset() setInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Apps().V1beta1().StatefulSets() @@ -534,12 +540,14 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet tidbUpgrader := NewFakeTiDBUpgrader() tidbFailover := NewFakeTiDBFailover() tidbControl := controller.NewFakeTiDBControl() + pdControl := pdapi.NewFakePDControl() operatorImage := "pingcap/tidb-operator:latest" tmm := &tidbMemberManager{ setControl, svcControl, tidbControl, + pdControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), @@ -549,7 +557,7 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet tidbFailover, tidbStatefulSetIsUpgrading, } - return tmm, setControl, podInformer.Informer().GetIndexer(), tidbControl + return tmm, setControl, podInformer.Informer().GetIndexer(), tidbControl, pdControl } func newTidbClusterForTiDB() *v1alpha1.TidbCluster { diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index c719393ba5..89608fd351 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -90,8 +90,8 @@ type PDClient interface { GetConfig() (*server.Config, error) // GetCluster returns used when syncing pod labels. GetCluster() (*metapb.Cluster, error) - // GetClusterStatus is used to determine if the cluster is initialized - GetClusterStatus() (*ClusterStatus, error) + // GetClusterInitialized is used to determine if the cluster is initialized + GetClusterInitialized() (*bool, error) // GetMembers returns all PD members from cluster GetMembers() (*MembersInfo, error) // GetStores lists all TiKV stores from cluster @@ -250,7 +250,7 @@ func (pc *pdClient) GetConfig() (*server.Config, error) { return config, nil } -func (pc *pdClient) GetClusterStatus() (*ClusterStatus, error) { +func (pc *pdClient) getClusterStatus() (*ClusterStatus, error) { apiURL := fmt.Sprintf("%s/%s", pc.url, clusterStatusPrefix) body, err := httputil.GetBodyOK(pc.httpClient, apiURL) if err != nil { @@ -264,6 +264,15 @@ func (pc *pdClient) GetClusterStatus() (*ClusterStatus, error) { return clusterStatus, nil } +func (pc *pdClient) GetClusterInitialized() (*bool, error) { + status, err := pc.getClusterStatus() + if err != nil { + f := false + return &f, err + } + return status.IsInitialized, nil +} + func (pc *pdClient) GetCluster() (*metapb.Cluster, error) { apiURL := fmt.Sprintf("%s/%s", pc.url, clusterIDPrefix) body, err := httputil.GetBodyOK(pc.httpClient, apiURL) @@ -630,7 +639,7 @@ const ( GetHealthActionType ActionType = "GetHealth" GetConfigActionType ActionType = "GetConfig" GetClusterActionType ActionType = "GetCluster" - GetClusterStatusActionType ActionType = "GetClusterStatus" + GetClusterInitializedActionType ActionType = "GetClusterInitialized" GetMembersActionType ActionType = "GetMembers" GetStoresActionType ActionType = "GetStores" GetTombStoneStoresActionType ActionType = "GetTombStoneStores" @@ -713,13 +722,13 @@ func (pc *FakePDClient) GetCluster() (*metapb.Cluster, error) { return result.(*metapb.Cluster), nil } -func (pc *FakePDClient) GetClusterStatus() (*ClusterStatus, error) { +func (pc *FakePDClient) GetClusterInitialized() (*bool, error) { action := &Action{} - result, err := pc.fakeAPI(GetClusterStatusActionType, action) + result, err := pc.fakeAPI(GetClusterInitializedActionType, action) if err != nil { return nil, err } - return result.(*ClusterStatus), nil + return result.(*bool), nil } func (pc *FakePDClient) GetMembers() (*MembersInfo, error) { From 85e03746ae7795618f14124aad1bc0cc4da0fa7c Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Fri, 13 Sep 2019 09:35:35 -0700 Subject: [PATCH 5/9] coordinate sync err with upgrade --- pkg/manager/member/pd_member_manager.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index ce6ce6dd82..c5943104bd 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -195,18 +195,20 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu } oldPDSet := oldPDSetTmp.DeepCopy() - - if err := pmm.syncTidbClusterStatus(tc, oldPDSet); err != nil { - return err - } + syncErr := pmm.syncTidbClusterStatus(tc, oldPDSet) if !tc.Status.PD.Synced { force := needForceUpgrade(tc) if force { + if syncErr != nil { + glog.Errorf("failed to sync TidbCluster: [%s/%s]'s status, error: %v", ns, tcName, syncErr) + } tc.Status.PD.Phase = v1alpha1.UpgradePhase setUpgradePartition(newPDSet, 0) errSTS := pmm.updateStatefulSet(tc, newPDSet, oldPDSet) return controller.RequeueErrorf("tidbcluster: [%s/%s]'s pd needs force upgrade, %v", ns, tcName, errSTS) + } else if syncErr != nil { + return syncErr } } From 37fdfeacfe9c2873235a44cbc65fa1c406bf6803 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Fri, 13 Sep 2019 10:56:37 -0700 Subject: [PATCH 6/9] don't change init containers when upgrading only add the wait for pd container on create --- pkg/manager/member/tidb_member_manager.go | 15 ++++++++++--- pkg/manager/member/tikv_member_manager.go | 26 +++++++++++++++++------ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 77145b515e..79caad4df8 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -140,9 +140,9 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid ns := tc.GetNamespace() tcName := tc.GetName() - newTiDBSet := tmm.getNewTiDBSetForTidbCluster(tc) oldTiDBSetTemp, err := tmm.setLister.StatefulSets(ns).Get(controller.TiDBMemberName(tcName)) if errors.IsNotFound(err) { + newTiDBSet := tmm.getNewTiDBSetForTidbCluster(tc, nil) err = SetLastAppliedConfigAnnotation(newTiDBSet) if err != nil { return err @@ -154,6 +154,7 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid tc.Status.TiDB.StatefulSet = &apps.StatefulSetStatus{} return nil } + newTiDBSet := tmm.getNewTiDBSetForTidbCluster(tc, oldTiDBSetTemp) initialized, err := controller.GetPDClient(tmm.pdControl, tc).GetClusterInitialized() if err != nil { return err @@ -233,7 +234,7 @@ func (tmm *tidbMemberManager) getNewTiDBHeadlessServiceForTidbCluster(tc *v1alph } } -func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster) *apps.StatefulSet { +func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet) *apps.StatefulSet { ns := tc.GetNamespace() tcName := tc.GetName() instanceName := tc.GetLabels()[label.InstanceLabelKey] @@ -370,6 +371,14 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust dnsPolicy = corev1.DNSClusterFirstWithHostNet } + // Don't change the init containers when upgrading + var initContainers []corev1.Container + if oldSet == nil { + initContainers = []corev1.Container{WaitForPDContainer(tc.GetName(), tmm.operatorImage, []string{"-" + pdapi.WaitForInitializationFlag})} + } else { + initContainers = oldSet.Spec.Template.Spec.InitContainers + } + tidbLabel := label.New().Instance(instanceName).TiDB() podAnnotations := CombineAnnotations(controller.AnnProm(10080), tc.Spec.TiDB.Annotations) tidbSet := &apps.StatefulSet{ @@ -391,7 +400,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.TiDB.Affinity, NodeSelector: tc.Spec.TiDB.NodeSelector, - InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tmm.operatorImage, []string{"-" + pdapi.WaitForInitializationFlag})}, + InitContainers: initContainers, HostNetwork: tc.Spec.PD.HostNetwork, DNSPolicy: dnsPolicy, Containers: containers, diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index f2fd634eab..b5058d2350 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -167,16 +167,15 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl ns := tc.GetNamespace() tcName := tc.GetName() - newSet, err := tkmm.getNewSetForTidbCluster(tc) - if err != nil { - return err - } - oldSetTmp, err := tkmm.setLister.StatefulSets(ns).Get(controller.TiKVMemberName(tcName)) if err != nil && !errors.IsNotFound(err) { return err } if errors.IsNotFound(err) { + newSet, err := tkmm.getNewSetForTidbCluster(tc, nil) + if err != nil { + return err + } err = SetLastAppliedConfigAnnotation(newSet) if err != nil { return err @@ -189,6 +188,11 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl return nil } + newSet, err := tkmm.getNewSetForTidbCluster(tc, oldSetTmp) + if err != nil { + return err + } + if !tc.PDIsAvailable() { return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) } @@ -279,7 +283,7 @@ func (tkmm *tikvMemberManager) getNewServiceForTidbCluster(tc *v1alpha1.TidbClus return &svc } -func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) (*apps.StatefulSet, error) { +func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet) (*apps.StatefulSet, error) { ns := tc.GetNamespace() tcName := tc.GetName() tikvConfigMap := controller.MemberConfigMapName(tc, v1alpha1.TiKVMemberType) @@ -351,6 +355,14 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) dnsPolicy = corev1.DNSClusterFirstWithHostNet } + // Don't change the init containers when upgrading + var initContainers []corev1.Container + if oldSet == nil { + initContainers = []corev1.Container{WaitForPDContainer(tc.GetName(), tkmm.operatorImage, []string{"-" + pdapi.WaitForLeaderFlag})} + } else { + initContainers = oldSet.Spec.Template.Spec.InitContainers + } + tikvset := &apps.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: setName, @@ -370,7 +382,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) SchedulerName: tc.Spec.SchedulerName, Affinity: tc.Spec.TiKV.Affinity, NodeSelector: tc.Spec.TiKV.NodeSelector, - InitContainers: []corev1.Container{WaitForPDContainer(tc.GetName(), tkmm.operatorImage, []string{"-" + pdapi.WaitForLeaderFlag})}, + InitContainers: initContainers, HostNetwork: tc.Spec.PD.HostNetwork, DNSPolicy: dnsPolicy, Containers: []corev1.Container{ From 8473ac6f8070b543500f976a8574e7a34ae252b0 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Fri, 13 Sep 2019 11:14:35 -0700 Subject: [PATCH 7/9] put go standard imports first --- cmd/wait-for-pd/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/wait-for-pd/main.go b/cmd/wait-for-pd/main.go index d1f18dd31d..e3d1d10b21 100644 --- a/cmd/wait-for-pd/main.go +++ b/cmd/wait-for-pd/main.go @@ -14,15 +14,15 @@ package main import ( - "github.com/golang/glog" - "github.com/pingcap/tidb-operator/pkg/pdapi" - "github.com/pingcap/tidb-operator/version" - "k8s.io/apiserver/pkg/util/logs" - "flag" "fmt" "os" "time" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/pdapi" + "github.com/pingcap/tidb-operator/version" + "k8s.io/apiserver/pkg/util/logs" ) const ( From 666e89ff50c9a409667c7d35eed0600b1da78b58 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Mon, 16 Sep 2019 13:01:26 -0700 Subject: [PATCH 8/9] fix merge TLS api change --- cmd/wait-for-pd/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/wait-for-pd/main.go b/cmd/wait-for-pd/main.go index e3d1d10b21..84656b9e8a 100644 --- a/cmd/wait-for-pd/main.go +++ b/cmd/wait-for-pd/main.go @@ -21,7 +21,7 @@ import ( "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/pdapi" - "github.com/pingcap/tidb-operator/version" + "github.com/pingcap/tidb-operator/pkg/version" "k8s.io/apiserver/pkg/util/logs" ) @@ -86,7 +86,9 @@ func main() { glog.Fatalf("Expected NAMESPACE env variable to be set") } - pdClient := pdapi.NewPDClient(pdapi.PdClientURL(pdapi.Namespace(namespace), tcName), timeout) + // This is just a check for intialization at startup, it doesn't seem valuable to try to hide it with https + // Just being able to observe that there is network traffic going on with this program pretty much tells you everything you would want to know here. + pdClient := pdapi.NewPDClient(pdapi.PdClientURL(pdapi.Namespace(namespace), tcName, "http"), timeout, false) var waitFunction func() bool From 46edcd984973b2d33af82d5c33a14678729f5c08 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Mon, 16 Sep 2019 13:00:43 -0700 Subject: [PATCH 9/9] put the PD api call into the PD sync loop save the initialized variable into the status --- pkg/apis/pingcap.com/v1alpha1/types.go | 1 + .../tidbcluster/tidb_cluster_controller.go | 1 - .../tidb_cluster_controller_test.go | 1 - pkg/manager/member/pd_member_manager.go | 28 ++++++++++++------ pkg/manager/member/pd_member_manager_test.go | 29 +++++++++++++------ pkg/manager/member/tidb_member_manager.go | 9 ++---- .../member/tidb_member_manager_test.go | 20 ++++--------- 7 files changed, 48 insertions(+), 41 deletions(-) diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index ec0a649577..1f756ce955 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -189,6 +189,7 @@ type PDStatus struct { Members map[string]PDMember `json:"members,omitempty"` Leader PDMember `json:"leader,omitempty"` FailureMembers map[string]PDFailureMember `json:"failureMembers,omitempty"` + Initialized *bool `json:"initialized,omitempty"` } // PDMember is PD member diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index d511605953..9c1404efdc 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -148,7 +148,6 @@ func NewController( setControl, svcControl, tidbControl, - pdControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller_test.go b/pkg/controller/tidbcluster/tidb_cluster_controller_test.go index 6e5fc8d683..41d694a4dc 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller_test.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller_test.go @@ -306,7 +306,6 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer) ), svcControl, tidbControl, - pdControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index c5943104bd..487e53bf99 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -275,10 +275,9 @@ func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set } pdClient := controller.GetPDClient(pmm.pdControl, tc) - - healthInfo, err := pdClient.GetHealth() - if err != nil { + handlePDAPIError := func(err error) error { tc.Status.PD.Synced = false + // get endpoints info eps, epErr := pmm.epsLister.Endpoints(ns).Get(controller.PDMemberName(tcName)) if epErr != nil { @@ -292,17 +291,29 @@ func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set return controller.RequeueErrorf("error getting PD health: %v", err) } + initialized, err := pdClient.GetClusterInitialized() + if err != nil { + return handlePDAPIError(err) + } + tc.Status.PD.Initialized = initialized + cluster, err := pdClient.GetCluster() if err != nil { - tc.Status.PD.Synced = false - return err + return handlePDAPIError(err) } + tc.Status.ClusterID = strconv.FormatUint(cluster.Id, 10) leader, err := pdClient.GetPDLeader() if err != nil { - tc.Status.PD.Synced = false - return err + return handlePDAPIError(err) + } + tc.Status.PD.Leader = tc.Status.PD.Members[leader.GetName()] + + healthInfo, err := pdClient.GetHealth() + if err != nil { + return handlePDAPIError(err) } + pdStatus := map[string]v1alpha1.PDMember{} for _, memberHealth := range healthInfo.Healths { id := memberHealth.MemberID @@ -334,10 +345,9 @@ func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set pdStatus[name] = status } + tc.Status.PD.Members = pdStatus tc.Status.PD.Synced = true - tc.Status.PD.Members = pdStatus - tc.Status.PD.Leader = tc.Status.PD.Members[leader.GetName()] return nil } diff --git a/pkg/manager/member/pd_member_manager_test.go b/pkg/manager/member/pd_member_manager_test.go index d4424f7aaf..9497430344 100644 --- a/pkg/manager/member/pd_member_manager_test.go +++ b/pkg/manager/member/pd_member_manager_test.go @@ -212,15 +212,10 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { pmm, fakeSetControl, fakeSvcControl, fakePDControl, _, _, _ := newFakePDMemberManager() pdClient := controller.NewFakePDClient(fakePDControl, tc) - if test.errWhenGetPDHealth { - pdClient.AddReaction(pdapi.GetHealthActionType, func(action *pdapi.Action) (interface{}, error) { - return nil, fmt.Errorf("failed to get health of pd cluster") - }) - } else { - pdClient.AddReaction(pdapi.GetHealthActionType, func(action *pdapi.Action) (interface{}, error) { - return test.pdHealth, nil - }) - } + + pdClient.AddReaction(pdapi.GetClusterInitializedActionType, func(action *pdapi.Action) (interface{}, error) { + return func() *bool { return nil }(), nil + }) if test.errWhenGetCluster { pdClient.AddReaction(pdapi.GetClusterActionType, func(action *pdapi.Action) (interface{}, error) { @@ -232,6 +227,16 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { }) } + if test.errWhenGetPDHealth { + pdClient.AddReaction(pdapi.GetHealthActionType, func(action *pdapi.Action) (interface{}, error) { + return nil, fmt.Errorf("failed to get health of pd cluster") + }) + } else { + pdClient.AddReaction(pdapi.GetHealthActionType, func(action *pdapi.Action) (interface{}, error) { + return test.pdHealth, nil + }) + } + if test.statusChange == nil { fakeSetControl.SetStatusChange(func(set *apps.StatefulSet) { set.Status.Replicas = *set.Spec.Replicas @@ -569,6 +574,9 @@ func TestPDMemberManagerUpgrade(t *testing.T) { pdClient.AddReaction(pdapi.GetClusterActionType, func(action *pdapi.Action) (interface{}, error) { return &metapb.Cluster{Id: uint64(1)}, nil }) + pdClient.AddReaction(pdapi.GetClusterInitializedActionType, func(action *pdapi.Action) (interface{}, error) { + return func() *bool { return nil }(), nil + }) fakeSetControl.SetStatusChange(test.statusChange) @@ -659,6 +667,9 @@ func TestPDMemberManagerSyncPDSts(t *testing.T) { pmm, fakeSetControl, _, fakePDControl, _, _, _ := newFakePDMemberManager() pdClient := controller.NewFakePDClient(fakePDControl, tc) + pdClient.AddReaction(pdapi.GetClusterInitializedActionType, func(action *pdapi.Action) (interface{}, error) { + return func() *bool { return nil }(), nil + }) pdClient.AddReaction(pdapi.GetHealthActionType, func(action *pdapi.Action) (interface{}, error) { return test.pdHealth, nil }) diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 79caad4df8..da023adea9 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -43,7 +43,6 @@ type tidbMemberManager struct { setControl controller.StatefulSetControlInterface svcControl controller.ServiceControlInterface tidbControl controller.TiDBControlInterface - pdControl pdapi.PDControlInterface setLister v1beta1.StatefulSetLister svcLister corelisters.ServiceLister podLister corelisters.PodLister @@ -58,7 +57,6 @@ type tidbMemberManager struct { func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, svcControl controller.ServiceControlInterface, tidbControl controller.TiDBControlInterface, - pdControl pdapi.PDControlInterface, setLister v1beta1.StatefulSetLister, svcLister corelisters.ServiceLister, podLister corelisters.PodLister, @@ -70,7 +68,6 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, setControl: setControl, svcControl: svcControl, tidbControl: tidbControl, - pdControl: pdControl, setLister: setLister, svcLister: svcLister, podLister: podLister, @@ -155,10 +152,8 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid return nil } newTiDBSet := tmm.getNewTiDBSetForTidbCluster(tc, oldTiDBSetTemp) - initialized, err := controller.GetPDClient(tmm.pdControl, tc).GetClusterInitialized() - if err != nil { - return err - } + + initialized := tc.Status.PD.Initialized // initialized is nil in TiDB versions (< 3.0) if (initialized != nil && !*initialized) || (initialized == nil && !tc.TiKVIsAvailable()) { return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName) diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index b2a6a99898..e6d9022c08 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -25,7 +25,6 @@ import ( informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" - "github.com/pingcap/tidb-operator/pkg/pdapi" apps "k8s.io/api/apps/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -63,7 +62,7 @@ func TestTiDBMemberManagerSyncCreate(t *testing.T) { test.prepare(tc) } - tmm, fakeSetControl, _, _, _ := newFakeTiDBMemberManager() + tmm, fakeSetControl, _, _ := newFakeTiDBMemberManager() if test.errWhenCreateStatefulSet { fakeSetControl.SetCreateStatefulSetError(errors.NewInternalError(fmt.Errorf("API server failed")), 0) @@ -141,7 +140,7 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { ns := tc.GetNamespace() tcName := tc.GetName() - tmm, fakeSetControl, _, _, fakePDControl := newFakeTiDBMemberManager() + tmm, fakeSetControl, _, _ := newFakeTiDBMemberManager() if test.statusChange == nil { fakeSetControl.SetStatusChange(func(set *apps.StatefulSet) { @@ -169,11 +168,6 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { fakeSetControl.SetUpdateStatefulSetError(errors.NewInternalError(fmt.Errorf("API server failed")), 0) } - pdClient := controller.NewFakePDClient(fakePDControl, tc) - pdClient.AddReaction(pdapi.GetClusterInitializedActionType, func(action *pdapi.Action) (interface{}, error) { - return func() *bool { return nil }(), nil - }) - err = tmm.Sync(tc1) if test.err { g.Expect(err).To(HaveOccurred()) @@ -245,7 +239,7 @@ func TestTiDBMemberManagerTiDBStatefulSetIsUpgrading(t *testing.T) { expectUpgrading bool } testFn := func(test *testcase, t *testing.T) { - pmm, _, podIndexer, _, _ := newFakeTiDBMemberManager() + pmm, _, podIndexer, _ := newFakeTiDBMemberManager() tc := newTidbClusterForTiDB() tc.Status.TiDB.StatefulSet = &apps.StatefulSetStatus{ UpdateRevision: "v3", @@ -358,7 +352,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { if test.updateTC != nil { test.updateTC(tc) } - pmm, _, _, tidbControl, _ := newFakeTiDBMemberManager() + pmm, _, _, tidbControl := newFakeTiDBMemberManager() if test.upgradingFn != nil { pmm.tidbStatefulSetIsUpgradingFn = test.upgradingFn @@ -527,7 +521,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { } } -func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSetControl, cache.Indexer, *controller.FakeTiDBControl, *pdapi.FakePDControl) { +func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSetControl, cache.Indexer, *controller.FakeTiDBControl) { cli := fake.NewSimpleClientset() kubeCli := kubefake.NewSimpleClientset() setInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Apps().V1beta1().StatefulSets() @@ -540,14 +534,12 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet tidbUpgrader := NewFakeTiDBUpgrader() tidbFailover := NewFakeTiDBFailover() tidbControl := controller.NewFakeTiDBControl() - pdControl := pdapi.NewFakePDControl() operatorImage := "pingcap/tidb-operator:latest" tmm := &tidbMemberManager{ setControl, svcControl, tidbControl, - pdControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), @@ -557,7 +549,7 @@ func newFakeTiDBMemberManager() (*tidbMemberManager, *controller.FakeStatefulSet tidbFailover, tidbStatefulSetIsUpgrading, } - return tmm, setControl, podInformer.Informer().GetIndexer(), tidbControl, pdControl + return tmm, setControl, podInformer.Informer().GetIndexer(), tidbControl } func newTidbClusterForTiDB() *v1alpha1.TidbCluster {