diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 4c4f99461c..ad3d685198 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -14,6 +14,7 @@ package controller import ( + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -27,12 +28,18 @@ const ( NotDDLOwnerError = "This node is not a ddl owner, can't be resigned." ) +type dbInfo struct { + IsOwner bool `json:"is_owner"` +} + // TiDBControlInterface is the interface that knows how to manage tidb peers type TiDBControlInterface interface { // GetHealth returns tidb's health info GetHealth(tc *v1alpha1.TidbCluster) map[string]bool // ResignDDLOwner resigns the ddl owner of tidb, if the tidb node is not a ddl owner returns (true,nil),else returns (false,err) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) + // Get TIDB info return tidb's dbInfo + GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) } // defaultTiDBControl is default implementation of TiDBControlInterface. @@ -89,6 +96,37 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal return false, err2 } +func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) { + tcName := tc.GetName() + ns := tc.GetNamespace() + + hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) + url := fmt.Sprintf("http://%s.%s.%s:10080/info", hostName, TiDBPeerMemberName(tcName), ns) + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return nil, err + } + res, err := tdc.httpClient.Do(req) + if err != nil { + return nil, err + } + defer DeferClose(res.Body, &err) + if res.StatusCode != http.StatusOK { + errMsg := fmt.Errorf(fmt.Sprintf("Error response %v", res.StatusCode)) + return nil, errMsg + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + info := dbInfo{} + err = json.Unmarshal(body, &info) + if err != nil { + return nil, err + } + return &info, nil +} + func (tdc *defaultTiDBControl) getBodyOK(apiURL string) ([]byte, error) { res, err := tdc.httpClient.Get(apiURL) if err != nil { @@ -112,6 +150,8 @@ type FakeTiDBControl struct { healthInfo map[string]bool resignDDLOwnerError error notDDLOwner bool + tidbInfo *dbInfo + getInfoError error } // NewFakeTiDBControl returns a FakeTiDBControl instance @@ -141,3 +181,7 @@ func (ftd *FakeTiDBControl) GetHealth(_ *v1alpha1.TidbCluster) map[string]bool { func (ftd *FakeTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) { return ftd.notDDLOwner, ftd.resignDDLOwnerError } + +func (ftd *FakeTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) { + return ftd.tidbInfo, ftd.getInfoError +} diff --git a/tests/actions.go b/tests/actions.go index 1d0163fc9e..7bc88f4775 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -14,6 +14,7 @@ package tests import ( + "crypto/tls" "database/sql" "encoding/json" "fmt" @@ -31,6 +32,8 @@ import ( "github.com/golang/glog" pingcapErrors "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb-operator/tests/pkg/apimachinery" + "github.com/pingcap/tidb-operator/tests/pkg/webhook" admissionV1beta1 "k8s.io/api/admissionregistration/v1beta1" "k8s.io/api/apps/v1beta1" batchv1 "k8s.io/api/batch/v1" @@ -125,7 +128,7 @@ type OperatorActions interface { RegisterWebHookAndService(info *OperatorConfig) error RegisterWebHookAndServiceOrDie(info *OperatorConfig) CleanWebHookAndService(info *OperatorConfig) error - StartValidatingAdmissionWebhookServerOrDie() + StartValidatingAdmissionWebhookServerOrDie(info *OperatorConfig) } type operatorActions struct { @@ -149,6 +152,7 @@ type OperatorConfig struct { WebhookServiceName string WebhookSecretName string WebhookConfigName string + Context *apimachinery.CertContext } type TidbClusterConfig struct { @@ -176,6 +180,16 @@ type TidbClusterConfig struct { GrafanaClient *metrics.Client } +func (oi *OperatorConfig) ConfigTLS() *tls.Config { + sCert, err := tls.X509KeyPair(oi.Context.Cert, oi.Context.Key) + if err != nil { + glog.Fatal(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + } +} + func (tc *TidbClusterConfig) BackupHelmSetString(m map[string]string) string { set := map[string]string{ @@ -629,6 +643,11 @@ func (oa *operatorActions) CheckScaledCorrectly(info *TidbClusterConfig, podUIDs } func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { + // record tikv leader count in webhook first + err := webhook.GetAllKVLeaders(oa.cli, info.Namespace, info.ClusterName) + if err != nil { + return err + } oa.emitEvent(info, "UpgradeTidbCluster") cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", @@ -1942,23 +1961,8 @@ func (oa *operatorActions) RegisterWebHookAndService(info *OperatorConfig) error namespace := os.Getenv("NAMESPACE") configName := info.WebhookConfigName - filePath := "/webhook.local.config/certificates/ca.crt" - fd, err := os.Open(filePath) - if err != nil { - glog.Errorf("file can't open file path %s err %v", filePath, err) - return err - } - defer fd.Close() - - ca, err := ioutil.ReadAll(fd) - - if err != nil { - glog.Errorf("file can't read file path %s err %v", filePath, err) - return err - } - - _, err = client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&admissionV1beta1.ValidatingWebhookConfiguration{ + _, err := client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&admissionV1beta1.ValidatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ Name: configName, }, @@ -1979,7 +1983,7 @@ func (oa *operatorActions) RegisterWebHookAndService(info *OperatorConfig) error Name: info.WebhookServiceName, Path: strPtr("/pods"), }, - CABundle: ca, + CABundle: info.Context.SigningCert, }, }, }, @@ -2076,13 +2080,21 @@ func (oa *operatorActions) drainerHealth(info *TidbClusterConfig, hostName strin return len(healths.PumpPos) > 0 && healths.Synced } -func (oa *operatorActions) StartValidatingAdmissionWebhookServerOrDie() { +func (oa *operatorActions) StartValidatingAdmissionWebhookServerOrDie(info *OperatorConfig) { + + context, err := apimachinery.SetupServerCert(os.Getenv("NAMESPACE"), info.WebhookServiceName) + if err != nil { + glog.Fatalf("fail to setup server cert: %v", err) + } + + info.Context = context + http.HandleFunc("/pods", webhook.ServePods) server := &http.Server{ Addr: ":443", - TLSConfig: oa.cfg.ConfigTLS(), + TLSConfig: info.ConfigTLS(), } - err := server.ListenAndServeTLS("", "") + err = server.ListenAndServeTLS("", "") if err != nil { glog.Errorf("fail to start webhook server err %v", err) os.Exit(4) diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index f9eaee9fe9..8afd114992 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -37,9 +37,6 @@ func main() { cli, kubeCli := client.NewCliOrDie() oa := tests.NewOperatorActions(cli, kubeCli, 5*time.Second, conf) - // start a http server in goruntine - go oa.StartValidatingAdmissionWebhookServerOrDie() - operatorInfo := &tests.OperatorConfig{ Namespace: "pingcap", ReleaseName: "operator", @@ -53,6 +50,9 @@ func main() { WebhookConfigName: "webhook-config", } + // start a http server in goruntine + go oa.StartValidatingAdmissionWebhookServerOrDie(operatorInfo) + initTidbVersion, err := conf.GetTiDBVersion() if err != nil { glog.Fatal(err) diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index ab43aa1d82..633d80a1cc 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -44,9 +44,6 @@ func main() { tidbVersion := conf.GetTiDBVersionOrDie() upgardeTiDBVersions := conf.GetUpgradeTidbVersionsOrDie() - // start a http server in goruntine - go oa.StartValidatingAdmissionWebhookServerOrDie() - // operator config operatorCfg := &tests.OperatorConfig{ Namespace: "pingcap", @@ -60,6 +57,9 @@ func main() { WebhookConfigName: "webhook-config", } + // start a http server in goruntine + go oa.StartValidatingAdmissionWebhookServerOrDie(operatorCfg) + // TODO remove this // create database and table and insert a column for test backup and restore initSql := `"create database record;use record;create table test(t char(32))"` diff --git a/tests/config.go b/tests/config.go index c22445aadc..7d49673891 100644 --- a/tests/config.go +++ b/tests/config.go @@ -1,7 +1,6 @@ package tests import ( - "crypto/tls" "flag" "fmt" "io/ioutil" @@ -67,11 +66,6 @@ func NewConfig() (*Config, error) { flag.StringVar(&cfg.TidbVersions, "tidb-versions", "v2.1.3,v2.1.4", "tidb versions") flag.StringVar(&cfg.OperatorTag, "operator-tag", "master", "operator tag used to choose charts") flag.StringVar(&cfg.OperatorImage, "operator-image", "pingcap/tidb-operator:latest", "operator image") - flag.StringVar(&cfg.CertFile, "tls-cert-file", cfg.CertFile, ""+ - "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ - "after server cert).") - flag.StringVar(&cfg.KeyFile, "tls-private-key-file", cfg.KeyFile, ""+ - "File containing the default x509 private key matching --tls-cert-file.") flag.StringVar(&cfg.OperatorRepoDir, "operator-repo-dir", "/tidb-operator", "local directory to which tidb-operator cloned") flag.Parse() @@ -150,16 +144,6 @@ func (c *Config) GetTiDBVersionOrDie() string { return v } -func (c *Config) ConfigTLS() *tls.Config { - sCert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) - if err != nil { - glog.Fatal(err) - } - return &tls.Config{ - Certificates: []tls.Certificate{sCert}, - } -} - func (c *Config) GetUpgradeTidbVersions() []string { tidbVersions := strings.Split(c.TidbVersions, ",") diff --git a/tests/manifests/e2e/e2e.yaml b/tests/manifests/e2e/e2e.yaml index db49d8da4d..9e20014ee9 100644 --- a/tests/manifests/e2e/e2e.yaml +++ b/tests/manifests/e2e/e2e.yaml @@ -1,15 +1,3 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: webhook-secret - namespace: tidb-operator-e2e -type: Opaque -data: - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM2ekNDQWRPZ0F3SUJBZ0lJV2xBRzZaeDVxRDR3RFFZSktvWklodmNOQVFFTEJRQXdFakVRTUE0R0ExVUUKQXhNSFkyRXRZMlZ5ZERBZUZ3MHhPVEEwTVRjd016TTRNVGRhRncweU1EQTBNVFl3TXpNNE1UZGFNREF4TGpBcwpCZ05WQkFNVEpYZGxZbWh2YjJzdGMyVnlkbWxqWlM1MGFXUmlMVzl3WlhKaGRHOXlMV1V5WlM1emRtTXdnZ0VpCk1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQzdGaHBoRWpRMWlMN1lKRjZBUmJhb1hHZEcKS21Lbkw3NHhvMkFnT2F3M1dmM3pVWXR3eUVsbUFzNDBLWXowL01saWFuMGRHalVRVVNQbEpFeEV5dUt1QVdPYQpnOXE4WUZKUklqNTQxUnZqZlI2NytwMDBPT0lTamJrUkhiemErcS93N1NTMkxjSUhBTXFsUVFNQnVUMVZScnUrCnF1R0M2K0VCLzVQMnB2RHRXWHo3dVFnYmhyOERFRmZsdFBWTUJWWnhOdDRMSUpTYm1UTkVsREdIVTl1R3NiZEsKd1FlNWp5T1dOOHNvT01PNEpkMzY4Y3MvSGFtenBOYktDZVpMbWVsTVpsc1Z3Y0tsOEt3dVdVSkM5dE11cHM1TQo1Zk1yMGE4OERhOFM3RkFVZ2ZZVkh5QTVXby9JOGlWQ21vUThrd2svaUxsNzd1bkkwdmVDZFhETFVsMHJBZ01CCkFBR2pKekFsTUE0R0ExVWREd0VCL3dRRUF3SUZvREFUQmdOVkhTVUVEREFLQmdnckJnRUZCUWNEQVRBTkJna3EKaGtpRzl3MEJBUXNGQUFPQ0FRRUFWSG41Z3IyeWpydkdhaGtCWXBZT0JYZFBOb0Z3Ykg5K1BraG9YOVJKRmtMMgoxbVQ1TUpvdHNTT2hRR0RaS2VvQm5tdlFoRUNJakVSb2xiVm5wdG9JYkN4elBFNFV0ZVc0NGoxU1AwL2xmZGtJCm5Cb2MycmhhQ0NmR3ZnMWgzWDVZZ0ZMblJaL1F1Y3puVno4b1RnVTB1dVBxTlhJOXFQc2dKT2laaWpVa1pwRzQKV090cDYrVEY2blRhY25PMnJ6bFl2RjJ2NkxZN2I1TmNXU0xUbnc0MUYzdVJDYnU4MHZxcm5yd2xJMkUrcXpySAo1MVdpWHI5VmJaMHBJUlowWEZIdXdoY2hmWVpEaENUVDFHMlh0b2ZqaTF1aUsrZWJsWVE3czJ0NWU1ZXJqTnZNCmxRUnVPNVUzemhxcHltZG1QaHJ0SFdBbVRSMXZwOER1NnZTakhmQzFXUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXhZYVlSSTBOWWkrMkNSZWdFVzJxRnhuUmlwaXB5KytNYU5nSURtc04xbjk4MUdMCmNNaEpaZ0xPTkNtTTlQekpZbXA5SFJvMUVGRWo1U1JNUk1yaXJnRmptb1BhdkdCU1VTSStlTlViNDMwZXUvcWQKTkRqaUVvMjVFUjI4MnZxdjhPMGt0aTNDQndES3BVRURBYms5VlVhN3ZxcmhndXZoQWYrVDlxYnc3Vmw4KzdrSQpHNGEvQXhCWDViVDFUQVZXY1RiZUN5Q1VtNWt6UkpReGgxUGJockczU3NFSHVZOGpsamZMS0RqRHVDWGQrdkhMClB4MnBzNlRXeWdubVM1bnBUR1piRmNIQ3BmQ3NMbGxDUXZiVExxYk9UT1h6SzlHdlBBMnZFdXhRRklIMkZSOGcKT1ZxUHlQSWxRcHFFUEpNSlA0aTVlKzdweU5MM2duVnd5MUpkS3dJREFRQUJBb0lCQURHdUJlVS9DMFFvQXQycwprcmVuUzREYndNVGVIb0pjNkRtUU04ZDY2U050cjBUOG8zV1lpZjBmdzVnUWJKRGx5NmhwdEwyVXB3Q2xPMDN1CjNKM3I3bFBjcEpGMGNCSlQxYWdiMnRFRmJqbHprVVREb1Jrci9jU0ZnOTVxc2lySUpRNXFPclJ4NURNdDM2SVEKYUhiOXRLNi9jTDJKN1FaeUVyY1FJajkrUno0UGI3L3NqZDFNYzBETHJJS0ZvRWoxdkNpcDRCWWZ4YlliQXZOWQozQTZzVU8rUENja0ZXaytxeE9BMndVUS83Tjc3UDBIMGRYeURmYitIVUIzL1RRbWhxWXh5S1kxRXo4cWZXNkg0Cmp6ZWhDYjhUSGhqVXB2WnhvTzQ3R0YvWmt4SVBISlozK0djc0UySnROODlBSTExSk9SNEZCckFKK1N5Y1BjRXkKbTgxdmpza0NnWUVBeVMvQ1hoeXpMb2UwU3ZyL05sK3M1eE1KUnZTSmhGcmZSR3I5T1UxQUFxZk52WTVVUDhsagpqK3RZOC9VSTE2YjdvZjJvVUdMZFd6cVBxeC93ajNXNzlVZGZyVDJobmZ0OFhRQ3JvZFN3WXdLMUJaK2FMclpqCjBzSTdRWHV6WlMyaWMrVDdiR04ycHdiaHJwRGlxQ0hVUzBmU1IwOFNCS3NmVWpLSklvQnlRcFVDZ1lFQTdnN2oKMmxtR09hakJpN05MZ3lINmE4WFRNMFAxU0wzUGQ4QnNiQXZoM2pLVG8zbWYwZmhjMFhyR0d4b0xkLzJ2NHdqUApLS3ptVWlvNGpiZXJsaW5XTXNWM3pNSXYvREVqVEFhUHJKUWJGUHRuTW92ZEU2K1pVdmZGQU9uS2VhczRqZFIxCk5vd2tOcTlRbE45Wmg0OHF2TnhkVHByeGp6NmFKZFAwcFg1TDhMOENnWUJoK3JWeE9nNzFrVGQzOE1kTUJzcGcKK3Y5Z3BBVTVCVHlJeUlZc1d2ZmFremg2b1k5Y1JVc01zelJ1RXg3TVQ5RnFzZXMvd3ZaRTBMOVpPc1BnU2hsUQp1Z0xaanhOZnFqT0Y0NmF5dUs5eWVNWUtTQkZCd0tmYTQ2Y1NIQmxoSkJsaTBkaTBqN2dnWGhTWS9JeTJEMHVoCm9nZkJuTHVNdEg0YmZPc1dkM0d1QlFLQmdRQ1J1NXZSTjZ6cjcxdE00bDMvMFBVMHRNNHVQQlFVaTk1T09RWW0KdnI5dS94ZFNwRW9xaUJpS1JOYXlFS2VrdFREUGs3ejk4WnF1QWhyTTV2dXIyY0MvSkJQS3piWUNkVEplZ0VYRQpLSWJMdVh2YmZiUEJNV1p6WENyRi9GbHZVbG8wdVROb1NUS0NKNkQxQWlZVXpwZ2pOZVFKRXVGK0I1em1PM014ClBMZlFrUUtCZ0NhOVZ3OTBsMzRUQ0ZQVHR2VFh6ZmdHUDY1eWhoaE80TURSY21LODdueHJmekxDbnJGZUZ6VmQKTUhVRDdmbUVzbEViUjVJRTNCbnBTR2dkTlQ1T2RvR3g4OTN4QllMKzdvODN1RnJ0MnpVSk5Vbm1TOXBkWGFIdwp5WU9sN0J6YVlrOVVodjRoS2REYVJYSnd1YVlRYzhINVlZN3N2Q1NKVk1rWWhTc3lKNFlCCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN3akNDQWFxZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFTTVJBd0RnWURWUVFERXdkallTMWoKWlhKME1CNFhEVEU1TURReE56QXpNemd4TjFvWERUSTVNRFF4TkRBek16Z3hOMW93RWpFUU1BNEdBMVVFQXhNSApZMkV0WTJWeWREQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU5tTDFRdGhGOVlVCklMOWlnNGJRc3F4bE42RVVuODRkWHg5akV0d0dQanNJU2xqVWtsSStjYmNRelhSd3BTK0JnUFZMQjdlNWtlTjQKYStlc0JISEN6WXZQZjZ5aUI1eXR1emc3VWJ0RnVTMXZkNmdTT01QcS9vVXpZYTVydkVTY2dMLzlWL2RLMVNObAp6R3poZlU4anlGeFNDMWt1U2lUMnI0VXBSZStwL1kyQnVwQ2U3UVhQUUVNYS9Qa0Z4UDBaekVyc3NnSmV0azJECnBPemlLcG5oQmhiNXN1amw2RWhqbStjakYwRWlxUDhrSXhSS3cxUE5lRFZ4WEdseHpaZzFLMjZyUXhCNUtJdW4KeHRmMFh2dDRZZ2NnMTJPVHVNMVRwcFlaaHhRM3k5TVBRdm9nK3lTRktoREZ4ZEJ6TUt0RDY0QXoxU1Q5VVUvawo1WjBSYVlyMlgrOENBd0VBQWFNak1DRXdEZ1lEVlIwUEFRSC9CQVFEQWdLa01BOEdBMVVkRXdFQi93UUZNQU1CCkFmOHdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBS21XczJYNXRuTm4vTmNnOENJSGpNMUIyRDN3SDE2d0ttb2oKelVQcFEvdXpBaWZ5ZW1WdytKNEVjWnVyVlpSUlpBUFp0Kyt1RTYvS2kxcUFPNmp1U0dSU0xXaVA3ZFA5SXF6dwpIRDZ6Nzl5dDBFMDJSRFlVUzhsbVJqWlFhbzJoNVJVTi81ZXVabmtqaWFFQ2VlOUMxVGkyaWwrMHdUZEdQbFl1CmFRTTZYMU45aXZiK3huSnROQlE2STJseTBDNG94aFZKM1o0SC80Ym9Vd3UzT3d0OEhmL21NYzA0bkl6N1lZOGwKUm12U3BrNWYxM0k1VjRLTUZuQ1V4WjAyWGVTbkY1dWVNYktpcUo0NXk3cVJ2SWIydlNBRVFHbWx4M1piRjNPRwpUdC9BWk1ITkM3ZnNWQmY5c1QrQUFEMGtTSXc3SERiMXBGOEFUVU84UHRHZzh5NGhJQTg9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K ---- apiVersion: v1 kind: Service metadata: @@ -61,14 +49,9 @@ spec: - --operator-tag=e2e - --operator-image=pingcap/tidb-operator:latest - --tidb-versions=v2.1.3,v2.1.4 - - --tls-cert-file=/webhook.local.config/certificates/tls.crt - - --tls-private-key-file=/webhook.local.config/certificates/tls.key volumeMounts: - mountPath: /logDir name: logdir - - name: webhook-certs - readOnly: true - mountPath: /webhook.local.config/certificates env: - name: NAMESPACE valueFrom: @@ -79,7 +62,4 @@ spec: hostPath: path: /var/log type: Directory - - name: webhook-certs - secret: - secretName: webhook-secret restartPolicy: Never diff --git a/tests/manifests/stability/stability.yaml b/tests/manifests/stability/stability.yaml index f6902be73f..fcecfff6af 100644 --- a/tests/manifests/stability/stability.yaml +++ b/tests/manifests/stability/stability.yaml @@ -1,15 +1,3 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: webhook-secret - namespace: tidb-operator-stability -type: Opaque -data: - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvRENDQWVTZ0F3SUJBZ0lJVS9pL0hYaGpvaXN3RFFZSktvWklodmNOQVFFTEJRQXdIVEViTUJrR0ExVUUKQXhNU1pUSmxMWE5sY25abGNpMWpaWEowTFdOaE1CNFhEVEU1TURReE5EQTJNVFUwTWxvWERUSXdNRFF4TXpBMgpNVFUwTTFvd05qRTBNRElHQTFVRUF4TXJkMlZpYUc5dmF5MXpaWEoyYVdObExuUnBaR0l0YjNCbGNtRjBiM0l0CmMzUmhZbWxzYVhSNUxuTjJZekNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFMVmwKZCtTR1dONDNSd3hNTUt3MVZhWG1vYWRtWXU1VjNsNlhVTjczZTNSYUZ2RDlYZEI0MUovc0EvQmp5NThqSm1OKwpoYURTczgvMVA1L1JHODFhVExpUUZDS3FWdTY3bDQvR3Y1a1gyZ3I4UHJMcjBkVW9hbGpyYUpzbm9VdVJ5VEJQCkJWSm1BQWppQlAySmJoWldIVy9HWCtvNjNTa3NucHZmZEcxVXA3M3NCS0hYRCt5dXEydnVMOHI0WTIveHU4SnYKZUFhbUE0K0hHdHNuNlUrQWVwc0V3QjdDcEQ3SWd6ejVROE9HVWtUcVg1NVJtNFJyM3dyOWJDY3ZyZmZNd0VZZgpLME9jWmNEcG9yRnkyNkxCeWx6WjJCOWdDRlhzdFN6V2xaSDZ2TmhJaDZ5OEl3YzlvWjRWZWFJOTkzUmF4cHpRCi8ySzhhbGVCZHl3TmNLeFZGbVVDQXdFQUFhTW5NQ1V3RGdZRFZSMFBBUUgvQkFRREFnV2dNQk1HQTFVZEpRUU0KTUFvR0NDc0dBUVVGQndNQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQXFFRlE2NzVEMTlVMy9TWENOdk1XWAptMy9kdE0vd2V2b2JNZWdwbEFPSUdNT20xSW5ScDQ3MU1mZURURmUvQWpEL1JVZ1J6QlI0bGQxMC9Cc1B4SkFtCkk5OGJYd3N3UVhRSEE0NEtGcXN0SmdqbVpncHR4eVhNb3ptMnU4WHVHTHQ5UWJLcUU4UEdFZWFhSXJlRFg3TVEKaWdxRFlwTDV1cVpOYnRZekxhemllekZKYTJQeWkvSG5aeFZzTU1vdUFPdk5hMlduQUdDTWV1eDlCTStDMSt6SApjWnJOWVpZVWFMU2doSWJYa1dHMVoyenZONzFtUHd1TGtBcGFoUHMycDl6c0FLN3pmVVgrZjQxTlJUNWR6MGo5CkJUYWlQVlF6MWFVM2grOUhpSlNSV3FoeEViQjMrSjVOQXN6d1N0WktHYlBtdmlISjJqQkZ3aGRleU9zdmlheUgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdFdWMzVJWlkzamRIREV3d3JEVlZwZWFocDJaaTdsWGVYcGRRM3ZkN2RGb1c4UDFkCjBIalVuK3dEOEdQTG55TW1ZMzZGb05LenovVS9uOUVielZwTXVKQVVJcXBXN3J1WGo4YS9tUmZhQ3Z3K3N1dlIKMVNocVdPdG9teWVoUzVISk1FOEZVbVlBQ09JRS9ZbHVGbFlkYjhaZjZqcmRLU3llbTk5MGJWU252ZXdFb2RjUAo3SzZyYSs0dnl2aGpiL0c3d205NEJxWURqNGNhMnlmcFQ0QjZtd1RBSHNLa1BzaURQUGxEdzRaU1JPcGZubEdiCmhHdmZDdjFzSnkrdDk4ekFSaDhyUTV4bHdPbWlzWExib3NIS1hObllIMkFJVmV5MUxOYVZrZnE4MkVpSHJMd2oKQnoyaG5oVjVvajMzZEZyR25ORC9ZcnhxVjRGM0xBMXdyRlVXWlFJREFRQUJBb0lCQUhCVWpaSXV3QW1jSVpzego4MUF4RndETmVYMjRvYnNGNGRhaHphckZodVhlaENab1FCaEJPeXB0ZFdMLzZiQjZpK05CRG04eEM3alVIcnBSCk8rNUc1UXZGN1RJcVdmc3NvQWFoQlRWQTEvM0ZodTQxcXBOZG03M0V5ZHFMQ0E5TUVGS2lIS0dTR0tObms3K04KdzJhQm5Xa0NaNU1kTUtlMndlRzAxdHg5ZUFMYmdsZ0VvQUVaR0g3c2lpUmpqd3hUN0dFd0FHSklTQWxobmUzUwo4UmhGY1dLNmp5QVRWS1RzY0Fhc2J3TVJXV21FcFFDMFhhNVhuZ01MR1piSmFCb20zM3RGbEd3dDFxdVhRbGkvCnV6UGZxNjhINVNaYisyOHArNS94ZmhuNlhPYVBaNVJ6QnpvVzBtV2tZOEFaR29STXVXLzV0TEVsdGpEQ1ZwZDkKaTVmQy8yRUNnWUVBMEc5TTl0Q2Z2eGpLSkU2UUcxL3hUcHNDNU0rbElERkFLUVVZWjFQamxWanUwcDkxc0FmUAoxNFRyakJCdlRJR2JGazhCN2tXQVI4Q3NSS0JPQVpqSkZFNmRGZFc4aStNK29PTzRVOTZNdDJIc0lKemxNT0JuCnk3ZWxSTjhXcGZCamNyK1QwWmtqOGV3M0g4QnhkOXZhYWE4WkRUWW1XaXhQM09CT0VTb0lYa2NDZ1lFQTNzcVgKZUgvZUhYblFlSlRsZ25MOTVoVUhJbEJ1eUU4NmlQb2k1SGVDWkZMUWgzR28rQmxLeXpBNm1BL3dwR2E0MEhlUwpnQkJwQTk3U3crYUtldGdoOEh0ODhEb0l6eWllZEdmVWkyQ0dPTWtNWDBLdjZ1c0RqcHlmWTB3KzdpMGFidURICkRWK1VTSjdtNjhzY05yNUFzVkx0anhqS0dFMk9Uc3FXQmpTL0gvTUNnWUFzK1djVTlvMFJrY3JFTE1PQmRLRS8Kd2NqTkVGVGo3bHlXdlVlM3UrMG1ZNHNjblZXcWh2VDgzdXhvUzMrSWRZcStOSXdKR3F3RVQzbWNVUzZqdjVEYgp0ZDdGUUZvdm9QZjVoVWxYcDNTYmVTQ1hKT043T1dDTUgzTWt1akpMMmVQTGRiVHlpK1dxcExwOE9tMEJYTW55Cjlkb2s5S012MzlIWHFmcU9UNUNBcndLQmdGVU5MRlFoSkc5R1FMSzN6UUpHMmV6TEhFVWFSYUNNdG9EeVZQMjUKTGZzVXJtejJsQlhhdWZYbHZJaXVsU1I5M3BJZkE3dUdDRUVsQUhzdStMQzY4QUg1Y3BIVzVlUUgwcTRIc1ZsZwpDUDJHcXdWMjFPZXV2bFhrTHVqZWc2dXpaa0xyNXJHUlNtK0swZ2MwSzlvdU9VNDRwVjRhalpSSGowcy9CWlRxCjhBZkhBb0dCQUtWeWpVV05wSHBZNzFFNlJxUjlsRE5sY2RjaE8zdjYwdnBUL1FjVkJyZUdqWmFBeWkremNvMlIKZlRwaUlPT0hhVUxCVVBqVmYxNTllMjRINi9PSkNMeGQrM0hzbE1lZDcvMndNUlZEWjBaV0EzbWR6SnRSMG56Zwo4ZTAweW41QTdTNm12NFJldVhvM1E1Tm1ESkxLZGk3eXNCNXVWMWpvSE9PZmRVQ3dSbDVkCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFkTVJzd0dRWURWUVFERXhKbE1tVXQKYzJWeWRtVnlMV05sY25RdFkyRXdIaGNOTVRrd05ERTBNRFl4TlRReVdoY05Namt3TkRFeE1EWXhOVFF5V2pBZApNUnN3R1FZRFZRUURFeEpsTW1VdGMyVnlkbVZ5TFdObGNuUXRZMkV3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUUMrS0hhN0xIV1IrbWkwWGsyUjEzcnRyTmxTNVBHSlYxV2psMXcyWTRraEZYakcKNGRXNCtKcndwRGZjb09DY0VlOG5rcTRWbDRSZTZZT1luUyszV2lWWVlDaXJNejQ0bGUrQU9EY3c5bDZkYTNINwpEczRzSllPSklZaWRxQldtY0FxZy9tKzZBaHBsdlRiZEdxcjZuMEQ4SVhacFhJUGNpMGpSTDlqaE96ZEpqNnFlClV5Y1hCQUVkK1ZpRGpmU3drMSt0cWxUY3RaTWZxbTJwcmllUVNXSjl2R21SZXlNUFNpMVdPZ3dCbDJFVzRhN0UKcEdHM2liUCtWeERaNmlYR0g4Y2dwWDVSRlB3VC9yeUJ4WkllSmtTbFZmUVpjeWV6cXJNYkRsNGJDWTkvMzIzSgpLODFjUDM2L1BYOHg4OVk3VFhmQUtWRHZicVk5b0VlbktBT2R5d2QxQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lDcERBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCZUZmcGkKYkQ3Mi96Nyt5a0w5SHJoOWtWL1pralM2WThkUVdnOTFQNzlJOVUzdXI2Vk9iaHJZY3VTdVZNdkFnZCtPeEEveApYTnpHYitpN2pzYzdzMXM0bXdpUnZjY0pBSnFKQmxXTWM0R1pLRVBxRW9XbUxlVXlBaEZ1WGZaVk5Yb3Y0VEJyCmdneEc0Zjcwb2VqcVh1VGNQUEdaREh1blhUQzhNMEhuaTZEY2huZ1ZVOHh4Q1cydWVYRC9paGNxSXBmcHNxa0oKL01EdnVJUjRWakZ2QVpveU1NcXhpQkYzMXlwQk5TU2RnMXRGQ05sa3hBNkZhaVM2VHhtckV5bUxFVmJ0MGxlUwo5ZzdENFdwcFo0RWRvbmtEd0lrSWpFd1BWMVFRVnhrbEZCTzFqbjMwVVBTcEpwR0RYRkNZQmo4bkZGYjV4aUtKCkNhcDRBak5xU3ZWMWZKSmsKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= ---- apiVersion: v1 kind: Service metadata: @@ -59,17 +47,12 @@ spec: command: - /usr/local/bin/stability-test - --config=/etc/tidb-operator-stability/config.yaml - - --tls-cert-file=/webhook.local.config/certificates/tls.crt - - --tls-private-key-file=/webhook.local.config/certificates/tls.key volumeMounts: - mountPath: /logDir name: logdir - name: config readOnly: true mountPath: /etc/tidb-operator-stability - - name: webhook-certs - readOnly: true - mountPath: /webhook.local.config/certificates env: - name: MY_NODE_NAME valueFrom: @@ -90,7 +73,4 @@ spec: items: - key: config path: config.yaml - - name: webhook-certs - secret: - secretName: webhook-secret restartPolicy: Never diff --git a/tests/pkg/apimachinery/certs.go b/tests/pkg/apimachinery/certs.go new file mode 100644 index 0000000000..a924a5573a --- /dev/null +++ b/tests/pkg/apimachinery/certs.go @@ -0,0 +1,84 @@ +package apimachinery + +import ( + "crypto/x509" + "github.com/golang/glog" + "io/ioutil" + "k8s.io/client-go/util/cert" + "os" +) + +type CertContext struct { + Cert []byte + Key []byte + SigningCert []byte +} + +// Setup the server cert. For example, user apiservers and admission webhooks +// can use the cert to prove their identify to the kube-apiserver +func SetupServerCert(namespaceName, serviceName string) (*CertContext, error) { + certDir, err := ioutil.TempDir("", "test-e2e-server-cert") + if err != nil { + glog.Errorf("Failed to create a temp dir for cert generation %v", err) + return nil, err + } + defer os.RemoveAll(certDir) + signingKey, err := cert.NewPrivateKey() + if err != nil { + glog.Errorf("Failed to create CA private key %v", err) + return nil, err + } + signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) + if err != nil { + glog.Errorf("Failed to create CA cert for apiserver %v", err) + return nil, err + } + caCertFile, err := ioutil.TempFile(certDir, "ca.crt") + if err != nil { + glog.Errorf("Failed to create a temp file for ca cert generation %v", err) + return nil, err + } + if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil { + glog.Errorf("Failed to write CA cert %v", err) + return nil, err + } + key, err := cert.NewPrivateKey() + if err != nil { + glog.Errorf("Failed to create private key for %v", err) + return nil, err + } + signedCert, err := cert.NewSignedCert( + cert.Config{ + CommonName: serviceName + "." + namespaceName + ".svc", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + key, signingCert, signingKey, + ) + if err != nil { + glog.Errorf("Failed to create cert%v", err) + return nil, err + } + certFile, err := ioutil.TempFile(certDir, "server.crt") + if err != nil { + glog.Errorf("Failed to create a temp file for cert generation %v", err) + return nil, err + } + keyFile, err := ioutil.TempFile(certDir, "server.key") + if err != nil { + glog.Errorf("Failed to create a temp file for key generation %v", err) + return nil, err + } + if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil { + glog.Errorf("Failed to write cert file %v", err) + return nil, err + } + if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil { + glog.Errorf("Failed to write key file %v", err) + return nil, err + } + return &CertContext{ + Cert: cert.EncodeCertPEM(signedCert), + Key: cert.EncodePrivateKeyPEM(key), + SigningCert: cert.EncodeCertPEM(signingCert), + }, nil +} diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go index 0e9e3fd395..905d628ce3 100644 --- a/tests/pkg/webhook/pods.go +++ b/tests/pkg/webhook/pods.go @@ -1,28 +1,68 @@ package webhook import ( - "encoding/json" "fmt" - "io/ioutil" - "net/http" "os" + "strconv" + "strings" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/tests/pkg/client" "k8s.io/api/admission/v1beta1" ) -type dbInfo struct { - IsOwner bool `json:"is_owner"` +var ( + // Pod name may the same in different namespaces + kvLeaderMap map[string]map[string]int +) + +func GetAllKVLeaders(versionCli versioned.Interface, namespace string, clusterName string) error { + + if kvLeaderMap == nil { + kvLeaderMap = make(map[string]map[string]int) + } + + if kvLeaderMap[namespace] == nil { + kvLeaderMap[namespace] = make(map[string]int) + } + + tc, err := versionCli.PingcapV1alpha1().TidbClusters(namespace).Get(clusterName, metav1.GetOptions{}) + + if err != nil { + glog.Infof("fail to get tc clustername %s namesapce %s %v", clusterName, namespace, err) + return err + } + + pdClient := controller.NewDefaultPDControl().GetPDClient(tc) + + for _, store := range tc.Status.TiKV.Stores { + storeID, err := strconv.ParseUint(store.ID, 10, 64) + if err != nil { + glog.Errorf("fail to convert string to int while deleting TIKV err %v", err) + return err + } + storeInfo, err := pdClient.GetStore(storeID) + if err != nil { + glog.Errorf("fail to read response %v", err) + return err + } + kvLeaderMap[namespace][store.PodName] = storeInfo.Status.LeaderCount + } + + return nil } // only allow pods to be delete when it is not ddlowner of tidb, not leader of pd and not // master of tikv. func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { glog.Infof("admitting pods") - httpClient := &http.Client{} + podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} if ar.Request.Resource != podResource { err := fmt.Errorf("expect resource to be %s", podResource) @@ -30,61 +70,114 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { return toAdmissionResponse(err) } - _, kubeCli := client.NewCliOrDie() + versionCli, kubeCli := client.NewCliOrDie() name := ar.Request.Name - nameSpace := ar.Request.Namespace + namespace := ar.Request.Namespace reviewResponse := v1beta1.AdmissionResponse{} - reviewResponse.Allowed = true + reviewResponse.Allowed = false - pod, err := kubeCli.CoreV1().Pods(nameSpace).Get(name, metav1.GetOptions{}) + pod, err := kubeCli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { - reviewResponse.Allowed = false - glog.Infof("%v", err) + glog.Infof("api server send wrong pod info namespace %s name %s err %v", namespace, name, err) return &reviewResponse } - glog.Infof("delete pod %s", pod.Labels["app.kubernetes.io/component"]) + glog.Infof("delete pod %s", pod.Labels[label.ComponentLabelKey]) - if pod.Labels["app.kubernetes.io/component"] == "tidb" { - podIP := pod.Status.PodIP - url := fmt.Sprintf("http://%s:10080/info", podIP) - req, err := http.NewRequest("POST", url, nil) - if err != nil { - glog.Errorf("fail to generator request %v", err) + tc, err := versionCli.PingcapV1alpha1().TidbClusters(namespace).Get(pod.Labels[label.InstanceLabelKey], metav1.GetOptions{}) + if err != nil { + glog.Infof("fail to fetch tidbcluster info namespace %s clustername(instance) %s err %v", namespace, pod.Labels[label.InstanceLabelKey], err) + return &reviewResponse + } + + pdClient := controller.NewDefaultPDControl().GetPDClient(tc) + tidbController := controller.NewDefaultTiDBControl() + + if pod.Labels[label.ComponentLabelKey] == "tidb" { + + // if tidb pod is deleting, allow pod delete operation + if pod.DeletionTimestamp != nil { + glog.Infof("TIDB pod status is namespace %s name %s timestamp %s", namespace, name, pod.DeletionTimestamp) + reviewResponse.Allowed = true return &reviewResponse } - res, err := httpClient.Do(req) + ordinal, err := strconv.ParseInt(strings.Split(name, "-")[len(strings.Split(name, "-"))-1], 10, 32) if err != nil { - glog.Errorf("fail to send request %v", err) + glog.Errorf("fail to convert string to int while deleting TiDB err %v", err) return &reviewResponse } - defer res.Body.Close() - content, err := ioutil.ReadAll(res.Body) + info, err := tidbController.GetInfo(tc, int32(ordinal)) if err != nil { - glog.Errorf("fail to read response %v", err) + glog.Errorf("fail to get tidb info error:%v", err) return &reviewResponse } - info := dbInfo{} - err = json.Unmarshal(content, &info) + if info.IsOwner && tc.Status.TiDB.StatefulSet.Replicas > 1 { + time.Sleep(10 * time.Second) + glog.Errorf("tidb is ddl owner, can't be deleted namespace %s name %s", namespace, name) + os.Exit(3) + } else { + glog.Infof("savely delete pod namespace %s name %s isowner %t", namespace, name, info.IsOwner) + } + + } else if pod.Labels[label.ComponentLabelKey] == "pd" { + + leader, err := pdClient.GetPDLeader() if err != nil { - glog.Errorf("unmarshal failed,namespace %s name %s error:%v", nameSpace, name, err) + glog.Errorf("fail to get pd leader %v", err) return &reviewResponse } - if !info.IsOwner { - glog.Infof("savely delete pod namespace %s name %s content [%s]", nameSpace, name, string(content)) + if leader.Name == name && tc.Status.TiDB.StatefulSet.Replicas > 1 { + time.Sleep(10 * time.Second) + glog.Errorf("pd is leader, can't be deleted namespace %s name %s", namespace, name) + os.Exit(3) + } else { + glog.Infof("savely delete pod namespace %s name %s leader name %s", namespace, name, leader.Name) } - if info.IsOwner { - glog.Errorf("tidb is ddl owner, can't be deleted namespace %s name %s", nameSpace, name) + } else if pod.Labels[label.ComponentLabelKey] == "tikv" { + + var storeID uint64 + storeID = 0 + for _, store := range tc.Status.TiKV.Stores { + if store.PodName == name { + storeID, err = strconv.ParseUint(store.ID, 10, 64) + if err != nil { + glog.Errorf("fail to convert string to int while deleting PD err %v", err) + return &reviewResponse + } + break + } + } + + // Fail to get store in stores + if storeID == 0 { + glog.Errorf("fail to find store in TIKV.Stores podname %s", name) + return &reviewResponse + } + + storeInfo, err := pdClient.GetStore(storeID) + if err != nil { + glog.Errorf("fail to read storeID %d response %v", storeID, err) + return &reviewResponse + } + + beforeCount := kvLeaderMap[namespace][name] + afterCount := storeInfo.Status.LeaderCount + + if beforeCount != 0 && beforeCount <= afterCount && tc.Status.TiKV.StatefulSet.Replicas > 1 { + time.Sleep(10 * time.Second) + glog.Errorf("kv leader is not zero, can't be deleted namespace %s name %s leaderCount %d", namespace, name, storeInfo.Status.LeaderCount) os.Exit(3) + } else { + glog.Infof("savely delete pod namespace %s name %s before count %d after count %d", namespace, name, beforeCount, afterCount) } } - + reviewResponse.Allowed = true return &reviewResponse }