From 851e659a580b3d7f22e1a34ee9cd635409b5fb58 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 11 Apr 2019 15:30:28 +0800 Subject: [PATCH 1/3] ensure test env normal and completed before cases run --- tests/cmd/stability/main.go | 6 ++- tests/fault.go | 46 +++++++++++++++++++ .../manager/static_pod_service.go | 9 ++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index e26177b88d..f16f1c23a1 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -39,6 +39,11 @@ func main() { conf := tests.ParseConfigOrDie() cli, kubeCli := client.NewCliOrDie() oa := tests.NewOperatorActions(cli, kubeCli, conf) + fta := tests.NewFaultTriggerAction(cli, kubeCli, conf) + err := fta.CheckAndRecoverEnv() + if err != nil { + glog.Fatal(err) + } tidbVersion := conf.GetTiDBVersionOrDie() upgardeTiDBVersions := conf.GetUpgradeTidbVersionsOrDie() @@ -203,7 +208,6 @@ func main() { backup.NewBackupCase(oa, clusterBackupFrom, clusterRestoreTo).RunOrDie() // stop a node and failover automatically - fta := tests.NewFaultTriggerAction(cli, kubeCli, conf) physicalNode, node, faultTime := fta.StopNodeOrDie() oa.CheckFailoverPendingOrDie(allClusters, &faultTime) oa.CheckFailoverOrDie(allClusters, node) diff --git a/tests/fault.go b/tests/fault.go index 4f8e75bea9..4125182441 100644 --- a/tests/fault.go +++ b/tests/fault.go @@ -23,6 +23,7 @@ const ( ) type FaultTriggerActions interface { + CheckAndRecoverEnv() error StopNode() (string, string, time.Time, error) StopNodeOrDie() (string, string, time.Time) StartNode(physicalNode string, node string) error @@ -62,6 +63,51 @@ type faultTriggerActions struct { cfg *Config } +func (fa *faultTriggerActions) CheckAndRecoverEnv() error { + // ensure all nodes are running + for _, physicalNode := range fa.cfg.Nodes { + for _, vNode := range physicalNode.Nodes { + err := fa.StartNode(physicalNode.PhysicalNode, vNode) + if err != nil { + return err + } + } + } + // ensure all etcds are running + err := fa.StartETCD() + if err != nil { + return err + } + // ensure all kubelets are running + for _, physicalNode := range fa.cfg.Nodes { + for _, vNode := range physicalNode.Nodes { + err := fa.StartKubelet(vNode) + if err != nil { + return err + } + } + } + // ensure all static pods are running + for _, physicalNode := range fa.cfg.APIServers { + for _, vNode := range physicalNode.Nodes { + err := fa.StartKubeAPIServer(vNode) + if err != nil { + return err + } + err = fa.StartKubeControllerManager(vNode) + if err != nil { + return err + } + err = fa.StartKubeScheduler(vNode) + if err != nil { + return err + } + } + } + + return nil +} + func (fa *faultTriggerActions) StopNode() (string, string, time.Time, error) { now := time.Now() node, err := getFaultNode(fa.kubeCli) diff --git a/tests/pkg/fault-trigger/manager/static_pod_service.go b/tests/pkg/fault-trigger/manager/static_pod_service.go index f30c38e6d1..79204951ec 100644 --- a/tests/pkg/fault-trigger/manager/static_pod_service.go +++ b/tests/pkg/fault-trigger/manager/static_pod_service.go @@ -15,6 +15,7 @@ package manager import ( "fmt" + "os" "os/exec" "github.com/golang/glog" @@ -74,6 +75,10 @@ func (m *Manager) StopKubeControllerManager() error { func (m *Manager) stopStaticPodService(serviceName string, fileName string) error { maniest := fmt.Sprintf("%s/%s", staticPodPath, fileName) + if _, err := os.Stat(maniest); os.IsNotExist(err) { + glog.Infof("%s had been stopped before", serviceName) + return nil + } shell := fmt.Sprintf("mkdir -p %s && mv %s %s", staticPodTmpPath, maniest, staticPodTmpPath) cmd := exec.Command("/bin/sh", "-c", shell) @@ -90,6 +95,10 @@ func (m *Manager) stopStaticPodService(serviceName string, fileName string) erro func (m *Manager) startStaticPodService(serviceName string, fileName string) error { maniest := fmt.Sprintf("%s/%s", staticPodTmpPath, fileName) + if _, err := os.Stat(maniest); err == nil { + glog.Infof("%s had been started before", serviceName) + return nil + } shell := fmt.Sprintf("mv %s %s", maniest, staticPodPath) cmd := exec.Command("/bin/sh", "-c", shell) From de811a5984d35a9eab60143d20517ee226cf6049 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Thu, 11 Apr 2019 16:24:36 +0800 Subject: [PATCH 2/3] add log --- tests/fault.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fault.go b/tests/fault.go index 4125182441..99eeb47f47 100644 --- a/tests/fault.go +++ b/tests/fault.go @@ -64,7 +64,7 @@ type faultTriggerActions struct { } func (fa *faultTriggerActions) CheckAndRecoverEnv() error { - // ensure all nodes are running + glog.Infof("ensure all nodes are running") for _, physicalNode := range fa.cfg.Nodes { for _, vNode := range physicalNode.Nodes { err := fa.StartNode(physicalNode.PhysicalNode, vNode) @@ -73,12 +73,12 @@ func (fa *faultTriggerActions) CheckAndRecoverEnv() error { } } } - // ensure all etcds are running + glog.Infof("ensure all etcds are running") err := fa.StartETCD() if err != nil { return err } - // ensure all kubelets are running + glog.Infof("ensure all kubelets are running") for _, physicalNode := range fa.cfg.Nodes { for _, vNode := range physicalNode.Nodes { err := fa.StartKubelet(vNode) @@ -87,7 +87,7 @@ func (fa *faultTriggerActions) CheckAndRecoverEnv() error { } } } - // ensure all static pods are running + glog.Infof("ensure all static pods are running") for _, physicalNode := range fa.cfg.APIServers { for _, vNode := range physicalNode.Nodes { err := fa.StartKubeAPIServer(vNode) From a8f1b4621d1bc0c273437f67dcb491c9a232987d Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Mon, 15 Apr 2019 17:00:47 +0800 Subject: [PATCH 3/3] address comment --- tests/cmd/stability/main.go | 5 +---- tests/fault.go | 7 +++++++ tests/pkg/fault-trigger/manager/static_pod_service.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index f16f1c23a1..199a486bb6 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -40,10 +40,7 @@ func main() { cli, kubeCli := client.NewCliOrDie() oa := tests.NewOperatorActions(cli, kubeCli, conf) fta := tests.NewFaultTriggerAction(cli, kubeCli, conf) - err := fta.CheckAndRecoverEnv() - if err != nil { - glog.Fatal(err) - } + fta.CheckAndRecoverEnvOrDie() tidbVersion := conf.GetTiDBVersionOrDie() upgardeTiDBVersions := conf.GetUpgradeTidbVersionsOrDie() diff --git a/tests/fault.go b/tests/fault.go index 99eeb47f47..0f97efdf26 100644 --- a/tests/fault.go +++ b/tests/fault.go @@ -24,6 +24,7 @@ const ( type FaultTriggerActions interface { CheckAndRecoverEnv() error + CheckAndRecoverEnvOrDie() StopNode() (string, string, time.Time, error) StopNodeOrDie() (string, string, time.Time) StartNode(physicalNode string, node string) error @@ -108,6 +109,12 @@ func (fa *faultTriggerActions) CheckAndRecoverEnv() error { return nil } +func (fa *faultTriggerActions) CheckAndRecoverEnvOrDie() { + if err:=fa.CheckAndRecoverEnv();err!=nil{ + glog.Fatal(err) + } +} + func (fa *faultTriggerActions) StopNode() (string, string, time.Time, error) { now := time.Now() node, err := getFaultNode(fa.kubeCli) diff --git a/tests/pkg/fault-trigger/manager/static_pod_service.go b/tests/pkg/fault-trigger/manager/static_pod_service.go index 79204951ec..b58e47cf89 100644 --- a/tests/pkg/fault-trigger/manager/static_pod_service.go +++ b/tests/pkg/fault-trigger/manager/static_pod_service.go @@ -95,7 +95,7 @@ func (m *Manager) stopStaticPodService(serviceName string, fileName string) erro func (m *Manager) startStaticPodService(serviceName string, fileName string) error { maniest := fmt.Sprintf("%s/%s", staticPodTmpPath, fileName) - if _, err := os.Stat(maniest); err == nil { + if _, err := os.Stat(maniest); os.IsNotExist(err) { glog.Infof("%s had been started before", serviceName) return nil }