Skip to content

Commit

Permalink
e2e: fine-grained control of waiting for apiservices/crds (#1511)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cofyc @sre-bot
cofyc authored and sre-bot committed Jan 15, 2020
1 parent a849b58 commit 0bf6259
Show file tree
Hide file tree
Showing 9 changed files with 170 additions and 46 deletions.
37 changes: 19 additions & 18 deletions tests/actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/pdapi"
"github.com/pingcap/tidb-operator/pkg/util"
e2eutil "github.com/pingcap/tidb-operator/tests/e2e/util"
utildiscovery "github.com/pingcap/tidb-operator/tests/e2e/util/discovery"
"github.com/pingcap/tidb-operator/tests/e2e/util/portforward"
"github.com/pingcap/tidb-operator/tests/e2e/util/proxiedpdclient"
Expand All @@ -59,6 +60,7 @@ import (
v1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
Expand All @@ -67,6 +69,7 @@ import (
"k8s.io/client-go/kubernetes"
typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
glog "k8s.io/klog"
aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
)
Expand All @@ -87,6 +90,8 @@ const (
func NewOperatorActions(cli versioned.Interface,
kubeCli kubernetes.Interface,
asCli asclientset.Interface,
aggrCli aggregatorclientset.Interface,
apiExtCli apiextensionsclientset.Interface,
pollInterval time.Duration,
operatorConfig *OperatorConfig,
cfg *Config,
Expand All @@ -106,6 +111,8 @@ func NewOperatorActions(cli versioned.Interface,
kubeCli: kubeCli,
pdControl: pdapi.NewDefaultPDControl(kubeCli),
asCli: asCli,
aggrCli: aggrCli,
apiExtCli: apiExtCli,
tcStsGetter: tcStsGetter,
pollInterval: pollInterval,
cfg: cfg,
Expand Down Expand Up @@ -239,6 +246,8 @@ type operatorActions struct {
cli versioned.Interface
kubeCli kubernetes.Interface
asCli asclientset.Interface
aggrCli aggregatorclientset.Interface
apiExtCli apiextensionsclientset.Interface
tcStsGetter typedappsv1.StatefulSetsGetter
pdControl pdapi.PDControlInterface
tidbControl controller.TiDBControlInterface
Expand Down Expand Up @@ -471,16 +480,8 @@ func (oa *operatorActions) InstallCRDOrDie() {
}
oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/crd.yaml"))
oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/data-resource-crd.yaml"))
out := oa.runKubectlOrDie([]string{"get", "crds", "--no-headers", `-ojsonpath={range .items[*]}{.metadata.name}{" "}{end}`}...)
waitArgs := []string{"wait", "--for=condition=Established"}
for _, crd := range strings.Split(out, " ") {
crd = strings.TrimSpace(crd)
if crd == "" {
continue
}
waitArgs = append(waitArgs, fmt.Sprintf("crds/%s", crd))
}
oa.runKubectlOrDie(waitArgs...)
glog.Infof("Wait for all CRDs are established")
e2eutil.WaitForCRDsEstablished(oa.apiExtCli, labels.Everything())
// workaround for https://github.com/kubernetes/kubernetes/issues/65517
glog.Infof("force sync kubectl cache")
cmdArgs := []string{"sh", "-c", "rm -rf ~/.kube/cache ~/.kube/http-cache"}
Expand Down Expand Up @@ -522,10 +523,8 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error {
return fmt.Errorf("failed to deploy operator: %v, %s", err, string(res))
}

// wait for all apiservices are available
// '-l a!=b' is a workaround solution for '--all' flag which is introduced only in kubectl 1.14+
oa.runKubectlOrDie("wait", "--for=condition=Available", "apiservices", "-l", "a!=b", "--timeout=60s")
return nil
glog.Infof("Wait for all apiesrvices are available")
return e2eutil.WaitForAPIServicesAvaiable(oa.aggrCli, labels.Everything())
}

func (oa *operatorActions) DeployOperatorOrDie(info *OperatorConfig) {
Expand Down Expand Up @@ -579,9 +578,11 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error {
return fmt.Errorf("failed to upgrade operator to: %s, %v, %s", info.Image, err, string(res))
}

// wait for all apiservices are available
// '-l a!=b' is a workaround solution for '--all' flag which is introduced only in kubectl 1.14+
oa.runKubectlOrDie("wait", "--for=condition=Available", "apiservices", "-l", "a!=b")
glog.Infof("Wait for all apiesrvices are available")
err = e2eutil.WaitForAPIServicesAvaiable(oa.aggrCli, labels.Everything())
if err != nil {
return err
}

if info.Tag == "e2e" {
return nil
Expand Down Expand Up @@ -3448,7 +3449,7 @@ func StartValidatingAdmissionWebhookServerOrDie(context *apimachinery.CertContex
panic(err)
}

versionCli, kubeCli, _ := client.NewCliOrDie()
versionCli, kubeCli, _, _, _ := client.NewCliOrDie()
wh := webhook.NewWebhook(kubeCli, versionCli, namespaces)
http.HandleFunc("/pods", wh.ServePods)
server := &http.Server{
Expand Down
4 changes: 2 additions & 2 deletions tests/cmd/stability/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func main() {
}

func run() {
cli, kubeCli, asCli := client.NewCliOrDie()
cli, kubeCli, asCli, aggrCli, apiExtCli := client.NewCliOrDie()

ocfg := newOperatorConfig()

Expand Down Expand Up @@ -115,7 +115,7 @@ func run() {
fta := tests.NewFaultTriggerAction(cli, kubeCli, cfg)
fta.CheckAndRecoverEnvOrDie()

oa := tests.NewOperatorActions(cli, kubeCli, asCli, tests.DefaultPollInterval, ocfg, cfg, allClusters, nil, nil)
oa := tests.NewOperatorActions(cli, kubeCli, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, cfg, allClusters, nil, nil)
oa.CheckK8sAvailableOrDie(nil, nil)
oa.LabelNodesOrDie()

Expand Down
8 changes: 7 additions & 1 deletion tests/e2e/e2e.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,14 @@ import (
e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config"
utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image"
v1 "k8s.io/api/core/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/component-base/logs"
"k8s.io/klog"
aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
Expand Down Expand Up @@ -157,6 +159,10 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
framework.ExpectNoError(err, "failed to create clientset")
kubeCli, err := kubernetes.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
aggrCli, err := aggregatorclientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
apiExtCli, err := apiextensionsclientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
asCli, err := asclientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
ginkgo.By("Recycle all local PVs")
Expand Down Expand Up @@ -186,7 +192,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
})
framework.ExpectNoError(err, "failed to wait for all PVs to be available")
ginkgo.By("Labeling nodes")
oa := tests.NewOperatorActions(cli, kubeCli, asCli, tests.DefaultPollInterval, nil, e2econfig.TestConfig, nil, nil, nil)
oa := tests.NewOperatorActions(cli, kubeCli, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, nil, e2econfig.TestConfig, nil, nil, nil)
oa.LabelNodesOrDie()
if e2econfig.TestConfig.InstallOperator {
ginkgo.By("Installing CRDs")
Expand Down
25 changes: 16 additions & 9 deletions tests/e2e/tidbcluster/serial.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,11 @@ import (
_ "net/http/pprof"
"time"

"github.com/onsi/gomega"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"k8s.io/klog"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
"k8s.io/utils/pointer"

"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1/helper"
asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/tests"
Expand All @@ -37,14 +33,19 @@ import (
"github.com/pingcap/tidb-operator/tests/e2e/util/portforward"
utilstatefulset "github.com/pingcap/tidb-operator/tests/e2e/util/statefulset"
v1 "k8s.io/api/core/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/klog"
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2esset "k8s.io/kubernetes/test/e2e/framework/statefulset"
"k8s.io/utils/pointer"
)

func mustToString(set sets.Int32) string {
Expand All @@ -62,6 +63,8 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
var c clientset.Interface
var cli versioned.Interface
var asCli asclientset.Interface
var aggrCli aggregatorclient.Interface
var apiExtCli apiextensionsclientset.Interface
var hc clientset.Interface
var cfg *tests.Config
var config *restclient.Config
Expand All @@ -78,6 +81,10 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
framework.ExpectNoError(err, "failed to create clientset")
asCli, err = asclientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
aggrCli, err = aggregatorclient.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
apiExtCli, err = apiextensionsclientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
clientRawConfig, err := e2econfig.LoadClientRawConfig()
framework.ExpectNoError(err, "failed to load raw config")
hc = helper.NewHijackClient(c, asCli)
Expand Down Expand Up @@ -114,7 +121,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
ImagePullPolicy: v1.PullIfNotPresent,
TestMode: true,
}
oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
ginkgo.By("Installing CRDs")
oa.CleanCRDOrDie()
oa.InstallCRDOrDie()
Expand Down Expand Up @@ -301,7 +308,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
PodWebhookEnabled: true,
StsWebhookEnabled: false,
}
oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
ginkgo.By("Installing CRDs")
oa.CleanCRDOrDie()
oa.InstallCRDOrDie()
Expand Down Expand Up @@ -357,7 +364,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
SchedulerReplicas: tests.IntPtr(0),
ControllerManagerReplicas: tests.IntPtr(0),
}
oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
ginkgo.By("Installing CRDs")
oa.CleanCRDOrDie()
oa.InstallCRDOrDie()
Expand Down
22 changes: 14 additions & 8 deletions tests/e2e/tidbcluster/tidbcluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,14 @@ import (
"strings"
"time"

"github.com/pingcap/tidb-operator/tests/pkg/fixture"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/pointer"

"github.com/pingcap/tidb-operator/pkg/label"

"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/features"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/manager/member"
"github.com/pingcap/tidb-operator/pkg/scheme"
operatorUtils "github.com/pingcap/tidb-operator/pkg/util"
Expand All @@ -46,19 +40,25 @@ import (
"github.com/pingcap/tidb-operator/tests/e2e/util/portforward"
"github.com/pingcap/tidb-operator/tests/pkg/apimachinery"
"github.com/pingcap/tidb-operator/tests/pkg/blockwriter"
"github.com/pingcap/tidb-operator/tests/pkg/fixture"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/klog"
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
"k8s.io/kubernetes/test/utils"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
)

Expand All @@ -69,6 +69,8 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
var c clientset.Interface
var cli versioned.Interface
var asCli asclientset.Interface
var aggrCli aggregatorclient.Interface
var apiExtCli apiextensionsclientset.Interface
var oa tests.OperatorActions
var cfg *tests.Config
var config *restclient.Config
Expand All @@ -89,6 +91,10 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
framework.ExpectNoError(err, "failed to create clientset")
genericCli, err = client.New(config, client.Options{Scheme: scheme.Scheme})
framework.ExpectNoError(err, "failed to create clientset")
aggrCli, err = aggregatorclient.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
apiExtCli, err = apiextensionsclientset.NewForConfig(config)
framework.ExpectNoError(err, "failed to create clientset")
clientRawConfig, err := e2econfig.LoadClientRawConfig()
framework.ExpectNoError(err, "failed to load raw config")
ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -97,7 +103,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
fwCancel = cancel
cfg = e2econfig.TestConfig
ocfg = e2econfig.NewDefaultOperatorConfig(cfg)
oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
})

ginkgo.AfterEach(func() {
Expand Down
Loading

0 comments on commit 0bf6259

Please sign in to comment.