From 0bf625985d2b8ec35e845fffe31a135567695fd8 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Wed, 15 Jan 2020 12:21:20 +0800 Subject: [PATCH] e2e: fine-grained control of waiting for apiservices/crds (#1511) --- tests/actions.go | 37 +++++------ tests/cmd/stability/main.go | 4 +- tests/e2e/e2e.go | 8 ++- tests/e2e/tidbcluster/serial.go | 25 +++++--- tests/e2e/tidbcluster/tidbcluster.go | 22 ++++--- tests/e2e/util/util.go | 93 ++++++++++++++++++++++++++++ tests/failover.go | 2 +- tests/pkg/client/client.go | 23 +++++-- tests/pkg/webhook/pods.go | 2 +- 9 files changed, 170 insertions(+), 46 deletions(-) create mode 100644 tests/e2e/util/util.go diff --git a/tests/actions.go b/tests/actions.go index c3260521e8..e9f2e5ed65 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -45,6 +45,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/pdapi" "github.com/pingcap/tidb-operator/pkg/util" + e2eutil "github.com/pingcap/tidb-operator/tests/e2e/util" utildiscovery "github.com/pingcap/tidb-operator/tests/e2e/util/discovery" "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" "github.com/pingcap/tidb-operator/tests/e2e/util/proxiedpdclient" @@ -59,6 +60,7 @@ import ( v1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -67,6 +69,7 @@ import ( "k8s.io/client-go/kubernetes" typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" glog "k8s.io/klog" + aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" ) @@ -87,6 +90,8 @@ const ( func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, asCli asclientset.Interface, + aggrCli aggregatorclientset.Interface, + apiExtCli apiextensionsclientset.Interface, pollInterval time.Duration, operatorConfig *OperatorConfig, cfg *Config, @@ -106,6 +111,8 @@ func NewOperatorActions(cli versioned.Interface, kubeCli: kubeCli, pdControl: pdapi.NewDefaultPDControl(kubeCli), asCli: asCli, + aggrCli: aggrCli, + apiExtCli: apiExtCli, tcStsGetter: tcStsGetter, pollInterval: pollInterval, cfg: cfg, @@ -239,6 +246,8 @@ type operatorActions struct { cli versioned.Interface kubeCli kubernetes.Interface asCli asclientset.Interface + aggrCli aggregatorclientset.Interface + apiExtCli apiextensionsclientset.Interface tcStsGetter typedappsv1.StatefulSetsGetter pdControl pdapi.PDControlInterface tidbControl controller.TiDBControlInterface @@ -471,16 +480,8 @@ func (oa *operatorActions) InstallCRDOrDie() { } oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/crd.yaml")) oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/data-resource-crd.yaml")) - out := oa.runKubectlOrDie([]string{"get", "crds", "--no-headers", `-ojsonpath={range .items[*]}{.metadata.name}{" "}{end}`}...) - waitArgs := []string{"wait", "--for=condition=Established"} - for _, crd := range strings.Split(out, " ") { - crd = strings.TrimSpace(crd) - if crd == "" { - continue - } - waitArgs = append(waitArgs, fmt.Sprintf("crds/%s", crd)) - } - oa.runKubectlOrDie(waitArgs...) + glog.Infof("Wait for all CRDs are established") + e2eutil.WaitForCRDsEstablished(oa.apiExtCli, labels.Everything()) // workaround for https://github.com/kubernetes/kubernetes/issues/65517 glog.Infof("force sync kubectl cache") cmdArgs := []string{"sh", "-c", "rm -rf ~/.kube/cache ~/.kube/http-cache"} @@ -522,10 +523,8 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { return fmt.Errorf("failed to deploy operator: %v, %s", err, string(res)) } - // wait for all apiservices are available - // '-l a!=b' is a workaround solution for '--all' flag which is introduced only in kubectl 1.14+ - oa.runKubectlOrDie("wait", "--for=condition=Available", "apiservices", "-l", "a!=b", "--timeout=60s") - return nil + glog.Infof("Wait for all apiesrvices are available") + return e2eutil.WaitForAPIServicesAvaiable(oa.aggrCli, labels.Everything()) } func (oa *operatorActions) DeployOperatorOrDie(info *OperatorConfig) { @@ -579,9 +578,11 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error { return fmt.Errorf("failed to upgrade operator to: %s, %v, %s", info.Image, err, string(res)) } - // wait for all apiservices are available - // '-l a!=b' is a workaround solution for '--all' flag which is introduced only in kubectl 1.14+ - oa.runKubectlOrDie("wait", "--for=condition=Available", "apiservices", "-l", "a!=b") + glog.Infof("Wait for all apiesrvices are available") + err = e2eutil.WaitForAPIServicesAvaiable(oa.aggrCli, labels.Everything()) + if err != nil { + return err + } if info.Tag == "e2e" { return nil @@ -3448,7 +3449,7 @@ func StartValidatingAdmissionWebhookServerOrDie(context *apimachinery.CertContex panic(err) } - versionCli, kubeCli, _ := client.NewCliOrDie() + versionCli, kubeCli, _, _, _ := client.NewCliOrDie() wh := webhook.NewWebhook(kubeCli, versionCli, namespaces) http.HandleFunc("/pods", wh.ServePods) server := &http.Server{ diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index 06c1a4a806..f50caeb363 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -73,7 +73,7 @@ func main() { } func run() { - cli, kubeCli, asCli := client.NewCliOrDie() + cli, kubeCli, asCli, aggrCli, apiExtCli := client.NewCliOrDie() ocfg := newOperatorConfig() @@ -115,7 +115,7 @@ func run() { fta := tests.NewFaultTriggerAction(cli, kubeCli, cfg) fta.CheckAndRecoverEnvOrDie() - oa := tests.NewOperatorActions(cli, kubeCli, asCli, tests.DefaultPollInterval, ocfg, cfg, allClusters, nil, nil) + oa := tests.NewOperatorActions(cli, kubeCli, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, cfg, allClusters, nil, nil) oa.CheckK8sAvailableOrDie(nil, nil) oa.LabelNodesOrDie() diff --git a/tests/e2e/e2e.go b/tests/e2e/e2e.go index e62e351b07..d53fd2dfec 100644 --- a/tests/e2e/e2e.go +++ b/tests/e2e/e2e.go @@ -35,12 +35,14 @@ import ( e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config" utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image" v1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeutils "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/component-base/logs" "k8s.io/klog" + aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -157,6 +159,10 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { framework.ExpectNoError(err, "failed to create clientset") kubeCli, err := kubernetes.NewForConfig(config) framework.ExpectNoError(err, "failed to create clientset") + aggrCli, err := aggregatorclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + apiExtCli, err := apiextensionsclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") asCli, err := asclientset.NewForConfig(config) framework.ExpectNoError(err, "failed to create clientset") ginkgo.By("Recycle all local PVs") @@ -186,7 +192,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { }) framework.ExpectNoError(err, "failed to wait for all PVs to be available") ginkgo.By("Labeling nodes") - oa := tests.NewOperatorActions(cli, kubeCli, asCli, tests.DefaultPollInterval, nil, e2econfig.TestConfig, nil, nil, nil) + oa := tests.NewOperatorActions(cli, kubeCli, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, nil, e2econfig.TestConfig, nil, nil, nil) oa.LabelNodesOrDie() if e2econfig.TestConfig.InstallOperator { ginkgo.By("Installing CRDs") diff --git a/tests/e2e/tidbcluster/serial.go b/tests/e2e/tidbcluster/serial.go index 593e555a52..6b714a4360 100644 --- a/tests/e2e/tidbcluster/serial.go +++ b/tests/e2e/tidbcluster/serial.go @@ -20,15 +20,11 @@ import ( _ "net/http/pprof" "time" - "github.com/onsi/gomega" - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" - "k8s.io/klog" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" - "k8s.io/utils/pointer" - "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1/helper" asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/tests" @@ -37,14 +33,19 @@ import ( "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" utilstatefulset "github.com/pingcap/tidb-operator/tests/e2e/util/statefulset" v1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/klog" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/test/e2e/framework" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2esset "k8s.io/kubernetes/test/e2e/framework/statefulset" + "k8s.io/utils/pointer" ) func mustToString(set sets.Int32) string { @@ -62,6 +63,8 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { var c clientset.Interface var cli versioned.Interface var asCli asclientset.Interface + var aggrCli aggregatorclient.Interface + var apiExtCli apiextensionsclientset.Interface var hc clientset.Interface var cfg *tests.Config var config *restclient.Config @@ -78,6 +81,10 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { framework.ExpectNoError(err, "failed to create clientset") asCli, err = asclientset.NewForConfig(config) framework.ExpectNoError(err, "failed to create clientset") + aggrCli, err = aggregatorclient.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + apiExtCli, err = apiextensionsclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") clientRawConfig, err := e2econfig.LoadClientRawConfig() framework.ExpectNoError(err, "failed to load raw config") hc = helper.NewHijackClient(c, asCli) @@ -114,7 +121,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { ImagePullPolicy: v1.PullIfNotPresent, TestMode: true, } - oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) + oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) ginkgo.By("Installing CRDs") oa.CleanCRDOrDie() oa.InstallCRDOrDie() @@ -301,7 +308,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { PodWebhookEnabled: true, StsWebhookEnabled: false, } - oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) + oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) ginkgo.By("Installing CRDs") oa.CleanCRDOrDie() oa.InstallCRDOrDie() @@ -357,7 +364,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { SchedulerReplicas: tests.IntPtr(0), ControllerManagerReplicas: tests.IntPtr(0), } - oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) + oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) ginkgo.By("Installing CRDs") oa.CleanCRDOrDie() oa.InstallCRDOrDie() diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go index 9992021e52..36cce82c6e 100644 --- a/tests/e2e/tidbcluster/tidbcluster.go +++ b/tests/e2e/tidbcluster/tidbcluster.go @@ -21,13 +21,6 @@ import ( "strings" "time" - "github.com/pingcap/tidb-operator/tests/pkg/fixture" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" - - "github.com/pingcap/tidb-operator/pkg/label" - "github.com/onsi/ginkgo" "github.com/onsi/gomega" asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" @@ -35,6 +28,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/features" + "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/manager/member" "github.com/pingcap/tidb-operator/pkg/scheme" operatorUtils "github.com/pingcap/tidb-operator/pkg/util" @@ -46,19 +40,25 @@ import ( "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" "github.com/pingcap/tidb-operator/tests/pkg/apimachinery" "github.com/pingcap/tidb-operator/tests/pkg/blockwriter" + "github.com/pingcap/tidb-operator/tests/pkg/fixture" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/klog" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/utils" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -69,6 +69,8 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { var c clientset.Interface var cli versioned.Interface var asCli asclientset.Interface + var aggrCli aggregatorclient.Interface + var apiExtCli apiextensionsclientset.Interface var oa tests.OperatorActions var cfg *tests.Config var config *restclient.Config @@ -89,6 +91,10 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { framework.ExpectNoError(err, "failed to create clientset") genericCli, err = client.New(config, client.Options{Scheme: scheme.Scheme}) framework.ExpectNoError(err, "failed to create clientset") + aggrCli, err = aggregatorclient.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + apiExtCli, err = apiextensionsclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") clientRawConfig, err := e2econfig.LoadClientRawConfig() framework.ExpectNoError(err, "failed to load raw config") ctx, cancel := context.WithCancel(context.Background()) @@ -97,7 +103,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { fwCancel = cancel cfg = e2econfig.TestConfig ocfg = e2econfig.NewDefaultOperatorConfig(cfg) - oa = tests.NewOperatorActions(cli, c, asCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) + oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) }) ginkgo.AfterEach(func() { diff --git a/tests/e2e/util/util.go b/tests/e2e/util/util.go new file mode 100644 index 0000000000..a26ff7f3b8 --- /dev/null +++ b/tests/e2e/util/util.go @@ -0,0 +1,93 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "time" + + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +// WaitForAPIServicesAvaiable waits for apiservices to be available +func WaitForAPIServicesAvaiable(client aggregatorclientset.Interface, selector labels.Selector) error { + isAvaiable := func(status apiregistrationv1.APIServiceStatus) bool { + if status.Conditions == nil { + return false + } + for _, condition := range status.Conditions { + if condition.Type == apiregistrationv1.Available { + return condition.Status == apiregistrationv1.ConditionTrue + } + } + return false + } + return wait.PollImmediate(5*time.Second, 3*time.Minute, func() (bool, error) { + apiServiceList, err := client.ApiregistrationV1().APIServices().List(metav1.ListOptions{ + LabelSelector: selector.String(), + }) + if err != nil { + return false, err + } + for _, apiService := range apiServiceList.Items { + if !isAvaiable(apiService.Status) { + framework.Logf("APIService %q is not available yet", apiService.Name) + return false, nil + } + } + for _, apiService := range apiServiceList.Items { + framework.Logf("APIService %q is available", apiService.Name) + } + return true, nil + }) +} + +// WaitForCRDsEstablished waits for all CRDs to be established +func WaitForCRDsEstablished(client apiextensionsclientset.Interface, selector labels.Selector) error { + isEstalbished := func(status apiextensionsv1beta1.CustomResourceDefinitionStatus) bool { + if status.Conditions == nil { + return false + } + for _, condition := range status.Conditions { + if condition.Type == apiextensionsv1beta1.Established { + return condition.Status == apiextensionsv1beta1.ConditionTrue + } + } + return false + } + return wait.PollImmediate(5*time.Second, 3*time.Minute, func() (bool, error) { + crdList, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().List(metav1.ListOptions{ + LabelSelector: selector.String(), + }) + if err != nil { + return false, err + } + for _, crd := range crdList.Items { + if !isEstalbished(crd.Status) { + framework.Logf("CRD %q is not established yet", crd.Name) + return false, nil + } + } + for _, crd := range crdList.Items { + framework.Logf("CRD %q is established", crd.Name) + } + return true, nil + }) +} diff --git a/tests/failover.go b/tests/failover.go index 56b1524e43..c61981d33c 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -233,7 +233,7 @@ func (oa *operatorActions) CheckFailoverPending(info *TidbClusterConfig, node st glog.Infof("pending failover,failed to get tidbcluster:[%s], error: %v", info.FullName(), err) if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") { glog.Info("create new client") - newCli, _, _ := client.NewCliOrDie() + newCli, _, _, _, _ := client.NewCliOrDie() oa.cli = newCli } return false, nil diff --git a/tests/pkg/client/client.go b/tests/pkg/client/client.go index 0357a44ea2..ed77d59338 100644 --- a/tests/pkg/client/client.go +++ b/tests/pkg/client/client.go @@ -19,16 +19,17 @@ import ( "os" "time" - exampleagg "github.com/pingcap/tidb-operator/tests/pkg/apiserver/client/clientset/versioned" - "github.com/pingcap/tidb-operator/tests/slack" - "github.com/juju/errors" asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/typed/pingcap/v1alpha1" + exampleagg "github.com/pingcap/tidb-operator/tests/pkg/apiserver/client/clientset/versioned" + "github.com/pingcap/tidb-operator/tests/slack" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" ) var ( @@ -44,7 +45,7 @@ func RegisterFlags() { "Only required if out-of-cluster.") } -func NewCliOrDie() (versioned.Interface, kubernetes.Interface, asclientset.Interface) { +func NewCliOrDie() (versioned.Interface, kubernetes.Interface, asclientset.Interface, aggregatorclientset.Interface, apiextensionsclientset.Interface) { cfg, err := GetConfig() if err != nil { slack.NotifyAndPanic(err) @@ -131,7 +132,7 @@ func LoadConfig() (*rest.Config, error) { return cfg, errors.Trace(err) } -func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interface, asclientset.Interface) { +func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interface, asclientset.Interface, aggregatorclientset.Interface, apiextensionsclientset.Interface) { cfg.Timeout = 30 * time.Second cli, err := versioned.NewForConfig(cfg) if err != nil { @@ -148,5 +149,15 @@ func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interf slack.NotifyAndPanic(err) } - return cli, kubeCli, asCli + aggrCli, err := aggregatorclientset.NewForConfig(cfg) + if err != nil { + slack.NotifyAndPanic(err) + } + + apiExtCli, err := apiextensionsclientset.NewForConfig(cfg) + if err != nil { + slack.NotifyAndPanic(err) + } + + return cli, kubeCli, asCli, aggrCli, apiExtCli } diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go index 8ac5f99cb1..799c5e8c87 100644 --- a/tests/pkg/webhook/pods.go +++ b/tests/pkg/webhook/pods.go @@ -42,7 +42,7 @@ func (wh *webhook) admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionRespo return toAdmissionResponse(err) } - versionCli, kubeCli, _ := client.NewCliOrDie() + versionCli, kubeCli, _, _, _ := client.NewCliOrDie() name := ar.Request.Name namespace := ar.Request.Namespace