From e35b5027946127af5718d342ddc96b53b9357827 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Tue, 6 Apr 2021 21:54:27 -0700 Subject: [PATCH 1/5] Address Excessive API Server calls from CNI Pods --- cmd/aws-k8s-agent/main.go | 17 +- cmd/cni-metrics-helper/main.go | 18 +- cmd/cni-metrics-helper/metrics/cni_metrics.go | 28 +- .../metrics/cni_metrics_test.go | 26 +- cmd/cni-metrics-helper/metrics/metrics.go | 26 +- .../metrics/metrics_test.go | 11 +- cmd/cni-metrics-helper/metrics/pod_watcher.go | 52 +++ go.mod | 10 +- pkg/apis/crd/v1alpha1/register.go | 37 -- pkg/apis/crd/v1alpha1/types.go | 35 -- .../crd/v1alpha1/zz_generated.deepcopy.go | 31 +- pkg/eniconfig/eniconfig.go | 190 +++------- pkg/eniconfig/eniconfig_test.go | 356 +++++++++--------- pkg/ipamd/introspect.go | 6 +- pkg/ipamd/ipamd.go | 88 +++-- pkg/ipamd/ipamd_test.go | 156 +++++--- pkg/k8sapi/discovery.go | 340 ----------------- pkg/k8sapi/generate_mocks.go | 16 - pkg/k8sapi/mocks/k8sapi_mocks.go | 78 ---- 19 files changed, 549 insertions(+), 972 deletions(-) create mode 100644 cmd/cni-metrics-helper/metrics/pod_watcher.go delete mode 100644 pkg/apis/crd/v1alpha1/register.go delete mode 100644 pkg/apis/crd/v1alpha1/types.go delete mode 100644 pkg/k8sapi/discovery.go delete mode 100644 pkg/k8sapi/generate_mocks.go delete mode 100644 pkg/k8sapi/mocks/k8sapi_mocks.go diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go index 90b7415c78..bcde990819 100644 --- a/cmd/aws-k8s-agent/main.go +++ b/cmd/aws-k8s-agent/main.go @@ -15,12 +15,10 @@ package main import ( - "os" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd" "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "os" ) var version string @@ -39,18 +37,17 @@ func _main() int { log.Infof("Starting L-IPAMD %s ...", version) - kubeClient, err := k8sapi.CreateKubeClient() - if err != nil { - log.Errorf("Failed to create client: %v", err) + //Check API Server Connectivity + if k8sapi.CheckAPIServerConnectivity() != nil{ return 1 } - eniConfigController := eniconfig.NewENIConfigController() - if ipamd.UseCustomNetworkCfg() { - go eniConfigController.Start() + standaloneK8SClient, k8sClient, err := k8sapi.CreateKubeClients() + if err != nil{ + return 1 } - ipamContext, err := ipamd.New(kubeClient, eniConfigController) + ipamContext, err := ipamd.New(standaloneK8SClient, k8sClient) if err != nil { log.Errorf("Initialization failure: %v", err) diff --git a/cmd/cni-metrics-helper/main.go b/cmd/cni-metrics-helper/main.go index 30984f27fe..e7b741d088 100644 --- a/cmd/cni-metrics-helper/main.go +++ b/cmd/cni-metrics-helper/main.go @@ -52,6 +52,8 @@ func main() { _, _ = fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) flags.PrintDefaults() } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() err := flags.Parse(os.Args) if err != nil { @@ -80,20 +82,17 @@ func main() { log.Infof("Starting CNIMetricsHelper. Sending metrics to CloudWatch: %v, LogLevel %s", options.submitCW, logConfig.LogLevel) - kubeClient, err := k8sapi.CreateKubeClient() + clientSet, err := k8sapi.GetKubeClientSet() + + _, k8sClient, err := k8sapi.CreateKubeClients() if err != nil { - log.Fatalf("Failed to create client: %v", err) + panic(err.Error()) } - discoverController := k8sapi.NewController(kubeClient) - go discoverController.DiscoverCNIK8SPods() var cw publisher.Publisher if options.submitCW { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cw, err = publisher.New(ctx) if err != nil { log.Fatalf("Failed to create publisher: %v", err) @@ -102,12 +101,13 @@ func main() { defer cw.Stop() } - var cniMetric = metrics.CNIMetricsNew(kubeClient, cw, discoverController, options.submitCW, log) + podWatcher := metrics.NewDefaultPodWatcher(k8sClient, log) + var cniMetric = metrics.CNIMetricsNew(clientSet, cw, options.submitCW, log, podWatcher) // metric loop var pullInterval = 30 // seconds for range time.Tick(time.Duration(pullInterval) * time.Second) { log.Info("Collecting metrics ...") - metrics.Handler(cniMetric) + metrics.Handler(ctx, cniMetric) } } diff --git a/cmd/cni-metrics-helper/metrics/cni_metrics.go b/cmd/cni-metrics-helper/metrics/cni_metrics.go index 95743f237b..edf134fafd 100644 --- a/cmd/cni-metrics-helper/metrics/cni_metrics.go +++ b/cmd/cni-metrics-helper/metrics/cni_metrics.go @@ -16,10 +16,10 @@ package metrics import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "golang.org/x/net/context" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" ) @@ -141,26 +141,27 @@ var InterestingCNIMetrics = map[string]metricsConvert{ type CNIMetricsTarget struct { interestingMetrics map[string]metricsConvert cwMetricsPublisher publisher.Publisher - kubeClient clientset.Interface - discoveryController *k8sapi.Controller + kubeClient kubernetes.Interface + podWatcher *defaultPodWatcher submitCW bool log logger.Logger } // CNIMetricsNew creates a new metricsTarget -func CNIMetricsNew(c clientset.Interface, cw publisher.Publisher, d *k8sapi.Controller, submitCW bool, l logger.Logger) *CNIMetricsTarget { +func CNIMetricsNew(k8sClient kubernetes.Interface, cw publisher.Publisher, submitCW bool, l logger.Logger, + watcher *defaultPodWatcher) *CNIMetricsTarget { return &CNIMetricsTarget{ interestingMetrics: InterestingCNIMetrics, cwMetricsPublisher: cw, - kubeClient: c, - discoveryController: d, + kubeClient: k8sClient, + podWatcher: watcher, submitCW: submitCW, log: l, } } -func (t *CNIMetricsTarget) grabMetricsFromTarget(cniPod string) ([]byte, error) { - output, err := getMetricsFromPod(t.kubeClient, cniPod, metav1.NamespaceSystem, metricsPort) +func (t *CNIMetricsTarget) grabMetricsFromTarget(ctx context.Context, cniPod string) ([]byte, error) { + output, err := getMetricsFromPod(ctx, t.kubeClient, cniPod, metav1.NamespaceSystem, metricsPort) if err != nil { t.log.Errorf("grabMetricsFromTarget: Failed to grab CNI endpoint: %v", err) return nil, err @@ -178,9 +179,12 @@ func (t *CNIMetricsTarget) getCWMetricsPublisher() publisher.Publisher { return t.cwMetricsPublisher } -func (t *CNIMetricsTarget) getTargetList() []string { - pods := t.discoveryController.GetCNIPods() - return pods +func (t *CNIMetricsTarget) getTargetList(ctx context.Context) ([]string, error) { + pods, err := t.podWatcher.GetCNIPods(ctx) + if err != nil { + return pods, err + } + return pods, nil } func (t *CNIMetricsTarget) submitCloudWatch() bool { diff --git a/cmd/cni-metrics-helper/metrics/cni_metrics_test.go b/cmd/cni-metrics-helper/metrics/cni_metrics_test.go index fe6ed62542..3835229307 100644 --- a/cmd/cni-metrics-helper/metrics/cni_metrics_test.go +++ b/cmd/cni-metrics-helper/metrics/cni_metrics_test.go @@ -1,15 +1,19 @@ package metrics import ( + eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + "golang.org/x/net/context" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher/mock_publisher" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" ) @@ -18,33 +22,37 @@ var logConfig = logger.Configuration{ LogLocation: "stdout", } -var log = logger.New(&logConfig) +var testLog = logger.New(&logConfig) type testMocks struct { - ctrl *gomock.Controller clientset *k8sfake.Clientset - discoverController *k8sapi.Controller + podWatcher *defaultPodWatcher mockPublisher *mock_publisher.MockPublisher } func setup(t *testing.T) *testMocks { ctrl := gomock.NewController(t) fakeClientset := k8sfake.NewSimpleClientset() + k8sSchema := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sSchema) + eniconfigscheme.AddToScheme(k8sSchema) + podWatcher := NewDefaultPodWatcher(testclient.NewFakeClientWithScheme(k8sSchema), testLog) return &testMocks{ - ctrl: ctrl, clientset: fakeClientset, - discoverController: k8sapi.NewController(fakeClientset), + podWatcher: podWatcher, mockPublisher: mock_publisher.NewMockPublisher(ctrl), } } func TestCNIMetricsNew(t *testing.T) { m := setup(t) - _, _ = m.clientset.CoreV1().Pods("kube-system").Create(&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "aws-node-1"}}) - cniMetric := CNIMetricsNew(m.clientset, m.mockPublisher, m.discoverController, false, log) + ctx := context.Background() + _, _ = m.clientset.CoreV1().Pods("kube-system").Create(ctx, &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "aws-node-1"}}, metav1.CreateOptions{}) + //cniMetric := CNIMetricsNew(m.clientset, m.mockPublisher, m.discoverController, false, log) + cniMetric := CNIMetricsNew(m.clientset, m.mockPublisher, false, testLog, m.podWatcher) assert.NotNil(t, cniMetric) assert.NotNil(t, cniMetric.getCWMetricsPublisher()) assert.NotEmpty(t, cniMetric.getInterestingMetrics()) - assert.Equal(t, log, cniMetric.getLogger()) + assert.Equal(t, testLog, cniMetric.getLogger()) assert.False(t, cniMetric.submitCloudWatch()) } diff --git a/cmd/cni-metrics-helper/metrics/metrics.go b/cmd/cni-metrics-helper/metrics/metrics.go index 1a6b7027d3..9913850502 100644 --- a/cmd/cni-metrics-helper/metrics/metrics.go +++ b/cmd/cni-metrics-helper/metrics/metrics.go @@ -15,8 +15,11 @@ package metrics import ( + "context" "bytes" "fmt" + "github.com/prometheus/common/log" + "k8s.io/client-go/kubernetes" "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" @@ -24,17 +27,16 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatch" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" - clientset "k8s.io/client-go/kubernetes" ) type metricMatcher func(metric *dto.Metric) bool type actionFuncType func(aggregatedValue *float64, sampleValue float64) type metricsTarget interface { - grabMetricsFromTarget(target string) ([]byte, error) + grabMetricsFromTarget(ctx context.Context, target string) ([]byte, error) getInterestingMetrics() map[string]metricsConvert getCWMetricsPublisher() publisher.Publisher - getTargetList() []string + getTargetList(ctx context.Context) ([]string, error) submitCloudWatch() bool getLogger() logger.Logger } @@ -81,14 +83,15 @@ func metricsMax(aggregatedValue *float64, sampleValue float64) { } } -func getMetricsFromPod(client clientset.Interface, podName string, namespace string, port int) ([]byte, error) { - rawOutput, err := client.CoreV1().RESTClient().Get(). +func getMetricsFromPod(ctx context.Context, k8sClient kubernetes.Interface, podName string, namespace string, port int) ([]byte, error) { + rawOutput, err := k8sClient.CoreV1().RESTClient().Get(). Namespace(namespace). Resource("pods"). SubResource("proxy"). Name(fmt.Sprintf("%v:%v", podName, port)). Suffix("metrics"). - Do().Raw() + Do(ctx).Raw() + if err != nil { return nil, err } @@ -365,16 +368,17 @@ func resetMetrics(interestingMetrics map[string]metricsConvert) { } } -func metricsListGrabAggregateConvert(t metricsTarget) (map[string]*dto.MetricFamily, map[string]metricsConvert, bool, error) { +func metricsListGrabAggregateConvert(ctx context.Context, t metricsTarget) (map[string]*dto.MetricFamily, map[string]metricsConvert, bool, error) { var resetDetected = false var families map[string]*dto.MetricFamily interestingMetrics := t.getInterestingMetrics() resetMetrics(interestingMetrics) - targetList := t.getTargetList() + targetList, _ := t.getTargetList(ctx) + log.Debugf("Total TargetList pod count:- %v", len(targetList)) for _, target := range targetList { - rawOutput, err := t.grabMetricsFromTarget(target) + rawOutput, err := t.grabMetricsFromTarget(ctx, target) if err != nil { // it may take times to remove some metric targets continue @@ -413,8 +417,8 @@ func metricsListGrabAggregateConvert(t metricsTarget) (map[string]*dto.MetricFam } // Handler grabs metrics from target, aggregates the metrics and convert them into cloudwatch metrics -func Handler(t metricsTarget) { - families, interestingMetrics, resetDetected, err := metricsListGrabAggregateConvert(t) +func Handler(ctx context.Context, t metricsTarget) { + families, interestingMetrics, resetDetected, err := metricsListGrabAggregateConvert(ctx, t) if err != nil || resetDetected { t.getLogger().Infof("Skipping 1st poll after reset, error: %v", err) diff --git a/cmd/cni-metrics-helper/metrics/metrics_test.go b/cmd/cni-metrics-helper/metrics/metrics_test.go index ab21f4c7ee..1b72a084f1 100644 --- a/cmd/cni-metrics-helper/metrics/metrics_test.go +++ b/cmd/cni-metrics-helper/metrics/metrics_test.go @@ -14,6 +14,7 @@ package metrics import ( + "golang.org/x/net/context" "io/ioutil" "testing" @@ -37,7 +38,7 @@ func newTestMetricsTarget(metricFile string, interestingMetrics map[string]metri interestingMetrics: interestingMetrics} } -func (target *testMetricsTarget) grabMetricsFromTarget(targetName string) ([]byte, error) { +func (target *testMetricsTarget) grabMetricsFromTarget(ctx context.Context, targetName string) ([]byte, error) { testMetrics, _ := ioutil.ReadFile(target.metricFile) return testMetrics, nil @@ -51,8 +52,8 @@ func (target *testMetricsTarget) getCWMetricsPublisher() publisher.Publisher { return nil } -func (target *testMetricsTarget) getTargetList() []string { - return []string{target.metricFile} +func (target *testMetricsTarget) getTargetList(ctx context.Context) ([]string, error) { + return []string{target.metricFile}, nil } func (target *testMetricsTarget) submitCloudWatch() bool { @@ -61,8 +62,8 @@ func (target *testMetricsTarget) submitCloudWatch() bool { func TestAPIServerMetric(t *testing.T) { testTarget := newTestMetricsTarget("cni_test1.data", InterestingCNIMetrics) - - _, _, resetDetected, err := metricsListGrabAggregateConvert(testTarget) + ctx := context.Background() + _, _, resetDetected, err := metricsListGrabAggregateConvert(ctx, testTarget) assert.NoError(t, err) assert.True(t, resetDetected) diff --git a/cmd/cni-metrics-helper/metrics/pod_watcher.go b/cmd/cni-metrics-helper/metrics/pod_watcher.go new file mode 100644 index 0000000000..41b7775939 --- /dev/null +++ b/cmd/cni-metrics-helper/metrics/pod_watcher.go @@ -0,0 +1,52 @@ +package metrics + +import ( + "context" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type PodWatcher interface { + GetCNIPods(ctx context.Context) ([]string, error) +} + +type defaultPodWatcher struct { + k8sClient client.Client + log logger.Logger +} + +// NewDefaultPodWatcher creates a new podWatcher +func NewDefaultPodWatcher(k8sClient client.Client, log logger.Logger) *defaultPodWatcher{ + return &defaultPodWatcher{ + k8sClient: k8sClient, + log: log, + } +} + +//Returns aws-node pod info. Below function assumes CNI pods follow aws-node* naming format +//and so the function has to be updated if the CNI pod name format changes. +func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error){ + var CNIPods []string + var podList corev1.PodList + listOptions := client.ListOptions{ + Namespace: metav1.NamespaceSystem, + } + + err := d.k8sClient.List(ctx, &podList, &listOptions) + if err != nil { + return CNIPods, err + } + + for _,pod := range podList.Items { + if strings.HasPrefix(pod.Name, "aws-node") { + CNIPods = append(CNIPods, pod.Name) + } + } + + d.log.Debugf("Total aws-node pod count:- ", len(CNIPods)) + return CNIPods, nil +} diff --git a/go.mod b/go.mod index 451073a846..2d22d687a1 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/json-iterator/go v1.1.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect - github.com/operator-framework/operator-sdk v0.0.7 github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v0.8.0 @@ -44,9 +43,10 @@ require ( google.golang.org/grpc v1.29.0 gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 - k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 - k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d - k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 + k8s.io/api v0.18.6 + k8s.io/apimachinery v0.18.6 + k8s.io/client-go v0.18.6 k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28 - k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect + k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92 // indirect + sigs.k8s.io/controller-runtime v0.6.3 ) diff --git a/pkg/apis/crd/v1alpha1/register.go b/pkg/apis/crd/v1alpha1/register.go deleted file mode 100644 index 58556d818b..0000000000 --- a/pkg/apis/crd/v1alpha1/register.go +++ /dev/null @@ -1,37 +0,0 @@ -package v1alpha1 - -import ( - sdkK8sutil "github.com/operator-framework/operator-sdk/pkg/util/k8sutil" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -const ( - version = "v1alpha1" - groupName = "crd.k8s.amazonaws.com" -) - -var ( - // SchemeBuilder knows the ENIConfig and ENIConfigList types - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme add the SchemeBuilder - AddToScheme = SchemeBuilder.AddToScheme - // SchemeGroupVersion is the group version used to register these objects. - SchemeGroupVersion = schema.GroupVersion{Group: groupName, Version: version} -) - -func init() { - sdkK8sutil.AddToSDKScheme(AddToScheme) -} - -// addKnownTypes adds the set of types defined in this package to the supplied scheme. -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &ENIConfig{}, - &ENIConfigList{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go deleted file mode 100644 index 3dc64f745a..0000000000 --- a/pkg/apis/crd/v1alpha1/types.go +++ /dev/null @@ -1,35 +0,0 @@ -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ENIConfigList is the ENI config list -type ENIConfigList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata"` - Items []ENIConfig `json:"items"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ENIConfig is the per ENI config -type ENIConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata"` - Spec ENIConfigSpec `json:"spec"` - Status ENIConfigStatus `json:"status,omitempty"` -} - -// ENIConfigSpec is the spec for this ENI -type ENIConfigSpec struct { - SecurityGroups []string `json:"securityGroups"` - Subnet string `json:"subnet"` -} - -// ENIConfigStatus is empty -type ENIConfigStatus struct { - // Fill me -} diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index feb45b9835..1a2497a83f 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,22 @@ // +build !ignore_autogenerated -// Code generated by deepcopy-gen. DO NOT EDIT. +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. package v1alpha1 @@ -13,9 +29,8 @@ func (in *ENIConfig) DeepCopyInto(out *ENIConfig) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) + out.Spec = in.Spec out.Status = in.Status - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ENIConfig. @@ -40,7 +55,7 @@ func (in *ENIConfig) DeepCopyObject() runtime.Object { func (in *ENIConfigList) DeepCopyInto(out *ENIConfigList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ENIConfig, len(*in)) @@ -48,7 +63,6 @@ func (in *ENIConfigList) DeepCopyInto(out *ENIConfigList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ENIConfigList. @@ -72,12 +86,6 @@ func (in *ENIConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ENIConfigSpec) DeepCopyInto(out *ENIConfigSpec) { *out = *in - if in.SecurityGroups != nil { - in, out := &in.SecurityGroups, &out.SecurityGroups - *out = make([]string, len(*in)) - copy(*out, *in) - } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ENIConfigSpec. @@ -93,7 +101,6 @@ func (in *ENIConfigSpec) DeepCopy() *ENIConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ENIConfigStatus) DeepCopyInto(out *ENIConfigStatus) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ENIConfigStatus. diff --git a/pkg/eniconfig/eniconfig.go b/pkg/eniconfig/eniconfig.go index 13087049d3..1f71da8382 100644 --- a/pkg/eniconfig/eniconfig.go +++ b/pkg/eniconfig/eniconfig.go @@ -16,18 +16,14 @@ package eniconfig import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" "os" - "runtime" - "sync" - "time" - - "github.com/operator-framework/operator-sdk/pkg/sdk" - "github.com/pkg/errors" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - sdkVersion "github.com/operator-framework/operator-sdk/version" - corev1 "k8s.io/api/core/v1" + "github.com/pkg/errors" ) const ( @@ -48,8 +44,8 @@ const ( // ENIConfig interface type ENIConfig interface { - MyENIConfig() (*v1alpha1.ENIConfigSpec, error) - Getter() *ENIConfigInfo + MyENIConfig(client.Client) (*v1alpha1.ENIConfigSpec, error) + GetENIConfigName(context.Context, client.Client) (string, error) } // ErrNoENIConfig is the missing ENIConfig error @@ -57,16 +53,6 @@ var ErrNoENIConfig = errors.New("eniconfig: eniconfig is not available") var log = logger.Get() -// ENIConfigController defines global context for ENIConfig controller -type ENIConfigController struct { - eni map[string]*v1alpha1.ENIConfigSpec - myENI string - eniLock sync.RWMutex - myNodeName string - eniConfigAnnotationDef string - eniConfigLabelDef string -} - // ENIConfigInfo returns locally cached ENIConfigs type ENIConfigInfo struct { ENI map[string]v1alpha1.ENIConfigSpec @@ -75,127 +61,28 @@ type ENIConfigInfo struct { EniConfigLabelDef string } -// NewENIConfigController creates a new ENIConfig controller -func NewENIConfigController() *ENIConfigController { - return &ENIConfigController{ - myNodeName: os.Getenv("MY_NODE_NAME"), - eni: make(map[string]*v1alpha1.ENIConfigSpec), - myENI: eniConfigDefault, - eniConfigAnnotationDef: getEniConfigAnnotationDef(), - eniConfigLabelDef: getEniConfigLabelDef(), - } -} - -// NewHandler creates a new handler for sdk -func NewHandler(controller *ENIConfigController) sdk.Handler { - return &Handler{controller: controller} -} - -// Handler stores the ENIConfigController -type Handler struct { - controller *ENIConfigController -} - -// Handle handles ENIConfig updates from API Server and store them in local cache -func (h *Handler) Handle(ctx context.Context, event sdk.Event) error { - switch o := event.Object.(type) { - case *v1alpha1.ENIConfig: - eniConfigName := o.GetName() - if event.Deleted { - log.Debugf("Deleting ENIConfig: %s", eniConfigName) - h.controller.eniLock.Lock() - defer h.controller.eniLock.Unlock() - delete(h.controller.eni, eniConfigName) - return nil - } - - curENIConfig := o.DeepCopy() - - log.Debugf("Handle ENIConfig Add/Update: %s, %v, %s", eniConfigName, curENIConfig.Spec.SecurityGroups, curENIConfig.Spec.Subnet) - - h.controller.eniLock.Lock() - defer h.controller.eniLock.Unlock() - h.controller.eni[eniConfigName] = &curENIConfig.Spec - - case *corev1.Node: - log.Debugf("Handle corev1.Node: %s, %v, %v", o.GetName(), o.GetAnnotations(), o.GetLabels()) - // Get annotations if not found get labels if not found fallback use default - if h.controller.myNodeName == o.GetName() { - val, ok := o.GetAnnotations()[h.controller.eniConfigAnnotationDef] - if !ok { - val, ok = o.GetLabels()[h.controller.eniConfigLabelDef] - if !ok { - val = eniConfigDefault - } - } - // If value changes - if h.controller.myENI != val { - h.controller.eniLock.Lock() - defer h.controller.eniLock.Unlock() - h.controller.myENI = val - log.Debugf("Setting myENI to: %s", val) - if val != eniConfigDefault { - labels := o.GetLabels() - labels["vpc.amazonaws.com/eniConfig"] = val - o.SetLabels(labels) - } - } - } - } - return nil -} - -func printVersion() { - log.Infof("Go Version: %s", runtime.Version()) - log.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) - log.Infof("operator-sdk Version: %v", sdkVersion.Version) -} - -// Start kicks off ENIConfig controller -func (eniCfg *ENIConfigController) Start() { - printVersion() - - sdk.ExposeMetricsPort() - - resource := "crd.k8s.amazonaws.com/v1alpha1" - kind := "ENIConfig" - resyncPeriod := time.Second * 5 - log.Infof("Watching %s, %s, every %v s", resource, kind, resyncPeriod.Seconds()) - sdk.Watch(resource, kind, "", resyncPeriod) - sdk.Watch("/v1", "Node", corev1.NamespaceAll, resyncPeriod) - sdk.Handle(NewHandler(eniCfg)) - sdk.Run(context.TODO()) -} - -func (eniCfg *ENIConfigController) Getter() *ENIConfigInfo { - output := &ENIConfigInfo{ - ENI: make(map[string]v1alpha1.ENIConfigSpec), +// MyENIConfig returns the ENIConfig applicable to the particular node +func MyENIConfig(ctx context.Context, k8sClient client.Client) (*v1alpha1.ENIConfigSpec, error) { + eniConfigName, err := GetNodeSpecificENIConfigName(ctx, k8sClient) + if err != nil { + log.Debugf("Error while retrieving Node name") } - eniCfg.eniLock.Lock() - defer eniCfg.eniLock.Unlock() - output.MyENI = eniCfg.myENI - output.EniConfigAnnotationDef = getEniConfigAnnotationDef() - output.EniConfigLabelDef = getEniConfigLabelDef() + log.Infof("Found ENI Config Name: %s", eniConfigName) - for name, val := range eniCfg.eni { - output.ENI[name] = *val + eniConfigsList := v1alpha1.ENIConfigList{} + err = k8sClient.List(ctx, &eniConfigsList) + if err != nil { + fmt.Errorf("Error while EniConfig List Get: %s", err) } - return output -} - -// MyENIConfig returns the security -func (eniCfg *ENIConfigController) MyENIConfig() (*v1alpha1.ENIConfigSpec, error) { - eniCfg.eniLock.Lock() - defer eniCfg.eniLock.Unlock() - - myENIConfig, ok := eniCfg.eni[eniCfg.myENI] - - if ok { - return &v1alpha1.ENIConfigSpec{ - SecurityGroups: myENIConfig.SecurityGroups, - Subnet: myENIConfig.Subnet, - }, nil + for _, eni := range eniConfigsList.Items { + if eniConfigName == eni.Name { + log.Debugf("Matching ENIConfig found: %s - %s - %s ", eni.Name, eni.Spec.Subnet, eni.Spec.SecurityGroups) + return &v1alpha1.ENIConfigSpec{ + SecurityGroups: eni.Spec.SecurityGroups, + Subnet: eni.Spec.Subnet, + }, nil + } } return nil, ErrNoENIConfig } @@ -227,3 +114,32 @@ func getEniConfigLabelDef() string { } return defaultEniConfigLabelDef } + +func GetNodeSpecificENIConfigName(ctx context.Context, k8sClient client.Client) (string, error) { + var eniConfigName string + nodeList := corev1.NodeList{} + err := k8sClient.List(ctx, &nodeList) + if err != nil { + fmt.Errorf("Error while Node List Get: %s", err) + } + log.Debugf("Node Count: ", len(nodeList.Items)) + for _, node := range nodeList.Items { + if node.Name == os.Getenv("MY_NODE_NAME") { + log.Debugf("Node Info: %s", node.Name) + val, ok := node.GetAnnotations()[getEniConfigAnnotationDef()] + if !ok { + val, ok = node.GetLabels()[getEniConfigLabelDef()] + if !ok { + val = eniConfigDefault + } + } + eniConfigName = val + if val != eniConfigDefault { + labels := node.GetLabels() + labels["vpc.amazonaws.com/eniConfig"] = eniConfigName + node.SetLabels(labels) + } + } + } + return eniConfigName, nil +} diff --git a/pkg/eniconfig/eniconfig_test.go b/pkg/eniconfig/eniconfig_test.go index 6ed50ae8ea..ab2ec1b27d 100644 --- a/pkg/eniconfig/eniconfig_test.go +++ b/pkg/eniconfig/eniconfig_test.go @@ -14,205 +14,215 @@ package eniconfig import ( "context" - "fmt" + //"fmt" "os" "testing" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" - "github.com/operator-framework/operator-sdk/pkg/sdk" "github.com/stretchr/testify/assert" + eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func updateENIConfig(hdlr sdk.Handler, name string, eniConfig v1alpha1.ENIConfigSpec, toDelete bool) { - event := sdk.Event{ - Object: &v1alpha1.ENIConfig{ - TypeMeta: metav1.TypeMeta{APIVersion: v1alpha1.SchemeGroupVersion.String()}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: eniConfig}, - Deleted: toDelete, - } - - _ = hdlr.Handle(context.TODO(), event) -} - -func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string, toDelete bool) { - node := corev1.Node{ - TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()}, +func TestMyENIConfig(t *testing.T) { + testNode := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Labels: make(map[string]string), + Name: "test-node", }, } - accessor, err := meta.Accessor(&node) - - if err != nil { - fmt.Printf("Failed to call meta.Access %v", err) - } - event := sdk.Event{ - Object: &node, - Deleted: toDelete, + testENIConfigAZ1 := &v1alpha1.ENIConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "az1", + }, + Spec: v1alpha1.ENIConfigSpec{ + SecurityGroups: []string{"SG1"}, + Subnet: "SB1", + }, } - eniAnnotations := make(map[string]string) - eniConfigAnnotationDef := getEniConfigAnnotationDef() - if !toDelete { - eniAnnotations[eniConfigAnnotationDef] = configName - } - accessor.SetAnnotations(eniAnnotations) - _ = hdlr.Handle(context.TODO(), event) -} - -func updateNodeLabel(hdlr sdk.Handler, nodeName string, configName string, toDelete bool) { - - node := corev1.Node{ - TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()}, + testENIConfigAZ2 := &v1alpha1.ENIConfig{ ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, + Name: "az2", + }, + Spec: v1alpha1.ENIConfigSpec{ + SecurityGroups: []string{"SG2"}, + Subnet: "SB2", }, } - accessor, err := meta.Accessor(&node) - if err != nil { - fmt.Printf("Failed to call meta.Access %v", err) + type env struct { + nodes []*corev1.Node + eniconfigs []*v1alpha1.ENIConfig + Labels map[string]string + Annotations map[string]string + eniConfigLabelKey string + eniConfigAnnotationKey string } - event := sdk.Event{ - Object: &node, - Deleted: toDelete, + tests := []struct { + name string + env env + want *v1alpha1.ENIConfigSpec + wantErr error + }{ + { + name: "Matching ENIConfig available - Using Default Labels", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, + Labels: map[string]string{ + "k8s.amazonaws.com/eniConfig": "az1", + }, + }, + want: &testENIConfigAZ1.Spec, + wantErr: nil, + }, + { + name: "No Matching ENIConfig available - Using Default Labels", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ2}, + Labels: map[string]string{ + "k8s.amazonaws.com/eniConfig": "az1", + }, + }, + want: nil, + wantErr: errors.New("eniconfig: eniconfig is not available"), + }, + { + name: "Matching ENIConfig available - Using Custom Label Key exposed via ENI_CONFIG_LABEL_DEF", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "az1", + }, + eniConfigLabelKey: "failure-domain.beta.kubernetes.io/zone", + }, + want: &testENIConfigAZ1.Spec, + wantErr: nil, + }, + { + name: "No Matching ENIConfig available - Using Custom Label Key exposed via ENI_CONFIG_LABEL_DEF", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "az2", + }, + eniConfigLabelKey: "failure-domain.beta.kubernetes.io/zone", + }, + want: nil, + wantErr: errors.New("eniconfig: eniconfig is not available"), + }, + { + name: "Matching ENIConfig available - Using Default Annotation", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "az2", + }, + Annotations: map[string]string{ + "k8s.amazonaws.com/eniConfig": "az1", + }, + }, + want: &testENIConfigAZ1.Spec, + wantErr: nil, + }, + { + name: "No Matching ENIConfig available - Using Default Annotation", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ2}, + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "az2", + }, + Annotations: map[string]string{ + "k8s.amazonaws.com/eniConfig": "az1", + }, + }, + want: nil, + wantErr: errors.New("eniconfig: eniconfig is not available"), + }, + { + name: "Matching ENIConfig available - Using Custom Annotation Key exposed via ENI_CONFIG_ANNOTATION_DEF", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "az2", + }, + Annotations: map[string]string{ + "k8s.amazonaws.com/myENIConfig": "az1", + }, + eniConfigAnnotationKey: "k8s.amazonaws.com/myENIConfig", + }, + want: &testENIConfigAZ1.Spec, + wantErr: nil, + }, + { + name: "No Matching ENIConfig available - Using Custom Label Key exposed via ENI_CONFIG_ANNOTATION_DEF", + env: env{ + nodes: []*corev1.Node{testNode}, + eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ2}, + Labels: map[string]string{ + "failure-domain.beta.kubernetes.io/zone": "az2", + }, + Annotations: map[string]string{ + "k8s.amazonaws.com/myENIConfig": "az1", + }, + eniConfigAnnotationKey: "k8s.amazonaws.com/myENIConfig", + }, + want: nil, + wantErr: errors.New("eniconfig: eniconfig is not available"), + }, } - eniLabels := make(map[string]string) - eniConfigLabelDef := getEniConfigLabelDef() - if !toDelete { - eniLabels[eniConfigLabelDef] = configName + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + k8sSchema := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sSchema) + eniconfigscheme.AddToScheme(k8sSchema) + k8sClient := testclient.NewFakeClientWithScheme(k8sSchema) + + for _, node := range tt.env.nodes { + _ = os.Setenv("MY_NODE_NAME", node.Name) + node.Labels = tt.env.Labels + node.Annotations = tt.env.Annotations + if tt.env.eniConfigAnnotationKey != "" { + _ = os.Setenv(envEniConfigAnnotationDef, tt.env.eniConfigAnnotationKey) + } + if tt.env.eniConfigLabelKey != "" { + _ = os.Setenv(envEniConfigLabelDef, tt.env.eniConfigLabelKey) + } + err := k8sClient.Create(ctx, node.DeepCopy()) + assert.NoError(t, err) + } + + for _, eniconfig := range tt.env.eniconfigs { + err := k8sClient.Create(ctx, eniconfig.DeepCopy()) + assert.NoError(t, err) + } + + myENIConfig, err := MyENIConfig(ctx, k8sClient) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, myENIConfig, tt.want) + } + }) } - accessor.SetLabels(eniLabels) - _ = hdlr.Handle(context.TODO(), event) -} - -func TestENIConfig(t *testing.T) { - testENIConfigController := NewENIConfigController() - - testHandler := NewHandler(testENIConfigController) - - // If there is no default ENI config - _, err := testENIConfigController.MyENIConfig() - assert.Error(t, err) - - // Start with default config - defaultSGs := []string{"sg1-id", "sg2-id"} - defaultSubnet := "subnet1" - defaultCfg := v1alpha1.ENIConfigSpec{ - SecurityGroups: defaultSGs, - Subnet: defaultSubnet} - - updateENIConfig(testHandler, eniConfigDefault, defaultCfg, false) - - outputCfg, err := testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, defaultCfg, *outputCfg) - - // Add one more ENI config, but it should NOT impact default - group1Cfg := v1alpha1.ENIConfigSpec{ - SecurityGroups: []string{"sg11-id", "sg12-id"}, - Subnet: "subnet11"} - group1Name := "group1ENIconfig" - updateENIConfig(testHandler, group1Name, group1Cfg, false) - - outputCfg, err = testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, defaultCfg, *outputCfg) - -} - -func TestNodeENIConfig(t *testing.T) { - myNodeName := "testMyNodeWithAnnotation" - myENIConfig := "testMyENIConfig" - _ = os.Setenv("MY_NODE_NAME", myNodeName) - testENIConfigController := NewENIConfigController() - - testHandler := NewHandler(testENIConfigController) - updateNodeAnnotation(testHandler, myNodeName, myENIConfig, false) - - // If there is no ENI config - _, err := testENIConfigController.MyENIConfig() - assert.Error(t, err) - - // Add eniconfig for myENIConfig - group1Cfg := v1alpha1.ENIConfigSpec{ - SecurityGroups: []string{"sg21-id", "sg22-id"}, - Subnet: "subnet21"} - updateENIConfig(testHandler, myENIConfig, group1Cfg, false) - outputCfg, err := testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, group1Cfg, *outputCfg) - - // Add default config - defaultSGs := []string{"sg1-id", "sg2-id"} - defaultSubnet := "subnet1" - defaultCfg := v1alpha1.ENIConfigSpec{ - SecurityGroups: defaultSGs, - Subnet: defaultSubnet} - updateENIConfig(testHandler, eniConfigDefault, defaultCfg, false) - outputCfg, err = testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, group1Cfg, *outputCfg) - - // Delete node's myENIConfig annotation, then the value should fallback to default - updateNodeAnnotation(testHandler, myNodeName, myENIConfig, true) - outputCfg, err = testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, defaultCfg, *outputCfg) - -} - -func TestNodeENIConfigLabel(t *testing.T) { - myNodeName := "testMyNodeWithLabel" - myENIConfig := "testMyENIConfig" - _ = os.Setenv("MY_NODE_NAME", myNodeName) - testENIConfigController := NewENIConfigController() - - testHandler := NewHandler(testENIConfigController) - updateNodeLabel(testHandler, myNodeName, myENIConfig, false) - - // If there is no ENI config - _, err := testENIConfigController.MyENIConfig() - assert.Error(t, err) - - // Add eniconfig for myENIConfig - group1Cfg := v1alpha1.ENIConfigSpec{ - SecurityGroups: []string{"sg21-id", "sg22-id"}, - Subnet: "subnet21"} - updateENIConfig(testHandler, myENIConfig, group1Cfg, false) - outputCfg, err := testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, group1Cfg, *outputCfg) - - // Add default config - defaultSGs := []string{"sg1-id", "sg2-id"} - defaultSubnet := "subnet1" - defaultCfg := v1alpha1.ENIConfigSpec{ - SecurityGroups: defaultSGs, - Subnet: defaultSubnet} - updateENIConfig(testHandler, eniConfigDefault, defaultCfg, false) - outputCfg, err = testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, group1Cfg, *outputCfg) - - // Delete node's myENIConfig annotation, then the value should fallback to default - updateNodeLabel(testHandler, myNodeName, myENIConfig, true) - outputCfg, err = testENIConfigController.MyENIConfig() - assert.NoError(t, err) - assert.Equal(t, defaultCfg, *outputCfg) - } func TestGetEniConfigAnnotationDefDefault(t *testing.T) { diff --git a/pkg/ipamd/introspect.go b/pkg/ipamd/introspect.go index eb71b1a381..b0762d11f0 100644 --- a/pkg/ipamd/introspect.go +++ b/pkg/ipamd/introspect.go @@ -15,6 +15,7 @@ package ipamd import ( "encoding/json" + "golang.org/x/net/context" "net" "net/http" "os" @@ -22,6 +23,7 @@ import ( "strings" "time" + "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/retry" ) @@ -142,7 +144,9 @@ func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - responseJSON, err := json.Marshal(ipam.eniConfig.Getter()) + ctx := context.Background() + myENIConfig, _ := eniconfig.GetNodeSpecificENIConfigName(ctx, ipam.k8sClient) + responseJSON, err := json.Marshal(myENIConfig) if err != nil { log.Errorf("Failed to marshal ENI config: %v", err) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 9237ceee15..df6fc19d74 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -14,17 +14,18 @@ package ipamd import ( + "context" "fmt" "net" "os" + "sigs.k8s.io/controller-runtime/pkg/client" "strconv" "strings" "sync" "sync/atomic" "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" @@ -35,9 +36,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" ) // The package ipamd is a long running daemon which manages a warm pool of available IP addresses. @@ -120,6 +121,9 @@ const ( // envEnablePodENI is used to attach a Trunk ENI to every node. Required in order to give Branch ENIs to pods. envEnablePodENI = "ENABLE_POD_ENI" + // envNodeName will be used to store Node name + envNodeName = "MY_NODE_NAME" + // vpcENIConfigLabel is used by the VPC resource controller to pick the right ENI config. vpcENIConfigLabel = "vpc.amazonaws.com/eniConfig" ) @@ -187,9 +191,9 @@ var ( type IPAMContext struct { awsClient awsutils.APIs dataStore *datastore.DataStore - k8sClient kubernetes.Interface + standalonek8sClient client.Client + k8sClient client.Client useCustomNetworking bool - eniConfig eniconfig.ENIConfig networkClient networkutils.NetworkAPIs maxIPsPerENI int maxENI int @@ -280,13 +284,12 @@ func prometheusRegister() { // New retrieves IP address usage information from Instance MetaData service and Kubelet // then initializes IP address pool data store -func New(k8sapiClient kubernetes.Interface, eniConfig *eniconfig.ENIConfigController) (*IPAMContext, error) { - prometheusRegister() +func New(k8sapiClient client.Client, k8sClient client.Client) (*IPAMContext, error) { prometheusRegister() c := &IPAMContext{} - c.k8sClient = k8sapiClient + c.standalonek8sClient = k8sapiClient + c.k8sClient = k8sClient c.networkClient = networkutils.New() - c.eniConfig = eniConfig c.useCustomNetworking = UseCustomNetworkCfg() client, err := awsutils.New(c.useCustomNetworking) @@ -329,6 +332,7 @@ func (c *IPAMContext) nodeInit() error { ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1)) var err error + ctx := context.Background() log.Debugf("Start node init") @@ -407,9 +411,10 @@ func (c *IPAMContext) nodeInit() error { vpcCIDRs = c.updateCIDRsRulesOnChange(vpcCIDRs) }, 30*time.Second) - if c.useCustomNetworking && c.eniConfig.Getter().MyENI != "default" { + eniConfigName,_ := eniconfig.GetNodeSpecificENIConfigName(ctx, c.k8sClient) + if c.useCustomNetworking && eniConfigName != "default" { // Signal to VPC Resource Controller that the node is using custom networking - err := c.SetNodeLabel(vpcENIConfigLabel, c.eniConfig.Getter().MyENI) + err := c.SetNodeLabel(ctx, vpcENIConfigLabel, eniConfigName) if err != nil { log.Errorf("Failed to set eniConfig node label", err) podENIErrInc("nodeInit") @@ -417,7 +422,7 @@ func (c *IPAMContext) nodeInit() error { } } else { // Remove the custom networking label - err := c.SetNodeLabel(vpcENIConfigLabel, "") + err := c.SetNodeLabel(ctx, vpcENIConfigLabel, "") if err != nil { log.Errorf("Failed to delete eniConfig node label", err) podENIErrInc("nodeInit") @@ -428,7 +433,7 @@ func (c *IPAMContext) nodeInit() error { // If we started on a node with a trunk ENI already attached, add the node label. if metadataResult.TrunkENI != "" { // Signal to VPC Resource Controller that the node has a trunk already - err := c.SetNodeLabel("vpc.amazonaws.com/has-trunk-attached", "true") + err := c.SetNodeLabel(ctx, "vpc.amazonaws.com/has-trunk-attached", "true") if err != nil { log.Errorf("Failed to set node label", err) podENIErrInc("nodeInit") @@ -437,7 +442,7 @@ func (c *IPAMContext) nodeInit() error { } } else { // Check if we want to ask for one - c.askForTrunkENIIfNeeded() + c.askForTrunkENIIfNeeded(ctx) } // For a new node, attach IPs @@ -495,20 +500,21 @@ func (c *IPAMContext) updateIPStats(unmanaged int) { // StartNodeIPPoolManager monitors the IP pool, add or del them when it is required. func (c *IPAMContext) StartNodeIPPoolManager() { sleepDuration := ipPoolMonitorInterval / 2 + ctx := context.Background() for { if !c.disableENIProvisioning { time.Sleep(sleepDuration) - c.updateIPPoolIfRequired() + c.updateIPPoolIfRequired(ctx) } time.Sleep(sleepDuration) - c.nodeIPPoolReconcile(nodeIPPoolReconcileInterval) + c.nodeIPPoolReconcile(ctx, nodeIPPoolReconcileInterval) } } -func (c *IPAMContext) updateIPPoolIfRequired() { - c.askForTrunkENIIfNeeded() +func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) { + c.askForTrunkENIIfNeeded(ctx) if c.nodeIPPoolTooLow() { - c.increaseIPPool() + c.increaseIPPool(ctx) } else if c.nodeIPPoolTooHigh() { c.decreaseIPPool(decreaseIPPoolInterval) } @@ -621,7 +627,7 @@ func (c *IPAMContext) findFreeableIPs(eni string) ([]string, error) { return freeableIPs, nil } -func (c *IPAMContext) increaseIPPool() { +func (c *IPAMContext) increaseIPPool(ctx context.Context) { log.Debug("Starting to increase IP pool size") ipamdActionsInprogress.WithLabelValues("increaseIPPool").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("increaseIPPool").Sub(float64(1)) @@ -652,7 +658,7 @@ func (c *IPAMContext) increaseIPPool() { } // If we did not add an IP, try to add an ENI instead. if c.dataStore.GetENIs() < (c.maxENI - c.unmanagedENI - reserveSlotForTrunkENI) { - if err = c.tryAllocateENI(); err == nil { + if err = c.tryAllocateENI(ctx); err == nil { c.updateLastNodeIPPoolAction() } } else { @@ -669,12 +675,12 @@ func (c *IPAMContext) updateLastNodeIPPoolAction() { logPoolStats(total, used, c.maxIPsPerENI) } -func (c *IPAMContext) tryAllocateENI() error { +func (c *IPAMContext) tryAllocateENI(ctx context.Context) error { var securityGroups []*string var subnet string if c.useCustomNetworking { - eniCfg, err := c.eniConfig.MyENIConfig() + eniCfg, err := eniconfig.MyENIConfig(ctx, c.k8sClient) if err != nil { log.Errorf("Failed to get pod ENI config") @@ -854,7 +860,7 @@ func logPoolStats(total, used, maxAddrsPerENI int) { log.Debugf("IP pool stats: total = %d, used = %d, c.maxIPsPerENI = %d", total, used, maxAddrsPerENI) } -func (c *IPAMContext) askForTrunkENIIfNeeded() { +func (c *IPAMContext) askForTrunkENIIfNeeded(ctx context.Context) { if c.enablePodENI && c.dataStore.GetTrunkENI() == "" { // Check that there is room for a trunk ENI to be attached: if c.dataStore.GetENIs() >= (c.maxENI - c.unmanagedENI) { @@ -862,7 +868,7 @@ func (c *IPAMContext) askForTrunkENIIfNeeded() { return } // We need to signal that VPC Resource Controller needs to attach a trunk ENI - err := c.SetNodeLabel("vpc.amazonaws.com/has-trunk-attached", "false") + err := c.SetNodeLabel(ctx, "vpc.amazonaws.com/has-trunk-attached", "false") if err != nil { podENIErrInc("askForTrunkENIIfNeeded") log.Errorf("Failed to set node label", err) @@ -927,7 +933,7 @@ func podENIErrInc(fn string) { } // nodeIPPoolReconcile reconcile ENI and IP info from metadata service and IP addresses in datastore -func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { +func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Duration) { curTime := time.Now() timeSinceLast := curTime.Sub(c.lastNodeIPPoolAction) if timeSinceLast <= interval { @@ -974,7 +980,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { if c.enablePodENI && metadataResult.TrunkENI != "" { // Label the node that we have a trunk - err = c.SetNodeLabel("vpc.amazonaws.com/has-trunk-attached", "true") + err = c.SetNodeLabel(ctx,"vpc.amazonaws.com/has-trunk-attached", "true") if err != nil { podENIErrInc("askForTrunkENIIfNeeded") log.Errorf("Failed to set node label for trunk. Aborting reconcile", err) @@ -1307,9 +1313,12 @@ func (c *IPAMContext) getTrunkLinkIndex() (int, error) { } // SetNodeLabel sets or deletes a node label -func (c *IPAMContext) SetNodeLabel(key, value string) error { +func (c *IPAMContext) SetNodeLabel(ctx context.Context, key, value string) error { + var node corev1.Node // Find my node - node, err := c.k8sClient.CoreV1().Nodes().Get(c.myNodeName, metav1.GetOptions{}) + err := c.k8sClient.Get(ctx, types.NamespacedName{Name: c.myNodeName}, &node) + log.Debugf("Node found %q - labels - %q", node.Name, len(node.Labels)) + if err != nil { log.Errorf("Failed to get node: %v", err) return err @@ -1319,6 +1328,7 @@ func (c *IPAMContext) SetNodeLabel(key, value string) error { log.Debugf("Node label %q is already %q", key, labelValue) return nil } + // Make deep copy for modification updateNode := node.DeepCopy() @@ -1332,15 +1342,27 @@ func (c *IPAMContext) SetNodeLabel(key, value string) error { } // Update node status to advertise the resource. - _, err = c.k8sClient.CoreV1().Nodes().Update(updateNode) + err = c.k8sClient.Update(ctx, updateNode) if err != nil { log.Errorf("Failed to update node %s with label %q: %q, error: %v", c.myNodeName, key, value, err) } - log.Infof("Updated node %s with label %q: %q", c.myNodeName, key, value) + log.Debugf("Updated node %s with label %q: %q", c.myNodeName, key, value) + return nil } // GetPod returns the pod matching the name and namespace -func (c *IPAMContext) GetPod(podName, namespace string) (*v1.Pod, error) { - return c.k8sClient.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{}) +func (c *IPAMContext) GetPod(podName, namespace string) (*corev1.Pod, error) { + ctx := context.Background() + var pod corev1.Pod + + podKey := types.NamespacedName{ + Namespace: namespace, + Name: podName, + } + err := c.standalonek8sClient.Get(ctx, podKey ,&pod) + if err != nil { + return nil, fmt.Errorf("Error while trying to retrieve Pod Info: %s", err) + } + return &pod, nil } \ No newline at end of file diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index e3cf7daf24..9a7579c3f1 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -16,6 +16,10 @@ package ipamd import ( "errors" "fmt" + "golang.org/x/net/context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "net" "os" "reflect" @@ -25,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" mock_awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks" mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks" @@ -35,7 +40,9 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" - k8s_fake "k8s.io/client-go/kubernetes/fake" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) const ( @@ -57,19 +64,25 @@ const ( ) type testMocks struct { - ctrl *gomock.Controller - awsutils *mock_awsutils.MockAPIs - clientset *k8s_fake.Clientset - network *mock_networkutils.MockNetworkAPIs - eniconfig *mock_eniconfig.MockENIConfig + ctrl *gomock.Controller + awsutils *mock_awsutils.MockAPIs + standaloneClient client.Client + k8sClient client.Client + network *mock_networkutils.MockNetworkAPIs + eniconfig *mock_eniconfig.MockENIConfig } func setup(t *testing.T) *testMocks { ctrl := gomock.NewController(t) + k8sSchema := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sSchema) + eniconfigscheme.AddToScheme(k8sSchema) + return &testMocks{ ctrl: ctrl, awsutils: mock_awsutils.NewMockAPIs(ctrl), - clientset: k8s_fake.NewSimpleClientset(), + standaloneClient: testclient.NewFakeClientWithScheme(k8sSchema), + k8sClient: testclient.NewFakeClientWithScheme(k8sSchema), network: mock_networkutils.NewMockNetworkAPIs(ctrl), eniconfig: mock_eniconfig.NewMockENIConfig(ctrl), } @@ -78,6 +91,7 @@ func setup(t *testing.T) *testMocks { func TestNodeInit(t *testing.T) { m := setup(t) defer m.ctrl.Finish() + ctx := context.Background() fakeCheckpoint := datastore.CheckpointData{ Version: datastore.CheckpointFormatVersion, @@ -87,17 +101,18 @@ func TestNodeInit(t *testing.T) { } mockContext := &IPAMContext{ - awsClient: m.awsutils, - k8sClient: m.clientset, - maxIPsPerENI: 14, - maxENI: 4, - warmENITarget: 1, - warmIPTarget: 3, - primaryIP: make(map[string]string), - terminating: int32(0), - networkClient: m.network, - dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint)), - myNodeName: myNodeName, + awsClient: m.awsutils, + standalonek8sClient: m.standaloneClient, + k8sClient: m.k8sClient, + maxIPsPerENI: 14, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint)), + myNodeName: myNodeName, } mockContext.dataStore.CheckpointMigrationPhase = 2 @@ -146,7 +161,8 @@ func TestNodeInit(t *testing.T) { Spec: v1.NodeSpec{}, Status: v1.NodeStatus{}, } - _, _ = m.clientset.CoreV1().Nodes().Create(&fakeNode) + //_, _ = m.clientset.CoreV1().Nodes().Create(&fakeNode) + _ = m.k8sClient.Create(ctx, &fakeNode) // Add IPs m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any()) @@ -201,21 +217,24 @@ func TestIncreaseIPPoolDefault(t *testing.T) { func TestIncreaseIPPoolCustomENI(t *testing.T) { _ = os.Setenv(envCustomNetworkCfg, "true") + _ = os.Setenv("MY_NODE_NAME", myNodeName) testIncreaseIPPool(t, true) } func testIncreaseIPPool(t *testing.T, useENIConfig bool) { m := setup(t) defer m.ctrl.Finish() + ctx := context.Background() mockContext := &IPAMContext{ awsClient: m.awsutils, + standalonek8sClient: m.standaloneClient, + k8sClient: m.k8sClient, maxIPsPerENI: 14, maxENI: 4, warmENITarget: 1, networkClient: m.network, useCustomNetworking: UseCustomNetworkCfg(), - eniConfig: m.eniconfig, primaryIP: make(map[string]string), terminating: int32(0), } @@ -241,7 +260,6 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { } if useENIConfig { - m.eniconfig.EXPECT().MyENIConfig().Return(podENIConfig, nil) m.awsutils.EXPECT().AllocENI(true, sg, podENIConfig.Subnet).Return(eni2, nil) } else { m.awsutils.EXPECT().AllocENI(false, nil, "").Return(eni2, nil) @@ -283,13 +301,42 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) m.awsutils.EXPECT().AllocIPAddresses(eni2, 14) - mockContext.increaseIPPool() + if mockContext.useCustomNetworking { + mockContext.myNodeName = myNodeName + + labels := map[string]string { + "k8s.amazonaws.com/eniConfig": "az1", + } + //Create a Fake Node + fakeNode := v1.Node{ + TypeMeta: metav1.TypeMeta{Kind: "Node"}, + ObjectMeta: metav1.ObjectMeta{Name: myNodeName, Labels: labels}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{}, + } + _ = m.k8sClient.Create(ctx, &fakeNode) + + //Create a dummy ENIConfig + fakeENIConfig := v1alpha1.ENIConfig{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "az1"}, + Spec: eniconfigscheme.ENIConfigSpec{ + Subnet: "subnet1", + SecurityGroups: []string{"sg1-id", "sg2-id"}, + }, + Status: eniconfigscheme.ENIConfigStatus{}, + } + _=m.k8sClient.Create(ctx, &fakeENIConfig) + } + + mockContext.increaseIPPool(ctx) } func TestTryAddIPToENI(t *testing.T) { _ = os.Unsetenv(envCustomNetworkCfg) m := setup(t) defer m.ctrl.Finish() + ctx := context.Background() primary := true notPrimary := false @@ -306,7 +353,6 @@ func TestTryAddIPToENI(t *testing.T) { warmENITarget: 1, warmIPTarget: warmIPTarget, networkClient: m.network, - eniConfig: m.eniconfig, primaryIP: make(map[string]string), terminating: int32(0), } @@ -349,12 +395,13 @@ func TestTryAddIPToENI(t *testing.T) { m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - mockContext.increaseIPPool() + mockContext.increaseIPPool(ctx) } func TestNodeIPPoolReconcile(t *testing.T) { m := setup(t) defer m.ctrl.Finish() + ctx := context.Background() mockContext := &IPAMContext{ awsClient: m.awsutils, @@ -384,7 +431,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp.MultiCardENIIDs).AnyTimes() - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx,0) curENIs := mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) @@ -407,7 +454,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil) - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) assert.Equal(t, 0, curENIs.TotalIPs) @@ -432,7 +479,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet) m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp2.MultiCardENIIDs).AnyTimes() - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) // Verify that we now have 2 ENIs, primary ENI with 0 secondary IPs, and secondary ENI with 1 secondary IP curENIs = mockContext.dataStore.GetENIInfos() @@ -442,7 +489,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { // Remove the secondary ENI in the IMDS metadata m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) assert.Equal(t, 0, curENIs.TotalIPs) @@ -539,7 +586,6 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { awsClient: m.awsutils, dataStore: tt.fields.datastore, useCustomNetworking: false, - eniConfig: m.eniconfig, networkClient: m.network, maxIPsPerENI: tt.fields.maxIPsPerENI, maxENI: -1, @@ -675,6 +721,7 @@ func TestPodENIConfigFlag(t *testing.T) { func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { m := setup(t) defer m.ctrl.Finish() + ctx := context.Background() mockContext := &IPAMContext{ awsClient: m.awsutils, @@ -701,7 +748,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { m.awsutils.EXPECT().IsCNIUnmanagedENI(eniID).Return(false).AnyTimes() // First reconcile, IMDS returns correct IPs so no change needed - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) // IMDS returns no secondary IPs, the EC2 call fails primary := true @@ -721,7 +768,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { // eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed")) - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) assert.Equal(t, 2, curENIs.TotalIPs) @@ -743,14 +790,14 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { // eniIPPoolReconcile() calls EC2 to get the actual count that should still be 2 m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Addresses, nil) - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) assert.Equal(t, 2, curENIs.TotalIPs) // If no ENI is found, we abort the reconcile m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil) - mockContext.nodeIPPoolReconcile(0) + mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) assert.Equal(t, 2, curENIs.TotalIPs) @@ -854,45 +901,56 @@ func TestIPAMContext_setupENI(t *testing.T) { func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { m := setup(t) defer m.ctrl.Finish() + ctx := context.Background() mockContext := &IPAMContext{ - k8sClient: m.clientset, - dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion})), - awsClient: m.awsutils, - networkClient: m.network, - primaryIP: make(map[string]string), - terminating: int32(0), - maxENI: 1, - myNodeName: myNodeName, + standalonek8sClient: m.standaloneClient, + k8sClient: m.k8sClient, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion})), + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + maxENI: 1, + myNodeName: myNodeName, } - labels := make(map[string]string) + labels := map[string]string { + "testKey": "testValue", + } fakeNode := v1.Node{ TypeMeta: metav1.TypeMeta{Kind: "Node"}, ObjectMeta: metav1.ObjectMeta{Name: myNodeName, Labels: labels}, Spec: v1.NodeSpec{}, Status: v1.NodeStatus{}, } - _, _ = m.clientset.CoreV1().Nodes().Create(&fakeNode) + _ = m.k8sClient.Create(ctx, &fakeNode) _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false) // If ENABLE_POD_ENI is not set, nothing happens - mockContext.askForTrunkENIIfNeeded() + mockContext.askForTrunkENIIfNeeded(ctx) mockContext.enablePodENI = true // Enabled, we should try to set the label if there is room - mockContext.askForTrunkENIIfNeeded() - notUpdatedNode, err := m.clientset.CoreV1().Nodes().Get(myNodeName, metav1.GetOptions{}) + mockContext.askForTrunkENIIfNeeded(ctx) + var notUpdatedNode corev1.Node + var updatedNode corev1.Node + NodeKey := types.NamespacedName{ + Namespace: "", + Name: myNodeName, + } + err := m.k8sClient.Get(ctx, NodeKey, ¬UpdatedNode) // Since there was no room, no label should be added assert.NoError(t, err) - assert.Equal(t, 0, len(notUpdatedNode.Labels)) + assert.Equal(t, 1, len(notUpdatedNode.Labels)) mockContext.maxENI = 4 // Now there is room! - mockContext.askForTrunkENIIfNeeded() + mockContext.askForTrunkENIIfNeeded(ctx) // Fetch the updated node and verify that the label is set - updatedNode, err := m.clientset.CoreV1().Nodes().Get(myNodeName, metav1.GetOptions{}) + //updatedNode, err := m.clientset.CoreV1().Nodes().Get(myNodeName, metav1.GetOptions{}) + err = m.k8sClient.Get(ctx, NodeKey, &updatedNode) assert.NoError(t, err) assert.Equal(t, "false", updatedNode.Labels["vpc.amazonaws.com/has-trunk-attached"]) } diff --git a/pkg/k8sapi/discovery.go b/pkg/k8sapi/discovery.go deleted file mode 100644 index 130784c510..0000000000 --- a/pkg/k8sapi/discovery.go +++ /dev/null @@ -1,340 +0,0 @@ -// Package k8sapi contains logic to retrieve pods running on local node -package k8sapi - -import ( - "fmt" - "os" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - - "github.com/operator-framework/operator-sdk/pkg/k8sclient" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type controller struct { - indexer cache.Indexer - queue workqueue.RateLimitingInterface - informer cache.Controller -} - -const ( - cniPodName = "aws-node" -) - -// K8SAPIs defines interface to use kubelet introspection API -type K8SAPIs interface { - SetNodeLabel(key, value string) error - GetPod(podName, namespace string) (*v1.Pod, error) -} - -// K8SPodInfo provides pod info -type K8SPodInfo struct { - // Name is pod's name - Name string - // Namespace is pod's namespace - Namespace string - // Sandbox is pod's sandbox id - Sandbox string - // IP is pod's ipv4 address - IP string - UID string -} - -var log = logger.Get() - -// Controller defines global context for discovery controller -type Controller struct { - workerPods map[string]*K8SPodInfo - workerPodsLock sync.RWMutex - - cniPods map[string]string - cniPodsLock sync.RWMutex - - controller *controller - kubeClient kubernetes.Interface - myNodeName string - synced bool -} - -// NewController creates a new DiscoveryController -func NewController(clientset kubernetes.Interface) *Controller { - return &Controller{kubeClient: clientset, - myNodeName: os.Getenv("MY_NODE_NAME"), - cniPods: make(map[string]string), - workerPods: make(map[string]*K8SPodInfo)} -} - -// CreateKubeClient creates a k8s client -func CreateKubeClient() (kubernetes.Interface, error) { - kubeClient := k8sclient.GetKubeClient() - // Informers don't seem to do a good job logging error messages when it - // can't reach the server, making debugging hard. This makes it easier to - // figure out if apiserver is configured incorrectly. - log.Infof("Testing communication with server") - v, err := kubeClient.Discovery().ServerVersion() - if err != nil { - return nil, fmt.Errorf("error communicating with apiserver: %v", err) - } - log.Infof("Successful communication with the Cluster! Cluster Version is: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s", - v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform) - - return kubeClient, nil -} - -// GetCNIPods return the list of CNI pod names -func (d *Controller) GetCNIPods() []string { - var cniPods []string - - log.Info("GetCNIPods start...") - - d.cniPodsLock.Lock() - defer d.cniPodsLock.Unlock() - - for k := range d.cniPods { - cniPods = append(cniPods, k) - } - - log.Infof("GetCNIPods discovered %v", cniPods) - return cniPods -} - -// DiscoverCNIK8SPods discovers CNI pods, aws-node, running in the cluster -func (d *Controller) DiscoverCNIK8SPods() { - // create the pod watcher - d.DiscoverK8SPods(cache.NewListWatchFromClient(d.kubeClient.CoreV1().RESTClient(), "pods", metav1.NamespaceSystem, fields.Everything())) -} - -// DiscoverK8SPods takes a watcher and updates the Controller cache -func (d *Controller) DiscoverK8SPods(podListWatcher *cache.ListWatch) { - // create the workqueue - queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - - // Bind the workqueue to a cache with the help of an informer. This way we make sure that - // whenever the cache is updated, the pod key is added to the workqueue. - // Note that when we finally process the item from the workqueue, we might see a newer version - // of the Pod than the version which was responsible for triggering the update. - indexer, informer := cache.NewIndexerInformer(podListWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err == nil { - queue.Add(key) - } - }, - UpdateFunc: func(old interface{}, new interface{}) { - key, err := cache.MetaNamespaceKeyFunc(new) - if err == nil { - queue.Add(key) - } - }, - DeleteFunc: func(obj interface{}) { - // IndexerInformer uses a delta queue, therefore for deletes we have to use this - // key function. - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err == nil { - queue.Add(key) - } - }, - }, cache.Indexers{}) - - d.controller = newController(queue, indexer, informer) - - // Now let's start the controller - stop := make(chan struct{}) - defer close(stop) - go d.run(1, stop) - - // Wait forever - select {} -} - -// SetNodeLabel sets or deletes a label on the current node -func (d *Controller) SetNodeLabel(key, value string) error { - // Find my node - node, err := d.kubeClient.CoreV1().Nodes().Get(d.myNodeName, metav1.GetOptions{}) - if err != nil { - log.Errorf("Failed to get node: %v", err) - return err - } - - if labelValue, ok := node.Labels[key]; ok && labelValue == value { - log.Debugf("Node label %q is already %q", key, labelValue) - return nil - } - // Make deep copy for modification - updateNode := node.DeepCopy() - - // Set node label - if value != "" { - updateNode.Labels[key] = value - } else { - // Empty value, delete the label - log.Debugf("Deleting label %q", key) - delete(updateNode.Labels, key) - } - - // Update node status to advertise the resource. - _, err = d.kubeClient.CoreV1().Nodes().Update(updateNode) - if err != nil { - log.Errorf("Failed to update node %s with label %q: %q, error: %v", d.myNodeName, key, value, err) - } - log.Infof("Updated node %s with label %q: %q", d.myNodeName, key, value) - return nil -} - -// GetPod returns a pod based on name and namespace -func (d *Controller) GetPod(podName, namespace string) (*v1.Pod, error) { - return d.kubeClient.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{}) -} - -// The rest of logic/code are taken from kubernetes/client-go/examples/workqueue -func newController(queue workqueue.RateLimitingInterface, indexer cache.Indexer, informer cache.Controller) *controller { - return &controller{ - informer: informer, - indexer: indexer, - queue: queue, - } -} - -func (d *Controller) processNextItem() bool { - // Wait until there is a new item in the working queue - key, quit := d.controller.queue.Get() - if quit { - return false - } - // Tell the queue that we are done with processing this key. This unblocks the key for other workers - // This allows safe parallel processing because two pods with the same key are never processed in - // parallel. - defer d.controller.queue.Done(key) - - // Invoke the method containing the business logic - err := d.handlePodUpdate(key.(string)) - // Handle the error if something went wrong during the execution of the business logic - d.controller.handleErr(err, key) - return true -} - -func (d *Controller) handlePodUpdate(key string) error { - obj, exists, err := d.controller.indexer.GetByKey(key) - if err != nil { - log.Errorf("fetching object with key %s from store failed with %v", key, err) - return err - } - - if !exists { - log.Infof("Pods deleted on my node: %v", key) - if strings.HasPrefix(key, metav1.NamespaceSystem+"/"+cniPodName) { - d.cniPodsLock.Lock() - defer d.cniPodsLock.Unlock() - delete(d.cniPods, key) - } else { - d.workerPodsLock.Lock() - defer d.workerPodsLock.Unlock() - delete(d.workerPods, key) - } - return nil - } - - pod, ok := obj.(*v1.Pod) - if !ok || pod == nil { - log.Errorf("updated object received was not a pod: %+v", obj) - return errors.New("received a non-pod object update") - } - // Note that you also have to check the uid if you have a local controlled resource, which - // is dependent on the actual instance, to detect that a Pod was recreated with the same name - podName := pod.GetName() - - // Check to see if this is a pod on this node - if d.myNodeName == pod.Spec.NodeName && !pod.Spec.HostNetwork { - d.workerPodsLock.Lock() - defer d.workerPodsLock.Unlock() - - log.Debugf("Update for pod %s: %v", podName, pod.Status.Phase) - - // Save pod info - d.workerPods[key] = &K8SPodInfo{ - Name: podName, - Namespace: pod.GetNamespace(), - IP: pod.Status.PodIP, - UID: string(pod.GetUID()), - } - - log.Infof("Add/Update for Pod %s on my node, namespace = %s, IP = %s", podName, d.workerPods[key].Namespace, d.workerPods[key].IP) - } else if strings.HasPrefix(key, metav1.NamespaceSystem+"/"+cniPodName) { - d.cniPodsLock.Lock() - defer d.cniPodsLock.Unlock() - - log.Infof("Add/Update for CNI pod %s", podName) - d.cniPods[podName] = podName - } - return nil -} - -// handleErr checks if an error happened and makes sure we will retry later. -func (c *controller) handleErr(err error, key interface{}) { - if err == nil { - // Forget about the #AddRateLimited history of the key on every successful synchronization. - // This ensures that future processing of updates for this key is not delayed because of - // an outdated error history. - c.queue.Forget(key) - return - } - - // This controller retries 5 times if something goes wrong. After that, it stops trying. - if c.queue.NumRequeues(key) < 5 { - log.Infof("Error syncing pod %v: %v", key, err) - - // Re-enqueue the key rate limited. Based on the rate limiter on the - // queue and the re-enqueue history, the key will be processed later again. - c.queue.AddRateLimited(key) - return - } - - c.queue.Forget(key) - // Report to an external entity that, even after several retries, we could not successfully process this key - runtime.HandleError(err) - log.Infof("Dropping pod %q out of the queue: %v", key, err) -} - -func (d *Controller) run(threadiness int, stopCh chan struct{}) { - // Let the workers stop when we are done - defer d.controller.queue.ShutDown() - log.Info("Starting Pod controller") - - go d.controller.informer.Run(stopCh) - - log.Info("Waiting for controller cache sync") - // Wait for all involved caches to be synced, before processing items from the queue is started - if !cache.WaitForCacheSync(stopCh, d.controller.informer.HasSynced) { - log.Error("Timed out waiting for caches to sync!") - runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) - return - } - - log.Info("Synced successfully with APIServer") - d.synced = true - - for i := 0; i < threadiness; i++ { - go wait.Until(d.runWorker, time.Second, stopCh) - } - - <-stopCh - log.Info("Stopping Pod controller") -} - -func (d *Controller) runWorker() { - for d.processNextItem() { - } -} diff --git a/pkg/k8sapi/generate_mocks.go b/pkg/k8sapi/generate_mocks.go deleted file mode 100644 index ead04f8033..0000000000 --- a/pkg/k8sapi/generate_mocks.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package k8sapi - -//go:generate go run github.com/golang/mock/mockgen -destination mocks/k8sapi_mocks.go -copyright_file ../../scripts/copyright.txt . K8SAPIs diff --git a/pkg/k8sapi/mocks/k8sapi_mocks.go b/pkg/k8sapi/mocks/k8sapi_mocks.go deleted file mode 100644 index eec43c320d..0000000000 --- a/pkg/k8sapi/mocks/k8sapi_mocks.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. -// - -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi (interfaces: K8SAPIs) - -// Package mock_k8sapi is a generated GoMock package. -package mock_k8sapi - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "k8s.io/api/core/v1" -) - -// MockK8SAPIs is a mock of K8SAPIs interface -type MockK8SAPIs struct { - ctrl *gomock.Controller - recorder *MockK8SAPIsMockRecorder -} - -// MockK8SAPIsMockRecorder is the mock recorder for MockK8SAPIs -type MockK8SAPIsMockRecorder struct { - mock *MockK8SAPIs -} - -// NewMockK8SAPIs creates a new mock instance -func NewMockK8SAPIs(ctrl *gomock.Controller) *MockK8SAPIs { - mock := &MockK8SAPIs{ctrl: ctrl} - mock.recorder = &MockK8SAPIsMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockK8SAPIs) EXPECT() *MockK8SAPIsMockRecorder { - return m.recorder -} - -// GetPod mocks base method -func (m *MockK8SAPIs) GetPod(arg0, arg1 string) (*v1.Pod, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPod", arg0, arg1) - ret0, _ := ret[0].(*v1.Pod) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPod indicates an expected call of GetPod -func (mr *MockK8SAPIsMockRecorder) GetPod(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPod", reflect.TypeOf((*MockK8SAPIs)(nil).GetPod), arg0, arg1) -} - -// SetNodeLabel mocks base method -func (m *MockK8SAPIs) SetNodeLabel(arg0, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetNodeLabel", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetNodeLabel indicates an expected call of SetNodeLabel -func (mr *MockK8SAPIsMockRecorder) SetNodeLabel(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNodeLabel", reflect.TypeOf((*MockK8SAPIs)(nil).SetNodeLabel), arg0, arg1) -} From 61245bd0ad7e39615c633e284c25ff81b8226295 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Wed, 7 Apr 2021 08:51:43 -0700 Subject: [PATCH 2/5] Missing files --- pkg/k8sapi/k8sutils.go | 77 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pkg/k8sapi/k8sutils.go diff --git a/pkg/k8sapi/k8sutils.go b/pkg/k8sapi/k8sutils.go new file mode 100644 index 0000000000..9639ae2687 --- /dev/null +++ b/pkg/k8sapi/k8sutils.go @@ -0,0 +1,77 @@ +package k8sapi + +import ( + "fmt" + eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var log = logger.Get() + +// CreateKubeClient creates a k8s client +func CreateKubeClients() (client.Client, client.Client, error) { + restCfg := ctrl.GetConfigOrDie() + vpcCniScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(vpcCniScheme) + eniconfigscheme.AddToScheme(vpcCniScheme) + + stopChan := ctrl.SetupSignalHandler() + cache, err := cache.New(restCfg, cache.Options{Scheme: vpcCniScheme}) + if err != nil { + return nil, nil, err + } + go func() { + cache.Start(stopChan) + }() + cache.WaitForCacheSync(stopChan) + + standaloneK8SClient, err := client.New(restCfg, client.Options{Scheme: vpcCniScheme}) + k8sClient := client.DelegatingClient{ + Reader: &client.DelegatingReader{ + CacheReader: cache, + ClientReader: standaloneK8SClient, + }, + Writer: standaloneK8SClient, + StatusClient: standaloneK8SClient, + } + return standaloneK8SClient, k8sClient, nil +} + +func GetKubeClientSet() (kubernetes.Interface, error) { + // creates the in-cluster config + config, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + + // creates the clientset + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + return clientSet, nil +} + +func CheckAPIServerConnectivity() error { + restCfg := ctrl.GetConfigOrDie() + clientSet,_ := kubernetes.NewForConfig(restCfg) + + log.Infof("Testing communication with server") + version, err := clientSet.Discovery().ServerVersion() + if err !=nil { + return fmt.Errorf("error communicating with apiserver: %v", err) + } + log.Infof("Successful communication with the Cluster! Cluster Version is: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s", + version.Major, version.Minor, version.GitVersion, version.GitTreeState, version.GitCommit, version.Platform) + + return nil +} + + From d21ad5a557b0fa10947937e71f21b32a7da5a4b3 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Wed, 14 Apr 2021 14:43:24 -0700 Subject: [PATCH 3/5] Addressed CR comments --- cmd/aws-k8s-agent/main.go | 16 +- cmd/cni-metrics-helper/main.go | 11 +- cmd/cni-metrics-helper/metrics/cni_metrics.go | 24 +- .../metrics/cni_metrics_test.go | 25 +- cmd/cni-metrics-helper/metrics/metrics.go | 11 +- .../metrics/metrics_test.go | 7 +- cmd/cni-metrics-helper/metrics/pod_watcher.go | 18 +- go.mod | 23 +- go.sum | 273 ++++++++++++++++++ pkg/apis/crd/v1alpha1/eniconfig_types.go | 60 ++++ pkg/apis/crd/v1alpha1/groupversion_info.go | 36 +++ pkg/awsutils/awsutils.go | 31 +- pkg/eniconfig/eniconfig.go | 76 +++-- pkg/eniconfig/eniconfig_test.go | 48 ++- pkg/ipamd/introspect.go | 10 +- pkg/ipamd/ipamd.go | 45 +-- pkg/ipamd/ipamd_test.go | 158 +++++----- pkg/k8sapi/k8sutils.go | 54 ++-- pkg/networkutils/network.go | 2 +- 19 files changed, 658 insertions(+), 270 deletions(-) create mode 100644 pkg/apis/crd/v1alpha1/eniconfig_types.go create mode 100644 pkg/apis/crd/v1alpha1/groupversion_info.go diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go index bcde990819..4b62fb770d 100644 --- a/cmd/aws-k8s-agent/main.go +++ b/cmd/aws-k8s-agent/main.go @@ -15,10 +15,11 @@ package main import ( + "os" + "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd" "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - "os" ) var version string @@ -38,16 +39,21 @@ func _main() int { log.Infof("Starting L-IPAMD %s ...", version) //Check API Server Connectivity - if k8sapi.CheckAPIServerConnectivity() != nil{ + if k8sapi.CheckAPIServerConnectivity() != nil { + return 1 + } + + rawK8SClient, err := k8sapi.CreateKubeClient() + if err != nil { return 1 } - standaloneK8SClient, k8sClient, err := k8sapi.CreateKubeClients() - if err != nil{ + cacheK8SClient, err := k8sapi.CreateCachedKubeClient(rawK8SClient) + if err != nil { return 1 } - ipamContext, err := ipamd.New(standaloneK8SClient, k8sClient) + ipamContext, err := ipamd.New(rawK8SClient, cacheK8SClient) if err != nil { log.Errorf("Initialization failure: %v", err) diff --git a/cmd/cni-metrics-helper/main.go b/cmd/cni-metrics-helper/main.go index e7b741d088..02a234c663 100644 --- a/cmd/cni-metrics-helper/main.go +++ b/cmd/cni-metrics-helper/main.go @@ -84,11 +84,16 @@ func main() { clientSet, err := k8sapi.GetKubeClientSet() - _, k8sClient, err := k8sapi.CreateKubeClients() + rawK8SClient, err := k8sapi.CreateKubeClient() if err != nil { - panic(err.Error()) + log.Fatalf("Error creating Kubernetes Client: %s", err) + os.Exit(1) + } + k8sClient, err := k8sapi.CreateCachedKubeClient(rawK8SClient) + if err != nil { + log.Fatalf("Error creating Cached Kubernetes Client: %s", err) + os.Exit(1) } - var cw publisher.Publisher diff --git a/cmd/cni-metrics-helper/metrics/cni_metrics.go b/cmd/cni-metrics-helper/metrics/cni_metrics.go index edf134fafd..85db5e984f 100644 --- a/cmd/cni-metrics-helper/metrics/cni_metrics.go +++ b/cmd/cni-metrics-helper/metrics/cni_metrics.go @@ -139,24 +139,24 @@ var InterestingCNIMetrics = map[string]metricsConvert{ // CNIMetricsTarget defines data structure for kube-state-metric target type CNIMetricsTarget struct { - interestingMetrics map[string]metricsConvert - cwMetricsPublisher publisher.Publisher - kubeClient kubernetes.Interface - podWatcher *defaultPodWatcher - submitCW bool - log logger.Logger + interestingMetrics map[string]metricsConvert + cwMetricsPublisher publisher.Publisher + kubeClient kubernetes.Interface + podWatcher *defaultPodWatcher + submitCW bool + log logger.Logger } // CNIMetricsNew creates a new metricsTarget func CNIMetricsNew(k8sClient kubernetes.Interface, cw publisher.Publisher, submitCW bool, l logger.Logger, watcher *defaultPodWatcher) *CNIMetricsTarget { return &CNIMetricsTarget{ - interestingMetrics: InterestingCNIMetrics, - cwMetricsPublisher: cw, - kubeClient: k8sClient, - podWatcher: watcher, - submitCW: submitCW, - log: l, + interestingMetrics: InterestingCNIMetrics, + cwMetricsPublisher: cw, + kubeClient: k8sClient, + podWatcher: watcher, + submitCW: submitCW, + log: l, } } diff --git a/cmd/cni-metrics-helper/metrics/cni_metrics_test.go b/cmd/cni-metrics-helper/metrics/cni_metrics_test.go index 3835229307..00cc200726 100644 --- a/cmd/cni-metrics-helper/metrics/cni_metrics_test.go +++ b/cmd/cni-metrics-helper/metrics/cni_metrics_test.go @@ -1,20 +1,21 @@ package metrics import ( - eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" - "golang.org/x/net/context" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" - "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher/mock_publisher" - "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "golang.org/x/net/context" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher/mock_publisher" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) var logConfig = logger.Configuration{ @@ -25,9 +26,9 @@ var logConfig = logger.Configuration{ var testLog = logger.New(&logConfig) type testMocks struct { - clientset *k8sfake.Clientset - podWatcher *defaultPodWatcher - mockPublisher *mock_publisher.MockPublisher + clientset *k8sfake.Clientset + podWatcher *defaultPodWatcher + mockPublisher *mock_publisher.MockPublisher } func setup(t *testing.T) *testMocks { @@ -38,9 +39,9 @@ func setup(t *testing.T) *testMocks { eniconfigscheme.AddToScheme(k8sSchema) podWatcher := NewDefaultPodWatcher(testclient.NewFakeClientWithScheme(k8sSchema), testLog) return &testMocks{ - clientset: fakeClientset, - podWatcher: podWatcher, - mockPublisher: mock_publisher.NewMockPublisher(ctrl), + clientset: fakeClientset, + podWatcher: podWatcher, + mockPublisher: mock_publisher.NewMockPublisher(ctrl), } } diff --git a/cmd/cni-metrics-helper/metrics/metrics.go b/cmd/cni-metrics-helper/metrics/metrics.go index 9913850502..e57d317e0f 100644 --- a/cmd/cni-metrics-helper/metrics/metrics.go +++ b/cmd/cni-metrics-helper/metrics/metrics.go @@ -15,18 +15,19 @@ package metrics import ( - "context" "bytes" + "context" "fmt" - "github.com/prometheus/common/log" - "k8s.io/client-go/kubernetes" - "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" - "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/log" + "k8s.io/client-go/kubernetes" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) type metricMatcher func(metric *dto.Metric) bool diff --git a/cmd/cni-metrics-helper/metrics/metrics_test.go b/cmd/cni-metrics-helper/metrics/metrics_test.go index 1b72a084f1..f84fd66eb7 100644 --- a/cmd/cni-metrics-helper/metrics/metrics_test.go +++ b/cmd/cni-metrics-helper/metrics/metrics_test.go @@ -14,13 +14,14 @@ package metrics import ( - "golang.org/x/net/context" + "context" "io/ioutil" "testing" + "github.com/stretchr/testify/assert" + "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - "github.com/stretchr/testify/assert" ) type testMetricsTarget struct { @@ -62,7 +63,7 @@ func (target *testMetricsTarget) submitCloudWatch() bool { func TestAPIServerMetric(t *testing.T) { testTarget := newTestMetricsTarget("cni_test1.data", InterestingCNIMetrics) - ctx := context.Background() + ctx := context.Background() _, _, resetDetected, err := metricsListGrabAggregateConvert(ctx, testTarget) assert.NoError(t, err) assert.True(t, resetDetected) diff --git a/cmd/cni-metrics-helper/metrics/pod_watcher.go b/cmd/cni-metrics-helper/metrics/pod_watcher.go index 41b7775939..b0b459a6d2 100644 --- a/cmd/cni-metrics-helper/metrics/pod_watcher.go +++ b/cmd/cni-metrics-helper/metrics/pod_watcher.go @@ -2,12 +2,13 @@ package metrics import ( "context" - "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strings" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) type PodWatcher interface { @@ -20,20 +21,21 @@ type defaultPodWatcher struct { } // NewDefaultPodWatcher creates a new podWatcher -func NewDefaultPodWatcher(k8sClient client.Client, log logger.Logger) *defaultPodWatcher{ +func NewDefaultPodWatcher(k8sClient client.Client, log logger.Logger) *defaultPodWatcher { return &defaultPodWatcher{ k8sClient: k8sClient, - log: log, + log: log, } } //Returns aws-node pod info. Below function assumes CNI pods follow aws-node* naming format //and so the function has to be updated if the CNI pod name format changes. -func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error){ +func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error) { var CNIPods []string var podList corev1.PodList listOptions := client.ListOptions{ - Namespace: metav1.NamespaceSystem, + Namespace: metav1.NamespaceSystem, + Limit: 500, } err := d.k8sClient.List(ctx, &podList, &listOptions) @@ -41,7 +43,7 @@ func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error){ return CNIPods, err } - for _,pod := range podList.Items { + for _, pod := range podList.Items { if strings.HasPrefix(pod.Name, "aws-node") { CNIPods = append(CNIPods, pod.Name) } diff --git a/go.mod b/go.mod index 2d22d687a1..56411e03b7 100644 --- a/go.mod +++ b/go.mod @@ -4,32 +4,18 @@ go 1.14 require ( github.com/aws/aws-sdk-go v1.37.23 - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/containernetworking/cni v0.8.0 github.com/containernetworking/plugins v0.9.0 github.com/coreos/go-iptables v0.4.5 - github.com/ghodss/yaml v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/mock v1.4.1 github.com/golang/protobuf v1.4.2 - github.com/google/btree v1.0.0 // indirect github.com/google/go-jsonnet v0.16.0 - github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/google/gopacket v1.1.18 - github.com/googleapis/gnostic v0.2.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect - github.com/hashicorp/golang-lru v0.5.1 // indirect - github.com/imdario/mergo v0.3.6 // indirect - github.com/json-iterator/go v1.1.5 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect - github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v0.8.0 - github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 - github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e - github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 // indirect - github.com/sirupsen/logrus v1.4.1 // indirect + github.com/prometheus/client_golang v1.0.0 + github.com/prometheus/client_model v0.2.0 + github.com/prometheus/common v0.4.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.5.1 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 @@ -38,15 +24,12 @@ require ( golang.org/x/mod v0.4.0 // indirect golang.org/x/net v0.0.0-20201110031124-69a78807bb2b golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637 - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect golang.org/x/tools v0.0.0-20210113180300-f96436850f18 // indirect google.golang.org/grpc v1.29.0 - gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.18.6 k8s.io/apimachinery v0.18.6 k8s.io/client-go v0.18.6 k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28 - k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92 // indirect sigs.k8s.io/controller-runtime v0.6.3 ) diff --git a/go.sum b/go.sum index afe8cd642c..346e32ef1d 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,65 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.35.27 h1:F0dUW+kouzchjt4X6kYfYMw1YtQPkA4pihpCDqQMrq8= github.com/aws/aws-sdk-go v1.35.27/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.37.23 h1:bO80NcSmRv52w+GFpBegoLdlP/Z0OwUqQ9bbeCLCy/0= github.com/aws/aws-sdk-go v1.37.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.7 h1:bU7QieuAp+sACI2vCzESJ3FoT860urYP+lThyZkb/2M= github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0= github.com/containernetworking/plugins v0.9.0 h1:c+1gegKhR7+d0Caum9pEHugZlyhXPOG6v3V6xJgIGCI= github.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= @@ -34,29 +68,96 @@ github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -73,6 +174,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= @@ -85,53 +187,103 @@ github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -140,47 +292,83 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/operator-framework/operator-sdk v0.0.7 h1:feujqHLhibLBbDVrSAFswpSzTVS5mEuarvywJ079mYE= github.com/operator-framework/operator-sdk v0.0.7/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -189,25 +377,45 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -223,42 +431,65 @@ golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -266,21 +497,30 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -294,9 +534,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -304,6 +549,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo= google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -314,11 +560,13 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -327,8 +575,11 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -337,22 +588,44 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 h1:hROmpFC7JMobXFXMmD7ZKZLhDKvr1IKfFJoYS/45G/8= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= +k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d h1:MZjlsu9igBoVPZkXpIGoxI6EonqNsXXZU7hhvfQLkd4= k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 h1:kQX7jEIMYrWV9XqFN4usRaBLzCu7fd/qsCXxbgf3+9g= k8s.io/client-go v0.0.0-20180806134042-1f13a808da65/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= +k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28 h1:dVG+VaWJ3eRA1UZUhCs59rcnqEDMky42bm5L8yyVCMw= k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28/go.mod h1:9a7E6pmKLfuq8ZL31k2PDpgvSdyZfUOH9czlEmpblFk= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 h1:f0BTap/vrgs21vVbJ1ySdsNtcivpA1x4ut6Wla9HKKw= k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22/go.mod h1:iU+ZGYsNlvU9XKUSso6SQfKTCCw7lFduMZy26Mgr2Fw= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= +sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= sigs.k8s.io/structured-merge-diff v0.0.0-20190426204423-ea680f03cc65/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/apis/crd/v1alpha1/eniconfig_types.go b/pkg/apis/crd/v1alpha1/eniconfig_types.go new file mode 100644 index 0000000000..4f161091d6 --- /dev/null +++ b/pkg/apis/crd/v1alpha1/eniconfig_types.go @@ -0,0 +1,60 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ENIConfigSpec defines the desired state of ENIConfig +type ENIConfigSpec struct { + SecurityGroups []string `json:"securityGroups"` + Subnet string `json:"subnet"` +} + +// ENIConfigStatus defines the observed state of ENIConfig +type ENIConfigStatus struct { + // Fill me +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// ENIConfig is the Schema for the eniconfigs API +type ENIConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ENIConfigSpec `json:"spec,omitempty"` + Status ENIConfigStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// ENIConfigList contains a list of ENIConfig +type ENIConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ENIConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ENIConfig{}, &ENIConfigList{}) +} diff --git a/pkg/apis/crd/v1alpha1/groupversion_info.go b/pkg/apis/crd/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..c5b2445d10 --- /dev/null +++ b/pkg/apis/crd/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the crd v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=crd.example.com +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "crd.k8s.amazonaws.com", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 48d121f49e..b92875247b 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -158,13 +158,12 @@ type APIs interface { //SetCNIunmanaged ENI SetCNIUnmanagedENIs(eniID []string) error - + //isCNIUnmanagedENI IsCNIUnmanagedENI(eniID string) bool - + //RefreshSGIDs - RefreshSGIDs(mac string) error - + RefreshSGIDs(mac string) error } // EC2InstanceMetadataCache caches instance metadata @@ -226,10 +225,10 @@ type TagMap map[string]string // DescribeAllENIsResult contains the fully type DescribeAllENIsResult struct { - ENIMetadata []ENIMetadata - TagMap map[string]TagMap - TrunkENI string - EFAENIs map[string]bool + ENIMetadata []ENIMetadata + TagMap map[string]TagMap + TrunkENI string + EFAENIs map[string]bool MultiCardENIIDs []string } @@ -409,7 +408,7 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) return err } log.Debugf("Found subnet-id: %s ", cache.subnetID) - + // We use the ctx here for testing, since we spawn go-routines above which will run forever. select { case <-ctx.Done(): @@ -1070,10 +1069,10 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, } } return DescribeAllENIsResult{ - ENIMetadata: verifiedENIs, - TagMap: tagMap, - TrunkENI: trunkENI, - EFAENIs: efaENIs, + ENIMetadata: verifiedENIs, + TagMap: tagMap, + TrunkENI: trunkENI, + EFAENIs: efaENIs, MultiCardENIIDs: multiCardENIIDs, }, nil } @@ -1520,9 +1519,9 @@ func (cache *EC2InstanceMetadataCache) getENIsFromPaginatedDescribeNetworkInterf //SetCNIUnmanagedENIs Set unmanaged ENI set func (cache *EC2InstanceMetadataCache) SetCNIUnmanagedENIs(eniID []string) error { - if len(eniID) != 0 { + if len(eniID) != 0 { cache.cniunmanagedENIs.Set(eniID) - } + } return nil } @@ -1532,4 +1531,4 @@ func (cache *EC2InstanceMetadataCache) IsCNIUnmanagedENI(eniID string) bool { return cache.cniunmanagedENIs.Has(eniID) } return false -} \ No newline at end of file +} diff --git a/pkg/eniconfig/eniconfig.go b/pkg/eniconfig/eniconfig.go index 1f71da8382..fbcb5b5bfd 100644 --- a/pkg/eniconfig/eniconfig.go +++ b/pkg/eniconfig/eniconfig.go @@ -17,13 +17,15 @@ package eniconfig import ( "context" "fmt" - corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "os" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - "github.com/pkg/errors" ) const ( @@ -45,7 +47,7 @@ const ( // ENIConfig interface type ENIConfig interface { MyENIConfig(client.Client) (*v1alpha1.ENIConfigSpec, error) - GetENIConfigName(context.Context, client.Client) (string, error) + GetENIConfigName(context.Context, client.Client) (string, error) } // ErrNoENIConfig is the missing ENIConfig error @@ -65,26 +67,21 @@ type ENIConfigInfo struct { func MyENIConfig(ctx context.Context, k8sClient client.Client) (*v1alpha1.ENIConfigSpec, error) { eniConfigName, err := GetNodeSpecificENIConfigName(ctx, k8sClient) if err != nil { - log.Debugf("Error while retrieving Node name") + log.Debugf("Error while retrieving Node name") } log.Infof("Found ENI Config Name: %s", eniConfigName) - - eniConfigsList := v1alpha1.ENIConfigList{} - err = k8sClient.List(ctx, &eniConfigsList) + var eniConfig v1alpha1.ENIConfig + err = k8sClient.Get(ctx, types.NamespacedName{Name: eniConfigName}, &eniConfig) if err != nil { - fmt.Errorf("Error while EniConfig List Get: %s", err) - } - for _, eni := range eniConfigsList.Items { - if eniConfigName == eni.Name { - log.Debugf("Matching ENIConfig found: %s - %s - %s ", eni.Name, eni.Spec.Subnet, eni.Spec.SecurityGroups) - return &v1alpha1.ENIConfigSpec{ - SecurityGroups: eni.Spec.SecurityGroups, - Subnet: eni.Spec.Subnet, - }, nil - } + fmt.Errorf("error while retrieving eniconfig: %s", err) + return nil, ErrNoENIConfig } - return nil, ErrNoENIConfig + + return &v1alpha1.ENIConfigSpec{ + SecurityGroups: eniConfig.Spec.SecurityGroups, + Subnet: eniConfig.Spec.Subnet, + }, nil } // getEniConfigAnnotationDef returns eniConfigAnnotation @@ -116,30 +113,31 @@ func getEniConfigLabelDef() string { } func GetNodeSpecificENIConfigName(ctx context.Context, k8sClient client.Client) (string, error) { - var eniConfigName string - nodeList := corev1.NodeList{} - err := k8sClient.List(ctx, &nodeList) + var eniConfigName string + + log.Infof("Get Node Info for: %s", os.Getenv("MY_NODE_NAME")) + var node corev1.Node + err := k8sClient.Get(ctx, types.NamespacedName{Name: os.Getenv("MY_NODE_NAME")}, &node) if err != nil { - fmt.Errorf("Error while Node List Get: %s", err) + fmt.Errorf("error retrieving node: %s", err) + return eniConfigName, err } - log.Debugf("Node Count: ", len(nodeList.Items)) - for _, node := range nodeList.Items { - if node.Name == os.Getenv("MY_NODE_NAME") { - log.Debugf("Node Info: %s", node.Name) - val, ok := node.GetAnnotations()[getEniConfigAnnotationDef()] - if !ok { - val, ok = node.GetLabels()[getEniConfigLabelDef()] - if !ok { - val = eniConfigDefault - } - } - eniConfigName = val - if val != eniConfigDefault { - labels := node.GetLabels() - labels["vpc.amazonaws.com/eniConfig"] = eniConfigName - node.SetLabels(labels) - } + + //Derive ENIConfig Name from either Node Annotations or Labels + val, ok := node.GetAnnotations()[getEniConfigAnnotationDef()] + if !ok { + val, ok = node.GetLabels()[getEniConfigLabelDef()] + if !ok { + val = eniConfigDefault } } + + eniConfigName = val + if val != eniConfigDefault { + labels := node.GetLabels() + labels["vpc.amazonaws.com/eniConfig"] = eniConfigName + node.SetLabels(labels) + } + return eniConfigName, nil } diff --git a/pkg/eniconfig/eniconfig_test.go b/pkg/eniconfig/eniconfig_test.go index ab2ec1b27d..1492b902ba 100644 --- a/pkg/eniconfig/eniconfig_test.go +++ b/pkg/eniconfig/eniconfig_test.go @@ -14,21 +14,19 @@ package eniconfig import ( "context" - //"fmt" "os" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" - "github.com/stretchr/testify/assert" - - eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" ) func TestMyENIConfig(t *testing.T) { @@ -44,7 +42,7 @@ func TestMyENIConfig(t *testing.T) { }, Spec: v1alpha1.ENIConfigSpec{ SecurityGroups: []string{"SG1"}, - Subnet: "SB1", + Subnet: "SB1", }, } @@ -54,7 +52,7 @@ func TestMyENIConfig(t *testing.T) { }, Spec: v1alpha1.ENIConfigSpec{ SecurityGroups: []string{"SG2"}, - Subnet: "SB2", + Subnet: "SB2", }, } @@ -76,57 +74,57 @@ func TestMyENIConfig(t *testing.T) { { name: "Matching ENIConfig available - Using Default Labels", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, Labels: map[string]string{ "k8s.amazonaws.com/eniConfig": "az1", }, }, - want: &testENIConfigAZ1.Spec, + want: &testENIConfigAZ1.Spec, wantErr: nil, }, { name: "No Matching ENIConfig available - Using Default Labels", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ2}, Labels: map[string]string{ "k8s.amazonaws.com/eniConfig": "az1", }, }, - want: nil, + want: nil, wantErr: errors.New("eniconfig: eniconfig is not available"), }, { name: "Matching ENIConfig available - Using Custom Label Key exposed via ENI_CONFIG_LABEL_DEF", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, Labels: map[string]string{ "failure-domain.beta.kubernetes.io/zone": "az1", }, eniConfigLabelKey: "failure-domain.beta.kubernetes.io/zone", }, - want: &testENIConfigAZ1.Spec, + want: &testENIConfigAZ1.Spec, wantErr: nil, }, { name: "No Matching ENIConfig available - Using Custom Label Key exposed via ENI_CONFIG_LABEL_DEF", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, Labels: map[string]string{ "failure-domain.beta.kubernetes.io/zone": "az2", }, eniConfigLabelKey: "failure-domain.beta.kubernetes.io/zone", }, - want: nil, + want: nil, wantErr: errors.New("eniconfig: eniconfig is not available"), }, { name: "Matching ENIConfig available - Using Default Annotation", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, Labels: map[string]string{ "failure-domain.beta.kubernetes.io/zone": "az2", @@ -135,13 +133,13 @@ func TestMyENIConfig(t *testing.T) { "k8s.amazonaws.com/eniConfig": "az1", }, }, - want: &testENIConfigAZ1.Spec, + want: &testENIConfigAZ1.Spec, wantErr: nil, }, { name: "No Matching ENIConfig available - Using Default Annotation", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ2}, Labels: map[string]string{ "failure-domain.beta.kubernetes.io/zone": "az2", @@ -150,13 +148,13 @@ func TestMyENIConfig(t *testing.T) { "k8s.amazonaws.com/eniConfig": "az1", }, }, - want: nil, + want: nil, wantErr: errors.New("eniconfig: eniconfig is not available"), }, { name: "Matching ENIConfig available - Using Custom Annotation Key exposed via ENI_CONFIG_ANNOTATION_DEF", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ1}, Labels: map[string]string{ "failure-domain.beta.kubernetes.io/zone": "az2", @@ -166,13 +164,13 @@ func TestMyENIConfig(t *testing.T) { }, eniConfigAnnotationKey: "k8s.amazonaws.com/myENIConfig", }, - want: &testENIConfigAZ1.Spec, + want: &testENIConfigAZ1.Spec, wantErr: nil, }, { name: "No Matching ENIConfig available - Using Custom Label Key exposed via ENI_CONFIG_ANNOTATION_DEF", env: env{ - nodes: []*corev1.Node{testNode}, + nodes: []*corev1.Node{testNode}, eniconfigs: []*v1alpha1.ENIConfig{testENIConfigAZ2}, Labels: map[string]string{ "failure-domain.beta.kubernetes.io/zone": "az2", @@ -182,7 +180,7 @@ func TestMyENIConfig(t *testing.T) { }, eniConfigAnnotationKey: "k8s.amazonaws.com/myENIConfig", }, - want: nil, + want: nil, wantErr: errors.New("eniconfig: eniconfig is not available"), }, } diff --git a/pkg/ipamd/introspect.go b/pkg/ipamd/introspect.go index b0762d11f0..402b835d62 100644 --- a/pkg/ipamd/introspect.go +++ b/pkg/ipamd/introspect.go @@ -15,7 +15,6 @@ package ipamd import ( "encoding/json" - "golang.org/x/net/context" "net" "net/http" "os" @@ -144,8 +143,13 @@ func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - myENIConfig, _ := eniconfig.GetNodeSpecificENIConfigName(ctx, ipam.k8sClient) + ctx := r.Context() + myENIConfig, err := eniconfig.GetNodeSpecificENIConfigName(ctx, ipam.cachedK8SClient) + if err != nil { + log.Errorf("Failed to get ENI config: %v", err) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } responseJSON, err := json.Marshal(myENIConfig) if err != nil { log.Errorf("Failed to marshal ENI config: %v", err) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index df6fc19d74..0f5441b170 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -25,20 +25,20 @@ import ( "sync/atomic" "time" - corev1 "k8s.io/api/core/v1" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" - "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" - "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" - "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" - "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" + "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" + "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" + "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) // The package ipamd is a long running daemon which manages a warm pool of available IP addresses. @@ -191,8 +191,8 @@ var ( type IPAMContext struct { awsClient awsutils.APIs dataStore *datastore.DataStore - standalonek8sClient client.Client - k8sClient client.Client + rawK8SClient client.Client + cachedK8SClient client.Client useCustomNetworking bool networkClient networkutils.NetworkAPIs maxIPsPerENI int @@ -284,11 +284,12 @@ func prometheusRegister() { // New retrieves IP address usage information from Instance MetaData service and Kubelet // then initializes IP address pool data store -func New(k8sapiClient client.Client, k8sClient client.Client) (*IPAMContext, error) { prometheusRegister() +func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContext, error) { + prometheusRegister() c := &IPAMContext{} - c.standalonek8sClient = k8sapiClient - c.k8sClient = k8sClient + c.rawK8SClient = rawK8SClient + c.cachedK8SClient = cachedK8SClient c.networkClient = networkutils.New() c.useCustomNetworking = UseCustomNetworkCfg() @@ -332,7 +333,7 @@ func (c *IPAMContext) nodeInit() error { ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1)) var err error - ctx := context.Background() + ctx := context.TODO() log.Debugf("Start node init") @@ -411,8 +412,8 @@ func (c *IPAMContext) nodeInit() error { vpcCIDRs = c.updateCIDRsRulesOnChange(vpcCIDRs) }, 30*time.Second) - eniConfigName,_ := eniconfig.GetNodeSpecificENIConfigName(ctx, c.k8sClient) - if c.useCustomNetworking && eniConfigName != "default" { + eniConfigName, err := eniconfig.GetNodeSpecificENIConfigName(ctx, c.cachedK8SClient) + if err == nil && c.useCustomNetworking && eniConfigName != "default" { // Signal to VPC Resource Controller that the node is using custom networking err := c.SetNodeLabel(ctx, vpcENIConfigLabel, eniConfigName) if err != nil { @@ -680,7 +681,7 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context) error { var subnet string if c.useCustomNetworking { - eniCfg, err := eniconfig.MyENIConfig(ctx, c.k8sClient) + eniCfg, err := eniconfig.MyENIConfig(ctx, c.cachedK8SClient) if err != nil { log.Errorf("Failed to get pod ENI config") @@ -980,7 +981,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur if c.enablePodENI && metadataResult.TrunkENI != "" { // Label the node that we have a trunk - err = c.SetNodeLabel(ctx,"vpc.amazonaws.com/has-trunk-attached", "true") + err = c.SetNodeLabel(ctx, "vpc.amazonaws.com/has-trunk-attached", "true") if err != nil { podENIErrInc("askForTrunkENIIfNeeded") log.Errorf("Failed to set node label for trunk. Aborting reconcile", err) @@ -1316,7 +1317,7 @@ func (c *IPAMContext) getTrunkLinkIndex() (int, error) { func (c *IPAMContext) SetNodeLabel(ctx context.Context, key, value string) error { var node corev1.Node // Find my node - err := c.k8sClient.Get(ctx, types.NamespacedName{Name: c.myNodeName}, &node) + err := c.cachedK8SClient.Get(ctx, types.NamespacedName{Name: c.myNodeName}, &node) log.Debugf("Node found %q - labels - %q", node.Name, len(node.Labels)) if err != nil { @@ -1342,7 +1343,7 @@ func (c *IPAMContext) SetNodeLabel(ctx context.Context, key, value string) error } // Update node status to advertise the resource. - err = c.k8sClient.Update(ctx, updateNode) + err = c.cachedK8SClient.Update(ctx, updateNode) if err != nil { log.Errorf("Failed to update node %s with label %q: %q, error: %v", c.myNodeName, key, value, err) } @@ -1353,16 +1354,16 @@ func (c *IPAMContext) SetNodeLabel(ctx context.Context, key, value string) error // GetPod returns the pod matching the name and namespace func (c *IPAMContext) GetPod(podName, namespace string) (*corev1.Pod, error) { - ctx := context.Background() + ctx := context.TODO() var pod corev1.Pod podKey := types.NamespacedName{ Namespace: namespace, Name: podName, } - err := c.standalonek8sClient.Get(ctx, podKey ,&pod) + err := c.rawK8SClient.Get(ctx, podKey, &pod) if err != nil { return nil, fmt.Errorf("Error while trying to retrieve Pod Info: %s", err) } return &pod, nil -} \ No newline at end of file +} diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 9a7579c3f1..696c60040e 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -14,19 +14,27 @@ package ipamd import ( + "context" "errors" "fmt" - "golang.org/x/net/context" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "net" "os" "reflect" "testing" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" @@ -35,14 +43,6 @@ import ( mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" mock_networkutils "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/vishvananda/netlink" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) const ( @@ -64,12 +64,12 @@ const ( ) type testMocks struct { - ctrl *gomock.Controller - awsutils *mock_awsutils.MockAPIs - standaloneClient client.Client - k8sClient client.Client - network *mock_networkutils.MockNetworkAPIs - eniconfig *mock_eniconfig.MockENIConfig + ctrl *gomock.Controller + awsutils *mock_awsutils.MockAPIs + rawK8SClient client.Client + cachedK8SClient client.Client + network *mock_networkutils.MockNetworkAPIs + eniconfig *mock_eniconfig.MockENIConfig } func setup(t *testing.T) *testMocks { @@ -79,12 +79,12 @@ func setup(t *testing.T) *testMocks { eniconfigscheme.AddToScheme(k8sSchema) return &testMocks{ - ctrl: ctrl, - awsutils: mock_awsutils.NewMockAPIs(ctrl), - standaloneClient: testclient.NewFakeClientWithScheme(k8sSchema), - k8sClient: testclient.NewFakeClientWithScheme(k8sSchema), - network: mock_networkutils.NewMockNetworkAPIs(ctrl), - eniconfig: mock_eniconfig.NewMockENIConfig(ctrl), + ctrl: ctrl, + awsutils: mock_awsutils.NewMockAPIs(ctrl), + rawK8SClient: testclient.NewFakeClientWithScheme(k8sSchema), + cachedK8SClient: testclient.NewFakeClientWithScheme(k8sSchema), + network: mock_networkutils.NewMockNetworkAPIs(ctrl), + eniconfig: mock_eniconfig.NewMockENIConfig(ctrl), } } @@ -101,18 +101,18 @@ func TestNodeInit(t *testing.T) { } mockContext := &IPAMContext{ - awsClient: m.awsutils, - standalonek8sClient: m.standaloneClient, - k8sClient: m.k8sClient, - maxIPsPerENI: 14, - maxENI: 4, - warmENITarget: 1, - warmIPTarget: 3, - primaryIP: make(map[string]string), - terminating: int32(0), - networkClient: m.network, - dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint)), - myNodeName: myNodeName, + awsClient: m.awsutils, + rawK8SClient: m.rawK8SClient, + cachedK8SClient: m.cachedK8SClient, + maxIPsPerENI: 14, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint)), + myNodeName: myNodeName, } mockContext.dataStore.CheckpointMigrationPhase = 2 @@ -137,10 +137,10 @@ func TestNodeInit(t *testing.T) { eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} resp := awsutils.DescribeAllENIsResult{ - ENIMetadata: eniMetadataSlice, - TagMap: map[string]awsutils.TagMap{}, - TrunkENI: "", - EFAENIs: make(map[string]bool), + ENIMetadata: eniMetadataSlice, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), MultiCardENIIDs: nil, } m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) @@ -162,7 +162,7 @@ func TestNodeInit(t *testing.T) { Status: v1.NodeStatus{}, } //_, _ = m.clientset.CoreV1().Nodes().Create(&fakeNode) - _ = m.k8sClient.Create(ctx, &fakeNode) + _ = m.cachedK8SClient.Create(ctx, &fakeNode) // Add IPs m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any()) @@ -228,8 +228,8 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { mockContext := &IPAMContext{ awsClient: m.awsutils, - standalonek8sClient: m.standaloneClient, - k8sClient: m.k8sClient, + rawK8SClient: m.rawK8SClient, + cachedK8SClient: m.cachedK8SClient, maxIPsPerENI: 14, maxENI: 4, warmENITarget: 1, @@ -302,9 +302,9 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { m.awsutils.EXPECT().AllocIPAddresses(eni2, 14) if mockContext.useCustomNetworking { - mockContext.myNodeName = myNodeName + mockContext.myNodeName = myNodeName - labels := map[string]string { + labels := map[string]string{ "k8s.amazonaws.com/eniConfig": "az1", } //Create a Fake Node @@ -314,19 +314,19 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { Spec: v1.NodeSpec{}, Status: v1.NodeStatus{}, } - _ = m.k8sClient.Create(ctx, &fakeNode) + _ = m.cachedK8SClient.Create(ctx, &fakeNode) //Create a dummy ENIConfig fakeENIConfig := v1alpha1.ENIConfig{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "az1"}, - Spec: eniconfigscheme.ENIConfigSpec{ - Subnet: "subnet1", + Spec: eniconfigscheme.ENIConfigSpec{ + Subnet: "subnet1", SecurityGroups: []string{"sg1-id", "sg2-id"}, }, - Status: eniconfigscheme.ENIConfigStatus{}, + Status: eniconfigscheme.ENIConfigStatus{}, } - _=m.k8sClient.Create(ctx, &fakeENIConfig) + _ = m.cachedK8SClient.Create(ctx, &fakeENIConfig) } mockContext.increaseIPPool(ctx) @@ -422,16 +422,16 @@ func TestNodeIPPoolReconcile(t *testing.T) { eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) resp := awsutils.DescribeAllENIsResult{ - ENIMetadata: eniMetadataList, - TagMap: map[string]awsutils.TagMap{}, - TrunkENI: "", - EFAENIs: make(map[string]bool), - MultiCardENIIDs : nil, + ENIMetadata: eniMetadataList, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + MultiCardENIIDs: nil, } m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp.MultiCardENIIDs).AnyTimes() - mockContext.nodeIPPoolReconcile(ctx,0) + mockContext.nodeIPPoolReconcile(ctx, 0) curENIs := mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) @@ -469,11 +469,11 @@ func TestNodeIPPoolReconcile(t *testing.T) { m.awsutils.EXPECT().IsUnmanagedENI(secENIid).Times(2).Return(false) m.awsutils.EXPECT().IsCNIUnmanagedENI(secENIid).Times(2).Return(false) resp2 := awsutils.DescribeAllENIsResult{ - ENIMetadata: twoENIs, - TagMap: map[string]awsutils.TagMap{}, - TrunkENI: "", - EFAENIs: make(map[string]bool), - MultiCardENIIDs : nil, + ENIMetadata: twoENIs, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + MultiCardENIIDs: nil, } m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet) @@ -678,12 +678,12 @@ func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { return false }).AnyTimes() - + mockAWSUtils.EXPECT().IsCNIUnmanagedENI(gomock.Any()).DoAndReturn( func(eni string) (unmanaged bool) { return false - - }).AnyTimes() + + }).AnyTimes() if got := c.filterUnmanagedENIs(tt.enis); !reflect.DeepEqual(got, tt.want) { t.Errorf("filterUnmanagedENIs() = %v, want %v", got, tt.want) @@ -904,27 +904,27 @@ func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { ctx := context.Background() mockContext := &IPAMContext{ - standalonek8sClient: m.standaloneClient, - k8sClient: m.k8sClient, - dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion})), - awsClient: m.awsutils, - networkClient: m.network, - primaryIP: make(map[string]string), - terminating: int32(0), - maxENI: 1, - myNodeName: myNodeName, + rawK8SClient: m.rawK8SClient, + cachedK8SClient: m.cachedK8SClient, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion})), + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + maxENI: 1, + myNodeName: myNodeName, } - labels := map[string]string { - "testKey": "testValue", - } + labels := map[string]string{ + "testKey": "testValue", + } fakeNode := v1.Node{ TypeMeta: metav1.TypeMeta{Kind: "Node"}, ObjectMeta: metav1.ObjectMeta{Name: myNodeName, Labels: labels}, Spec: v1.NodeSpec{}, Status: v1.NodeStatus{}, } - _ = m.k8sClient.Create(ctx, &fakeNode) + _ = m.cachedK8SClient.Create(ctx, &fakeNode) _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false) // If ENABLE_POD_ENI is not set, nothing happens @@ -939,7 +939,7 @@ func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { Namespace: "", Name: myNodeName, } - err := m.k8sClient.Get(ctx, NodeKey, ¬UpdatedNode) + err := m.cachedK8SClient.Get(ctx, NodeKey, ¬UpdatedNode) // Since there was no room, no label should be added assert.NoError(t, err) assert.Equal(t, 1, len(notUpdatedNode.Labels)) @@ -950,7 +950,7 @@ func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { // Fetch the updated node and verify that the label is set //updatedNode, err := m.clientset.CoreV1().Nodes().Get(myNodeName, metav1.GetOptions{}) - err = m.k8sClient.Get(ctx, NodeKey, &updatedNode) + err = m.cachedK8SClient.Get(ctx, NodeKey, &updatedNode) assert.NoError(t, err) assert.Equal(t, "false", updatedNode.Labels["vpc.amazonaws.com/has-trunk-attached"]) } diff --git a/pkg/k8sapi/k8sutils.go b/pkg/k8sapi/k8sutils.go index 9639ae2687..69e1355352 100644 --- a/pkg/k8sapi/k8sutils.go +++ b/pkg/k8sapi/k8sutils.go @@ -16,8 +16,29 @@ import ( var log = logger.Get() // CreateKubeClient creates a k8s client -func CreateKubeClients() (client.Client, client.Client, error) { - restCfg := ctrl.GetConfigOrDie() +func CreateKubeClient() (client.Client, error) { + restCfg, err := ctrl.GetConfig() + if err != nil { + return nil, err + } + vpcCniScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(vpcCniScheme) + eniconfigscheme.AddToScheme(vpcCniScheme) + + rawK8SClient, err := client.New(restCfg, client.Options{Scheme: vpcCniScheme}) + if err != nil { + return nil, err + } + + return rawK8SClient, nil +} + +// CreateKubeClient creates a k8s client +func CreateCachedKubeClient(rawK8SClient client.Client) (client.Client, error) { + restCfg, err := ctrl.GetConfig() + if err != nil { + return nil, err + } vpcCniScheme := runtime.NewScheme() clientgoscheme.AddToScheme(vpcCniScheme) eniconfigscheme.AddToScheme(vpcCniScheme) @@ -25,30 +46,28 @@ func CreateKubeClients() (client.Client, client.Client, error) { stopChan := ctrl.SetupSignalHandler() cache, err := cache.New(restCfg, cache.Options{Scheme: vpcCniScheme}) if err != nil { - return nil, nil, err + return nil, err } go func() { cache.Start(stopChan) }() cache.WaitForCacheSync(stopChan) - standaloneK8SClient, err := client.New(restCfg, client.Options{Scheme: vpcCniScheme}) - k8sClient := client.DelegatingClient{ + cachedK8SClient := client.DelegatingClient{ Reader: &client.DelegatingReader{ CacheReader: cache, - ClientReader: standaloneK8SClient, + ClientReader: rawK8SClient, }, - Writer: standaloneK8SClient, - StatusClient: standaloneK8SClient, + Writer: rawK8SClient, + StatusClient: rawK8SClient, } - return standaloneK8SClient, k8sClient, nil + return cachedK8SClient, nil } - func GetKubeClientSet() (kubernetes.Interface, error) { // creates the in-cluster config config, err := rest.InClusterConfig() if err != nil { - panic(err.Error()) + return nil, err } // creates the clientset @@ -60,18 +79,19 @@ func GetKubeClientSet() (kubernetes.Interface, error) { } func CheckAPIServerConnectivity() error { - restCfg := ctrl.GetConfigOrDie() - clientSet,_ := kubernetes.NewForConfig(restCfg) + restCfg, err := ctrl.GetConfig() + if err != nil { + return err + } + clientSet, _ := kubernetes.NewForConfig(restCfg) log.Infof("Testing communication with server") version, err := clientSet.Discovery().ServerVersion() - if err !=nil { + if err != nil { return fmt.Errorf("error communicating with apiserver: %v", err) } log.Infof("Successful communication with the Cluster! Cluster Version is: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s", version.Major, version.Minor, version.GitVersion, version.GitTreeState, version.GitCommit, version.Platform) - return nil + return nil } - - diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index 350fff3d6b..029ddce93c 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -310,7 +310,7 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr } var allCIDRs []snatCIDR for _, cidr := range vpcCIDRs { - log.Debugf("Adding %s CIDR to NAT chain", cidr) + log.Debugf("Adding %s CIDR to NAT chain", cidr) allCIDRs = append(allCIDRs, snatCIDR{cidr: cidr, isExclusion: false}) } for _, cidr := range n.excludeSNATCIDRs { From 388358be43270529b7aa018276c9ce3ee0037d48 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Tue, 20 Apr 2021 10:11:01 -0700 Subject: [PATCH 4/5] Scope down Metrics Server Pod list call by labels --- cmd/cni-metrics-helper/metrics/pod_watcher.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/cni-metrics-helper/metrics/pod_watcher.go b/cmd/cni-metrics-helper/metrics/pod_watcher.go index b0b459a6d2..84e101997f 100644 --- a/cmd/cni-metrics-helper/metrics/pod_watcher.go +++ b/cmd/cni-metrics-helper/metrics/pod_watcher.go @@ -2,7 +2,6 @@ package metrics import ( "context" - "strings" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,9 +32,15 @@ func NewDefaultPodWatcher(k8sClient client.Client, log logger.Logger) *defaultPo func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error) { var CNIPods []string var podList corev1.PodList + labelSelector, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": "aws-node", + }, + }) + listOptions := client.ListOptions{ Namespace: metav1.NamespaceSystem, - Limit: 500, + LabelSelector: labelSelector, } err := d.k8sClient.List(ctx, &podList, &listOptions) @@ -44,11 +49,9 @@ func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error) { } for _, pod := range podList.Items { - if strings.HasPrefix(pod.Name, "aws-node") { - CNIPods = append(CNIPods, pod.Name) - } + CNIPods = append(CNIPods, pod.Name) } - d.log.Debugf("Total aws-node pod count:- ", len(CNIPods)) + d.log.Infof("Total aws-node pod count:- ", len(CNIPods)) return CNIPods, nil } From cca12100d093e3be786e0eacb1b9b1d300d14147 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Tue, 20 Apr 2021 23:49:25 -0700 Subject: [PATCH 5/5] Scope down Metrics Server Pod list call by labels --- cmd/cni-metrics-helper/metrics/pod_watcher.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/cni-metrics-helper/metrics/pod_watcher.go b/cmd/cni-metrics-helper/metrics/pod_watcher.go index 84e101997f..0ea1242953 100644 --- a/cmd/cni-metrics-helper/metrics/pod_watcher.go +++ b/cmd/cni-metrics-helper/metrics/pod_watcher.go @@ -32,18 +32,22 @@ func NewDefaultPodWatcher(k8sClient client.Client, log logger.Logger) *defaultPo func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error) { var CNIPods []string var podList corev1.PodList - labelSelector, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + labelSelector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{ "k8s-app": "aws-node", }, }) + if err != nil { + panic(err.Error()) + } + listOptions := client.ListOptions{ Namespace: metav1.NamespaceSystem, LabelSelector: labelSelector, } - err := d.k8sClient.List(ctx, &podList, &listOptions) + err = d.k8sClient.List(ctx, &podList, &listOptions) if err != nil { return CNIPods, err }