From 5b133a1b2e13004386f28a454013a1680158d879 Mon Sep 17 00:00:00 2001 From: Satish Matti Date: Fri, 20 Dec 2019 16:14:46 -0800 Subject: [PATCH 1/4] Add Ingress usage metrics --- pkg/context/context.go | 6 + pkg/controller/controller.go | 12 + pkg/metrics/features.go | 237 ++++++++++++ pkg/metrics/metrics.go | 259 +++++++++++++ pkg/metrics/metrics_test.go | 708 +++++++++++++++++++++++++++++++++++ pkg/metrics/types.go | 37 ++ 6 files changed, 1259 insertions(+) create mode 100644 pkg/metrics/features.go create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/metrics/metrics_test.go create mode 100644 pkg/metrics/types.go diff --git a/pkg/context/context.go b/pkg/context/context.go index 97e4f9e9ea..d0e2127f10 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -40,6 +40,7 @@ import ( "k8s.io/ingress-gce/pkg/common/typed" frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned" informerfrontendconfig "k8s.io/ingress-gce/pkg/frontendconfig/client/informers/externalversions/frontendconfig/v1beta1" + "k8s.io/ingress-gce/pkg/metrics" "k8s.io/ingress-gce/pkg/utils" "k8s.io/ingress-gce/pkg/utils/namer" "k8s.io/klog" @@ -75,6 +76,8 @@ type ControllerContext struct { DestinationRuleInformer cache.SharedIndexInformer ConfigMapInformer cache.SharedIndexInformer + ControllerMetrics *metrics.ControllerMetrics + healthChecks map[string]func() error lock sync.Mutex @@ -114,6 +117,7 @@ func NewControllerContext( Cloud: cloud, ClusterNamer: namer, KubeSystemUID: kubeSystemUID, + ControllerMetrics: metrics.NewControllerMetrics(), ControllerContextConfig: config, IngressInformer: informerv1beta1.NewIngressInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()), ServiceInformer: informerv1.NewServiceInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()), @@ -293,6 +297,8 @@ func (ctx *ControllerContext) Start(stopCh chan struct{}) { if ctx.EnableASMConfigMap && ctx.ConfigMapInformer != nil { go ctx.ConfigMapInformer.Run(stopCh) } + // Export ingress usage metrics. + go ctx.ControllerMetrics.Run(stopCh) } // Ingresses returns the store of Ingresses. diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 5a2f49c2fe..d0fc01640f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -45,6 +45,7 @@ import ( "k8s.io/ingress-gce/pkg/healthchecks" "k8s.io/ingress-gce/pkg/instances" "k8s.io/ingress-gce/pkg/loadbalancers" + "k8s.io/ingress-gce/pkg/metrics" negtypes "k8s.io/ingress-gce/pkg/neg/types" ingsync "k8s.io/ingress-gce/pkg/sync" "k8s.io/ingress-gce/pkg/tls" @@ -89,6 +90,9 @@ type LoadBalancerController struct { // Ingress sync + GC implementation ingSyncer ingsync.Syncer + + // Ingress usage metrics. + metrics metrics.IngressMetricsCollector } // NewLoadBalancerController creates a controller for gce loadbalancers. @@ -119,6 +123,7 @@ func NewLoadBalancerController( backendSyncer: backends.NewBackendSyncer(backendPool, healthChecker, ctx.Cloud), negLinker: backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud), igLinker: backends.NewInstanceGroupLinker(instancePool, backendPool), + metrics: ctx.ControllerMetrics, } lbc.ingSyncer = ingsync.NewIngressSyncer(&lbc) @@ -531,6 +536,10 @@ func (lbc *LoadBalancerController) sync(key string) error { if err != nil && ingExists { lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "GC", fmt.Sprintf("Error during GC: %v", err)) } + // Delete the ingress state for metrics after GC is successful. + if err == nil && ingExists { + lbc.metrics.DeleteIngress(key) + } return err } @@ -555,6 +564,9 @@ func (lbc *LoadBalancerController) sync(key string) error { syncErr := lbc.ingSyncer.Sync(syncState) if syncErr != nil { lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "Sync", fmt.Sprintf("Error during sync: %v", syncErr.Error())) + } else { + // Insert/update the ingress state for metrics after successful sync. + lbc.metrics.SetIngress(key, metrics.NewIngressState(ing, urlMap.AllServicePorts())) } // Garbage collection will occur regardless of an error occurring. If an error occurred, diff --git a/pkg/metrics/features.go b/pkg/metrics/features.go new file mode 100644 index 0000000000..8e2be96ebd --- /dev/null +++ b/pkg/metrics/features.go @@ -0,0 +1,237 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "strconv" + + "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/klog" +) + +type feature string + +func (f feature) String() string { + return string(f) +} + +const ( + // WARNING: Please keep the following constants in sync with + // pkg/annotations/ingress.go + // allowHTTPKey tells the Ingress controller to allow/block HTTP access. + allowHTTPKey = "kubernetes.io/ingress.allow-http" + ingressClassKey = "kubernetes.io/ingress.class" + gceIngressClass = "gce" + gceMultiIngressClass = "gce-multi-cluster" + gceL7ILBIngressClass = "gce-internal" + // preSharedCertKey represents the specific pre-shared SSL + // certificate for the Ingress controller to use. + preSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert" + managedCertKey = "networking.gke.io/managed-certificates" + // staticIPKey is the annotation key used by controller to record GCP static ip. + staticIPKey = "ingress.kubernetes.io/static-ip" + + ingress = feature("Ingress") + externalIngress = feature("ExternalIngress") + internalIngress = feature("InternalIngress") + httpEnabled = feature("HTTPEnabled") + hostBasedRouting = feature("HostBasedRouting") + pathBasedRouting = feature("PathBasedRouting") + tlsTermination = feature("TLSTermination") + secretBasedCertsForTLS = feature("SecretBasedCertsForTLS") + preSharedCertsForTLS = feature("PreSharedCertsForTLS") + managedCertsForTLS = feature("ManagedCertsForTLS") + staticGlobalIP = feature("StaticGlobalIP") + + servicePort = feature("L7LBServicePort") + externalServicePort = feature("L7XLBServicePort") + internalServicePort = feature("L7ILBServicePort") + neg = feature("NEG") + cloudCDN = feature("CloudCDN") + cloudArmor = feature("CloudArmor") + cloudIAP = feature("CloudIAP") + backendTimeout = feature("BackendTimeout") + backendConnectionDraining = feature("BackendConnectionDraining") + clientIPAffinity = feature("ClientIPAffinity") + cookieAffinity = feature("CookieAffinity") + customRequestHeaders = feature("CustomRequestHeaders") +) + +// featuresForIngress returns the list of features for given ingress. +func featuresForIngress(ing *v1beta1.Ingress) []feature { + features := []feature{ingress} + + ingKey := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) + klog.V(4).Infof("Listing features for Ingress %s", ingKey) + ingAnnotations := ing.Annotations + + // Determine the type of ingress based on ingress class. + ingClass := ingAnnotations[ingressClassKey] + klog.V(6).Infof("Ingress class value for ingress %s: %s", ingKey, ingClass) + switch ingClass { + case "", gceIngressClass, gceMultiIngressClass: + features = append(features, externalIngress) + case gceL7ILBIngressClass: + features = append(features, internalIngress) + } + + // Determine if http is enabled. + if val, ok := ingAnnotations[allowHTTPKey]; !ok { + klog.V(6).Infof("Annotation %s does not exist for ingress %s", allowHTTPKey, ingKey) + features = append(features, httpEnabled) + } else { + klog.V(6).Infof("User specified value for annotation %s on ingress %s: %s", allowHTTPKey, ingKey, val) + v, err := strconv.ParseBool(val) + if err != nil { + klog.Errorf("Failed to parse %s for annotation %s on ingress %s", val, allowHTTPKey, ingKey) + } + if err == nil && v { + features = append(features, httpEnabled) + } + } + + // An ingress without a host or http-path is ignored. + hostBased, pathBased := false, false + if len(ing.Spec.Rules) == 0 { + klog.V(6).Infof("Neither host-based nor path-based routing rules are setup for ingress %s", ingKey) + } + for _, rule := range ing.Spec.Rules { + if rule.HTTP != nil && len(rule.HTTP.Paths) > 0 { + klog.V(6).Infof("User specified http paths for ingress %s: %v", ingKey, rule.HTTP.Paths) + pathBased = true + } + if rule.Host != "" { + klog.V(6).Infof("User specified host for ingress %s: %v", ingKey, rule.Host) + hostBased = true + } + if pathBased && hostBased { + break + } + } + if hostBased { + features = append(features, hostBasedRouting) + } + if pathBased { + features = append(features, pathBasedRouting) + } + + // SSL certificate based features. + sslConfigured := false + if val, ok := ingAnnotations[preSharedCertKey]; ok { + klog.V(6).Infof("Specified pre-shared certs for ingress %s: %v", ingKey, val) + sslConfigured = true + features = append(features, preSharedCertsForTLS) + } + if val, ok := ingAnnotations[managedCertKey]; ok { + klog.V(6).Infof("Specified google managed certs for ingress %s: %v", ingKey, val) + sslConfigured = true + features = append(features, managedCertsForTLS) + } + if hasSecretBasedCerts(ing) { + sslConfigured = true + features = append(features, secretBasedCertsForTLS) + } + if sslConfigured { + klog.V(6).Infof("TLS termination is configured for ingress %s", ingKey) + features = append(features, tlsTermination) + } + + // Both user specified and ingress controller managed global static ips are reported. + if val, ok := ingAnnotations[staticIPKey]; ok && val != "" { + klog.V(6).Infof("Specified static for ingress %s: %s", ingKey, val) + features = append(features, staticGlobalIP) + } + klog.V(4).Infof("Features for ingress %s/%s: %v", ing.Namespace, ing.Name, features) + return features +} + +// hasSecretBasedCerts returns true if ingress spec contains a secret based cert. +func hasSecretBasedCerts(ing *v1beta1.Ingress) bool { + for _, tlsSecret := range ing.Spec.TLS { + if tlsSecret.SecretName == "" { + continue + } + klog.V(6).Infof("User specified secret for ingress %s/%s: %s", ing.Namespace, ing.Name, tlsSecret.SecretName) + return true + } + return false +} + +// featuresForServicePort returns the list of features for given service port. +func featuresForServicePort(sp utils.ServicePort) []feature { + features := []feature{servicePort} + svcPortKey := newServicePortKey(sp).string() + klog.V(4).Infof("Listing features for service port %s", svcPortKey) + if sp.L7ILBEnabled { + klog.V(6).Infof("L7 ILB is enabled for service port %s", svcPortKey) + features = append(features, internalServicePort) + } else { + features = append(features, externalServicePort) + } + if sp.NEGEnabled { + klog.V(6).Infof("NEG is enabled for service port %s", svcPortKey) + features = append(features, neg) + } + if sp.BackendConfig == nil { + klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features) + return features + } + + beConfig := fmt.Sprintf("%s/%s", sp.BackendConfig.Namespace, sp.BackendConfig.Name) + klog.V(6).Infof("Backend config specified for service port %s: %s", svcPortKey, beConfig) + + if sp.BackendConfig.Spec.Cdn != nil && sp.BackendConfig.Spec.Cdn.Enabled { + klog.V(6).Infof("Cloud CDN is enabled for service port %s", svcPortKey) + features = append(features, cloudCDN) + } + if sp.BackendConfig.Spec.Iap != nil && sp.BackendConfig.Spec.Iap.Enabled { + klog.V(6).Infof("Cloud IAP is enabled for service port %s", svcPortKey) + features = append(features, cloudIAP) + } + // Possible list of Affinity types: + // NONE, CLIENT_IP, GENERATED_COOKIE, CLIENT_IP_PROTO, or CLIENT_IP_PORT_PROTO. + if sp.BackendConfig.Spec.SessionAffinity != nil { + affinityType := sp.BackendConfig.Spec.SessionAffinity.AffinityType + switch affinityType { + case "GENERATED_COOKIE": + features = append(features, cookieAffinity) + case "CLIENT_IP", "CLIENT_IP_PROTO", "CLIENT_IP_PORT_PROTO": + features = append(features, clientIPAffinity) + } + klog.V(6).Infof("Session affinity %s is configured for service port %s", affinityType, svcPortKey) + } + if sp.BackendConfig.Spec.SecurityPolicy != nil { + klog.V(6).Infof("Security policy %s is configured for service port %s", sp.BackendConfig.Spec.SecurityPolicy, svcPortKey) + features = append(features, cloudArmor) + } + if sp.BackendConfig.Spec.TimeoutSec != nil { + klog.V(6).Infof("Backend timeout(%v secs) is configured for service port %s", sp.BackendConfig.Spec.TimeoutSec, svcPortKey) + features = append(features, backendTimeout) + } + if sp.BackendConfig.Spec.ConnectionDraining != nil { + klog.V(6).Infof("Backend connection draining(%v secs) is configured for service port %s", sp.BackendConfig.Spec.ConnectionDraining.DrainingTimeoutSec, svcPortKey) + features = append(features, backendConnectionDraining) + } + if sp.BackendConfig.Spec.CustomRequestHeaders != nil { + klog.V(6).Infof("Custom request headers configured for service port %s: %v", svcPortKey, sp.BackendConfig.Spec.CustomRequestHeaders.Headers) + features = append(features, customRequestHeaders) + } + klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features) + return features +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000000..83bc7569ef --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,259 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/klog" +) + +const ( + label = "Feature" +) + +var ( + metricsInterval = 10 * time.Minute + ingressCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "number_of_ingresses", + Help: "Number of Ingresses", + }, + []string{label}, + ) + servicePortCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "number_of_service_ports", + Help: "Number of Service Ports", + }, + []string{label}, + ) +) + +// init registers ingress usage metrics. +func init() { + klog.V(3).Infof("Registering Ingress usage metrics %v and %v", ingressCount, servicePortCount) + prometheus.MustRegister(ingressCount, servicePortCount) +} + +// NewIngressState returns ingress state for given ingress and service ports. +func NewIngressState(ing *v1beta1.Ingress, svcPorts []utils.ServicePort) IngressState { + return IngressState{ingress: ing, servicePorts: svcPorts} +} + +// ControllerMetrics contains the state of the all ingresses. +type ControllerMetrics struct { + ingressMap map[string]IngressState + sync.Mutex +} + +// NewControllerMetrics initializes ControllerMetrics and starts a go routine to compute and export metrics periodically. +func NewControllerMetrics() *ControllerMetrics { + return &ControllerMetrics{ingressMap: make(map[string]IngressState)} +} + +// servicePortKey defines a service port uniquely. +// Note that same service port combination used by ILB and XLB are treated as separate service ports. +type servicePortKey struct { + svcPortID utils.ServicePortID + isL7ILBEnabled bool +} + +func newServicePortKey(svcPort utils.ServicePort) servicePortKey { + return servicePortKey{svcPortID: svcPort.ID, isL7ILBEnabled: svcPort.L7ILBEnabled} +} + +func (spk servicePortKey) string() string { + if spk.isL7ILBEnabled { + return fmt.Sprintf("%s/%s", spk.svcPortID, "ILB") + } + return fmt.Sprintf("%s/%s", spk.svcPortID, "XLB") +} + +func (im *ControllerMetrics) Run(stopCh <-chan struct{}) { + klog.V(3).Infof("Ingress Metrics initialized. Metrics will be exported at an interval of %v", metricsInterval) + // Compute and export metrics periodically. + go func() { + // Wait for ingress states to be populated in the cache before computing metrics. + time.Sleep(metricsInterval) + wait.Until(im.export, metricsInterval, stopCh) + }() + <-stopCh +} + +// SetIngress implements ControllerMetrics. +func (im *ControllerMetrics) SetIngress(ingKey string, ing IngressState) { + im.Lock() + defer im.Unlock() + + if im.ingressMap == nil { + klog.Fatalf("Ingress Metrics failed to initialize correctly.") + } + im.ingressMap[ingKey] = ing +} + +// DeleteIngress implements ControllerMetrics. +func (im *ControllerMetrics) DeleteIngress(ingKey string) { + im.Lock() + defer im.Unlock() + + delete(im.ingressMap, ingKey) +} + +// export computes and exports ingress usage metrics. +func (im *ControllerMetrics) export() { + ingCount, svcPortCount := im.computeMetrics() + + klog.V(3).Infof("Exporting ingress usage metrics. Ingress Count: %#v, Service Port count: %#v", ingCount, svcPortCount) + for feature, count := range ingCount { + ingressCount.With(prometheus.Labels{label: feature.String()}).Set(float64(count)) + } + + for feature, count := range svcPortCount { + servicePortCount.With(prometheus.Labels{label: feature.String()}).Set(float64(count)) + } + klog.V(3).Infof("Ingress usage metrics exported.") +} + +// computeMetrics traverses all ingresses and computes, +// 1. Count of GCE ingresses for each feature. +// 2. Count of service-port pairs that backs up a GCE ingress for each feature. +func (im *ControllerMetrics) computeMetrics() (map[feature]int, map[feature]int) { + ingCount, svcPortCount := initializeCounts() + // servicePortFeatures tracks the list of service-ports and their features. + // This is to avoid re-computing features for a service-port. + svcPortFeatures := make(map[servicePortKey][]feature) + klog.V(4).Infof("Computing Ingress usage metrics from ingress state map: %#v", im.ingressMap) + im.Lock() + defer im.Unlock() + + for ingKey, ingState := range im.ingressMap { + // Both frontend and backend associated ingress features are tracked. + currIngFeatures := make(map[feature]bool) + klog.V(6).Infof("Computing frontend based features for ingress %s", ingKey) + // Add frontend associated ingress features. + for _, feature := range featuresForIngress(ingState.ingress) { + currIngFeatures[feature] = true + } + klog.V(6).Infof("Frontend based features for ingress %s: %v", ingKey, currIngFeatures) + klog.V(6).Infof("Computing backend based features for ingress %s", ingKey) + for _, svcPort := range ingState.servicePorts { + svcPortKey := newServicePortKey(svcPort) + klog.V(6).Infof("Computing features for service-port %s", svcPortKey.string()) + svcFeatures, ok := svcPortFeatures[svcPortKey] + if !ok { + svcFeatures = featuresForServicePort(svcPort) + } + // Add backend associated ingress features. + for _, sf := range svcFeatures { + // Ignore features specific to the service. + if !isServiceFeature(sf) { + currIngFeatures[sf] = true + } + } + if ok { + // Skip re-computing features for a service port. + klog.V(4).Infof("Features for service port %s exists, skipping.", svcPortKey.string()) + continue + } + svcPortFeatures[svcPortKey] = svcFeatures + klog.V(6).Infof("Features for service port %s: %v", svcPortKey.string(), svcFeatures) + // Update service port feature counts. + updateServicePortCount(svcPortCount, svcFeatures) + } + klog.V(6).Infof("Features for ingress %s: %v", ingKey, currIngFeatures) + // Merge current ingress to update ingress feature counts. + updateIngressCount(ingCount, currIngFeatures) + } + + klog.V(4).Infof("Ingress usage metrics computed.") + return ingCount, svcPortCount +} + +// initializeCounts initializes feature count maps for ingress and service ports. +// This is required in order to reset counts for features that do not exist now +// but existed before. +func initializeCounts() (map[feature]int, map[feature]int) { + return map[feature]int{ + ingress: 0, + externalIngress: 0, + internalIngress: 0, + httpEnabled: 0, + hostBasedRouting: 0, + pathBasedRouting: 0, + tlsTermination: 0, + secretBasedCertsForTLS: 0, + preSharedCertsForTLS: 0, + managedCertsForTLS: 0, + staticGlobalIP: 0, + neg: 0, + cloudCDN: 0, + cloudArmor: 0, + cloudIAP: 0, + backendTimeout: 0, + backendConnectionDraining: 0, + clientIPAffinity: 0, + cookieAffinity: 0, + customRequestHeaders: 0, + }, + // service port counts + map[feature]int{ + servicePort: 0, + externalServicePort: 0, + internalServicePort: 0, + neg: 0, + cloudCDN: 0, + cloudArmor: 0, + cloudIAP: 0, + backendTimeout: 0, + backendConnectionDraining: 0, + clientIPAffinity: 0, + cookieAffinity: 0, + customRequestHeaders: 0, + } +} + +// updateServicePortCount inserts/increments service port counts by 1 for given features. +func updateServicePortCount(svcPortCount map[feature]int, features []feature) { + for _, feature := range features { + svcPortCount[feature] += 1 + } +} + +// updateIngressCount inserts/increments ingress counts by 1 for given feature map. +func updateIngressCount(ingCount map[feature]int, features map[feature]bool) { + for feature := range features { + ingCount[feature] += 1 + } +} + +// isServiceFeature returns true if given feature applies only to service-port but +// not to the ingress that references this service-port. +func isServiceFeature(ftr feature) bool { + serviceFeatures := map[feature]bool{ + servicePort: true, + externalServicePort: true, + internalServicePort: true, + } + return serviceFeatures[ftr] +} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 0000000000..27e3bf6590 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,708 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + backendconfigv1beta1 "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1" + "k8s.io/ingress-gce/pkg/utils" +) + +var ( + testTTL = int64(10) + defaultNamespace = "default" + testServicePorts = []utils.ServicePort{ + { + ID: utils.ServicePortID{ + Service: types.NamespacedName{ + Name: "dummy-service", + Namespace: defaultNamespace, + }, + Port: intstr.FromInt(80), + }, + BackendConfig: &backendconfigv1beta1.BackendConfig{ + Spec: backendconfigv1beta1.BackendConfigSpec{ + Cdn: &backendconfigv1beta1.CDNConfig{ + Enabled: true, + CachePolicy: &backendconfigv1beta1.CacheKeyPolicy{}, + }, + SessionAffinity: &backendconfigv1beta1.SessionAffinityConfig{ + AffinityType: "GENERATED_COOKIE", + AffinityCookieTtlSec: &testTTL, + }, + SecurityPolicy: &backendconfigv1beta1.SecurityPolicyConfig{ + Name: "security-policy-1", + }, + ConnectionDraining: &backendconfigv1beta1.ConnectionDrainingConfig{ + DrainingTimeoutSec: testTTL, + }, + }, + }, + }, + { + ID: utils.ServicePortID{ + Service: types.NamespacedName{ + Name: "foo-service", + Namespace: defaultNamespace, + }, + Port: intstr.FromInt(80), + }, + NEGEnabled: true, + BackendConfig: &backendconfigv1beta1.BackendConfig{ + Spec: backendconfigv1beta1.BackendConfigSpec{ + Iap: &backendconfigv1beta1.IAPConfig{ + Enabled: true, + }, + SessionAffinity: &backendconfigv1beta1.SessionAffinityConfig{ + AffinityType: "CLIENT_IP", + AffinityCookieTtlSec: &testTTL, + }, + TimeoutSec: &testTTL, + CustomRequestHeaders: &backendconfigv1beta1.CustomRequestHeadersConfig{ + Headers: []string{}, + }, + }, + }, + }, + // NEG default backend. + { + ID: utils.ServicePortID{ + Service: types.NamespacedName{ + Name: "dummy-service", + Namespace: defaultNamespace, + }, + Port: intstr.FromInt(80), + }, + NEGEnabled: true, + L7ILBEnabled: true, + }, + { + ID: utils.ServicePortID{ + Service: types.NamespacedName{ + Name: "bar-service", + Namespace: defaultNamespace, + }, + Port: intstr.FromInt(5000), + }, + NEGEnabled: true, + L7ILBEnabled: true, + BackendConfig: &backendconfigv1beta1.BackendConfig{ + Spec: backendconfigv1beta1.BackendConfigSpec{ + Iap: &backendconfigv1beta1.IAPConfig{ + Enabled: true, + }, + SessionAffinity: &backendconfigv1beta1.SessionAffinityConfig{ + AffinityType: "GENERATED_COOKIE", + AffinityCookieTtlSec: &testTTL, + }, + ConnectionDraining: &backendconfigv1beta1.ConnectionDrainingConfig{ + DrainingTimeoutSec: testTTL, + }, + }, + }, + }, + } + ingressStates = []struct { + desc string + ing *v1beta1.Ingress + frontendFeatures []feature + svcPorts []utils.ServicePort + backendFeatures []feature + }{ + { + "empty spec", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress0", + }, + }, + []feature{ingress, externalIngress, httpEnabled}, + []utils.ServicePort{}, + nil, + }, + { + "http disabled", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress1", + Annotations: map[string]string{ + allowHTTPKey: "false"}, + }, + }, + []feature{ingress, externalIngress}, + []utils.ServicePort{}, + nil, + }, + { + "default backend", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress2", + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{}, + }, + }, + []feature{ingress, externalIngress, httpEnabled}, + []utils.ServicePort{testServicePorts[0]}, + []feature{servicePort, externalServicePort, cloudCDN, + cookieAffinity, cloudArmor, backendConnectionDraining}, + }, + { + "host rule only", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress3", + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "foo.bar", + }, + }, + }, + }, + []feature{ingress, externalIngress, httpEnabled, hostBasedRouting}, + []utils.ServicePort{}, + nil, + }, + { + "both host and path rules", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress4", + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "foo.bar", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/foo", + Backend: v1beta1.IngressBackend{ + ServiceName: "foo-service", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + }, + []feature{ingress, externalIngress, httpEnabled, + hostBasedRouting, pathBasedRouting}, + []utils.ServicePort{testServicePorts[1]}, + []feature{servicePort, externalServicePort, neg, cloudIAP, + clientIPAffinity, backendTimeout, customRequestHeaders}, + }, + { + "default backend and host rule", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress5", + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{ + { + Host: "foo.bar", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/foo", + Backend: v1beta1.IngressBackend{ + ServiceName: "foo-service", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + }, + []feature{ingress, externalIngress, httpEnabled, + hostBasedRouting, pathBasedRouting}, + testServicePorts[:2], + []feature{servicePort, externalServicePort, cloudCDN, + cookieAffinity, cloudArmor, backendConnectionDraining, neg, cloudIAP, + clientIPAffinity, backendTimeout, customRequestHeaders}, + }, + { + "tls termination with pre-shared certs", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress6", + Annotations: map[string]string{ + preSharedCertKey: "pre-shared-cert1,pre-shared-cert2", + }, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{}, + }, + }, + []feature{ingress, externalIngress, httpEnabled, + preSharedCertsForTLS, tlsTermination}, + []utils.ServicePort{testServicePorts[0]}, + []feature{servicePort, externalServicePort, cloudCDN, + cookieAffinity, cloudArmor, backendConnectionDraining}, + }, + { + "tls termination with google managed certs", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress7", + Annotations: map[string]string{ + managedCertKey: "managed-cert1,managed-cert2", + }, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{}, + }, + }, + []feature{ingress, externalIngress, httpEnabled, + managedCertsForTLS, tlsTermination}, + []utils.ServicePort{testServicePorts[0]}, + []feature{servicePort, externalServicePort, cloudCDN, + cookieAffinity, cloudArmor, backendConnectionDraining}, + }, + { + "tls termination with pre-shared and google managed certs", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress8", + Annotations: map[string]string{ + preSharedCertKey: "pre-shared-cert1,pre-shared-cert2", + managedCertKey: "managed-cert1,managed-cert2", + }, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{}, + }, + }, + []feature{ingress, externalIngress, httpEnabled, + preSharedCertsForTLS, managedCertsForTLS, tlsTermination}, + []utils.ServicePort{testServicePorts[0]}, + []feature{servicePort, externalServicePort, cloudCDN, + cookieAffinity, cloudArmor, backendConnectionDraining}, + }, + { + "tls termination with pre-shared and secret based certs", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress9", + Annotations: map[string]string{ + preSharedCertKey: "pre-shared-cert1,pre-shared-cert2", + }, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "foo.bar", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/foo", + Backend: v1beta1.IngressBackend{ + ServiceName: "foo-service", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + TLS: []v1beta1.IngressTLS{ + { + Hosts: []string{"foo.bar"}, + SecretName: "secret-1", + }, + }, + }, + }, + []feature{ingress, externalIngress, httpEnabled, hostBasedRouting, + pathBasedRouting, preSharedCertsForTLS, secretBasedCertsForTLS, tlsTermination}, + []utils.ServicePort{testServicePorts[1]}, + []feature{servicePort, externalServicePort, neg, cloudIAP, + clientIPAffinity, backendTimeout, customRequestHeaders}, + }, + { + "global static ip", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress10", + Annotations: map[string]string{ + preSharedCertKey: "pre-shared-cert1,pre-shared-cert2", + staticIPKey: "10.0.1.2", + }, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{}, + }, + }, + []feature{ingress, externalIngress, httpEnabled, + preSharedCertsForTLS, tlsTermination, staticGlobalIP}, + []utils.ServicePort{testServicePorts[0]}, + []feature{servicePort, externalServicePort, cloudCDN, + cookieAffinity, cloudArmor, backendConnectionDraining}, + }, + { + "default backend, host rule for internal load-balancer", + &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Namespace: defaultNamespace, + Name: "ingress11", + Annotations: map[string]string{ + ingressClassKey: gceL7ILBIngressClass, + }, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "dummy-service", + ServicePort: intstr.FromInt(80), + }, + Rules: []v1beta1.IngressRule{ + { + Host: "bar", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/bar", + Backend: v1beta1.IngressBackend{ + ServiceName: "bar-service", + ServicePort: intstr.FromInt(5000), + }, + }, + }, + }, + }, + }, + }, + }, + }, + []feature{ingress, internalIngress, httpEnabled, + hostBasedRouting, pathBasedRouting}, + []utils.ServicePort{testServicePorts[2], testServicePorts[3]}, + []feature{servicePort, internalServicePort, neg, cloudIAP, + cookieAffinity, backendConnectionDraining}, + }, + } +) + +func TestFeaturesForIngress(t *testing.T) { + t.Parallel() + for _, tc := range ingressStates { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + gotFrontendFeatures := featuresForIngress(tc.ing) + if diff := cmp.Diff(tc.frontendFeatures, gotFrontendFeatures); diff != "" { + t.Fatalf("Got diff for frontend features (-want +got):\n%s", diff) + } + }) + } +} + +func TestFeaturesForServicePort(t *testing.T) { + t.Parallel() + for _, tc := range ingressStates { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + backendFeatureMap := make(map[feature]bool) + var gotBackendFeatures []feature + for _, svcPort := range tc.svcPorts { + for _, feature := range featuresForServicePort(svcPort) { + if backendFeatureMap[feature] { + continue + } + backendFeatureMap[feature] = true + gotBackendFeatures = append(gotBackendFeatures, feature) + } + } + if diff := cmp.Diff(tc.backendFeatures, gotBackendFeatures); diff != "" { + t.Fatalf("Got diff for backend features (-want +got):\n%s", diff) + } + }) + } +} + +func TestComputeMetrics(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + desc string + ingressStates []IngressState + expectIngressCount map[feature]int + expectSvcPortCount map[feature]int + }{ + { + "frontend features only", + []IngressState{ + NewIngressState(ingressStates[0].ing, ingressStates[0].svcPorts), + NewIngressState(ingressStates[1].ing, ingressStates[1].svcPorts), + NewIngressState(ingressStates[3].ing, ingressStates[3].svcPorts), + }, + map[feature]int{ + backendConnectionDraining: 0, + backendTimeout: 0, + clientIPAffinity: 0, + cloudArmor: 0, + cloudCDN: 0, + cloudIAP: 0, + cookieAffinity: 0, + customRequestHeaders: 0, + externalIngress: 3, + httpEnabled: 2, + hostBasedRouting: 1, + ingress: 3, + internalIngress: 0, + managedCertsForTLS: 0, + neg: 0, + pathBasedRouting: 0, + preSharedCertsForTLS: 0, + secretBasedCertsForTLS: 0, + staticGlobalIP: 0, + tlsTermination: 0, + }, + map[feature]int{ + backendConnectionDraining: 0, + backendTimeout: 0, + clientIPAffinity: 0, + cloudArmor: 0, + cloudCDN: 0, + cloudIAP: 0, + cookieAffinity: 0, + customRequestHeaders: 0, + internalServicePort: 0, + servicePort: 0, + externalServicePort: 0, + neg: 0, + }, + }, + { + "features for internal and external load-balancers", + []IngressState{ + NewIngressState(ingressStates[0].ing, ingressStates[0].svcPorts), + NewIngressState(ingressStates[1].ing, ingressStates[1].svcPorts), + NewIngressState(ingressStates[3].ing, ingressStates[3].svcPorts), + NewIngressState(ingressStates[11].ing, ingressStates[11].svcPorts), + }, + map[feature]int{ + backendConnectionDraining: 1, + backendTimeout: 0, + clientIPAffinity: 0, + cloudArmor: 0, + cloudCDN: 0, + cloudIAP: 1, + cookieAffinity: 1, + customRequestHeaders: 0, + externalIngress: 3, + httpEnabled: 3, + hostBasedRouting: 2, + ingress: 4, + internalIngress: 1, + managedCertsForTLS: 0, + neg: 1, + pathBasedRouting: 1, + preSharedCertsForTLS: 0, + secretBasedCertsForTLS: 0, + staticGlobalIP: 0, + tlsTermination: 0, + }, + map[feature]int{ + backendConnectionDraining: 1, + backendTimeout: 0, + clientIPAffinity: 0, + cloudArmor: 0, + cloudCDN: 0, + cloudIAP: 1, + cookieAffinity: 1, + customRequestHeaders: 0, + internalServicePort: 2, + servicePort: 2, + externalServicePort: 0, + neg: 2, + }, + }, + { + "frontend and backend features", + []IngressState{ + NewIngressState(ingressStates[2].ing, ingressStates[2].svcPorts), + NewIngressState(ingressStates[4].ing, ingressStates[4].svcPorts), + NewIngressState(ingressStates[6].ing, ingressStates[6].svcPorts), + NewIngressState(ingressStates[8].ing, ingressStates[8].svcPorts), + NewIngressState(ingressStates[10].ing, ingressStates[10].svcPorts), + }, + map[feature]int{ + backendConnectionDraining: 4, + backendTimeout: 1, + clientIPAffinity: 1, + cloudArmor: 4, + cloudCDN: 4, + cloudIAP: 1, + cookieAffinity: 4, + customRequestHeaders: 1, + externalIngress: 5, + httpEnabled: 5, + hostBasedRouting: 1, + ingress: 5, + internalIngress: 0, + managedCertsForTLS: 1, + neg: 1, + pathBasedRouting: 1, + preSharedCertsForTLS: 3, + secretBasedCertsForTLS: 0, + staticGlobalIP: 1, + tlsTermination: 3, + }, + map[feature]int{ + backendConnectionDraining: 1, + backendTimeout: 1, + clientIPAffinity: 1, + cloudArmor: 1, + cloudCDN: 1, + cloudIAP: 1, + cookieAffinity: 1, + customRequestHeaders: 1, + internalServicePort: 0, + servicePort: 2, + externalServicePort: 2, + neg: 1, + }, + }, + { + "all ingress features", + []IngressState{ + NewIngressState(ingressStates[0].ing, ingressStates[0].svcPorts), + NewIngressState(ingressStates[1].ing, ingressStates[1].svcPorts), + NewIngressState(ingressStates[2].ing, ingressStates[2].svcPorts), + NewIngressState(ingressStates[3].ing, ingressStates[3].svcPorts), + NewIngressState(ingressStates[4].ing, ingressStates[4].svcPorts), + NewIngressState(ingressStates[5].ing, ingressStates[5].svcPorts), + NewIngressState(ingressStates[6].ing, ingressStates[6].svcPorts), + NewIngressState(ingressStates[7].ing, ingressStates[7].svcPorts), + NewIngressState(ingressStates[8].ing, ingressStates[8].svcPorts), + NewIngressState(ingressStates[9].ing, ingressStates[9].svcPorts), + NewIngressState(ingressStates[10].ing, ingressStates[10].svcPorts), + NewIngressState(ingressStates[11].ing, ingressStates[11].svcPorts), + }, + map[feature]int{ + backendConnectionDraining: 7, + backendTimeout: 3, + clientIPAffinity: 3, + cloudArmor: 6, + cloudCDN: 6, + cloudIAP: 4, + cookieAffinity: 7, + customRequestHeaders: 3, + externalIngress: 11, + httpEnabled: 11, + hostBasedRouting: 5, + ingress: 12, + internalIngress: 1, + managedCertsForTLS: 2, + neg: 4, + pathBasedRouting: 4, + preSharedCertsForTLS: 4, + secretBasedCertsForTLS: 1, + staticGlobalIP: 1, + tlsTermination: 5, + }, + map[feature]int{ + backendConnectionDraining: 2, + backendTimeout: 1, + clientIPAffinity: 1, + cloudArmor: 1, + cloudCDN: 1, + cloudIAP: 2, + cookieAffinity: 2, + customRequestHeaders: 1, + internalServicePort: 2, + servicePort: 4, + externalServicePort: 2, + neg: 3, + }, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + newMetrics := NewControllerMetrics() + for _, ingState := range tc.ingressStates { + ingKey := fmt.Sprintf("%s/%s", defaultNamespace, ingState.ingress.Name) + newMetrics.SetIngress(ingKey, ingState) + } + gotIngressCount, gotSvcPortCount := newMetrics.computeMetrics() + if diff := cmp.Diff(tc.expectIngressCount, gotIngressCount); diff != "" { + t.Errorf("Got diff for ingress features count (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.expectSvcPortCount, gotSvcPortCount); diff != "" { + t.Fatalf("Got diff for service port features count (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go new file mode 100644 index 0000000000..8fd2d439bb --- /dev/null +++ b/pkg/metrics/types.go @@ -0,0 +1,37 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-gce/pkg/utils" +) + +// IngressState defines an ingress and its associated service ports. +type IngressState struct { + ingress *v1beta1.Ingress + servicePorts []utils.ServicePort +} + +// IngressMetricsCollector is an interface to update/delete ingress states in the cache +// that is used for computing ingress usage metrics. +type IngressMetricsCollector interface { + // SetIngress adds/updates ingress state for given ingress key. + SetIngress(ingKey string, ing IngressState) + // DeleteIngress removes the given ingress key. + DeleteIngress(ingKey string) +} From 19571bb2cbb90b2235e8691d7f5167436c7484ab Mon Sep 17 00:00:00 2001 From: Satish Matti Date: Thu, 30 Jan 2020 17:45:58 -0800 Subject: [PATCH 2/4] Make metrics label Snake case --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 83bc7569ef..c124d082e4 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -29,7 +29,7 @@ import ( ) const ( - label = "Feature" + label = "feature" ) var ( From 352ca2ca83b0ed31bdcc05c7ce2df3765e0a604f Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Wed, 8 Jan 2020 11:59:38 -0800 Subject: [PATCH 3/4] add neg metrics --- pkg/metrics/features.go | 13 +++++-- pkg/metrics/metrics.go | 65 ++++++++++++++++++++++++++++++--- pkg/metrics/metrics_test.go | 73 ++++++++++++++++++++++++++++++++++++- pkg/metrics/types.go | 19 ++++++++++ pkg/neg/controller.go | 16 +++++++- 5 files changed, 172 insertions(+), 14 deletions(-) diff --git a/pkg/metrics/features.go b/pkg/metrics/features.go index 8e2be96ebd..07977ab208 100644 --- a/pkg/metrics/features.go +++ b/pkg/metrics/features.go @@ -59,10 +59,11 @@ const ( managedCertsForTLS = feature("ManagedCertsForTLS") staticGlobalIP = feature("StaticGlobalIP") - servicePort = feature("L7LBServicePort") - externalServicePort = feature("L7XLBServicePort") - internalServicePort = feature("L7ILBServicePort") - neg = feature("NEG") + servicePort = feature("L7LBServicePort") + externalServicePort = feature("L7XLBServicePort") + internalServicePort = feature("L7ILBServicePort") + neg = feature("NEG") + cloudCDN = feature("CloudCDN") cloudArmor = feature("CloudArmor") cloudIAP = feature("CloudIAP") @@ -71,6 +72,10 @@ const ( clientIPAffinity = feature("ClientIPAffinity") cookieAffinity = feature("CookieAffinity") customRequestHeaders = feature("CustomRequestHeaders") + + standaloneNeg = feature("StandaloneNEG") + ingressNeg = feature("IngressNEG") + asmNeg = feature("AsmNEG") ) // featuresForIngress returns the list of features for given ingress. diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index c124d082e4..93ce715aff 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -48,6 +48,13 @@ var ( }, []string{label}, ) + networkEndpointGroupCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "number_of_negs", + Help: "Number of NEGs", + }, + []string{label}, + ) ) // init registers ingress usage metrics. @@ -63,13 +70,16 @@ func NewIngressState(ing *v1beta1.Ingress, svcPorts []utils.ServicePort) Ingress // ControllerMetrics contains the state of the all ingresses. type ControllerMetrics struct { + // ingressMap is a map between ingress key to ingress state ingressMap map[string]IngressState + // negMap is a map between service key to neg state + negMap map[string]NegServiceState sync.Mutex } // NewControllerMetrics initializes ControllerMetrics and starts a go routine to compute and export metrics periodically. func NewControllerMetrics() *ControllerMetrics { - return &ControllerMetrics{ingressMap: make(map[string]IngressState)} + return &ControllerMetrics{ingressMap: make(map[string]IngressState), negMap: make(map[string]NegServiceState)} } // servicePortKey defines a service port uniquely. @@ -101,7 +111,7 @@ func (im *ControllerMetrics) Run(stopCh <-chan struct{}) { <-stopCh } -// SetIngress implements ControllerMetrics. +// SetIngress implements IngressMetricsCollector. func (im *ControllerMetrics) SetIngress(ingKey string, ing IngressState) { im.Lock() defer im.Unlock() @@ -112,7 +122,7 @@ func (im *ControllerMetrics) SetIngress(ingKey string, ing IngressState) { im.ingressMap[ingKey] = ing } -// DeleteIngress implements ControllerMetrics. +// DeleteIngress implements IngressMetricsCollector. func (im *ControllerMetrics) DeleteIngress(ingKey string) { im.Lock() defer im.Unlock() @@ -120,9 +130,29 @@ func (im *ControllerMetrics) DeleteIngress(ingKey string) { delete(im.ingressMap, ingKey) } +// SetIngress implements NegMetricsCollector. +func (im *ControllerMetrics) SetNegService(svcKey string, negState NegServiceState) { + im.Lock() + defer im.Unlock() + + if im.negMap == nil { + klog.Fatalf("Ingress Metrics failed to initialize correctly.") + } + im.negMap[svcKey] = negState +} + +// DeleteIngress implements NegMetricsCollector. +func (im *ControllerMetrics) DeleteNegService(svcKey string) { + im.Lock() + defer im.Unlock() + + delete(im.negMap, svcKey) +} + // export computes and exports ingress usage metrics. func (im *ControllerMetrics) export() { - ingCount, svcPortCount := im.computeMetrics() + ingCount, svcPortCount := im.computeIngressMetrics() + negCount := im.computeNegMetrics() klog.V(3).Infof("Exporting ingress usage metrics. Ingress Count: %#v, Service Port count: %#v", ingCount, svcPortCount) for feature, count := range ingCount { @@ -132,13 +162,17 @@ func (im *ControllerMetrics) export() { for feature, count := range svcPortCount { servicePortCount.With(prometheus.Labels{label: feature.String()}).Set(float64(count)) } + + for feature, count := range negCount { + networkEndpointGroupCount.With(prometheus.Labels{label: feature.String()}).Set(float64(count)) + } klog.V(3).Infof("Ingress usage metrics exported.") } -// computeMetrics traverses all ingresses and computes, +// computeIngressMetrics traverses all ingresses and computes, // 1. Count of GCE ingresses for each feature. // 2. Count of service-port pairs that backs up a GCE ingress for each feature. -func (im *ControllerMetrics) computeMetrics() (map[feature]int, map[feature]int) { +func (im *ControllerMetrics) computeIngressMetrics() (map[feature]int, map[feature]int) { ingCount, svcPortCount := initializeCounts() // servicePortFeatures tracks the list of service-ports and their features. // This is to avoid re-computing features for a service-port. @@ -190,6 +224,25 @@ func (im *ControllerMetrics) computeMetrics() (map[feature]int, map[feature]int) return ingCount, svcPortCount } +// computeNegMetrics aggregates NEG metrics in the cache +func (im *ControllerMetrics) computeNegMetrics() map[feature]int { + counts := map[feature]int{ + standaloneNeg: 0, + ingressNeg: 0, + asmNeg: 0, + neg: 0, + } + + for key, negState := range im.negMap { + klog.V(6).Infof("For service %s, it has standaloneNegs:%v, ingressNegs:%v and asmNeg:%v", key, negState.StandaloneNeg, negState.IngressNeg, negState.AsmNeg) + counts[standaloneNeg] += negState.StandaloneNeg + counts[ingressNeg] += negState.IngressNeg + counts[asmNeg] += negState.AsmNeg + counts[neg] += negState.AsmNeg + negState.StandaloneNeg + negState.IngressNeg + } + return counts +} + // initializeCounts initializes feature count maps for ingress and service ports. // This is required in order to reset counts for features that do not exist now // but existed before. diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 27e3bf6590..7559861c35 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" backendconfigv1beta1 "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1" "k8s.io/ingress-gce/pkg/utils" + "reflect" ) var ( @@ -491,7 +492,7 @@ func TestFeaturesForServicePort(t *testing.T) { } } -func TestComputeMetrics(t *testing.T) { +func TestComputeIngressMetrics(t *testing.T) { t.Parallel() for _, tc := range []struct { desc string @@ -696,7 +697,7 @@ func TestComputeMetrics(t *testing.T) { ingKey := fmt.Sprintf("%s/%s", defaultNamespace, ingState.ingress.Name) newMetrics.SetIngress(ingKey, ingState) } - gotIngressCount, gotSvcPortCount := newMetrics.computeMetrics() + gotIngressCount, gotSvcPortCount := newMetrics.computeIngressMetrics() if diff := cmp.Diff(tc.expectIngressCount, gotIngressCount); diff != "" { t.Errorf("Got diff for ingress features count (-want +got):\n%s", diff) } @@ -706,3 +707,71 @@ func TestComputeMetrics(t *testing.T) { }) } } + +func TestComputeNegMetrics(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + desc string + negStates []NegServiceState + expectNegCount map[feature]int + }{ + { + "empty input", + []NegServiceState{}, + map[feature]int{ + standaloneNeg: 0, + ingressNeg: 0, + asmNeg: 0, + neg: 0, + }, + }, + { + "one neg service", + []NegServiceState{ + newNegState(0, 0, 1), + }, + map[feature]int{ + standaloneNeg: 0, + ingressNeg: 0, + asmNeg: 1, + neg: 1, + }, + }, + { + "many neg services", + []NegServiceState{ + newNegState(0, 0, 1), + newNegState(0, 1, 0), + newNegState(5, 0, 0), + newNegState(5, 3, 2), + }, + map[feature]int{ + standaloneNeg: 10, + ingressNeg: 4, + asmNeg: 3, + neg: 17, + }, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + newMetrics := NewControllerMetrics() + for i, negState := range tc.negStates { + newMetrics.SetNegService(string(i), negState) + } + output := newMetrics.computeNegMetrics() + if !reflect.DeepEqual(output, tc.expectNegCount) { + t.Errorf("For case %q, expect output %v, but got %v", tc.desc, tc.expectNegCount, output) + } + }) + } +} + +func newNegState(standalone, ingress, asm int) NegServiceState { + return NegServiceState{ + IngressNeg: ingress, + StandaloneNeg: standalone, + AsmNeg: asm, + } +} diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index 8fd2d439bb..4fe3d3c75e 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -27,6 +27,16 @@ type IngressState struct { servicePorts []utils.ServicePort } +// NegServiceState contains all the neg usage associated with one service +type NegServiceState struct { + // standaloneNeg is the count of standalone NEG + StandaloneNeg int + // ingressNeg is the count of NEGs created for ingress + IngressNeg int + // asmNeg is the count of NEGs created for ASM + AsmNeg int +} + // IngressMetricsCollector is an interface to update/delete ingress states in the cache // that is used for computing ingress usage metrics. type IngressMetricsCollector interface { @@ -35,3 +45,12 @@ type IngressMetricsCollector interface { // DeleteIngress removes the given ingress key. DeleteIngress(ingKey string) } + +// NegMetricsCollector is an interface to update/delete Neg states in the cache +// that is used for computing neg usage metrics. +type NegMetricsCollector interface { + // SetNegService adds/updates neg state for given service key. + SetNegService(svcKey string, negState NegServiceState) + // DeleteNegService removes the given service key. + DeleteNegService(svcKey string) +} diff --git a/pkg/neg/controller.go b/pkg/neg/controller.go index d865ef9079..975d8ebe01 100644 --- a/pkg/neg/controller.go +++ b/pkg/neg/controller.go @@ -40,6 +40,7 @@ import ( "k8s.io/ingress-gce/pkg/context" "k8s.io/ingress-gce/pkg/controller/translator" "k8s.io/ingress-gce/pkg/flags" + usage "k8s.io/ingress-gce/pkg/metrics" "k8s.io/ingress-gce/pkg/neg/metrics" "k8s.io/ingress-gce/pkg/neg/readiness" negtypes "k8s.io/ingress-gce/pkg/neg/types" @@ -86,6 +87,9 @@ type Controller struct { // reflector handles NEG readiness gate and conditions for pods in NEG. reflector readiness.Reflector + + // collector collects NEG usage metrics + collector usage.NegMetricsCollector } // NewController returns a network endpoint group controller. @@ -133,6 +137,7 @@ func NewController( endpointQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), syncTracker: utils.NewTimeTracker(), reflector: reflector, + collector: ctx.ControllerMetrics, } ctx.IngressInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -317,6 +322,7 @@ func (c *Controller) processService(key string) error { return err } if !exists { + c.collector.DeleteNegService(key) c.manager.StopSyncer(namespace, name) return nil } @@ -325,22 +331,26 @@ func (c *Controller) processService(key string) error { if service == nil { return fmt.Errorf("cannot convert to Service (%T)", obj) } - + negUsage := usage.NegServiceState{} svcPortInfoMap := make(negtypes.PortInfoMap) if err := c.mergeDefaultBackendServicePortInfoMap(key, svcPortInfoMap); err != nil { return err } + negUsage.IngressNeg = len(svcPortInfoMap) if err := c.mergeIngressPortInfo(service, types.NamespacedName{Namespace: namespace, Name: name}, svcPortInfoMap); err != nil { return err } + negUsage.IngressNeg = len(svcPortInfoMap) if err := c.mergeStandaloneNEGsPortInfo(service, types.NamespacedName{Namespace: namespace, Name: name}, svcPortInfoMap); err != nil { return err } - + negUsage.StandaloneNeg = len(svcPortInfoMap) - negUsage.IngressNeg csmSVCPortInfoMap, destinationRulesPortInfoMap, err := c.getCSMPortInfoMap(namespace, name, service) if err != nil { return err } + negUsage.AsmNeg = len(csmSVCPortInfoMap) + len(destinationRulesPortInfoMap) + // merges csmSVCPortInfoMap, because eventually those NEG will sync with the service annotation. // merges destinationRulesPortInfoMap later, because we only want them sync with the DestinationRule annotation. if err := svcPortInfoMap.Merge(csmSVCPortInfoMap); err != nil { @@ -356,11 +366,13 @@ func (c *Controller) processService(key string) error { if err := svcPortInfoMap.Merge(destinationRulesPortInfoMap); err != nil { return fmt.Errorf("failed to merge service ports referenced by Istio:DestinationRule (%v): %v", destinationRulesPortInfoMap, err) } + c.collector.SetNegService(key, negUsage) return c.manager.EnsureSyncers(namespace, name, svcPortInfoMap) } // do not need Neg klog.V(4).Infof("Service %q does not need any NEG. Skipping", key) + c.collector.DeleteNegService(key) // neg annotation is not found or NEG is not enabled c.manager.StopSyncer(namespace, name) // delete the annotation From 448ce01b0ca6f1d91abe7d12b674b1e4f49d2a98 Mon Sep 17 00:00:00 2001 From: Satish Matti Date: Mon, 13 Jan 2020 12:01:16 -0800 Subject: [PATCH 4/4] Register NEG usage metrics --- pkg/metrics/metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 93ce715aff..2e5e2a7126 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -59,8 +59,8 @@ var ( // init registers ingress usage metrics. func init() { - klog.V(3).Infof("Registering Ingress usage metrics %v and %v", ingressCount, servicePortCount) - prometheus.MustRegister(ingressCount, servicePortCount) + klog.V(3).Infof("Registering Ingress usage metrics %v, %v and %v", ingressCount, servicePortCount, networkEndpointGroupCount) + prometheus.MustRegister(ingressCount, servicePortCount, networkEndpointGroupCount) } // NewIngressState returns ingress state for given ingress and service ports. @@ -154,7 +154,7 @@ func (im *ControllerMetrics) export() { ingCount, svcPortCount := im.computeIngressMetrics() negCount := im.computeNegMetrics() - klog.V(3).Infof("Exporting ingress usage metrics. Ingress Count: %#v, Service Port count: %#v", ingCount, svcPortCount) + klog.V(3).Infof("Exporting ingress usage metrics. Ingress Count: %#v, Service Port count: %#v, NEG count: %#v", ingCount, svcPortCount, negCount) for feature, count := range ingCount { ingressCount.With(prometheus.Labels{label: feature.String()}).Set(float64(count)) }