Cherrypick #976, #984, #990 and #1012 [Ingress Usage metrics] into release 1.8

pkg/context/context.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/ingress-gce/pkg/common/typed"
 	frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
 	informerfrontendconfig "k8s.io/ingress-gce/pkg/frontendconfig/client/informers/externalversions/frontendconfig/v1beta1"
+	"k8s.io/ingress-gce/pkg/metrics"
 	"k8s.io/ingress-gce/pkg/utils"
 	"k8s.io/ingress-gce/pkg/utils/namer"
 	"k8s.io/klog"
@@ -75,6 +76,8 @@ type ControllerContext struct {
 	DestinationRuleInformer cache.SharedIndexInformer
 	ConfigMapInformer       cache.SharedIndexInformer
 
+	ControllerMetrics *metrics.ControllerMetrics
+
 	healthChecks map[string]func() error
 
 	lock sync.Mutex
@@ -114,6 +117,7 @@ func NewControllerContext(
 		Cloud:                   cloud,
 		ClusterNamer:            namer,
 		KubeSystemUID:           kubeSystemUID,
+		ControllerMetrics:       metrics.NewControllerMetrics(),
 		ControllerContextConfig: config,
 		IngressInformer:         informerv1beta1.NewIngressInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
 		ServiceInformer:         informerv1.NewServiceInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
@@ -293,6 +297,8 @@ func (ctx *ControllerContext) Start(stopCh chan struct{}) {
 	if ctx.EnableASMConfigMap && ctx.ConfigMapInformer != nil {
 		go ctx.ConfigMapInformer.Run(stopCh)
 	}
+	// Export ingress usage metrics.
+	go ctx.ControllerMetrics.Run(stopCh)
 }
 
 // Ingresses returns the store of Ingresses.

pkg/controller/controller.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/ingress-gce/pkg/healthchecks"
 	"k8s.io/ingress-gce/pkg/instances"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
+	"k8s.io/ingress-gce/pkg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	ingsync "k8s.io/ingress-gce/pkg/sync"
 	"k8s.io/ingress-gce/pkg/tls"
@@ -89,6 +90,9 @@ type LoadBalancerController struct {
 
 	// Ingress sync + GC implementation
 	ingSyncer ingsync.Syncer
+
+	// Ingress usage metrics.
+	metrics metrics.IngressMetricsCollector
 }
 
 // NewLoadBalancerController creates a controller for gce loadbalancers.
@@ -119,6 +123,7 @@ func NewLoadBalancerController(
 		backendSyncer: backends.NewBackendSyncer(backendPool, healthChecker, ctx.Cloud),
 		negLinker:     backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud),
 		igLinker:      backends.NewInstanceGroupLinker(instancePool, backendPool),
+		metrics:       ctx.ControllerMetrics,
 	}
 	lbc.ingSyncer = ingsync.NewIngressSyncer(&lbc)
 
@@ -531,6 +536,10 @@ func (lbc *LoadBalancerController) sync(key string) error {
 		if err != nil && ingExists {
 			lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "GC", fmt.Sprintf("Error during GC: %v", err))
 		}
+		// Delete the ingress state for metrics after GC is successful.
+		if err == nil && ingExists {
+			lbc.metrics.DeleteIngress(key)
+		}
 		return err
 	}
 
@@ -555,6 +564,9 @@ func (lbc *LoadBalancerController) sync(key string) error {
 	syncErr := lbc.ingSyncer.Sync(syncState)
 	if syncErr != nil {
 		lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "Sync", fmt.Sprintf("Error during sync: %v", syncErr.Error()))
+	} else {
+		// Insert/update the ingress state for metrics after successful sync.
+		lbc.metrics.SetIngress(key, metrics.NewIngressState(ing, urlMap.AllServicePorts()))
 	}
 
 	// Garbage collection will occur regardless of an error occurring. If an error occurred,

pkg/metrics/features.go

@@ -0,0 +1,242 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"fmt"
+	"strconv"
+
+	"k8s.io/api/networking/v1beta1"
+	"k8s.io/ingress-gce/pkg/utils"
+	"k8s.io/klog"
+)
+
+type feature string
+
+func (f feature) String() string {
+	return string(f)
+}
+
+const (
+	// WARNING: Please keep the following constants in sync with
+	// pkg/annotations/ingress.go
+	// allowHTTPKey tells the Ingress controller to allow/block HTTP access.
+	allowHTTPKey         = "kubernetes.io/ingress.allow-http"
+	ingressClassKey      = "kubernetes.io/ingress.class"
+	gceIngressClass      = "gce"
+	gceMultiIngressClass = "gce-multi-cluster"
+	gceL7ILBIngressClass = "gce-internal"
+	// preSharedCertKey represents the specific pre-shared SSL
+	// certificate for the Ingress controller to use.
+	preSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
+	managedCertKey   = "networking.gke.io/managed-certificates"
+	// staticIPKey is the annotation key used by controller to record GCP static ip.
+	staticIPKey = "ingress.kubernetes.io/static-ip"
+
+	ingress                = feature("Ingress")
+	externalIngress        = feature("ExternalIngress")
+	internalIngress        = feature("InternalIngress")
+	httpEnabled            = feature("HTTPEnabled")
+	hostBasedRouting       = feature("HostBasedRouting")
+	pathBasedRouting       = feature("PathBasedRouting")
+	tlsTermination         = feature("TLSTermination")
+	secretBasedCertsForTLS = feature("SecretBasedCertsForTLS")
+	preSharedCertsForTLS   = feature("PreSharedCertsForTLS")
+	managedCertsForTLS     = feature("ManagedCertsForTLS")
+	staticGlobalIP         = feature("StaticGlobalIP")
+
+	servicePort         = feature("L7LBServicePort")
+	externalServicePort = feature("L7XLBServicePort")
+	internalServicePort = feature("L7ILBServicePort")
+	neg                 = feature("NEG")
+
+	cloudCDN                  = feature("CloudCDN")
+	cloudArmor                = feature("CloudArmor")
+	cloudIAP                  = feature("CloudIAP")
+	backendTimeout            = feature("BackendTimeout")
+	backendConnectionDraining = feature("BackendConnectionDraining")
+	clientIPAffinity          = feature("ClientIPAffinity")
+	cookieAffinity            = feature("CookieAffinity")
+	customRequestHeaders      = feature("CustomRequestHeaders")
+
+	standaloneNeg = feature("StandaloneNEG")
+	ingressNeg    = feature("IngressNEG")
+	asmNeg        = feature("AsmNEG")
+)
+
+// featuresForIngress returns the list of features for given ingress.
+func featuresForIngress(ing *v1beta1.Ingress) []feature {
+	features := []feature{ingress}
+
+	ingKey := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)
+	klog.V(4).Infof("Listing features for Ingress %s", ingKey)
+	ingAnnotations := ing.Annotations
+
+	// Determine the type of ingress based on ingress class.
+	ingClass := ingAnnotations[ingressClassKey]
+	klog.V(6).Infof("Ingress class value for ingress %s: %s", ingKey, ingClass)
+	switch ingClass {
+	case "", gceIngressClass, gceMultiIngressClass:
+		features = append(features, externalIngress)
+	case gceL7ILBIngressClass:
+		features = append(features, internalIngress)
+	}
+
+	// Determine if http is enabled.
+	if val, ok := ingAnnotations[allowHTTPKey]; !ok {
+		klog.V(6).Infof("Annotation %s does not exist for ingress %s", allowHTTPKey, ingKey)
+		features = append(features, httpEnabled)
+	} else {
+		klog.V(6).Infof("User specified value for annotation %s on ingress %s: %s", allowHTTPKey, ingKey, val)
+		v, err := strconv.ParseBool(val)
+		if err != nil {
+			klog.Errorf("Failed to parse %s for annotation %s on ingress %s", val, allowHTTPKey, ingKey)
+		}
+		if err == nil && v {
+			features = append(features, httpEnabled)
+		}
+	}
+
+	// An ingress without a host or http-path is ignored.
+	hostBased, pathBased := false, false
+	if len(ing.Spec.Rules) == 0 {
+		klog.V(6).Infof("Neither host-based nor path-based routing rules are setup for ingress %s", ingKey)
+	}
+	for _, rule := range ing.Spec.Rules {
+		if rule.HTTP != nil && len(rule.HTTP.Paths) > 0 {
+			klog.V(6).Infof("User specified http paths for ingress %s: %v", ingKey, rule.HTTP.Paths)
+			pathBased = true
+		}
+		if rule.Host != "" {
+			klog.V(6).Infof("User specified host for ingress %s: %v", ingKey, rule.Host)
+			hostBased = true
+		}
+		if pathBased && hostBased {
+			break
+		}
+	}
+	if hostBased {
+		features = append(features, hostBasedRouting)
+	}
+	if pathBased {
+		features = append(features, pathBasedRouting)
+	}
+
+	// SSL certificate based features.
+	sslConfigured := false
+	if val, ok := ingAnnotations[preSharedCertKey]; ok {
+		klog.V(6).Infof("Specified pre-shared certs for ingress %s: %v", ingKey, val)
+		sslConfigured = true
+		features = append(features, preSharedCertsForTLS)
+	}
+	if val, ok := ingAnnotations[managedCertKey]; ok {
+		klog.V(6).Infof("Specified google managed certs for ingress %s: %v", ingKey, val)
+		sslConfigured = true
+		features = append(features, managedCertsForTLS)
+	}
+	if hasSecretBasedCerts(ing) {
+		sslConfigured = true
+		features = append(features, secretBasedCertsForTLS)
+	}
+	if sslConfigured {
+		klog.V(6).Infof("TLS termination is configured for ingress %s", ingKey)
+		features = append(features, tlsTermination)
+	}
+
+	// Both user specified and ingress controller managed global static ips are reported.
+	if val, ok := ingAnnotations[staticIPKey]; ok && val != "" {
+		klog.V(6).Infof("Specified static for ingress %s: %s", ingKey, val)
+		features = append(features, staticGlobalIP)
+	}
+	klog.V(4).Infof("Features for ingress %s/%s: %v", ing.Namespace, ing.Name, features)
+	return features
+}
+
+// hasSecretBasedCerts returns true if ingress spec contains a secret based cert.
+func hasSecretBasedCerts(ing *v1beta1.Ingress) bool {
+	for _, tlsSecret := range ing.Spec.TLS {
+		if tlsSecret.SecretName == "" {
+			continue
+		}
+		klog.V(6).Infof("User specified secret for ingress %s/%s: %s", ing.Namespace, ing.Name, tlsSecret.SecretName)
+		return true
+	}
+	return false
+}
+
+// featuresForServicePort returns the list of features for given service port.
+func featuresForServicePort(sp utils.ServicePort) []feature {
+	features := []feature{servicePort}
+	svcPortKey := newServicePortKey(sp).string()
+	klog.V(4).Infof("Listing features for service port %s", svcPortKey)
+	if sp.L7ILBEnabled {
+		klog.V(6).Infof("L7 ILB is enabled for service port %s", svcPortKey)
+		features = append(features, internalServicePort)
+	} else {
+		features = append(features, externalServicePort)
+	}
+	if sp.NEGEnabled {
+		klog.V(6).Infof("NEG is enabled for service port %s", svcPortKey)
+		features = append(features, neg)
+	}
+	if sp.BackendConfig == nil {
+		klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features)
+		return features
+	}
+
+	beConfig := fmt.Sprintf("%s/%s", sp.BackendConfig.Namespace, sp.BackendConfig.Name)
+	klog.V(6).Infof("Backend config specified for service port %s: %s", svcPortKey, beConfig)
+
+	if sp.BackendConfig.Spec.Cdn != nil && sp.BackendConfig.Spec.Cdn.Enabled {
+		klog.V(6).Infof("Cloud CDN is enabled for service port %s", svcPortKey)
+		features = append(features, cloudCDN)
+	}
+	if sp.BackendConfig.Spec.Iap != nil && sp.BackendConfig.Spec.Iap.Enabled {
+		klog.V(6).Infof("Cloud IAP is enabled for service port %s", svcPortKey)
+		features = append(features, cloudIAP)
+	}
+	// Possible list of Affinity types:
+	// NONE, CLIENT_IP, GENERATED_COOKIE, CLIENT_IP_PROTO, or CLIENT_IP_PORT_PROTO.
+	if sp.BackendConfig.Spec.SessionAffinity != nil {
+		affinityType := sp.BackendConfig.Spec.SessionAffinity.AffinityType
+		switch affinityType {
+		case "GENERATED_COOKIE":
+			features = append(features, cookieAffinity)
+		case "CLIENT_IP", "CLIENT_IP_PROTO", "CLIENT_IP_PORT_PROTO":
+			features = append(features, clientIPAffinity)
+		}
+		klog.V(6).Infof("Session affinity %s is configured for service port %s", affinityType, svcPortKey)
+	}
+	if sp.BackendConfig.Spec.SecurityPolicy != nil {
+		klog.V(6).Infof("Security policy %s is configured for service port %s", sp.BackendConfig.Spec.SecurityPolicy, svcPortKey)
+		features = append(features, cloudArmor)
+	}
+	if sp.BackendConfig.Spec.TimeoutSec != nil {
+		klog.V(6).Infof("Backend timeout(%v secs) is configured for service port %s", sp.BackendConfig.Spec.TimeoutSec, svcPortKey)
+		features = append(features, backendTimeout)
+	}
+	if sp.BackendConfig.Spec.ConnectionDraining != nil {
+		klog.V(6).Infof("Backend connection draining(%v secs) is configured for service port %s", sp.BackendConfig.Spec.ConnectionDraining.DrainingTimeoutSec, svcPortKey)
+		features = append(features, backendConnectionDraining)
+	}
+	if sp.BackendConfig.Spec.CustomRequestHeaders != nil {
+		klog.V(6).Infof("Custom request headers configured for service port %s: %v", svcPortKey, sp.BackendConfig.Spec.CustomRequestHeaders.Headers)
+		features = append(features, customRequestHeaders)
+	}
+	klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features)
+	return features
+}