Support for prometheus metrics (#3179)

docs/cis-3.x/README.md

@@ -63,4 +63,15 @@ kubectl delete -f ./docs/cis-3.x/deployment/cis-deployment.yaml
 kubectl delete secret f5-bigip-ctlr-login -n kube-system
 kubectl delete -f ./docs/config_examples/customResourceDefinitions/incubator/customresourcedefinitions.yml
 kubectl delete -f ./docs/cis-3.x/rbac/clusterrole.yaml
-```
+```
+
+Prometheus Metrics
+------------------
+
+| Name                                     | Type  | Default Status | Description                                                               | Labels                                   |
+|------------------------------------------|-------|----------------|---------------------------------------------------------------------------|------------------------------------------|
+| k8s_bigip_ctlr_managed_services          | Gauge | Enabled        | The total number of managed services by the CIS Controller                | -                                        |
+| k8s_bigip_ctlr_managed_transport_servers | Gauge | Enabled        | The total number of managed transport servers by the CIS Controller       | -                                        |
+| k8s_bigip_ctlr_configuration_warnings    | Gauge | Enabled        | The total number of configuration warnings by the CIS Controller          | ["kind" ,"namespace", "name", "warning"] |
+| k8s_bigip_ctlr_managed_bigips            | Gauge | Enabled        | The total number of bigips where the CIS Controller posts the declaration | -                                        |
+| k8s_bigip_ctlr_monitored_nodes           | Gauge | Enabled        | The total number of monitored nodes by the CIS Controller                 | ["nodeselector"]                         |

pkg/controller/controller.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"fmt"
 	cisapiv1 "github.com/F5Networks/k8s-bigip-ctlr/v3/config/apis/cis/v1"
+	"github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/prometheus"
 	"github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/tokenmanager"
 	"os"
 	"strings"
@@ -101,13 +102,11 @@ func NewController(params Params) *Controller {
 	}
 	// setup agents for bigip label
 	for bigip, _ := range ctlr.bigIpMap {
-		ctlr.AgentParams.Partition = bigip.DefaultPartition
-		agent := NewAgent(ctlr.AgentParams, bigip.BigIpLabel)
-		//Maintain map of agent per bigipLabel
-		ctlr.AgentMap[bigip.BigIpLabel] = agent
-		go ctlr.responseHandler(agent.respChan)
+
+		ctlr.startAgent(bigip)
+
 		// enable http endpoint
-		go ctlr.enableHttpEndpoint(params.HttpAddress, bigip.BigIpLabel)
+		go ctlr.enableHttpEndpoint(params.HttpAddress, bigip)
 	}
 
 	ctlr.setupIPAM(params)
@@ -286,3 +285,9 @@ func (ctlr *Controller) Stop() {
 	}
 
 }
+
+// Set the resource count for prometheus metrics
+func (ctlr *Controller) setPrometheusResourceCount() {
+	prometheus.ManagedServices.Set(float64(len(ctlr.resources.poolMemCache)))
+	prometheus.ManagedTransportServers.Set(float64(len(ctlr.TeemData.ResourceType.TransportServer) + len(ctlr.TeemData.ResourceType.IPAMTS)))
+}

pkg/controller/informerManager.go

@@ -33,6 +33,8 @@ func (ctlr *Controller) initInformers() {
 	}
 	ctlr.updateResourceSelectorConfig(configCR.Spec.BaseConfig)
 	ctlr.updateBigIpConfigMap(configCR.Spec.BigIpConfig)
+	// update the agent params
+	ctlr.AgentParams.PostParams.AS3Config = configCR.Spec.AS3Config
 	if ctlr.managedResources.ManageRoutes {
 		// initialize the processed host-path map
 		var processedHostPath ProcessedHostPath

pkg/controller/metrics.go

@@ -18,16 +18,17 @@ package controller
 
 import (
 	"context"
+	cisapiv1 "github.com/F5Networks/k8s-bigip-ctlr/v3/config/apis/cis/v1"
 	bigIPPrometheus "github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/prometheus"
 	log "github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/vlogger"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"net/http"
 )
 
-func (ctlr *Controller) enableHttpEndpoint(httpAddress string, bigipLabel string) {
+func (ctlr *Controller) enableHttpEndpoint(httpAddress string, bigip cisapiv1.BigIpConfig) {
 	// Expose Prometheus metrics
 	http.Handle("/metrics", promhttp.Handler())
-	bigIPPrometheus.RegisterMetrics(ctlr.AgentMap[bigipLabel].PostManager.HTTPClientMetrics)
+	bigIPPrometheus.RegisterMetrics(ctlr.AgentMap[bigip.BigIpLabel].PostManager.HTTPClientMetrics, bigip.BigIpAddress)
 	// Expose cis health endpoint
 	http.Handle("/health", ctlr.CISHealthCheckHandler())
 	log.Fatal(http.ListenAndServe(httpAddress, nil).Error())

pkg/controller/nativeResourceWorker.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	cisapiv1 "github.com/F5Networks/k8s-bigip-ctlr/v3/config/apis/cis/v1"
 	"github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/clustermanager"
+	"github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"os"
@@ -1688,6 +1689,7 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 		if processedRouteTimestamp.Before(&route.ObjectMeta.CreationTimestamp) {
 			message := fmt.Sprintf("Discarding route %v as other route already exposes URI %v%v and is older ", route.Name, route.Spec.Host, route.Spec.Path)
 			log.Warningf(message)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "HostAlreadyClaimed", message, v1.ConditionFalse)
 			return false
 		}
@@ -1700,12 +1702,14 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 		if len(plcSSLProfiles.serverSSLs) == 0 && route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt {
 			message := fmt.Sprintf("Missing server SSL profile in the policy %v/%v", plcSSLProfiles.plcNamespace, plcSSLProfiles.plcName)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	case AnnotationSSLOption:
 		if _, ok := route.ObjectMeta.Annotations[F5ServerSslProfileAnnotation]; !ok && route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt {
 			message := fmt.Sprintf("Missing server SSL profile in the annotation")
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	case RouteCertificateSSLOption:
@@ -1715,22 +1719,26 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 			//Invalid certificate and key
 			message := fmt.Sprintf("Invalid certificate and key for route: %v", route.ObjectMeta.Name)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	case DefaultSSLOption:
 		if ctlr.resources.baseRouteConfig.DefaultTLS.ClientSSL == "" {
 			message := fmt.Sprintf("Missing client SSL profile %s reference in the ConfigCR - BaseRouteSpec", ctlr.resources.baseRouteConfig.DefaultTLS.Reference)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 		if ctlr.resources.baseRouteConfig.DefaultTLS.ServerSSL == "" && route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt {
 			message := fmt.Sprintf("Missing server SSL profile %s reference in the ConfigCR - BaseRouteSpec", ctlr.resources.baseRouteConfig.DefaultTLS.Reference)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	default:
 		message := fmt.Sprintf("Missing certificate/key/SSL profile annotation/defaultSSL for route: %v", route.ObjectMeta.Name)
 		go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse)
+		prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 		return false
 	}
 
@@ -1740,12 +1748,14 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 			message := fmt.Sprintf("Discarding route %v as annotation %v is empty", route.Name, F5VsAppRootAnnotation)
 			log.Warningf(message)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "InvalidAnnotation", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 		if route.Spec.Path != "" && route.Spec.Path != "/" {
 			message := fmt.Sprintf("Invalid annotation: %v=%v can not target path for app-root annotation for route %v, skipping", F5VsAppRootAnnotation, appRootPath, route.Name)
 			log.Warningf(message)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "InvalidAnnotation", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	}
@@ -1756,6 +1766,7 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 			message := fmt.Sprintf("Discarding route %v as annotation %v is empty", route.Name, F5VsWAFPolicy)
 			log.Warningf(message)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "InvalidAnnotation", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	}
@@ -1776,6 +1787,7 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 				F5VsAllowSourceRangeAnnotation)
 			log.Warningf(message)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "InvalidAnnotation", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	}
@@ -1800,6 +1812,7 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 				message := fmt.Sprintf("unable to parse annotation %v for route %v/%v", MultiClusterServicesAnnotation, route.Name, route.Namespace)
 				log.Warningf(message)
 				go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "InvalidAnnotation", message, v1.ConditionFalse)
+				prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 				return false
 			}
 		}
@@ -1812,10 +1825,11 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, plcSSLProfiles rg
 			log.Warningf(message)
 			go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%s/%s", route.Namespace, route.Name),
 				"ServiceNotFound", message, v1.ConditionFalse)
+			prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, message).Set(1)
 			return false
 		}
 	}
-
+	prometheus.ConfigurationWarnings.WithLabelValues(Route, route.ObjectMeta.Namespace, route.ObjectMeta.Name, "").Set(0)
 	return true
 }
 

pkg/controller/postManager.go

@@ -133,7 +133,7 @@ func (postMgr *PostManager) publishConfig(cfg as3Config) {
 
 func (postMgr *PostManager) postConfig(cfg *as3Config) {
 	// log as3 request if it's set
-	if postMgr.LogAS3Request {
+	if postMgr.AS3Config.DebugAS3 {
 		postMgr.logAS3Request(cfg.data)
 	}
 	httpReqBody := bytes.NewBuffer([]byte(cfg.data))
@@ -196,7 +196,7 @@ func (postMgr *PostManager) httpPOST(request *http.Request) (*http.Response, map
 	err = json.Unmarshal(body, &response)
 	if err != nil {
 		log.Errorf("[AS3]%v Response body unmarshal failed: %v\n", postMgr.postManagerPrefix, err)
-		if postMgr.LogAS3Response {
+		if postMgr.AS3Config.DebugAS3 {
 			log.Errorf("[AS3]%v Raw response from Big-IP: %v", postMgr.postManagerPrefix, string(body))
 		}
 		return nil, nil
@@ -302,14 +302,14 @@ func (postMgr *PostManager) handleResponseStatusNotFound(responseMap map[string]
 	} else {
 		log.Errorf("[AS3]%v Big-IP Responded with error code: %v", postMgr.postManagerPrefix, http.StatusNotFound)
 	}
-	if postMgr.LogAS3Response {
+	if postMgr.AS3Config.DebugAS3 {
 		postMgr.logAS3Response(responseMap)
 	}
 	postMgr.updateTenantResponseCode(http.StatusNotFound, "", "", false)
 }
 
 func (postMgr *PostManager) handleResponseOthers(responseMap map[string]interface{}) {
-	if postMgr.LogAS3Response {
+	if postMgr.AS3Config.DebugAS3 {
 		postMgr.logAS3Response(responseMap)
 	}
 	if results, ok := (responseMap["results"]).([]interface{}); ok {
@@ -441,7 +441,7 @@ func (postMgr *PostManager) httpReq(request *http.Request) (*http.Response, map[
 	err = json.Unmarshal(body, &response)
 	if err != nil {
 		log.Errorf("[AS3]%v Response body unmarshal failed: %v\n", postMgr.postManagerPrefix, err)
-		if postMgr.LogAS3Response {
+		if postMgr.AS3Config.DebugAS3 {
 			log.Errorf("[AS3]%v Raw response from Big-IP: %v", postMgr.postManagerPrefix, string(body))
 		}
 		return nil, nil

pkg/controller/postManager_test.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"fmt"
+	cisapiv1 "github.com/F5Networks/k8s-bigip-ctlr/v3/config/apis/cis/v1"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"net/http"
@@ -12,8 +13,8 @@ var _ = Describe("PostManager Tests", func() {
 	BeforeEach(func() {
 		mockPM = newMockPostManger()
 		mockPM.tenantResponseMap = make(map[string]tenantResponse)
-		mockPM.LogAS3Response = true
-		mockPM.AS3PostDelay = 2
+		mockPM.AS3Config = cisapiv1.AS3Config{DebugAS3: true,
+			PostDelayAS3: 2}
 	})
 
 	It("Setup Client", func() {
@@ -27,7 +28,7 @@ var _ = Describe("PostManager Tests", func() {
 			mockPM.CMUsername = "user"
 			mockPM.CMPassword = "pswd"
 			agentCfg = as3Config{
-				data:      "{}",
+				data:      `{"declaration": {"test": {"Shared": {"class": "application"}}}}`,
 				as3APIURL: mockPM.getAS3APIURL([]string{"test"}),
 				id:        0,
 			}

pkg/controller/requestHandler.go

@@ -27,10 +27,10 @@ import (
 func (agent *Agent) requestHandler() {
 	for rsConfig := range agent.reqChan {
 		// For the very first post after starting controller, need not wait to post
-		if !agent.firstPost && agent.AS3PostDelay != 0 {
+		if !agent.firstPost && agent.AS3Config.PostDelayAS3 != 0 {
 			// Time (in seconds) that CIS waits to post the AS3 declaration to BIG-IP.
-			log.Debugf("[AS3] Delaying post to BIG-IP for %v seconds ", agent.AS3PostDelay)
-			_ = <-time.After(time.Duration(agent.AS3PostDelay) * time.Second)
+			log.Debugf("[AS3] Delaying post to BIG-IP for %v seconds ", agent.AS3Config.PostDelayAS3)
+			_ = <-time.After(time.Duration(agent.AS3Config.PostDelayAS3) * time.Second)
 		}
 
 		// Fetch the latest config from channel
@@ -124,7 +124,6 @@ func NewAgent(params AgentParams, bigiplabel string) *Agent {
 	// retryWorker runs as a separate go routine
 	// blocks on retryChan ; retries failed declarations and polls for accepted tenant statuses
 	go agent.retryWorker()
-
 	return agent
 }
 

pkg/controller/types.go

@@ -765,9 +765,7 @@ type (
 		LogLevel           string
 		VerifyInterval     int
 		VXLANName          string
-		PythonBaseDir      string
 		UserAgent          string
-		HttpAddress        string
 		EnableIPV6         bool
 		StaticRoutingMode  bool
 		SharedStaticRoutes bool
@@ -805,15 +803,12 @@ type (
 	}
 
 	PostParams struct {
-		CMUsername   string
-		CMPassword   string
-		CMURL        string
-		TrustedCerts string
-		SSLInsecure  bool
-		AS3PostDelay int
-		// Log the AS3 response body in Controller logs
-		LogAS3Response    bool
-		LogAS3Request     bool
+		CMUsername        string
+		CMPassword        string
+		CMURL             string
+		TrustedCerts      string
+		SSLInsecure       bool
+		AS3Config         cisapiv1.AS3Config
 		HTTPClientMetrics bool
 		httpClient        *http.Client
 	}