Antrea Prometheus integration

build/yamls/antrea-eks.yml

@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-t4m46b8f6h
+  name: antrea-config-522bfh947f
   namespace: kube-system
 ---
 apiVersion: v1
@@ -453,7 +459,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-522bfh947f
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -651,7 +657,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-522bfh947f
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-gke.yml

@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-5754dg84hf
+  name: antrea-config-t5f26b6m9f
   namespace: kube-system
 ---
 apiVersion: v1
@@ -453,7 +459,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-t5f26b6m9f
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -651,7 +657,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-t5f26b6m9f
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-ipsec.yml

@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-c7579447k2
+  name: antrea-config-hdhc998mb8
   namespace: kube-system
 ---
 apiVersion: v1
@@ -462,7 +468,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-hdhc998mb8
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -692,7 +698,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-hdhc998mb8
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea.yml

@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-428d4tg64g
+  name: antrea-config-m8cb9g82tf
   namespace: kube-system
 ---
 apiVersion: v1
@@ -453,7 +459,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-m8cb9g82tf
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -651,7 +657,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-m8cb9g82tf
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/base/conf/antrea-agent.conf

@@ -46,3 +46,6 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-agent` container must be set to the same value.
 #apiPort: 10350
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false

build/yamls/base/conf/antrea-controller.conf

@@ -2,3 +2,6 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-controller` container must be set to the same value.
 #apiPort: 10349
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false

cmd/antrea-agent/agent.go

@@ -30,6 +30,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute"
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
+	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/querier"
 	"github.com/vmware-tanzu/antrea/pkg/agent/route"
@@ -161,11 +162,19 @@ func run(o *Options) error {
 		ovsBridgeClient,
 		networkPolicyController)
 
+	if o.config.EnablePrometheusMetrics {
+		metrics.InitializePrometheusMetrics(o.config.OVSBridge, ifaceStore, ofClient)
+	}
+
 	agentMonitor := monitor.NewAgentMonitor(crdClient, agentQuerier)
 
 	go agentMonitor.Run(stopCh)
 
-	apiServer, err := apiserver.New(agentQuerier, networkPolicyController, o.config.APIPort)
+	apiServer, err := apiserver.New(
+		agentQuerier,
+		networkPolicyController,
+		o.config.APIPort,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error when creating agent API server: %v", err)
 	}

cmd/antrea-agent/config.go

@@ -75,4 +75,7 @@ type AgentConfig struct {
 	// APIPort is the port for the antrea-agent APIServer to serve on.
 	// Defaults to 10350.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
 }

cmd/antrea-controller/config.go

@@ -25,4 +25,7 @@ type ControllerConfig struct {
 	// APIPort is the port for the antrea-controller APIServer to serve on.
 	// Defaults to 10349.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
 }

cmd/antrea-controller/controller.go

@@ -31,6 +31,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/apiserver"
 	"github.com/vmware-tanzu/antrea/pkg/apiserver/openapi"
 	"github.com/vmware-tanzu/antrea/pkg/apiserver/storage"
+	"github.com/vmware-tanzu/antrea/pkg/controller/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy/store"
 	"github.com/vmware-tanzu/antrea/pkg/controller/querier"
@@ -81,7 +82,8 @@ func run(o *Options) error {
 		addressGroupStore,
 		appliedToGroupStore,
 		networkPolicyStore,
-		controllerQuerier)
+		controllerQuerier,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error creating API server config: %v", err)
 	}
@@ -103,6 +105,10 @@ func run(o *Options) error {
 
 	go apiServer.GenericAPIServer.PrepareRun().Run(stopCh)
 
+	if o.config.EnablePrometheusMetrics {
+		metrics.InitializePrometheusMetrics()
+	}
+
 	<-stopCh
 	klog.Info("Stopping Antrea controller")
 	return nil
@@ -113,7 +119,8 @@ func createAPIServerConfig(kubeconfig string,
 	addressGroupStore storage.Interface,
 	appliedToGroupStore storage.Interface,
 	networkPolicyStore storage.Interface,
-	controllerQuerier querier.ControllerQuerier) (*apiserver.Config, error) {
+	controllerQuerier querier.ControllerQuerier,
+	enableMetrics bool) (*apiserver.Config, error) {
 	// TODO:
 	// 1. Support user-provided certificate.
 	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
@@ -155,6 +162,7 @@ func createAPIServerConfig(kubeconfig string,
 		openapi.GetOpenAPIDefinitions,
 		genericopenapi.NewDefinitionNamer(apiserver.Scheme))
 	serverConfig.OpenAPIConfig.Info.Title = "Antrea"
+	serverConfig.EnableMetrics = enableMetrics
 
 	return apiserver.NewConfig(
 		serverConfig,

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/j-keck/arping v1.0.0
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd
+	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.4.1
 	github.com/spf13/cobra v0.0.5

pkg/agent/agent.go

@@ -37,12 +37,11 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/types"
 	"github.com/vmware-tanzu/antrea/pkg/agent/util"
 	"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
+	"github.com/vmware-tanzu/antrea/pkg/util/env"
 )
 
 const (
 	maxRetryForHostLink = 5
-	// nodeNameEnvKey is environment variable.
-	nodeNameEnvKey = "NODE_NAME"
 	// ipsecPSKEnvKey is environment variable.
 	ipsecPSKEnvKey          = "ANTREA_IPSEC_PSK"
 	roundNumKey             = "roundNum" // round number key in externalIDs.
@@ -436,7 +435,7 @@ func (i *Initializer) setupDefaultTunnelInterface(tunnelPortName string) error {
 // initNodeLocalConfig retrieves node's subnet CIDR from node.spec.PodCIDR, which is used for IPAM and setup
 // host gateway interface.
 func (i *Initializer) initNodeLocalConfig() error {
-	nodeName, err := getNodeName()
+	nodeName, err := env.GetNodeName()
 	if err != nil {
 		return err
 	}
@@ -476,24 +475,6 @@ func (i *Initializer) initNodeLocalConfig() error {
 	return nil
 }
 
-// getNodeName returns the node's name used in Kubernetes, based on the priority:
-// - Environment variable NODE_NAME, which should be set by Downward API
-// - OS's hostname
-func getNodeName() (string, error) {
-	nodeName := os.Getenv(nodeNameEnvKey)
-	if nodeName != "" {
-		return nodeName, nil
-	}
-	klog.Infof("Environment variable %s not found, using hostname instead", nodeNameEnvKey)
-	var err error
-	nodeName, err = os.Hostname()
-	if err != nil {
-		klog.Errorf("Failed to get local hostname: %v", err)
-		return "", err
-	}
-	return nodeName, nil
-}
-
 // readIPSecPSK reads the IPSec PSK value from environment variable
 // ANTREA_IPSEC_PSK, when enableIPSecTunnel is set to true.
 func (i *Initializer) readIPSecPSK() error {

pkg/agent/agent_test.go

@@ -17,7 +17,6 @@ package agent
 import (
 	"fmt"
 	"net"
-	"os"
 	"testing"
 
 	mock "github.com/golang/mock/gomock"
@@ -30,38 +29,6 @@ import (
 	ovsconfigtest "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig/testing"
 )
 
-func TestGetNodeName(t *testing.T) {
-	hostName, err := os.Hostname()
-	if err != nil {
-		t.Fatalf("Failed to retrieve hostname, %v", err)
-	}
-	testTable := map[string]string{
-		"node1":     "node1",
-		"node_12":   "node_12",
-		"":          hostName,
-		"node-1234": "node-1234",
-	}
-
-	for k, v := range testTable {
-		compareNodeName(k, v, t)
-	}
-}
-
-func compareNodeName(k, v string, t *testing.T) {
-	if k != "" {
-		_ = os.Setenv(nodeNameEnvKey, k)
-		defer os.Unsetenv(nodeNameEnvKey)
-	}
-	nodeName, err := getNodeName()
-	if err != nil {
-		t.Errorf("Failure with expected name %s, %v", k, err)
-		return
-	}
-	if nodeName != v {
-		t.Errorf("Failed to retrieve nodename, want: %s, get: %s", v, nodeName)
-	}
-}
-
 func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer {
 	return &Initializer{ovsBridgeClient: ovsBridgeClient, ifaceStore: ifaceStore, hostGateway: "gw0"}
 }