Update CNI metrics

.gitignore

@@ -6,4 +6,5 @@ verify-network
 .idea/
 *.iml
 .DS_Store
+cni-metrics-helper/cni-metrics-helper
 portmap

Makefile

@@ -12,7 +12,7 @@
 # language governing permissions and limitations under the License.
 #
 
-.PHONY: build-linux clean docker docker-build lint unit-test vet download-portmap build-docker-test
+.PHONY: all build-linux clean docker docker-build lint unit-test vet download-portmap build-docker-test build-metrics docker-metrics metrics-unit-test docker-metrics-test docker-vet
 
 IMAGE   ?= amazon/amazon-k8s-cni
 VERSION ?= $(shell git describe --tags --always --dirty)
@@ -42,8 +42,8 @@ build-linux:
 	GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-k8s-agent -ldflags "$(LDFLAGS)"
 	GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-cni -ldflags "$(LDFLAGS)" ./plugins/routed-eni/
 
-# Build docker image
-docker: 
+# Build CNI Docker image
+docker:
 	@docker build --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.release -t "$(IMAGE):$(VERSION)" .
 	@echo "Built Docker image \"$(IMAGE):$(VERSION)\""
 
@@ -64,6 +64,27 @@ docker-unit-test: build-docker-test
 	docker run -e GO111MODULE=on \
 		amazon-k8s-cni-test:latest make unit-test
 
+# Build metrics
+build-metrics:
+	GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o cni-metrics-helper/cni-metrics-helper cni-metrics-helper/cni-metrics-helper.go
+
+# Build metrics Docker image
+docker-metrics:
+	@docker build --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.metrics -t "amazon/cni-metrics-helper:$(VERSION)" .
+	@echo "Built Docker image \"amazon/cni-metrics-helper:$(VERSION)\""
+
+metrics-unit-test:
+	GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./cni-metrics-helper/metrics/...
+
+docker-metrics-test:
+	docker run -v $(shell pwd):/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
+		--workdir=/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
+		--env GOPATH=/usr/src/app \
+		golang:1.10 make metrics-unit-test
+
+# Build both CNI and metrics helper
+all: docker docker-metrics
+
 # golint
 # To install: go get -u golang.org/x/lint/golint
 lint:
@@ -86,4 +107,5 @@ docker-vet: build-docker-test
 clean:
 	rm -f aws-k8s-agent
 	rm -f aws-cni
+	rm -f cni-metrics-helper/cni-metrics-helper
 	rm -f portmap

cni-metrics-helper/cni-metrics-helper.go

@@ -0,0 +1,115 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/spf13/pflag"
+
+	"github.com/aws/amazon-vpc-cni-k8s/cni-metrics-helper/metrics"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/publisher"
+)
+
+type options struct {
+	kubeconfig   string
+	pullInterval int
+	pullCNI      bool
+	submitCW     bool
+	help         bool
+}
+
+func main() {
+	options := &options{}
+	flags := pflag.NewFlagSet("", pflag.ExitOnError)
+	// Add glog flags
+	flags.AddGoFlagSet(flag.CommandLine)
+	_ = flags.Lookup("logtostderr").Value.Set("true")
+	flags.Lookup("logtostderr").DefValue = "true"
+	flags.Lookup("logtostderr").NoOptDefVal = "true"
+	flags.BoolVar(&options.submitCW, "cloudwatch", true, "a bool")
+
+	flags.Usage = func() {
+		_, _ = fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+		flags.PrintDefaults()
+	}
+
+	err := flags.Parse(os.Args)
+	if err != nil {
+		glog.Fatalf("Error on parsing parameters: %s", err)
+	}
+
+	err = flag.CommandLine.Parse([]string{})
+	if err != nil {
+		glog.Fatalf("Error on parsing commandline: %s", err)
+	}
+
+	if options.help {
+		flags.Usage()
+		os.Exit(1)
+	}
+
+	cwENV, found := os.LookupEnv("USE_CLOUDWATCH")
+	if found {
+		if strings.Compare(cwENV, "yes") == 0 {
+			options.submitCW = true
+		}
+		if strings.Compare(cwENV, "no") == 0 {
+			options.submitCW = false
+		}
+	}
+
+	glog.Infof("Starting CNIMetricsHelper, cloudwatch: %v...", options.submitCW)
+
+	kubeClient, err := k8sapi.CreateKubeClient()
+	if err != nil {
+		glog.Errorf("Failed to create client: %v", err)
+		os.Exit(1)
+	}
+
+	discoverController := k8sapi.NewController(kubeClient)
+	go discoverController.DiscoverK8SPods()
+
+	var cw publisher.Publisher
+
+	if options.submitCW {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		cw, err = publisher.New(ctx)
+		if err != nil {
+			glog.Errorf("Failed to create publisher: %v", err)
+			os.Exit(1)
+		}
+		go cw.Start()
+		defer cw.Stop()
+	}
+
+	var cniMetric *metrics.CNIMetricsTarget
+	cniMetric = metrics.CNIMetricsNew(kubeClient, cw, discoverController, options.submitCW)
+
+	// metric loop
+	var pullInterval = 30 // seconds
+	for range time.Tick(time.Duration(pullInterval) * time.Second) {
+		glog.Info("Collecting metrics ...")
+		metrics.Handler(cniMetric)
+	}
+}

cni-metrics-helper/metrics/cni_metrics.go

@@ -0,0 +1,167 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Package metrics handles the processing of all metrics. This file handles metrics for ipamd
+package metrics
+
+import (
+	"github.com/golang/glog"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/publisher"
+)
+
+// Port where prometheus metrics are published.
+const metricsPort = 61678
+
+// InterestingCNIMetrics defines metrics parsing definition for kube-state-metrics
+var InterestingCNIMetrics = map[string]metricsConvert{
+	"awscni_assigned_ip_addresses": {
+		actions: []metricsAction{
+			{cwMetricName: "assignIPAddresses",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{}}}},
+	"awscni_total_ip_addresses": {
+		actions: []metricsAction{
+			{cwMetricName: "totalIPAddresses",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{}}}},
+	"awscni_eni_allocated": {
+		actions: []metricsAction{
+			{cwMetricName: "eniAllocated",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{}}}},
+	"awscni_eni_max": {
+		actions: []metricsAction{
+			{cwMetricName: "eniMaxAvailable",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{}}}},
+	"awscni_ip_max": {
+		actions: []metricsAction{
+			{cwMetricName: "maxIPAddresses",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{}}}},
+	"awscni_aws_api_latency_ms": {
+		actions: []metricsAction{
+			{cwMetricName: "awsAPILatency",
+				matchFunc:  matchAny,
+				actionFunc: metricsMax,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_aws_api_error_count": {
+		actions: []metricsAction{
+			{cwMetricName: "awsAPIErr",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_aws_utils_error_count": {
+		actions: []metricsAction{
+			{cwMetricName: "awsUtilErr",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_ipamd_error_count": {
+		actions: []metricsAction{
+			{cwMetricName: "ipamdErr",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_ipamd_action_inprogress": {
+		actions: []metricsAction{
+			{cwMetricName: "ipamdActionInProgress",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_reconcile_count": {
+		actions: []metricsAction{
+			{cwMetricName: "reconcileCount",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_add_ip_req_count": {
+		actions: []metricsAction{
+			{cwMetricName: "addReqCount",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+	"awscni_del_ip_req_count": {
+		actions: []metricsAction{
+			{cwMetricName: "delReqCount",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
+}
+
+// CNIMetricsTarget defines data structure for kube-state-metric target
+type CNIMetricsTarget struct {
+	interestingMetrics  map[string]metricsConvert
+	cwMetricsPublisher  publisher.Publisher
+	kubeClient          clientset.Interface
+	cniPods             []string
+	discoveryController *k8sapi.Controller
+	submitCW            bool
+}
+
+// CNIMetricsNew creates a new metricsTarget
+func CNIMetricsNew(c clientset.Interface, cw publisher.Publisher, d *k8sapi.Controller, submitCW bool) *CNIMetricsTarget {
+	return &CNIMetricsTarget{
+		interestingMetrics:  InterestingCNIMetrics,
+		cwMetricsPublisher:  cw,
+		kubeClient:          c,
+		discoveryController: d,
+		submitCW:            submitCW,
+	}
+}
+
+func (t *CNIMetricsTarget) grabMetricsFromTarget(cniPod string) ([]byte, error) {
+	glog.Infof("Grabbing metrics from CNI: %s", cniPod)
+	output, err := getMetricsFromPod(t.kubeClient, cniPod, metav1.NamespaceSystem, metricsPort)
+	if err != nil {
+		glog.Errorf("grabMetricsFromTarget: Failed to grab CNI endpoint: %v", err)
+		return nil, err
+	}
+
+	glog.V(5).Infof("cni-metrics text output: %s", string(output))
+	return output, nil
+}
+
+func (t *CNIMetricsTarget) getInterestingMetrics() map[string]metricsConvert {
+	return InterestingCNIMetrics
+}
+
+func (t *CNIMetricsTarget) getCWMetricsPublisher() publisher.Publisher {
+	return t.cwMetricsPublisher
+}
+
+func (t *CNIMetricsTarget) getTargetList() []string {
+	pods := t.discoveryController.GetCNIPods()
+	return pods
+}
+
+func (t *CNIMetricsTarget) submitCloudWatch() bool {
+	return t.submitCW
+}