Enable metric endpoints creation

.circleci/config.yml

@@ -60,6 +60,7 @@ jobs:
             k3d create --publish 8200:30028 --image rancher/k3s:v1.0.0 -t=0
             export KUBECONFIG="$(k3d get-kubeconfig)"
             k3d import-images patoarvizu/kms-vault-operator:latest
+            kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/prometheus-operator/crds.yaml
             kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-operator.yaml
             kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/cert-manager/cert-manager-v0.14.1.yaml
             kubectl rollout status -n cert-manager deployment/cert-manager-webhook -w
@@ -70,6 +71,7 @@ jobs:
               sleep 1
             done
             kubectl apply -f test/manifests/global/operator.yaml
+            operator-sdk test local ./test/e2e/ --namespace "vault" --global-manifest ./test/manifests/global/operator.yaml --go-test-flags '-v -run TestMonitoringObjectsCreated'
             operator-sdk test local ./test/e2e/ --namespace "vault" --global-manifest ./test/manifests/global/operator.yaml --go-test-flags '-v -run .*V1'
             kubectl delete -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-cluster-v1.yaml
             kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-cluster-v2.yaml

README.md

@@ -18,6 +18,7 @@
         - [Partial secrets](#partial-secrets)
         - [Empty secrets](#empty-secrets)
         - [Validating webhook](#validating-webhook)
+        - [Monitoring](#monitoring)
     - [For security nerds](#for-security-nerds)
         - [Docker images are signed and published to Docker Hub's Notary server](#docker-images-are-signed-and-published-to-docker-hubs-notary-server)
         - [Docker images are labeled with Git and GPG metadata](#docker-images-are-labeled-with-git-and-gpg-metadata)
@@ -145,6 +146,10 @@ The Docker image contains another binary (`kms-vault-validating-webhook`) that c
 
 Keep in mind that a `ValidatingWebhookConfiguration` requires a valid CA bundle to trust the webhook over TLS. While this can be any certificate generated offline, you can also use [`cert-manager`](https://github.com/jetstack/cert-manager/) to make it easy to generate certificates as Kubernetes `Secret`s and mount them on containers (like the webhook), or to inject the corresponding CA bundle in `ValidatingWebhookConfiguration`s.
 
+### Monitoring
+
+If your Kubernetes cluster is running the Prometheus [operator](https://github.com/coreos/prometheus-operator), this operator will automatically create an additional `Service` called `kms-vault-operator-metrics` and a corresponding `ServiceMonitor` of the same name. This monitor will scrape the operator for metrics on two different ports. Port 8383 will post general metrics about the running process, while port 8686 will post metrics about the custom resources managed by the operator. More information can be found on the Operator SDK [website](https://sdk.operatorframework.io/docs/golang/monitoring/prometheus/).
+
 ## For security nerds
 
 ### Docker images are signed and published to Docker Hub's Notary server

cmd/manager/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -16,10 +17,15 @@ import (
 	"github.com/patoarvizu/kms-vault-operator/pkg/controller/kmsvaultsecret"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
+	kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
 	"github.com/operator-framework/operator-sdk/pkg/log/zap"
+	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	"github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -28,8 +34,9 @@ import (
 
 // Change below variables to serve metrics on different host or port.
 var (
-	metricsHost       = "0.0.0.0"
-	metricsPort int32 = 8383
+	metricsHost               = "0.0.0.0"
+	metricsPort         int32 = 8383
+	operatorMetricsPort int32 = 8686
 )
 var log = logf.Log.WithName("cmd")
 
@@ -110,6 +117,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	addMetrics(ctx, cfg)
+
 	log.Info("Starting the Cmd.")
 
 	// Start the Cmd
@@ -118,3 +127,53 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func addMetrics(ctx context.Context, cfg *rest.Config) {
+	if err := serveCRMetrics(cfg); err != nil {
+		if errors.Is(err, k8sutil.ErrRunLocal) {
+			log.Info("Skipping CR metrics server creation; not running in a cluster.")
+			return
+		}
+		log.Info("Could not generate and serve custom resource metrics", "error", err.Error())
+	}
+
+	servicePorts := []v1.ServicePort{
+		{Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: metricsPort}},
+		{Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: operatorMetricsPort}},
+	}
+
+	service, err := metrics.CreateMetricsService(ctx, cfg, servicePorts)
+	if err != nil {
+		log.Info("Could not create metrics Service", "error", err.Error())
+	}
+
+	services := []*v1.Service{service}
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		log.Info("Could not get operator namespace", "error", err.Error())
+	}
+	_, err = metrics.CreateServiceMonitors(cfg, operatorNs, services)
+	if err != nil {
+		log.Info("Could not create ServiceMonitor object", "error", err.Error())
+		if err == metrics.ErrServiceMonitorNotPresent {
+			log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error())
+		}
+	}
+}
+
+func serveCRMetrics(cfg *rest.Config) error {
+	filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(apis.AddToScheme)
+	if err != nil {
+		return err
+	}
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return err
+	}
+	ns := []string{operatorNs}
+	err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort)
+	if err != nil {
+		return err
+	}
+	return nil
+}

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/aws/aws-sdk-go v1.17.7
+	github.com/coreos/prometheus-operator v0.34.0
 	github.com/go-logr/logr v0.1.0
 	github.com/go-openapi/spec v0.19.4
 	github.com/hashicorp/vault/api v1.0.4

go.sum

@@ -137,6 +137,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
 github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/prometheus-operator v0.34.0 h1:TF9qaydNeUamLKs0hniaapa4FBz8U8TIlRRtJX987A4=
 github.com/coreos/prometheus-operator v0.34.0/go.mod h1:Li6rMllG/hYIyXfMuvUwhyC+hqwJVHdsDdP21hypT1M=
 github.com/coreos/rkt v1.30.0/go.mod h1:O634mlH6U7qk87poQifK6M2rsFNt+FyUTWNMnP1hF1U=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
@@ -1049,6 +1050,7 @@ k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d h1:Xpe6sK+RY4ZgCTyZ3y273U
 k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kube-proxy v0.0.0-20191016114407-2e83b6f20229/go.mod h1:2Hxci1uzXO5ipP0h9n2+h18fvNkBTpYlckk5dOPu8zg=
 k8s.io/kube-scheduler v0.0.0-20191016114748-65049c67a58b/go.mod h1:BgDUHHC5Wl0xcBUQgo2XEprE5nG5i9tlRR4iNgEFbL0=
+k8s.io/kube-state-metrics v1.7.2 h1:6vdtgXrrRRMSgnyDmgua+qvgCYv954JNfxXAtDkeLVQ=
 k8s.io/kube-state-metrics v1.7.2/go.mod h1:U2Y6DRi07sS85rmVPmBFlmv+2peBcL8IWGjM+IjYA/E=
 k8s.io/kubectl v0.0.0-20191016120415-2ed914427d51/go.mod h1:gL826ZTIfD4vXTGlmzgTbliCAT9NGiqpCqK2aNYv5MQ=
 k8s.io/kubelet v0.0.0-20191016114556-7841ed97f1b2/go.mod h1:SBvrtLbuePbJygVXGGCMtWKH07+qrN2dE1iMnteSG8E=

test/e2e/kmsvaultoperator_test.go

@@ -11,6 +11,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	vaultapi "github.com/hashicorp/vault/api"
 	"github.com/operator-framework/operator-sdk/pkg/test"
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
@@ -19,6 +20,7 @@ import (
 	operator "github.com/patoarvizu/kms-vault-operator/pkg/apis/k8s/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	dynclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const encryptedSecret = "AQICAHgKbLYZWOFlPGwA/1foMoxcBOxv7LddQQW9biqG70YNkwF+dKr15L/4Pl/d26uDd7KqAAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMz0gfMT1P5MBTd/fGAgEQgCANG/RycP+0ZXj2qZORafZO4fGdU7KGFINsrs1JDnx1mg=="
@@ -218,6 +220,72 @@ func authenticatedVaultClient() (*vaultapi.Client, error) {
 	return vaultClient, nil
 }
 
+func TestMonitoringObjectsCreated(t *testing.T) {
+	ctx := framework.NewTestCtx(t)
+	setup(t, ctx)
+	metricsService := &v1.Service{}
+	err := wait.Poll(time.Second*2, time.Second*60, func() (done bool, err error) {
+		err = framework.Global.Client.Get(context.TODO(), dynclient.ObjectKey{Namespace: "vault", Name: "kms-vault-operator-metrics"}, metricsService)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatal("Could not get metrics Service")
+	}
+	httpMetricsPortFound := false
+	crMetricsPortFound := false
+	for _, p := range metricsService.Spec.Ports {
+		if p.Name == "http-metrics" && p.Port == 8383 {
+			httpMetricsPortFound = true
+			continue
+		}
+		if p.Name == "cr-metrics" && p.Port == 8686 {
+			crMetricsPortFound = true
+			continue
+		}
+	}
+	if !httpMetricsPortFound {
+		t.Fatal("Service kms-vault-operator-metrics doesn't have http-metrics port 8383")
+	}
+	if !crMetricsPortFound {
+		t.Fatal("Service kms-vault-operator-metrics doesn't have cr-metrics port 8686")
+	}
+
+	framework.Global.Scheme.AddKnownTypes(monitoringv1.SchemeGroupVersion, &monitoringv1.ServiceMonitor{})
+	serviceMonitor := &monitoringv1.ServiceMonitor{}
+	err = wait.Poll(time.Second*2, time.Second*60, func() (done bool, err error) {
+		err = framework.Global.Client.Client.Get(context.TODO(), dynclient.ObjectKey{Namespace: "vault", Name: "kms-vault-operator-metrics"}, serviceMonitor)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatal("Could not find metrics ServiceMonitor")
+	}
+	httpMetricsEndpointFound := false
+	crMetricsEndpointFound := false
+	for _, e := range serviceMonitor.Spec.Endpoints {
+		if e.Port == "http-metrics" {
+			httpMetricsEndpointFound = true
+			continue
+		}
+		if e.Port == "cr-metrics" {
+			crMetricsEndpointFound = true
+			continue
+		}
+	}
+	if !httpMetricsEndpointFound {
+		t.Error("ServiceMonitor kms-vault-operator-metrics doesn't have endpoint http-metrics")
+	}
+	if !crMetricsEndpointFound {
+		t.Error("ServiceMonitor kms-vault-operator-metrics doesn't have endpoint cr-metrics")
+	}
+	ctx.Cleanup()
+}
+
 func TestKMSVaultSecretV1(t *testing.T) {
 	ctx := framework.NewTestCtx(t)
 	setup(t, ctx)