✨ Add OpenCensus controller metrics

pkg/controller/metrics/default_views.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+var (
+	// DefaultPrometheusDistribution is an OpenCensus Distribution with the same
+	// buckets as the default buckets in the Prometheus client.
+	DefaultPrometheusDistribution = view.Distribution(.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10)
+
+	// ViewReconcileTotal counts ReconcileTotal with Controller and Result tags.
+	ViewReconcileTotal = view.View{
+		Name:        "controller_runtime_reconcile_total",
+		Measure:     MeasureReconcileTotal,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{TagController, TagResult},
+	}
+
+	// ViewReconcileError counts ReconcileError with a Controller tag.
+	ViewReconcileError = view.View{
+		Name:        "controller_runtime_reconcile_errors_total",
+		Measure:     MeasureReconcileErrors,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{TagController},
+	}
+	// ViewReconcileTime is a histogram of ReconcileTime with a Controller tag.
+	ViewReconcileTime = view.View{
+		Name:        "controller_runtime_reconcile_time_seconds",
+		Measure:     MeasureReconcileTime,
+		Aggregation: DefaultPrometheusDistribution,
+		TagKeys:     []tag.Key{TagController},
+	}
+
+	// DefaultViews is an array of OpenCensus views that can be registered
+	// using view.Register(metrics.DefaultViews...) to export default metrics.
+	DefaultViews = []*view.View{
+		&ViewReconcileTotal,
+		&ViewReconcileError,
+		&ViewReconcileTime,
+	}
+)

pkg/controller/metrics/default_views_test.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"testing"
+
+	"go.opencensus.io/stats/view"
+)
+
+func TestRegisterUnregisterDefaultViews(t *testing.T) {
+	if err := view.Register(DefaultViews...); err != nil {
+		t.Fatalf("Error registering views: %v", err)
+	}
+	if view.Find(ViewReconcileTotal.Name) == nil {
+		t.Errorf("Couldn't find view: %s", ViewReconcileTotal.Name)
+	}
+	if view.Find(ViewReconcileError.Name) == nil {
+		t.Errorf("Couldn't find view: %s", ViewReconcileError.Name)
+	}
+	if view.Find(ViewReconcileTime.Name) == nil {
+		t.Errorf("Couldn't find view %s", ViewReconcileTime.Name)
+	}
+
+	view.Unregister(DefaultViews...)
+	if view.Find(ViewReconcileTotal.Name) != nil {
+		t.Errorf("view %s was not unregistered", ViewReconcileTotal.Name)
+	}
+	if view.Find(ViewReconcileError.Name) != nil {
+		t.Errorf("view %s was not unregistered", ViewReconcileError.Name)
+	}
+	if view.Find(ViewReconcileTime.Name) != nil {
+		t.Errorf("view %s was not unregistered", ViewReconcileTime.Name)
+	}
+
+}

pkg/controller/metrics/oc_metrics.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+)
+
+var (
+	// MeasureReconcileTotal is a counter which records the total number of reconciliations
+	// per controller. It has two tags: controller refers
+	// to the controller name and result refers to the reconcile result e.g.
+	// success, error, requeue, requeue_after.
+	MeasureReconcileTotal = stats.Int64(
+		"sigs.kubernetes.io/controller-runtime/measures/reconcile_total",
+		"Total number of reconciliations per controller",
+		stats.UnitNone,
+	)
+
+	// MeasureReconcileErrors is a counter which records the total
+	// number of errors from the Reconciler. It has one tag: controller refers
+	// to the controller name. TODO is this necessary when we have the above with
+	// error tag?
+	MeasureReconcileErrors = stats.Int64(
+		"sigs.kubernetes.io/controller-runtime/measures/reconcile_errors_total",
+		"Total number of reconciliation errors per controller",
+		stats.UnitNone,
+	)
+
+	// MeasureReconcileTime is a measure which keeps track of the duration
+	// of reconciliations. It has one tag: controller refers
+	// to the controller name. // TODO should this be milliseconds?
+	MeasureReconcileTime = stats.Float64(
+		"sigs.kubernetes.io/controller-runtime/measures/reconcile_time_seconds",
+		"Length of time per reconciliation per controller",
+		"s",
+	)
+
+	// Tag keys must conform to the restrictions described in
+	// go.opencensus.io/tag/validate.go. Currently those restrictions are:
+	// - length between 1 and 255 inclusive
+	// - characters are printable US-ASCII
+
+	// TagController is a tag referring to the controller name that produced a
+	// measurement.
+	TagController = mustNewTagKey("controller")
+
+	// TagResult is a tag referring to the reconcile result of a reconciler.
+	TagResult = mustNewTagKey("result")
+)
+
+func mustNewTagKey(k string) tag.Key {
+	tagKey, err := tag.NewKey(k)
+	if err != nil {
+		panic(err)
+	}
+	return tagKey
+}

pkg/internal/controller/controller.go

@@ -17,10 +17,13 @@ limitations under the License.
 package controller
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
 
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -29,6 +32,7 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
 	logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
@@ -173,10 +177,18 @@ func (c *Controller) Start(stop <-chan struct{}) error {
 func (c *Controller) processNextWorkItem() bool {
 	// This code copy-pasted from the sample-Controller.
 
+	// Create a context used for tagging OpenCensus measurements.
+	metricsCtx, _ := tag.New(context.Background(),
+		tag.Insert(metrics.TagController, c.Name),
+	)
+
 	// Update metrics after processing each item
+	// TODO if no reconcile occurred, should the time metric be updated?
 	reconcileStartTS := time.Now()
 	defer func() {
-		c.updateMetrics(time.Now().Sub(reconcileStartTS))
+		reconcileTime := time.Now().Sub(reconcileStartTS)
+		c.updatePromMetrics(reconcileTime)
+		c.updateOCMetrics(metricsCtx, reconcileTime)
 	}()
 
 	obj, shutdown := c.Queue.Get()
@@ -210,21 +222,31 @@ func (c *Controller) processNextWorkItem() bool {
 		return true
 	}
 
+	// Defer recording that a reconcile occurred. Result checks will set the
+	// correct tag in metricsCtx.
+	defer func() {
+		stats.Record(metricsCtx, metrics.MeasureReconcileTotal.M(1))
+	}()
 	// RunInformersAndControllers the syncHandler, passing it the namespace/Name string of the
 	// resource to be synced.
 	if result, err := c.Do.Reconcile(req); err != nil {
 		c.Queue.AddRateLimited(req)
 		log.Error(err, "Reconciler error", "controller", c.Name, "request", req)
 		ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
 		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "error").Inc()
+		stats.Record(metricsCtx, metrics.MeasureReconcileErrors.M(1))
+		metricsCtx, _ = tag.New(metricsCtx, tag.Insert(metrics.TagResult, "error"))
 		return false
 	} else if result.RequeueAfter > 0 {
 		c.Queue.AddAfter(req, result.RequeueAfter)
 		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue_after").Inc()
+		metricsCtx, _ = tag.New(metricsCtx, tag.Insert(metrics.TagResult, "requeue_after"))
 		return true
 	} else if result.Requeue {
 		c.Queue.AddRateLimited(req)
 		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue").Inc()
+		metricsCtx, _ = tag.New(metricsCtx, tag.Insert(metrics.TagResult, "requeue"))
+
 		return true
 	}
 
@@ -236,6 +258,7 @@ func (c *Controller) processNextWorkItem() bool {
 	log.V(1).Info("Successfully Reconciled", "controller", c.Name, "request", req)
 
 	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "success").Inc()
+	metricsCtx, _ = tag.New(metricsCtx, tag.Insert(metrics.TagResult, "success"))
 	// Return true, don't take a break
 	return true
 }
@@ -246,7 +269,12 @@ func (c *Controller) InjectFunc(f inject.Func) error {
 	return nil
 }
 
-// updateMetrics updates prometheus metrics within the controller
-func (c *Controller) updateMetrics(reconcileTime time.Duration) {
+// updatePromMetrics updates prometheus metrics within the controller
+func (c *Controller) updatePromMetrics(reconcileTime time.Duration) {
 	ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds())
 }
+
+// updateOCMetrics updates OpenCensus metrics within the controller
+func (c *Controller) updateOCMetrics(ctx context.Context, reconcileTime time.Duration) {
+	stats.Record(ctx, metrics.MeasureReconcileTime.M(reconcileTime.Seconds()))
+}