POC tracking job metrics calling DB Api

controllers/djob_controller_databricks.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 
 	databricksv1alpha1 "github.com/microsoft/azure-databricks-operator/api/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus"
+	models "github.com/xinsnake/databricks-sdk-golang/azure/models"
 	"k8s.io/apimachinery/pkg/types"
 )
 
@@ -31,7 +33,8 @@ func (r *DjobReconciler) submit(instance *databricksv1alpha1.Djob) error {
 
 	instance.Spec.Name = instance.GetName()
 
-	job, err := r.APIClient.Jobs().Create(*instance.Spec)
+	job, err := createJob(r, instance)
+
 	if err != nil {
 		return err
 	}
@@ -48,7 +51,8 @@ func (r *DjobReconciler) refresh(instance *databricksv1alpha1.Djob) error {
 
 	jobID := instance.Status.JobStatus.JobID
 
-	job, err := r.APIClient.Jobs().Get(jobID)
+	job, err := getJob(r, jobID)
+
 	if err != nil {
 		return err
 	}
@@ -88,13 +92,44 @@ func (r *DjobReconciler) delete(instance *databricksv1alpha1.Djob) error {
 
 	jobID := instance.Status.JobStatus.JobID
 
+	_, err := getJob(r, jobID)
+
 	// Check if the job exists before trying to delete it
-	if _, err := r.APIClient.Jobs().Get(jobID); err != nil {
+	if err != nil {
 		if strings.Contains(err.Error(), "does not exist") {
 			return nil
 		}
 		return err
 	}
 
-	return r.APIClient.Jobs().Delete(jobID)
+	return trackExecutionTime(djobDeleteDuration, func() error {
+		return r.APIClient.Jobs().Delete(jobID)
+	})
+}
+
+func getJob(r *DjobReconciler, jobID int64) (job models.Job, err error) {
+	return trackJob(djobGetDuration, djobGetSuccess, djobGetFailure, func() (models.Job, error) {
+		return r.APIClient.Jobs().Get(jobID)
+	})
+}
+
+func createJob(r *DjobReconciler, instance *databricksv1alpha1.Djob) (job models.Job, err error) {
+	return trackJob(djobCreateDuration, djobCreateSuccess, djobCreateFailure, func() (models.Job, error) {
+		return r.APIClient.Jobs().Create(*instance.Spec)
+	})
+}
+
+func trackJob(duration prometheus.Histogram, success prometheus.Counter, failure prometheus.Counter, 
+	f func() (models.Job, error)) (job models.Job, err error) {
+		job, err = trackJobExecutionTime(duration, func() (models.Job, error) {
+			return f()
+		})
+
+		if err != nil {
+			failure.Inc()
+		} else {
+			success.Inc()
+		}
+
+		return
 }

controllers/djob_metrics.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2019 microsoft.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	djobCreateSuccess = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "djob_success_total",
+			Help: "Number of create djob success",
+		},
+	)
+	djobCreateFailure = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "djob_failures_total",
+			Help: "Number of create djob failures",
+		},
+	)
+
+	djobGetSuccess = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "djob_success_total",
+			Help: "Number of create djob success",
+		},
+	)
+	djobGetFailure = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "djob_failures_total",
+			Help: "Number of create djob failures",
+		},
+	)
+
+	djobCreateDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "djob_creation_duration",
+		Help:    "Duration of DB api create calls.",
+		Buckets: prometheus.LinearBuckets(100, 10, 20), 
+	})
+
+	djobGetDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "djob_get_duration",
+		Help:    "Duration of DB api get calls.",
+		Buckets: prometheus.LinearBuckets(100, 10, 20),  // 5 buckets, each 5 centigrade wide.
+	})
+
+	djobDeleteDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "djob_delete_duration",
+		Help:    "Duration of DB api delete calls.",
+		Buckets: prometheus.LinearBuckets(100, 10, 20),  // 5 buckets, each 5 centigrade wide.
+	})
+)
+
+func init() {
+	// Register custom metrics with the global prometheus registry
+	metrics.Registry.MustRegister(djobCreateSuccess, djobCreateFailure, djobGetDuration)
+}

controllers/metrics.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2019 microsoft.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import(
+	"time"
+	"github.com/prometheus/client_golang/prometheus"
+	models "github.com/xinsnake/databricks-sdk-golang/azure/models"
+)
+
+func trackJobExecutionTime(histogram prometheus.Histogram, f func() (models.Job, error)) (models.Job, error) {
+	startTime := time.Now()
+
+	defer trackMetric(startTime, histogram)
+
+	return f()
+}
+
+func trackExecutionTime(histogram prometheus.Histogram, f func() error) error {
+	startTime := time.Now()
+
+	defer trackMetric(startTime, histogram)
+
+	return f()
+}
+
+func trackMetric(startTime time.Time, histogram prometheus.Histogram) {
+	endTime := float64(time.Now().Sub(startTime).Nanoseconds() / int64(time.Millisecond))
+	histogram.Observe(endTime)
+}