Initialize schedule Prometheus metrics to have them created beforehan…

…d (see https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics) Signed-off-by: Alex Lemaresquier <alex+github@lemaresquier.org>
vmware-tanzu · Aug 10, 2018 · 9f80f01 · 9f80f01
1 parent 0acd368
commit 9f80f01
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 0 deletions.
diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go
@@ -651,6 +651,7 @@ func (s *server) runControllers(config *api.Config) error {
 			s.sharedInformerFactory.Ark().V1().Schedules(),
 			config.ScheduleSyncPeriod.Duration,
 			s.logger,
+			s.metrics,
 		)
 		wg.Add(1)
 		go func() {

diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go
@@ -41,6 +41,7 @@ import (
 	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/metrics"
 	kubeutil "github.com/heptio/ark/pkg/util/kube"
 )
 
@@ -55,6 +56,7 @@ type scheduleController struct {
 	syncPeriod            time.Duration
 	clock                 clock.Clock
 	logger                logrus.FieldLogger
+	metrics               *metrics.ServerMetrics
 }
 
 func NewScheduleController(
@@ -64,6 +66,7 @@ func NewScheduleController(
 	schedulesInformer informers.ScheduleInformer,
 	syncPeriod time.Duration,
 	logger logrus.FieldLogger,
+	metrics *metrics.ServerMetrics,
 ) *scheduleController {
 	if syncPeriod < time.Minute {
 		logger.WithField("syncPeriod", syncPeriod).Info("Provided schedule sync period is too short. Setting to 1 minute")
@@ -80,6 +83,7 @@ func NewScheduleController(
 		syncPeriod: syncPeriod,
 		clock:      clock.RealClock{},
 		logger:     logger,
+		metrics:    metrics,
 	}
 
 	c.syncHandler = c.processSchedule
@@ -106,6 +110,10 @@ func NewScheduleController(
 					return
 				}
 				c.queue.Add(key)
+				scheduleName := schedule.GetName()
+				c.logger.Info("Creating schedule ", scheduleName)
+				//Init Prometheus metrics to 0 to have them flowing up
+				metrics.InitSchedule(scheduleName)
 			},
 		},
 	)

diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go
@@ -34,6 +34,7 @@ import (
 	api "github.com/heptio/ark/pkg/apis/ark/v1"
 	"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
+	"github.com/heptio/ark/pkg/metrics"
 	"github.com/heptio/ark/pkg/util/collections"
 	arktest "github.com/heptio/ark/pkg/util/test"
 )
@@ -129,6 +130,7 @@ func TestProcessSchedule(t *testing.T) {
 				sharedInformers.Ark().V1().Schedules(),
 				time.Duration(0),
 				logger,
+				metrics.NewServerMetrics(),
 			)
 
 			var (

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -106,6 +106,18 @@ func (m *ServerMetrics) RegisterAllMetrics() {
 	}
 }
 
+func (m *ServerMetrics) InitSchedule(scheduleName string) {
+	if c, ok := m.metrics[backupAttemptCount].(*prometheus.CounterVec); ok {
+		c.WithLabelValues(scheduleName).Set(0)
+	}
+	if c, ok := m.metrics[backupSuccessCount].(*prometheus.CounterVec); ok {
+		c.WithLabelValues(scheduleName).Set(0)
+	}
+	if c, ok := m.metrics[backupFailureCount].(*prometheus.CounterVec); ok {
+		c.WithLabelValues(scheduleName).Set(0)
+	}
+}
+
 // SetBackupTarballSizeBytesGauge records the size, in bytes, of a backup tarball.
 func (m *ServerMetrics) SetBackupTarballSizeBytesGauge(backupSchedule string, size int64) {
 	if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok {