reloader: Expose metrics to give info about last operation result/time (

thanos-io#4594)

Signed-off-by: Philip Gough <philip.p.gough@gmail.com>

CHANGELOG.md

@@ -20,6 +20,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4469](https://github.com/thanos-io/thanos/pull/4469) Compact: Add flag `compact.skip-block-with-out-of-order-chunks` to skip blocks with out-of-order chunks during compaction instead of halting
 - [#4506](https://github.com/thanos-io/thanos/pull/4506) `Baidu BOS` object storage, see [documents](docs/storage.md#baidu-bos) for further information.
 - [#4552](https://github.com/thanos-io/thanos/pull/4552) Compact: Adds `thanos_compact_downsample_duration_seconds` histogram.
+- [#4594](https://github.com/thanos-io/thanos/pull/4594) reloader: Expose metrics in config reloader to give info on the last operation.
 
 ### Fixed
 

pkg/reloader/reloader.go

@@ -7,7 +7,7 @@
 // Reloader type is useful when you want to:
 //
 // 	* Watch on changes against certain file e.g (`cfgFile`).
-// 	* Optionally, specify different different output file for watched `cfgFile` (`cfgOutputFile`).
+// 	* Optionally, specify different output file for watched `cfgFile` (`cfgOutputFile`).
 // 	This will also try decompress the `cfgFile` if needed and substitute ALL the envvars using Kubernetes substitution format: (`$(var)`)
 // 	* Watch on changes against certain directories (`watchedDirs`).
 //
@@ -95,10 +95,12 @@ type Reloader struct {
 	lastCfgHash         []byte
 	lastWatchedDirsHash []byte
 
-	reloads           prometheus.Counter
-	reloadErrors      prometheus.Counter
-	configApplyErrors prometheus.Counter
-	configApply       prometheus.Counter
+	reloads                    prometheus.Counter
+	reloadErrors               prometheus.Counter
+	lastReloadSuccess          prometheus.Gauge
+	lastReloadSuccessTimestamp prometheus.Gauge
+	configApplyErrors          prometheus.Counter
+	configApply                prometheus.Counter
 }
 
 // Options bundles options for the Reloader.
@@ -154,6 +156,18 @@ func New(logger log.Logger, reg prometheus.Registerer, o *Options) *Reloader {
 				Help: "Total number of reload requests that failed.",
 			},
 		),
+		lastReloadSuccess: promauto.With(reg).NewGauge(
+			prometheus.GaugeOpts{
+				Name: "reloader_last_reload_successful",
+				Help: "Whether the last reload attempt was successful",
+			},
+		),
+		lastReloadSuccessTimestamp: promauto.With(reg).NewGauge(
+			prometheus.GaugeOpts{
+				Name: "reloader_last_reload_success_timestamp_seconds",
+				Help: "Timestamp of the last successful reload",
+			},
+		),
 		configApply: promauto.With(reg).NewCounter(
 			prometheus.CounterOpts{
 				Name: "reloader_config_apply_operations_total",
@@ -243,6 +257,7 @@ func (r *Reloader) Watch(ctx context.Context) error {
 		if err := r.apply(applyCtx); err != nil {
 			r.configApplyErrors.Inc()
 			level.Error(r.logger).Log("msg", "apply error", "err", err)
+			continue
 		}
 	}
 }
@@ -347,6 +362,7 @@ func (r *Reloader) apply(ctx context.Context) error {
 		r.reloads.Inc()
 		if err := r.triggerReload(ctx); err != nil {
 			r.reloadErrors.Inc()
+			r.lastReloadSuccess.Set(0)
 			return errors.Wrap(err, "trigger reload")
 		}
 
@@ -357,6 +373,8 @@ func (r *Reloader) apply(ctx context.Context) error {
 			"cfg_in", r.cfgFile,
 			"cfg_out", r.cfgOutputFile,
 			"watched_dirs", strings.Join(r.watchedDirs, ", "))
+		r.lastReloadSuccess.Set(1)
+		r.lastReloadSuccessTimestamp.SetToCurrentTime()
 		return nil
 	}); err != nil {
 		level.Error(r.logger).Log("msg", "Failed to trigger reload. Retrying.", "err", err)