feat: argo_events_action_retries_failed_total metric (argoproj#3190)

.gitignore

@@ -15,3 +15,5 @@ debug.test
 site/
 /go-diagrams/
 argo-events
+.swo
+.swp

docs/metrics.md

@@ -84,6 +84,11 @@ How many actions have been triggered successfully.
 
 How many actions failed.
 
+#### argo_events_action_retries_failed_total
+
+How many actions failed after the retries have been exhausted.  
+This is also incremented if there is no `retryStrategy` specified.
+
 #### argo_events_action_duration_milliseconds
 
 Action triggering duration.
@@ -140,6 +145,7 @@ of monitoring your applications running with Argo Events.
   - `argo_events_events_processing_failed_total`
   - `argo_events_events_sent_failed_total`
   - `argo_events_action_failed_total`
+  - `argo_events_action_retries_failed_total`
 
 - Saturation
 

metrics/metrics.go

@@ -45,6 +45,7 @@ type Metrics struct {
 	eventProcessingDuration *prometheus.SummaryVec
 	actionTriggered         *prometheus.CounterVec
 	actionFailed            *prometheus.CounterVec
+	actionRetriesFailed     *prometheus.CounterVec
 	actionDuration          *prometheus.SummaryVec
 }
 
@@ -108,6 +109,14 @@ func NewMetrics(namespace string) *Metrics {
 				labelNamespace: namespace,
 			},
 		}, []string{labelSensorName, labelTriggerName}),
+		actionRetriesFailed: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: prefix,
+			Name:      "action_retries_failed_total",
+			Help:      "How many actions failed after the retries have been exhausted. https://argoproj.github.io/argo-events/metrics/#action_retries_failed_total",
+			ConstLabels: prometheus.Labels{
+				labelNamespace: namespace,
+			},
+		}, []string{labelSensorName, labelTriggerName}),
 		actionDuration: prometheus.NewSummaryVec(prometheus.SummaryOpts{
 			Namespace: prefix,
 			Name:      "action_duration_milliseconds",
@@ -127,6 +136,7 @@ func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
 	m.eventProcessingDuration.Collect(ch)
 	m.actionTriggered.Collect(ch)
 	m.actionFailed.Collect(ch)
+	m.actionRetriesFailed.Collect(ch)
 	m.actionDuration.Collect(ch)
 }
 
@@ -138,6 +148,7 @@ func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
 	m.eventProcessingDuration.Describe(ch)
 	m.actionTriggered.Describe(ch)
 	m.actionFailed.Describe(ch)
+	m.actionRetriesFailed.Describe(ch)
 	m.actionDuration.Describe(ch)
 }
 
@@ -173,6 +184,10 @@ func (m *Metrics) ActionFailed(sensorName, triggerName string) {
 	m.actionFailed.WithLabelValues(sensorName, triggerName).Inc()
 }
 
+func (m *Metrics) ActionRetriesFailed(sensorName, triggerName string) {
+	m.actionRetriesFailed.WithLabelValues(sensorName, triggerName).Inc()
+}
+
 func (m *Metrics) ActionDuration(sensorName, triggerName string, num float64) {
 	m.actionDuration.WithLabelValues(sensorName, triggerName).Observe(num)
 }

sensors/listener.go

@@ -220,6 +220,7 @@ func (sensorCtx *SensorContext) listenEvents(ctx context.Context) error {
 				})
 				if err != nil {
 					triggerLogger.Warnf("failed to trigger actions, %v", err)
+					sensorCtx.metrics.ActionRetriesFailed(sensor.Name, trigger.Template.Name)
 				}
 			}