From bd7d904f6f051b1449550a0058d295cf149c3ba7 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kristoffer@tailscale.com>
Date: Fri, 17 May 2024 16:14:05 -0400
Subject: [PATCH] add debug metric for last sent to node

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
---
 hscontrol/metrics.go | 15 +++++++++++++++
 hscontrol/poll.go    |  6 ++++++
 2 files changed, 21 insertions(+)

diff --git a/hscontrol/metrics.go b/hscontrol/metrics.go
index 0e18ed3464..835a6aac0a 100644
--- a/hscontrol/metrics.go
+++ b/hscontrol/metrics.go
@@ -7,8 +7,23 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"tailscale.com/envknob"
 )
 
+var debugHighCardinalityMetrics = envknob.Bool("HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS")
+
+var mapResponseLastSentSeconds *prometheus.GaugeVec
+
+func init() {
+	if debugHighCardinalityMetrics {
+		mapResponseLastSentSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: prometheusNamespace,
+			Name:      "mapresponse_last_sent_seconds",
+			Help:      "last sent metric to node.id",
+		}, []string{"type", "id"})
+	}
+}
+
 const prometheusNamespace = "headscale"
 
 var (
diff --git a/hscontrol/poll.go b/hscontrol/poll.go
index 377c7c3885..557a5cd4e9 100644
--- a/hscontrol/poll.go
+++ b/hscontrol/poll.go
@@ -344,6 +344,9 @@ func (m *mapSession) serveLongPoll() {
 
 				log.Trace().Str("node", m.node.Hostname).TimeDiff("timeSpent", time.Now(), startWrite).Str("mkey", m.node.MachineKey.String()).Msg("finished writing mapresp to node")
 
+				if debugHighCardinalityMetrics {
+					mapResponseLastSentSeconds.WithLabelValues(updateType, m.node.ID.String()).Set(float64(time.Now().Unix()))
+				}
 				mapResponseSent.WithLabelValues("ok", updateType).Inc()
 				m.tracef("update sent")
 				m.resetKeepAlive()
@@ -369,6 +372,9 @@ func (m *mapSession) serveLongPoll() {
 				return
 			}
 
+			if debugHighCardinalityMetrics {
+				mapResponseLastSentSeconds.WithLabelValues("keepalive", m.node.ID.String()).Set(float64(time.Now().Unix()))
+			}
 			mapResponseSent.WithLabelValues("ok", "keepalive").Inc()
 		}
 	}