From 540205509c24f79581cea6d08c7226eeb31785f5 Mon Sep 17 00:00:00 2001
From: Jay Yu <103467857+jayy04@users.noreply.github.com>
Date: Tue, 10 Oct 2023 16:14:40 -0400
Subject: [PATCH] Remove in memory sink from fanout and add latency sampling
 methods

---
 store/iavl/store.go  |  2 +-
 telemetry/metrics.go |  4 +++-
 telemetry/wrapper.go | 21 +++++++++++++++++++++
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/store/iavl/store.go b/store/iavl/store.go
index 77f61138053..e7fc71983dc 100644
--- a/store/iavl/store.go
+++ b/store/iavl/store.go
@@ -202,7 +202,7 @@ func (st *Store) Set(key, value []byte) {
 
 // Implements types.KVStore.
 func (st *Store) Get(key []byte) []byte {
-	defer telemetry.MeasureSince(time.Now(), "store", "iavl", "get")
+	defer telemetry.MeasureSinceWithSampling(time.Now(), 0.01, "store", "iavl", "get")
 	value, err := st.tree.Get(key)
 	if err != nil {
 		panic(err)
diff --git a/telemetry/metrics.go b/telemetry/metrics.go
index a0cb15b11fa..c27dd5c5aed 100644
--- a/telemetry/metrics.go
+++ b/telemetry/metrics.go
@@ -98,7 +98,7 @@ func New(cfg Config) (_ *Metrics, rerr error) {
 	}()
 
 	m := &Metrics{memSink: memSink}
-	fanout := metrics.FanoutSink{memSink}
+	fanout := metrics.FanoutSink{}
 
 	if cfg.PrometheusRetentionTime > 0 {
 		m.prometheusEnabled = true
@@ -112,6 +112,8 @@ func New(cfg Config) (_ *Metrics, rerr error) {
 		}
 
 		fanout = append(fanout, promSink)
+	} else {
+		fanout = append(fanout, memSink)
 	}
 
 	if _, err := metrics.NewGlobal(metricsConf, fanout); err != nil {
diff --git a/telemetry/wrapper.go b/telemetry/wrapper.go
index fa6aae10221..7a429b7ee93 100644
--- a/telemetry/wrapper.go
+++ b/telemetry/wrapper.go
@@ -1,6 +1,7 @@
 package telemetry
 
 import (
+	"math/rand"
 	"time"
 
 	"github.com/armon/go-metrics"
@@ -19,6 +20,18 @@ func NewLabel(name, value string) metrics.Label {
 	return metrics.Label{Name: name, Value: value}
 }
 
+// ModuleMeasureSinceWithSampling samples latency metrics given the sample rate.
+// This is intended to be used in hot code paths.
+func ModuleMeasureSinceWithSampling(module string, start time.Time, sampleRate float64, keys ...string) {
+	if rand.Float64() < sampleRate {
+		metrics.MeasureSinceWithLabels(
+			keys,
+			start.UTC(),
+			append([]metrics.Label{NewLabel(MetricLabelNameModule, module)}, globalLabels...),
+		)
+	}
+}
+
 // ModuleMeasureSince provides a short hand method for emitting a time measure
 // metric for a module with a given set of keys. If any global labels are defined,
 // they will be added to the module label.
@@ -70,3 +83,11 @@ func SetGaugeWithLabels(keys []string, val float32, labels []metrics.Label) {
 func MeasureSince(start time.Time, keys ...string) {
 	metrics.MeasureSinceWithLabels(keys, start.UTC(), globalLabels)
 }
+
+// MeasureSinceWithSampling provides a wrapper functionality for emitting a a time measure
+// metric with sampling.
+func MeasureSinceWithSampling(start time.Time, sampleRate float64, keys ...string) {
+	if rand.Float64() < sampleRate {
+		metrics.MeasureSinceWithLabels(keys, start.UTC(), globalLabels)
+	}
+}