Remove in memory sink from fanout and add latency sampling methods

store/iavl/store.go

@@ -202,7 +202,7 @@ func (st *Store) Set(key, value []byte) {
 
 // Implements types.KVStore.
 func (st *Store) Get(key []byte) []byte {
-	defer telemetry.MeasureSince(time.Now(), "store", "iavl", "get")
+	defer telemetry.MeasureSinceWithSampling(time.Now(), 0.01, "store", "iavl", "get")
 	value, err := st.tree.Get(key)
 	if err != nil {
 		panic(err)

telemetry/metrics.go

@@ -98,7 +98,7 @@ func New(cfg Config) (_ *Metrics, rerr error) {
 	}()
 
 	m := &Metrics{memSink: memSink}
-	fanout := metrics.FanoutSink{memSink}
+	fanout := metrics.FanoutSink{}
 
 	if cfg.PrometheusRetentionTime > 0 {
 		m.prometheusEnabled = true
@@ -112,6 +112,8 @@ func New(cfg Config) (_ *Metrics, rerr error) {
 		}
 
 		fanout = append(fanout, promSink)
+	} else {
+		fanout = append(fanout, memSink)
 	}
 
 	if _, err := metrics.NewGlobal(metricsConf, fanout); err != nil {

telemetry/wrapper.go

@@ -1,6 +1,7 @@
 package telemetry
 
 import (
+	"math/rand"
 	"time"
 
 	"github.com/armon/go-metrics"
@@ -19,6 +20,18 @@ func NewLabel(name, value string) metrics.Label {
 	return metrics.Label{Name: name, Value: value}
 }
 
+// ModuleMeasureSinceWithSampling samples latency metrics given the sample rate.
+// This is intended to be used in hot code paths.
+func ModuleMeasureSinceWithSampling(module string, start time.Time, sampleRate float64, keys ...string) {
+	if rand.Float64() < sampleRate {
+		metrics.MeasureSinceWithLabels(
+			keys,
+			start.UTC(),
+			append([]metrics.Label{NewLabel(MetricLabelNameModule, module)}, globalLabels...),
+		)
+	}
+}
+
 // ModuleMeasureSince provides a short hand method for emitting a time measure
 // metric for a module with a given set of keys. If any global labels are defined,
 // they will be added to the module label.
@@ -70,3 +83,11 @@ func SetGaugeWithLabels(keys []string, val float32, labels []metrics.Label) {
 func MeasureSince(start time.Time, keys ...string) {
 	metrics.MeasureSinceWithLabels(keys, start.UTC(), globalLabels)
 }
+
+// MeasureSinceWithSampling provides a wrapper functionality for emitting a a time measure
+// metric with sampling.
+func MeasureSinceWithSampling(start time.Time, sampleRate float64, keys ...string) {
+	if rand.Float64() < sampleRate {
+		metrics.MeasureSinceWithLabels(keys, start.UTC(), globalLabels)
+	}
+}