From a8c2f78463c1afc5675c7b3cac796da73c4fa9d5 Mon Sep 17 00:00:00 2001
From: Nathan J Mehl <70606471+n-oden@users.noreply.github.com>
Date: Fri, 29 Apr 2022 10:49:09 -0400
Subject: [PATCH] Document and fix recorded metric forwarding (#266)

This should address most of https://github.com/Stackdriver/stackdriver-prometheus-sidecar/issues/104

- fix target cache lookup for recorded metrics
- update documentation with examples

Also: allow overriding docker FROM target with DOCKER_IMAGE_BASE env var
(helpful for testing and runtime scenarios where /bin/sh is needed)
---
 Dockerfile       |  3 ++-
 Makefile         |  3 ++-
 README.md        | 24 +++++++++++++++++++++++-
 targets/cache.go |  2 +-
 4 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 858318d6..223049b3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,5 @@
-FROM  gcr.io/distroless/static:latest
+ARG DOCKER_IMAGE_BASE=gcr.io/distroless/static:latest
+FROM  $DOCKER_IMAGE_BASE
 LABEL maintainer "Stackdriver Engineering <engineering@stackdriver.com>"
 
 COPY stackdriver-prometheus-sidecar         /bin/stackdriver-prometheus-sidecar
diff --git a/Makefile b/Makefile
index f4bcf90f..b14e5193 100644
--- a/Makefile
+++ b/Makefile
@@ -43,6 +43,7 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 PREFIX                  ?= $(shell pwd)
 BIN_DIR                 ?= $(shell pwd)
 # Private repo.
+DOCKER_IMAGE_BASE       ?= gcr.io/distroless/static:latest
 DOCKER_IMAGE_NAME       ?= gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar
 DOCKER_IMAGE_TAG        ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
 
@@ -110,7 +111,7 @@ tarball: promu
 
 docker: build-linux-amd64
 	@echo ">> building docker image"
-	docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
+	docker build --build-arg "DOCKER_IMAGE_BASE=$(DOCKER_IMAGE_BASE)" -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
 
 push: test docker
 	@echo ">> pushing docker image"
diff --git a/README.md b/README.md
index 0d847e84..9c46779b 100644
--- a/README.md
+++ b/README.md
@@ -70,7 +70,7 @@ stackdriver-prometheus-sidecar --include='{__name__!~"cadvisor_.+",job="k8s"}' .
 
 This drops all series which do not have a `job` label `k8s` and all metrics that have a name starting with `cadvisor_`.
 
-For equality filter on metric name you can use the simpler notation, e.g. `--include='metric_name{label="foo"}'`.
+For equality filter on metric name you can use the simpler notation, e.g., `--include='metric_name{label="foo"}'`.
 
 The flag may be repeated to provide several sets of filters, in which case the metric will be forwarded if it matches at least one of them. Please note that inclusion filters only apply to Prometheus metrics proxied directly, and do not apply to [aggregated counters](#counter-aggregator).
 
@@ -95,6 +95,28 @@ static_metadata:
   * All `static_metadata` entries must have `type` specified. This specifies the Stackdriver metric type and overrides the metric type chosen by the Prometheus client.
   * If `value_type` is specified, it will override the default value type for counters and gauges. All Prometheus metrics have a default type of double.
 
+#### Dealing with recording rules
+
+The default Prometheus naming format for [recording rules](https://prometheus.io/docs/practices/rules/) is `level:metric:operations`, e.g., `instance:requests_total:sum`, but colons are not allowed in Stackdriver metric descriptor names.  To forward a recorded Prometheus metric to Stackdriver, you must use the `metric_renames` feature to replace the colon characters:
+
+```yaml
+metric_renames:
+  - from: instance:requests_total:sum
+    to: recorded_instance_requests_total_sum
+```
+
+Additionally, the sidecar assumes that any recorded metrics are gauges.  If this is not the case (e.g., for a Prometheus counter metric) you will need to specify that in `static_metadata`:
+
+```yaml
+static_metadata:
+  - metric: recorded_instance_requests_total_sum
+    type: counter
+    value_type: int64
+    help: an arbitrary help string
+```
+
+*Warning:* recorded metrics _must_ have minimally an "instance" and "job" label, otherwise they will not be forwarded.
+
 #### Counter Aggregator
 
 Counter Aggregator is an advanced feature of the sidecar that can be used to export a sum of multiple Prometheus counters to Stackdriver as a single CUMULATIVE metric.
diff --git a/targets/cache.go b/targets/cache.go
index 17789d2f..f89cf738 100644
--- a/targets/cache.go
+++ b/targets/cache.go
@@ -179,7 +179,7 @@ func targetMatch(targets []*Target, lset labels.Labels) (*Target, bool) {
 Outer:
 	for _, t := range targets {
 		for _, tl := range t.Labels {
-			if lset.Get(tl.Name) != tl.Value {
+			if v := lset.Get(tl.Name); v != "" && v != tl.Value {
 				continue Outer
 			}
 		}