From e07e7454b58b2ff630a5a169fccab5cdf851e846 Mon Sep 17 00:00:00 2001
From: "Nathan J. Mehl" <n@oden.io>
Date: Fri, 15 Jan 2021 15:57:38 -0500
Subject: [PATCH] Document and fix recorded metric forwarding

This should address most of https://github.com/Stackdriver/stackdriver-prometheus-sidecar/issues/104

- fix target cache lookup for recorded metrics
- update documentation with examples

Also: allow overriding docker FROM target with DOCKER_IMAGE_BASE env var
(helpful for testing and runtime scenarios where /bin/sh is needed)
---
 Dockerfile       |  3 ++-
 Makefile         |  3 ++-
 README.md        | 22 ++++++++++++++++++++++
 targets/cache.go |  2 +-
 4 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 858318d6..223049b3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,5 @@
-FROM  gcr.io/distroless/static:latest
+ARG DOCKER_IMAGE_BASE=gcr.io/distroless/static:latest
+FROM  $DOCKER_IMAGE_BASE
 LABEL maintainer "Stackdriver Engineering <engineering@stackdriver.com>"
 
 COPY stackdriver-prometheus-sidecar         /bin/stackdriver-prometheus-sidecar
diff --git a/Makefile b/Makefile
index f4bcf90f..b14e5193 100644
--- a/Makefile
+++ b/Makefile
@@ -43,6 +43,7 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 PREFIX                  ?= $(shell pwd)
 BIN_DIR                 ?= $(shell pwd)
 # Private repo.
+DOCKER_IMAGE_BASE       ?= gcr.io/distroless/static:latest
 DOCKER_IMAGE_NAME       ?= gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar
 DOCKER_IMAGE_TAG        ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
 
@@ -110,7 +111,7 @@ tarball: promu
 
 docker: build-linux-amd64
 	@echo ">> building docker image"
-	docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
+	docker build --build-arg "DOCKER_IMAGE_BASE=$(DOCKER_IMAGE_BASE)" -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
 
 push: test docker
 	@echo ">> pushing docker image"
diff --git a/README.md b/README.md
index c5603c96..6f340180 100644
--- a/README.md
+++ b/README.md
@@ -86,6 +86,28 @@ static_metadata:
   * All `static_metadata` entries must have `type` specified. This specifies the Stackdriver metric type and overrides the metric type chosen by the Prometheus client.
   * If `value_type` is specified, it will override the default value type for counters and gauges. All Prometheus metrics have a default type of double.
 
+#### Dealing with recording rules
+
+The default Prometheus naming format for [recording rules](https://prometheus.io/docs/practices/rules/) is `level:metric:operations`, e.g. `instance:requests_total:sum`, but colons are not allowed in Stackdriver metric descriptor names.  To forward a recorded Prometheus metric to Stackdriver, you must use the `metric_renames` feature to replace the colon characters:
+
+```yaml
+metric_renames:
+  - from: instance:requests_total:sum
+    to: recorded_instance_requests_total_sum
+```
+
+Additionally, the sidecar assumes that any recorded metrics are gauges.  If this is not the case (e.g. for a Prometheus counter metric) you will need to specify that in `static_metadata`:
+
+```yaml
+static_metadata:
+  - metric: recorded_instance_requests_total_sum
+    type: counter
+    value_type: int64
+    help: an arbitrary help string
+```
+
+*Warning:* recorded metrics _must_ have minimally an "instance" and "job" label, otherwise they will not be forwarded.
+
 #### Counter Aggregator
 
 Counter Aggregator is an advanced feature of the sidecar that can be used to export a sum of multiple Prometheus counters to Stackdriver as a single CUMULATIVE metric.
diff --git a/targets/cache.go b/targets/cache.go
index 17789d2f..f89cf738 100644
--- a/targets/cache.go
+++ b/targets/cache.go
@@ -179,7 +179,7 @@ func targetMatch(targets []*Target, lset labels.Labels) (*Target, bool) {
 Outer:
 	for _, t := range targets {
 		for _, tl := range t.Labels {
-			if lset.Get(tl.Name) != tl.Value {
+			if v := lset.Get(tl.Name); v != "" && v != tl.Value {
 				continue Outer
 			}
 		}