Document and fix recorded metric forwarding

This should address most of #104

- fix target cache lookup for recorded metrics
- update documentation with examples

Also: allow overriding docker FROM target with DOCKER_IMAGE_BASE env var
(helpful for testing and runtime scenarios where /bin/sh is needed)

Dockerfile

@@ -1,4 +1,5 @@
-FROM  gcr.io/distroless/static:latest
+ARG DOCKER_IMAGE_BASE=gcr.io/distroless/static:latest
+FROM  $DOCKER_IMAGE_BASE
 LABEL maintainer "Stackdriver Engineering <engineering@stackdriver.com>"
 
 COPY stackdriver-prometheus-sidecar         /bin/stackdriver-prometheus-sidecar

Makefile

@@ -43,6 +43,7 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 PREFIX                  ?= $(shell pwd)
 BIN_DIR                 ?= $(shell pwd)
 # Private repo.
+DOCKER_IMAGE_BASE       ?= gcr.io/distroless/static:latest
 DOCKER_IMAGE_NAME       ?= gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar
 DOCKER_IMAGE_TAG        ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
 
@@ -110,7 +111,7 @@ tarball: promu
 
 docker: build-linux-amd64
 	@echo ">> building docker image"
-	docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
+	docker build --build-arg "DOCKER_IMAGE_BASE=$(DOCKER_IMAGE_BASE)" -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
 
 push: test docker
 	@echo ">> pushing docker image"

README.md

@@ -86,6 +86,28 @@ static_metadata:
   * All `static_metadata` entries must have `type` specified. This specifies the Stackdriver metric type and overrides the metric type chosen by the Prometheus client.
   * If `value_type` is specified, it will override the default value type for counters and gauges. All Prometheus metrics have a default type of double.
 
+#### Dealing with recording rules
+
+The default Prometheus naming format for [recording rules](https://prometheus.io/docs/practices/rules/) is `level:metric:operations`, e.g. `instance:requests_total:sum`, but colons are not allowed in Stackdriver metric descriptor names.  To forward a recorded Prometheus metric to Stackdriver, you must use the `metric_renames` feature to replace the colon characters:
+
+```yaml
+metric_renames:
+  - from: instance:requests_total:sum
+    to: recorded_instance_requests_total_sum
+```
+
+Additionally, the sidecar assumes that any recorded metrics are gauges.  If this is not the case (e.g. for a Prometheus counter metric) you will need to specify that in `static_metadata`:
+
+```yaml
+static_metadata:
+  - metric: recorded_instance_requests_total_sum
+    type: counter
+    value_type: int64
+    help: an arbitrary help string
+```
+
+*Warning:* recorded metrics _must_ have minimally an "instance" and "job" label, otherwise they will not be forwarded.
+
 #### Counter Aggregator
 
 Counter Aggregator is an advanced feature of the sidecar that can be used to export a sum of multiple Prometheus counters to Stackdriver as a single CUMULATIVE metric.

targets/cache.go

@@ -179,7 +179,7 @@ func targetMatch(targets []*Target, lset labels.Labels) (*Target, bool) {
 Outer:
 	for _, t := range targets {
 		for _, tl := range t.Labels {
-			if lset.Get(tl.Name) != tl.Value {
+			if v := lset.Get(tl.Name); v != "" && v != tl.Value {
 				continue Outer
 			}
 		}