diff --git a/README.md b/README.md index 34901e52..e5d73d36 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,14 @@ static_metadata: # - ... ``` +#### Forwarding recorded metrics + +The [suggested format](https://prometheus.io/docs/practices/rules/) for Prometheus metrics generated by recording rules is `level:metric:operation`, but that name format is not compatible with Stackdriver: Stackdriver's [metric name rules](https://cloud.google.com/monitoring/api/v3/metrics-details#label_names) specify that only upper and lowercase letters, digits and underscores may be used in metric names. + +The sidecar will, therefore, treat any Prometheus metric name prefixed with the value of the `--stackdriver.recorded-metric-prefix` flag (by default, `recorded_`) as a recorded metric, which will be created as a gauge on the Stackdriver side. + +Note also that it is not currently possible to forward recorded metrics that lack an `instance` and `job` label, as those tags are used as cache keys. + #### Counter Aggregator Counter Aggregator is an advanced feature of the sidecar that can be used to export a sum of multiple Prometheus counters to Stackdriver as a single CUMULATIVE metric. diff --git a/cmd/stackdriver-prometheus-sidecar/main.go b/cmd/stackdriver-prometheus-sidecar/main.go index 4d237a1d..61044920 100644 --- a/cmd/stackdriver-prometheus-sidecar/main.go +++ b/cmd/stackdriver-prometheus-sidecar/main.go @@ -182,6 +182,7 @@ type mainConfig struct { GenericLabels genericConfig StackdriverAddress *url.URL MetricsPrefix string + RecordedMetricPrefix string UseGKEResource bool StoreInFilesDirectory string WALDirectory string @@ -240,6 +241,9 @@ func main() { a.Flag("stackdriver.metrics-prefix", "Customized prefix for Stackdriver metrics. If not set, external.googleapis.com/prometheus will be used"). StringVar(&cfg.MetricsPrefix) + a.Flag("stackdriver.recorded-metric-prefix", "Prometheus metric name prefix used to detect recorded metrics. If not set, 'recorded_' will be used."). + StringVar(&cfg.RecordedMetricPrefix) + a.Flag("stackdriver.use-gke-resource", "Whether to use the legacy gke_container MonitoredResource type instead of k8s_container"). Default("false").BoolVar(&cfg.UseGKEResource) @@ -383,7 +387,7 @@ func main() { if err != nil { panic(err) } - metadataCache := metadata.NewCache(httpClient, metadataURL, cfg.StaticMetadata) + metadataCache := metadata.NewCache(httpClient, metadataURL, cfg.RecordedMetricPrefix, cfg.StaticMetadata) // We instantiate a context here since the tailer is used by two other components. // The context will be used in the lifecycle of prometheusReader further down. diff --git a/metadata/cache.go b/metadata/cache.go index 703e8628..692a4c12 100644 --- a/metadata/cache.go +++ b/metadata/cache.go @@ -37,9 +37,10 @@ type Cache struct { promURL *url.URL client *http.Client - metadata map[string]*metadataEntry - seenJobs map[string]struct{} - staticMetadata map[string]scrape.MetricMetadata + metadata map[string]*metadataEntry + seenJobs map[string]struct{} + staticMetadata map[string]scrape.MetricMetadata + recordedMetricPrefix string } // DefaultEndpointPath is the default HTTP path on which Prometheus serves @@ -53,16 +54,17 @@ const MetricTypeUntyped = "untyped" // NewCache returns a new cache that gets populated by the metadata endpoint // at the given URL. // It uses the default endpoint path if no specific path is provided. -func NewCache(client *http.Client, promURL *url.URL, staticMetadata []scrape.MetricMetadata) *Cache { +func NewCache(client *http.Client, promURL *url.URL, recordedMetricPrefix string, staticMetadata []scrape.MetricMetadata) *Cache { if client == nil { client = http.DefaultClient } c := &Cache{ - promURL: promURL, - client: client, - staticMetadata: map[string]scrape.MetricMetadata{}, - metadata: map[string]*metadataEntry{}, - seenJobs: map[string]struct{}{}, + promURL: promURL, + client: client, + recordedMetricPrefix: recordedMetricPrefix, + staticMetadata: map[string]scrape.MetricMetadata{}, + metadata: map[string]*metadataEntry{}, + seenJobs: map[string]struct{}{}, } for _, m := range staticMetadata { c.staticMetadata[m.Metric] = m @@ -121,10 +123,18 @@ func (c *Cache) Get(ctx context.Context, job, instance, metric string) (*scrape. if md != nil && md.found { return &md.MetricMetadata, nil } - // The metric might also be produced by a recording rule, which by convention - // contain at least one `:` character. In that case we can generally assume that - // it is a gauge. We leave the help text empty. + + // The suggested format for recorded metric names is `level:metric:operation`, + // but stackdriver metric names cannot have colon characters in them, so + // return an error. if strings.Contains(metric, ":") { + return nil, errors.New(fmt.Sprintf("metric name '%s' cannot be forwarded due to illegal characters", metric)) + } + + // Treat metric names prefixed with the flagged prefix as recorded metrics. In that + // case we can generally assume that it is a gauge. We leave the help text + // empty. + if strings.HasPrefix(metric, c.recordedMetricPrefix) { return &scrape.MetricMetadata{ Metric: metric, Type: textparse.MetricTypeGauge, diff --git a/metadata/cache_test.go b/metadata/cache_test.go index 7564d610..6bbb077a 100644 --- a/metadata/cache_test.go +++ b/metadata/cache_test.go @@ -67,7 +67,7 @@ func TestCache_Get(t *testing.T) { t.Fatal(err) } // Create cache with static metadata. - c := NewCache(nil, u, []scrape.MetricMetadata{ + c := NewCache(nil, u, "recorded_", []scrape.MetricMetadata{ {Metric: "static_metric1", Type: textparse.MetricTypeCounter, Help: "help_static1"}, {Metric: "static_metric2", Type: textparse.MetricTypeCounter, Help: "help_static2"}, {Metric: "metric_with_override", Type: textparse.MetricTypeCounter, Help: "help_metric_override"}, @@ -237,17 +237,26 @@ func TestCache_Get(t *testing.T) { handler = func(qMetric, qMatch string) *apiResponse { return nil } - md, err = c.Get(ctx, "prometheus", "localhost:9090", "some:recording:rule") + md, err = c.Get(ctx, "prometheus", "localhost:9090", "recorded_some_rule") if err != nil { t.Fatal(err) } want = &scrape.MetricMetadata{ - Metric: "some:recording:rule", + Metric: "recorded_some_rule", Type: textparse.MetricTypeGauge, } if !reflect.DeepEqual(md, want) { t.Fatalf("expected metadata %v but got %v", want, md) } + + // Test prometheus-style recording rule + handler = func(qMetric, qMatch string) *apiResponse { + return nil + } + md, err = c.Get(ctx, "prometheus", "localhost:9090", "some:recording:rule") + if err == nil { + t.Fatal(err) + } } func TestNewCache(t *testing.T) { @@ -255,7 +264,7 @@ func TestNewCache(t *testing.T) { {Metric: "a", Help: "a"}, {Metric: "b", Help: "b"}, } - c := NewCache(nil, nil, static) + c := NewCache(nil, nil, "recorded_", static) want := map[string]scrape.MetricMetadata{ "a": {Metric: "a", Help: "a"}, diff --git a/targets/cache.go b/targets/cache.go index 17789d2f..db5386af 100644 --- a/targets/cache.go +++ b/targets/cache.go @@ -179,7 +179,8 @@ func targetMatch(targets []*Target, lset labels.Labels) (*Target, bool) { Outer: for _, t := range targets { for _, tl := range t.Labels { - if lset.Get(tl.Name) != tl.Value { + v := lset.Get(tl.Name) + if v != "" && v != tl.Value { continue Outer } }