move GetClusterShardingStrategy to common/monitoring; add query argument to QueryTSDBHeadSeries

CHANGELOG.md

@@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add helm chart templating test in ci pipeline.
 - Add support for Alloy to be used as monitoring agent in-place of Prometheus Agent. This is configurable via the `--monitoring-agent` flag.
 
+### Changed
+
+- Move GetClusterShardingStrategy to common/monitoring package
+- Add query argument to QueryTSDBHeadSeries
+
 ## [0.3.1] - 2024-07-22
 
 ### Fixed

pkg/common/monitoring/monitoring.go

@@ -2,13 +2,17 @@ package monitoring
 
 import (
 	"context"
+	"strconv"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/yaml"
+
+	"github.com/giantswarm/observability-operator/pkg/monitoring/prometheusagent/sharding"
 )
 
 const (
@@ -84,3 +88,22 @@ func readMimirAuthPasswordFromSecret(secret corev1.Secret) (string, error) {
 		return secretData, nil
 	}
 }
+
+func GetClusterShardingStrategy(cluster metav1.Object) (*sharding.Strategy, error) {
+	var err error
+	var scaleUpSeriesCount, scaleDownPercentage float64
+	if value, ok := cluster.GetAnnotations()["monitoring.giantswarm.io/prometheus-agent-scale-up-series-count"]; ok {
+		if scaleUpSeriesCount, err = strconv.ParseFloat(value, 64); err != nil {
+			return nil, err
+		}
+	}
+	if value, ok := cluster.GetAnnotations()["monitoring.giantswarm.io/prometheus-agent-scale-down-percentage"]; ok {
+		if scaleDownPercentage, err = strconv.ParseFloat(value, 64); err != nil {
+			return nil, err
+		}
+	}
+	return &sharding.Strategy{
+		ScaleUpSeriesCount:  scaleUpSeriesCount,
+		ScaleDownPercentage: scaleDownPercentage,
+	}, nil
+}

pkg/monitoring/mimir/querier/querier.go

@@ -3,7 +3,6 @@ package querier
 import (
 	"context"
 	"errors"
-	"fmt"
 	"time"
 
 	"github.com/prometheus/client_golang/api"
@@ -19,7 +18,7 @@ var (
 )
 
 // QueryTSDBHeadSeries performs an instant query against Mimir.
-func QueryTSDBHeadSeries(ctx context.Context, clusterName string) (float64, error) {
+func QueryTSDBHeadSeries(ctx context.Context, query string) (float64, error) {
 	config := api.Config{
 		Address: "http://mimir-gateway.mimir.svc/prometheus",
 	}
@@ -34,7 +33,6 @@ func QueryTSDBHeadSeries(ctx context.Context, clusterName string) (float64, erro
 	api := v1.NewAPI(c)
 
 	queryContext, cancel := context.WithTimeout(ctx, 2*time.Minute)
-	query := fmt.Sprintf("sum(max_over_time(prometheus_agent_active_series{cluster_id=\"%s\"}[6h]))", clusterName)
 	val, _, err := api.Query(queryContext, query, time.Now())
 	cancel()
 	if err != nil {

pkg/monitoring/prometheusagent/configmap.go

@@ -3,7 +3,6 @@ package prometheusagent
 import (
 	"context"
 	"fmt"
-	"strconv"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
@@ -16,7 +15,6 @@ import (
 	commonmonitoring "github.com/giantswarm/observability-operator/pkg/common/monitoring"
 	"github.com/giantswarm/observability-operator/pkg/metrics"
 	"github.com/giantswarm/observability-operator/pkg/monitoring/mimir/querier"
-	"github.com/giantswarm/observability-operator/pkg/monitoring/prometheusagent/sharding"
 )
 
 func (pas PrometheusAgentService) buildRemoteWriteConfig(ctx context.Context,
@@ -47,13 +45,14 @@ func (pas PrometheusAgentService) buildRemoteWriteConfig(ctx context.Context,
 	}
 
 	// Compute the number of shards based on the number of series.
-	headSeries, err := querier.QueryTSDBHeadSeries(ctx, cluster.Name)
+	query := fmt.Sprintf("sum(max_over_time(prometheus_agent_active_series{cluster_id=\"%s\"}[6h]))", cluster.Name)
+	headSeries, err := querier.QueryTSDBHeadSeries(ctx, query)
 	if err != nil {
 		logger.Error(err, "failed to query head series")
 		metrics.MimirQueryErrors.WithLabelValues().Inc()
 	}
 
-	clusterShardingStrategy, err := getClusterShardingStrategy(cluster)
+	clusterShardingStrategy, err := commonmonitoring.GetClusterShardingStrategy(cluster)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -96,25 +95,6 @@ func getPrometheusAgentRemoteWriteConfigName(cluster *clusterv1.Cluster) string
 	return fmt.Sprintf("%s-remote-write-config", cluster.Name)
 }
 
-func getClusterShardingStrategy(cluster metav1.Object) (*sharding.Strategy, error) {
-	var err error
-	var scaleUpSeriesCount, scaleDownPercentage float64
-	if value, ok := cluster.GetAnnotations()["monitoring.giantswarm.io/prometheus-agent-scale-up-series-count"]; ok {
-		if scaleUpSeriesCount, err = strconv.ParseFloat(value, 64); err != nil {
-			return nil, err
-		}
-	}
-	if value, ok := cluster.GetAnnotations()["monitoring.giantswarm.io/prometheus-agent-scale-down-percentage"]; ok {
-		if scaleDownPercentage, err = strconv.ParseFloat(value, 64); err != nil {
-			return nil, err
-		}
-	}
-	return &sharding.Strategy{
-		ScaleUpSeriesCount:  scaleUpSeriesCount,
-		ScaleDownPercentage: scaleDownPercentage,
-	}, nil
-}
-
 func readCurrentShardsFromConfig(configMap corev1.ConfigMap) (int, error) {
 	remoteWriteConfig := RemoteWriteConfig{}
 	err := yaml.Unmarshal([]byte(configMap.Data["values"]), &remoteWriteConfig)