refactor(bloom planner): Compute gaps and build tasks from metas and …

…TSDBs (grafana#12994)

Signed-off-by: Joel Takvorian <jtakvori@redhat.com>

docs/sources/shared/configuration.md

@@ -333,6 +333,23 @@ bloom_build:
   [enabled: <boolean> | default = false]
 
   planner:
+    # Interval at which to re-run the bloom creation planning.
+    # CLI flag: -bloom-build.planner.interval
+    [planning_interval: <duration> | default = 8h]
+
+    # Newest day-table offset (from today, inclusive) to build blooms for.
+    # Increase to lower cost by not re-writing data to object storage too
+    # frequently since recent data changes more often at the cost of not having
+    # blooms available as quickly.
+    # CLI flag: -bloom-build.planner.min-table-offset
+    [min_table_offset: <int> | default = 1]
+
+    # Oldest day-table offset (from today, inclusive) to compact. This can be
+    # used to lower cost by not trying to compact older data which doesn't
+    # change. This can be optimized by aligning it with the maximum
+    # `reject_old_samples_max_age` setting of any tenant.
+    # CLI flag: -bloom-build.planner.max-table-offset
+    [max_table_offset: <int> | default = 2]
 
   builder:
 
@@ -3382,6 +3399,16 @@ shard_streams:
 # CLI flag: -bloom-compactor.max-bloom-size
 [bloom_compactor_max_bloom_size: <int> | default = 128MB]
 
+# Experimental. Whether to create blooms for the tenant.
+# CLI flag: -bloom-build.enable
+[bloom_creation_enabled: <boolean> | default = false]
+
+# Experimental. Number of splits to create for the series keyspace when building
+# blooms. The series keyspace is split into this many parts to parallelize bloom
+# creation.
+# CLI flag: -bloom-build.split-keyspace-by
+[bloom_split_series_keyspace_by: <int> | default = 256]
+
 # Experimental. Length of the n-grams created when computing blooms from log
 # lines.
 # CLI flag: -bloom-compactor.ngram-length

operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -180,7 +180,7 @@ metadata:
     operatorframework.io/arch.arm64: supported
     operatorframework.io/arch.ppc64le: supported
     operatorframework.io/arch.s390x: supported
-  name: loki-operator.v0.1.0
+  name: loki-operator.v0.0.0
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -1703,8 +1703,8 @@ spec:
                 - name: RELATED_IMAGE_GATEWAY
                   value: quay.io/observatorium/api:latest
                 - name: RELATED_IMAGE_OPA
-                  value: quay.io/observatorium/opa-openshift:latest
-                image: quay.io/openshift-logging/loki-operator:0.1.0
+                  value: quay.io/jotak/observatorium-opa-openshift:latest
+                image: quay.io/jotak/loki-operator:0.0.0
                 imagePullPolicy: IfNotPresent
                 livenessProbe:
                   httpGet:
@@ -1827,9 +1827,9 @@ spec:
     name: loki
   - image: quay.io/observatorium/api:latest
     name: gateway
-  - image: quay.io/observatorium/opa-openshift:latest
+  - image: quay.io/jotak/observatorium-opa-openshift:latest
     name: opa
-  version: 0.1.0
+  version: 0.0.0
   webhookdefinitions:
   - admissionReviewVersions:
     - v1

operator/config/manager/kustomization.yaml

@@ -5,5 +5,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/openshift-logging/loki-operator
-  newTag: 0.1.0
+  newName: quay.io/jotak/loki-operator
+  newTag: 0.0.0

operator/config/overlays/openshift/manager_related_image_patch.yaml

@@ -13,4 +13,4 @@ spec:
           - name: RELATED_IMAGE_GATEWAY
             value: quay.io/observatorium/api:latest
           - name: RELATED_IMAGE_OPA
-            value: quay.io/observatorium/opa-openshift:latest
+            value: quay.io/jotak/observatorium-opa-openshift:latest

operator/internal/manifests/openshift/opa_openshift.go

@@ -58,6 +58,7 @@ func newOPAOpenShiftContainer(mode lokiv1.ModeType, secretVolumeName, tlsDir, mi
 		args = append(args, []string{
 			fmt.Sprintf("--opa.matcher=%s", opaNetworkLabelMatchers),
 			"--opa.matcher-op=or",
+			"--opa.skip-namespace-inference=true",
 		}...)
 	}
 

pkg/bloombuild/planner/config.go

@@ -1,21 +1,40 @@
 package planner
 
-import "flag"
+import (
+	"flag"
+	"fmt"
+	"time"
+)
 
 // Config configures the bloom-planner component.
 type Config struct {
-	// TODO: Add config
+	PlanningInterval time.Duration `yaml:"planning_interval"`
+	MinTableOffset   int           `yaml:"min_table_offset"`
+	MaxTableOffset   int           `yaml:"max_table_offset"`
 }
 
 // RegisterFlagsWithPrefix registers flags for the bloom-planner configuration.
-func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) {
-	// TODO: Register flags with flagsPrefix
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.DurationVar(&cfg.PlanningInterval, prefix+".interval", 8*time.Hour, "Interval at which to re-run the bloom creation planning.")
+	f.IntVar(&cfg.MinTableOffset, prefix+".min-table-offset", 1, "Newest day-table offset (from today, inclusive) to build blooms for. Increase to lower cost by not re-writing data to object storage too frequently since recent data changes more often at the cost of not having blooms available as quickly.")
+	// TODO(owen-d): ideally we'd set this per tenant based on their `reject_old_samples_max_age` setting,
+	// but due to how we need to discover tenants, we can't do that yet. Tenant+Period discovery is done by
+	// iterating the table periods in object storage and looking for tenants within that period.
+	// In order to have this done dynamically, we'd need to account for tenant specific overrides, which are also
+	// dynamically reloaded.
+	// I'm doing it the simple way for now.
+	f.IntVar(&cfg.MaxTableOffset, prefix+".max-table-offset", 2, "Oldest day-table offset (from today, inclusive) to compact. This can be used to lower cost by not trying to compact older data which doesn't change. This can be optimized by aligning it with the maximum `reject_old_samples_max_age` setting of any tenant.")
 }
 
 func (cfg *Config) Validate() error {
+	if cfg.MinTableOffset > cfg.MaxTableOffset {
+		return fmt.Errorf("min-table-offset (%d) must be less than or equal to max-table-offset (%d)", cfg.MinTableOffset, cfg.MaxTableOffset)
+	}
+
 	return nil
 }
 
 type Limits interface {
-	// TODO: Add limits
+	BloomCreationEnabled(tenantID string) bool
+	BloomSplitSeriesKeyspaceBy(tenantID string) int
 }

pkg/bloombuild/planner/metrics.go

@@ -8,10 +8,19 @@ import (
 const (
 	metricsNamespace = "loki"
 	metricsSubsystem = "bloomplanner"
+
+	statusSuccess = "success"
+	statusFailure = "failure"
 )
 
 type Metrics struct {
 	running prometheus.Gauge
+
+	buildStarted   prometheus.Counter
+	buildCompleted *prometheus.CounterVec
+	buildTime      *prometheus.HistogramVec
+
+	tenantsDiscovered prometheus.Counter
 }
 
 func NewMetrics(r prometheus.Registerer) *Metrics {
@@ -22,5 +31,32 @@ func NewMetrics(r prometheus.Registerer) *Metrics {
 			Name:      "running",
 			Help:      "Value will be 1 if bloom planner is currently running on this instance",
 		}),
+
+		buildStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: metricsSubsystem,
+			Name:      "build_started_total",
+			Help:      "Total number of builds started",
+		}),
+		buildCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: metricsSubsystem,
+			Name:      "build_completed_total",
+			Help:      "Total number of builds completed",
+		}, []string{"status"}),
+		buildTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: metricsSubsystem,
+			Name:      "build_time_seconds",
+			Help:      "Time spent during a builds cycle.",
+			Buckets:   prometheus.DefBuckets,
+		}, []string{"status"}),
+
+		tenantsDiscovered: promauto.With(r).NewCounter(prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: metricsSubsystem,
+			Name:      "tenants_discovered_total",
+			Help:      "Number of tenants discovered during the current build iteration",
+		}),
 	}
 }