feat: allow users to specify Kafka topic name prefix in values.yaml (#…

…1544)
sentry-kubernetes · Oct 25, 2024 · 5693406 · 5693406
1 parent 35f779a
commit 5693406
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 6 deletions.
diff --git a/charts/sentry/templates/relay/_helper-sentry-relay.tpl b/charts/sentry/templates/relay/_helper-sentry-relay.tpl
@@ -94,8 +94,27 @@ config.yml: |-
     {{- else }}
     redis: "{{ $redisProto }}://{{ $redisHost }}:{{ $redisPort }}/{{ $redisDb }}"
     {{- end }}
+
+    {{- if ((.Values.kafkaTopicOverrides).prefix) }}
+    topics:
+      metrics_sessions: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-metrics"
+      events: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-attachments"
+      transactions: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-transactions"
+      outcomes: "{{ default "" .Values.kafkaTopicOverrides.prefix }}outcomes"
+      outcomes_billing: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-outcomes"
+      metrics_generic: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-performance-metrics"
+      profiles: "{{ default "" .Values.kafkaTopicOverrides.prefix }}profiles"
+      replay_events: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-replay-events"
+      replay_recordings: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-replay-recordings"
+      monitors: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-monitors"
+      spans: "{{ default "" .Values.kafkaTopicOverrides.prefix }}snuba-spans"
+      metrics_summaries: "{{ default "" .Values.kafkaTopicOverrides.prefix }}snuba-metrics-summaries"
+      cogs: "{{ default "" .Values.kafkaTopicOverrides.prefix }}shared-resources-usage"
+      feedback: "{{ default "" .Values.kafkaTopicOverrides.prefix }}ingest-feedback-events"
+    {{- else }}
     topics:
-      metrics_sessions: ingest-metrics
+      metrics_sessions: "ingest-metrics"
+    {{- end }}
 
   {{ .Values.config.relay | nindent 2 }}
 {{- end -}}
diff --git a/charts/sentry/templates/sentry/_helper-sentry.tpl b/charts/sentry/templates/sentry/_helper-sentry.tpl
@@ -252,6 +252,14 @@ sentry.conf.py: |-
   SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream"
   SENTRY_EVENTSTREAM_OPTIONS = {"producer_configuration": DEFAULT_KAFKA_OPTIONS}
 
+  {{- if ((.Values.kafkaTopicOverrides).prefix) }}
+  SENTRY_CHARTS_KAFKA_TOPIC_PREFIX = {{ .Values.kafkaTopicOverrides.prefix | quote }}
+
+  from sentry.conf.types.kafka_definition import Topic
+  for topic in Topic:
+    KAFKA_TOPIC_OVERRIDES[topic.value] = f"{SENTRY_CHARTS_KAFKA_TOPIC_PREFIX}{topic.value}"
+  {{- end }}
+
   KAFKA_CLUSTERS["default"] = DEFAULT_KAFKA_OPTIONS
 
   ###############

diff --git a/...sentry/post-process-forwarder/errors/deployment-sentry-post-process-forwarder-errors.yaml b/...sentry/post-process-forwarder/errors/deployment-sentry-post-process-forwarder-errors.yaml
@@ -87,7 +87,7 @@ spec:
           - "post-process-forwarder-errors"
           - "--consumer-group"
           - "post-process-forwarder"
-          - "--synchronize-commit-log-topic=snuba-commit-log"
+          - "--synchronize-commit-log-topic={{ default "" ((.Values.kafkaTopicOverrides).prefix) }}snuba-commit-log"
           - "--synchronize-commit-group=snuba-consumers"
           {{- if .Values.sentry.postProcessForwardErrors.autoOffsetReset }}
           - "--auto-offset-reset"

diff --git a/...ess-forwarder/issue-platform/deployment-sentry-post-process-forwarder-issue-platform.yaml b/...ess-forwarder/issue-platform/deployment-sentry-post-process-forwarder-issue-platform.yaml
@@ -87,7 +87,7 @@ spec:
           - "post-process-forwarder-issue-platform"
           - "--consumer-group"
           - "post-process-forwarder"
-          - "--synchronize-commit-log-topic=snuba-generic-events-commit-log"
+          - "--synchronize-commit-log-topic={{ default "" ((.Values.kafkaTopicOverrides).prefix) }}snuba-generic-events-commit-log"
           - "--synchronize-commit-group"
           - "generic_events_group"
           {{- if .Values.sentry.postProcessForwardIssuePlatform.autoOffsetReset }}

diff --git a/...process-forwarder/transactions/deployment-sentry-post-process-forwarder-transactions.yaml b/...process-forwarder/transactions/deployment-sentry-post-process-forwarder-transactions.yaml
@@ -87,7 +87,7 @@ spec:
           - "post-process-forwarder-transactions"
           - "--consumer-group"
           - "post-process-forwarder"
-          - "--synchronize-commit-log-topic=snuba-transactions-commit-log"
+          - "--synchronize-commit-log-topic={{ default "" ((.Values.kafkaTopicOverrides).prefix) }}snuba-transactions-commit-log"
           - "--synchronize-commit-group"
           - "transactions_group"
           {{- if .Values.sentry.postProcessForwardTransactions.autoOffsetReset }}

diff --git a/charts/sentry/templates/snuba/_helper-snuba.tpl b/charts/sentry/templates/snuba/_helper-snuba.tpl
@@ -23,6 +23,14 @@ settings.py: |
   {{- end -}}
 {{- end }}
 
+  {{- if ((.Values.kafkaTopicOverrides).prefix) }}
+  SENTRY_CHARTS_KAFKA_TOPIC_PREFIX = {{ .Values.kafkaTopicOverrides.prefix | quote }}
+
+  from snuba.utils.streams.topics import Topic
+  for topic in Topic:
+    KAFKA_TOPIC_MAP[topic.value] = f"{SENTRY_CHARTS_KAFKA_TOPIC_PREFIX}{topic.value}"
+  {{- end }}
+
   # Clickhouse Options
   CLUSTERS = [
     {

diff --git a/charts/sentry/templates/snuba/deployment-snuba-outcomes-billing-consumer.yaml b/charts/sentry/templates/snuba/deployment-snuba-outcomes-billing-consumer.yaml
@@ -90,8 +90,7 @@ spec:
           - "--auto-offset-reset"
           - "{{ .Values.snuba.outcomesBillingConsumer.autoOffsetReset }}"
           {{- end }}
-          - "--raw-events-topic"
-          - "outcomes-billing"
+          - "--raw-events-topic={{ default "" ((.Values.kafkaTopicOverrides).prefix) }}outcomes-billing"
           {{- if .Values.snuba.outcomesBillingConsumer.maxBatchSize }}
           - "--max-batch-size"
           - "{{ .Values.snuba.outcomesBillingConsumer.maxBatchSize }}"

diff --git a/charts/sentry/values.yaml b/charts/sentry/values.yaml
@@ -158,6 +158,11 @@ relay:
     #  - name: compression.type
     #    value: "lz4"
 
+# Override custom Kafka topic names
+# WARNING: If you update this and you are also using the Kafka subchart, you need to update the provisioned Topic names in this values as well!
+# kafkaTopicOverrides:
+#  prefix: ""
+
 # enable and reference the volume
 geodata:
   accountID: ""