feat: added different types of snuba consumers

sentry/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: sentry
 description: A Helm chart for Kubernetes
 type: application
-version: 2.0.1
+version: 2.1.0
 appVersion: 10.0.0
 dependencies:
   - name: redis

sentry/templates/deployment-snuba-consumer.yaml

@@ -50,7 +50,7 @@ spec:
       - name: {{ .Chart.Name }}-snuba
         image: "{{ .Values.images.snuba.repository }}:{{ .Values.images.snuba.tag }}"
         imagePullPolicy: {{ .Values.images.snuba.pullPolicy }}
-        command: ["snuba", "consumer", "--auto-offset-reset=latest", "--max-batch-time-ms", "750"]
+        command: ["snuba", "consumer", "--storage", "events", "--auto-offset-reset=latest", "--max-batch-time-ms", "750"]
         ports:
         - containerPort: {{ template "snuba.port" }}
         env:

sentry/templates/deployment-snuba-outcomes-consumers.yaml

@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "sentry.fullname" . }}-snuba-outcomes-consumer
+  labels:
+    app: {{ template "sentry.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  selector:
+    matchLabels:
+        app: {{ template "sentry.fullname" . }}
+        release: "{{ .Release.Name }}"
+        role: snuba-outcomes-consumer
+  replicas: {{ .Values.snuba.outcomesConsumer.replicas }}
+  template:
+    metadata:
+      annotations:
+        checksum/snubaSettingsPy: {{ .Values.config.snubaSettingsPy | sha256sum }}
+        checksum/config.yaml: {{ include (print $.Template.BasePath "/configmap-snuba.yaml") . | sha256sum }}
+        {{- if .Values.snuba.outcomesConsumer.annotations }}
+{{ toYaml .Values.snuba.outcomesConsumer.annotations | indent 8 }}
+        {{- end }}
+      labels:
+        app: {{ template "sentry.fullname" . }}
+        release: "{{ .Release.Name }}"
+        role: snuba-outcomes-consumer
+        {{- if .Values.snuba.outcomesConsumer.podLabels }}
+{{ toYaml .Values.snuba.outcomesConsumer.podLabels | indent 8 }}
+        {{- end }}
+    spec:
+      affinity:
+      {{- if .Values.snuba.outcomesConsumer.affinity }}
+{{ toYaml .Values.snuba.outcomesConsumer.affinity | indent 8 }}
+      {{- end }}
+      {{- if .Values.snuba.outcomesConsumer.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.snuba.outcomesConsumer.nodeSelector | indent 8 }}
+      {{- end }}
+      {{- if .Values.snuba.outcomesConsumer.tolerations }}
+      tolerations:
+{{ toYaml .Values.snuba.outcomesConsumer.tolerations | indent 8 }}
+      {{- end }}
+      {{- if .Values.images.snuba.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.images.snuba.imagePullSecrets | indent 8 }}
+      {{- end }}
+      containers:
+      - name: {{ .Chart.Name }}-snuba
+        image: "{{ .Values.images.snuba.repository }}:{{ .Values.images.snuba.tag }}"
+        imagePullPolicy: {{ .Values.images.snuba.pullPolicy }}
+        command: ["snuba", "consumer", "--storage", "outcomes_raw", "--auto-offset-reset=latest", "--max-batch-size",  "3"]
+        ports:
+        - containerPort: {{ template "snuba.port" }}
+        env:
+        - name: SNUBA_SETTINGS
+          value: /etc/snuba/settings.py
+{{- if .Values.snuba.outcomesConsumer.env }}
+{{ toYaml .Values.snuba.outcomesConsumer.env | indent 8 }}
+{{- end }}
+        volumeMounts:
+        - mountPath: /etc/snuba
+          name: config
+          readOnly: true
+        resources:
+{{ toYaml .Values.snuba.outcomesConsumer.resources | indent 12 }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ template "sentry.fullname" . }}-snuba

sentry/templates/deployment-snuba-replacer.yaml

@@ -50,7 +50,7 @@ spec:
       - name: {{ .Chart.Name }}-snuba
         image: "{{ .Values.images.snuba.repository }}:{{ .Values.images.snuba.tag }}"
         imagePullPolicy: {{ .Values.images.snuba.pullPolicy }}
-        command: ["snuba", "replacer", "--auto-offset-reset=latest", "--max-batch-size",  "3"]
+        command: ["snuba", "replacer", "--storage", "events", "--auto-offset-reset=latest", "--max-batch-size",  "3"]
         ports:
         - containerPort: {{ template "snuba.port" }}
         env:

sentry/templates/deployment-snuba-sessions-consumer.yaml

@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "sentry.fullname" . }}-sessions-consumer
+  labels:
+    app: {{ template "sentry.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  selector:
+    matchLabels:
+        app: {{ template "sentry.fullname" . }}
+        release: "{{ .Release.Name }}"
+        role: sessions-consumer
+  replicas: {{ .Values.snuba.sessionsConsumer.replicas }}
+  template:
+    metadata:
+      annotations:
+        checksum/snubaSettingsPy: {{ .Values.config.snubaSettingsPy | sha256sum }}
+        checksum/config.yaml: {{ include (print $.Template.BasePath "/configmap-snuba.yaml") . | sha256sum }}
+        {{- if .Values.snuba.sessionsConsumer.annotations }}
+{{ toYaml .Values.snuba.sessionsConsumer.annotations | indent 8 }}
+        {{- end }}
+      labels:
+        app: {{ template "sentry.fullname" . }}
+        release: "{{ .Release.Name }}"
+        role: sessions-consumer
+        {{- if .Values.snuba.sessionsConsumer.podLabels }}
+{{ toYaml .Values.snuba.sessionsConsumer.podLabels | indent 8 }}
+        {{- end }}
+    spec:
+      affinity:
+      {{- if .Values.snuba.sessionsConsumer.affinity }}
+{{ toYaml .Values.snuba.sessionsConsumer.affinity | indent 8 }}
+      {{- end }}
+      {{- if .Values.snuba.sessionsConsumer.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.snuba.sessionsConsumer.nodeSelector | indent 8 }}
+      {{- end }}
+      {{- if .Values.snuba.sessionsConsumer.tolerations }}
+      tolerations:
+{{ toYaml .Values.snuba.sessionsConsumer.tolerations | indent 8 }}
+      {{- end }}
+      {{- if .Values.images.snuba.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.images.snuba.imagePullSecrets | indent 8 }}
+      {{- end }}
+      containers:
+      - name: {{ .Chart.Name }}-snuba
+        image: "{{ .Values.images.snuba.repository }}:{{ .Values.images.snuba.tag }}"
+        imagePullPolicy: {{ .Values.images.snuba.pullPolicy }}
+        command: ["snuba", "consumer", "--storage", "sessions_raw", "--auto-offset-reset=latest", "--max-batch-time-ms", "750"]
+        ports:
+        - containerPort: {{ template "snuba.port" }}
+        env:
+        - name: SNUBA_SETTINGS
+          value: /etc/snuba/settings.py
+{{- if .Values.snuba.sessionsConsumer.env }}
+{{ toYaml .Values.snuba.sessionsConsumer.env | indent 8 }}
+{{- end }}
+        volumeMounts:
+        - mountPath: /etc/snuba
+          name: config
+          readOnly: true
+        resources:
+{{ toYaml .Values.snuba.sessionsConsumer.resources | indent 12 }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ template "sentry.fullname" . }}-snuba

sentry/values.yaml

@@ -93,6 +93,25 @@ snuba:
     nodeSelector: {}
     # tolerations: []
     # podLabels: []
+
+  outcomesConsumer:
+    replicas: 1
+    env: {}
+    resources: {}
+    affinity: {}
+    nodeSelector: {}
+    # tolerations: []
+    # podLabels: []
+
+  sessionsConsumer:
+    replicas: 1
+    env: {}
+    resources: {}
+    affinity: {}
+    nodeSelector: {}
+    # tolerations: []
+    # podLabels: []
+
   replacer:
     env: {}
     resources: {}