feat(helm): add fluent-bit daemonset (reanahub#827)

jlemesh · Aug 30, 2024 · 049691c · 049691c
1 parent 4096e82
commit 049691c
Show file tree

Hide file tree

Showing 3 changed files with 133 additions and 0 deletions.
diff --git a/helm/configurations/values-dev.yaml b/helm/configurations/values-dev.yaml
@@ -56,3 +56,11 @@ opensearch:
       memory: "2Gi"
   secretMounts: []
   customSecurityConfig:
+
+# FluentBit configuration for dev environment
+fluent-bit:
+  outputConfig:
+    tls: Off
+    httpPassword: ReanaOS1=
+  extraVolumes: []
+  extraVolumeMounts: []
diff --git a/helm/reana/Chart.yaml b/helm/reana/Chart.yaml
@@ -39,3 +39,7 @@ dependencies:
     version: 2.22.1
     repository: https://opensearch-project.github.io/helm-charts/
     condition: opensearch.enabled
+  - name: fluent-bit
+    version: 0.47.7
+    repository: https://fluent.github.io/helm-charts
+    condition: fluent-bit.enabled
diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml
@@ -351,3 +351,124 @@ opensearch:
         backend_roles:
           - readall
         and_backend_roles: []
+
+# FluentBit chart values.yaml
+fluent-bit:
+  enabled: false
+  inputConfig:
+    refreshInterval: 2
+    rotateWait: 5
+    skipLongLines: On
+    skipEmptyLines: On
+  filterConfig:
+    bufferSize: 512k
+    kubeUrl: https://kubernetes.default.svc:443
+    kubeCaFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    kubeTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+  outputConfig:
+    host: opensearch-cluster-master
+    httpUser: fluentbit
+    httpPassword:
+    enableTls: true
+    tls: On
+    tlsVerify: On
+    tlsVerifyHostname: Off
+    tlsCaFile: /fluent-bit/etc/certs/ca.crt
+    tlsCrtFile: ""
+    tlsKeyFile: ""
+    tlsKeyPassword: ""
+  ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file
+  config:
+    service: |
+      [SERVICE]
+          Daemon Off
+          Flush {{ .Values.flush }}
+          Log_Level {{ .Values.logLevel }}
+          Parsers_File /fluent-bit/etc/parsers.conf
+          Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
+          HTTP_Server On
+          HTTP_Listen 0.0.0.0
+          HTTP_Port {{ .Values.metricsPort }}
+          Health_Check On
+
+    # ## https://docs.fluentbit.io/manual/pipeline/inputs
+    inputs: |
+      [INPUT]
+          Name tail
+          Path /var/log/containers/reana-run-job-*.log
+          multiline.parser docker, cri
+          Tag kube.*
+          Skip_Long_Lines {{ .Values.inputConfig.skipLongLines }}
+          Skip_Empty_Lines {{ .Values.inputConfig.skipEmptyLines }}
+          Refresh_Interval {{ .Values.inputConfig.refreshInterval }}
+          Rotate_Wait {{ .Values.inputConfig.rotateWait }}
+
+      [INPUT]
+          Name tail
+          Path /var/log/containers/reana-run-batch-*_workflow-engine-*.log
+          multiline.parser docker, cri
+          Tag kube.*
+          Skip_Long_Lines {{ .Values.inputConfig.skipLongLines }}
+          Skip_Empty_Lines {{ .Values.inputConfig.skipEmptyLines }}
+          Refresh_Interval {{ .Values.inputConfig.refreshInterval }}
+          Rotate_Wait {{ .Values.inputConfig.rotateWait }}
+
+    ## https://docs.fluentbit.io/manual/pipeline/filters
+    filters: |
+      [FILTER]
+          Name kubernetes
+          Buffer_Size {{ .Values.filterConfig.bufferSize }}
+          Match kube.*
+          Kube_Tag_Prefix kube.var.log.containers.
+          Kube_URL {{ .Values.filterConfig.kubeUrl }}
+          Kube_CA_File {{ .Values.filterConfig.kubeCaFile }}
+          Kube_Token_File {{ .Values.filterConfig.kubeTokenFile }}
+
+    ## https://docs.fluentbit.io/manual/pipeline/outputs
+    outputs: |
+      [OUTPUT]
+          Name opensearch
+          Match kube.var.log.containers.reana-run-job-*
+          Host {{ .Values.outputConfig.host }}
+          {{- if .Values.outputConfig.httpPassword }}
+          HTTP_User {{ .Values.outputConfig.httpUser }}
+          HTTP_Passwd {{ .Values.outputConfig.httpPassword }}
+          {{- end }}
+          Index fluentbit-job_log
+          Suppress_Type_Name On
+          tls {{ .Values.outputConfig.tls }}
+          tls.verify {{ .Values.outputConfig.tlsVerify }}
+          tls.verify_hostname {{ .Values.outputConfig.tlsVerifyHostname }}
+          {{ if .Values.outputConfig.tlsCaFile }}tls.ca_file {{ .Values.outputConfig.tlsCaFile }}{{- end }}
+          {{ if .Values.outputConfig.tlsCrtFile }}tls.crt_file {{ .Values.outputConfig.tlsCrtFile }}{{- end }}
+          {{ if .Values.outputConfig.tlsKeyFile }}tls.key_file {{ .Values.outputConfig.tlsKeyFile }}{{- end }}
+          {{ if .Values.outputConfig.tlsKeyPassword }}tls.key_password {{ .Values.outputConfig.tlsKeyPassword }}{{- end }}
+
+      [OUTPUT]
+          Name opensearch
+          Match kube.var.log.containers.reana-run-batch-*_workflow-engine-*.log
+          Host {{ .Values.outputConfig.host }}
+          {{- if .Values.outputConfig.httpPassword }}
+          HTTP_User {{ .Values.outputConfig.httpUser }}
+          HTTP_Passwd {{ .Values.outputConfig.httpPassword }}
+          {{- end }}
+          Index fluentbit-workflow_log
+          Suppress_Type_Name On
+          tls {{ .Values.outputConfig.tls }}
+          tls.verify {{ .Values.outputConfig.tlsVerify }}
+          tls.verify_hostname {{ .Values.outputConfig.tlsVerifyHostname }}
+          {{ if .Values.outputConfig.tlsCaFile }}tls.ca_file {{ .Values.outputConfig.tlsCaFile }}{{- end }}
+          {{ if .Values.outputConfig.tlsCrtFile }}tls.crt_file {{ .Values.outputConfig.tlsCrtFile }}{{- end }}
+          {{ if .Values.outputConfig.tlsKeyFile }}tls.key_file {{ .Values.outputConfig.tlsKeyFile }}{{- end }}
+          {{ if .Values.outputConfig.tlsKeyPassword }}tls.key_password {{ .Values.outputConfig.tlsKeyPassword }}{{- end }}
+  extraVolumes:
+    - name: reana-opensearch-ca
+      secret:
+        secretName: reana-opensearch-tls-secret
+        items:
+        - key: ca.crt
+          path: ca.crt
+  extraVolumeMounts:
+    - name: reana-opensearch-ca
+      mountPath: /fluent-bit/etc/certs
+      readOnly: true