feat: add experimental otelcol log collector

This is an alpha feature without a stable API. It only supports container logs from docker-shim. chore: update changelog
SumoLogic · Jan 3, 2022 · e99294c · e99294c
1 parent 066fe63
commit e99294c
Show file tree

Hide file tree

Showing 20 changed files with 622 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- feat: add experimental otelcol log collector [#1986][#1986]
 - feat: add option to disable pod owners enrichment [#1959][#1959]
 
 ### Changed
@@ -24,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - fix(helm): add metrics port to otelcol pods [#1992][#1992]
 
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.3.1...main
+[#1986]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/1986
 [#1959]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/1959
 [#1974]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/1974
 [#1973]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/1973

diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
@@ -0,0 +1 @@
+{{ tpl (toYaml .Values.otellogs.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }}
diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl
@@ -51,6 +51,42 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 {{- end -}}
 
+{{- define "sumologic.logs.collector.name" -}}
+{{- template "sumologic.fullname" . }}-otelcol-logs-collector
+{{- end -}}
+
+{{- define "sumologic.logs.collector.name.configmap" -}}
+{{- template "sumologic.logs.collector.name" . }}
+{{- end -}}
+
+{{- define "sumologic.logs.collector.name.serviceaccount" -}}
+{{- template "sumologic.logs.collector.name" . }}
+{{- end -}}
+
+{{- define "sumologic.logs.collector.name.daemonset" -}}
+{{- template "sumologic.logs.collector.name" . }}
+{{- end -}}
+
+{{- define "sumologic.labels.app.logs.collector" -}}
+{{- template "sumologic.fullname" . }}-otelcol-logs-collector
+{{- end -}}
+
+{{- define "sumologic.labels.app.logs.collector.configmap" -}}
+{{- template "sumologic.labels.app.logs.collector" . }}
+{{- end -}}
+
+{{- define "sumologic.labels.app.logs.collector.serviceaccount" -}}
+{{- template "sumologic.labels.app.logs.collector" . }}
+{{- end -}}
+
+{{- define "sumologic.labels.app.logs.collector.daemonset" -}}
+{{- template "sumologic.labels.app.logs.collector" . }}
+{{- end -}}
+
+{{- define "sumologic.labels.app.logs.collector.pod" -}}
+{{- template "sumologic.labels.app.logs.collector" . }}
+{{- end -}}
+
 {{- define "sumologic.labels.app.logs.pod" -}}
 {{- template "sumologic.labels.app.logs" . }}
 {{- end -}}

diff --git a/deploy/helm/sumologic/templates/logs/collector/otelcol/configmap.yaml b/deploy/helm/sumologic/templates/logs/collector/otelcol/configmap.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.otellogs.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "sumologic.logs.collector.name.configmap" . }}
+  labels:
+    app: {{ template "sumologic.labels.app.logs.collector.configmap" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}
+data:
+  {{- (tpl (.Files.Glob "conf/logs/collector/otelcol/config.yaml").AsConfig .) | nindent 2 }}
+{{- end }}
diff --git a/deploy/helm/sumologic/templates/logs/collector/otelcol/daemonset.yaml b/deploy/helm/sumologic/templates/logs/collector/otelcol/daemonset.yaml
@@ -0,0 +1,109 @@
+{{- if .Values.otellogs.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ template "sumologic.logs.collector.name.daemonset" . }}
+  labels:
+    app: {{ template "sumologic.labels.app.logs.collector.daemonset" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "sumologic.labels.app.logs.collector.pod" . }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/logs/collector/otelcol/configmap.yaml") . | sha256sum }}
+{{- if .Values.sumologic.podAnnotations }}
+{{ toYaml .Values.sumologic.podAnnotations | indent 8 }}
+{{- end }}
+{{- if .Values.otellogs.daemonset.podAnnotations }}
+{{ toYaml .Values.otellogs.daemonset.podAnnotations | indent 8 }}
+{{- end }}
+      labels:
+        app.kubernetes.io/name: {{ template "sumologic.labels.app.logs.collector.pod" . }}
+        {{- include "sumologic.labels.common" . | nindent 8 }}
+{{- if .Values.sumologic.podLabels }}
+{{ toYaml .Values.sumologic.podLabels | indent 8 }}
+{{- end }}
+{{- if .Values.otellogs.daemonset.podLabels }}
+{{ toYaml .Values.otellogs.daemonset.podLabels | indent 8 }}
+{{- end }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.otellogs.daemonset.securityContext | nindent 8 }}
+      {{- if .Values.otellogs.daemonset.priorityClassName }}
+      priorityClassName: {{ .Values.metadata.logs.daemonset.priorityClassName | quote }}
+      {{- end }}
+      containers:
+      - args:
+        - --config=/etc/otelcol/config.yaml
+        image: {{ .Values.otellogs.image.repository }}:{{ .Values.otellogs.image.tag }}
+        imagePullPolicy: {{ .Values.otellogs.image.pullPolicy }}
+        name: otelcol
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 13133 # Health Check extension default port.
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 13133 # Health Check extension default port.
+        volumeMounts:
+        - mountPath: /etc/otelcol
+          name: otelcol-config
+        - mountPath: /var/log/pods
+          name: varlogpods
+          readOnly: true
+        - mountPath: /var/lib/docker/containers
+          name: varlibdockercontainers
+          readOnly: true
+        - mountPath: {{ .Values.otellogs.config.extensions.file_storage.directory }}
+          name: file-storage
+        env:
+        - name: LOGS_METADATA_SVC
+          valueFrom:
+            configMapKeyRef:
+              name: sumologic-configmap
+              key: fluentdLogs
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        securityContext: 
+          {{- toYaml .Values.otellogs.daemonset.containers.otelcol.securityContext | nindent 10 }}
+      initContainers: # ensure the host path is owned by the otel user group
+      - name: changeowner
+        image: busybox
+        command:
+        - "sh"
+        - "-c"
+        - |
+          chown -R \
+            {{ .Values.otellogs.daemonset.securityContext.fsGroup }}:{{ .Values.otellogs.daemonset.securityContext.fsGroup }} \
+            {{ .Values.otellogs.config.extensions.file_storage.directory }}
+        volumeMounts:
+        - mountPath: {{ .Values.otellogs.config.extensions.file_storage.directory }}
+          name: file-storage
+      volumes:
+      - configMap:
+          defaultMode: 420
+          items:
+          - key: config.yaml
+            path: config.yaml
+          name: {{ template "sumologic.logs.collector.name.configmap" . }}
+        name: otelcol-config
+      - hostPath:
+          path: /var/log/pods
+          type: ""
+        name: varlogpods
+      - hostPath:
+          path: /var/lib/docker/containers
+          type: ""
+        name: varlibdockercontainers
+      - hostPath:
+          path: /var/lib/otc
+          type: DirectoryOrCreate
+        name: file-storage
+      serviceAccountName: {{ template "sumologic.logs.collector.name.serviceaccount" . }}
+{{- end }}
diff --git a/deploy/helm/sumologic/templates/logs/collector/otelcol/serviceaccount.yaml b/deploy/helm/sumologic/templates/logs/collector/otelcol/serviceaccount.yaml
@@ -0,0 +1,9 @@
+{{- if .Values.otellogs.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "sumologic.logs.collector.name.serviceaccount" . }}
+  labels:
+    app: {{ template "sumologic.labels.app.logs.collector.serviceaccount" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}
+{{- end }}
diff --git a/deploy/helm/sumologic/templates/logs/common/service-headless.yaml b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml
@@ -14,6 +14,10 @@ spec:
     app: {{ template "sumologic.labels.app.logs.pod" . }}
   clusterIP: None
   ports:
+  - name: otlphttp
+    port: 4318
+    targetPort: 4318
+    protocol: TCP
   {{- if .Values.sumologic.traces.enabled }}
   - name: zipkin-write
     port: 9411

diff --git a/deploy/helm/sumologic/templates/logs/common/service.yaml b/deploy/helm/sumologic/templates/logs/common/service.yaml
@@ -14,6 +14,10 @@ spec:
   selector:
     app: {{ template "sumologic.labels.app.logs.pod" . }}
   ports:
+  - name: otlphttp
+    port: 4318
+    targetPort: 4318
+    protocol: TCP
   {{- if .Values.sumologic.traces.enabled }}
   - name: zipkin-write
     port: 9411

diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml
@@ -130,6 +130,9 @@ spec:
           containerPort: 8888
           protocol: TCP
         {{- end }}
+        - name: otlphttp
+          containerPort: 4318
+          protocol: TCP
         livenessProbe:
           httpGet:
             path: /
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{{ tpl (toYaml .Values.otellogs.config \| replace ": '{{" ": {{" \| replace "}}'" "}}") . \| nindent 2 }}