Skip to content

Commit

Permalink
otelcol: add systemd logs pipeline (#1767)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pmalek
pmalek authored Sep 20, 2021
1 parent 79a0ea7 commit e7f0722
Show file tree
Hide file tree
Showing 2 changed files with 179 additions and 43 deletions.
143 changes: 108 additions & 35 deletions deploy/helm/sumologic/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2592,7 +2592,7 @@ otelcol:
statefulset:
image:
repository: public.ecr.aws/sumologic/sumologic-otel-collector
tag: 0.0.25-beta.0
tag: 0.0.27-beta.0
pullPolicy: IfNotPresent
metadata:
metrics:
Expand Down Expand Up @@ -2769,7 +2769,7 @@ otelcol:
extensions:
health_check: {}
exporters:
sumologic:
sumologic/containers:
log_format: json
endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
## ToDo: Move sources to sourceprocessor
Expand All @@ -2789,64 +2789,85 @@ otelcol:
- host
- node
- pod
sumologic/systemd:
log_format: json
endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
## ToDo: Move sources to sourceprocessor
source_name: "%{_sourceName}"
source_category: "%{_sourceCategory}"
source_host: "%{_sourceHost}"
sending_queue:
enabled: true
metadata_attributes:
- _collector
- _sourceCategory
- _sourceHost
- _sourceName

processors:
attributes:
## Common processors
attributes/remove_fluent_tag:
actions:
- action: extract
key: fluent.tag
pattern: ^containers\.var\.log\.containers\.(?P<k8s_pod_name>[^_]+)_(?P<k8s_namespace>[^_]+)_(?P<k8s_container_name>.+)-(?P<container_id>[a-f0-9]{64})\.log$
- action: delete
key: fluent.tag
## The memory_limiter processor is used to prevent out of memory situations on the collector.
memory_limiter:
## check_interval is the time between measurements of memory usage for the
## purposes of avoiding going over the limits. Defaults to zero, so no
## checks will be performed. Values below 1 second are not recommended since
## it can result in unnecessary CPU consumption.
check_interval: 5s

## Maximum amount of memory, in MiB, targeted to be allocated by the process heap.
## Note that typically the total memory usage of process will be about 50MiB higher
## than this value.
limit_mib: 1900
## The batch processor accepts spans and places them into batches grouped by node and resource
batch:
## Number of spans after which a batch will be sent regardless of time
send_batch_size: 256
## Time duration after which a batch will be sent regardless of size
timeout: 5s

## Containers related processors
filter/include_fluent_tag_containers:
logs:
include:
match_type: regexp
record_attributes:
- key: fluent.tag
value: containers\..+
attributes/containers:
actions:
- action: extract
key: fluent.tag
pattern: ^containers\.var\.log\.containers\.(?P<k8s_pod_name>[^_]+)_(?P<k8s_namespace>[^_]+)_(?P<k8s_container_name>.+)-(?P<container_id>[a-f0-9]{64})\.log$
- action: insert
key: k8s.container.id
from_attribute: container_id
- action: delete
key: container_id

- action: insert
key: k8s.pod.name
from_attribute: k8s_pod_name
- action: delete
key: k8s_pod_name

- action: insert
key: k8s.namespace.name
from_attribute: k8s_namespace
- action: delete
key: k8s_namespace

- action: insert
key: k8s.container.name
from_attribute: k8s_container_name
- action: delete
key: k8s_container_name
groupbyattrs:
groupbyattrs/containers:
keys:
- k8s.container.id
- k8s.container.name
- k8s.namespace.name
- k8s.pod.name

## The memory_limiter processor is used to prevent out of memory situations on the collector.
memory_limiter:
## check_interval is the time between measurements of memory usage for the
## purposes of avoiding going over the limits. Defaults to zero, so no
## checks will be performed. Values below 1 second are not recommended since
## it can result in unnecessary CPU consumption.
check_interval: 5s

## Maximum amount of memory, in MiB, targeted to be allocated by the process heap.
## Note that typically the total memory usage of process will be about 50MiB higher
## than this value.
limit_mib: 1900

## The batch processor accepts spans and places them into batches grouped by node and resource
batch:
## Number of spans after which a batch will be sent regardless of time
send_batch_size: 256
## Time duration after which a batch will be sent regardless of size
timeout: 5s
k8s_tagger:
## Has to be false to enrich metadata
passthrough: false
Expand Down Expand Up @@ -2879,7 +2900,7 @@ otelcol:
delimiter: "_"
pod_association:
- from: build_hostname
source:
source/containers:
collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}'
source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}'
source_category: '{{ .Values.fluentd.logs.containers.sourceCategory | quote }}'
Expand All @@ -2897,6 +2918,41 @@ otelcol:
pod_key: "k8s.pod.name"
container_key: "k8s.container.name"
source_host_key: "k8s.pod.hostname"

## Systemd related processors
filter/include_fluent_tag_host:
logs:
include:
match_type: regexp
record_attributes:
- key: fluent.tag
value: host\..+
filter/include_systemd:
logs:
include:
match_type: regexp
record_attributes:
- key: _SYSTEMD_UNIT
value: .+
filter/exclude_kubelet:
logs:
exclude:
match_type: service
record_attributes:
- key: _SYSTEMD_UNIT
value: kubelet.service
groupbyattrs/systemd:
keys:
- _SYSTEMD_UNIT
- _HOSTNAME
source/systemd:
collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}'
source_name: '{{ .Values.fluentd.logs.systemd.sourceName | quote }}'
source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}'
source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}'
source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}'
source_host_key: "_HOSTNAME"

service:
extensions:
- health_check
Expand All @@ -2907,13 +2963,30 @@ otelcol:
- fluentforward
processors:
- memory_limiter
- attributes
- groupbyattrs
- filter/include_fluent_tag_containers
- attributes/containers
- attributes/remove_fluent_tag
- groupbyattrs/containers
- k8s_tagger
- source
- source/containers
- batch
exporters:
- sumologic
- sumologic/containers
logs/systemd:
receivers:
- fluentforward
processors:
- memory_limiter
- filter/include_fluent_tag_host
- attributes/remove_fluent_tag
- filter/include_systemd
- filter/exclude_kubelet
- groupbyattrs/systemd
- source/systemd
- batch
exporters:
- sumologic/systemd

deployment:
nodeSelector: {}
tolerations: []
Expand Down
79 changes: 71 additions & 8 deletions vagrant/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ kube-prometheus-stack:
dashboardproviders.yaml:
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards/default
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards/default
sidecar:
image:
sha: ''
Expand Down Expand Up @@ -111,6 +111,7 @@ fluent-bit:
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Systemd_Filter _SYSTEMD_UNIT=snap.microk8s.daemon-kubelite.service
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
Expand Down Expand Up @@ -148,3 +149,65 @@ fluentd:
</match>
multiline:
enabled: false

otelcol:
metadata:
logs:
config:
processors:
# Filter out receiver-mock logs to prevent snowball effect
filter/exclude_fluent_tag_receiver_mock_container:
logs:
exclude:
match_type: regexp
record_attributes:
- key: fluent.tag
value: containers\.var\.log\.containers\.receiver-mock.*
filter/exclude_systemd_snap_kubelite:
logs:
exclude:
match_type: strict
record_attributes:
- key: _SYSTEMD_UNIT
value: snap.microk8s.daemon-kubelite.service
service:
extensions:
- health_check
# - sumologic
pipelines:
logs:
receivers:
- fluentforward
processors:
- memory_limiter
- filter/include_fluent_tag_containers

# Vagrant specific
- filter/exclude_fluent_tag_receiver_mock_container

- attributes/containers
- attributes/remove_fluent_tag
- groupbyattrs/containers
- k8s_tagger
- source/containers
- batch
exporters:
- sumologic/containers
logs/systemd:
receivers:
- fluentforward
processors:
- memory_limiter
- filter/include_fluent_tag_host

# Vagrant specific
- filter/exclude_fluent_tag_receiver_mock_container

- attributes/remove_fluent_tag
- filter/include_systemd
- filter/exclude_systemd_snap_kubelite
- groupbyattrs/systemd
- source/systemd
- batch
exporters:
- sumologic/systemd

0 comments on commit e7f0722

Please sign in to comment.