From 4fca9b832f07e5ae7a79f18ff274588c946e961a Mon Sep 17 00:00:00 2001 From: azro352 <35503478+azro352@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:25:09 +0200 Subject: [PATCH] Add logstash helm chart (#7143) * Add logstash helm chart with examples and documentation * Update eck-stack chart to include logstash chart. * Update eck-stack values to include logstash chart, disabled by default. * Include logstash tests. --------- Signed-off-by: Michael Montgomery Co-authored-by: Michael Montgomery Co-authored-by: Peter Brachwitz --- deploy/eck-stack/Chart.yaml | 3 + deploy/eck-stack/README.md | 2 + .../eck-stack/charts/eck-logstash/.helmignore | 24 +++ .../eck-stack/charts/eck-logstash/Chart.yaml | 10 ++ deploy/eck-stack/charts/eck-logstash/LICENSE | 93 ++++++++++ .../eck-logstash/examples/basic-eck.yaml | 44 +++++ .../charts/eck-logstash/examples/es-role.yaml | 25 +++ .../eck-logstash/examples/monitored.yaml | 49 +++++ .../charts/eck-logstash/examples/multi.yaml | 78 ++++++++ .../charts/eck-logstash/examples/volumes.yaml | 107 +++++++++++ .../charts/eck-logstash/templates/NOTES.txt | 6 + .../eck-logstash/templates/_helpers.tpl | 51 ++++++ .../eck-logstash/templates/logstash.yaml | 51 ++++++ .../templates/tests/logstash_test.yaml | 168 ++++++++++++++++++ .../eck-stack/charts/eck-logstash/values.yaml | 106 +++++++++++ .../examples/logstash/basic-eck.yaml | 114 ++++++++++++ deploy/eck-stack/values.yaml | 5 + .../stack-helm-chart.asciidoc | 15 +- hack/helm/release/README.md | 1 + 19 files changed, 951 insertions(+), 1 deletion(-) create mode 100644 deploy/eck-stack/charts/eck-logstash/.helmignore create mode 100644 deploy/eck-stack/charts/eck-logstash/Chart.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/LICENSE create mode 100644 deploy/eck-stack/charts/eck-logstash/examples/basic-eck.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/examples/es-role.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/examples/monitored.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/examples/multi.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/examples/volumes.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/templates/NOTES.txt create mode 100644 deploy/eck-stack/charts/eck-logstash/templates/_helpers.tpl create mode 100644 deploy/eck-stack/charts/eck-logstash/templates/logstash.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/templates/tests/logstash_test.yaml create mode 100644 deploy/eck-stack/charts/eck-logstash/values.yaml create mode 100644 deploy/eck-stack/examples/logstash/basic-eck.yaml diff --git a/deploy/eck-stack/Chart.yaml b/deploy/eck-stack/Chart.yaml index ede8ebe399..a7412c4a20 100644 --- a/deploy/eck-stack/Chart.yaml +++ b/deploy/eck-stack/Chart.yaml @@ -21,3 +21,6 @@ dependencies: - name: eck-beats condition: eck-beats.enabled version: "0.8.0-SNAPSHOT" + - name: eck-logstash + condition: eck-logstash.enabled + version: "0.8.0-SNAPSHOT" diff --git a/deploy/eck-stack/README.md b/deploy/eck-stack/README.md index 31cc0bdb34..e0b4bc7ca9 100644 --- a/deploy/eck-stack/README.md +++ b/deploy/eck-stack/README.md @@ -12,6 +12,7 @@ The following Elastic Stack resources are currently supported. - Elastic Agent - Fleet Server - Beats +- Logstash Additional resources will be supported in future releases of this Helm Chart. @@ -75,6 +76,7 @@ The following table lists the configurable parameters of the eck-stack chart and | `eck-kibana.enabled` | If `true`, create a Kibana resource (using the eck-kibana Chart) | `true` | | `eck-agent.enabled` | If `true`, create an Elastic Agent resource (using the eck-agent Chart) | `false` | | `eck-fleet-server.enabled` | If `true`, create a Fleet Server resource (using the eck-fleet-server Chart) | `false` | +| `eck-logstash.enabled` | If `true`, create a Logstash resource (using the eck-logstash Chart) | `false` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. diff --git a/deploy/eck-stack/charts/eck-logstash/.helmignore b/deploy/eck-stack/charts/eck-logstash/.helmignore new file mode 100644 index 0000000000..f1568daf25 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +templates/tests diff --git a/deploy/eck-stack/charts/eck-logstash/Chart.yaml b/deploy/eck-stack/charts/eck-logstash/Chart.yaml new file mode 100644 index 0000000000..b7143c9750 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: eck-logstash +description: Logstash managed by the ECK operator +kubeVersion: ">= 1.21.0-0" +type: application +version: 0.8.0-SNAPSHOT +sources: + - https://github.com/elastic/cloud-on-k8s + - https://github.com/elastic/logstash +icon: https://helm.elastic.co/icons/logstash.png diff --git a/deploy/eck-stack/charts/eck-logstash/LICENSE b/deploy/eck-stack/charts/eck-logstash/LICENSE new file mode 100644 index 0000000000..92503a7217 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/LICENSE @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. \ No newline at end of file diff --git a/deploy/eck-stack/charts/eck-logstash/examples/basic-eck.yaml b/deploy/eck-stack/charts/eck-logstash/examples/basic-eck.yaml new file mode 100644 index 0000000000..ea7349005c --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/examples/basic-eck.yaml @@ -0,0 +1,44 @@ +--- +# values corresponding to config/recipes/logstash/logstash-eck.yaml +version: 8.11.0-SNAPSHOT + +elasticsearchRefs: + - clusterName: eck + name: elasticsearch + +pipelines: + - pipeline.id: main + config.string: | + input { + beats { + port => 5044 + } + } + filter { + grok { + match => { "message" => "%{HTTPD_COMMONLOG}"} + } + geoip { + source => "[source][address]" + target => "[source]" + } + } + output { + elasticsearch { + hosts => [ "${ECK_ES_HOSTS}" ] + user => "${ECK_ES_USER}" + password => "${ECK_ES_PASSWORD}" + ssl_certificate_authorities => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + +services: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 diff --git a/deploy/eck-stack/charts/eck-logstash/examples/es-role.yaml b/deploy/eck-stack/charts/eck-logstash/examples/es-role.yaml new file mode 100644 index 0000000000..6e725fae17 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/examples/es-role.yaml @@ -0,0 +1,25 @@ +--- +# values corresponding to config/recipes/logstash/logstash-es-role.yaml +version: 8.11.0-SNAPSHOT + +elasticsearchRefs: + - clusterName: eck + name: elasticsearch + +pipelines: + - pipeline.id: main + config.string: | + input { exec { command => "uptime" interval => 10 } } + output { + elasticsearch { + hosts => [ "${ECK_ES_HOSTS}" ] + ssl_enabled => true + ssl_certificate_authorities => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}" + user => "${ECK_ES_USER}" + password => "${ECK_ES_PASSWORD}" + index => "my-index" + data_stream => false + ilm_enabled => false + manage_template => false + } + } diff --git a/deploy/eck-stack/charts/eck-logstash/examples/monitored.yaml b/deploy/eck-stack/charts/eck-logstash/examples/monitored.yaml new file mode 100644 index 0000000000..dc6b52ae53 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/examples/monitored.yaml @@ -0,0 +1,49 @@ +--- +# values corresponding to config/recipes/logstash/logstash-monitored.yaml +version: 8.11.0-SNAPSHOT + +monitoring: + metrics: + elasticsearchRefs: + - name: elasticsearch-monitoring + +pipelines: + - pipeline.id: main + config.string: | + input { + beats { + port => 5044 + } + } + filter { + grok { + match => { "message" => "%{HTTPD_COMMONLOG}"} + } + geoip { + source => "[source][address]" + target => "[source]" + } + } + output { + elasticsearch { + hosts => [ "${ECK_ES_HOSTS}" ] + user => "${ECK_ES_USER}" + password => "${ECK_ES_PASSWORD}" + ssl_certificate_authorities => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + +elasticsearchRefs: + - clusterName: eck + name: elasticsearch + +services: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 diff --git a/deploy/eck-stack/charts/eck-logstash/examples/multi.yaml b/deploy/eck-stack/charts/eck-logstash/examples/multi.yaml new file mode 100644 index 0000000000..cebc9c9023 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/examples/multi.yaml @@ -0,0 +1,78 @@ +--- +# values corresponding to config/recipes/logstash/logstash-multi.yaml +version: 8.11.0-SNAPSHOT + +pipelines: + - pipeline.id: main + config.string: | + input { + beats { + port => 5044 + } + } + filter { + grok { + match => { "message" => "%{HTTPD_COMMONLOG}"} + } + geoip { + source => "[source][address]" + target => "[source]" + } + } + output { + pipeline { + send_to => 'prod' + } + pipeline { + send_to => 'qa' + } + } + - pipeline.id: production + config.string: | + input { + pipeline { + address => 'prod' + } + } + output { + elasticsearch { + hosts => [ "${PROD_ES_ES_HOSTS}" ] + user => "${PROD_ES_ES_USER}" + password => "${PROD_ES_ES_PASSWORD}" + ssl_certificate_authorities => "${PROD_ES_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + - pipeline.id: qa + config.string: | + input { + pipeline { + address => 'qa' + } + } + output { + elasticsearch { + hosts => [ "${QA_ES_ES_HOSTS}" ] + user => "${QA_ES_ES_USER}" + password => "${QA_ES_ES_PASSWORD}" + ssl_certificate_authorities => "${QA_ES_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + +elasticsearchRefs: + - clusterName: prod-es + name: production + - clusterName: qa-es + name: qa + namespace: qa + +services: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 + diff --git a/deploy/eck-stack/charts/eck-logstash/examples/volumes.yaml b/deploy/eck-stack/charts/eck-logstash/examples/volumes.yaml new file mode 100644 index 0000000000..f36cf2c127 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/examples/volumes.yaml @@ -0,0 +1,107 @@ +--- +# values corresponding to config/recipes/logstash/logstash-volumes.yaml +version: 8.11.0-SNAPSHOT + +config: + log.level: info + queue.type: persisted + path.queue: /usr/share/logstash/pq + +podTemplate: + spec: + containers: + - name: logstash + volumeMounts: + - mountPath: /usr/share/logstash/pq + name: pq + readOnly: false + - mountPath: /usr/share/logstash/dlq + name: dlq + readOnly: false + +pipelines: + - pipeline.id: dlq_read + dead_letter_queue.enable: false + config.string: | + input { + dead_letter_queue { + path => "/usr/share/logstash/dlq" + commit_offsets => true + pipeline_id => "beats" + clean_consumed => true + } + } + filter { + mutate { + remove_field => "[geoip][location]" + } + } + output { + elasticsearch { + hosts => [ "${ECK_ES_HOSTS}" ] + user => "${ECK_ES_USER}" + password => "${ECK_ES_PASSWORD}" + ssl_certificate_authorities => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + - pipeline.id: beats + dead_letter_queue.enable: true + path.dead_letter_queue: /usr/share/logstash/dlq + config.string: | + input { + beats { + port => 5044 + } + } + filter { + grok { + match => { "message" => "%{HTTPD_COMMONLOG}"} + } + geoip { + source => "[source][address]" + target => "[source]" + } + } + output { + elasticsearch { + hosts => [ "${ECK_ES_HOSTS}" ] + user => "${ECK_ES_USER}" + password => "${ECK_ES_PASSWORD}" + ssl_certificate_authorities => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + +volumeClaimTemplates: + - metadata: + name: pq + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - metadata: + name: dlq + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + + +elasticsearchRefs: + - clusterName: eck + name: elasticsearch + +services: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 + diff --git a/deploy/eck-stack/charts/eck-logstash/templates/NOTES.txt b/deploy/eck-stack/charts/eck-logstash/templates/NOTES.txt new file mode 100644 index 0000000000..c2f255af99 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/templates/NOTES.txt @@ -0,0 +1,6 @@ + +1. Check Logstash status + $ kubectl get logstash {{ include "logstash.fullname" . }} -n {{ .Release.Namespace }} + +2. Check Logstash pod status + $ kubectl get pods --namespace={{ .Release.Namespace }} -l logstash.k8s.elastic.co/name={{ include "logstash.fullname" . }} diff --git a/deploy/eck-stack/charts/eck-logstash/templates/_helpers.tpl b/deploy/eck-stack/charts/eck-logstash/templates/_helpers.tpl new file mode 100644 index 0000000000..7efd669fdb --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "logstash.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "logstash.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "logstash.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "logstash.labels" -}} +helm.sh/chart: {{ include "logstash.chart" . }} +{{ include "logstash.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.labels }} +{{ toYaml .Values.labels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "logstash.selectorLabels" -}} +app.kubernetes.io/name: {{ include "logstash.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deploy/eck-stack/charts/eck-logstash/templates/logstash.yaml b/deploy/eck-stack/charts/eck-logstash/templates/logstash.yaml new file mode 100644 index 0000000000..36131f010e --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/templates/logstash.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: logstash.k8s.elastic.co/v1alpha1 +kind: Logstash +metadata: + name: {{ include "logstash.fullname" . }} + labels: + {{- include "logstash.labels" . | nindent 4 }} + annotations: + eck.k8s.elastic.co/license: enterprise + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + version: {{ required "A Logstash version is required" .Values.version }} + count: {{ required "A pod count is required" .Values.count }} + {{- with .Values.image }} + image: {{ . }} + {{- end }} + {{- with .Values.serviceAccountName }} + serviceAccountName: {{ . }} + {{- end }} + {{- with .Values.revisionHistoryLimit }} + revisionHistoryLimit: {{ . }} + {{- end }} + + {{- with .Values.config }} + config: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.configRef }} + configRef: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.pipelinesRef }} + pipelinesRef: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.podTemplate }} + podTemplate: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.monitoring }} + monitoring: + {{- toYaml . | nindent 4 }} + {{- end }} + + pipelines: {{ toYaml .Values.pipelines | nindent 4 }} + volumeClaimTemplates: {{ toYaml .Values.volumeClaimTemplates | nindent 4 }} + elasticsearchRefs: {{ toYaml .Values.elasticsearchRefs | nindent 4 }} + services: {{ toYaml .Values.services | nindent 4 }} + secureSettings: {{ toYaml .Values.secureSettings | nindent 4 }} diff --git a/deploy/eck-stack/charts/eck-logstash/templates/tests/logstash_test.yaml b/deploy/eck-stack/charts/eck-logstash/templates/tests/logstash_test.yaml new file mode 100644 index 0000000000..9a8584a5a5 --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/templates/tests/logstash_test.yaml @@ -0,0 +1,168 @@ +suite: test logstash +templates: + - templates/logstash.yaml +tests: + - it: should render quickstart properly + set: + version: 8.2.0 + release: + name: quickstart + asserts: + - isKind: + of: Logstash + - equal: + path: metadata.name + value: quickstart-eck-logstash + - equal: + path: spec.version + value: 8.2.0 + - it: name override should work properly + set: + nameOverride: override + release: + name: quickstart + asserts: + - isKind: + of: Logstash + - equal: + path: metadata.name + value: quickstart-override + - it: fullname override should work properly + set: + fullnameOverride: override + release: + name: quickstart + asserts: + - isKind: + of: Logstash + - equal: + path: metadata.name + value: override + - it: setting volumeClaimDeletePolicy should work properly + set: + volumeClaimTemplates: + - metadata: + name: pq + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + release: + name: quickstart + asserts: + - isKind: + of: Logstash + - equal: + path: spec.volumeClaimTemplates + value: + - metadata: + name: pq + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - it: should render labels, annotations, and additional custom values properly + set: + labels: + test: label + annotations: + test: annotation + monitoring: + metrics: + elasticsearchRefs: + - name: monitoring + namespace: observability + logs: + elasticsearchRefs: + - name: monitoring + namespace: observability + services: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 + release: + name: quickstart + asserts: + - isKind: + of: Logstash + - equal: + path: metadata.labels + value: + app.kubernetes.io/instance: quickstart + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: eck-logstash + helm.sh/chart: eck-logstash-0.8.0-SNAPSHOT + test: label + - equal: + path: metadata.annotations + value: + eck.k8s.elastic.co/license: enterprise + test: annotation + - equal: + path: spec.services + value: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 + - it: should render image properly + set: + image: my.regis.try/logstash:8 + asserts: + - equal: + path: spec.image + value: my.regis.try/logstash:8 + - it: should render image properly + set: + image: my.registry.com/elastic/logstash:8.9.0 + asserts: + - equal: + path: spec.image + value: my.registry.com/elastic/logstash:8.9.0 + - it: should render serviceAccountName properly + set: + serviceAccountName: my-sa + asserts: + - equal: + path: spec.serviceAccountName + value: my-sa + - it: should render revisionHistoryLimit properly + set: + revisionHistoryLimit: 5 + asserts: + - equal: + path: spec.revisionHistoryLimit + value: 5 + - it: should render configRef properly + set: + configRef: + secretRef: + secretName: configLogstashSecretName + asserts: + - equal: + path: spec.configRef.secretRef.secretName + value: configLogstashSecretName + - it: should render pipelinesRef properly + set: + pipelinesRef: + secretRef: + secretName: pipelineLogstashSecretName + asserts: + - equal: + path: spec.pipelinesRef.secretRef.secretName + value: pipelineLogstashSecretName diff --git a/deploy/eck-stack/charts/eck-logstash/values.yaml b/deploy/eck-stack/charts/eck-logstash/values.yaml new file mode 100644 index 0000000000..10324c419a --- /dev/null +++ b/deploy/eck-stack/charts/eck-logstash/values.yaml @@ -0,0 +1,106 @@ +--- +# Default values for eck-logstash. +# This is a YAML-formatted file. + +# Overridable names of the Logstash resource. +# By default, this is the Release name set for the chart, +# followed by 'eck-logstash'. +# +# nameOverride will override the name of the Chart with the name set here, +# so nameOverride: quickstart, would convert to '{{ Release.name }}-quickstart' +# +# nameOverride: "quickstart" +# +# fullnameOverride will override both the release name, and the chart name, +# and will name the Logstash resource exactly as specified. +# +# fullnameOverride: "quickstart" + +# Version of Logstash. +# +version: 8.11.0-SNAPSHOT + +# Logstash Docker image to deploy +# +# image: + +# Used to check access from the current resource to a resource (for ex. a remote Elasticsearch cluster) in a different namespace. +# Can only be used if ECK is enforcing RBAC on references. +# +# serviceAccountName: "" + +# Labels that will be applied to Logstash. +# +labels: {} + +# Annotations that will be applied to Logstash. +# +annotations: {} + +# Number of revisions to retain to allow rollback in the underlying StatefulSets. +# By default, if not set, Kubernetes sets 10. +# +# revisionHistoryLimit: 2 + +# Controlling the number of pods. +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-logstash-scaling-logstash.html +# +count: 1 + +# The logstash configuration, the ECK equivalent to logstash.yml +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-logstash-configuration.html#k8s-logstash-configuring-logstash +# +config: {} + +configRef: {} +# secretRef: +# secretName: '' + +# Set podTemplate to customize the pod used by Logstash +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-customize-pods.html +# +podTemplate: {} + +# Settings for configuring stack monitoring. +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-stack-monitoring.html +# +monitoring: {} + # metrics: + # elasticsearchRefs: + # - name: monitoring + # namespace: observability + # logs: + # elasticsearchRefs: + # - name: monitoring + # namespace: observability + +# The Logstash pipelines, the ECK equivalent to pipelines.yml +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-logstash-configuration.html#k8s-logstash-pipelines +# +pipelines: [] + +pipelinesRef: {} +# secretRef: +# secretName: '' + + +# volumeClaimTemplates +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-logstash-configuration.html#k8s-volume-claim-settings +# +volumeClaimTemplates: [] + +# ElasticsearchRefs are references to Elasticsearch clusters running in the same Kubernetes cluster. +# Ensure that the 'clusterName' field matches what is referenced in the pipeline. +# ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-logstash-configuration.html#k8s-logstash-pipelines-es +# +elasticsearchRefs: [] +# - namespace: '' +# name: '' +# clusterName: '' +# serviceName: '' +# secretName: '' + +services: [] + +# SecureSettings is a list of references to Kubernetes Secrets containing sensitive configuration options for the Logstash +secureSettings: [] diff --git a/deploy/eck-stack/examples/logstash/basic-eck.yaml b/deploy/eck-stack/examples/logstash/basic-eck.yaml new file mode 100644 index 0000000000..00f0b94f9e --- /dev/null +++ b/deploy/eck-stack/examples/logstash/basic-eck.yaml @@ -0,0 +1,114 @@ +--- +eck-elasticsearch: + nodeSets: + - name: default + count: 3 + config: + # Comment out when setting the vm.max_map_count via initContainer, as these are mutually exclusive. + # For production workloads, it is strongly recommended to increase the kernel setting vm.max_map_count to 262144 + # and leave node.store.allow_mmap unset. + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html + # + node.store.allow_mmap: false + podTemplate: + spec: + containers: + - name: elasticsearch + resources: + limits: + memory: 2Gi + requests: + memory: 2Gi +eck-kibana: + enabled: true + spec: + count: 1 + elasticsearchRef: + name: elasticsearch +eck-beats: + enabled: true + spec: + type: filebeat + daemonSet: null + config: + filebeat.inputs: + - type: log + paths: + - /data/logstash-tutorial.log + processors: + - add_host_metadata: {} + - add_cloud_metadata: {} + output.logstash: + # This needs to be {{logstash-name}}-ls-beats:5044 + hosts: ["logstash-ls-beats-ls-beats:5044"] + deployment: + podTemplate: + spec: + automountServiceAccountToken: true + initContainers: + - name: download-tutorial + image: curlimages/curl + command: ["/bin/sh"] + args: ["-c", "curl -L https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz | gunzip -c > /data/logstash-tutorial.log"] + volumeMounts: + - name: data + mountPath: /data + containers: + - name: filebeat + securityContext: + runAsUser: 1000 + volumeMounts: + - name: data + mountPath: /data + - name: beat-data + mountPath: /usr/share/filebeat/data + volumes: + - name: data + emptydir: {} + - name: beat-data + emptydir: {} +eck-logstash: + enabled: true + # This is required to be able to set the logstash + # output of beats in a consistent manner. + fullnameOverride: "logstash-ls-beats" + elasticsearchRefs: + # This clusterName is required to match the environment variables + # used in the below config.string output section. + - clusterName: eck + name: elasticsearch + pipelines: + - pipeline.id: main + config.string: | + input { + beats { + port => 5044 + } + } + filter { + grok { + match => { "message" => "%{HTTPD_COMMONLOG}"} + } + geoip { + source => "[source][address]" + target => "[source]" + } + } + output { + elasticsearch { + hosts => [ "${ECK_ES_HOSTS}" ] + user => "${ECK_ES_USER}" + password => "${ECK_ES_PASSWORD}" + ssl_certificate_authorities => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}" + } + } + services: + - name: beats + service: + spec: + type: ClusterIP + ports: + - port: 5044 + name: "filebeat" + protocol: TCP + targetPort: 5044 diff --git a/deploy/eck-stack/values.yaml b/deploy/eck-stack/values.yaml index d7753ac223..381566c632 100644 --- a/deploy/eck-stack/values.yaml +++ b/deploy/eck-stack/values.yaml @@ -34,3 +34,8 @@ eck-fleet-server: # eck-beats: enabled: false + +# If enabled, will use the eck-logstash chart and deploy a Logstash resource. +# +eck-logstash: + enabled: false diff --git a/docs/orchestrating-elastic-stack-applications/stack-helm-chart.asciidoc b/docs/orchestrating-elastic-stack-applications/stack-helm-chart.asciidoc index 2e6373968b..d9adf02778 100644 --- a/docs/orchestrating-elastic-stack-applications/stack-helm-chart.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/stack-helm-chart.asciidoc @@ -60,6 +60,19 @@ helm install eck-stack-with-fleet elastic/eck-stack \ --values https://raw.githubusercontent.com/elastic/cloud-on-k8s/{eck_release_branch}/deploy/eck-stack/examples/agent/fleet-agents.yaml -n elastic-stack ---- +[float] +[id="{p}-install-logstash-elasticsearch-kibana-helm"] +== Installing Logstash along with Elasticsearch, Kibana and Beats using the eck-stack Helm Chart + +The following section builds upon the previous sections, and allows installing Logstash along with Elasticsearch, Kibana and Beats. + +[source,sh,subs="attributes"] +---- +# Install an eck-managed Elasticsearch, Kibana, Beats and Logstash using custom values. +helm install eck-stack-with-logstash elastic/eck-stack \ + --values https://raw.githubusercontent.com/elastic/cloud-on-k8s/{eck_release_branch}/deploy/eck-stack/examples/logstash/basic-eck.yaml -n elastic-stack +---- + [float] [id="{p}-eck-stack-individual-components"] === Installing individual components of the Elastic Stack using the Helm Charts @@ -81,4 +94,4 @@ helm install es-quickstart elastic/eck-stack -n elastic-stack --create-namespace [source,sh] ---- helm install es-quickstart elastic/eck-elasticsearch -n elastic-stack --create-namespace ----- \ No newline at end of file +---- diff --git a/hack/helm/release/README.md b/hack/helm/release/README.md index a849498a98..7b0fe66b0a 100644 --- a/hack/helm/release/README.md +++ b/hack/helm/release/README.md @@ -35,6 +35,7 @@ ECK Helm charts are grouped under 2 parent charts: `eck-operator` and `eck-stack │   │   ├── eck-elasticsearch │   │   ├── eck-fleet-server │   │   └── eck-kibana +│   │   └── eck-logstash ``` All subcharts are managed locally through their parent's `charts/` directory.