diff --git a/elastic-agent/elastic-agent-providers.asciidoc b/elastic-agent/elastic-agent-providers.asciidoc index 44125efe34..6fd2d8e518 100644 --- a/elastic-agent/elastic-agent-providers.asciidoc +++ b/elastic-agent/elastic-agent-providers.asciidoc @@ -299,3 +299,185 @@ inputs: - add_fields: container.name: other-container ---- + +[[kubernetes-provider]] +==== Kubernetes Provider + +Provides inventory information from Kubernetes. The available keys are: + + +|=== +|Key |Type |Description + +|`kubernetes.namespace` +|`string` +|Namespace of the Pod + +|`kubernetes.pod.name` +|`string` +|Name of the Pod + +|`kubernetes.pod.uuid` +|`string` +|UUID of the Pod + +|`kubernetes.pod.ip` +|`string` +|IP of the Pod + +|`kubernetes.pod.labels` +|`object` +|Object of labels of the Pod + +|`kubernetes.container.name` +|`string` +|Name of the container + +|`kubernetes.container.runtime` +|`string` +|Runtime of the container + +|`kubernetes.container.id` +|`string` +|ID of the container + +|`kubernetes.container.image` +|`string` +|Image of the container +|=== + +Imagine that the Kubernetes provider provides the following inventory: + +[source,json] +---- +[ + { + "id": "1", + "mapping:": {"namespace": "kube-system", "pod": {"name": "kube-controllermanger"}}, + "processors": {"add_fields": {"container.name": "my-container"}} + }, + { + "id": "2", + "mapping:": {"namespace": "kube-system", "pod": {"name": "kube-scheduler"}}, + "processors": {"add_fields": {"kuberentes.namespace": "kube-system", "kubernetes.pod": {"name": "kube-scheduler"}} + } +] +---- + +{agent} automatically prefixes the result with `kuberentes`: + + +[source,json] +--- +[ + {"kubernetes": {"id": "1", "namespace": "kube-system", "pod": {"name": "kube-controllermanger"}}, + {"kubernetes": {"id": "2", "namespace": "kube-system", "pod": {"name": "kube-scheduler"}}, +] +--- + +===== Provider configuration + +[source,yaml] +---- +providers.kubernetes: + node: ${NODE_NAME} + scope: node + #kube_config: /Users/elastic-agent/.kube/config + #sync_period: 600 + #cleanup_timeout: 60 +---- + +`node`:: (Optional) Specify the node to scope {agent} to in case it +cannot be accurately detected, as when running {agent} in host network +mode. +`cleanup_timeout`:: (Optional) Specify the time of inactivity before stopping the +running configuration for a container, 60s by default. +`sync_period`:: (Optional) Specify timeout for listing historical resources. +`kube_config`:: (Optional) Use given config file as configuration for Kubernetes +client. If kube_config is not set, KUBECONFIG environment variable will be +checked and if not present it will fall back to InCluster. +`scope`:: (Optional) Specify at what level autodiscover needs to be done at. `scope` can +either take `node` or `cluster` as values. `node` scope allows discovery of resources in +the specified node. `cluster` scope allows cluster wide discovery. Only `pod` and `node` resources +can be discovered at node scope. + +===== Autodiscover target Pods + +To set the target host dynamically only for a targeted Pod based on its labels, use a variable in the +{agent} policy to return path information from the provider: + +[source,yaml] +---- +- data_stream: + dataset: kubernetes.scheduler + type: metrics + metricsets: + - scheduler + hosts: + - '${kubernetes.pod.ip}:10251' + period: 10s + condition: ${kubernetes.pod.labels.component} == 'kube-scheduler' +---- + +The policy generated by this configuration looks like: + +[source,yaml] +---- +- hosts: + - 172.18.0.4:10251 + metricsets: + - scheduler + module: kubernetes + period: 10s + processors: + - add_fields: + fields: + namespace: kube-system + pod: + ip: 172.18.0.4 + labels: + component: kube-scheduler + tier: control-plane + name: kube-scheduler-kind-control-plane + uid: 6da04645-04b4-4cb2-b203-2ad58abc6cdf + target: kubernetes +---- + +To set the log path of Pods dynamically in the configuration, use a variable in the +{agent} policy to return path information from the provider: + +[source,yaml] +---- +streams: + - data_stream: + dataset: generic + symlinks: true + paths: + - /var/log/containers/*${kubernetes.container.id}.log +---- + +The policy generated by this configuration looks like: + +[source,yaml] +---- +- paths: + - /var/log/containers/*c957652eca53594ce496c7b237d19f05be339ebfe281b99ce1c0a0401e48ce3a.log + processors: + - add_fields: + fields: + container: + id: c957652eca53594ce496c7b237d19f05be339ebfe281b99ce1c0a0401e48ce3a + image: k8s.gcr.io/kube-apiserver:v1.18.2 + name: kube-apiserver + runtime: containerd + namespace: kube-system + pod: + ip: 172.18.0.4 + labels: + component: kube-apiserver + tier: control-plane + name: kube-apiserver-kind-control-plane + uid: f8743f90-50a4-4ef8-9fe9-78c245eb8bf3 + target: kubernetes + symlinks: true +---- \ No newline at end of file diff --git a/elastic-agent/running-on-kubernetes-standalone.asciidoc b/elastic-agent/running-on-kubernetes-standalone.asciidoc new file mode 100644 index 0000000000..87ce15bef9 --- /dev/null +++ b/elastic-agent/running-on-kubernetes-standalone.asciidoc @@ -0,0 +1,138 @@ +[[running-on-kubernetes]] +[role="xpack"] +=== Run {agent} on Kubernetes + +You can use {agent} https://www.docker.elastic.co/r/beats/elastic-agent[Docker images] on Kubernetes to +retrieve cluster metrics. + +ifeval::["{release-state}"=="unreleased"] + +However, version {version} of {agent} has not yet been +released, so no Docker image is currently available for this version. + +endif::[] + + +[float] +==== Kubernetes deploy manifests + +You deploy {agent} in two different ways at the same time: + +* As a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] +to ensure that there's a running instance on each node of the cluster. These +instances are used to retrieve most metrics from the host, such as system +metrics, Docker stats, and metrics from all the services running on top of +Kubernetes. + +* As a single {agent} instance created using a https://kubernetes.io/docs/concepts/workloads/controllers/Deployment/[Deployment]. +This instance is used to retrieve metrics that are unique for the whole +cluster, such as Kubernetes events or +https://github.com/kubernetes/kube-state-metrics[kube-state-metrics]. If `kube-state-metrics` is not already +running, deploy it now (see the +https://github.com/kubernetes/kube-state-metrics#kubernetes-deployment[Kubernetes +deployment] docs) + +Everything is deployed under the `kube-system` namespace by default. To change +the namespace, modify the manifest file. + +To download the manifest file, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +------------------------------------------------ + +This manifest includes Kubernetes integration to collect Kubernetes metrics, +System integration to collect system level metrics/logs from nodes and +Pod's log collection using <>. + +[float] +==== Settings + +User need to set Elasticsearch settings before deploying the manifest: + +[source,yaml] +------------------------------------------------ +- name: ES_USERNAME + value: "elastic" +- name: ES_PASSWORD + value: "passpassMyStr0ngP@ss" +- name: ES_HOST + value: "https://somesuperhostiduuid.europe-west1.gcp.cloud.es.io:443" +------------------------------------------------ + +[float] +===== Running {agent} on master nodes + +Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. The manifest for standalone {agent} defines +tolerations so as to run on master nodes too and being able to collect metrics from the control plane +components of Kuberentes (scheduler, controller manager) +To disable {agent} from running on master nodes remove the following part of the Daemonset spec: + +[source,yaml] +------------------------------------------------ +spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule +------------------------------------------------ + + +[float] +==== Deploy +To deploy to Kubernetes, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +kubectl create -f elastic-agent-standalone-kubernetes.yaml +------------------------------------------------ + +To check the status, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ kubectl -n kube-system get pods -l app=elastic-agent +NAME READY STATUS RESTARTS AGE +elastic-agent-4665d 1/1 Running 0 81m +elastic-agent-9f466c4b5-l8cm8 1/1 Running 0 81m +elastic-agent-fj2z9 1/1 Running 0 81m +elastic-agent-hs4pb 1/1 Running 0 81m +------------------------------------------------ + +[float] +==== Autodiscover targeted Pods + +It is possible to define autodiscover conditions so as to allow {agent} to automatically +identify Pods and start collecting from them using predefined integrations. For example if a +user want to automatically identify a Redis Pod and start monitoring it using the Redis integration +the following configuration should be added as an extra input in the Daemonset manifest: + +[source,yaml] +------------------------------------------------ +- name: redis + type: redis/metrics + use_output: default + meta: + package: + name: redis + version: 0.3.6 + data_stream: + namespace: default + streams: + - data_stream: + dataset: redis.info + type: metrics + metricsets: + - info + hosts: + - '${kubernetes.pod.ip}:6379' + idle_timeout: 20s + maxconn: 10 + network: tcp + period: 10s + condition: ${kubernetes.pod.labels.app} == 'redis' +------------------------------------------------ + +Users can find more information about how to shape their dynamic inputs +for autodiscover at <> docs. \ No newline at end of file