diff --git a/docs/tasks/debug-application-cluster/audit.md b/docs/tasks/debug-application-cluster/audit.md index 3105b7cb15375..69eff766a8463 100644 --- a/docs/tasks/debug-application-cluster/audit.md +++ b/docs/tasks/debug-application-cluster/audit.md @@ -359,57 +359,66 @@ Note that this example requries json format output support in Kubernetes 1.8. 1. install [fluentd, fluent-plugin-forest and fluent-plugin-rewrite-tag-filter][fluentd_install_doc] in the kube-apiserver node 1. create a config file for fluentd - $ cat < /etc/fluentd/config - # fluentd conf runs in the same host with kube-apiserver - - @type tail - # audit log path of kube-apiserver - path /var/log/audit - pos_file /var/log/audit.pos - format json - time_key time - time_format %Y-%m-%dT%H:%M:%S.%N%z - tag audit - + ```shell + $ cat < /etc/fluentd/config + # fluentd conf runs in the same host with kube-apiserver + + @type tail + # audit log path of kube-apiserver + path /var/log/audit + pos_file /var/log/audit.pos + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%N%z + tag audit + - - #https://github.com/fluent/fluent-plugin-rewrite-tag-filter/issues/13 - type record_transformer - enable_ruby - - namespace ${record["objectRef"].nil? ? "none":(record["objectRef"]["namespace"].nil? ? "none":record["objectRef"]["namespace"])} - - + + #https://github.com/fluent/fluent-plugin-rewrite-tag-filter/issues/13 + type record_transformer + enable_ruby + + namespace ${record["objectRef"].nil? ? "none":(record["objectRef"]["namespace"].nil? ? "none":record["objectRef"]["namespace"])} + + - - # route audit according to namespace element in context - @type rewrite_tag_filter - rewriterule1 namespace ^(.+) ${tag}.$1 - + + # route audit according to namespace element in context + @type rewrite_tag_filter + rewriterule1 namespace ^(.+) ${tag}.$1 + - - @type record_transformer - remove_keys namespace - + + @type record_transformer + remove_keys namespace + - - @type forest - subtype file - remove_prefix audit - - + + @type forest + subtype file + remove_prefix audit + + + ``` + 1. start fluentd - $ fluentd -c /etc/fluentd/config -vv + ```shell + $ fluentd -c /etc/fluentd/config -vv + ``` + 1. start kube-apiserver with the following options: - --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kube-audit --audit-log-format=json + ```shell + --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kube-audit --audit-log-format=json + ``` + 1. check audits for different namespaces in /var/log/audit-*.log #### Use logstash to collect and distribute audit events from webhook backend @@ -421,56 +430,68 @@ different users into different files. 1. install [logstash][logstash_install_doc] 1. create config file for logstash - $ cat < /etc/logstash/config - input{ - http{ - #TODO, figure out a way to use kubeconfig file to authenticate to logstash - #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http.html#plugins-inputs-http-ssl - port=>8888 - } - } - filter{ - split{ - # Webhook audit backend sends several events together with EventList - # split each event here. - field=>[items] - # We only need event subelement, remove others. - remove_field=>[headers, metadata, apiVersion, "@timestamp", kind, "@version", host] - } - mutate{ - rename => {items=>event} - } - } - output{ - file{ - # Audit events from different users will be saved into different files. - path=>"/var/log/kube-audit-%{[event][user][username]}/audit" - } - } + ```shell + $ cat < /etc/logstash/config + input{ + http{ + #TODO, figure out a way to use kubeconfig file to authenticate to logstash + #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http.html#plugins-inputs-http-ssl + port=>8888 + } + } + filter{ + split{ + # Webhook audit backend sends several events together with EventList + # split each event here. + field=>[items] + # We only need event subelement, remove others. + remove_field=>[headers, metadata, apiVersion, "@timestamp", kind, "@version", host] + } + mutate{ + rename => {items=>event} + } + } + output{ + file{ + # Audit events from different users will be saved into different files. + path=>"/var/log/kube-audit-%{[event][user][username]}/audit" + } + } + ``` + 1. start logstash - $ bin/logstash -f /etc/logstash/config --path.settings /etc/logstash/ + ```shell + $ bin/logstash -f /etc/logstash/config --path.settings /etc/logstash/ + ``` + 1. create a [kubeconfig file](/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) for kube-apiserver webhook audit backend - $ cat < /etc/kubernetes/audit-webhook-kubeconfig - apiVersion: v1 - clusters: - - cluster: - server: http://:8888 - name: logstash - contexts: - - context: - cluster: logstash - user: "" - name: default-context - current-context: default-context - kind: Config - preferences: {} - users: [] - EOF + ```shell + $ cat < /etc/kubernetes/audit-webhook-kubeconfig + apiVersion: v1 + clusters: + - cluster: + server: http://:8888 + name: logstash + contexts: + - context: + cluster: logstash + user: "" + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: [] + EOF + ``` + 1. start kube-apiserver with the following options: - --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-webhook-config-file=/etc/kubernetes/audit-webhook-kubeconfig + ```shell + --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-webhook-config-file=/etc/kubernetes/audit-webhook-kubeconfig + ``` + 1. check audits in logstash node's directories /var/log/kube-audit-*/audit Note that in addition to file output plugin, logstash has a variety of outputs that