From cd4bc37db7cd84ab891245d0e1fab85de0c78bcc Mon Sep 17 00:00:00 2001
From: Quentin Pradet <quentin.pradet@elastic.co>
Date: Thu, 28 Oct 2021 14:57:16 +0400
Subject: [PATCH] Update log4j configuration to current master

Specific revision is 0d3cebe812080ea153d124b313fb2c184c7df75b.
---
 .../templates/config/log4j2.properties        | 94 +++++--------------
 .../templates/config/log4j2.properties        | 34 +++++--
 2 files changed, 47 insertions(+), 81 deletions(-)

diff --git a/cars/v1/vanilla/templates/config/log4j2.properties b/cars/v1/vanilla/templates/config/log4j2.properties
index ac633cb..1efbae8 100644
--- a/cars/v1/vanilla/templates/config/log4j2.properties
+++ b/cars/v1/vanilla/templates/config/log4j2.properties
@@ -1,9 +1,7 @@
+{# Don't parse this file using jinja2 #}
+{% raw %}
 status = error
 
-# log action execution errors for easier debugging
-logger.action.name = org.elasticsearch.action
-logger.action.level = debug
-
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
@@ -13,8 +11,8 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%ma
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json
-appender.rolling.layout.type = ESJsonLayout
-appender.rolling.layout.type_name = server
+appender.rolling.layout.type = ECSJsonLayout
+appender.rolling.layout.dataset = elasticsearch.server
 
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz
 appender.rolling.policies.type = Policies
@@ -61,22 +59,14 @@ rootLogger.appenderRef.console.ref = console
 rootLogger.appenderRef.rolling.ref = rolling
 rootLogger.appenderRef.rolling_old.ref = rolling_old
 
-{% if verbose_imc_logging_enabled is defined and verbose_imc_logging_enabled %}
-logger.verbose_imc.name = org.elasticsearch.indices.IndexingMemoryController
-logger.verbose_imc.level = debug
-# don't spam console
-# logger.verbose_imc.appenderRef.console.ref = console
-logger.verbose_imc.appenderRef.rolling.ref = rolling
-logger.verbose_imc.appenderRef.rolling_old.ref = rolling_old
-logger.verbose_imc.additivity = false
-{%- endif %}
-
 ######## Deprecation JSON #######################
 appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json
-appender.deprecation_rolling.layout.type = ESJsonLayout
-appender.deprecation_rolling.layout.type_name = deprecation
+appender.deprecation_rolling.layout.type = ECSJsonLayout
+# Intentionally follows a different pattern to above
+appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch
+appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter
 
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz
 appender.deprecation_rolling.policies.type = Policies
@@ -84,26 +74,15 @@ appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
 appender.deprecation_rolling.policies.size.size = 1GB
 appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
 appender.deprecation_rolling.strategy.max = 4
+
+appender.header_warning.type = HeaderWarningAppender
+appender.header_warning.name = header_warning
 #################################################
-######## Deprecation -  old style pattern #######
-appender.deprecation_rolling_old.type = RollingFile
-appender.deprecation_rolling_old.name = deprecation_rolling_old
-appender.deprecation_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
-appender.deprecation_rolling_old.layout.type = PatternLayout
-appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
-
-appender.deprecation_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
-  _deprecation-%i.log.gz
-appender.deprecation_rolling_old.policies.type = Policies
-appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
-appender.deprecation_rolling_old.policies.size.size = 1GB
-appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy
-appender.deprecation_rolling_old.strategy.max = 4
-#################################################
+
 logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = warn
+logger.deprecation.level = WARN
 logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
-logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old
+logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
 ######## Search slowlog JSON ####################
@@ -111,8 +90,8 @@ appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
   .cluster_name}_index_search_slowlog.json
-appender.index_search_slowlog_rolling.layout.type = ESJsonLayout
-appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog
+appender.index_search_slowlog_rolling.layout.type = ECSJsonLayout
+appender.index_search_slowlog_rolling.layout.dataset = elasticsearch.index_search_slowlog
 
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
   .cluster_name}_index_search_slowlog-%i.json.gz
@@ -122,26 +101,11 @@ appender.index_search_slowlog_rolling.policies.size.size = 1GB
 appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy
 appender.index_search_slowlog_rolling.strategy.max = 4
 #################################################
-######## Search slowlog -  old style pattern ####
-appender.index_search_slowlog_rolling_old.type = RollingFile
-appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old
-appender.index_search_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
-  _index_search_slowlog.log
-appender.index_search_slowlog_rolling_old.layout.type = PatternLayout
-appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
-
-appender.index_search_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
-  _index_search_slowlog-%i.log.gz
-appender.index_search_slowlog_rolling_old.policies.type = Policies
-appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
-appender.index_search_slowlog_rolling_old.policies.size.size = 1GB
-appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy
-appender.index_search_slowlog_rolling_old.strategy.max = 4
+
 #################################################
 logger.index_search_slowlog_rolling.name = index.search.slowlog
 logger.index_search_slowlog_rolling.level = trace
 logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
-logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old
 logger.index_search_slowlog_rolling.additivity = false
 
 ######## Indexing slowlog JSON ##################
@@ -149,8 +113,9 @@ appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
   _index_indexing_slowlog.json
-appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout
-appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog
+appender.index_indexing_slowlog_rolling.layout.type = ECSJsonLayout
+appender.index_indexing_slowlog_rolling.layout.dataset = elasticsearch.index_indexing_slowlog
+
 
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
   _index_indexing_slowlog-%i.json.gz
@@ -160,25 +125,10 @@ appender.index_indexing_slowlog_rolling.policies.size.size = 1GB
 appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy
 appender.index_indexing_slowlog_rolling.strategy.max = 4
 #################################################
-######## Indexing slowlog -  old style pattern ##
-appender.index_indexing_slowlog_rolling_old.type = RollingFile
-appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old
-appender.index_indexing_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
-  _index_indexing_slowlog.log
-appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
-
-appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\
-  _index_indexing_slowlog-%i.log.gz
-appender.index_indexing_slowlog_rolling_old.policies.type = Policies
-appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
-appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB
-appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy
-appender.index_indexing_slowlog_rolling_old.strategy.max = 4
-#################################################
+
 
 logger.index_indexing_slowlog.name = index.indexing.slowlog.index
 logger.index_indexing_slowlog.level = trace
 logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
-logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old
 logger.index_indexing_slowlog.additivity = false
+{% endraw %}
diff --git a/cars/v1/x_pack/security/templates/config/log4j2.properties b/cars/v1/x_pack/security/templates/config/log4j2.properties
index cfc3961..aa5a8b6 100644
--- a/cars/v1/x_pack/security/templates/config/log4j2.properties
+++ b/cars/v1/x_pack/security/templates/config/log4j2.properties
@@ -13,17 +13,19 @@ appender.audit_rolling.layout.pattern = {\
                 %varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\
                 %varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\
                 %varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\
+                %varsNotEmpty{, "authentication.type":"%enc{%map{authentication.type}}{JSON}"}\
                 %varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\
                 %varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\
                 %varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\
                 %varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\
                 %varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\
                 %varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\
+                %varsNotEmpty{, "user.roles":%map{user.roles}}\
                 %varsNotEmpty{, "apikey.id":"%enc{%map{apikey.id}}{JSON}"}\
                 %varsNotEmpty{, "apikey.name":"%enc{%map{apikey.name}}{JSON}"}\
-                %varsNotEmpty{, "user.roles":%map{user.roles}}\
+                %varsNotEmpty{, "authentication.token.name":"%enc{%map{authentication.token.name}}{JSON}"}\
+                %varsNotEmpty{, "authentication.token.type":"%enc{%map{authentication.token.type}}{JSON}"}\
                 %varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\
-                %varsNotEmpty{, "authentication.type":"%enc{%map{authentication.type}}{JSON}"}\
                 %varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\
                 %varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\
                 %varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\
@@ -38,14 +40,19 @@ appender.audit_rolling.layout.pattern = {\
                 %varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\
                 %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\
                 %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\
-                %varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\
+                %varsNotEmpty{, "put":%map{put}}\
+                %varsNotEmpty{, "delete":%map{delete}}\
+                %varsNotEmpty{, "change":%map{change}}\
+                %varsNotEmpty{, "create":%map{create}}\
+                %varsNotEmpty{, "invalidate":%map{invalidate}}\
                 }%n
 # "node.name" node name from the `elasticsearch.yml` settings
 # "node.id" node id which should not change between cluster restarts
 # "host.name" unresolved hostname of the local node
 # "host.ip" the local bound ip (i.e. the ip listening for connections)
-# "event.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal)
+# "origin.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal)
 # "event.action" the name of the audited event, eg. "authentication_failed", "access_granted", "run_as_granted", etc.
+# "authentication.type" one of "realm", "api_key", "token", "anonymous" or "internal"
 # "user.name" the subject name as authenticated by a realm
 # "user.run_by.name" the original authenticated subject name that is impersonating another one.
 # "user.run_as.name" if this "event.action" is of a run_as type, this is the subject name to be impersonated as.
@@ -53,28 +60,37 @@ appender.audit_rolling.layout.pattern = {\
 # "user.run_by.realm" the realm name of the impersonating subject ("user.run_by.name")
 # "user.run_as.realm" if this "event.action" is of a run_as type, this is the realm name the impersonated user is looked up from
 # "user.roles" the roles array of the user; these are the roles that are granting privileges
-# "origin.type" it is "rest" if the event is originating (is in relation to) a REST request; possible other values are "transport" and "ip_filter"
+# "apikey.id" this field is present if and only if the "authentication.type" is "api_key"
+# "apikey.name" this field is present if and only if the "authentication.type" is "api_key"
+# "authentication.token.name" this field is present if and only if the authenticating credential is a service account token
+# "authentication.token.type" this field is present if and only if the authenticating credential is a service account token
+# "event.type" informs about what internal system generated the event; possible values are "rest", "transport", "ip_filter" and "security_config_change"
 # "origin.address" the remote address and port of the first network hop, i.e. a REST proxy or another cluster node
 # "realm" name of a realm that has generated an "authentication_failed" or an "authentication_successful"; the subject is not yet authenticated
 # "url.path" the URI component between the port and the query string; it is percent (URL) encoded
 # "url.query" the URI component after the path and before the fragment; it is percent (URL) encoded
 # "request.method" the method of the HTTP request, i.e. one of GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT
 # "request.body" the content of the request body entity, JSON escaped
-# "request.id" a synthentic identifier for the incoming request, this is unique per incoming request, and consistent across all audit events generated by that request
+# "request.id" a synthetic identifier for the incoming request, this is unique per incoming request, and consistent across all audit events generated by that request
 # "action" an action is the most granular operation that is authorized and this identifies it in a namespaced way (internal)
 # "request.name" if the event is in connection to a transport message this is the name of the request class, similar to how rest requests are identified by the url path (internal)
 # "indices" the array of indices that the "action" is acting upon
 # "opaque_id" opaque value conveyed by the "X-Opaque-Id" request header
 # "x_forwarded_for" the addresses from the "X-Forwarded-For" request header, as a verbatim string value (not an array)
 # "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event
-# "rule" name of the applied rulee if the "origin.type" is "ip_filter"
-# "event.category" fixed value "elasticsearch-audit"
+# "rule" name of the applied rule if the "origin.type" is "ip_filter"
+# the "put", "delete", "change", "create", "invalidate" fields are only present
+# when the "event.type" is "security_config_change" and contain the security config change (as an object) taking effect
 
-appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}.json
+appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}-%i.json.gz
 appender.audit_rolling.policies.type = Policies
 appender.audit_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.audit_rolling.policies.time.interval = 1
 appender.audit_rolling.policies.time.modulate = true
+appender.audit_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.audit_rolling.policies.size.size = 1GB
+appender.audit_rolling.strategy.type = DefaultRolloverStrategy
+appender.audit_rolling.strategy.fileIndex = nomax
 
 logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail
 logger.xpack_security_audit_logfile.level = info