diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0f8a30f735f6..b18f82c01ca4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v5.0.0-beta1...master[Check the HEAD di *Affecting all Beats* +- A dynamic mapping rule is added to the default Elasticsearch template to treat strings as keywords by default. {pull}2688[2688] + *Metricbeat* *Packetbeat* diff --git a/filebeat/filebeat.template-es2x.json b/filebeat/filebeat.template-es2x.json index 91d779555eb3..f3f4932736ee 100644 --- a/filebeat/filebeat.template-es2x.json +++ b/filebeat/filebeat.template-es2x.json @@ -11,14 +11,13 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, - "match_mapping_type": "string", - "path_match": "fields.*" + "match_mapping_type": "string" } } ], diff --git a/filebeat/filebeat.template.json b/filebeat/filebeat.template.json index 3bf6c46280a3..f4204f549d84 100644 --- a/filebeat/filebeat.template.json +++ b/filebeat/filebeat.template.json @@ -9,13 +9,12 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "type": "keyword" }, - "match_mapping_type": "string", - "path_match": "fields.*" + "match_mapping_type": "string" } } ], diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py index 5dc4cd34af99..cc4b22245a9b 100644 --- a/libbeat/scripts/generate_template.py +++ b/libbeat/scripts/generate_template.py @@ -68,6 +68,30 @@ def fields_to_es_template(args, input, output, index, version): properties = {} dynamic_templates = [] + + # Make strings keywords by default + if args.es2x: + dynamic_templates.append({ + "strings_as_keyword": { + "mapping": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 1024 + }, + "match_mapping_type": "string", + } + }) + else: + dynamic_templates.append({ + "strings_as_keyword": { + "mapping": { + "type": "keyword", + "ignore_above": 1024 + }, + "match_mapping_type": "string", + } + }) + for section in docs["fields"]: prop, dynamic = fill_section_properties(args, section, defaults, "") @@ -200,9 +224,9 @@ def fill_field_properties(args, field, defaults, path): field.get("scaling_factor", 1000) elif field["type"] in ["dict", "list"]: - if field.get("dict-type") == "keyword": + if field.get("dict-type") == "text": # add a dynamic template to set all members of - # the dict as keywords + # the dict as text if len(path) > 0: name = path + "." + field["name"] else: @@ -213,8 +237,7 @@ def fill_field_properties(args, field, defaults, path): name: { "mapping": { "type": "string", - "index": "not_analyzed", - "ignore_above": 1024 + "index": "analyzed", }, "match_mapping_type": "string", "path_match": name + ".*" @@ -224,8 +247,7 @@ def fill_field_properties(args, field, defaults, path): dynamic_templates.append({ name: { "mapping": { - "type": "keyword", - "ignore_above": 1024 + "type": "text", }, "match_mapping_type": "string", "path_match": name + ".*" diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 7cc00cf10d48..901e9409e133 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -11,14 +11,13 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, - "match_mapping_type": "string", - "path_match": "fields.*" + "match_mapping_type": "string" } } ], diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 95e31280e43d..2c323bb53fb4 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -9,13 +9,12 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "type": "keyword" }, - "match_mapping_type": "string", - "path_match": "fields.*" + "match_mapping_type": "string" } } ], diff --git a/packetbeat/packetbeat.template-es2x.json b/packetbeat/packetbeat.template-es2x.json index 5676c3469950..89da89a66c89 100644 --- a/packetbeat/packetbeat.template-es2x.json +++ b/packetbeat/packetbeat.template-es2x.json @@ -11,58 +11,13 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - }, - { - "amqp.headers": { - "mapping": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "match_mapping_type": "string", - "path_match": "amqp.headers.*" - } - }, - { - "cassandra.response.supported": { - "mapping": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "match_mapping_type": "string", - "path_match": "cassandra.response.supported.*" - } - }, - { - "http.request.headers": { - "mapping": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "match_mapping_type": "string", - "path_match": "http.request.headers.*" - } - }, - { - "http.response.headers": { - "mapping": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "match_mapping_type": "string", - "path_match": "http.response.headers.*" + "match_mapping_type": "string" } } ], diff --git a/packetbeat/packetbeat.template.json b/packetbeat/packetbeat.template.json index 24269575f358..af72411fc851 100644 --- a/packetbeat/packetbeat.template.json +++ b/packetbeat/packetbeat.template.json @@ -9,53 +9,12 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "type": "keyword" }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - }, - { - "amqp.headers": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "amqp.headers.*" - } - }, - { - "cassandra.response.supported": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "cassandra.response.supported.*" - } - }, - { - "http.request.headers": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "http.request.headers.*" - } - }, - { - "http.response.headers": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "http.response.headers.*" + "match_mapping_type": "string" } } ], diff --git a/winlogbeat/winlogbeat.template-es2x.json b/winlogbeat/winlogbeat.template-es2x.json index 206f5a9caba9..43439b762dac 100644 --- a/winlogbeat/winlogbeat.template-es2x.json +++ b/winlogbeat/winlogbeat.template-es2x.json @@ -11,36 +11,13 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - }, - { - "event_data": { - "mapping": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "match_mapping_type": "string", - "path_match": "event_data.*" - } - }, - { - "user_data": { - "mapping": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "match_mapping_type": "string", - "path_match": "user_data.*" + "match_mapping_type": "string" } } ], diff --git a/winlogbeat/winlogbeat.template.json b/winlogbeat/winlogbeat.template.json index d7643fe550ed..9bdfd91b6b15 100644 --- a/winlogbeat/winlogbeat.template.json +++ b/winlogbeat/winlogbeat.template.json @@ -9,33 +9,12 @@ }, "dynamic_templates": [ { - "fields": { + "strings_as_keyword": { "mapping": { "ignore_above": 1024, "type": "keyword" }, - "match_mapping_type": "string", - "path_match": "fields.*" - } - }, - { - "event_data": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "event_data.*" - } - }, - { - "user_data": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string", - "path_match": "user_data.*" + "match_mapping_type": "string" } } ],