diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4c213504bad1..dad73445fcae 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* +- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] *Heartbeat* diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json index f582bdbdf603..ae88869d0c43 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json @@ -9,7 +9,7 @@ "INDEXNAME": "[a-zA-Z0-9_.-]*" }, "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" + "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{DATA:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" ] } }, diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log b/filebeat/module/elasticsearch/slowlog/test/test.log index 3d6d1ebae79c..52cbd3741ed3 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log +++ b/filebeat/module/elasticsearch/slowlog/test/test.log @@ -35,3 +35,4 @@ "name":"Rados-MacBook-Pro.local" } }] +[2019-11-14T21:18:40,269][TRACE][index.search.slowlog.query] [exp-data-elasticsearc-2] [exp_v3_1_current][3] took[516.4ms], took_millis[516], types[encounter], stats[], search_type[QUERY_THEN_FETCH], total_shards[10], source[{"size":1000,"query":{"constant_score":{"filter":{"bool":{"must":[{"bool":{"should":[{"nested":{"query":{"constant_score":{"filter":{"bool":{"must":[{"term":{"diagnosis.dx_rank":{"value":1,"boost":1.0}}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}},"boost":1.0}},"path":"diagnosis","ignore_unmapped":true,"score_mode":"avg","boost":1.0}},{"nested":{"query":{"constant_score":{"filter":{"bool":{"must":[{"term":{"procedure.px_rank":{"value":1,"boost":1.0}}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}},"boost":1.0}},"path":"procedure","ignore_unmapped":true,"score_mode":"avg","boost":1.0}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}}],"must_not":[{"exists":{"field":"primary_px_key","boost":1.0}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}},"boost":1.0}},"version":true,"sort":[{"_doc":{"order":"asc"}}]}] diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index 4b534272ea5d..55fb7a6c3b62 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -140,5 +140,28 @@ "log.offset": 4766, "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[", "service.type": "elasticsearch" + }, + { + "@timestamp": "2019-11-14T21:18:40.269-02:00", + "elasticsearch.index.name": "exp_v3_1_current", + "elasticsearch.node.name": "exp-data-elasticsearc-2", + "elasticsearch.shard.id": "3", + "elasticsearch.slowlog.logger": "index.search.slowlog.query", + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source_query": "{\"size\":1000,\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"diagnosis.dx_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"diagnosis\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}},{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"procedure.px_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"procedure\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}}],\"must_not\":[{\"exists\":{\"field\":\"primary_px_key\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"version\":true,\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}", + "elasticsearch.slowlog.stats": "", + "elasticsearch.slowlog.took": "516.4ms", + "elasticsearch.slowlog.total_shards": 10, + "elasticsearch.slowlog.types": "encounter", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 516000000, + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "TRACE", + "log.offset": 5638, + "message": "[2019-11-14T21:18:40,269][TRACE][index.search.slowlog.query] [exp-data-elasticsearc-2] [exp_v3_1_current][3] took[516.4ms], took_millis[516], types[encounter], stats[], search_type[QUERY_THEN_FETCH], total_shards[10], source[{\"size\":1000,\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"diagnosis.dx_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"diagnosis\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}},{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"procedure.px_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"procedure\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}}],\"must_not\":[{\"exists\":{\"field\":\"primary_px_key\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"version\":true,\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}]", + "service.type": "elasticsearch" } ] \ No newline at end of file