diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 18a9c184c9bc..0b259be433e8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,8 +50,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d instead of saving the parsed date. Now aligned with `event.created` semantics elsewhere. {pull}10139[10139] - Rename `mysql.error.thread_id` and `mysql.slowlog.id` to `mysql.thread_id`. {pull}10161[10161] - Remove `mysql.error.timestamp` and `mysql.slowlog.timestamp`. {pull}10161[10161] -- Migrate multiple fields to `event.duration`, from modules "iis", "kibana" and "traefik", - including `http.response.elapsed_time` (ECS). {pull}10188[10188] +- Migrate multiple fields to `event.duration`, from modules "apache", "elasticsearch", + "haproxy", "iis", "kibana", "mysql", "nginx", "postgresql" and "traefik", + including `http.response.elapsed_time` (ECS). {pull}10188[10188], {pull}10274[10274] - Rename multiple fields to `http.response.body.bytes`, from modules "apache", "iis", "kibana", "nginx" and "traefik", including `http.response.content_length` (ECS). {pull}10188[10188] diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index cad2b9ab0d1a..e52512d20467 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -304,6 +304,11 @@ to: url.original alias: true +- from: elasticsearch.slowlog.took_millis + to: event.duration + alias: false + scale: 1000000 + ## IIS module - from: iis.access.server_ip @@ -445,6 +450,16 @@ ## HAProxy module +- from: haproxy.total_waiting_time_ms + to: event.duration + alias: false + scale: 1000000 + +- from: haproxy.http.request.time_active_ms + to: event.duration + alias: false + scale: 1000000 + - from: haproxy.client.ip to: source.address alias: true @@ -612,6 +627,11 @@ to: mysql.thread_id alias: true +- from: mysql.slowlog.query_time.sec + to: event.duration + alias: false + scale: 1000000000 + ## NGINX module - from: nginx.access.user_name @@ -696,6 +716,11 @@ ## PostgreSQL module +- from: postgresql.log.duration + to: event.duration + alias: false + scale: 1000000 + - from: postgresql.log.timezone to: event.timezone alias: true diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index da553d2629f1..584cf418ed98 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4456,17 +4456,6 @@ Extra source information -- -*`elasticsearch.slowlog.took_millis`*:: -+ --- -type: keyword - -example: 42 - -Time took in milliseconds - --- - *`elasticsearch.slowlog.total_hits`*:: + -- @@ -4910,15 +4899,6 @@ Complete HTTP request line, including the method, request and HTTP version strin -- -*`haproxy.http.request.time_active_ms`*:: -+ --- -type: long - -Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent. - --- - *`haproxy.http.request.time_wait_without_data_ms`*:: + -- @@ -4943,15 +4923,6 @@ Total time in milliseconds spent waiting for a full HTTP request from the client TCP log format -*`haproxy.tcp.processing_time_ms`*:: -+ --- -type: long - -Total time in milliseconds elapsed between the accept and the last close - --- - *`haproxy.tcp.connection_waiting_time_ms`*:: + -- @@ -6356,16 +6327,6 @@ Contains fields from the MySQL slow logs. -*`mysql.slowlog.query_time.sec`*:: -+ --- -type: float - -The total time the query took, in seconds, as a floating point number. - - --- - *`mysql.slowlog.lock_time.sec`*:: + -- @@ -10351,17 +10312,6 @@ Name of database -- -*`postgresql.log.duration`*:: -+ --- -type: float - -example: 30.0 - -Duration of a query. - --- - *`postgresql.log.query`*:: + -- diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json index 7f40df72788f..5511a09e38c3 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test.log-expected.json @@ -151,4 +151,4 @@ "source.ip": "192.168.2.1", "user.name": "username" } -] +] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index e69336565621..f78593aff008 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzUml9v2zgSwN/7KQZ+2gKJzk7S3MYPB+x507TF9c82aRe7riHQ1FhiTZEKSdnxLfrdDyRlR5Yl2cqluZ5fEksk5zd/OBySPoY5roaAnGjDqEaiaPIMwDDDcQi9ree9ZwARaqpYZpgUQ/jHMwDY7gtvZZRzfAYwY8gjPXRNjkGQFHfF2I9ZZTiEWMk8K57UyNgerjykkBEG9t/Nm8oAvXckRZAzMAm61r1SS7wjaeY0Xfz+7o3+87T80pPNcbWUKtoRzESEd+2SX9smrnm9zBnjOEVijg1qc8xElpuu8lm0RzqL6mWT91fxr8vpp4+z0ecXf//lmt5OR/HycPE6ISpqFR+tje6a1lP0DxdI8oiZndbluGmMHaiJn/LQnKxQbb2pKnOToG8FMyVTWCaMJmASpgEXKAxIxWImiMFoCAq1OQKjiNCZVPYdsCycMW5Q9SpS7i1he1Xf1hukTO7lhrZhK//vCSp0zlB4m6PeJYafdt94ZQl8vLy+gV8+vF53fl5Wb9NvSTQopMgWGIEUTtp9M5oQIZA/PwIuKeGhnYnwk23jvruZCUzrHKMy5/Nmi92P091uCglP93qc5CZBYRgl9qHv1MDTs1OKGrbAMGIKqZFq9QAsyVG3Yn20LcDITQgiZIoJyjLCYYpcilg3mmwMvTmbEkHCXKPqHUHPZh8dkihlogeTWl6iFFnV0lp9pdhrRVHKv74LmIT4eME7pLnBqNnLlOfaoBqmUjAj1d9SwkR3wzIRMbrHtK99GyCUotYYwXRV8LYZdCbl8Ul/cBH0B0H/zNp068npzpPz7oYu5sJeS9txrKXXM7KDmUccibqmSnL+0ffubmSXCutyUaEfZ6TqgYyYpOgY1HRMWayIV9CoHNtSIIkihbo6/j7JWuaKYsCyBwjOFesoLVc8KLIrf4DAwq3hVEarjpITY7Kg6B/Y/gGVwqCoxtQhGJt801V7japaL7XJXMuLMFPoU/DjLf/rwWP68DEBrkZgaziNphAQHFhxZAnR9bOkKn0Pgf28dIJAZ0jZjFG7OlyNvIig0riOqcxV4x5onfsHAdpPuQq/GgGVnKNfDGpBS+7PfXSEGmkj2oxLUo3lA8FGFZKNQLsQSBUxEVuLWu43ZEFgwZTJCYeU0ISJFnBNVT4N9SqdSh4aMuUYGpbi99IDPpBcI1gRwARopFJEGihHIqwOeQaeBRyL3gtuFBPxE4AfwO1Q9nIvkcxDhTMdZkraNdzxf0fyG8usM7sBuJfoMEDhDBUKW0/cK9WMnhFFOEceKtSUiKeiLtk7JWpu6TlbIMjpV6RG2yKTI5As4+samGnQRmYZRs3KUE60DnPBJYmeShMvzcWLyG3x5iAOtD7NcsfZyFiXlA9k/OADA0YfPvkYL+IF1Uyq1ALfp8IaxOaUXVbALq0NRoa9hj5QEfupKCFzo1nk95VzVAJ5nQKlxLLS/wNKJqqQ0Eppt3lPgXkjDeGAnGQ2XivQRgKVtjY3nry0XrpNuTZEuVYzJphOgtoq4+siDVUuGqZgsyJ7FHBbDYvqSN58flvQ5Flpth0B0UD88DbKM8mEAZGnU1T1tCZRSCIdGmuX0GaZpuTxYPIroqYk3rJmIRWcVJfbCjfUJY1NINsU6FaXNfNjm9giGCnn1sUequBs5TIkrt8B1Zdu+6w1Ai7j2C+9cYPIBEk1Mz64kH2FJAPCuSwWGyKitV/YvzvXsrZPOJ82JnUmDMY7534HYMJm8lrlnRwb+HPG5XRl2ioUuzJ9N6RPNo04omaYzaaZR2GM1bObBzvuPY8gRoFF4SwpzTMi6OrH96BznpxZg5Q1+AHc2WjT/d5dyVzEj+nfP+yA/+ceXlV1+AF83GLXerqN3VAttoRuHwZeu9c2e7vzid1LlmoM7PppUxvLNJOielK0Le5fMr5vt31ueH/CKAMMaJAGb9GQX4khI4XEoLuksriMVo7xmxau2pObKpFfuuoG3I3+tnMaFzRtc6XnXXg1aj5arT9IrZuF9bNlk7PF7gZlm6UqqY1izcHljoKbamIpn0LgRr8FqgRJFGq8bTX5Nd7mdmtdVJONlj89O7u4uDipNX8jxX1pGK4PgoK0/eJge0N9NTqyf1LGOSuKtUbCwXm/f2DJuLHS1M590g3QJUJX1lojF3dHpSJ4SXQxMEYd6H8+iH6Ts7hcchk3Jy3/3t8GaL+5uKz+/mAHojc+6Q9+Pu6fH59c3Az6w/75cHB2dHF6Ohm/fvfyPUzG/qLcDxEUEMFtjmo1gfEi/Pwm+fp5AuMUjWLUXcefB6dB/9iOG/TPg5Pzybg/cdX4+Cx4kerJkfsSeiONz9x3u2dJmNHjwcXZ6Qv7aJWhHk+O7ObI+H8cgrsgGf/26fLjH+HNq8t34cvLm9GrzRjuslyPB7a9u5wY//Wl52i/9IZ/femlxNAkJJz7r1MptfnSGw6C/rdv3yZH/02qt8V+ZSXbyfMxqp0fNJS9UWvsGZpt7+3P7tbALSRuyjGz2SIVF1xuq+yM1cR32u+nug7F4J2p47BebAOx75uEdVPZxUmLqGtDDHOzoYu8Br1Ksdgm0v+mx7ZqklkN5I46uxAPncvaOLhctvu1wyTpYCW8M4qEHrIF79I2K3QBJmZSpWT33vhBfiolmn3Twc2F1ty9ITg76TgZ19mtjcHviJl5VKE+He4Va33PMPI/NWoCOOkGoGRuWKVKqP4Yw7VocrPuD179efLbP+cXX5dnsYnJSyO6TQ8WNUt/HT1K1tmTAW5apn4kaZus/wQAAP//KBSxQg==" + return "eJzUml9v2zgSwN/7KQZ+2gKJzk7S3MYPB+x507TF9c82aRe7riHQ5FhiTZEqSdnxLfrdD6RkW5Yl2cq1uZ5fEksk5zd/OBySPoU5roaAghjLqUGiafwEwHIrcAi9nee9JwAMDdU8tVzJIfzjCQDs9oXXimUCnwDMOApmhr7JKUiS4L4Y97GrFIcQaZWlxZMaGbvDlYeUimHg/t28qQzQe0MSBDUDG6Nv3Su1xHuSpF7Txe9vXpk/z8svc7I5rpZKsz3BXDK8b5f80jXxzetlzrjAKRJ7atHYUy7TzHaVz9kB6ZzVyyZvb6Jfl9MP72ejj8/+/sst/TIdRcvjxZuYaNYqnq2N7pvWU/SPF0gyxu1e63LcNMYO1MRPeWhBVqh33lSVuYsxbwUzrRJYxpzGYGNuABcoLSjNIy6JRTYEjcaegNVEmlRp9w54Gs64sKh7FSlbS7he1bf1BimT53JD17CV//cYNXpnaPySodknhp/23+TKEnh/fXsHv7x7ue78tKzept+SGNBIkS+QgZJe2rYZjYmUKJ6egFCUiNDNRPjJtfHf/cwEbkyGrMz5tNli23G6200jEclBj5PMxigtp8Q9zDs18PTclKKWLzBkXCO1Sq8egKUEmlas964FWLUJQYRUc0l5SgRMUSgZmUaTjaE351MiSZgZ1L0T6LnsY0LCEi57MKnlJVqTVS2t01fJg1aUpfybdwEbkzxe8B5pZpE1e5mKzFjUw0RJbpX+W0K47G5YLhmnB0z7Mm8DhFI0BhlMVwVvm0FnSp2e9QdXQX8Q9C+cTXeenO89uexu6GIuHLS0G8dZej0jO5h5JJDoW6qVEO/z3t2N7FNhXS4q9BOcVD2QEhsXHYOajgmPNMkVtDrDthRIGNNoquMfkmxUpikGPH2A4EzzjtIyLYIiu4oHCCzcGk4VW3WUHFubBkX/wPUPqJIWZTWmjsHY5Juu2hvU1XqpTeZaHsNUY56Cv93yvx48og8fE+BmBK6GM2gLAcGRFUcaE1M/S6rSDxC4z3MvCEyKlM84davDzSgXEVQa1zGVuWrcA61z/yhA9ylX4TcjoEoIzBeDWtCS+7M8OkKDtBFtJhSpxvKRYKMKyUagWwiUZlxGzqKO+xVZEFhwbTMiICE05rIF3FCdTUOzSqZKhJZMBYaWJ/i99IB3JDMITgRwCQapkswAFUik0yFLIWcBz2IOglvNZfQI4Edwe5SD3Esk81DjzISpVm4N9/zfkfzOMZvUbQC2Ej0GaJyhRunqia1Szegp0UQIFKFGQ4l8LOqSvROi545e8AWCmn5Gao0rMgUCSVOxroG5AWNVmiJrVoYKYkyYSaEIeyxNcmk+XmTmijcPcaT1aZp5zkbGuqR8JOO7PDBg9O5DHuNFvKCeKZ044G0qrEFsTtllBdzS2mBkOGjoIxVxn4oSKrOGs3xfOUctUdQpUEosK/M/oOSyCgmtlG6b9xiYd8oSAShI6uK1Am0VUOVqc5uTl9ZLvyk3lmjfasYlN3FQW2V8XiShzmTDFGxW5IACfqvhUD3Jq4+vC5osLc22EyAGSD68i/JUcWlBZskUdT2tjTUSZkLr7BK6LNOUPB5MfkP0lEQ71iykgpfqc1vhhrqksQlklwL96rJm/tYmdghWqblzcQ5VcLZyWRLV74DqS7dD1hqBUFGUL71Rg8gYSTUzPriQfYEkBSKEKhYbItnaL/zfnWtZ1yecTxuTOpcWo71zvyMwYTN5nfJejgv8ORdqurJtFYpbmb4b0geXRjxRM8xm0yxYGGH17ObBjnsrGEQosSicFaVZSiRd/fge9M5TM2eQsgY/gDsbbXrYuyuVyehb+vcPN+D/uYdXVR1+AB+32LWebmM31IsdobuHgbf+tcve/nxi/5KlGgP7ftrUxipJlayeFO2K+5eKtu12zw23J4wqwIAGSfAaLfmVWDLSSCz6SyqHy2nlGL9p4ao9uakS5UtX3YD70d92TuODpm2u9HIX3oyaj1brD1LrZmH9bNnkbLm/QdllqUpqo1hzCLWn4KaaWKrHELjRb4E6RsJCg19aTX6LXzK3tS6qyUbLn19cXF1dndWav5FiWxqG64OgIGm/ONjdUN+MTtyfhAvBi2KtkXBw2e8fWTJurDR1c590A/SJ0Je1zsjF3VGpCF4SUwyMrAP9z0fRb3KWUEuhouaklb/PbwNMvrm4rv7+YA+iNz7rD34+7V+enl3dDfrD/uVwcHFydX4+Gb988/wtTMb5RXk+RFBABF8y1KsJjBfhx1fx548TGCdoNaf+Ov4yOA/6p27coH8ZnF1Oxv2Jr8bHF8GzxExO/JcwN9L4wn93e5aYWzMeXF2cP3OPVima8eTEbY5s/o9H8Bck498+XL//I7x7cf0mfH59N3qxGcNflpvxwLX3lxPjvz71PO2n3vCvT72EWBqHRIj861QpYz/1hoOg//Xr18nJf5PqXbFfWcn28nyEeu8HDWVv1Bp7hnbXe4ezuzNwC4mfctxutkjFBZffKntjNfGd9/uJqUOxeG/rOJwX20Dc+yZh3VT2cdIi6tYSy/1s6CKvQa9SLLaJzH/T41o1yawGckedfYiH3mVtHEIt2/3aYZJ0sBLeW03CHLIF79o1K3QBLmdKJ2T/3vhBftomlrYgzDej3DZFxsVZx8m3zUQHxTqzc2T5r3yaAM66AWiVWV5ZoKu/g/Atmixs+oMXf5799s/51eflRWQj8tzKbpHJWbP0l+ybTPgDk++uZdYxRdtk/ScAAP//bQSM1A==" } diff --git a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml index 36c5119e9854..77a3f9e0ffd0 100644 --- a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml +++ b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml @@ -31,10 +31,6 @@ description: "Extra source information" example: "" type: text - - name: took_millis - description: "Time took in milliseconds" - example: 42 - type: keyword - name: total_hits description: "Total hits" example: 42 diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json index 87696f84c6d2..fbfec49e08c5 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json @@ -15,7 +15,7 @@ "INDEXNAME": "[a-zA-Z0-9_.-]*" }, "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.took_millis:int}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" + "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:temp.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" ] } }, @@ -35,12 +35,19 @@ "field": "elasticsearch.slowlog.timestamp" } }, + { "script": { "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.took_millis * params.scale)", + "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", "params": { "scale": 1000000 }, - "if": "ctx.elasticsearch.slowlog.containsKey('took_millis')" + "if": "ctx.temp?.duration != null" + } + }, + { + "remove": { + "field": "temp.duration", + "ignore_missing": true } } ], diff --git a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json index 6267be7a7f7f..32dda026a239 100644 --- a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json @@ -10,7 +10,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"module\":\"system\",\"rtt\":9610,\"name\":\"network\"},\"system\":{\"network\":{\"name\":\"bridg\",\"in\":{\"packets\":0,\"errors\":0,\"dropped\":0,\"bytes\":0},\"out\":{\"errors\":0,\"dropped\":0,\"packets\":1,\"bytes\":342}}},\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "221micros", - "elasticsearch.slowlog.took_millis": 0, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -33,7 +32,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"rtt\":9616,\"name\":\"network\",\"module\":\"system\"},\"system\":{\"network\":{\"name\":\"utun0\",\"in\":{\"dropped\":0,\"bytes\":0,\"packets\":0,\"errors\":0},\"out\":{\"packets\":2,\"bytes\":200,\"errors\":0,\"dropped\":0}}},\"beat\":{\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "388.6micros", - "elasticsearch.slowlog.took_millis": 0, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -56,7 +54,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"rtt\":9640,\"name\":\"network\",\"module\":\"system\"},\"system\":{\"network\":{\"name\":\"utun1\",\"in\":{\"packets\":200,\"errors\":0,\"dropped\":0,\"bytes\":44296},\"out\":{\"errors\":0,\"dropped\":0,\"packets\":208,\"bytes\":59626}}},\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "287.1micros", - "elasticsearch.slowlog.took_millis": 0, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -79,7 +76,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }", "elasticsearch.slowlog.took": "1.7ms", - "elasticsearch.slowlog.took_millis": 1, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, @@ -105,7 +101,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:42.117Z\",\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"},\"metricset\":{\"module\":\"system\",\"rtt\":39463,\"name\":\"process\"},\"system\":{\"process\":{\"state\":\"running\",\"pid\":6274,\"name\":\"iTerm2\",\"cmdline\":\"/Applications/iTerm.app/Contents/MacOS/iTerm2\",\"ppid\":1,\"pgid\":6274,\"username\":\"rado\",\"memory\":{\"size\":6263349248,\"rss\":{\"bytes\":226975744,\"pct\":0.0132},\"share\":0},\"cpu\":{\"total\":{\"value\":921790,\"pct\":0.1368,\"norm\":{\"pct\":0.0342}},\"start_time\":\"2018-07-02T10:40:29.756Z\"}}}}", "elasticsearch.slowlog.took": "560.6micros", - "elasticsearch.slowlog.took_millis": 0, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, @@ -128,7 +123,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:42.117Z\",\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":39476},\"system\":{\"process\":{\"username\":\"rado\",\"state\":\"running\",\"cmdline\":\"com.docker.hyperkit -A -u -F vms/0/hyperkit.pid -c 2 -m 6144M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-vpnkit,path=s50,uuid=18fcb277-636a-4fd7-99d2-9bd2dd50a58c -U b1496a26-aed9-4ee1-818d-a3683593b754 -s 2:0,ahci-hd,file:///Users/rado/Library/Containers/com.docker.docker/Data/vms/0/Docker.qcow2?sync=os\\u0026buffered=1,format=qcow,qcow-config=discard=true;compact_after_unmaps=262144;keep_erased=262144;runtime_asserts=false -s 3,virtio-sock,guest_cid=3,path=vms/0,guest_forwards=2376;1525 -s 4,ahci-cd,/Applications/Docker.app/Contents/Resources/linuxkit/docker-for-mac.iso -s 5,ahci-cd,vms/0/config.iso -s 6,virtio-rnd -s 7,virtio-9p,path=s51,tag=port -l com1,autopty=vms/0/tty,log=vms/0/console-ring -f bootrom,/Applications/Docker.app/Contents/Resources/uefi/UEFI.fd,,\",\"ppid\":559,\"pgid\":555,\"name\":\"com.docker.hype\",\"cpu\":{\"total\":{\"pct\":0.1181,\"norm\":{\"pct\":0.0295},\"value\":8.7575e+06},\"start_time\":\"2018-07-01T22:13:07.748Z\"},\"pid\":567,\"memory\":{\"share\":0,\"size\":11128897536,\"rss\":{\"pct\":0.0205,\"bytes\":352854016}}}}}", "elasticsearch.slowlog.took": "469.9micros", - "elasticsearch.slowlog.took_millis": 0, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index 460c6b5d07b1..4cda0f2d5140 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -10,7 +10,6 @@ "elasticsearch.slowlog.source_query": "{\"query\":{\"match_all\":{\"boost\":1.0}}}", "elasticsearch.slowlog.stats": "", "elasticsearch.slowlog.took": "4.5ms", - "elasticsearch.slowlog.took_millis": 4, "elasticsearch.slowlog.total_hits": 19435, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -35,7 +34,6 @@ "elasticsearch.slowlog.source_query": "{\"query\":{\"match_all\":{\"boost\":1.0}}}", "elasticsearch.slowlog.stats": "", "elasticsearch.slowlog.took": "10.8ms", - "elasticsearch.slowlog.took_millis": 10, "elasticsearch.slowlog.total_hits": 19435, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -60,7 +58,6 @@ "elasticsearch.slowlog.source_query": "{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}", "elasticsearch.slowlog.stats": "", "elasticsearch.slowlog.took": "124.3ms", - "elasticsearch.slowlog.took_millis": 124, "elasticsearch.slowlog.total_hits": 0, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -85,7 +82,6 @@ "elasticsearch.slowlog.source_query": "{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}", "elasticsearch.slowlog.stats": "", "elasticsearch.slowlog.took": "7.2ms", - "elasticsearch.slowlog.took_millis": 7, "elasticsearch.slowlog.total_hits": 0, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", @@ -110,7 +106,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T13:47:50.747Z\",\"system\":{\"process\":{\"ppid\":34526,\"state\":\"running\",\"cpu\":{\"total\":{\"value\":734879,\"pct\":0.0173,\"norm\":{\"pct\":0.0043}},\"start_time\":\"2018-07-04T06:56:34.863Z\"},\"pgid\":34526,\"cmdline\":\"/Applications/Firefox.app/Contents/MacOS/plugin-container.app/Contents/MacOS/plugin-container -childID 1 -isForBrowser -prefsLen 22119 -schedulerPrefs 0001,2 -greomni /Applications/Firefox.app/Contents/Resources/omni.ja -appomni /Applications/Firefox.app/Contents/Resources/browser/omni.ja -appdir /Applications/Firefox.app/Contents/Resources/browser -profile /Users/rado/Library/Application Support/Firefox/Profiles/pt6eoq1j.default-1484133908360 34526 gecko-crash-server-pipe.34526 org.mozilla.machname.231926932 tab\",\"name\":\"plugin-containe\",\"memory\":{\"size\":7489249280,\"rss\":{\"bytes\":567619584,\"pct\":0.033},\"share\":0},\"pid\":34528,\"username\":\"rado\"}},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":43856},\"beat\":{\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "1.4ms", - "elasticsearch.slowlog.took_millis": 1, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, @@ -133,7 +128,6 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.source_query": "\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }", "elasticsearch.slowlog.took": "1.7ms", - "elasticsearch.slowlog.took_millis": 1, "elasticsearch.slowlog.type": "doc", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, diff --git a/filebeat/module/haproxy/fields.go b/filebeat/module/haproxy/fields.go index 1797a39e9d99..4b36485924ff 100644 --- a/filebeat/module/haproxy/fields.go +++ b/filebeat/module/haproxy/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzMWdtu2zwSvs9TDHKzLZD6AQJsgUW2uy3w/20ucm/Q5FgalCJVcpTUb/+DB8k62lbiovWdRc7MxzkP+QG+4+EeSlE7+/NwA8DEGu/hNn+5vQFQ6KWjmsmae/h4AwDtfvjbqkbjDcCeUCt/Hxc/gBEV9pmGHx9qvIfC2abOX2b4Hhnlv0due2cNo1Hb8LdbHXH5KioEuwcusSOAd9aBJs9o0L2Hl5JkCQ4l0jMqEEZB7axE71FFOmmNQRn4baYodkJ+XwMi75/F8CI8eNQoOUi2UAkjChxhCAvhi0f3jG4GUVq4GJAWnjNNYJ2QjEQmYIZnpLFlobcvgphMsWWqcFv5JblPYTOETUAGKtKaPEprlAdfo2HIfMJqgPAsHNnGw48GG+xzTb6jrSmmiI64I6yrYdpbN2MK9Cx2mnzZ2mVPRuis0AsR7w6MfutQqNMgTVPt0AXDRQpgJ4yviLO7RHCaIuYSkwa1LYA8YNq1uRBQVFlU+q+x5CocbYRlvf8OO94BGakbFcgdsqPLj5Cj8QJlHq3r8EeDnn2bFtBhLyXtcG9dyArkwRpsNZxDOAq6FFyr2l+PLkv6l4dC253QK3HSigybsmowlVDKoffjDH82p6Nz1m0r9F4UiyI/hU2QN4U4K8LpD/D5P4+xEpIBKXxEFflNz8r4k2ccxjZO4vzmJQuV2IlN5J0u7FxUoavIiJghPQtePOKDNYpSgEQH874tBmSOKab9jkaFDDMRV1m17jiBALgUPKzawaFqdCIG8bunh0ewDj4/PT2+P1UDFvP+gzUsyPguNUnbGPZBcz1qEJLpuXPj7OdTW/bbGIBh+zMGl1gOls6EnLRGNs6FLNbHZgegphYJlkqOuRlJmws2mOmtrgeyM+MVUeakcgWQpTBKp/jtN2lXxDppCF4L1TNp3bqlHeT+K8LNVe4U3q99pG0lzXSAP2t0hEa2SiV/RBVhukOIY7Yt9bivPY+3C/bY8yzF+Rezt66K2Q7Ezjbc75OUjSBKbKvaKwK7hUH1zRSx0CSGWqwFl22W3+QKNdhQUeFEws5uUJZbSbV1vF7WhGosaKLYnFjGlXdeWpLVJsgRyXlRpFZKGFKcFaDQcy56S57y3+MWoKPXvMElVhqqh/G8tabSVjlgX9aI8KwuC7QDknm1zE7z6dfV3v+j/fLYVzYUgkt0IWuI0E7mBJfCdUN1UvYwO3wz+gC1wzChAqXGJzH+FIZbkh6FkyXUuinIhD5CPAvSYqcxTPsDXo0f5spTdVxaw2TQ8DhATml/FJQF2s1JPstm7wNpDLvDlrzdymG3tRrKSU6XgNFWjsNmJYgFDpcId1iErvZt9lhmcpExiA9vdYgFFis08HZXOM1oCUp31ce8mAseNYbBSCjV/746zc5kQIe+tsYPwQ5k/y8yAYda9G5NwhQxTzwP5UxeEDU3DtVWWvudTjb6H0eLAN/iitBwG5j9+1noBm8BQ1QCGUUyzT/deJQbmVKo1GQlme280p5pM6e1GcQlCoXuZNs3hfwXeQ5tYCbuuI1BgGqw1XhK18dB9TYTHTcnbrcQJ9QqpvbhJDHfIg4Gy/lOnAU3/jXREZx606l0mc9ScMx67LjtXOOwU9o/3l/zqFKKcCZunEEFovXaF+ISiH17tN/sthHDhV6b9v4yp3XiZZuFbDWZk0Z6sFWtkYdeAoGqf48ZUFXIpVV33R5hVCJ6RhdHNc+OTPFa0PH6Nk2qwwv4CeQnqnCgdIeVoOgb3f1Lfj26y9d55CMBL1/7ohZ1upLkF8yT8Z6c53h93tqwlfgSPbL3CNQ9jrS7u9R0fA6Z00scUs/rJb5NBIe3DW+VYHFORWvvt49vOh5NCLJ9o/Ww1N2BsZzax0AZULz9TFc8xhBzstPe2apf+d4NjrCz6vAexJ7Rje3dN/CaU3YnlIsdzdPDY3xpSWPMWwbGNNzOP6XNwlyj4LmAEFJizUOHl9r6uQFz9LL2R6D8JwAA///iRBtm" + return "eJzMWc2O2zYQvu9TDPbSBEj8AAs0QLFNmwBtsoe9G2NyLBGhSIUc7cZvX/BHsn5ta9dB6pslzszH+fk4Q72Hb3S4gxJrZ38cbgBYsaY7uM1Pbm8AJHnhVM3Kmjv4cAMA7Xr418pG0w3AXpGW/i6+fA8GK+orDT8+1HQHhbNNnZ/M6D0qyn+P2vbOGiYjt+Fv93ak5QtWBHYPXFInAG+sA608kyH3Fp5LJUpwJEg9kQQ0EmpnBXlPMsoJawyJoG8zRbFD8W0NiLx+FsMzevCkSXCwbKFCgwWNMIQX4Ykn90RuBlF6cTEgjZ6zTFCdkIxMJmCGZ6yxZdTbZ1SsTLFlVdG28kt2H8NiCItAGaiU1sqTsEZ68DUZhqwnvA0QntAp23j43lBDfa0pd7Q1xRTREXeEdTVMe+tmQkGecaeVL9u47JVBnR16IeLdgclvHaE8DdI01Y5cCFyUAHZofKU4p0sEp1XEXFLyoLYFKA+UVm0uBBRdFp3+cyK5CkdbYdnvvyKO70AZoRsZxB2xU5dvIVfjBc48RtfR94Y8+5YWyFGPkna0ty6wgvJgDbUeziUcDV0KrnXtz0eXLf3modB2h3olTrWCYROrhlChlI68HzP8WU4n56zbVuQ9FosmP4ZFkBeFOivC7g/w6Y+HeBIqAwJ9RBX1TffK9INnEsY2TtD84qUIldSZTeKdL+xcVZGrlMHIkJ6RF7d4b41UqUBignnfHgbKHCmmfU5GBoaZmKusXLedIABcIg9P7ZBQNTmMRfzm8f4BrINPj48Pb0+dAYu8f28NozK+oyZhG8M+eK4nDShYPXVpnPN8Gst+GwMwbH/G4JLKwaszJSesEY1zgcX62OwA1DQiIVIpMTcja3PFBjO91fVAdmG8IspMKlcAWaKROtVvv0m7ItZJQ/BSqJ6V1m1a2gH3XxFuPuVO4f3SR9qepFkO6EdNTpERrVOVP6KKMN0h1DHbVnrc157H2xV77HmW6vyz2VtXRbYD3NmG+32StBFESe2p9oLCbmGo+maKGLXCoRdr5LJl+U0+oQYLKlU4TNjZDY7l1lJtHa+3NZEaG5o4NhPL+OSdt5ZstQQ5EjlvSsmVFoYSZw1I8pwPvaVM+fO4BNQxa16REisD1cN4PlpTa6sSsG9rJHjWlwXZgci8W2an+fTrzt6/yX5+6DsbCuSSXGANDO1kJrhUrhtVJ2cP2eGr0QeoHYUJFVRqfJLij2G4VcITOlFCrZtCmdBH4BMqjTtNYdof6Gr8kCtPnePCGlaGDI8L5JT3R0VZkN2c1LMc9j6QxrA7bJW3WzHstlZDOanpEjDainHZrASxoOES446K0NW+Lh7LSi4KhuLDaxNiQcUKD7w+FU4rWoLSXfUxL3LBg6YwGKGU/eeraXaGAR352ho/BDuw/VdUAo409m5NwhQxLzwP5QwvYM2NI7kV1n5TJxv9D6OXAF/jG9RwG5T9/oS6oVugUJWgjFQizT/deJQbmRJlarKSzXZeafe0mfPaDOKSUJI72fZNIf+jPIc2MAt32sYgQDbUejzR9XFQvc1Cx8VJ2y3ECbWK1D6cJOZbxMFgOd+JM3LjX1IdIak3nUuX9SwVx2zGjtvONQk7lf3f52seVUoMe+LGGZKAbdY+Ky5BsW+39ovTNmK4MGvT2p+WtA6ft9nIVitzMkj3tqo18TBLIEj17zEDqoq4tPJdtwaNTEJP5OKo5tkpU7wUdLy+jXfwIbC24a1ExuFd/AT96nvc47cLH2Z2hH2j9ZDS34GxnNqkIBlQnBgwL9zTFbcxxJxisXe26jP8m8EWdlYe3gLumVy+rXae42eBOPS3d51rdtntUCye3I/3D/GLQmrXXzEYjb7RzH86moW7xtGksU530vxM+WoEhaA6ZXr39Uto6+nmvwAAAP//o2OPYg==" } diff --git a/filebeat/module/haproxy/log/_meta/fields.yml b/filebeat/module/haproxy/log/_meta/fields.yml index 6a7c8e8f4774..9176a8ae7282 100644 --- a/filebeat/module/haproxy/log/_meta/fields.yml +++ b/filebeat/module/haproxy/log/_meta/fields.yml @@ -38,10 +38,6 @@ description: Complete HTTP request line, including the method, request and HTTP version string. type: text - - name: time_active_ms - description: Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent. - type: long - - name: time_wait_without_data_ms description: Total time in milliseconds spent waiting for the server to send a full HTTP response, not counting data. type: long @@ -54,9 +50,6 @@ description: TCP log format type: group fields: - - name: processing_time_ms - type: long - description: Total time in milliseconds elapsed between the accept and the last close - name: connection_waiting_time_ms type: long description: Total time in milliseconds elapsed between the accept and the last close diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index 08f4315c88e5..38ed994296d9 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -7,11 +7,11 @@ "patterns": [ "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", - "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" + "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" ], "ignore_missing": false, "pattern_definitions": { @@ -73,7 +73,15 @@ { "script": { "lang": "painless", - "source": "if (ctx.haproxy.http?.request?.time_active_ms != null) { ctx.event.duration = ctx.haproxy.http.request.time_active_ms * 1000000 } else { if (ctx.haproxy.tcp?.processing_time_ms != null) { ctx.event.duration = ctx.haproxy.tcp.processing_time_ms * 1000000 } }" + "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.temp?.duration != null" + } + }, + { + "remove": { + "field": "temp.duration", + "ignore_missing": true } }, diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index b269eb0e09ff..6ebe02ae203e 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -20,7 +20,6 @@ "docs.example.internal" ], "haproxy.http.request.raw_request_line": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1", - "haproxy.http.request.time_active_ms": 2, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": 0, "haproxy.http.response.captured_cookie": "-", diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json index 5f80d850cfc1..970b30f90bfe 100644 --- a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json @@ -17,7 +17,6 @@ "haproxy.frontend_name": "http-webservices", "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.raw_request_line": "GET / HTTP/1.1", - "haproxy.http.request.time_active_ms": 0, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": -1, "haproxy.http.response.captured_cookie": "-", @@ -54,7 +53,6 @@ "haproxy.frontend_name": "http-webservices", "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", - "haproxy.http.request.time_active_ms": 0, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": -1, "haproxy.http.response.captured_cookie": "-", @@ -94,7 +92,6 @@ "localhost:8888" ], "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", - "haproxy.http.request.time_active_ms": 0, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": -1, "haproxy.http.response.captured_cookie": "-", diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index d5ffb2e3f13e..439e4dc21683 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -18,7 +18,6 @@ "haproxy.server_name": "", "haproxy.server_queue": 0, "haproxy.source": "127.0.0.1", - "haproxy.tcp.processing_time_ms": 1, "haproxy.termination_state": "SC", "haproxy.total_waiting_time_ms": -1, "input.type": "log", diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go index ebdc2cbff030..53015978245c 100644 --- a/filebeat/module/mysql/fields.go +++ b/filebeat/module/mysql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEmN+P2zgOx9/nryB6L3fANNN7OBwwDwdcWyy2QLvAogX20WUs2tZGFl2Rnkz61y8kOz8a24kzM9n6oUAnFr8f0hJJ8TWsaHMP9Ua+uRsAteroHl592nz+/eOrGwBDkgfbqGV/D/+7AQD4xKZ1BAUHaDCI9SVoRZCWgOMSCutIFjcAUnHQLGdf2PIeNLR0A1BYckbuk6nX4LGmvXx8dNPQPZSB26b/ywhDfH5JhqAIXE8BxOdQ71BTq0BoMmt2v2y1Hfvy4I8T8vH5UhHk7D3l8Vfg0FuFD+9TfCLWt5bCZjGQpxA4DKQP3T6j/Y69ovXSO3gch2Q/RkMWB8uOg3EuIHsydBbl6JcGteo/3mJqdW3LgB1/vwGGyo4eyF2o6rhcjK2bo1eTCJZ0qZ+jq6b0tlrieO24vOKXjgoXfei0IzO1NS2E8tEoFI5Rj345QQj9WVBWdBAt77c+KPPqFqwHoZy9kVtAAewkYvJo2HoF39ZLCovx/cH56iq8WHPrFbg4Zl6jVTK7MxwBQBmWBPiA1uHS0SJaGFh9QNcSWHmWv4HXkgn5Y49G09NMVzu96Gq0DoG0DZ4MLDdjWWpAQ49YW0/jueFFiCRHfwEQFgXlek2gmo0t7Cyi5Ubpsi9WcKhR+5WXo4r9Tmnb7nZsIGmdjuOlN54gEhPLCa9/QqWQvKIaR2VXtFlzODY6x81kE9YVhS4F7P2GNQrQI+WtkhkPQt6GQF6zVii8HNe7zipgqxV5tTnGdBQ1buO/JiYjQ0ohHkrAPCcRaIJ9sI5KkgW8Qw/GFgWFfbnoclPMadHQRKpF0YxC8PxyznxEUdj3JELkx8VX1rmJE/20KLLZnZJAKOzBFj9keek19zwnKmaOeUVZZccP+ZLZEfrLCP+oSCs6aBQhiSSyyk4cZ62bTGMJenkOBKW64YCxbkeJRLLdcYGE3QOdy4Y7vox9ZqysrhkvT2Qi3Y/YAhz3v6zOEB7nqZnlAwZgv+1qx4AkD4T7ZsLKSDo+5IrUz4J7ClsfrSPGc583VqHZkLMr3oD/87bYzYvsOHW6E3K4wtndWgZu1Nb2e6phu3NzmuZ6J+Q0FaA3YPVpp6cJloPVTfatpfYqSWirAElhTx0/9NaxicC2zmWxlbwGVjTep8Uokbj6CG5vCud6psT3J9vr8UXjp8ngn57BekOPJLCOXc8uuHGx/GscvaZQUtagyBUyUzIOnfFdxzU1Q4Ef7oZlFl3LAiplEeQFu8NNkxLP9oYNUQOcra3extMT98CS4KuQiGX/ddtd7F8bmLQC2DQuXisair1QWnkLHOBr8jIZsdq/Jemt2e6PaT7D/89jfmPfR3IB/37zBmpCL6AVKrCneO1FqFpvApnEbQebBYB3jgtgIFhSvBc7LsvJTuwfUKk2cn93t16vFw2FnD0ucq7vDOd3/f9fC4UHCnf/Wfz3zlgsPYvaXO5SgOhRyRsyi0prN36f8Z7NcjR8x9OaGcE7ntoEcqg2dk8MH7zn92+BfGn98X4dm9wcUmp4HN6d4MyHnsEbny8BvWA3zPzwfhLBchYyboYf9kQmmEmwzwgNlqlrN8ANdRdDWZxGGmstzkKd6ktmQr+NSztW04Y047kcfo1WR+ZbZ/ln8P2ajrEvY15R5lXXy6MBdG5bJgwqdldFUQ5Y0jRuoDxLI7m/A3l0JhfVT8QzdQ0/h04ZyGt/Q+mPedfExI6rGyHuyhv7fDhBjI/1W8vtie8QN5nE9lGtz4el5vlu/r9pAj/aOjXZ+XZUmmT7mUPvcefQWKkeTbSTs5Lp0VGaWMTVTxgaVSzjRXFaTrgNOS0M1zho1eZI2uNiMVNwsG5K7OavAAAA//8Czml+" + return "eJzEmM+P27gOx+/zVxB9l/eAqdN3eHhADgtsWyy2QLvAogX26DIWbWsji64oTyb96xeSnR+N5cSZTlofCnRi8fshLVIUX8KatktotvLF3AF47Q0t4cWH7cc/37+4A1AkhdOt12yX8MsdAMAHVp0hKNlBi060rcDXBHEJGK6g1IYkuwOQmp3PC7alrpbgXUd3AKUmo2QZTb0Eiw0d5MPjty0toXLctcNfEgzh+S0agtJxMwUQnmO9Y01fO0KVa7X/Zadt2FZHf5yQD8+nmqBga6kIvwK7wSq8exvjE7C+dOS22UienGM3kj52+4L2G7YetZXBwdM4RPshGpIdLTsNxqWAHMjQaJSTX1r09fDxsqnVja4c9vzDBhgrG3ogc6Wq4SpLrZuj15AIVnStn8lVU3o7LTG8MVzd8EsHhas+tOFinXvdUCZUJINQGkZ/8ssZQBhSARvurAcuIVg/7H7YoPak9ikRAMAzrAjwAbXBlaEsWBhZfUDTEWgBbUGoYKvkHlAAe8RQe1rW1oPtmhW5LOmv443kQvbUo2S2z3S11wuuBuvgyHfOkoLVNpX0Ixp6xEZbSqfasxBJgfYKICxLKvwtgRpWutSziFZbT9d9sZJdg35YeT2q6K8Ut+1+xzqSzvg0XnzjCSIhT894/RMKrxQ1NZiUXdN2w+7U6Bw3o03Y1OT6EnDwGzYoQI9UdJ5UOghF5xxZn3dC7vm43vRWATtfk/W6wFCOgsZ9+FeFYqTIkwtJCVgUJAKt0w/aUEWSwRu0oHRZkjtU3742hZoWDKXdMSg+J+csP58z71E8HI54IbJp8bU2ZiKjnxZFVvsscYTCFnT5TZWXQfPAM51AeYFFTXmt00m+YjaE9jrCv2ryNR31XRBFIlmtJ9LZN23uwxH0/BwInpqWHbotRIlIsttxjoTNA12qhnu+nG2utKxvGS9LpALdt9gCHPa/rC8Qntappx4ff+yPjhFI4QgPvYSWRDU+xgrQP5xtCNYJ46WvGw6h2ZBzD7wR/sfdUTcvsGnoeMFid4PM3VkGbr1u9Nd4gu2z5jzN7fLjPBWgVaD903KndZqd9tv8S0fdTUrQTgGiwoE6fOidYxOB7YzJQyN5C6xgfCiKQSJyDRHc3RMudUyR72/Wt+MLxs+Twb8tg7aKHklgE3qefXDDYvlPGr0hV1HeosgNClM0Dr3xfb81NZCAb26GVR5cyx16ygPIM/aG2zYWnt11FYIGGN1ofx+yJ+yBFcFnIRHN9vOutzi8NjKpBbBtTbhUtBQ6objyHtjB5+hlNKL98JbEt2a7n9L8Dv8/pvzGoYvkEv776hU0hFbA1+iBLYVLL0LdWeVIRW492iwAvHdcAB3BisKt2HBVTfZh/4La+1aWi8Vms8lacgVbzApuFoqLxfD/l0Lugdzif9n/F0pjZVm8LmQRA0SPnqwildW+MenbjLWsVsnwnY4+ZgTvdATiyKDXoXdieGctv30NZCttT/dragxyTOnd4/jmBBc+9Aze8HxyaAX7yeC7t5MImnOXczv+sGcqwUyCQ0VosYo9uwJuqb8WSnYeKdVZXIQ615bMhH4dlvasqnNxwnM9/Aa1T0y3LvLP4Ps9prGtQl3xzOu+k0cFaMzumFDosb8oimeHFU3jOiryOJD7EcjJiVxQPxPP2DX8HDrPQNYP95MhzfsmJnRc/QBxf7yxLcbzw/Bou7PcnfkOYZNJaB+9tsX4qPl+N39tW8ePuolNdrEblEbZYeIweNw7lDqqk4V2clIyPTiK84qw+gkjo5olfShOywl3rqBMcYOjVm2OpD49LGYKjtZNid39EwAA//+S3DeD" } diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml index 60145e644102..8f3bb2590184 100644 --- a/filebeat/module/mysql/slowlog/_meta/fields.yml +++ b/filebeat/module/mysql/slowlog/_meta/fields.yml @@ -3,10 +3,6 @@ description: > Contains fields from the MySQL slow logs. fields: - - name: query_time.sec - type: float - description: > - The total time the query took, in seconds, as a floating point number. - name: lock_time.sec type: float description: > @@ -67,7 +63,7 @@ Whether the query needed temporary tables on disk. - name: tmp_tables type: long - description: > + description: > Number of temporary tables created for this query - name: tmp_disk_tables type: long @@ -76,7 +72,7 @@ - name: tmp_table_sizes type: long format: bytes - description: + description: Size of temporary tables created for this query. - name: filesort type: boolean diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index b51bcad5a2f8..4849863bf052 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" + "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" ], "pattern_definitions" : { "GREEDYMULTILINE": "(.|\n)*", @@ -39,10 +39,15 @@ } } }, { - "script":{ + "script": { "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.mysql.slowlog.query_time.sec * 1000000) * 1000", - "if": "ctx.mysql.slowlog.query_time?.sec != null" + "source": "ctx.event.duration = Math.round(ctx.temp.duration * 1000000) * 1000", + "if": "ctx.temp?.duration != null" + } + }, { + "remove": { + "field": "temp.duration", + "ignore_missing": true } }, { "date": { diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json index 3e979082548f..24b9728d6fb9 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json @@ -15,7 +15,6 @@ "mysql.slowlog.lock_time.sec": 0.0, "mysql.slowlog.query": "select sleep(2);", "mysql.slowlog.query_cache_hit": false, - "mysql.slowlog.query_time.sec": 2.000652, "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, "mysql.thread_id": "5", diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json index 1bab861381fd..7a331caecc60 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json @@ -15,7 +15,6 @@ "mysql.slowlog.lock_time.sec": 0.0, "mysql.slowlog.query": "select sleep(2)\nAS foo;", "mysql.slowlog.query_cache_hit": false, - "mysql.slowlog.query_time.sec": 2.000227, "mysql.slowlog.rows_affected": 0, "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, @@ -46,7 +45,6 @@ "mysql.slowlog.priority_queue": false, "mysql.slowlog.query": "call PROC('blah');", "mysql.slowlog.query_cache_hit": false, - "mysql.slowlog.query_time.sec": 178.30602, "mysql.slowlog.rows_affected": 3062, "mysql.slowlog.rows_examined": 53022772, "mysql.slowlog.rows_sent": 0, diff --git a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json index 40ee56904bd8..4050aa4ddc23 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json @@ -15,7 +15,6 @@ "mysql.slowlog.lock_time.sec": 0.000337, "mysql.slowlog.query": "select count(*) from customer, orders, nation\n where c_custkey=o_custkey\n and c_nationkey=n_nationkey\n and n_name='GERMANY';", "mysql.slowlog.query_cache_hit": false, - "mysql.slowlog.query_time.sec": 5.524103, "mysql.slowlog.rows_examined": 65633, "mysql.slowlog.rows_sent": 1, "mysql.slowlog.schema": "dbt3sf1", diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json index 68c950eefc62..3036d6644357 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json @@ -14,7 +14,6 @@ "mysql.slowlog.current_user": "root", "mysql.slowlog.lock_time.sec": 0.0, "mysql.slowlog.query": "select sleep(15);", - "mysql.slowlog.query_time.sec": 15.000223, "mysql.slowlog.rows_examined": 0, "mysql.slowlog.rows_sent": 1, "mysql.thread_id": 7234, @@ -37,7 +36,6 @@ "mysql.slowlog.current_user": "debian-sys-maint", "mysql.slowlog.lock_time.sec": 6.1e-05, "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", - "mysql.slowlog.query_time.sec": 0.000153, "mysql.slowlog.rows_examined": 5, "mysql.slowlog.rows_sent": 1, "service.type": "mysql", @@ -59,7 +57,6 @@ "mysql.slowlog.current_user": "appuser", "mysql.slowlog.lock_time.sec": 0.000212, "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = 'CYB'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;", - "mysql.slowlog.query_time.sec": 4.071491, "mysql.slowlog.rows_examined": 1489615, "mysql.slowlog.rows_sent": 1000, "mysql.thread_id": 10997316, @@ -83,7 +80,6 @@ "mysql.slowlog.current_user": "appuser", "mysql.slowlog.lock_time.sec": 3.6e-05, "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');", - "mysql.slowlog.query_time.sec": 10.346539, "mysql.slowlog.rows_examined": 4751313, "mysql.slowlog.rows_sent": 0, "mysql.thread_id": 10999834, diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json index a45e9be1c781..770d0d449805 100644 --- a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json @@ -25,7 +25,6 @@ "mysql.slowlog.merge_passes": 0, "mysql.slowlog.query": "SELECT t.table_schema, t.table_name, column_name, `auto_increment`,\n pow(2, case data_type\n when 'tinyint' then 7\n when 'smallint' then 15\n when 'mediumint' then 23\n when 'int' then 31\n when 'bigint' then 63\n end+(column_type like '% unsigned'))-1 as max_int\n FROM information_schema.tables t\n JOIN information_schema.columns c\n ON BINARY t.table_schema = c.table_schema AND BINARY t.table_name = c.table_name\n WHERE c.extra = 'auto_increment' AND t.auto_increment IS NOT NULL;", "mysql.slowlog.query_cache_hit": false, - "mysql.slowlog.query_time.sec": 0.050365, "mysql.slowlog.rows_affected": 0, "mysql.slowlog.rows_examined": 3146, "mysql.slowlog.rows_sent": 16, @@ -72,7 +71,6 @@ "mysql.slowlog.merge_passes": 0, "mysql.slowlog.query": "UPDATE test SET test.state = 'NOT_RELEVANT', modified = now() WHERE test.id IN (26328833, 390, 149386, 152268, 160997, 165304, 168524, 184105, 193022, 194533, 194862, 196469, 196487, 246398, 256594, 260566, 261862, 262342, 263701, 264166, 264607, 267671, 274879, 276704, 280964, 284366, 289323, 289843, 290004, 298999, 301213, 303494, 307920, 311905, 316311, 318404, 330846, 340751, 341433, 357191, 369184, 376876, 378360, 378492, 379470, 382131, 384077, 388368, 396815, 396881, 398272, 398950, 399589, 401299, 408787, 411293, 419109, 425953, 427659, 433183, 437030, 438332, 438386, 447037, 454231, 455257, 455344, 456385, 460420, 460425, 461252, 462338, 462531, 462684, 463104, 463395, 471073, 480069, 480078, 482399, 485205, 487971, 497191, 500261, 501855, 517585, 519310, 519654, 522575, 538425, 543560, 562315, 573934, 583466, 583490, 583502, 597605, 600875, 601546, 603879, 604467, 604619, 757786, 797285, 799155, 802905, 806268, 806798, 811974, 819684, 822629, 826406, 837733, 840128, 840131, 840251, 840277, 840302, 842966, 844294, 844300, 847837, 852503, 854272, 854299, 862983, 881405, 881461, 881467, 881560, 881908, 882435, 882453, 882651, 882711, 882811, 888265, 888286, 914091, 916288, 916316, 917708, 918238, 918887, 919222, 926607, 976977, 977010, 977067, 977131, 977185, 988249, 988276, 988336, 988360, 988504, 990994);", "mysql.slowlog.query_cache_hit": false, - "mysql.slowlog.query_time.sec": 153.88348, "mysql.slowlog.rows_affected": 19198, "mysql.slowlog.rows_examined": 120309968, "mysql.slowlog.rows_sent": 0, diff --git a/filebeat/module/postgresql/fields.go b/filebeat/module/postgresql/fields.go index e9e2539b165c..ec3f465f3324 100644 --- a/filebeat/module/postgresql/fields.go +++ b/filebeat/module/postgresql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyck0FP3DAQhe/5FU97rESE1Fsq9ULhBG0p3NGwmThW7YzxOKjbX185AXbJJivSOcZ67/sycc7wm3cVgmgykfXJFUCyyXGFzc/x4d3t9aYAatZttCFZ6Sp8LQDgRureMRqJCBTVdgapZexzcGLQWMdaFoC2EtPDVrrGmgop9lwAjWVXazX0naEjzxObPGkXuIKJ0oeXJzM241wNfWii+InI4JDnEHmIdWIOio6ZJ7nvyafW8DpTjUOVZD1rIh/enZ7E57lveR/dm2S8sx2Xs6ytRH6w9aRsfH0nnVmncCGRMWl7JdWU6JGUJyH+Qz4MN87v6sfNCd4R7Tt5hjTzzW/YPlKOz75h44TSotDn8/J8ldC3F1aWIjz1HHfzWx+OFrl3l9eXF/f4hKtfP27QK0f9ssrjNtdDEyX23KW9w+J1+yvd9MOMGyJnSScngVJbgZ9z9ULYWzOu4u1Xn+G2kaleunzL4BBly6plOEp+hJrXuRKYI2VO/wfO8TO7lTwnppzLfYTnWZXM2m85n5ry/gUAAP//D0KTig==" + return "eJyck0FP3DAQhe/5FU97rER+QCr1guAEbSnckdlMHKu2x3gc1O2vr7xhtcGbRKRztPXe9/ImvsJvOjQILElHkldbAckkSw12P8fDx4e7XQW0JPtoQjLsG3yrAOCe28ESOo4IKorxGqknnHWwrNEZS1JXgPQc0/OefWd0gxQHqoDOkG2lOfpdwStHRZo86RCogY48hPeTmTTj3B790EV2RZBjhjxT5BRrWU+MLpmr3I/ktRpOU8aYRknGkSTlwofbVXyep57O0nOSjLfGUz3L2nOkZ9MWZuPnW/Z6W4RrjoTC7URqVVIvSqgQ0R/lwvGPc4f2ZbfCu6B9V47A3bzzCfs6UDwsMh9v7m6un/AFt79+3GMQivJ1U4aHbA9JKpEjn84lL272L/uyg7FuZY2S4iao1Degt2y9IHZGRzUu5P1VzXD7SKpd2vMyOETek0gdLpSfoeY6NwKzpM7q/8BZeiO7kWdZ13O6z/AciSi9dZfzqpL3LwAA//+FnHBP" } diff --git a/filebeat/module/postgresql/log/_meta/fields.yml b/filebeat/module/postgresql/log/_meta/fields.yml index 92b48ffb8c45..d7ee240dcfb3 100644 --- a/filebeat/module/postgresql/log/_meta/fields.yml +++ b/filebeat/module/postgresql/log/_meta/fields.yml @@ -14,11 +14,6 @@ example: "mydb" description: Name of database - - name: duration - type: float - example: "30.0" - description: - Duration of a query. - name: query example: "SELECT * FROM users;" description: diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 4a3fb4937a69..5101d751a98c 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:event.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:event.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:temp.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", @@ -24,13 +24,17 @@ ], "ignore_failure": true } - }, - { + }, { "script": { - "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.postgresql.log.duration * params.scale)", - "params": { "scale": 1000000 }, - "if": "ctx.postgresql.log.containsKey('duration')" + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.temp?.duration != null" + } + }, { + "remove": { + "field": "temp.duration", + "ignore_missing": true } } ], diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index d1a1b109da2a..6dfe9c270c87 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -92,7 +92,6 @@ "log.offset": 445, "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": 37.118, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "process.pid": 4983, @@ -115,7 +114,6 @@ "log.offset": 873, "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": 2.895, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", "process.pid": 4986, @@ -138,7 +136,6 @@ "log.offset": 1300, "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": 2.809, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", "process.pid": 4989, @@ -195,7 +192,6 @@ "log.offset": 1907, "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "postgres", - "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", "process.pid": 5404, @@ -215,7 +211,6 @@ "log.offset": 2620, "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "postgresql.log.database": "clients", - "postgresql.log.duration": 9.482, "postgresql.log.query": "select * from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", "process.pid": 5502, @@ -235,7 +230,6 @@ "log.offset": 2733, "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "postgresql.log.database": "clients", - "postgresql.log.duration": 0.765, "postgresql.log.query": "select id from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", "process.pid": 5502, @@ -258,7 +252,6 @@ "log.offset": 2847, "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "clients", - "postgresql.log.duration": 26.082, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", "process.pid": 5502, @@ -278,7 +271,6 @@ "log.offset": 3559, "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients", - "postgresql.log.duration": 36.162, "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", "process.pid": 5502, @@ -298,7 +290,6 @@ "log.offset": 3751, "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.database": "c$lients", - "postgresql.log.duration": 10.54, "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "process.pid": 5502, @@ -318,7 +309,6 @@ "log.offset": 3908, "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.database": "_clients$db", - "postgresql.log.duration": 5.156, "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "process.pid": 5502, @@ -338,7 +328,6 @@ "log.offset": 4069, "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients_db", - "postgresql.log.duration": 25.871, "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "process.pid": 5502, diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index b3f008c4dffe..eb8be342a872 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -52,7 +52,6 @@ "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.core_id": 2, "postgresql.log.database": "postgres", - "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", "process.pid": 5404,