From 1402b324f780382b767e19fcf53b4b549e4b88b6 Mon Sep 17 00:00:00 2001 From: urso Date: Mon, 11 Sep 2017 17:41:52 +0200 Subject: [PATCH 1/3] Update queue.mem defaults - Enable flush based memory queue by default: Increases chance of pushing bigger batches of events to the outputs. Especially if number of CPUs is limited to 1 or 2 this helps with throughput, as outputs and producers go-routines will interleave less. - Set flush.events=2048 and flush.timeout=1s --- CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 4 ++-- filebeat/filebeat.reference.yml | 4 ++-- heartbeat/heartbeat.reference.yml | 4 ++-- libbeat/_meta/config.reference.yml | 4 ++-- libbeat/publisher/queue/memqueue/config.go | 4 ++-- metricbeat/metricbeat.reference.yml | 4 ++-- packetbeat/packetbeat.reference.yml | 4 ++-- winlogbeat/winlogbeat.reference.yml | 4 ++-- 9 files changed, 17 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index b4c94ff9474..d6bb8816708 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -68,6 +68,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] +- Enable flush timeout by defauly. {pull}5150[5150] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 7ad40e0c424..6e37829e218 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -101,11 +101,11 @@ auditbeat.modules: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 11f0fe833be..05c76a7e7b6 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -521,11 +521,11 @@ filebeat.prospectors: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index ac73a44c4f1..23350d2ac1c 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -250,11 +250,11 @@ heartbeat.scheduler: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 8f69c25573d..8248a2ead01 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -36,11 +36,11 @@ # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/libbeat/publisher/queue/memqueue/config.go b/libbeat/publisher/queue/memqueue/config.go index ef843484a37..54da7aeb8f1 100644 --- a/libbeat/publisher/queue/memqueue/config.go +++ b/libbeat/publisher/queue/memqueue/config.go @@ -13,8 +13,8 @@ type config struct { var defaultConfig = config{ Events: 4 * 1024, - FlushMinEvents: 0, - FlushTimeout: 0, + FlushMinEvents: 2 * 1024, + FlushTimeout: 1 * time.Second, } func (c *config) Validate() error { diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0a9a09b6fa1..6356dd3d70a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -480,11 +480,11 @@ metricbeat.modules: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 633549190e5..262cbce5c19 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -488,11 +488,11 @@ packetbeat.protocols: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 6d2a0570081..96ba9830c88 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -65,11 +65,11 @@ winlogbeat.event_logs: # before providing a batch of events to the outputs. # A value of 0 (the default) ensures events are immediately available # to be sent to the outputs. - #flush.min_events: 0 + #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. - #flush.timeout: 0s + #flush.timeout: 1s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. From a45a17f621c46637f79003b08137dd18411826d9 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 12 Sep 2017 16:41:48 +0200 Subject: [PATCH 2/3] set queue flush parameters in system tests update queue flush parameters to min_events of 8 and a timeout of 0.1s. This hopefully stabilizes some time sensitive tests. --- auditbeat/tests/system/config/auditbeat.yml.j2 | 5 +++++ filebeat/tests/system/config/filebeat.yml.j2 | 7 +++++++ heartbeat/tests/system/config/heartbeat.yml.j2 | 9 +++++++++ libbeat/tests/system/config/mockbeat.yml.j2 | 6 ++++++ metricbeat/tests/system/config/metricbeat.yml.j2 | 7 +++++++ packetbeat/tests/system/config/packetbeat.yml.j2 | 7 +++++++ winlogbeat/tests/system/config/winlogbeat.yml.j2 | 7 +++++++ 7 files changed, 48 insertions(+) diff --git a/auditbeat/tests/system/config/auditbeat.yml.j2 b/auditbeat/tests/system/config/auditbeat.yml.j2 index 21911d7d039..0cdbf5bbf84 100644 --- a/auditbeat/tests/system/config/auditbeat.yml.j2 +++ b/auditbeat/tests/system/config/auditbeat.yml.j2 @@ -17,6 +17,11 @@ auditbeat.modules: {% endif -%} {%- endfor %} +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + output.file: path: '{{ output_file_path|default(beat.working_dir + "/output") }}' filename: {{ output_file_filename|default("auditbeat") }} diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index a41e1db935f..9a7330b35a9 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -156,6 +156,13 @@ processors: {%- endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/heartbeat/tests/system/config/heartbeat.yml.j2 b/heartbeat/tests/system/config/heartbeat.yml.j2 index 664715c60b6..42407d56841 100644 --- a/heartbeat/tests/system/config/heartbeat.yml.j2 +++ b/heartbeat/tests/system/config/heartbeat.yml.j2 @@ -56,6 +56,15 @@ fields: {% endfor -%} {% endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + +#================================ Outputs ===================================== + output.file: path: {{ output_file_path|default(beat.working_dir + "/output") }} filename: "{{ output_file_filename|default("heartbeat") }}" diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 03909dfb3ac..a0415a953d2 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -18,6 +18,12 @@ tags: [ {%- endfor -%} {%- endif -%}] +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s ############################# Output ############################################ diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 519bf3ef1fe..2c02f4b78c9 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -139,6 +139,13 @@ processors: {%- endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 354958c38d3..2523d2ea69f 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -193,6 +193,13 @@ processors: {%- endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. diff --git a/winlogbeat/tests/system/config/winlogbeat.yml.j2 b/winlogbeat/tests/system/config/winlogbeat.yml.j2 index f221376cbff..8ca238a13ba 100644 --- a/winlogbeat/tests/system/config/winlogbeat.yml.j2 +++ b/winlogbeat/tests/system/config/winlogbeat.yml.j2 @@ -77,6 +77,13 @@ fields: {% endfor -%} {% endif %} +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: 8 + flush.timeout: 0.1s + #================================ Outputs ===================================== # Configure what outputs to use when sending the data collected by the beat. From 2afc1490f54bc067cabc0c0b4a39542daa679cfc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 12 Sep 2017 12:16:08 -0400 Subject: [PATCH 3/3] Fix typo in CHANGELOG --- CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d6bb8816708..65fc539eac4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -68,7 +68,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] -- Enable flush timeout by defauly. {pull}5150[5150] +- Enable flush timeout by default. {pull}5150[5150] *Auditbeat*