From d6fd3317d2e99b67e0589a8eafbbe9b224450fca Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 11 Feb 2016 17:32:11 +0100 Subject: [PATCH 1/2] Add shipper.max_procs option --- CHANGELOG.asciidoc | 1 + libbeat/beat/beat.go | 7 +++++++ libbeat/docs/shipperconfig.asciidoc | 5 +++++ libbeat/publisher/publish.go | 2 ++ 4 files changed, 15 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 5e41b85a806..f9626ad0657 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -62,6 +62,7 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff] - Add option to elasticsearch output to pass http parameters in index operations {issue}805[805] - Improve logstash and elasticsearch backoff behavior. {pull}927[927] - Add experimental Kafka output. {pull}942[942] +- Add config file option to configure GOMAXPROCS. {pull}969[969] *Packetbeat* - Change the DNS library used throughout the dns package to github.com/miekg/dns. {pull}803[803] diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index b00f4f3d1e6..3736b29fae1 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -220,6 +220,13 @@ func (b *Beat) LoadConfig() error { logp.Debug("beat", "Initializing output plugins") + if b.Config.Shipper.MaxProcs != nil { + maxProcs := *b.Config.Shipper.MaxProcs + if maxProcs > 0 { + runtime.GOMAXPROCS(maxProcs) + } + } + pub, err := publisher.New(b.Name, b.Config.Output, b.Config.Shipper) if err != nil { return fmt.Errorf("error Initialising publisher: %v\n", err) diff --git a/libbeat/docs/shipperconfig.asciidoc b/libbeat/docs/shipperconfig.asciidoc index 2862b54fc23..6fe734e48d9 100644 --- a/libbeat/docs/shipperconfig.asciidoc +++ b/libbeat/docs/shipperconfig.asciidoc @@ -153,6 +153,11 @@ value is 1000. (DO NOT TOUCH) The internal queue size for bulk events in the processing pipeline. The default value is 0. +===== max_procs + +Sets the maximum number of CPUs that can be executing simultaneously. The +default is the number of logical CPUs available in the system. + ===== geoip.paths This configuration option is currently used by Packetbeat only. diff --git a/libbeat/publisher/publish.go b/libbeat/publisher/publish.go index e3e3692b51c..fbef3413de4 100644 --- a/libbeat/publisher/publish.go +++ b/libbeat/publisher/publish.go @@ -86,6 +86,8 @@ type ShipperConfig struct { // internal publisher queue sizes QueueSize *int `yaml:"queue_size"` BulkQueueSize *int `yaml:"bulk_queue_size"` + + MaxProcs *int `yaml:"max_procs"` } type Topology struct { From e0e5c9e5462818490dfb4f17f013a2551d88b836 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 11 Feb 2016 17:36:20 +0100 Subject: [PATCH 2/2] Add max_procs to libbeat.yml --- filebeat/filebeat.yml | 4 ++++ libbeat/etc/libbeat.yml | 4 ++++ metricbeat/metricbeat.yml | 9 ++++++--- packetbeat/packetbeat.yml | 4 ++++ topbeat/topbeat.yml | 4 ++++ winlogbeat/winlogbeat.yml | 4 ++++ 6 files changed, 26 insertions(+), 3 deletions(-) diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 7176679cda2..2254fbfcb22 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -376,6 +376,10 @@ shipper: # Internal queue size for single events in processing pipeline #queue_size: 1000 + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + # Configure local GeoIP database support. # If no paths are not configured geoip is disabled. #geoip: diff --git a/libbeat/etc/libbeat.yml b/libbeat/etc/libbeat.yml index 6ed5e5d162d..9ff9c4427cb 100644 --- a/libbeat/etc/libbeat.yml +++ b/libbeat/etc/libbeat.yml @@ -205,6 +205,10 @@ shipper: # Internal queue size for single events in processing pipeline #queue_size: 1000 + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + # Configure local GeoIP database support. # If no paths are not configured geoip is disabled. #geoip: diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 57f16d64580..c6c05ea808f 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -5,8 +5,7 @@ metricbeat: # expvar: # period: 10s apache: - hosts: - - "http://192.168.99.100/" + hosts: "http://127.0.0.1/" metricsets: status: period: 1s @@ -40,7 +39,7 @@ output: # Scheme and port can be left out and will be set to the default (http and 9200) # In case you specify and additional path, the scheme is required: http://localhost:9200/path # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 - hosts: ["192.168.99.100:9200"] + hosts: ["localhost:9200"] # Optional protocol and basic auth credentials. #protocol: "https" @@ -231,6 +230,10 @@ shipper: # Internal queue size for single events in processing pipeline #queue_size: 1000 + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + # Configure local GeoIP database support. # If no paths are not configured geoip is disabled. #geoip: diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 482e5c68b3a..a7d293e19ca 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -348,6 +348,10 @@ shipper: # Internal queue size for single events in processing pipeline #queue_size: 1000 + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + # Configure local GeoIP database support. # If no paths are not configured geoip is disabled. #geoip: diff --git a/topbeat/topbeat.yml b/topbeat/topbeat.yml index 3f1a5bb750f..0995cd19666 100644 --- a/topbeat/topbeat.yml +++ b/topbeat/topbeat.yml @@ -231,6 +231,10 @@ shipper: # Internal queue size for single events in processing pipeline #queue_size: 1000 + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + # Configure local GeoIP database support. # If no paths are not configured geoip is disabled. #geoip: diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 13e62b30e19..91bf3c099fa 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -230,6 +230,10 @@ shipper: # Internal queue size for single events in processing pipeline #queue_size: 1000 + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + # Configure local GeoIP database support. # If no paths are not configured geoip is disabled. #geoip: