diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index d5b5bfd5f59..9ef45505011 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -143,7 +143,7 @@ system-tests: buildbeat.test prepare-tests python-env # Runs system tests without coverage reports and in parallel .PHONY: fast-system-tests fast-system-tests: buildbeat.test python-env - . ${PYTHON_ENV}/bin/activate; nosetests -w tests/system --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) + . ${PYTHON_ENV}/bin/activate; nosetests -w tests/system --processes=$(PROCESSES) --process-timeout=$(TIMEOUT) -a '!integration' # Run benchmark tests .PHONY: benchmark-tests diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index cf5b8dbe4a6..41d749baf1f 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -264,6 +264,18 @@ def wait_until(self, cond, max_timeout=10, poll_interval=0.1, name="cond"): "Waited {} seconds.".format(max_timeout)) time.sleep(poll_interval) + def get_log(self, logfile=None): + """ + Returns the log as a string. + """ + if logfile is None: + logfile = self.beat_name + ".log" + + with open(os.path.join(self.working_dir, logfile), 'r') as f: + data=f.read() + + return data + def log_contains(self, msg, logfile=None): """ Returns true if the give logfile contains the given message. diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 994ee34962e..e2963f1ab89 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -27,29 +27,28 @@ for each MetricSet to prevent type conflicts. Also all values are stored under t package beater import ( + "fmt" + "github.com/elastic/beats/libbeat/beat" - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/include" ) type Metricbeat struct { - done chan struct{} - MbConfig *Config + done chan struct{} + config *Config } -// New creates a new Metricbeat instance +// New creates and returns a new Metricbeat instance. func New() *Metricbeat { return &Metricbeat{} } func (mb *Metricbeat) Config(b *beat.Beat) error { - - mb.MbConfig = &Config{} - err := b.RawConfig.Unpack(mb.MbConfig) + mb.config = &Config{} + err := b.RawConfig.Unpack(mb.config) if err != nil { - logp.Err("Error reading configuration file: %v", err) - return err + return fmt.Errorf("error reading configuration file. %v", err) } // List all registered modules and metricsets @@ -64,9 +63,8 @@ func (mb *Metricbeat) Setup(b *beat.Beat) error { } func (mb *Metricbeat) Run(b *beat.Beat) error { - // Checks all defined metricsets and starts a module for each entry with the defined metricsets - for _, moduleConfig := range mb.MbConfig.Metricbeat.Modules { + for _, moduleConfig := range mb.config.Metricbeat.Modules { module, err := helper.Registry.GetModule(moduleConfig) if err != nil { diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4322fbedb9f..2ffc9958a43 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -44,6 +44,11 @@ required: True The timestamp when the log line was read. The precision is in milliseconds. The timezone is UTC. +==== metricset-host + +Hostname of the machine from which the metricset was collected. This field may not be present when the data was collected locally. + + ==== rtt type: long diff --git a/metricbeat/etc/fields.yml b/metricbeat/etc/fields.yml index 9ced459272c..c9e5f40b55f 100644 --- a/metricbeat/etc/fields.yml +++ b/metricbeat/etc/fields.yml @@ -38,6 +38,11 @@ common: The timestamp when the log line was read. The precision is in milliseconds. The timezone is UTC. + - name: metricset-host + description: > + Hostname of the machine from which the metricset was collected. This + field may not be present when the data was collected locally. + - name: rtt type: long required: true diff --git a/metricbeat/etc/fields_base.yml b/metricbeat/etc/fields_base.yml index cfca79d65a2..3bc3fd1a414 100644 --- a/metricbeat/etc/fields_base.yml +++ b/metricbeat/etc/fields_base.yml @@ -38,6 +38,11 @@ common: The timestamp when the log line was read. The precision is in milliseconds. The timezone is UTC. + - name: metricset-host + description: > + Hostname of the machine from which the metricset was collected. This + field may not be present when the data was collected locally. + - name: rtt type: long required: true diff --git a/metricbeat/module/redis/info/data.go b/metricbeat/module/redis/info/data.go index 3040381f1d9..7a0e2e152de 100644 --- a/metricbeat/module/redis/info/data.go +++ b/metricbeat/module/redis/info/data.go @@ -35,7 +35,7 @@ func eventMapping(info map[string]string) common.MapStr { "used_memory_lua": toInt(info["used_memory_lua"]), "mem_allocator": info["mem_allocator"], // Could be moved to server as it rarely changes }, - "presistence": common.MapStr{ + "persistence": common.MapStr{ "loading": toBool(info["loading"]), "rdb_changes_since_last_save": toInt(info["rdb_changes_since_last_save"]), "rdb_bgsave_in_progress": toBool(info["rdb_bgsave_in_progress"]), @@ -85,8 +85,8 @@ func eventMapping(info map[string]string) common.MapStr { "instantaneous_ops_per_sec": toInt(info["instantaneous_ops_per_sec"]), "total_net_input_bytes": toInt(info["total_net_input_bytes"]), "total_net_output_bytes": toInt(info["total_net_output_bytes"]), - "instantaneous_input_kbps": toInt(info["instantaneous_input_kbps"]), - "instantaneous_output_kbps": toInt(info["instantaneous_output_kbps"]), + "instantaneous_input_kbps": toFloat(info["instantaneous_input_kbps"]), + "instantaneous_output_kbps": toFloat(info["instantaneous_output_kbps"]), "rejected_connections": toInt(info["rejected_connections"]), "sync_full": toInt(info["sync_full"]), "sync_partial_ok": toInt(info["sync_partial_ok"]), diff --git a/metricbeat/module/redis/info/info.go b/metricbeat/module/redis/info/info.go index 936b23ada61..30835071441 100644 --- a/metricbeat/module/redis/info/info.go +++ b/metricbeat/module/redis/info/info.go @@ -12,6 +12,10 @@ import ( "github.com/elastic/beats/metricbeat/helper" ) +var ( + debugf = logp.MakeDebug("redis") +) + func init() { helper.Registry.AddMetricSeter("redis", "info", New) } @@ -75,14 +79,13 @@ func createPool(host, password, network string, maxConn int, timeout time.Durati } func (m *MetricSeter) Fetch(ms *helper.MetricSet, host string) (events common.MapStr, err error) { - // Fetch default INFO info, err := m.fetchRedisStats(host, "default") - if err != nil { return nil, err } + debugf("Redis INFO from %s: %+v", host, info) return eventMapping(info), nil } diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 6828c9d1df8..432e1861456 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -3,7 +3,7 @@ metricbeat: {% if redis %} - module: redis hosts: - - "{{ redis_host | default("127.0.0.1", true) }}:6379" + - "{{ redis_host|default("127.0.0.1", true) }}:6379" metricsets: - info period: 2s @@ -12,7 +12,7 @@ metricbeat: {% endif %} {% if mysql %} - module: mysql: - # This expectd a full mysql dsn + # This expects a full MySQL Data Source Name (DSN). # Example: [username[:password]@][protocol[(address)]]/ hosts: - "@tcp(127.0.0.1:3306)/" @@ -22,44 +22,11 @@ metricbeat: enabled: true {% endif %} - -############################# Output ############################################ - -# Configure what outputs to use when sending the data collected by metricbeat. -# You can enable one or multiple outputs by setting enabled option to true. output: - - # Elasticsearch as output - # Options: - # host, port: where Elasticsearch is listening on - # save_topology: specify if the topology is saved in Elasticsearch - #elasticsearch: - # enabled: false - # host: localhost - # port: 9200 - # save_topology: true - - # Redis as output - # Options: - # host, port: where Redis is listening on - # save_topology: specify if the topology is saved in Redis - #redis: - # enabled: false - # host: localhost - # port: 6379 - # save_topology: true - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path file: enabled: true path: {{ output_file_path|default(beat.working_dir + "/output") }} filename: "{{ output_file_filename|default("metricbeat") }}" rotate_every_kb: 1000 - #number_of_files: 7 # vim: set ft=jinja: diff --git a/metricbeat/tests/system/metricbeat.py b/metricbeat/tests/system/metricbeat.py new file mode 100644 index 00000000000..200114f7395 --- /dev/null +++ b/metricbeat/tests/system/metricbeat.py @@ -0,0 +1,26 @@ +import sys + +sys.path.append('../../../libbeat/tests/system') +from beat.beat import TestCase + +COMMON_FIELDS = ["@timestamp", "beat", "metricset", "metricset-host", + "module", "rtt", "type"] + + +class BaseTest(TestCase): + @classmethod + def setUpClass(self): + self.beat_name = "metricbeat" + self.build_path = "../../build/system-tests/" + self.beat_path = "../../metricbeat.test" + + def assert_fields_are_documented(self, evt): + """ + Assert that all keys present in evt are documented in fields.yml. + """ + expected_fields, _ = self.load_fields() + flat = self.flatten_object(evt, []) + + for key in flat.keys(): + if key not in expected_fields: + raise Exception("Key '{}' found in event is not documented!".format(key)) diff --git a/metricbeat/tests/system/metricbeat/__init__.py b/metricbeat/tests/system/metricbeat/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/metricbeat/tests/system/metricbeat/metricbeat.py b/metricbeat/tests/system/metricbeat/metricbeat.py deleted file mode 100644 index 8ee7e8bef38..00000000000 --- a/metricbeat/tests/system/metricbeat/metricbeat.py +++ /dev/null @@ -1,16 +0,0 @@ - -import sys - -sys.path.append('../../../libbeat/tests/system') - -from beat.beat import TestCase - - -class BaseTest(TestCase): - - @classmethod - def setUpClass(self): - self.beat_name = "metricbeat" - self.build_path = "../../build/system-tests/" - self.beat_path = "../../metricbeat.test" - diff --git a/metricbeat/tests/system/redis/__init__.py b/metricbeat/tests/system/redis/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/metricbeat/tests/system/redis/test_info.py b/metricbeat/tests/system/redis/test_info.py deleted file mode 100644 index 115296492a1..00000000000 --- a/metricbeat/tests/system/redis/test_info.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import sys - -# Load parent directory -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - -from metricbeat.metricbeat import BaseTest - -class Test(BaseTest): - - def test_base(self): - """ - Basic test with exiting metricbeat with redis info metricset normally - """ - self.render_config_template( - redis=True, - redis_host=os.getenv('REDIS_HOST') - ) - - proc = self.start_beat() - self.wait_until( - lambda: self.output_has(lines=1) - ) - - exit_code = proc.kill_and_wait() - assert exit_code == 0 - - def test_selectors(self): - """ - Test if selectors reduce the output as expected - """ - self.render_config_template( - redis=True, - redis_host=os.getenv('REDIS_HOST'), - redis_selectors=["clients", "cpu"] - ) - - proc = self.start_beat() - self.wait_until( - lambda: self.output_has(lines=1) - ) - - output = self.read_output_json() - event = output[0] - redis_info = event["redis-info"] - - assert len(redis_info) == 2 - assert len(redis_info["clients"]) == 4 - assert len(redis_info["cpu"]) == 4 - - - exit_code = proc.kill_and_wait() - assert exit_code == 0 diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index 052d634077a..7e9b156c513 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -1,15 +1,24 @@ -from metricbeat.metricbeat import BaseTest +import re +from metricbeat import BaseTest -class Test(BaseTest): - def test_base(self): +class Test(BaseTest): + def test_start_stop(self): """ - Basic test with exiting Mockbeat normally + Metricbeat starts and stops without error. """ - self.render_config_template( - ) - + self.render_config_template() proc = self.start_beat() - self.wait_until( lambda: self.log_contains("Setup Beat")) - exit_code = proc.kill_and_wait() - assert exit_code == 0 + self.wait_until(lambda: self.log_contains("Setup Beat")) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log, "ERR|WARN") + + # Ensure all Beater stages are used. + self.assertRegexpMatches(log, re.compile(".*".join([ + "Setup Beat: metricbeat", + "metricbeat start running", + "metricbeat cleanup" + ]), re.DOTALL)) diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py new file mode 100644 index 00000000000..4677265d08a --- /dev/null +++ b/metricbeat/tests/system/test_redis.py @@ -0,0 +1,79 @@ +import os +import metricbeat +from nose.plugins.attrib import attr + +REDIS_FIELDS = metricbeat.COMMON_FIELDS + ["redis-info"] + +REDIS_INFO_FIELDS = ["clients", "cluster", "cpu", "keyspace", "memory", + "persistence", "replication", "server", "stats"] + +CPU_FIELDS = ["used_cpu_sys", "used_cpu_sys_children", "used_cpu_user", + "used_cpu_user_children"] + +CLIENTS_FIELDS = ["blocked_clients", "client_biggest_input_buf", + "client_longest_output_list", "connected_clients"] + + +class RedisInfoTest(metricbeat.BaseTest): + @attr('integration') + def test_output(self): + """ + Redis module outputs an event. + """ + self.render_config_template( + redis=True, + redis_host=os.getenv('REDIS_HOST') + ) + proc = self.start_beat() + self.wait_until( + lambda: self.output_has(lines=1) + ) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log, "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(REDIS_FIELDS, evt.keys()) + redis_info = evt["redis-info"] + self.assertItemsEqual(REDIS_INFO_FIELDS, redis_info.keys()) + self.assertItemsEqual(CLIENTS_FIELDS, redis_info["clients"].keys()) + self.assertItemsEqual(CPU_FIELDS, redis_info["cpu"].keys()) + + # TODO: After fields.yml is updated this can be uncommented. + #self.assert_fields_are_documented(evt) + + @attr('integration') + def test_selectors(self): + """ + Redis selectors filter the event. + """ + selectors = ["clients", "cpu"] + self.render_config_template( + redis=True, + redis_host=os.getenv('REDIS_HOST'), + redis_selectors=selectors + ) + proc = self.start_beat() + self.wait_until( + lambda: self.output_has(lines=1) + ) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log, "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(REDIS_FIELDS, evt.keys()) + redis_info = evt["redis-info"] + self.assertItemsEqual(selectors, redis_info.keys()) + self.assertItemsEqual(CLIENTS_FIELDS, redis_info["clients"].keys()) + self.assertItemsEqual(CPU_FIELDS, redis_info["cpu"].keys())