From e9052c457cee4d0594609c28c42d10aade352697 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 29 Jul 2020 16:40:12 +0200 Subject: [PATCH 1/5] Add an explicit system test for processes on unix systems Some fields are only available on some operating systems, and for processes Metricbeat has permissions to read. Check these fields only on these operating systems and on processes owned by the same user running the test. --- metricbeat/module/system/test_system.py | 77 ++++++++++++++++++++----- 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index cc6b731b113..2f41b4d745b 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -392,8 +392,63 @@ def test_process(self): self.assertGreater(len(output), 0) found_cmdline = False - found_env = False + for evt in output: + process = evt["system"]["process"] + + # Remove 'env' prior to checking documented fields because its keys are dynamic. + process.pop("env", None) + self.assert_fields_are_documented(evt) + + # Remove optional keys. + process.pop("cgroup", None) + process.pop("fd", None) + process.pop("cwd", None) + + cmdline = process.pop("cmdline", None) + if cmdline is not None: + found_cmdline = True + + self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) + + self.assertTrue(found_cmdline, "cmdline not found in any process events") + + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os") + def test_process_unix(self): + """ + Test system/process output for fields specific of unix systems. + """ + import getpass + + self.render_config_template(modules=[{ + "name": "system", + "metricsets": ["process"], + "period": "5s", + "extras": { + "process.env.whitelist": ["PATH"], + "process.include_cpu_ticks": True, + + # Remove 'percpu' prior to checking documented fields because its keys are dynamic. + "process.include_per_cpu": False, + }, + }], + # Some info is only guaranteed in processes with permissions, check + # only on own processes. + processors=[{ + "drop_event": { + "when": "not.equals.user.name: " + getpass.getuser(), + }, + }], + ) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertGreater(len(output), 0) + found_fd = False + found_env = False found_cwd = not sys.platform.startswith("linux") for evt in output: process = evt["system"]["process"] @@ -407,26 +462,18 @@ def test_process(self): # Remove optional keys. process.pop("cgroup", None) - cmdline = process.pop("cmdline", None) - if cmdline is not None: - found_cmdline = True - fd = process.pop("fd", None) - if fd is not None: - found_fd = True + process.pop("cmdline", None) cwd = process.pop("cwd", None) if cwd is not None: found_cwd = True + fd = process.pop("fd", None) + if fd is not None: + found_fd = True self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) - self.assertTrue(found_cmdline, "cmdline not found in any process events") - - if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): - self.assertTrue(found_fd, "fd not found in any process events") - - if sys.platform.startswith("linux") or sys.platform.startswith("freebsd")\ - or sys.platform.startswith("darwin"): - self.assertTrue(found_env, "env not found in any process events") + self.assertTrue(found_fd, "fd not found in any process events") + self.assertTrue(found_env, "env not found in any process events") @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process_metricbeat(self): From e2ba02b40c583d4c474628705c58dff51bb4e23e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 29 Jul 2020 17:00:11 +0200 Subject: [PATCH 2/5] Small refactor --- metricbeat/module/system/test_system.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 2f41b4d745b..1ea11d5bde6 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -394,6 +394,7 @@ def test_process(self): found_cmdline = False for evt in output: process = evt["system"]["process"] + found_cmdline |= "cmdline" in process # Remove 'env' prior to checking documented fields because its keys are dynamic. process.pop("env", None) @@ -402,11 +403,8 @@ def test_process(self): # Remove optional keys. process.pop("cgroup", None) process.pop("fd", None) - process.pop("cwd", None) - - cmdline = process.pop("cmdline", None) - if cmdline is not None: - found_cmdline = True + process.pop("working_directory", None) + process.pop("cmdline", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) @@ -452,23 +450,19 @@ def test_process_unix(self): found_cwd = not sys.platform.startswith("linux") for evt in output: process = evt["system"]["process"] + found_fd |= "fd" in process + found_env |= "env" in process + found_cwd |= "working_directory" in process # Remove 'env' prior to checking documented fields because its keys are dynamic. env = process.pop("env", None) - if env is not None: - found_env = True - self.assert_fields_are_documented(evt) # Remove optional keys. process.pop("cgroup", None) process.pop("cmdline", None) - cwd = process.pop("cwd", None) - if cwd is not None: - found_cwd = True - fd = process.pop("fd", None) - if fd is not None: - found_fd = True + process.pop("working_directory", None) + process.pop("fd", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) From 83b84e7eec870e0e65e4e9d2d648bb2c5dd464e8 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 29 Jul 2020 17:04:58 +0200 Subject: [PATCH 3/5] Actually fix check on working directory --- metricbeat/module/system/test_system.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 1ea11d5bde6..44775dba7c9 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -403,7 +403,6 @@ def test_process(self): # Remove optional keys. process.pop("cgroup", None) process.pop("fd", None) - process.pop("working_directory", None) process.pop("cmdline", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) @@ -449,10 +448,11 @@ def test_process_unix(self): found_env = False found_cwd = not sys.platform.startswith("linux") for evt in output: + found_cwd |= "working_directory" in evt["process"] + process = evt["system"]["process"] found_fd |= "fd" in process found_env |= "env" in process - found_cwd |= "working_directory" in process # Remove 'env' prior to checking documented fields because its keys are dynamic. env = process.pop("env", None) @@ -461,13 +461,13 @@ def test_process_unix(self): # Remove optional keys. process.pop("cgroup", None) process.pop("cmdline", None) - process.pop("working_directory", None) process.pop("fd", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) self.assertTrue(found_fd, "fd not found in any process events") self.assertTrue(found_env, "env not found in any process events") + self.assertTrue(found_cwd, "working_directory not found in any process events") @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process_metricbeat(self): From 8c1ece9a1f85ca04b005ccc30763bebb7713dbfc Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 29 Jul 2020 17:24:03 +0200 Subject: [PATCH 4/5] Fix format in python test --- metricbeat/module/system/test_system.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 44775dba7c9..e5dbec73f96 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -416,17 +416,18 @@ def test_process_unix(self): """ import getpass - self.render_config_template(modules=[{ - "name": "system", - "metricsets": ["process"], - "period": "5s", - "extras": { - "process.env.whitelist": ["PATH"], - "process.include_cpu_ticks": True, - - # Remove 'percpu' prior to checking documented fields because its keys are dynamic. - "process.include_per_cpu": False, - }, + self.render_config_template( + modules=[{ + "name": "system", + "metricsets": ["process"], + "period": "5s", + "extras": { + "process.env.whitelist": ["PATH"], + "process.include_cpu_ticks": True, + + # Remove 'percpu' prior to checking documented fields because its keys are dynamic. + "process.include_per_cpu": False, + }, }], # Some info is only guaranteed in processes with permissions, check # only on own processes. From 113909ecb53580d7d8b1f9ceb74bf564e1e15f8f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 29 Jul 2020 20:34:20 +0200 Subject: [PATCH 5/5] Skip check on darwin --- metricbeat/module/system/test_system.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index e5dbec73f96..82e59e71af4 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -466,7 +466,9 @@ def test_process_unix(self): self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) - self.assertTrue(found_fd, "fd not found in any process events") + if not sys.platform.startswith("darwin"): + self.assertTrue(found_fd, "fd not found in any process events") + self.assertTrue(found_env, "env not found in any process events") self.assertTrue(found_cwd, "working_directory not found in any process events")