From 97ef47857985c7e59f284fc248d2490aae8e947e Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 27 Nov 2024 16:02:29 +0800 Subject: [PATCH 1/3] Dev: report: Add ~/.config/crm/crm.conf to the list of collected files --- crmsh/config.py | 9 +++++++-- crmsh/report/core.py | 2 +- etc/crm.conf.in | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crmsh/config.py b/crmsh/config.py index 6979760d2..e0e7f9bae 100644 --- a/crmsh/config.py +++ b/crmsh/config.py @@ -298,8 +298,13 @@ def get(self, value): 'from_time': opt_string('-12H'), 'compress': opt_boolean('yes'), 'speed_up': opt_boolean('no'), - 'collect_extra_logs': opt_string('/var/log/messages \ - /var/log/crmsh/crmsh.log /etc/crm/profiles.yml /etc/crm/crm.conf'), + 'collect_extra_logs': opt_list([ + '/var/log/messages', + '/var/log/crmsh/crmsh.log', + '/etc/crm/profiles.yml', + '/etc/crm/crm.conf', + '~/.config/crm/crm.conf' + ]), 'remove_exist_dest': opt_boolean('no'), 'single_node': opt_boolean('no'), 'sanitize_rule': opt_string('passw.*'), diff --git a/crmsh/report/core.py b/crmsh/report/core.py index b98152d08..e0b53422e 100644 --- a/crmsh/report/core.py +++ b/crmsh/report/core.py @@ -40,7 +40,7 @@ def load(self) -> None: self.to_time: float = utils.now() self.no_compress: bool = not config.report.compress self.speed_up: bool = config.report.speed_up - self.extra_log_list: List[str] = config.report.collect_extra_logs.split() + self.extra_log_list: List[str] = config.report.collect_extra_logs self.rm_exist_dest: bool = config.report.remove_exist_dest self.single: bool= config.report.single_node self.sensitive_regex_list: List[str] = [] diff --git a/etc/crm.conf.in b/etc/crm.conf.in index 3570985c2..f7aefee14 100644 --- a/etc/crm.conf.in +++ b/etc/crm.conf.in @@ -83,7 +83,7 @@ ocf_root = @OCF_ROOT_DIR@ ; from_time = -12H ; compress = yes ; speed_up = no -; collect_extra_logs = /var/log/messages /var/log/pacemaker.log +; collect_extra_logs = /var/log/messages, /var/log/crmsh/crmsh.log, /etc/crm/profiles.yml, /etc/crm/crm.conf, ~/.config/crm/crm.conf ; remove_exist_dest = no ; single_node = no ; From cf2b6404a7f69955eff7b5f701048e12f47b31ab Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 27 Nov 2024 21:44:57 +0800 Subject: [PATCH 2/3] Dev: report: Handle collect files with the same name Create {basename}.d directory in report result and put those files in it. Then files like `/etc/crm/crm.conf` and `/root/.config/crm/crm.conf` will under `crm.conf.d` directory. --- crmsh/report/collect.py | 6 ++++-- crmsh/report/utils.py | 25 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crmsh/report/collect.py b/crmsh/report/collect.py index 3e76388ee..ed4bf1722 100644 --- a/crmsh/report/collect.py +++ b/crmsh/report/collect.py @@ -65,9 +65,11 @@ def collect_ha_logs(context: core.Context) -> None: Collect pacemaker, corosync and extra logs """ log_list = [get_pcmk_log(), get_corosync_log()] + context.extra_log_list - for log in log_list: + log_list = [os.path.expanduser(log) for log in log_list] + log_list_marked_same_basename = utils.mark_duplicate_basenames(log_list) + for log, same_basename in log_list_marked_same_basename: if log and os.path.isfile(log): - utils.dump_logset(context, log) + utils.dump_logset(context, log, create_dir=same_basename) def collect_journal_logs(context: core.Context) -> None: diff --git a/crmsh/report/utils.py b/crmsh/report/utils.py index 2b5d0e675..4ffd34469 100644 --- a/crmsh/report/utils.py +++ b/crmsh/report/utils.py @@ -10,6 +10,7 @@ import traceback from dateutil import tz from enum import Enum +from collections import Counter from typing import Optional, List, Tuple from crmsh import utils as crmutils @@ -228,7 +229,7 @@ def get_distro_info() -> str: return res.group(1) if res else "Unknown" -def dump_logset(context: core.Context, logf: str) -> None: +def dump_logset(context: core.Context, logf: str, create_dir: bool = False) -> None: """ Dump the log set into the specified output file """ @@ -255,7 +256,17 @@ def dump_logset(context: core.Context, logf: str) -> None: out_string += print_logseg(newest, 0, context.to_time) if out_string: - outf = os.path.join(context.work_dir, os.path.basename(logf)) + if create_dir: + basename, dirname = os.path.basename(logf), os.path.dirname(logf) + dest_dir = os.path.join( + context.work_dir, + f'{basename}.d', + dirname.lstrip('/') + ) + crmutils.mkdirp(dest_dir) + outf = os.path.join(dest_dir, basename) + else: + outf = os.path.join(context.work_dir, os.path.basename(logf)) crmutils.str2file(out_string.strip('\n'), outf) logger.debug(f"Dump {logf} into {real_path(outf)}") @@ -775,4 +786,14 @@ def print_traceback(): def real_path(target_file: str) -> str: return '/'.join(target_file.split('/')[3:]) + + +def mark_duplicate_basenames(file_path_list: List[str]) -> List[Tuple[str, bool]]: + """ + Mark the paths with the same basename + Return a list of tuples, each tuple contains the path and a boolean value + indicating whether the path has the same basename with others + """ + counter = Counter([os.path.basename(path) for path in file_path_list]) + return [(path, counter[os.path.basename(path)] > 1) for path in file_path_list] # vim:ts=4:sw=4:et: From a0fba5910b1caed34c11ffbe333a49f3608255bd Mon Sep 17 00:00:00 2001 From: xin liang Date: Wed, 27 Nov 2024 22:31:50 +0800 Subject: [PATCH 3/3] Dev: unittests: Adjust unit test for previous commit --- test/unittests/test_report_collect.py | 11 ++++++++--- test/unittests/test_report_core.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/unittests/test_report_collect.py b/test/unittests/test_report_collect.py index e13443236..7f57dac50 100644 --- a/test/unittests/test_report_collect.py +++ b/test/unittests/test_report_collect.py @@ -41,13 +41,18 @@ def test_get_pcmk_log(self, mock_isfile, mock_read, mock_warning): ]) mock_read.assert_called_once_with(constants.PCMKCONF) + @mock.patch('crmsh.report.utils.mark_duplicate_basenames') @mock.patch('crmsh.report.utils.dump_logset') @mock.patch('os.path.isfile') @mock.patch('crmsh.report.collect.get_pcmk_log') @mock.patch('crmsh.report.collect.get_corosync_log') - def test_collect_ha_logs(self, mock_corosync_log, mock_get_log, mock_isfile, mock_dump): + def test_collect_ha_logs(self, mock_corosync_log, mock_get_log, mock_isfile, mock_dump, mock_mark): mock_corosync_log.return_value = "/var/log/cluster/corosync.log" mock_get_log.return_value = "/var/pacemaker.log" + mock_mark.return_value = [ + (mock_get_log.return_value, False), + (mock_corosync_log.return_value, False) + ] mock_isfile.side_effect = [True, True] mock_ctx_inst = mock.Mock(extra_log_list=[]) @@ -59,8 +64,8 @@ def test_collect_ha_logs(self, mock_corosync_log, mock_get_log, mock_isfile, moc mock.call(mock_corosync_log.return_value) ]) mock_dump.assert_has_calls([ - mock.call(mock_ctx_inst, mock_get_log.return_value), - mock.call(mock_ctx_inst, mock_corosync_log.return_value) + mock.call(mock_ctx_inst, mock_get_log.return_value, create_dir=False), + mock.call(mock_ctx_inst, mock_corosync_log.return_value, create_dir=False) ]) @mock.patch('logging.Logger.warning') diff --git a/test/unittests/test_report_core.py b/test/unittests/test_report_core.py index 347169778..e57cd6e65 100644 --- a/test/unittests/test_report_core.py +++ b/test/unittests/test_report_core.py @@ -39,7 +39,7 @@ def setUp(self, mock_config, mock_now, mock_parse_to_timestamp): mock_config.report = mock.Mock( from_time="20230101", compress=False, - collect_extra_logs="file1 file2", + collect_extra_logs=["file1", "file2"], remove_exist_dest=False, single_node=False )