From 822dec71129bdfeb057d32f480f82fcd2a28551b Mon Sep 17 00:00:00 2001 From: Shriya Deshmukh Date: Wed, 24 Aug 2022 21:18:30 -0600 Subject: [PATCH] CORTX-33037: Apply support bundle filter limit_size Signed-off-by: Shriya Deshmukh --- src/rgw/const.py | 3 + src/rgw/support/rgw_support_bundle | 201 +++++++++++++++++++++-------- 2 files changed, 149 insertions(+), 55 deletions(-) diff --git a/src/rgw/const.py b/src/rgw/const.py index 4171c2f..28a9058 100644 --- a/src/rgw/const.py +++ b/src/rgw/const.py @@ -174,6 +174,9 @@ SVC_THREAD_POOL_SIZE_KEY = f'{COMPONENT_NAME} thread pool size' SVC_CONCURRENT_MAX_REQ_KEY = f'{COMPONENT_NAME} max concurrent request' +# Support Bundle +SVC_FILE_PERCENTAGE = 95 + class RgwEndpoint(Enum): """Enum class to define rgw endpoints provided by hare.""" diff --git a/src/rgw/support/rgw_support_bundle b/src/rgw/support/rgw_support_bundle index 3bec359..4913583 100755 --- a/src/rgw/support/rgw_support_bundle +++ b/src/rgw/support/rgw_support_bundle @@ -31,7 +31,8 @@ from cortx.utils.process import SimpleProcess from cortx.utils.conf_store.conf_store import Conf, MappedConf from cortx.utils.support_framework.log_filters import FilterLog from cortx.rgw.const import ( - CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE, RGW_CORE_FILE_DIR_NAME) + CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE, + SVC_FILE_PERCENTAGE) class SupportBundleError(BaseError): @@ -51,7 +52,7 @@ class RGWSupportBundle: Log.info(f"{COMPONENT_NAME} support bundle generation started!!") coredumps = filters.get('coredumps', False) stacktrace = filters.get('stacktrace', False) - duration = filters.get('duration',False) + duration = filters.get('duration', False) size_limit = filters.get('size_limit') machine_id = Conf.machine_id cortx_config_store = MappedConf(cluster_conf) @@ -65,11 +66,19 @@ class RGWSupportBundle: os.makedirs(RGWSupportBundle._tmp_src, exist_ok=True) # copy configuration files if os.path.exists(config_file): - shutil.copyfile(config_file,\ + shutil.copyfile(config_file, os.path.join(RGWSupportBundle._tmp_src, RGW_CONF_FILE)) else: - Log.error(f"{config_file} is not present." - f"Skipping the config file collection for {COMPONENT_NAME}") + Log.error(f"{config_file} is not present. Skipping the config file" + f" collection for {COMPONENT_NAME}") + + # add cortx components rpm version + cmd = "rpm -qa | grep cortx" + output, _, rc = SimpleProcess(cmd).run() + if rc == 0: + infile = os.path.join(RGWSupportBundle._tmp_src, 'cortx-rpms') + with open(infile, 'w') as fin: + fin.write(output.decode("utf-8")) # copy rgw client log files log_dir = os.path.join(log_base, f'rgw/{machine_id}') @@ -78,7 +87,12 @@ class RGWSupportBundle: for file in os.listdir(log_dir): if regex.match(file): infile = os.path.join(log_dir, file) - outfile = os.path.join(RGWSupportBundle._tmp_src, file) + if size_limit: + tmp_client_log_dir = os.path.join('/tmp', 'rgw_log') + os.makedirs(tmp_client_log_dir, exist_ok=True) + outfile = os.path.join(tmp_client_log_dir, file) + else: + outfile = os.path.join(RGWSupportBundle._tmp_src, file) if os.path.isfile(infile): shutil.copyfile(infile, outfile) @@ -88,7 +102,7 @@ class RGWSupportBundle: # files in delim_T use timestamp syntax as # 2022-07-10T05:28:35.570+0000 from cortx.utils.support_framework.log_filters import FilterLog - from cortx.rgw.const import ( RGW_STARTUP_LOG, RGW_SETUP_LOG, + from cortx.rgw.const import (RGW_STARTUP_LOG, RGW_SETUP_LOG, RGW_SUPPORT_BUNDLE_LOG, RADOSGW_ADMIN_LOG, RGW_1_LOG, LOG_DATE_REGEX, LOG_TIME_REGEX, DATETIME_DATE_REGEX, DATETIME_TIME_REGEX) @@ -119,51 +133,94 @@ class RGWSupportBundle: else: Log.error("RGW log file does not exists hence skipping log file collection.") - # remaining size limit in folder - remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( - RGWSupportBundle._tmp_src, size_limit) # apply truncate logic on all textual logs if size_limit: from cortx.utils.support_framework.log_filters import FilterLog - tmp_path_for_copy = os.path.join(RGWSupportBundle._tmp_src, 'tmp_dir') - FilterLog.limit_size( - src_dir=RGWSupportBundle._tmp_src, - dest_dir= tmp_path_for_copy, - size=remaining_size_limit, - file_name_reg_ex='rgw*') - remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( - RGWSupportBundle._tmp_src, size_limit) - FilterLog.limit_size( - src_dir=RGWSupportBundle._tmp_src, - dest_dir= tmp_path_for_copy, - size=remaining_size_limit, - file_name_reg_ex='radosgw*') - - for file in os.listdir(tmp_path_for_copy): - shutil.move(os.path.join(tmp_path_for_copy, file), - os.path.join(RGWSupportBundle._tmp_src, file)) - shutil.rmtree(tmp_path_for_copy) - - # copy addb logs - addb_log_path = os.path.join(log_dir, 'addb_files-*') - addb_dirs = glob.glob(addb_log_path) - for addb_dir in addb_dirs: - shutil.copytree(addb_dir, - os.path.join(RGWSupportBundle._tmp_src, addb_dir.split('/')[-1])) + Log.info(f'Size_limit allocated to {COMPONENT_NAME} is {size_limit}') + list_of_files = filter(lambda f: os.path.isfile(os.path.join( + tmp_client_log_dir, f)), os.listdir(tmp_client_log_dir)) + list_of_files = sorted(list_of_files, + key=lambda f: os.path.getmtime(os.path.join(tmp_client_log_dir, f)), + reverse=True) + size_limit_in_byte = FilterLog._get_size_in_bytes(size_limit) + dest_dir_size = RGWSupportBundle._get_dir_size( + RGWSupportBundle._tmp_src) + client_log_dir_size = RGWSupportBundle._get_dir_size(tmp_client_log_dir) + Log.info(f'Total size of textual log files: {client_log_dir_size}') + remaining_size_limit = size_limit_in_byte - dest_dir_size + Log.info(f'Remaining size_limit for textual log : {remaining_size_limit}') + if (remaining_size_limit >= client_log_dir_size): + for file_regex in ['rgw*', 'radosgw*']: + RGWSupportBundle._apply_size_filter( + tmp_client_log_dir, size_limit, file_regex) + else: + tmp_svc_files_dir = os.path.join('/tmp', 'rgw_svc_log') + os.makedirs(tmp_svc_files_dir, exist_ok=True) + for file in os.listdir(tmp_client_log_dir): + if file.startswith('rgw-') and file.endswith('.log'): + shutil.move(os.path.join(tmp_client_log_dir, file), + os.path.join(tmp_svc_files_dir, file)) + # Allocate 95% of size from total size_limit to + # rgw svc log(rgw-1.log) files + allocated_size = str((( + size_limit_in_byte * SVC_FILE_PERCENTAGE)/100)) + 'B' + Log.info(f'Allocated size for {COMPONENT_NAME} svc files:{allocated_size}') + RGWSupportBundle._apply_size_filter( + tmp_svc_files_dir, allocated_size, 'rgw-*', + check_remaining_size=False) + RGWSupportBundle._copy_compressed_files(tmp_client_log_dir, + allocated_size) + RGWSupportBundle._apply_size_filter( + tmp_client_log_dir, size_limit, 'rgw_*') + RGWSupportBundle._apply_size_filter( + tmp_client_log_dir, size_limit, 'radosgw*') + shutil.rmtree(tmp_svc_files_dir) + shutil.rmtree(tmp_client_log_dir) # copy motr trace log files motr_trace_dir = os.path.join(log_dir, 'motr_trace_files') if os.path.exists(motr_trace_dir): + tmp_motr_trace_dir = os.path.join(RGWSupportBundle._tmp_src, 'motr_trace_files') # include the latest 5 log files of motr traces in support bundle - latest_files = RGWSupportBundle._get_latest_files(motr_trace_dir) + latest_files = RGWSupportBundle._get_latest_files(directory=motr_trace_dir) + Log.info('Bundle motr trace files') for file in latest_files: infile = os.path.join(motr_trace_dir, file) - tmp_motr_trace_dir = os.path.join(RGWSupportBundle._tmp_src, 'motr_trace_files') - os.makedirs(tmp_motr_trace_dir, exist_ok=True) outfile = os.path.join(tmp_motr_trace_dir, file) - # Convert m0trace file to human readable yaml format - SimpleProcess(f'm0trace -i {infile} -Y -o {outfile}.yaml').run() - SimpleProcess(f'xz {outfile}.yaml').run() # compress the output file + if size_limit: + remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( + RGWSupportBundle._tmp_src, size_limit) + if (os.path.getsize(infile) >= FilterLog._get_size_in_bytes( + remaining_size_limit)): + Log.info('size_limit is full.') + break + os.makedirs(tmp_motr_trace_dir, exist_ok=True) + # Convert m0trace file to human readable yaml format + SimpleProcess(f'm0trace -i {infile} -Y -o {outfile}.yaml').run() + SimpleProcess(f'xz {outfile}.yaml').run() # compress the output file + else: + os.makedirs(tmp_motr_trace_dir, exist_ok=True) + # Convert m0trace file to human readable yaml format + SimpleProcess(f'm0trace -i {infile} -Y -o {outfile}.yaml').run() + SimpleProcess(f'xz {outfile}.yaml').run() # compress the output file + + # copy addb logs + addb_log_path = os.path.join(log_dir, 'addb_files-*') + addb_dirs = glob.glob(addb_log_path) + for addb_dir in addb_dirs: + Log.info('Bundle addb files.') + if size_limit: + remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( + RGWSupportBundle._tmp_src, size_limit) + if (os.path.getsize(addb_dir) >= FilterLog._get_size_in_bytes( + remaining_size_limit)): + Log.info('size_limit is full.') + break + shutil.copytree(addb_dir, os.path.join( + RGWSupportBundle._tmp_src, addb_dir.split('/')[-1])) + else: + shutil.copytree(addb_dir, os.path.join( + RGWSupportBundle._tmp_src, addb_dir.split('/')[-1])) if stacktrace: from subprocess import check_output @@ -211,19 +268,50 @@ class RGWSupportBundle: else: shutil.copytree(crash_dump_dir, dest_dump_dir) - # add cortx components rpm version - cmd = "rpm -qa | grep cortx" - output, _, rc = SimpleProcess(cmd).run() - if rc == 0: - infile = os.path.join(RGWSupportBundle._tmp_src, 'cortx-rpms') - with open(infile, 'w') as fin: - fin.write(output.decode("utf-8")) RGWSupportBundle._generate_tar(bundle_id, target_path) RGWSupportBundle._cleanup() @staticmethod - def _get_remaining_folder_size(directory:str, size_limit:str) -> str: - """Returns size of directory in bytes. eg '3B', '100B', '0B'.""" + def _get_dir_size(dir_path: str): + """Calculate total directory size.""" + dir_size = 0 + for path, _, files in os.walk(dir_path): + for file in files: + file_path = os.path.join(path, file) + dir_size += os.path.getsize(file_path) + return dir_size + + @staticmethod + def _apply_size_filter(src_dir: str, size_limit: str, file_name_reg_ex: str, + check_remaining_size: bool = True): + """Apply limit_size filter on given files.""" + Log.info(f'Bundle {file_name_reg_ex} files.') + remaining_size_limit = size_limit + if check_remaining_size: + remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( + RGWSupportBundle._tmp_src, size_limit) + if remaining_size_limit == '0B': + Log.info(f'size_limit is full for {file_name_reg_ex} files.') + return + FilterLog.limit_size(src_dir=src_dir, dest_dir=RGWSupportBundle._tmp_src, + size=remaining_size_limit, file_name_reg_ex=file_name_reg_ex) + + @staticmethod + def _copy_compressed_files(src_dir: str, size_limit: str): + """Copy compressed log files in support bundle.""" + for file_path in glob.glob(os.path.join(src_dir, 'rgw-*.gz')): + remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( + RGWSupportBundle._tmp_src, size_limit) + file_size = os.path.getsize(file_path) + if FilterLog._get_size_in_bytes(remaining_size_limit) < file_size: + Log.info('size_limit is full.') + break + shutil.move(file_path, os.path.join( + RGWSupportBundle._tmp_src, os.path.basename(file_path))) + + @staticmethod + def _get_remaining_folder_size(directory: str, size_limit: str) -> str: + """Returns remaining size of log directory in bytes. eg '3B', '100B', '0B'.""" current_size = 0 for path, _, files in os.walk(directory): for file in files: @@ -237,11 +325,13 @@ class RGWSupportBundle: return str(remaining_size_in_byte) + 'B' @staticmethod - def _get_latest_files(directory:str,number_of_files:int=5) -> list: - all_files = filter(lambda f: os.path.isfile(os.path.join(directory, f)), + def _get_latest_files(directory: str, number_of_files: int = 5) -> list: + all_files = filter( + lambda f: os.path.isfile(os.path.join(directory, f)), os.listdir(directory)) # sort the files based on last modification time - sorted_files = sorted(all_files, + sorted_files = sorted( + all_files, key=lambda f: os.path.getmtime(os.path.join(directory, f)), reverse=True) return sorted_files[0:number_of_files] @@ -255,7 +345,8 @@ class RGWSupportBundle: if not os.path.exists(target_path): os.makedirs(target_path) with tarfile.open(tar_file_name, 'w:gz') as tar: - tar.add(RGWSupportBundle._tmp_src, + tar.add( + RGWSupportBundle._tmp_src, arcname=os.path.basename(RGWSupportBundle._tmp_src)) @staticmethod @@ -284,7 +375,7 @@ class RGWSupportBundle: parser.add_argument('--coredumps', type=RGWSupportBundle.str2bool, default=False, dest='coredumps', help="Include/Exclude Coredumps, Default = False") parser.add_argument('--stacktrace', type=RGWSupportBundle.str2bool, default=False, dest='stacktrace', - help="Include/Exclude stacktrace, Default = False") + help="Include/Exclude stacktrasce, Default = False") parser.add_argument('--modules', dest='modules', help="list of components & services to generate support bundle.") parser.add_argument('--all', type=RGWSupportBundle.str2bool, default=False,