Skip to content
This repository has been archived by the owner on Feb 8, 2024. It is now read-only.

Commit

Permalink
CORTX-33037: Apply support bundle filter limit_size
Browse files Browse the repository at this point in the history 
Signed-off-by: Shriya Deshmukh <shriya.deshmukh@seagate.com>
  • Loading branch information
@shriya-deshmukh
shriya-deshmukh committed Aug 25, 2022
1 parent caac053 commit 822dec7
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 55 deletions.
3 changes: 3 additions & 0 deletions src/rgw/const.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,9 @@
SVC_THREAD_POOL_SIZE_KEY = f'{COMPONENT_NAME} thread pool size'
SVC_CONCURRENT_MAX_REQ_KEY = f'{COMPONENT_NAME} max concurrent request'

# Support Bundle
SVC_FILE_PERCENTAGE = 95

class RgwEndpoint(Enum):
"""Enum class to define rgw endpoints provided by hare."""

Expand Down
201 changes: 146 additions & 55 deletions src/rgw/support/rgw_support_bundle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ from cortx.utils.process import SimpleProcess
from cortx.utils.conf_store.conf_store import Conf, MappedConf
from cortx.utils.support_framework.log_filters import FilterLog
from cortx.rgw.const import (
CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE, RGW_CORE_FILE_DIR_NAME)
CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE,
SVC_FILE_PERCENTAGE)


class SupportBundleError(BaseError):
Expand All @@ -51,7 +52,7 @@ class RGWSupportBundle:
Log.info(f"{COMPONENT_NAME} support bundle generation started!!")
coredumps = filters.get('coredumps', False)
stacktrace = filters.get('stacktrace', False)
duration = filters.get('duration',False)
duration = filters.get('duration', False)
size_limit = filters.get('size_limit')
machine_id = Conf.machine_id
cortx_config_store = MappedConf(cluster_conf)
Expand All @@ -65,11 +66,19 @@ class RGWSupportBundle:
os.makedirs(RGWSupportBundle._tmp_src, exist_ok=True)
# copy configuration files
if os.path.exists(config_file):
shutil.copyfile(config_file,\
shutil.copyfile(config_file,
os.path.join(RGWSupportBundle._tmp_src, RGW_CONF_FILE))
else:
Log.error(f"{config_file} is not present."
f"Skipping the config file collection for {COMPONENT_NAME}")
Log.error(f"{config_file} is not present. Skipping the config file"
f" collection for {COMPONENT_NAME}")

# add cortx components rpm version
cmd = "rpm -qa | grep cortx"
output, _, rc = SimpleProcess(cmd).run()
if rc == 0:
infile = os.path.join(RGWSupportBundle._tmp_src, 'cortx-rpms')
with open(infile, 'w') as fin:
fin.write(output.decode("utf-8"))

# copy rgw client log files
log_dir = os.path.join(log_base, f'rgw/{machine_id}')
Expand All @@ -78,7 +87,12 @@ class RGWSupportBundle:
for file in os.listdir(log_dir):
if regex.match(file):
infile = os.path.join(log_dir, file)
outfile = os.path.join(RGWSupportBundle._tmp_src, file)
if size_limit:
tmp_client_log_dir = os.path.join('/tmp', 'rgw_log')
os.makedirs(tmp_client_log_dir, exist_ok=True)
outfile = os.path.join(tmp_client_log_dir, file)
else:
outfile = os.path.join(RGWSupportBundle._tmp_src, file)
if os.path.isfile(infile):
shutil.copyfile(infile, outfile)

Expand All @@ -88,7 +102,7 @@ class RGWSupportBundle:
# files in delim_T use timestamp syntax as
# 2022-07-10T05:28:35.570+0000
from cortx.utils.support_framework.log_filters import FilterLog
from cortx.rgw.const import ( RGW_STARTUP_LOG, RGW_SETUP_LOG,
from cortx.rgw.const import (RGW_STARTUP_LOG, RGW_SETUP_LOG,
RGW_SUPPORT_BUNDLE_LOG, RADOSGW_ADMIN_LOG, RGW_1_LOG,
LOG_DATE_REGEX, LOG_TIME_REGEX, DATETIME_DATE_REGEX,
DATETIME_TIME_REGEX)
Expand Down Expand Up @@ -119,51 +133,94 @@ class RGWSupportBundle:
else:
Log.error("RGW log file does not exists hence skipping log file collection.")

# remaining size limit in folder
remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
RGWSupportBundle._tmp_src, size_limit)
# apply truncate logic on all textual logs
if size_limit:
from cortx.utils.support_framework.log_filters import FilterLog
tmp_path_for_copy = os.path.join(RGWSupportBundle._tmp_src, 'tmp_dir')
FilterLog.limit_size(
src_dir=RGWSupportBundle._tmp_src,
dest_dir= tmp_path_for_copy,
size=remaining_size_limit,
file_name_reg_ex='rgw*')
remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
RGWSupportBundle._tmp_src, size_limit)
FilterLog.limit_size(
src_dir=RGWSupportBundle._tmp_src,
dest_dir= tmp_path_for_copy,
size=remaining_size_limit,
file_name_reg_ex='radosgw*')

for file in os.listdir(tmp_path_for_copy):
shutil.move(os.path.join(tmp_path_for_copy, file),
os.path.join(RGWSupportBundle._tmp_src, file))
shutil.rmtree(tmp_path_for_copy)

# copy addb logs
addb_log_path = os.path.join(log_dir, 'addb_files-*')
addb_dirs = glob.glob(addb_log_path)
for addb_dir in addb_dirs:
shutil.copytree(addb_dir,
os.path.join(RGWSupportBundle._tmp_src, addb_dir.split('/')[-1]))
Log.info(f'Size_limit allocated to {COMPONENT_NAME} is {size_limit}')
list_of_files = filter(lambda f: os.path.isfile(os.path.join(
tmp_client_log_dir, f)), os.listdir(tmp_client_log_dir))
list_of_files = sorted(list_of_files,
key=lambda f: os.path.getmtime(os.path.join(tmp_client_log_dir, f)),
reverse=True)
size_limit_in_byte = FilterLog._get_size_in_bytes(size_limit)
dest_dir_size = RGWSupportBundle._get_dir_size(
RGWSupportBundle._tmp_src)
client_log_dir_size = RGWSupportBundle._get_dir_size(tmp_client_log_dir)
Log.info(f'Total size of textual log files: {client_log_dir_size}')
remaining_size_limit = size_limit_in_byte - dest_dir_size
Log.info(f'Remaining size_limit for textual log : {remaining_size_limit}')
if (remaining_size_limit >= client_log_dir_size):
for file_regex in ['rgw*', 'radosgw*']:
RGWSupportBundle._apply_size_filter(
tmp_client_log_dir, size_limit, file_regex)
else:
tmp_svc_files_dir = os.path.join('/tmp', 'rgw_svc_log')
os.makedirs(tmp_svc_files_dir, exist_ok=True)
for file in os.listdir(tmp_client_log_dir):
if file.startswith('rgw-') and file.endswith('.log'):
shutil.move(os.path.join(tmp_client_log_dir, file),
os.path.join(tmp_svc_files_dir, file))
# Allocate 95% of size from total size_limit to
# rgw svc log(rgw-1.log) files
allocated_size = str(((
size_limit_in_byte * SVC_FILE_PERCENTAGE)/100)) + 'B'
Log.info(f'Allocated size for {COMPONENT_NAME} svc files:{allocated_size}')
RGWSupportBundle._apply_size_filter(
tmp_svc_files_dir, allocated_size, 'rgw-*',
check_remaining_size=False)
RGWSupportBundle._copy_compressed_files(tmp_client_log_dir,
allocated_size)
RGWSupportBundle._apply_size_filter(
tmp_client_log_dir, size_limit, 'rgw_*')
RGWSupportBundle._apply_size_filter(
tmp_client_log_dir, size_limit, 'radosgw*')
shutil.rmtree(tmp_svc_files_dir)
shutil.rmtree(tmp_client_log_dir)

# copy motr trace log files
motr_trace_dir = os.path.join(log_dir, 'motr_trace_files')
if os.path.exists(motr_trace_dir):
tmp_motr_trace_dir = os.path.join(RGWSupportBundle._tmp_src, 'motr_trace_files')
# include the latest 5 log files of motr traces in support bundle
latest_files = RGWSupportBundle._get_latest_files(motr_trace_dir)
latest_files = RGWSupportBundle._get_latest_files(directory=motr_trace_dir)
Log.info('Bundle motr trace files')
for file in latest_files:
infile = os.path.join(motr_trace_dir, file)
tmp_motr_trace_dir = os.path.join(RGWSupportBundle._tmp_src, 'motr_trace_files')
os.makedirs(tmp_motr_trace_dir, exist_ok=True)
outfile = os.path.join(tmp_motr_trace_dir, file)
# Convert m0trace file to human readable yaml format
SimpleProcess(f'm0trace -i {infile} -Y -o {outfile}.yaml').run()
SimpleProcess(f'xz {outfile}.yaml').run() # compress the output file
if size_limit:
remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
RGWSupportBundle._tmp_src, size_limit)
if (os.path.getsize(infile) >= FilterLog._get_size_in_bytes(
remaining_size_limit)):
Log.info('size_limit is full.')
break
os.makedirs(tmp_motr_trace_dir, exist_ok=True)
# Convert m0trace file to human readable yaml format
SimpleProcess(f'm0trace -i {infile} -Y -o {outfile}.yaml').run()
SimpleProcess(f'xz {outfile}.yaml').run() # compress the output file
else:
os.makedirs(tmp_motr_trace_dir, exist_ok=True)
# Convert m0trace file to human readable yaml format
SimpleProcess(f'm0trace -i {infile} -Y -o {outfile}.yaml').run()
SimpleProcess(f'xz {outfile}.yaml').run() # compress the output file

# copy addb logs
addb_log_path = os.path.join(log_dir, 'addb_files-*')
addb_dirs = glob.glob(addb_log_path)
for addb_dir in addb_dirs:
Log.info('Bundle addb files.')
if size_limit:
remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
RGWSupportBundle._tmp_src, size_limit)
if (os.path.getsize(addb_dir) >= FilterLog._get_size_in_bytes(
remaining_size_limit)):
Log.info('size_limit is full.')
break
shutil.copytree(addb_dir, os.path.join(
RGWSupportBundle._tmp_src, addb_dir.split('/')[-1]))
else:
shutil.copytree(addb_dir, os.path.join(
RGWSupportBundle._tmp_src, addb_dir.split('/')[-1]))

if stacktrace:
from subprocess import check_output
Expand Down Expand Up @@ -211,19 +268,50 @@ class RGWSupportBundle:
else:
shutil.copytree(crash_dump_dir, dest_dump_dir)

# add cortx components rpm version
cmd = "rpm -qa | grep cortx"
output, _, rc = SimpleProcess(cmd).run()
if rc == 0:
infile = os.path.join(RGWSupportBundle._tmp_src, 'cortx-rpms')
with open(infile, 'w') as fin:
fin.write(output.decode("utf-8"))
RGWSupportBundle._generate_tar(bundle_id, target_path)
RGWSupportBundle._cleanup()

@staticmethod
def _get_remaining_folder_size(directory:str, size_limit:str) -> str:
"""Returns size of directory in bytes. eg '3B', '100B', '0B'."""
def _get_dir_size(dir_path: str):
"""Calculate total directory size."""
dir_size = 0
for path, _, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(path, file)
dir_size += os.path.getsize(file_path)
return dir_size

@staticmethod
def _apply_size_filter(src_dir: str, size_limit: str, file_name_reg_ex: str,
check_remaining_size: bool = True):
"""Apply limit_size filter on given files."""
Log.info(f'Bundle {file_name_reg_ex} files.')
remaining_size_limit = size_limit
if check_remaining_size:
remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
RGWSupportBundle._tmp_src, size_limit)
if remaining_size_limit == '0B':
Log.info(f'size_limit is full for {file_name_reg_ex} files.')
return
FilterLog.limit_size(src_dir=src_dir, dest_dir=RGWSupportBundle._tmp_src,
size=remaining_size_limit, file_name_reg_ex=file_name_reg_ex)

@staticmethod
def _copy_compressed_files(src_dir: str, size_limit: str):
"""Copy compressed log files in support bundle."""
for file_path in glob.glob(os.path.join(src_dir, 'rgw-*.gz')):
remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
RGWSupportBundle._tmp_src, size_limit)
file_size = os.path.getsize(file_path)
if FilterLog._get_size_in_bytes(remaining_size_limit) < file_size:
Log.info('size_limit is full.')
break
shutil.move(file_path, os.path.join(
RGWSupportBundle._tmp_src, os.path.basename(file_path)))

@staticmethod
def _get_remaining_folder_size(directory: str, size_limit: str) -> str:
"""Returns remaining size of log directory in bytes. eg '3B', '100B', '0B'."""
current_size = 0
for path, _, files in os.walk(directory):
for file in files:
Expand All @@ -237,11 +325,13 @@ class RGWSupportBundle:
return str(remaining_size_in_byte) + 'B'

@staticmethod
def _get_latest_files(directory:str,number_of_files:int=5) -> list:
all_files = filter(lambda f: os.path.isfile(os.path.join(directory, f)),
def _get_latest_files(directory: str, number_of_files: int = 5) -> list:
all_files = filter(
lambda f: os.path.isfile(os.path.join(directory, f)),
os.listdir(directory))
# sort the files based on last modification time
sorted_files = sorted(all_files,
sorted_files = sorted(
all_files,
key=lambda f: os.path.getmtime(os.path.join(directory, f)),
reverse=True)
return sorted_files[0:number_of_files]
Expand All @@ -255,7 +345,8 @@ class RGWSupportBundle:
if not os.path.exists(target_path):
os.makedirs(target_path)
with tarfile.open(tar_file_name, 'w:gz') as tar:
tar.add(RGWSupportBundle._tmp_src,
tar.add(
RGWSupportBundle._tmp_src,
arcname=os.path.basename(RGWSupportBundle._tmp_src))

@staticmethod
Expand Down Expand Up @@ -284,7 +375,7 @@ class RGWSupportBundle:
parser.add_argument('--coredumps', type=RGWSupportBundle.str2bool, default=False, dest='coredumps',
help="Include/Exclude Coredumps, Default = False")
parser.add_argument('--stacktrace', type=RGWSupportBundle.str2bool, default=False, dest='stacktrace',
help="Include/Exclude stacktrace, Default = False")
help="Include/Exclude stacktrasce, Default = False")
parser.add_argument('--modules', dest='modules',
help="list of components & services to generate support bundle.")
parser.add_argument('--all', type=RGWSupportBundle.str2bool, default=False,
Expand Down

0 comments on commit 822dec7

Please sign in to comment.