diff --git a/setup.py b/setup.py index 55aaf81..7130276 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ mini_prov_files = glob.glob('./src/setup/*.py') logrotate_tmpl_file = 'src/rgw/setup/templates/rgw.logrotate.tmpl' logrotate_service_tmpl = 'src/rgw/setup/templates/logrotate.service.tmpl' +core_file_logrotate_tmpl = 'src/rgw/setup/templates/rgw_core_logrotate.sh' data_file_list=[ ('%s/mini-provisioner' % RGW_INSTALL_PATH, mini_prov_files), ('%s/bin' % RGW_INSTALL_PATH, @@ -45,7 +46,7 @@ ('%s/mini-provisioner' % RGW_INSTALL_PATH,['VERSION']), ('%s/conf' % RGW_INSTALL_PATH,['conf/cortx_rgw.conf', logrotate_tmpl_file, 'src/rgw/support/support.yaml', - logrotate_service_tmpl]) + logrotate_service_tmpl, core_file_logrotate_tmpl]) ] # Add addb plugin to rpm if its generated duirng build process. diff --git a/src/rgw/const.py b/src/rgw/const.py index 8abeb90..4171c2f 100644 --- a/src/rgw/const.py +++ b/src/rgw/const.py @@ -34,6 +34,7 @@ COMPONENT_SVC_NAME = 'rgw_s3' DECRYPTION_KEY = 'cortx' SERVICE_NAME = f'{COMPONENT_NAME}_setup' # rgw_setup +RGW_CORE_FILE_DIR_NAME = f'{COMPONENT_NAME}_debug' # rgw_debug INSTALL_PATH = '/opt/seagate/cortx' RGW_INSTALL_PATH = f'{INSTALL_PATH}/{COMPONENT_NAME}' ADMIN_CREATION_TIMEOUT = 600 @@ -63,6 +64,11 @@ REQUIRED_RPMS = ['cortx-hare', 'cortx-py-utils', 'ceph-radosgw'] ADMIN_PARAMETERS = {'MOTR_ADMIN_FID':'motr admin fid', 'MOTR_ADMIN_ENDPOINT':'motr admin endpoint', 'RGW_FRONTENDS': 'rgw frontends'} +# core files rotate script +CORE_LOGROTATE_FILE_NAME =f'{COMPONENT_NAME}_core_logrotate.sh' # rgw_core_logrotate.sh +CORE_LOGROTATE_FILE = f'{CRON_DIR}/{CORE_LOGROTATE_FILE_NAME}' +CORE_LOGROTATE_TMPL = f'{RGW_INSTALL_PATH}/conf/{CORE_LOGROTATE_FILE_NAME}' + # CORTX cluster confstore keys SVC_NAME_KEY = 'node>%s>components[%s]>services[0]' NUM_COMPONENTS_KEY = 'node>%s>num_components' diff --git a/src/rgw/setup/rgw.py b/src/rgw/setup/rgw.py index 2ea0e4b..4811131 100644 --- a/src/rgw/setup/rgw.py +++ b/src/rgw/setup/rgw.py @@ -151,7 +151,7 @@ def start(conf: MappedConf, index: str): os.makedirs(motr_trace_dir, exist_ok=True) os.makedirs(addb_dir, exist_ok=True) # Create rgw crash file directory - rgw_core_dir = os.path.join(log_path, 'rgw_debug') + rgw_core_dir = os.path.join(log_path, const.RGW_CORE_FILE_DIR_NAME) os.makedirs(rgw_core_dir, exist_ok=True) Log.info('Starting radosgw service.') @@ -395,7 +395,7 @@ def _change_working_dir(conf: MappedConf): """Change current working directory to crash directory path.""" log_path = Rgw._get_log_dir_path(conf) # Create svc crash file directory - svc_core_dir = os.path.join(log_path, 'rgw_debug') + svc_core_dir = os.path.join(log_path, const.RGW_CORE_FILE_DIR_NAME) os.makedirs(svc_core_dir, exist_ok=True) os.chdir(svc_core_dir) @@ -827,6 +827,19 @@ def _logrotate_generic(conf: MappedConf): Log.info(f'{const.LOGROTATE_TMPL} file copied to {const.LOGROTATE_CONF}') except Exception as e: Log.error(f"Failed to configure logrotate for {const.COMPONENT_NAME}. ERROR:{e}") + + # Copy log rotate script for rgw core files into system's logrotate directory. + core_dir_path = os.path.join(log_file_path, const.RGW_CORE_FILE_DIR_NAME) + try: + with open(const.CORE_LOGROTATE_TMPL, 'r') as f: + content = f.read() + content = content.replace('TEMP_CORE_FILE_DIR_PATH', core_dir_path) + with open(const.CORE_LOGROTATE_FILE, 'w') as f: + f.write(content) + Log.info(f'{const.CORE_LOGROTATE_TMPL} file copied to {const.CORE_LOGROTATE_FILE}') + except Exception as e: + Log.error(f"Failed to configure core file's logrotate for {const.COMPONENT_NAME}. ERROR:{e}") + # start cron.d service try: os.system(f"chmod +x {const.CRON_LOGROTATE}") diff --git a/src/rgw/setup/templates/rgw_core_logrotate.sh b/src/rgw/setup/templates/rgw_core_logrotate.sh new file mode 100644 index 0000000..aa2886e --- /dev/null +++ b/src/rgw/setup/templates/rgw_core_logrotate.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# +# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. +# + +# script is used to delete the old rgw core files in /etc/cortx/log/rgw//rgw_debug directory +# script will retain latest 5 core file and remove rest of core files. +# argument1: +# Default number of latest log files is 2 +# ./rgw_core_logrotate.sh -n 2 + +usage() { echo "Usage: bash $(basename "$0")[--help|-h] + [-n maxRgwCoreFileCount] +Retain recent generated rgw core files of given count and remove rest of core files. + +where: +-n number of latest rgw core files to retain (Default count for core files is 2) +--help|-h display this help and exit" 1>&2; exit 1; } + +# max rgw core files count in each rgw log directory +core_files_max_count=2 +rgw_core_dir="TEMP_CORE_FILE_DIR_PATH" + +while getopts ":n:" option; do + case "${option}" in + n) + core_files_max_count=${OPTARG} + if [ -z "${core_files_max_count}" ] + then + usage + fi + ;; + *) + usage + ;; + esac +done + +echo "max RGW core file to be preserved: $core_files_max_count" +echo + +echo "Checking for core files in $rgw_core_dir directory" +if [[ -n "$rgw_core_dir" && -d "$rgw_core_dir" ]] +then + # Find core files + core_files=$(find "$rgw_core_dir" -maxdepth 1 -type f -name "core.*") + core_files_count=$(echo "$core_files" | grep -v "^$" | wc -l) + echo "## found $core_files_count file(s) in log directory($rgw_core_dir) ##" + # check core files count is greater than max core file count or not + if [ "$core_files_count" -gt "$core_files_max_count" ] + then + # get files sort by date - oldest will come on top + remove_file_count=$(("$core_files_count"-"$core_files_max_count")) + if [ "$remove_file_count" -gt 0 ] + then + echo "## ($remove_file_count) rgw core file(s) can be removed from directory :$rgw_core_dir ##" + # get the files sorted by time modified (most recently modified comes last), + # that is oldest files will come on top. + files_to_remove=$(ls -tr "$rgw_core_dir" | grep core | head -n "$remove_file_count") + for file in $files_to_remove + do + rm -f "$rgw_core_dir/$file" + done + echo "## deleted ($remove_file_count) core file(s) from directory: $rgw_core_dir ##" + fi + fi +else + echo "ERROR !!! Invalid directory path : $rgw_core_dir. Please add proper path of rgw core directory." + exit +fi + +echo "Done with rgw core file rotation" diff --git a/src/rgw/support/rgw_support_bundle b/src/rgw/support/rgw_support_bundle index 1713fea..7a9be66 100755 --- a/src/rgw/support/rgw_support_bundle +++ b/src/rgw/support/rgw_support_bundle @@ -30,7 +30,7 @@ from cortx.utils.errors import BaseError from cortx.utils.process import SimpleProcess from cortx.utils.conf_store.conf_store import Conf, MappedConf from cortx.rgw.const import ( - CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE) + CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE, RGW_CORE_FILE_DIR_NAME) class SupportBundleError(BaseError): @@ -164,10 +164,10 @@ class RGWSupportBundle: # copy ceph crash-dump files if coredumps: - crash_dump_dir = os.path.join(log_dir, 'rgw_debug') + crash_dump_dir = os.path.join(log_dir, RGW_CORE_FILE_DIR_NAME) if os.path.exists(crash_dump_dir): shutil.copytree(crash_dump_dir,\ - os.path.join(RGWSupportBundle._tmp_src, 'rgw_debug')) + os.path.join(RGWSupportBundle._tmp_src, RGW_CORE_FILE_DIR_NAME)) # add cortx components rpm version cmd = "rpm -qa | grep cortx"