CORTX-32039: Add rgw core file clean up script as part of system's logrotate directory.

setup.py

@@ -37,6 +37,7 @@
 mini_prov_files = glob.glob('./src/setup/*.py')
 logrotate_tmpl_file = 'src/rgw/setup/templates/rgw.logrotate.tmpl'
 logrotate_service_tmpl = 'src/rgw/setup/templates/logrotate.service.tmpl'
+core_file_logrotate_tmpl = 'src/rgw/setup/templates/rgw_core_logrotate.sh'
 
 data_file_list=[ ('%s/mini-provisioner' % RGW_INSTALL_PATH, mini_prov_files),
                     ('%s/bin' % RGW_INSTALL_PATH,
@@ -45,7 +46,7 @@
                     ('%s/mini-provisioner' % RGW_INSTALL_PATH,['VERSION']),
                     ('%s/conf' % RGW_INSTALL_PATH,['conf/cortx_rgw.conf',
                     logrotate_tmpl_file, 'src/rgw/support/support.yaml',
-                    logrotate_service_tmpl])
+                    logrotate_service_tmpl, core_file_logrotate_tmpl])
                   ]
 
 # Add addb plugin to rpm if its generated duirng build process.

src/rgw/const.py

@@ -34,6 +34,7 @@
 COMPONENT_SVC_NAME = 'rgw_s3'
 DECRYPTION_KEY = 'cortx'
 SERVICE_NAME = f'{COMPONENT_NAME}_setup' # rgw_setup
+RGW_CORE_FILE_DIR_NAME = f'{COMPONENT_NAME}_debug' # rgw_debug
 INSTALL_PATH = '/opt/seagate/cortx'
 RGW_INSTALL_PATH = f'{INSTALL_PATH}/{COMPONENT_NAME}'
 ADMIN_CREATION_TIMEOUT = 600
@@ -63,6 +64,11 @@
 REQUIRED_RPMS = ['cortx-hare', 'cortx-py-utils', 'ceph-radosgw']
 ADMIN_PARAMETERS = {'MOTR_ADMIN_FID':'motr admin fid', 'MOTR_ADMIN_ENDPOINT':'motr admin endpoint', 'RGW_FRONTENDS': 'rgw frontends'}
 
+# core files rotate script
+CORE_LOGROTATE_FILE_NAME =f'{COMPONENT_NAME}_core_logrotate.sh' # rgw_core_logrotate.sh
+CORE_LOGROTATE_FILE = f'{CRON_DIR}/{CORE_LOGROTATE_FILE_NAME}'
+CORE_LOGROTATE_TMPL = f'{RGW_INSTALL_PATH}/conf/{CORE_LOGROTATE_FILE_NAME}'
+
 # CORTX cluster confstore keys
 SVC_NAME_KEY = 'node>%s>components[%s]>services[0]'
 NUM_COMPONENTS_KEY = 'node>%s>num_components'

src/rgw/setup/rgw.py

@@ -151,7 +151,7 @@ def start(conf: MappedConf, index: str):
         os.makedirs(motr_trace_dir, exist_ok=True)
         os.makedirs(addb_dir, exist_ok=True)
         # Create rgw crash file directory
-        rgw_core_dir = os.path.join(log_path, 'rgw_debug')
+        rgw_core_dir = os.path.join(log_path, const.RGW_CORE_FILE_DIR_NAME)
         os.makedirs(rgw_core_dir, exist_ok=True)
 
         Log.info('Starting radosgw service.')
@@ -395,7 +395,7 @@ def _change_working_dir(conf: MappedConf):
         """Change current working directory to crash directory path."""
         log_path = Rgw._get_log_dir_path(conf)
         # Create svc crash file directory
-        svc_core_dir = os.path.join(log_path, 'rgw_debug')
+        svc_core_dir = os.path.join(log_path, const.RGW_CORE_FILE_DIR_NAME)
         os.makedirs(svc_core_dir, exist_ok=True)
         os.chdir(svc_core_dir)
 
@@ -827,6 +827,22 @@ def _logrotate_generic(conf: MappedConf):
             Log.info(f'{const.LOGROTATE_TMPL} file copied to {const.LOGROTATE_CONF}')
         except Exception as e:
             Log.error(f"Failed to configure logrotate for {const.COMPONENT_NAME}. ERROR:{e}")
+
+        # Copy log rotate script for rgw core files into system's logrotate directory.
+        old_core_logrotate_file = os.path.join(const.LOGROTATE_DIR, const.CORE_LOGROTATE_FILE_NAME)
+        core_dir_path = os.path.join(log_file_path, const.RGW_CORE_FILE_DIR_NAME)
+        if os.path.exists(old_core_logrotate_file):
+            os.remove(old_core_logrotate_file)
+        try:
+            with open(const.CORE_LOGROTATE_TMPL, 'r') as f:
+                content = f.read()
+            content = content.replace('TEMP_CORE_FILE_DIR_PATH', core_dir_path)
+            with open(const.CORE_LOGROTATE_FILE, 'w') as f:
+                f.write(content)
+            Log.info(f'{const.CORE_LOGROTATE_TMPL} file copied to {const.CORE_LOGROTATE_FILE}')
+        except Exception as e:
+            Log.error(f"Failed to configure core file's logrotate for {const.COMPONENT_NAME}. ERROR:{e}")
+
         # start cron.d service
         try:
             os.system(f"chmod +x {const.CRON_LOGROTATE}")

src/rgw/setup/templates/rgw_core_logrotate.sh

@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+#
+
+# script is used to delete the old rgw core files in /etc/cortx/log/rgw/<machine-id>/rgw_debug directory
+# script will retain latest 5 core file and remove rest of core files.
+# argument1: <number of latest rgw core files to retain>
+# Default number of latest log files is 5
+# ./rgw_core_logrotate.sh -n 5
+
+usage() { echo "Usage: bash $(basename "$0")[--help|-h]
+                   [-n maxRgwCoreFileCount]
+Retain recent generated rgw core files of given count and remove rest of core files.
+
+where:
+-n            number of latest rgw core files to retain (Default count for core files is 5)
+--help|-h     display this help and exit" 1>&2; exit 1; }
+
+# max rgw core files count in each rgw log directory
+core_files_max_count=5
+rgw_core_dir="TEMP_CORE_FILE_DIR_PATH"
+
+while getopts ":n:" option; do
+    case "${option}" in
+        n)
+            core_files_max_count=${OPTARG}
+            if [ -z ${core_files_max_count} ]
+            then
+              usage
+            fi
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+
+echo "max RGW core file to be collected: $core_files_max_count"
+echo
+
+echo "Checking for core files in $rgw_core_dir directory"
+if [[ -n "$rgw_core_dir" && -d "$rgw_core_dir" ]]
+then
+    # Find core files
+     core_files=`find $rgw_core_dir -maxdepth 1 -type f -name "core.*"`
+     core_files_count=`echo "$core_files" | grep -v "^$" | wc -l`
+     echo "## found $core_files_count file(s) in log directory($rgw_core_dir) ##"
+     # check core files count is greater than max core file count or not
+     if [ $core_files_count -gt $core_files_max_count ]
+     then
+        # get files sort by date - oldest will come on top
+        remove_file_count=`expr $core_files_count - $core_files_max_count`
+        if [ $remove_file_count -gt 0 ]
+        then
+            echo "## ($remove_file_count) rgw core file(s) can be removed from directory :$rgw_core_dir ##"
+            # get the files sorted by time modified (most recently modified comes last),
+            # that is oldest files will come on top.
+            files_to_remove=`ls -tr "$rgw_core_dir" | grep core | head -n $remove_file_count`
+            for file in $files_to_remove
+            do
+              rm -f "$rgw_core_dir/$file"
+            done
+            echo "## deleted ($remove_file_count) core file(s) from directory: $rgw_core_dir ##"
+        else
+            echo "## No rgw core files to remove ##"
+        fi
+     else
+         echo "## No rgw core files to remove ##"
+     fi
+else
+    echo "## No rgw core files to remove ##"
+fi

src/rgw/support/rgw_support_bundle

@@ -30,7 +30,7 @@ from cortx.utils.errors import BaseError
 from cortx.utils.process import SimpleProcess
 from cortx.utils.conf_store.conf_store import Conf, MappedConf
 from cortx.rgw.const import (
-    CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE)
+    CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE, RGW_CORE_FILE_DIR_NAME)
 
 
 class SupportBundleError(BaseError):
@@ -164,10 +164,10 @@ class RGWSupportBundle:
 
         # copy ceph crash-dump files
         if coredumps:
-            crash_dump_dir = os.path.join(log_dir, 'rgw_debug')
+            crash_dump_dir = os.path.join(log_dir, RGW_CORE_FILE_DIR_NAME)
             if os.path.exists(crash_dump_dir):
                 shutil.copytree(crash_dump_dir,\
-                    os.path.join(RGWSupportBundle._tmp_src, 'rgw_debug'))
+                    os.path.join(RGWSupportBundle._tmp_src, RGW_CORE_FILE_DIR_NAME))
 
         # add cortx components rpm version
         cmd = "rpm -qa | grep cortx"