From 8db9ea489af0dbbce8a5634cab3ad7ba90c7095c Mon Sep 17 00:00:00 2001 From: yuanyu Date: Mon, 30 May 2022 09:21:37 +0800 Subject: [PATCH 01/13] Add collect performance interface --- .../vnx/vnx_block/component_handler.py | 378 ++++++++++++++++++ .../drivers/dell_emc/vnx/vnx_block/consts.py | 170 ++++++++ .../dell_emc/vnx/vnx_block/navi_handler.py | 104 +++++ .../dell_emc/vnx/vnx_block/vnx_block.py | 24 ++ .../performance_file/vnx_block/__init__.py | 0 .../dell_emc/vnx/vnx_block/test_vnx_block.py | 141 +++++++ 6 files changed, 817 insertions(+) create mode 100644 delfin/drivers/utils/performance_file/vnx_block/__init__.py diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index 914885c22..46aec08d9 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import csv +import os import re +import time +from io import StringIO import six from oslo_log import log @@ -21,6 +25,7 @@ from delfin import exception from delfin.common import constants from delfin.drivers.dell_emc.vnx.vnx_block import consts +from delfin.drivers.utils.tools import Tools LOG = log.getLogger(__name__) @@ -565,3 +570,376 @@ def list_storage_hosts(self, storage_id): } host_list.append(host_model) return host_list + + def collect_perf_metrics(self, storage_id, resource_metrics, + start_time, end_time): + metrics = [] + archive_file_list = [] + try: + if (end_time - start_time) < consts.EXEC_TIME_INTERVAL: + LOG.warn("not exe collert, start_time-end_time={}".format( + (end_time - start_time))) + return metrics + archive_file_list = self._get__archive_file(start_time, end_time) + LOG.info("Get archive files: {}".format(archive_file_list)) + if not archive_file_list: + LOG.warn("The required performance file was not found!") + return metrics + resources_map, resources_type_map = self._get_resources_map( + resource_metrics) + if not resources_map or not resources_type_map: + LOG.warn("Resource object not found!") + return metrics + performance_lines_map = self._filter_performance_data( + archive_file_list, resources_map, start_time, end_time) + if not performance_lines_map: + LOG.warn("The required performance data was not found!") + return metrics + metrics = self.create_metrics(storage_id, resource_metrics, + resources_map, resources_type_map, + performance_lines_map) + metrics = self._handle_replenish_point(metrics, start_time) + except exception.DelfinException as err: + err_msg = "Failed to collect metrics from VnxBlockStor: %s" % \ + (six.text_type(err)) + LOG.error(err_msg) + raise err + except Exception as err: + err_msg = "Failed to collect metrics from VnxBlockStor: %s" % \ + (six.text_type(err)) + LOG.error(err_msg) + raise exception.InvalidResults(err_msg) + finally: + self._remove_archive_file(archive_file_list) + return metrics + + def create_metrics(self, storage_id, resource_metrics, resources_map, + resources_type_map, performance_lines_map): + metrics = [] + for resource_obj in resources_type_map.keys(): + if not resources_map.get(resource_obj) \ + or not resources_type_map.get(resource_obj): + continue + resources_type = resources_type_map.get(resource_obj) + labels = { + 'storage_id': storage_id, + 'resource_type': resources_type, + 'resource_id': resources_map.get(resource_obj), + 'type': 'RAW', + 'unit': '' + } + if not performance_lines_map.get(resource_obj): + continue + metric_model_list = self._get_metric_model( + resource_metrics.get(resources_type), labels, + performance_lines_map.get(resource_obj), + consts.RESOURCES_TYPE_TO_METRIC_CAP.get( + resources_type), resources_type) + if metric_model_list: + metrics.extend(metric_model_list) + return metrics + + def _handle_replenish_point(self, metrics, start_time): + if not metrics: + return metrics + nar_interval = self.navi_handler.get_nar_interval() + LOG.info("Get nar_interval:{}".format(nar_interval)) + replenish_point_interval = nar_interval * units.k + check_nar_interval = nar_interval * units.k * 1.5 + for m in metrics: + new_values_map = {} + previous_time = '' + previous_value = '' + for collection_time in m.values.keys(): + if previous_time == '': + previous_time = collection_time + previous_value = m.values.get(collection_time) + if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ + < previous_time: + new_values_map[previous_time] = \ + float('%.6f' % previous_value) + continue + next_time = collection_time + replenish_point_value = 0 + if previous_value and m.values.get(next_time): + replenish_point_value = \ + (previous_value + m.values.get(next_time)) / 2 + while (next_time - previous_time) > check_nar_interval: + replenish_point_time = next_time - replenish_point_interval + if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ + < replenish_point_time: + new_values_map[replenish_point_time] = \ + float('%.6f' % replenish_point_value) + next_time = replenish_point_time + if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ + < collection_time: + new_values_map[collection_time] = \ + float('%.6f' % m.values.get(collection_time)) + previous_time = collection_time + previous_value = m.values.get(collection_time) + sorted_map = sorted(new_values_map.items(), key=lambda x: x[0]) + m.values.clear() + m.values.update(sorted_map) + return metrics + + def _get__archive_file(self, start_time, end_time): + archive_file_list = [] + archives = self.navi_handler.get_archives() + tools = Tools() + for archive_info in (archives or []): + collection_timestamp = tools.time_str_to_timestamp( + archive_info.get('collection_time'), consts.TIME_PATTERN) + if collection_timestamp > start_time: + archive_file_list.append(archive_info.get('archive_name')) + if collection_timestamp > end_time: + break + return archive_file_list + + def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, + resources_type): + metric_model_list = [] + if not metric_values: + return metric_model_list + tools = Tools() + for metric_name in (metric_list or []): + values = {} + obj_labels = copy.deepcopy(labels) + obj_labels['unit'] = obj_cap.get(metric_name).get('unit') + for metric_value in metric_values: + metric_value_infos = metric_value + if not consts.METRIC_MAP.get(resources_type): + continue + if not consts.METRIC_MAP.get(resources_type).get(metric_name): + continue + value = metric_value_infos[ + consts.METRIC_MAP.get(resources_type).get(metric_name)] + if not value: + value = '0' + collection_timestamp = tools.time_str_to_timestamp( + metric_value_infos[1], consts.TIME_PATTERN) + values[collection_timestamp] = float('%.6f' % float(value)) + if values: + metric_model = constants.metric_struct(name=metric_name, + labels=obj_labels, + values=values) + metric_model_list.append(metric_model) + return metric_model_list + + def _get_resources_map(self, resource_metrics): + resources_map = {} + resources_type_map = {} + for resource_type_key in resource_metrics.keys(): + sub_resources_map = {} + sub_resources_type_map = {} + if resource_type_key == constants.ResourceType.CONTROLLER: + sub_resources_map, sub_resources_type_map = \ + self._get_controllers_map() + elif resource_type_key == constants.ResourceType.PORT: + sub_resources_map, sub_resources_type_map = \ + self._get_ports_map() + elif resource_type_key == constants.ResourceType.DISK: + sub_resources_map, sub_resources_type_map = \ + self._get_disks_map() + elif resource_type_key == constants.ResourceType.VOLUME: + sub_resources_map, sub_resources_type_map = \ + self._get_volumes_map() + if sub_resources_map and sub_resources_type_map: + resources_map.update(sub_resources_map) + resources_type_map.update(sub_resources_type_map) + return resources_map, resources_type_map + + def _get_controllers_map(self): + resources_map = {} + resources_type_map = {} + controllers = self.navi_handler.get_controllers() + for controller in (controllers or []): + resources_map[controller.get('sp_name')] = controller.get( + 'signature_for_the_sp') + resources_type_map[controller.get('sp_name')] = \ + constants.ResourceType.CONTROLLER + return resources_map, resources_type_map + + def _get_ports_map(self): + resources_map = {} + resources_type_map = {} + ports = self.navi_handler.get_ports() + for port in (ports or []): + port_id = port.get('sp_port_id') + sp_name = port.get('sp_name').replace('SP ', '') + name = '%s-%s' % (sp_name, port_id) + port_id = 'Port %s [ %s ]' % (port_id, port.get('sp_uid')) + resources_map[port_id] = name + resources_type_map[port_id] = constants.ResourceType.PORT + return resources_map, resources_type_map + + def _get_disks_map(self): + resources_map = {} + resources_type_map = {} + disks = self.navi_handler.get_disks() + for disk in (disks or []): + disk_name = disk.get('disk_name') + disk_name = disk_name.replace(' Disk', ' Disk') + resources_map[disk_name] = disk.get('disk_id') + resources_type_map[disk_name] = constants.ResourceType.DISK + return resources_map, resources_type_map + + def _get_volumes_map(self): + resources_map = {} + resources_type_map = {} + volumes = self.navi_handler.get_all_lun() + for volume in (volumes or []): + if not volume.get('name'): + continue + volume_name = '%s [%s]' % ( + volume.get('name'), volume.get('logical_unit_number')) + resources_map[volume_name] = str(volume.get('logical_unit_number')) + resources_type_map[volume_name] = constants.ResourceType.VOLUME + return resources_map, resources_type_map + + def _filter_performance_data(self, archive_file_list, resources_map, + start_time, end_time): + performance_lines_map = {} + try: + if not resources_map: + return performance_lines_map + tools = Tools() + for archive_file in (archive_file_list or []): + self.navi_handler.download_archives(archive_file) + archive_name_infos = archive_file.split('.') + file_path = '%s%s.csv' % ( + self.navi_handler.get_local_file_path(), + archive_name_infos[0]) + with open(file_path) as file: + f_csv = csv.reader(StringIO(file.read())) + next(f_csv) + for row in f_csv: + self._package_performance_data(row, resources_map, + start_time, end_time, + tools, + performance_lines_map) + except Exception as err: + err_msg = "Failed to filter performance data: %s" % \ + (six.text_type(err)) + LOG.error(err_msg) + raise exception.StorageBackendException(err_msg) + return performance_lines_map + + def _package_performance_data(self, row, resources_map, start_time, + end_time, tools, performance_lines_map): + resource_obj_name = row[0] + resource_obj_name = self._package_resource_obj_name(resource_obj_name) + if resource_obj_name in resources_map.keys(): + obj_collection_timestamp = tools.time_str_to_timestamp( + row[1], consts.TIME_PATTERN) + if (start_time - consts.TIME_INTERVAL_FLUCTUATION) \ + <= obj_collection_timestamp \ + and obj_collection_timestamp \ + <= (end_time + consts.TIME_INTERVAL_FLUCTUATION): + if performance_lines_map.get(resource_obj_name): + performance_lines_map.get(resource_obj_name).append(row) + else: + obj_performance_list = [] + obj_performance_list.append(row) + performance_lines_map[ + resource_obj_name] = obj_performance_list + + def _package_resource_obj_name(self, source_name): + target_name = source_name + if 'Port ' in target_name: + target_name = re.sub('(\\[.*;)', '[', target_name) + elif '; ' in target_name: + target_name = re.sub('(; .*])', ']', target_name) + return target_name + + def _remove_archive_file(self, archive_file_list): + try: + for archive_file in archive_file_list: + nar_file_path = '%s%s' % ( + self.navi_handler.get_local_file_path(), archive_file) + archive_name_infos = archive_file.split('.') + csv_file_path = '%s%s.csv' % ( + self.navi_handler.get_local_file_path(), + archive_name_infos[0]) + for file_path in [nar_file_path, csv_file_path]: + LOG.info("Delete file :{}".format(file_path)) + if os.path.exists(file_path): + os.remove(file_path) + else: + err_msg = 'no such file:%s' % file_path + LOG.error(err_msg) + raise exception.StorageBackendException(err_msg) + except Exception as err: + err_msg = "Failed to remove archive file: %s" % \ + (six.text_type(err)) + LOG.error(err_msg) + raise exception.StorageBackendException(err_msg) + + def get_latest_perf_timestamp(self, storage_id): + latest_time = 0 + num = 0 + tools = Tools() + while latest_time <= 0: + num += 1 + latest_time, file_latest_time = self.check_latest_timestamp( + storage_id) + if num > consts.EXEC_MAX_NUM: + latest_time = file_latest_time + LOG.warn("Storage：{}，Exit after {} executions.".format( + storage_id, consts.EXEC_MAX_NUM)) + break + if latest_time <= 0: + wait_time = tools.timestamp_to_time_str( + time.time() * units.k, + consts.ARCHIVE_FILE_NAME_TIME_PATTERN) + LOG.warn("Storage：{} No new file found, " + "wait for next execution：{}".format(storage_id, + wait_time)) + time.sleep(consts.SLEEP_TIME_SECONDS) + return latest_time + + def get_data_latest_timestamp(self, storage_id): + archive_file_list = [] + try: + tools = Tools() + archive_name = self.navi_handler.create_archives(storage_id) + LOG.info("Create archive_name: {}".format(archive_name)) + archive_file_list.append(archive_name) + archive_name_infos = archive_name.split('.') + file_path = '%s%s.csv' % ( + self.navi_handler.get_local_file_path(), archive_name_infos[0]) + resource_obj_name = '' + collection_time = '' + with open(file_path) as file: + f_csv = csv.reader(file) + next(f_csv) + for row in f_csv: + if not resource_obj_name or resource_obj_name == row[0]: + resource_obj_name = row[0] + collection_time = row[1] + else: + break + latest_time = tools.time_str_to_timestamp(collection_time, + consts.TIME_PATTERN) + except Exception as err: + err_msg = "Failed to get latest perf timestamp " \ + "from VnxBlockStor: %s" % (six.text_type(err)) + LOG.error(err_msg) + raise exception.InvalidResults(err_msg) + finally: + self._remove_archive_file(archive_file_list) + return latest_time + + def check_latest_timestamp(self, storage_id): + latest_time = 0 + file_latest_time = self.get_data_latest_timestamp(storage_id) + sys_time = self.navi_handler.get_sp_time() + LOG.info("Get sys_time=={},file_latest_time=={}".format( + sys_time, file_latest_time)) + if sys_time > 0 and file_latest_time > 0: + LOG.info("(sys_time - file_latest_time)={}".format( + (sys_time - file_latest_time))) + if (sys_time - file_latest_time) < \ + consts.CREATE_FILE_TIME_INTERVAL: + latest_time = file_latest_time + time.sleep(consts.CHECK_WAITE_TIME_SECONDS) + return latest_time, file_latest_time diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/consts.py b/delfin/drivers/dell_emc/vnx/vnx_block/consts.py index d012a74bb..9fbfc73d2 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/consts.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/consts.py @@ -154,3 +154,173 @@ } ALU_PAIRS_PATTERN = '^[0-9]+\\s+[0-9]+$' HBA_UID_PATTERN = "^\\s*HBA UID\\s+SP Name\\s+SPPort" + +IOPS_DESCRIPTION = { + "unit": "IOPS", + "description": "Input/output operations per second" +} +READ_IOPS_DESCRIPTION = { + "unit": "IOPS", + "description": "Read input/output operations per second" +} +WRITE_IOPS_DESCRIPTION = { + "unit": "IOPS", + "description": "Write input/output operations per second" +} +THROUGHPUT_DESCRIPTION = { + "unit": "MB/s", + "description": "Represents how much data is " + "successfully transferred in MB/s" +} +READ_THROUGHPUT_DESCRIPTION = { + "unit": "MB/s", + "description": "Represents how much data read is " + "successfully transferred in MB/s" +} +WRITE_THROUGHPUT_DESCRIPTION = { + "unit": "MB/s", + "description": "Represents how much data write is " + "successfully transferred in MB/s" +} +RESPONSE_TIME_DESCRIPTION = { + "unit": "ms", + "description": "Average time taken for an IO " + "operation in ms" +} +CACHE_HIT_RATIO_DESCRIPTION = { + "unit": "%", + "description": "Percentage of io that are cache hits" +} +READ_CACHE_HIT_RATIO_DESCRIPTION = { + "unit": "%", + "description": "Percentage of read ops that are cache hits" +} +WRITE_CACHE_HIT_RATIO_DESCRIPTION = { + "unit": "%", + "description": "Percentage of write ops that are cache hits" +} +IO_SIZE_DESCRIPTION = { + "unit": "KB", + "description": "The average size of IO requests in KB" +} +READ_IO_SIZE_DESCRIPTION = { + "unit": "KB", + "description": "The average size of read IO requests in KB" +} +WRITE_IO_SIZE_DESCRIPTION = { + "unit": "KB", + "description": "The average size of write IO requests in KB" +} +CONTROLLER_CAP = { + "iops": IOPS_DESCRIPTION, + "readIops": READ_IOPS_DESCRIPTION, + "writeIops": WRITE_IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION +} +VOLUME_CAP = { + "iops": IOPS_DESCRIPTION, + "readIops": READ_IOPS_DESCRIPTION, + "writeIops": WRITE_IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION, + "cacheHitRatio": CACHE_HIT_RATIO_DESCRIPTION, + "readCacheHitRatio": READ_CACHE_HIT_RATIO_DESCRIPTION, + "writeCacheHitRatio": WRITE_CACHE_HIT_RATIO_DESCRIPTION, + "ioSize": IO_SIZE_DESCRIPTION, + "readIoSize": READ_IO_SIZE_DESCRIPTION, + "writeIoSize": WRITE_IO_SIZE_DESCRIPTION +} +PORT_CAP = { + "iops": IOPS_DESCRIPTION, + "readIops": READ_IOPS_DESCRIPTION, + "writeIops": WRITE_IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION +} +DISK_CAP = { + "iops": IOPS_DESCRIPTION, + "readIops": READ_IOPS_DESCRIPTION, + "writeIops": WRITE_IOPS_DESCRIPTION, + "throughput": THROUGHPUT_DESCRIPTION, + "readThroughput": READ_THROUGHPUT_DESCRIPTION, + "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, + "responseTime": RESPONSE_TIME_DESCRIPTION +} +RESOURCES_TYPE_TO_METRIC_CAP = { + constants.ResourceType.CONTROLLER: CONTROLLER_CAP, + constants.ResourceType.PORT: PORT_CAP, + constants.ResourceType.DISK: DISK_CAP, + constants.ResourceType.VOLUME: VOLUME_CAP, +} +METRIC_MAP = { + constants.ResourceType.CONTROLLER: { + "iops": 16, + "readIops": 25, + "writeIops": 34, + "throughput": 13, + "readThroughput": 19, + "writeThroughput": 28, + "responseTime": 10 + }, + constants.ResourceType.PORT: { + "iops": 16, + "readIops": 25, + "writeIops": 34, + "throughput": 13, + "readThroughput": 19, + "writeThroughput": 28 + }, + constants.ResourceType.DISK: { + "iops": 16, + "readIops": 25, + "writeIops": 34, + "throughput": 13, + "readThroughput": 19, + "writeThroughput": 28, + "responseTime": 10 + }, + constants.ResourceType.VOLUME: { + "iops": 16, + "readIops": 25, + "writeIops": 34, + "throughput": 13, + "readThroughput": 19, + "writeThroughput": 28, + "responseTime": 10, + "readCacheHitRatio": 42, + "writeCacheHitRatio": 45, + "readIoSize": 22, + "writeIoSize": 31 + } +} + +ARCHIVE_FILE_NAME = '%s_SPA_%s.nar' +GET_SP_TIME = 'getsptime' +GET_NAR_INTERVAL_API = 'analyzer -get -narinterval' +GET_ARCHIVE_API = 'analyzer -archive -list' +CREATE_ARCHIVE_API = 'analyzer -archiveretrieve -file %s -location %s ' \ + '-overwrite y -retry 3' +DOWNLOAD_ARCHIVE_API = 'analyzer -archive -file %s -path %s -o' +ARCHIVEDUMP_API = 'analyzer -archivedump -data %s%s -out %s%s.csv' +ARCHIVE_FILE_DIR = "/delfin/drivers/utils/performance_file/vnx_block/" +GET_SP_TIME_PATTERN = '%m/%d/%y %H:%M:%S' +ARCHIVE_FILE_NAME_TIME_PATTERN = '%Y_%m_%d_%H_%M_%S' +# Unit: s +SLEEP_TIME_SECONDS = 60 +# Unit: ms +CREATE_FILE_TIME_INTERVAL = 150000 +# Unit: ms +EXEC_TIME_INTERVAL = 240000 +EXEC_MAX_NUM = 50 +# Unit: ms +TIME_INTERVAL_FLUCTUATION = 3000 +REPLACE_PATH = "/delfin/drivers/dell_emc/vnx/vnx_block" +# Unit: s +CHECK_WAITE_TIME_SECONDS = 15 diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/navi_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/navi_handler.py index 7483ec955..db99ead52 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/navi_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/navi_handler.py @@ -11,16 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import re import threading +import time import six from oslo_log import log as logging from delfin import cryptor +from oslo_utils import units + from delfin import exception from delfin.drivers.dell_emc.vnx.vnx_block import consts from delfin.drivers.dell_emc.vnx.vnx_block.navicli_client import NaviClient +from delfin.drivers.utils.tools import Tools LOG = logging.getLogger(__name__) @@ -700,3 +705,102 @@ def cli_hba_to_list(self, resource_info): LOG.error(err_msg) raise exception.InvalidResults(err_msg) return obj_list + + def get_archives(self): + return self.get_resources_info(consts.GET_ARCHIVE_API, + self.cli_archives_to_list) + + def cli_archives_to_list(self, resource_info): + obj_list = [] + try: + obj_infos = resource_info.split('\n') + for obj_info in obj_infos: + str_line = obj_info.strip() + if str_line: + archive_infos = str_line.split() + if archive_infos and len(archive_infos) == 5: + obj_model = {} + obj_model['collection_time'] = \ + "%s %s" % (archive_infos[2], archive_infos[3]) + obj_model['archive_name'] = archive_infos[4] + obj_list.append(obj_model) + except Exception as e: + err_msg = "arrange archives info error: %s", six.text_type(e) + LOG.error(err_msg) + raise exception.InvalidResults(err_msg) + return obj_list + + def download_archives(self, archive_name): + download_archive_api = consts.DOWNLOAD_ARCHIVE_API % ( + archive_name, self.get_local_file_path()) + self.get_resources_info(download_archive_api, self.cli_res_to_list) + archive_name_infos = archive_name.split('.') + archivedump_api = consts.ARCHIVEDUMP_API % ( + self.get_local_file_path(), archive_name, + self.get_local_file_path(), archive_name_infos[0]) + self.get_resources_info(archivedump_api, self.cli_res_to_list) + + def get_local_file_path(self): + driver_path = os.path.abspath(os.path.join(os.getcwd())) + driver_path = driver_path.replace("\\", "/") + driver_path = driver_path.replace(consts.REPLACE_PATH, "") + local_path = '%s%s' % (driver_path, consts.ARCHIVE_FILE_DIR) + return local_path + + def get_sp_time(self): + return self.get_resources_info(consts.GET_SP_TIME, + self.analysis_sp_time) + + def analysis_sp_time(self, resource_info): + system_time = 0 + try: + tools = Tools() + obj_infos = resource_info.split('\n') + for obj_info in obj_infos: + str_line = obj_info.strip() + if "Time on SP A:" in str_line: + time_str = str_line.replace("Time on SP A:", "").strip() + system_time = tools.time_str_to_timestamp( + time_str, consts.GET_SP_TIME_PATTERN) + except Exception as e: + err_msg = "analysis sp time error: %s", six.text_type(e) + LOG.error(err_msg) + raise exception.InvalidResults(err_msg) + return system_time + + def get_nar_interval(self): + return self.get_resources_info(consts.GET_NAR_INTERVAL_API, + self.analysis_nar_interval) + + def analysis_nar_interval(self, resource_info): + nar_interval = 60 + try: + if resource_info and ":" in resource_info: + nar_interval_str = resource_info.split(":")[1].strip() + nar_interval = int(nar_interval_str) + except Exception as e: + err_msg = "analysis sp time error: %s", six.text_type(e) + LOG.error(err_msg) + raise exception.InvalidResults(err_msg) + return nar_interval + + def get_archive_file_name(self, storage_id): + tools = Tools() + create_time = tools.timestamp_to_time_str( + time.time() * units.k, consts.ARCHIVE_FILE_NAME_TIME_PATTERN) + archive_file_name = consts.ARCHIVE_FILE_NAME % (storage_id, + create_time) + return archive_file_name + + def create_archives(self, storage_id): + archive_name = self.get_archive_file_name(storage_id) + create_archive_api = consts.CREATE_ARCHIVE_API % ( + archive_name, self.get_local_file_path()) + self.get_resources_info(create_archive_api, self.cli_res_to_list) + + archive_name_infos = archive_name.split('.') + archivedump_api = consts.ARCHIVEDUMP_API % ( + self.get_local_file_path(), archive_name, + self.get_local_file_path(), archive_name_infos[0]) + self.get_resources_info(archivedump_api, self.cli_res_to_list) + return archive_name diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/vnx_block.py b/delfin/drivers/dell_emc/vnx/vnx_block/vnx_block.py index 467094208..4a5203d61 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/vnx_block.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/vnx_block.py @@ -13,7 +13,9 @@ # limitations under the License. from oslo_log import log +from delfin.common import constants from delfin.drivers import driver +from delfin.drivers.dell_emc.vnx.vnx_block import consts from delfin.drivers.dell_emc.vnx.vnx_block.alert_handler import AlertHandler from delfin.drivers.dell_emc.vnx.vnx_block.component_handler import \ ComponentHandler @@ -86,3 +88,25 @@ def list_storage_hosts(self, context): def list_masking_views(self, context): return self.com_handler.list_masking_views(self.storage_id) + + def collect_perf_metrics(self, context, storage_id, resource_metrics, + start_time, end_time): + return self.com_handler.collect_perf_metrics(storage_id, + resource_metrics, + start_time, end_time) + + @staticmethod + def get_capabilities(context, filters=None): + """Get capability of supported driver""" + return { + 'is_historic': True, + 'resource_metrics': { + constants.ResourceType.CONTROLLER: consts.CONTROLLER_CAP, + constants.ResourceType.VOLUME: consts.VOLUME_CAP, + constants.ResourceType.PORT: consts.PORT_CAP, + constants.ResourceType.DISK: consts.DISK_CAP + } + } + + def get_latest_perf_timestamp(self, context): + return self.com_handler.get_latest_perf_timestamp(self.storage_id) diff --git a/delfin/drivers/utils/performance_file/vnx_block/__init__.py b/delfin/drivers/utils/performance_file/vnx_block/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py index 0f137da28..ffb6acb0f 100644 --- a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py +++ b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py @@ -11,10 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import sys import time from unittest import TestCase, mock +from unittest.mock import patch, mock_open +from delfin.common import constants from delfin.drivers.dell_emc.vnx.vnx_block import consts from delfin.drivers.dell_emc.vnx.vnx_block.alert_handler import AlertHandler from delfin.drivers.utils.tools import Tools @@ -394,6 +397,47 @@ StorageGroup Name: None """ +ARCHIVE_DATAS = """ +Index Size in KB Last Modified Filename +2 46 07/08/2021 01:20:29 CETV2135000041_SPA_2021-07-07_17-20-26-GMT_P08-00.nar +3 40 07/08/2021 03:56:28 CETV2135000041_SPA_2021-07-07_19-56-25-GMT_P08-00.nar +4 31 07/08/2021 06:32:29 CETV2135000041_SPA_2021-07-07_22-32-26-GMT_P08-00.nar +5 02 07/08/2021 09:08:29 CETV2135000041_SPA_2021-07-08_01-08-26-GMT_P08-00.nar +6 76 07/08/2021 11:44:29 CETV2135000041_SPA_2021-07-08_03-44-26-GMT_P08-00.nar +7 48 07/08/2021 14:20:28 CETV2135000041_SPA_2021-07-08_06-20-26-GMT_P08-00.nar +8 34 07/08/2021 16:31:13 CETV2135000041_SPA_2021-07-08_08-31-11-GMT_P08-00.nar +""" +PERFORMANCE_FILE = """ +SP A,07/08/2021 12:15:56,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 +SP A,07/08/2021 12:16:56,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 +SP A,07/08/2021 12:17:55,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 +SP A,07/08/2021 12:18:56,,,,,,,,,0,,,0.28,,,0.73,,,0,,,0,,,0,,,0.28,,,,,,0.73 +SP A,07/08/2021 12:19:56,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 +SP B,07/08/2021 12:15:56,,,,,,,,,0,,,0.9,,,2.6,,,0.9,,,2.4,,,1,,,0.7,,,,,,0.2 +SP B,07/08/2021 12:16:56,,,,,,,,,0,,,0.1,,,5.6,,,0.2,,,6.7,,,2,,,1.6,,,,,,1.4 +SP B,07/08/2021 12:17:55,,,,,,,,,0,,,0.2,,,4.6,,,0.3,,,1.7,,,3,,,2.6,,,,,,2.4 +SP B,07/08/2021 12:18:56,,,,,,,,,0,,,0.3,,,6.6,,,0.4,,,2.7,,,4,,,3.6,,,,,,3.4 +SP B,07/08/2021 12:19:56,,,,,,,,,0,,,0.4,,,7.6,,,0.5,,,3.7,,,5,,,4.6,,,,,,4.4 +L1 [230],07/08/2021 12:15:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +L1 [230],07/08/2021 12:16:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +L1 [230],07/08/2021 12:17:55,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +L1 [230],07/08/2021 12:18:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +L1 [230],07/08/2021 12:19:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +Bus 0 Enclosur 0 Disk 0,07/08/2021 12:15:56,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,, +Bus 0 Enclosur 0 Disk 0,07/08/2021 12:16:56,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,, +Bus 0 Enclosur 0 Disk 0,07/08/2021 12:17:55,,,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,, +Bus 0 Enclosur 0 Disk 0,07/08/2021 12:18:56,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, +Bus 0 Enclosur 0 Disk 0,07/08/2021 12:19:56,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, +Port 6 [FC; 50:08:1E ],07/08/2021 12:15:56,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,, +Port 6 [FC; 50:08:1E ],07/08/2021 12:16:56,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,, +Port 6 [FC; 50:08:1E ],07/08/2021 12:17:55,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,, +Port 6 [FC; 50:08:1E ],07/08/2021 12:18:56,,,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,, +Port 6 [FC; 50:08:1E ],07/08/2021 12:19:56,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, +""" +NAR_INTERVAL_DATAS = """ +Archive Poll Interval (sec): 60 +""" + AGENT_RESULT = { 'agent_rev': '7.33.1 (0.38)', 'name': 'K10', @@ -589,6 +633,56 @@ 'status': 'normal', 'ip_address': '8.44.129.26' }] +METRICS_RESULT = [ + constants.metric_struct(name='iops', labels={ + 'storage_id': '12345', + 'resource_type': 'controller', + 'resource_id': '3600485', + 'type': 'RAW', + 'unit': 'IOPS' + }, values={ + 1625717816000: 0.0, + 1625717875000: 0.0, + 1625717936000: 0.73, + 1625717996000: 0.0 + }), + constants.metric_struct(name='iops', labels={ + 'storage_id': '12345', + 'resource_type': 'port', + 'resource_id': 'A-6', + 'type': 'RAW', + 'unit': 'IOPS' + }, values={ + 1625717816000: 3.0, + 1625717875000: 4.0, + 1625717936000: 5.0, + 1625717996000: 6.0 + }), + constants.metric_struct(name='iops', labels={ + 'storage_id': '12345', + 'resource_type': 'disk', + 'resource_id': 'Bus0Enclosur0Disk0', + 'type': 'RAW', + 'unit': 'IOPS' + }, values={ + 1625717816000: 4.0, + 1625717875000: 5.0, + 1625717936000: 6.0, + 1625717996000: 6.0 + }), + constants.metric_struct(name='iops', labels={ + 'storage_id': '12345', + 'resource_type': 'volume', + 'resource_id': '230', + 'type': 'RAW', + 'unit': 'IOPS' + }, values={ + 1625717816000: 0.0, + 1625717875000: 0.0, + 1625717936000: 0.0, + 1625717996000: 0.0 + }) +] def create_driver(): @@ -784,3 +878,50 @@ def test_get_hosts(self): NaviClient.exec = mock.Mock(side_effect=[HBA_DATAS]) hosts = self.driver.list_storage_hosts(context) self.assertDictEqual(hosts[0], HOST_RESULT[0]) + + @patch("builtins.open", new_callable=mock_open, read_data=PERFORMANCE_FILE) + def test_get_perf_metrics(self, mock_file): + driver = create_driver() + resource_metrics = { + 'controller': [ + 'iops', 'readIops', 'writeIops', + 'throughput', 'readThroughput', 'writeThroughput', + 'responseTime' + ], + 'port': [ + 'iops', 'readIops', 'writeIops', + 'throughput', 'readThroughput', 'writeThroughput', + 'responseTime' + ], + 'disk': [ + 'iops', 'readIops', 'writeIops', + 'throughput', 'readThroughput', 'writeThroughput', + 'responseTime' + ], + 'volume': [ + 'iops', 'readIops', 'writeIops', + 'throughput', 'readThroughput', 'writeThroughput', + 'responseTime', + 'cacheHitRatio', 'readCacheHitRatio', 'writeCacheHitRatio', + 'ioSize', 'readIoSize', 'writeIoSize', + ] + } + start_time = 1625717756000 + end_time = 1625717996000 + NaviClient.exec = mock.Mock( + side_effect=[ARCHIVE_DATAS, SP_DATAS, PORT_DATAS, DISK_DATAS, + GET_ALL_LUN_INFOS, '', '', NAR_INTERVAL_DATAS]) + os.path.exists = mock.Mock(return_value=True) + os.remove = mock.Mock(return_value=True) + metrics = driver.collect_perf_metrics(context, '12345', + resource_metrics, start_time, + end_time) + self.assertEqual(metrics[0][1]["resource_id"], '3600485') + + def test_get_capabilities(self): + cap = VnxBlockStorDriver.get_capabilities(context) + self.assertIsNotNone(cap.get('resource_metrics')) + self.assertIsNotNone(cap.get('resource_metrics').get('controller')) + self.assertIsNotNone(cap.get('resource_metrics').get('volume')) + self.assertIsNotNone(cap.get('resource_metrics').get('port')) + self.assertIsNotNone(cap.get('resource_metrics').get('disk')) From 045c5f530fd43050fa811011490dbd568d104997 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Mon, 30 May 2022 10:32:13 +0800 Subject: [PATCH 02/13] Add collect performance interface --- .../vnx/vnx_block/component_handler.py | 2 +- .../dell_emc/vnx/vnx_block/test_vnx_block.py | 73 ++++++++++--------- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index 46aec08d9..ddfa94781 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -577,7 +577,7 @@ def collect_perf_metrics(self, storage_id, resource_metrics, archive_file_list = [] try: if (end_time - start_time) < consts.EXEC_TIME_INTERVAL: - LOG.warn("not exe collert, start_time-end_time={}".format( + LOG.warn("not exe collert, end_time-start_time={}".format( (end_time - start_time))) return metrics archive_file_list = self._get__archive_file(start_time, end_time) diff --git a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py index ffb6acb0f..457bbee84 100644 --- a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py +++ b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py @@ -11,15 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import sys import time from unittest import TestCase, mock -from unittest.mock import patch, mock_open from delfin.common import constants from delfin.drivers.dell_emc.vnx.vnx_block import consts from delfin.drivers.dell_emc.vnx.vnx_block.alert_handler import AlertHandler +from delfin.drivers.dell_emc.vnx.vnx_block.component_handler import \ + ComponentHandler from delfin.drivers.utils.tools import Tools sys.modules['delfin.cryptor'] = mock.Mock() @@ -407,33 +407,38 @@ 7 48 07/08/2021 14:20:28 CETV2135000041_SPA_2021-07-08_06-20-26-GMT_P08-00.nar 8 34 07/08/2021 16:31:13 CETV2135000041_SPA_2021-07-08_08-31-11-GMT_P08-00.nar """ -PERFORMANCE_FILE = """ -SP A,07/08/2021 12:15:56,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 -SP A,07/08/2021 12:16:56,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 -SP A,07/08/2021 12:17:55,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 -SP A,07/08/2021 12:18:56,,,,,,,,,0,,,0.28,,,0.73,,,0,,,0,,,0,,,0.28,,,,,,0.73 -SP A,07/08/2021 12:19:56,,,,,,,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0,,,0 -SP B,07/08/2021 12:15:56,,,,,,,,,0,,,0.9,,,2.6,,,0.9,,,2.4,,,1,,,0.7,,,,,,0.2 -SP B,07/08/2021 12:16:56,,,,,,,,,0,,,0.1,,,5.6,,,0.2,,,6.7,,,2,,,1.6,,,,,,1.4 -SP B,07/08/2021 12:17:55,,,,,,,,,0,,,0.2,,,4.6,,,0.3,,,1.7,,,3,,,2.6,,,,,,2.4 -SP B,07/08/2021 12:18:56,,,,,,,,,0,,,0.3,,,6.6,,,0.4,,,2.7,,,4,,,3.6,,,,,,3.4 -SP B,07/08/2021 12:19:56,,,,,,,,,0,,,0.4,,,7.6,,,0.5,,,3.7,,,5,,,4.6,,,,,,4.4 -L1 [230],07/08/2021 12:15:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -L1 [230],07/08/2021 12:16:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -L1 [230],07/08/2021 12:17:55,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -L1 [230],07/08/2021 12:18:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -L1 [230],07/08/2021 12:19:56,,,,,,,,,,,,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -Bus 0 Enclosur 0 Disk 0,07/08/2021 12:15:56,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,, -Bus 0 Enclosur 0 Disk 0,07/08/2021 12:16:56,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,, -Bus 0 Enclosur 0 Disk 0,07/08/2021 12:17:55,,,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,, -Bus 0 Enclosur 0 Disk 0,07/08/2021 12:18:56,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, -Bus 0 Enclosur 0 Disk 0,07/08/2021 12:19:56,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, -Port 6 [FC; 50:08:1E ],07/08/2021 12:15:56,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,, -Port 6 [FC; 50:08:1E ],07/08/2021 12:16:56,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,, -Port 6 [FC; 50:08:1E ],07/08/2021 12:17:55,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,, -Port 6 [FC; 50:08:1E ],07/08/2021 12:18:56,,,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,, -Port 6 [FC; 50:08:1E ],07/08/2021 12:19:56,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, -""" +PERFORMANCE_LINES_MAP = { + 'SP A': [['SP A', '07/08/2021 12:15:56', '', '', '', '', '', '', '', '', + '0', '', '', '0', '', '', '0', '', '', '0', '', '', '0', '', + '', '0', '', '', '0', '', '', '0', '', '', '0'], + ['SP A', '07/08/2021 12:16:56', '', '', '', '', '', '', '', '', + '0', '', '', '0', '', '', '0', '', '', '0', '', '', '0', '', '', + '0', '', '', '0', '', '', '0', '', '', '0'], + ['SP A', '07/08/2021 12:17:55', '', '', '', '', '', '', '', + '', '0', '', '', '0', '', '', '0', '', '', '0', '', '', '0', + '', '', '0', '', '', '0', '', '', '0', '', '', '0'], + ['SP A', '07/08/2021 12:18:56', '', '', '', '', '', '', '', '', + '0', '', '', '0.28', '', '', '0.73', '', '', '0', '', '', + '0', '', '', '0', '', '', '0.28', '', '', '', '', '', '0.73'], + ['SP A', '07/08/2021 12:19:56', '', '', '', '', '', '', '', '', + '0', '', '', '0', '', '', '0', '', '', '0', '', '', '0', '', + '', '0', '', '', '0', '', '', '0', '', '', '0']], + 'SP B': [['SP B', '07/08/2021 12:15:56', '', '', '', '', '', '', '', '', + '0', '', '', '0.9', '', '', '2.6', '', '', '0.9', '', '', + '2.4', '', '', '1', '', '', '0.7', '', '', '', '', '', '0.2'], + ['SP B', '07/08/2021 12:16:56', '', '', '', '', '', '', '', '', + '0', '', '', '0.1', '', '', '5.6', '', '', '0.2', '', '', '6.7', + '', '', '2', '', '', '1.6', '', '', '', '', '', '1.4'], + ['SP B', '07/08/2021 12:17:55', '', '', '', '', '', '', '', + '', '0', '', '', '0.2', '', '', '4.6', '', '', '0.3', '', '', + '1.7', '', '', '3', '', '', '2.6', '', '', '', '', '', '2.4'], + ['SP B', '07/08/2021 12:18:56', '', '', '', '', '', '', '', '', + '0', '', '', '0.3', '', '', '6.6', '', '', '0.4', '', '', + '2.7', '', '', '4', '', '', '3.6', '', '', '', '', '', '3.4'], + ['SP B', '07/08/2021 12:19:56', '', '', '', '', '', '', '', '', + '0', '', '', '0.4', '', '', '7.6', '', '', '0.5', '', '', + '3.7', '', '', '5', '', '', '4.6', '', '', '', '', '', '4.4']] +} NAR_INTERVAL_DATAS = """ Archive Poll Interval (sec): 60 """ @@ -879,8 +884,7 @@ def test_get_hosts(self): hosts = self.driver.list_storage_hosts(context) self.assertDictEqual(hosts[0], HOST_RESULT[0]) - @patch("builtins.open", new_callable=mock_open, read_data=PERFORMANCE_FILE) - def test_get_perf_metrics(self, mock_file): + def test_get_perf_metrics(self): driver = create_driver() resource_metrics = { 'controller': [ @@ -908,11 +912,12 @@ def test_get_perf_metrics(self, mock_file): } start_time = 1625717756000 end_time = 1625717996000 + ComponentHandler._filter_performance_data = mock.Mock( + side_effect=[PERFORMANCE_LINES_MAP]) NaviClient.exec = mock.Mock( side_effect=[ARCHIVE_DATAS, SP_DATAS, PORT_DATAS, DISK_DATAS, - GET_ALL_LUN_INFOS, '', '', NAR_INTERVAL_DATAS]) - os.path.exists = mock.Mock(return_value=True) - os.remove = mock.Mock(return_value=True) + GET_ALL_LUN_INFOS, NAR_INTERVAL_DATAS]) + ComponentHandler._remove_archive_file = mock.Mock(return_value="") metrics = driver.collect_perf_metrics(context, '12345', resource_metrics, start_time, end_time) From acd4563b868e047990ae04cf555c68208e55fe7a Mon Sep 17 00:00:00 2001 From: yuanyu Date: Mon, 30 May 2022 14:12:06 +0800 Subject: [PATCH 03/13] Add collect performance interface --- .../vnx/vnx_block/component_handler.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index ddfa94781..399b2d74c 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -646,14 +646,14 @@ def _handle_replenish_point(self, metrics, start_time): LOG.info("Get nar_interval:{}".format(nar_interval)) replenish_point_interval = nar_interval * units.k check_nar_interval = nar_interval * units.k * 1.5 - for m in metrics: + for metric in metrics: new_values_map = {} previous_time = '' previous_value = '' - for collection_time in m.values.keys(): + for collection_time in metric.values.keys(): if previous_time == '': previous_time = collection_time - previous_value = m.values.get(collection_time) + previous_value = metric.values.get(collection_time) if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ < previous_time: new_values_map[previous_time] = \ @@ -661,9 +661,9 @@ def _handle_replenish_point(self, metrics, start_time): continue next_time = collection_time replenish_point_value = 0 - if previous_value and m.values.get(next_time): + if previous_value and metric.values.get(next_time): replenish_point_value = \ - (previous_value + m.values.get(next_time)) / 2 + (previous_value + metric.values.get(next_time)) / 2 while (next_time - previous_time) > check_nar_interval: replenish_point_time = next_time - replenish_point_interval if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ @@ -674,12 +674,12 @@ def _handle_replenish_point(self, metrics, start_time): if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ < collection_time: new_values_map[collection_time] = \ - float('%.6f' % m.values.get(collection_time)) + float('%.6f' % metric.values.get(collection_time)) previous_time = collection_time - previous_value = m.values.get(collection_time) + previous_value = metric.values.get(collection_time) sorted_map = sorted(new_values_map.items(), key=lambda x: x[0]) - m.values.clear() - m.values.update(sorted_map) + metric.values.clear() + metric.values.update(sorted_map) return metrics def _get__archive_file(self, start_time, end_time): From 1930b81595b750856e347f5811b36ddfb6e969c6 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Wed, 1 Jun 2022 09:39:11 +0800 Subject: [PATCH 04/13] Add collect performance interface --- .../vnx/vnx_block/component_handler.py | 52 +++---------------- .../drivers/dell_emc/vnx/vnx_block/consts.py | 1 + 2 files changed, 7 insertions(+), 46 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index 399b2d74c..4d6a1ddf7 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -598,7 +598,6 @@ def collect_perf_metrics(self, storage_id, resource_metrics, metrics = self.create_metrics(storage_id, resource_metrics, resources_map, resources_type_map, performance_lines_map) - metrics = self._handle_replenish_point(metrics, start_time) except exception.DelfinException as err: err_msg = "Failed to collect metrics from VnxBlockStor: %s" % \ (six.text_type(err)) @@ -639,49 +638,6 @@ def create_metrics(self, storage_id, resource_metrics, resources_map, metrics.extend(metric_model_list) return metrics - def _handle_replenish_point(self, metrics, start_time): - if not metrics: - return metrics - nar_interval = self.navi_handler.get_nar_interval() - LOG.info("Get nar_interval:{}".format(nar_interval)) - replenish_point_interval = nar_interval * units.k - check_nar_interval = nar_interval * units.k * 1.5 - for metric in metrics: - new_values_map = {} - previous_time = '' - previous_value = '' - for collection_time in metric.values.keys(): - if previous_time == '': - previous_time = collection_time - previous_value = metric.values.get(collection_time) - if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ - < previous_time: - new_values_map[previous_time] = \ - float('%.6f' % previous_value) - continue - next_time = collection_time - replenish_point_value = 0 - if previous_value and metric.values.get(next_time): - replenish_point_value = \ - (previous_value + metric.values.get(next_time)) / 2 - while (next_time - previous_time) > check_nar_interval: - replenish_point_time = next_time - replenish_point_interval - if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ - < replenish_point_time: - new_values_map[replenish_point_time] = \ - float('%.6f' % replenish_point_value) - next_time = replenish_point_time - if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ - < collection_time: - new_values_map[collection_time] = \ - float('%.6f' % metric.values.get(collection_time)) - previous_time = collection_time - previous_value = metric.values.get(collection_time) - sorted_map = sorted(new_values_map.items(), key=lambda x: x[0]) - metric.values.clear() - metric.values.update(sorted_map) - return metrics - def _get__archive_file(self, start_time, end_time): archive_file_list = [] archives = self.navi_handler.get_archives() @@ -717,7 +673,11 @@ def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, value = '0' collection_timestamp = tools.time_str_to_timestamp( metric_value_infos[1], consts.TIME_PATTERN) - values[collection_timestamp] = float('%.6f' % float(value)) + collection_time_str = tools.timestamp_to_time_str( + collection_timestamp, consts.COLLECTION_TIME_PATTERN) + collection_timestamp = tools.time_str_to_timestamp( + collection_time_str, consts.COLLECTION_TIME_PATTERN) + values[collection_timestamp] = float('%.6f' % (float(value))) if values: metric_model = constants.metric_struct(name=metric_name, labels=obj_labels, @@ -831,7 +791,7 @@ def _package_performance_data(self, row, resources_map, start_time, if resource_obj_name in resources_map.keys(): obj_collection_timestamp = tools.time_str_to_timestamp( row[1], consts.TIME_PATTERN) - if (start_time - consts.TIME_INTERVAL_FLUCTUATION) \ + if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ <= obj_collection_timestamp \ and obj_collection_timestamp \ <= (end_time + consts.TIME_INTERVAL_FLUCTUATION): diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/consts.py b/delfin/drivers/dell_emc/vnx/vnx_block/consts.py index 9fbfc73d2..ad36eeec0 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/consts.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/consts.py @@ -324,3 +324,4 @@ REPLACE_PATH = "/delfin/drivers/dell_emc/vnx/vnx_block" # Unit: s CHECK_WAITE_TIME_SECONDS = 15 +COLLECTION_TIME_PATTERN = '%m/%d/%Y %H:%M:00' From 8c82538314691f2bda48910d0eb6c1353f91c1b7 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Wed, 15 Jun 2022 10:07:05 +0800 Subject: [PATCH 05/13] Modify bug --- delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py | 6 ++++-- .../unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index 4d6a1ddf7..d5aef2e6c 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -256,8 +256,10 @@ def list_disks(self, storage_id): hot_spare = disk.get('hot_spare', '') if hot_spare and hot_spare != 'N/A': logical_type = constants.DiskLogicalType.HOTSPARE + disk_name = disk.get('disk_name') + disk_name = disk_name.replace(' Disk', ' Disk') disk_model = { - 'name': disk.get('disk_name'), + 'name': disk_name, 'storage_id': storage_id, 'native_disk_id': disk.get('disk_id'), 'serial_number': disk.get('serial_number'), @@ -273,7 +275,7 @@ def list_disks(self, storage_id): 'logical_type': logical_type, 'health_score': None, 'native_disk_group_id': None, - 'location': disk.get('disk_name') + 'location': disk_name } disk_list.append(disk_model) return disk_list diff --git a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py index 457bbee84..31d6d4bc7 100644 --- a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py +++ b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py @@ -552,7 +552,7 @@ } DISK_RESULT = [ { - 'name': 'Bus 0 Enclosure 0 Disk 0', + 'name': 'Bus 0 Enclosure 0 Disk 0', 'storage_id': '12345', 'native_disk_id': 'Bus0Enclosure0Disk0', 'serial_number': 'KSJEX35J', @@ -566,7 +566,7 @@ 'logical_type': 'unknown', 'health_score': None, 'native_disk_group_id': None, - 'location': 'Bus 0 Enclosure 0 Disk 0' + 'location': 'Bus 0 Enclosure 0 Disk 0' }] SP_RESULT = [ { From f9b382e03bd5f98313a60a8fd424e7aab47e2407 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Thu, 16 Jun 2022 11:54:35 +0800 Subject: [PATCH 06/13] Modify bug --- delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index d5aef2e6c..dab2915c2 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -257,7 +257,7 @@ def list_disks(self, storage_id): if hot_spare and hot_spare != 'N/A': logical_type = constants.DiskLogicalType.HOTSPARE disk_name = disk.get('disk_name') - disk_name = disk_name.replace(' Disk', ' Disk') + disk_name = ' '.join(disk_name.strip().split()) disk_model = { 'name': disk_name, 'storage_id': storage_id, @@ -740,7 +740,7 @@ def _get_disks_map(self): disks = self.navi_handler.get_disks() for disk in (disks or []): disk_name = disk.get('disk_name') - disk_name = disk_name.replace(' Disk', ' Disk') + disk_name = ' '.join(disk_name.strip().split()) resources_map[disk_name] = disk.get('disk_id') resources_type_map[disk_name] = constants.ResourceType.DISK return resources_map, resources_type_map From 3e14813a50858d284ee0c9fdf101124752db1049 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Thu, 16 Jun 2022 16:51:43 +0800 Subject: [PATCH 07/13] Modify bug --- .../vnx/vnx_block/component_handler.py | 6 +- .../drivers/dell_emc/vnx/vnx_block/consts.py | 278 ++++++++++-------- 2 files changed, 162 insertions(+), 122 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index dab2915c2..e068efb6e 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -679,7 +679,11 @@ def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, collection_timestamp, consts.COLLECTION_TIME_PATTERN) collection_timestamp = tools.time_str_to_timestamp( collection_time_str, consts.COLLECTION_TIME_PATTERN) - values[collection_timestamp] = float('%.6f' % (float(value))) + if "iops" in metric_name.lower(): + value = int(float(value)) + else: + value = float('%.6f' % (float(value))) + values[collection_timestamp] = value if values: metric_model = constants.metric_struct(name=metric_name, labels=obj_labels, diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/consts.py b/delfin/drivers/dell_emc/vnx/vnx_block/consts.py index ad36eeec0..c1c77683a 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/consts.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/consts.py @@ -13,6 +13,8 @@ # limitations under the License. from delfin import exception from delfin.common import constants +from delfin.common.constants import ControllerMetric, DiskMetric, PortMetric, \ + VolumeMetric SOCKET_TIMEOUT = 30 LOGIN_SOCKET_TIMEOUT = 10 @@ -155,103 +157,137 @@ ALU_PAIRS_PATTERN = '^[0-9]+\\s+[0-9]+$' HBA_UID_PATTERN = "^\\s*HBA UID\\s+SP Name\\s+SPPort" -IOPS_DESCRIPTION = { - "unit": "IOPS", - "description": "Input/output operations per second" -} -READ_IOPS_DESCRIPTION = { - "unit": "IOPS", - "description": "Read input/output operations per second" -} -WRITE_IOPS_DESCRIPTION = { - "unit": "IOPS", - "description": "Write input/output operations per second" -} -THROUGHPUT_DESCRIPTION = { - "unit": "MB/s", - "description": "Represents how much data is " - "successfully transferred in MB/s" -} -READ_THROUGHPUT_DESCRIPTION = { - "unit": "MB/s", - "description": "Represents how much data read is " - "successfully transferred in MB/s" -} -WRITE_THROUGHPUT_DESCRIPTION = { - "unit": "MB/s", - "description": "Represents how much data write is " - "successfully transferred in MB/s" -} -RESPONSE_TIME_DESCRIPTION = { - "unit": "ms", - "description": "Average time taken for an IO " - "operation in ms" -} -CACHE_HIT_RATIO_DESCRIPTION = { - "unit": "%", - "description": "Percentage of io that are cache hits" -} -READ_CACHE_HIT_RATIO_DESCRIPTION = { - "unit": "%", - "description": "Percentage of read ops that are cache hits" -} -WRITE_CACHE_HIT_RATIO_DESCRIPTION = { - "unit": "%", - "description": "Percentage of write ops that are cache hits" -} -IO_SIZE_DESCRIPTION = { - "unit": "KB", - "description": "The average size of IO requests in KB" -} -READ_IO_SIZE_DESCRIPTION = { - "unit": "KB", - "description": "The average size of read IO requests in KB" -} -WRITE_IO_SIZE_DESCRIPTION = { - "unit": "KB", - "description": "The average size of write IO requests in KB" -} CONTROLLER_CAP = { - "iops": IOPS_DESCRIPTION, - "readIops": READ_IOPS_DESCRIPTION, - "writeIops": WRITE_IOPS_DESCRIPTION, - "throughput": THROUGHPUT_DESCRIPTION, - "readThroughput": READ_THROUGHPUT_DESCRIPTION, - "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, - "responseTime": RESPONSE_TIME_DESCRIPTION + ControllerMetric.IOPS.name: { + "unit": ControllerMetric.IOPS.unit, + "description": ControllerMetric.IOPS.description + }, + ControllerMetric.READ_IOPS.name: { + "unit": ControllerMetric.READ_IOPS.unit, + "description": ControllerMetric.READ_IOPS.description + }, + ControllerMetric.WRITE_IOPS.name: { + "unit": ControllerMetric.WRITE_IOPS.unit, + "description": ControllerMetric.WRITE_IOPS.description + }, + ControllerMetric.THROUGHPUT.name: { + "unit": ControllerMetric.THROUGHPUT.unit, + "description": ControllerMetric.THROUGHPUT.description + }, + ControllerMetric.READ_THROUGHPUT.name: { + "unit": ControllerMetric.READ_THROUGHPUT.unit, + "description": ControllerMetric.READ_THROUGHPUT.description + }, + ControllerMetric.WRITE_THROUGHPUT.name: { + "unit": ControllerMetric.WRITE_THROUGHPUT.unit, + "description": ControllerMetric.WRITE_THROUGHPUT.description + }, + ControllerMetric.RESPONSE_TIME.name: { + "unit": ControllerMetric.RESPONSE_TIME.unit, + "description": ControllerMetric.RESPONSE_TIME.description + } } VOLUME_CAP = { - "iops": IOPS_DESCRIPTION, - "readIops": READ_IOPS_DESCRIPTION, - "writeIops": WRITE_IOPS_DESCRIPTION, - "throughput": THROUGHPUT_DESCRIPTION, - "readThroughput": READ_THROUGHPUT_DESCRIPTION, - "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, - "responseTime": RESPONSE_TIME_DESCRIPTION, - "cacheHitRatio": CACHE_HIT_RATIO_DESCRIPTION, - "readCacheHitRatio": READ_CACHE_HIT_RATIO_DESCRIPTION, - "writeCacheHitRatio": WRITE_CACHE_HIT_RATIO_DESCRIPTION, - "ioSize": IO_SIZE_DESCRIPTION, - "readIoSize": READ_IO_SIZE_DESCRIPTION, - "writeIoSize": WRITE_IO_SIZE_DESCRIPTION + VolumeMetric.IOPS.name: { + "unit": VolumeMetric.IOPS.unit, + "description": VolumeMetric.IOPS.description + }, + VolumeMetric.READ_IOPS.name: { + "unit": VolumeMetric.READ_IOPS.unit, + "description": VolumeMetric.READ_IOPS.description + }, + VolumeMetric.WRITE_IOPS.name: { + "unit": VolumeMetric.WRITE_IOPS.unit, + "description": VolumeMetric.WRITE_IOPS.description + }, + VolumeMetric.THROUGHPUT.name: { + "unit": VolumeMetric.THROUGHPUT.unit, + "description": VolumeMetric.THROUGHPUT.description + }, + VolumeMetric.READ_THROUGHPUT.name: { + "unit": VolumeMetric.READ_THROUGHPUT.unit, + "description": VolumeMetric.READ_THROUGHPUT.description + }, + VolumeMetric.WRITE_THROUGHPUT.name: { + "unit": VolumeMetric.WRITE_THROUGHPUT.unit, + "description": VolumeMetric.WRITE_THROUGHPUT.description + }, + VolumeMetric.RESPONSE_TIME.name: { + "unit": VolumeMetric.RESPONSE_TIME.unit, + "description": VolumeMetric.RESPONSE_TIME.description + }, + VolumeMetric.READ_CACHE_HIT_RATIO.name: { + "unit": VolumeMetric.READ_CACHE_HIT_RATIO.unit, + "description": VolumeMetric.READ_CACHE_HIT_RATIO.description + }, + VolumeMetric.WRITE_CACHE_HIT_RATIO.name: { + "unit": VolumeMetric.WRITE_CACHE_HIT_RATIO.unit, + "description": VolumeMetric.WRITE_CACHE_HIT_RATIO.description + }, + VolumeMetric.READ_IO_SIZE.name: { + "unit": VolumeMetric.READ_IO_SIZE.unit, + "description": VolumeMetric.READ_IO_SIZE.description + }, + VolumeMetric.WRITE_IO_SIZE.name: { + "unit": VolumeMetric.WRITE_IO_SIZE.unit, + "description": VolumeMetric.WRITE_IO_SIZE.description + } } PORT_CAP = { - "iops": IOPS_DESCRIPTION, - "readIops": READ_IOPS_DESCRIPTION, - "writeIops": WRITE_IOPS_DESCRIPTION, - "throughput": THROUGHPUT_DESCRIPTION, - "readThroughput": READ_THROUGHPUT_DESCRIPTION, - "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, - "responseTime": RESPONSE_TIME_DESCRIPTION + PortMetric.IOPS.name: { + "unit": PortMetric.IOPS.unit, + "description": PortMetric.IOPS.description + }, + PortMetric.READ_IOPS.name: { + "unit": PortMetric.READ_IOPS.unit, + "description": PortMetric.READ_IOPS.description + }, + PortMetric.WRITE_IOPS.name: { + "unit": PortMetric.WRITE_IOPS.unit, + "description": PortMetric.WRITE_IOPS.description + }, + PortMetric.THROUGHPUT.name: { + "unit": PortMetric.THROUGHPUT.unit, + "description": PortMetric.THROUGHPUT.description + }, + PortMetric.READ_THROUGHPUT.name: { + "unit": PortMetric.READ_THROUGHPUT.unit, + "description": PortMetric.READ_THROUGHPUT.description + }, + PortMetric.WRITE_THROUGHPUT.name: { + "unit": PortMetric.WRITE_THROUGHPUT.unit, + "description": PortMetric.WRITE_THROUGHPUT.description + } } DISK_CAP = { - "iops": IOPS_DESCRIPTION, - "readIops": READ_IOPS_DESCRIPTION, - "writeIops": WRITE_IOPS_DESCRIPTION, - "throughput": THROUGHPUT_DESCRIPTION, - "readThroughput": READ_THROUGHPUT_DESCRIPTION, - "writeThroughput": WRITE_THROUGHPUT_DESCRIPTION, - "responseTime": RESPONSE_TIME_DESCRIPTION + DiskMetric.IOPS.name: { + "unit": DiskMetric.IOPS.unit, + "description": DiskMetric.IOPS.description + }, + DiskMetric.READ_IOPS.name: { + "unit": DiskMetric.READ_IOPS.unit, + "description": DiskMetric.READ_IOPS.description + }, + DiskMetric.WRITE_IOPS.name: { + "unit": DiskMetric.WRITE_IOPS.unit, + "description": DiskMetric.WRITE_IOPS.description + }, + DiskMetric.THROUGHPUT.name: { + "unit": DiskMetric.THROUGHPUT.unit, + "description": DiskMetric.THROUGHPUT.description + }, + DiskMetric.READ_THROUGHPUT.name: { + "unit": DiskMetric.READ_THROUGHPUT.unit, + "description": DiskMetric.READ_THROUGHPUT.description + }, + DiskMetric.WRITE_THROUGHPUT.name: { + "unit": DiskMetric.WRITE_THROUGHPUT.unit, + "description": DiskMetric.WRITE_THROUGHPUT.description + }, + DiskMetric.RESPONSE_TIME.name: { + "unit": DiskMetric.RESPONSE_TIME.unit, + "description": DiskMetric.RESPONSE_TIME.description + } } RESOURCES_TYPE_TO_METRIC_CAP = { constants.ResourceType.CONTROLLER: CONTROLLER_CAP, @@ -261,43 +297,43 @@ } METRIC_MAP = { constants.ResourceType.CONTROLLER: { - "iops": 16, - "readIops": 25, - "writeIops": 34, - "throughput": 13, - "readThroughput": 19, - "writeThroughput": 28, - "responseTime": 10 + ControllerMetric.IOPS.name: 16, + ControllerMetric.READ_IOPS.name: 25, + ControllerMetric.WRITE_IOPS.name: 34, + ControllerMetric.THROUGHPUT.name: 13, + ControllerMetric.READ_THROUGHPUT.name: 19, + ControllerMetric.WRITE_THROUGHPUT.name: 28, + ControllerMetric.RESPONSE_TIME.name: 10 }, constants.ResourceType.PORT: { - "iops": 16, - "readIops": 25, - "writeIops": 34, - "throughput": 13, - "readThroughput": 19, - "writeThroughput": 28 + PortMetric.IOPS.name: 16, + PortMetric.READ_IOPS.name: 25, + PortMetric.WRITE_IOPS.name: 34, + PortMetric.THROUGHPUT.name: 13, + PortMetric.READ_THROUGHPUT.name: 19, + PortMetric.WRITE_THROUGHPUT.name: 28 }, constants.ResourceType.DISK: { - "iops": 16, - "readIops": 25, - "writeIops": 34, - "throughput": 13, - "readThroughput": 19, - "writeThroughput": 28, - "responseTime": 10 + DiskMetric.IOPS.name: 16, + DiskMetric.READ_IOPS.name: 25, + DiskMetric.WRITE_IOPS.name: 34, + DiskMetric.THROUGHPUT.name: 13, + DiskMetric.READ_THROUGHPUT.name: 19, + DiskMetric.WRITE_THROUGHPUT.name: 28, + DiskMetric.RESPONSE_TIME.name: 10 }, constants.ResourceType.VOLUME: { - "iops": 16, - "readIops": 25, - "writeIops": 34, - "throughput": 13, - "readThroughput": 19, - "writeThroughput": 28, - "responseTime": 10, - "readCacheHitRatio": 42, - "writeCacheHitRatio": 45, - "readIoSize": 22, - "writeIoSize": 31 + VolumeMetric.IOPS.name: 16, + VolumeMetric.READ_IOPS.name: 25, + VolumeMetric.WRITE_IOPS.name: 34, + VolumeMetric.THROUGHPUT.name: 13, + VolumeMetric.READ_THROUGHPUT.name: 19, + VolumeMetric.WRITE_THROUGHPUT.name: 28, + VolumeMetric.RESPONSE_TIME.name: 10, + VolumeMetric.READ_CACHE_HIT_RATIO.name: 42, + VolumeMetric.WRITE_CACHE_HIT_RATIO.name: 45, + VolumeMetric.READ_IO_SIZE.name: 22, + VolumeMetric.WRITE_IO_SIZE.name: 31 } } From 4bfaf1bd427262efb4ec4088b387851ccb0cfac7 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Fri, 17 Jun 2022 09:35:56 +0800 Subject: [PATCH 08/13] Modify bug --- .../vnx/vnx_block/component_handler.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index e068efb6e..c109ad3e1 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -16,7 +16,6 @@ import os import re import time -from io import StringIO import six from oslo_log import log @@ -578,10 +577,8 @@ def collect_perf_metrics(self, storage_id, resource_metrics, metrics = [] archive_file_list = [] try: - if (end_time - start_time) < consts.EXEC_TIME_INTERVAL: - LOG.warn("not exe collert, end_time-start_time={}".format( - (end_time - start_time))) - return metrics + LOG.warn("开始采集, storage:%s, start time:%s, end time:%s" + % (storage_id, start_time, end_time)) archive_file_list = self._get__archive_file(start_time, end_time) LOG.info("Get archive files: {}".format(archive_file_list)) if not archive_file_list: @@ -600,6 +597,8 @@ def collect_perf_metrics(self, storage_id, resource_metrics, metrics = self.create_metrics(storage_id, resource_metrics, resources_map, resources_type_map, performance_lines_map) + LOG.warn("采集完成, storage:%s, start time:%s, end time:%s, length of metrics:%s " + % (storage_id, start_time, end_time, len(metrics))) except exception.DelfinException as err: err_msg = "Failed to collect metrics from VnxBlockStor: %s" % \ (six.text_type(err)) @@ -661,13 +660,11 @@ def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, tools = Tools() for metric_name in (metric_list or []): values = {} - obj_labels = copy.deepcopy(labels) + obj_labels = copy.copy(labels) obj_labels['unit'] = obj_cap.get(metric_name).get('unit') for metric_value in metric_values: metric_value_infos = metric_value - if not consts.METRIC_MAP.get(resources_type): - continue - if not consts.METRIC_MAP.get(resources_type).get(metric_name): + if not consts.METRIC_MAP.get(resources_type, {}).get(metric_name): continue value = metric_value_infos[ consts.METRIC_MAP.get(resources_type).get(metric_name)] @@ -679,7 +676,7 @@ def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, collection_timestamp, consts.COLLECTION_TIME_PATTERN) collection_timestamp = tools.time_str_to_timestamp( collection_time_str, consts.COLLECTION_TIME_PATTERN) - if "iops" in metric_name.lower(): + if "iops" == obj_cap.get(metric_name).get('unit').lower(): value = int(float(value)) else: value = float('%.6f' % (float(value))) @@ -766,17 +763,15 @@ def _filter_performance_data(self, archive_file_list, resources_map, start_time, end_time): performance_lines_map = {} try: - if not resources_map: - return performance_lines_map tools = Tools() - for archive_file in (archive_file_list or []): + for archive_file in archive_file_list: self.navi_handler.download_archives(archive_file) archive_name_infos = archive_file.split('.') file_path = '%s%s.csv' % ( self.navi_handler.get_local_file_path(), archive_name_infos[0]) with open(file_path) as file: - f_csv = csv.reader(StringIO(file.read())) + f_csv = csv.reader(file) next(f_csv) for row in f_csv: self._package_performance_data(row, resources_map, @@ -850,15 +845,15 @@ def get_latest_perf_timestamp(self, storage_id): storage_id) if num > consts.EXEC_MAX_NUM: latest_time = file_latest_time - LOG.warn("Storage：{}，Exit after {} executions.".format( + LOG.warn("Storage:{}, Exit after {} executions.".format( storage_id, consts.EXEC_MAX_NUM)) break if latest_time <= 0: wait_time = tools.timestamp_to_time_str( time.time() * units.k, consts.ARCHIVE_FILE_NAME_TIME_PATTERN) - LOG.warn("Storage：{} No new file found, " - "wait for next execution：{}".format(storage_id, + LOG.warn("Storage:{} No new file found, " + "wait for next execution:{}".format(storage_id, wait_time)) time.sleep(consts.SLEEP_TIME_SECONDS) return latest_time From 36dc5009640ef9c85682a594e4c33711b1e1ed0e Mon Sep 17 00:00:00 2001 From: yuanyu Date: Fri, 17 Jun 2022 10:18:37 +0800 Subject: [PATCH 09/13] Modify bug --- .../vnx/vnx_block/component_handler.py | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index c109ad3e1..c3a128554 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -577,7 +577,7 @@ def collect_perf_metrics(self, storage_id, resource_metrics, metrics = [] archive_file_list = [] try: - LOG.warn("开始采集, storage:%s, start time:%s, end time:%s" + LOG.warn("Start collection, storage:%s, start time:%s, end time:%s" % (storage_id, start_time, end_time)) archive_file_list = self._get__archive_file(start_time, end_time) LOG.info("Get archive files: {}".format(archive_file_list)) @@ -597,7 +597,8 @@ def collect_perf_metrics(self, storage_id, resource_metrics, metrics = self.create_metrics(storage_id, resource_metrics, resources_map, resources_type_map, performance_lines_map) - LOG.warn("采集完成, storage:%s, start time:%s, end time:%s, length of metrics:%s " + LOG.warn("Collection complete, storage:%s, start time:%s, " + "end time:%s, length of metrics:%s " % (storage_id, start_time, end_time, len(metrics))) except exception.DelfinException as err: err_msg = "Failed to collect metrics from VnxBlockStor: %s" % \ @@ -616,25 +617,24 @@ def collect_perf_metrics(self, storage_id, resource_metrics, def create_metrics(self, storage_id, resource_metrics, resources_map, resources_type_map, performance_lines_map): metrics = [] - for resource_obj in resources_type_map.keys(): + for resource_obj, resource_type in resources_type_map.items(): if not resources_map.get(resource_obj) \ - or not resources_type_map.get(resource_obj): + or not resource_type: + continue + if not performance_lines_map.get(resource_obj): continue - resources_type = resources_type_map.get(resource_obj) labels = { 'storage_id': storage_id, - 'resource_type': resources_type, + 'resource_type': resource_type, 'resource_id': resources_map.get(resource_obj), 'type': 'RAW', 'unit': '' } - if not performance_lines_map.get(resource_obj): - continue metric_model_list = self._get_metric_model( - resource_metrics.get(resources_type), labels, + resource_metrics.get(resource_type), labels, performance_lines_map.get(resource_obj), - consts.RESOURCES_TYPE_TO_METRIC_CAP.get( - resources_type), resources_type) + consts.RESOURCES_TYPE_TO_METRIC_CAP.get(resource_type), + resource_type) if metric_model_list: metrics.extend(metric_model_list) return metrics @@ -655,8 +655,6 @@ def _get__archive_file(self, start_time, end_time): def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, resources_type): metric_model_list = [] - if not metric_values: - return metric_model_list tools = Tools() for metric_name in (metric_list or []): values = {} @@ -664,7 +662,8 @@ def _get_metric_model(self, metric_list, labels, metric_values, obj_cap, obj_labels['unit'] = obj_cap.get(metric_name).get('unit') for metric_value in metric_values: metric_value_infos = metric_value - if not consts.METRIC_MAP.get(resources_type, {}).get(metric_name): + if not consts.METRIC_MAP.get(resources_type, {}).get( + metric_name): continue value = metric_value_infos[ consts.METRIC_MAP.get(resources_type).get(metric_name)] @@ -789,27 +788,22 @@ def _package_performance_data(self, row, resources_map, start_time, end_time, tools, performance_lines_map): resource_obj_name = row[0] resource_obj_name = self._package_resource_obj_name(resource_obj_name) - if resource_obj_name in resources_map.keys(): + if resource_obj_name in resources_map: obj_collection_timestamp = tools.time_str_to_timestamp( row[1], consts.TIME_PATTERN) if (start_time + consts.TIME_INTERVAL_FLUCTUATION) \ <= obj_collection_timestamp \ and obj_collection_timestamp \ <= (end_time + consts.TIME_INTERVAL_FLUCTUATION): - if performance_lines_map.get(resource_obj_name): - performance_lines_map.get(resource_obj_name).append(row) - else: - obj_performance_list = [] - obj_performance_list.append(row) - performance_lines_map[ - resource_obj_name] = obj_performance_list + performance_lines_map.setdefault(resource_obj_name, []).append( + row) def _package_resource_obj_name(self, source_name): target_name = source_name if 'Port ' in target_name: - target_name = re.sub('(\\[.*;)', '[', target_name) + return re.sub(r'(\[.*;)', '[', target_name) elif '; ' in target_name: - target_name = re.sub('(; .*])', ']', target_name) + return re.sub(r'(; .*])', ']', target_name) return target_name def _remove_archive_file(self, archive_file_list): From 4c98fcb5a4ddd46857e218b75ede9741671a86e2 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Fri, 17 Jun 2022 17:53:51 +0800 Subject: [PATCH 10/13] Modify bug --- .../dell_emc/vnx/vnx_block/component_handler.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index c3a128554..fee6eadf6 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -577,27 +577,27 @@ def collect_perf_metrics(self, storage_id, resource_metrics, metrics = [] archive_file_list = [] try: - LOG.warn("Start collection, storage:%s, start time:%s, end time:%s" + LOG.info("Start collection, storage:%s, start time:%s, end time:%s" % (storage_id, start_time, end_time)) archive_file_list = self._get__archive_file(start_time, end_time) LOG.info("Get archive files: {}".format(archive_file_list)) if not archive_file_list: - LOG.warn("The required performance file was not found!") + LOG.warning("The required performance file was not found!") return metrics resources_map, resources_type_map = self._get_resources_map( resource_metrics) if not resources_map or not resources_type_map: - LOG.warn("Resource object not found!") + LOG.warning("Resource object not found!") return metrics performance_lines_map = self._filter_performance_data( archive_file_list, resources_map, start_time, end_time) if not performance_lines_map: - LOG.warn("The required performance data was not found!") + LOG.warning("The required performance data was not found!") return metrics metrics = self.create_metrics(storage_id, resource_metrics, resources_map, resources_type_map, performance_lines_map) - LOG.warn("Collection complete, storage:%s, start time:%s, " + LOG.info("Collection complete, storage:%s, start time:%s, " "end time:%s, length of metrics:%s " % (storage_id, start_time, end_time, len(metrics))) except exception.DelfinException as err: @@ -839,14 +839,14 @@ def get_latest_perf_timestamp(self, storage_id): storage_id) if num > consts.EXEC_MAX_NUM: latest_time = file_latest_time - LOG.warn("Storage:{}, Exit after {} executions.".format( + LOG.warning("Storage:{}, Exit after {} executions.".format( storage_id, consts.EXEC_MAX_NUM)) break if latest_time <= 0: wait_time = tools.timestamp_to_time_str( time.time() * units.k, consts.ARCHIVE_FILE_NAME_TIME_PATTERN) - LOG.warn("Storage:{} No new file found, " + LOG.warning("Storage:{} No new file found, " "wait for next execution:{}".format(storage_id, wait_time)) time.sleep(consts.SLEEP_TIME_SECONDS) From 8e3ed34229f3f6ffeb3888708014f98608df2b1b Mon Sep 17 00:00:00 2001 From: yuanyu Date: Fri, 17 Jun 2022 17:57:33 +0800 Subject: [PATCH 11/13] Modify bug --- delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index fee6eadf6..25324dfa5 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -847,8 +847,8 @@ def get_latest_perf_timestamp(self, storage_id): time.time() * units.k, consts.ARCHIVE_FILE_NAME_TIME_PATTERN) LOG.warning("Storage:{} No new file found, " - "wait for next execution:{}".format(storage_id, - wait_time)) + "wait for next execution:{}".format(storage_id, + wait_time)) time.sleep(consts.SLEEP_TIME_SECONDS) return latest_time From 1b5246f20f9c2a852eb8c9ddedbf57372e470883 Mon Sep 17 00:00:00 2001 From: yuanyu Date: Mon, 20 Jun 2022 18:02:35 +0800 Subject: [PATCH 12/13] Modify bug --- delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py | 6 +++--- .../unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index 25324dfa5..b89225b76 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -358,7 +358,7 @@ def get_ports(self, storage_id, io_configs, iscsi_port_map): ipv4 = iscsi_port.get('ip_address') ipv4_mask = iscsi_port.get('subnet_mask') port_model = { - 'name': name, + 'name': location, 'storage_id': storage_id, 'native_port_id': name, 'location': location, @@ -401,7 +401,7 @@ def get_bus_ports(self, storage_id, io_configs): location = '%s %s,Port %s' % ( bus_port.get('i/o_module_slot'), sp_name, bus_port.get('physical_port_id')) - native_port_id = location.replace(' ', '') + native_port_id = name.replace(' ', '') native_port_id = native_port_id.replace(',', '') module_key = '%s_%s' % ( sp_name, bus_port.get('i/o_module_slot')) @@ -415,7 +415,7 @@ def get_bus_ports(self, storage_id, io_configs): state = bus_port_state_map.get(port_state_key, '') port_model = { - 'name': name, + 'name': location, 'storage_id': storage_id, 'native_port_id': native_port_id, 'location': location, diff --git a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py index 31d6d4bc7..0bbdfcefa 100644 --- a/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py +++ b/delfin/tests/unit/drivers/dell_emc/vnx/vnx_block/test_vnx_block.py @@ -591,7 +591,7 @@ }] PORT_RESULT = [ { - 'name': 'A-6', + 'name': 'Slot A3,Port 0', 'storage_id': '12345', 'native_port_id': 'A-6', 'location': 'Slot A3,Port 0', @@ -625,7 +625,7 @@ 'native_storage_host_initiator_id': '20:00:00:00:C9:9B:57:79:10:' '00:00:00:C9:9B:57:79', 'wwn': '20:00:00:00:C9:9B:57:79:10:00:00:00:C9:9B:57:79', - 'type': 'fc', + 'type': 'unknown', 'status': 'online', 'native_storage_host_id': 'aix_ma' }] From a2a0f1c9e5ee98f498127cfeb59f78f01be8729a Mon Sep 17 00:00:00 2001 From: yuanyu Date: Tue, 21 Jun 2022 09:12:27 +0800 Subject: [PATCH 13/13] Modify bug --- delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py index b89225b76..220fc72a6 100644 --- a/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py +++ b/delfin/drivers/dell_emc/vnx/vnx_block/component_handler.py @@ -396,12 +396,10 @@ def get_bus_ports(self, storage_id, io_configs): sps = bus_port.get('sps') for sp in (sps or []): sp_name = sp.replace('sp', '').upper() - name = '%s-%s' % (sp_name, - bus_port.get('bus_name')) location = '%s %s,Port %s' % ( bus_port.get('i/o_module_slot'), sp_name, bus_port.get('physical_port_id')) - native_port_id = name.replace(' ', '') + native_port_id = location.replace(' ', '') native_port_id = native_port_id.replace(',', '') module_key = '%s_%s' % ( sp_name, bus_port.get('i/o_module_slot'))