diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index c2b2ab9d4..c099b48f3 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -95,6 +95,82 @@ PARSE_ALERT_SEVERITY_MAP = {'1': constants.Severity.WARNING, '2': constants.Severity.INFORMATIONAL} +# collect_perf_metrics method +SIXTY = 60 +LIST_METRICS = -1 + +STORAGE_CAP = { + constants.StorageMetric.IOPS.name: { + "unit": constants.StorageMetric.IOPS.unit, + "description": constants.StorageMetric.IOPS.description + }, + constants.StorageMetric.READ_IOPS.name: { + "unit": constants.StorageMetric.READ_IOPS.unit, + "description": constants.StorageMetric.READ_IOPS.description + }, + constants.StorageMetric.WRITE_IOPS.name: { + "unit": constants.StorageMetric.WRITE_IOPS.unit, + "description": constants.StorageMetric.WRITE_IOPS.description + }, + constants.StorageMetric.THROUGHPUT.name: { + "unit": constants.StorageMetric.THROUGHPUT.unit, + "description": constants.StorageMetric.THROUGHPUT.description + }, + constants.StorageMetric.READ_THROUGHPUT.name: { + "unit": constants.StorageMetric.READ_THROUGHPUT.unit, + "description": constants.StorageMetric.READ_THROUGHPUT.description + }, + constants.StorageMetric.WRITE_THROUGHPUT.name: { + "unit": constants.StorageMetric.WRITE_THROUGHPUT.unit, + "description": constants.StorageMetric.WRITE_THROUGHPUT.description + }, + constants.StorageMetric.READ_RESPONSE_TIME.name: { + "unit": constants.StorageMetric.READ_RESPONSE_TIME.unit, + "description": constants.StorageMetric.READ_RESPONSE_TIME.description + }, + constants.StorageMetric.WRITE_RESPONSE_TIME.name: { + "unit": constants.StorageMetric.WRITE_RESPONSE_TIME.unit, + "description": constants.StorageMetric.WRITE_RESPONSE_TIME.description + } +} +VOLUME_CAP = { + constants.VolumeMetric.IOPS.name: { + "unit": constants.VolumeMetric.IOPS.unit, + "description": constants.VolumeMetric.IOPS.description + }, + constants.VolumeMetric.READ_IOPS.name: { + "unit": constants.VolumeMetric.READ_IOPS.unit, + "description": constants.VolumeMetric.READ_IOPS.description + }, + constants.VolumeMetric.WRITE_IOPS.name: { + "unit": constants.VolumeMetric.WRITE_IOPS.unit, + "description": constants.VolumeMetric.WRITE_IOPS.description + }, + constants.VolumeMetric.THROUGHPUT.name: { + "unit": constants.VolumeMetric.THROUGHPUT.unit, + "description": constants.VolumeMetric.THROUGHPUT.description + }, + constants.VolumeMetric.READ_THROUGHPUT.name: { + "unit": constants.VolumeMetric.READ_THROUGHPUT.unit, + "description": constants.VolumeMetric.READ_THROUGHPUT.description + }, + constants.VolumeMetric.WRITE_THROUGHPUT.name: { + "unit": constants.VolumeMetric.WRITE_THROUGHPUT.unit, + "description": constants.VolumeMetric.WRITE_THROUGHPUT.description + }, + constants.VolumeMetric.READ_RESPONSE_TIME.name: { + "unit": constants.VolumeMetric.READ_RESPONSE_TIME.unit, + "description": constants.VolumeMetric.READ_RESPONSE_TIME.description + }, + constants.VolumeMetric.WRITE_RESPONSE_TIME.name: { + "unit": constants.VolumeMetric.WRITE_RESPONSE_TIME.unit, + "description": constants.VolumeMetric.WRITE_RESPONSE_TIME.description + } +} + +# Timestamp format conversion +PURE_TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + HOST_OS_TYPES_MAP = { 'linux': constants.HostOSTypes.LINUX, 'windows': constants.HostOSTypes.WINDOWS, diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index f5a9bfb5f..82f8144c3 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -16,6 +16,7 @@ import time from oslo_log import log +from oslo_utils import units from delfin import exception, utils from delfin.common import constants @@ -124,13 +125,12 @@ def list_alerts(self, context, query_para=None): for alert in alerts: alerts_model = dict() opened = alert.get('opened') - time_difference = time.mktime( - time.localtime()) - time.mktime(time.gmtime()) - timestamp = (int(datetime.datetime.strptime - (opened, '%Y-%m-%dT%H:%M:%SZ').timestamp() - + time_difference) * - consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) if \ - opened is not None else None + time_difference = self.get_time_difference() + timestamp = (int(datetime.datetime.strptime( + opened, consts.PURE_TIME_FORMAT).timestamp() + + time_difference) * consts. + DEFAULT_LIST_ALERTS_TIME_CONVERSION)\ + if opened is not None else None if query_para is not None: try: if timestamp is None or timestamp \ @@ -158,6 +158,12 @@ def list_alerts(self, context, query_para=None): alerts_list.append(alerts_model) return alerts_list + @staticmethod + def get_time_difference(): + time_difference = time.mktime( + time.localtime()) - time.mktime(time.gmtime()) + return time_difference + @staticmethod def parse_alert(context, alert): try: @@ -363,6 +369,210 @@ def reset_connection(self, context, **kwargs): def get_access_url(): return 'https://{ip}' + def collect_perf_metrics(self, context, storage_id, resource_metrics, + start_time, end_time): + LOG.info('The system(storage_id: %s) starts to collect storage and' + ' volume performance, start_time: %s, end_time: %s', + storage_id, start_time, end_time) + metrics = [] + if resource_metrics.get(constants.ResourceType.STORAGE): + storage_metrics = self.get_storage_metrics( + storage_id, + resource_metrics.get(constants.ResourceType.STORAGE), + start_time, end_time, + constants.ResourceType.STORAGE) + metrics.extend(storage_metrics) + LOG.info('The system(storage_id: %s) stop to collect storage' + ' performance, The length is: %s', + storage_id, len(storage_metrics)) + if resource_metrics.get(constants.ResourceType.VOLUME): + volume_metrics = self.get_volume_metrics( + storage_id, + resource_metrics.get(constants.ResourceType.VOLUME), + start_time, end_time, + constants.ResourceType.VOLUME) + metrics.extend(volume_metrics) + LOG.info('The system(storage_id: %s) stop to collect volume' + ' performance, The length is: %s', + storage_id, len(volume_metrics)) + return metrics + + def get_storage_metrics(self, storage_id, resource_metrics, start_time, + end_time, resource_type): + metrics = [] + arrays_id, arrays_name = self.get_array() + packaging_data = self.get_packaging_storage_data( + end_time, start_time, resource_type) + if not arrays_id or not arrays_name or not packaging_data or\ + end_time < start_time: + return metrics + for resource_key in resource_metrics.keys(): + labels = { + 'storage_id': storage_id, + 'resource_type': resource_type, + 'resource_id': arrays_id, + 'resource_name': arrays_name, + 'type': 'RAW', + 'unit': resource_metrics[resource_key]['unit'] + } + resource_value = {} + for about_timestamp in packaging_data.keys(): + metrics_data = packaging_data.get(about_timestamp) + resource_value[about_timestamp] = \ + metrics_data.get(resource_key) + metrics_res = constants.metric_struct( + name=resource_key, labels=labels, values=resource_value) + metrics.append(metrics_res) + return metrics + + def get_packaging_storage_data(self, end_time, start_time, resource_type): + duplicate = set() + packaging_data = {} + list_metrics = self.rest_handler.rest_call( + self.rest_handler.REST_ARRAY_HISTORICAL_URL) + for storage_metrics in (list_metrics or []): + about_timestamp = self.checkout_data( + storage_metrics, start_time, end_time, resource_type, + duplicate) + if about_timestamp is None: + continue + metrics_data = self.get_metrics_data( + storage_metrics, about_timestamp) + packaging_data[about_timestamp] = metrics_data + return packaging_data + + def checkout_data(self, storage_metrics, start_time, end_time, + resource_type, duplicate): + opened = storage_metrics.get('time') + if opened is None: + return None + timestamp_s = self.get_timestamp_s(opened) + timestamp_ms = timestamp_s * units.k + if timestamp_ms < start_time or timestamp_ms >= end_time: + return None + about_timestamp = \ + int(timestamp_s / consts.SIXTY) * consts.SIXTY * units.k + duplicate_value = self.get_duplicate_value( + about_timestamp, resource_type, storage_metrics) + if duplicate_value in duplicate: + return None + duplicate.add(duplicate_value) + return about_timestamp + + def get_volume_metrics(self, storage_id, resource_metrics, start_time, + end_time, resource_type): + metrics = [] + packaging_data = self.get_packaging_volume_data( + end_time, resource_type, start_time) + if end_time < start_time or not packaging_data: + return metrics + for volume_name in packaging_data.keys(): + for resource_key in resource_metrics.keys(): + labels = { + 'storage_id': storage_id, + 'resource_type': resource_type, + 'resource_id': volume_name, + 'resource_name': volume_name, + 'type': 'RAW', + 'unit': resource_metrics[resource_key]['unit'] + } + resource_value = {} + for volume_metrics in (packaging_data.get(volume_name) or []): + resource_value[volume_metrics.get('time')] = \ + volume_metrics.get(resource_key) + metrics_res = constants.metric_struct( + name=resource_key, labels=labels, values=resource_value) + metrics.append(metrics_res) + return metrics + + def get_packaging_volume_data(self, end_time, resource_type, start_time): + duplicate = set() + packaging_data = {} + list_metrics = self.rest_handler.rest_call( + self.rest_handler.REST_VOLUME_HISTORICAL_URL) + for volume_metrics in (list_metrics or []): + about_timestamp = self.checkout_data( + volume_metrics, start_time, end_time, resource_type, + duplicate) + if about_timestamp is None: + continue + volume_metrics_data = self.get_metrics_data( + volume_metrics, about_timestamp) + volume_metrics_list = packaging_data.get( + volume_metrics.get('name')) + if not volume_metrics_list: + volume_metrics_list = [] + volume_metrics_list.append(volume_metrics_data) + packaging_data[volume_metrics.get('name')] = volume_metrics_list + return packaging_data + + def get_timestamp_s(self, opened): + time_difference = self.get_time_difference() + timestamp_s = int( + datetime.datetime.strptime(opened, consts.PURE_TIME_FORMAT) + .timestamp() + time_difference) + return timestamp_s + + @staticmethod + def get_duplicate_value(about_timestamp, resource_type, storage_metrics): + duplicate_value = None + if resource_type == constants.ResourceType.VOLUME: + duplicate_value = '{}{}'.format( + storage_metrics.get('name'), about_timestamp) + if resource_type == constants.ResourceType.STORAGE: + duplicate_value = about_timestamp + return duplicate_value + + @staticmethod + def get_metrics_data(metrics, about_timestamp): + read_iop = metrics.get('reads_per_sec') + write_iop = metrics.get('writes_per_sec') + read_throughput = metrics.get('output_per_sec') / units.Mi + write_throughput = metrics.get('input_per_sec') / units.Mi + read_response_time = metrics.get('usec_per_read_op') / units.k + write_response_time = metrics.get('usec_per_write_op') / units.k + metrics_data = { + 'iops': round(read_iop + write_iop, 3), + "readIops": round(read_iop, 3), + "writeIops": round(write_iop, 3), + "throughput": round(read_throughput + write_throughput, 3), + "readThroughput": round(read_throughput, 3), + "writeThroughput": round(write_throughput, 3), + "readResponseTime": round(read_response_time, 3), + "writeResponseTime": round(write_response_time, 3), + 'time': about_timestamp + } + return metrics_data + + def get_array(self): + arrays_id = None + arrays_name = None + arrays = self.rest_handler.rest_call( + self.rest_handler.REST_ARRAY_URL) + if arrays: + arrays_id = arrays.get('id') + arrays_name = arrays.get('array_name') + return arrays_id, arrays_name + + @staticmethod + def get_capabilities(context, filters=None): + return { + 'is_historic': True, + 'resource_metrics': { + constants.ResourceType.STORAGE: consts.STORAGE_CAP, + constants.ResourceType.VOLUME: consts.VOLUME_CAP + } + } + + def get_latest_perf_timestamp(self, context): + list_metrics = self.rest_handler.rest_call( + self.rest_handler.REST_ARRAY_HISTORICAL_URL) + opened = list_metrics[consts.LIST_METRICS].get('time') + timestamp_s = self.get_timestamp_s(opened) + timestamp_ms = \ + int(timestamp_s / consts.SIXTY) * consts.SIXTY * units.k + return timestamp_ms + def list_storage_host_initiators(self, context): list_initiators = [] initiators = self.rest_handler.rest_call( @@ -402,7 +612,7 @@ def list_storage_hosts(self, ctx): self.rest_handler.REST_HOST_PERSONALITY_URL) for host in (hosts or []): name = host.get('name') - personality = host.get('personality').lower()\ + personality = host.get('personality').lower() \ if host.get('personality') else None h = { "name": name, @@ -496,7 +706,7 @@ def list_masking_views(self, context): host_id = masking_view.get('name') native_volume_id = masking_view.get('vol') hgroup_name = '{}{}'.format(hgroup, native_volume_id) - if view_id_dict.get(hgroup_name) is not None and\ + if view_id_dict.get(hgroup_name) is not None and \ view_id_dict.get(hgroup_name) in hgroup: continue native_masking_view_id = '{}{}{}'.format( diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index b46cb0740..301b559a1 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -41,6 +41,9 @@ class RestHandler(RestClient): REST_HGROUP_CONNECT_URL = '/api/1.17/hgroup?connect=true' REST_HGROUP_URL = '/api/1.17/hgroup' REST_VOLUME_GROUP_URL = '/api/1.17/vgroup' + REST_ARRAY_HISTORICAL_URL = '/api/1.17/array?action=monitor&historical=1h' + REST_VOLUME_HISTORICAL_URL =\ + '/api/1.17/volume?action=monitor&historical=1h' def __init__(self, **kwargs): super(RestHandler, self).__init__(**kwargs) diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index 1dd9d1374..54a3f4a79 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys +import time from unittest import TestCase, mock import six from oslo_log import log +from oslo_utils import units + +from delfin.common import constants +from delfin.drivers.pure.flasharray import consts sys.modules['delfin.cryptor'] = mock.Mock() from delfin import context @@ -1619,6 +1624,53 @@ 'name': 'hosttestNonenc::136_connect', 'native_storage_host_id': 'hosttest', 'native_volume_id': 'nc::136_connect', 'storage_id': '12345'}] +storage_resource_metrics = { + constants.ResourceType.STORAGE: consts.STORAGE_CAP, +} +volume_resource_metrics = { + constants.ResourceType.VOLUME: consts.VOLUME_CAP +} +drive_metrics = [ + { + "writes_per_sec": 0, + "output_per_sec": 0, + "usec_per_write_op": 0, + "local_queue_usec_per_op": 0, + "time": "2022-04-25T02:24:46Z", + "reads_per_sec": 0, + "input_per_sec": 0, + "usec_per_read_op": 0, + "queue_depth": 0 + }, { + "writes_per_sec": 1856, + "output_per_sec": 0, + "usec_per_write_op": 653021.569741, + "local_queue_usec_per_op": 43158, + "time": "2022-04-25T02:25:46Z", + "reads_per_sec": 0, + "input_per_sec": 0, + "usec_per_read_op": 5360, + "queue_depth": 0 + }] +volume_metrics_info = [{ + "writes_per_sec": 1864, + "name": "136_connect", + "usec_per_write_op": 46200000, + "output_per_sec": 0, + "reads_per_sec": 0, + "input_per_sec": 5620302, + "time": "2022-04-12T02:12:16Z", + "usec_per_read_op": 0 +}, { + "writes_per_sec": 1864, + "name": "136_connect", + "usec_per_write_op": 46200000, + "output_per_sec": 0, + "reads_per_sec": 0, + "input_per_sec": 5620302, + "time": "2022-04-12T02:13:16Z", + "usec_per_read_op": 0 +}] def create_driver(): @@ -1721,3 +1773,217 @@ def test_list_masking_views(self): side_effect=[HGROUP_CONNECT_INFO, HOSTS_CONNECT_INFO]) views = self.driver.list_masking_views(context) self.assertListEqual(views, views_data) + + def test_collect_perf_metrics(self): + RestHandler.rest_call = mock.Mock( + side_effect=[storage_id_info, drive_metrics]) + localtime = time.mktime(time.localtime()) * units.k + storage_id = 12345 + start_time = localtime - 1000 * 60 * 60 * 24 * 364 + end_time = localtime + metrics = self.driver.collect_perf_metrics( + context, storage_id, storage_resource_metrics, start_time, + end_time) + storage_metrics = [ + constants.metric_struct( + name='iops', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1650853440000: 0, 1650853500000: 1856} + ), + constants.metric_struct( + name='readIops', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1650853440000: 0, 1650853500000: 0} + ), + constants.metric_struct( + name='writeIops', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1650853440000: 0, 1650853500000: 1856} + ), + constants.metric_struct( + name='throughput', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'MB/s'}, + values={1650853440000: 0.0, 1650853500000: 0.0} + ), + constants.metric_struct( + name='readThroughput', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'MB/s'}, + values={1650853440000: 0.0, 1650853500000: 0.0} + ), + constants.metric_struct( + name='writeThroughput', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'MB/s'}, + values={1650853440000: 0.0, 1650853500000: 0.0} + ), + constants.metric_struct( + name='readResponseTime', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'ms'}, + values={1650853440000: 0.0, 1650853500000: 5.36} + ), + constants.metric_struct( + name='writeResponseTime', + labels={ + 'storage_id': 12345, + 'resource_type': 'storage', + 'resource_id': 'dlmkk15xcfdf4v5', + 'resource_name': 'pure01', + 'type': 'RAW', + 'unit': 'ms'}, + values={1650853440000: 0.0, 1650853500000: 653.022} + ) + ] + self.assertListEqual(metrics, storage_metrics) + volume_metrics = [ + constants.metric_struct( + name='iops', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1649729520000: 1864, 1649729580000: 1864} + ), + constants.metric_struct( + name='readIops', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1649729520000: 0, 1649729580000: 0} + ), + constants.metric_struct( + name='writeIops', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'IOPS'}, + values={1649729520000: 1864, 1649729580000: 1864} + ), + constants.metric_struct( + name='throughput', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'MB/s'}, + values={1649729520000: 5.36, 1649729580000: 5.36} + ), + constants.metric_struct( + name='readThroughput', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'MB/s'}, + values={1649729520000: 0.0, 1649729580000: 0.0} + ), + constants.metric_struct( + name='writeThroughput', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'MB/s'}, + values={1649729520000: 5.36, 1649729580000: 5.36} + ), + constants.metric_struct( + name='readResponseTime', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'ms'}, + values={1649729520000: 0.0, 1649729580000: 0.0} + ), + constants.metric_struct( + name='writeResponseTime', + labels={ + 'storage_id': 12345, + 'resource_type': 'volume', + 'resource_id': '136_connect', + 'resource_name': '136_connect', + 'type': 'RAW', + 'unit': 'ms'}, + values={1649729520000: 46200.0, 1649729580000: 46200.0} + ) + ] + RestHandler.rest_call = mock.Mock( + side_effect=[volume_metrics_info]) + metrics = self.driver.collect_perf_metrics( + context, storage_id, volume_resource_metrics, start_time, + end_time) + self.assertListEqual(metrics, volume_metrics) + + def test_get_capabilities(self): + err = None + try: + self.driver.get_capabilities(context) + except Exception as e: + err = six.text_type(e) + LOG.error("test_get_capabilities error: %s", err) + self.assertEqual(err, None) + + def test_get_latest_perf_timestamp(self): + RestHandler.rest_call = mock.Mock( + side_effect=[drive_metrics]) + timestamp = self.driver.get_latest_perf_timestamp(context) + times = 1650853500000 + self.assertEqual(timestamp, times)