From 202624fe7f95b24637f6b634d8eebe2e3f554754 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 14 Oct 2021 16:13:41 +0800 Subject: [PATCH 01/69] pure --- delfin/drivers/pure/storage/__init__.py | 0 delfin/drivers/pure/storage/rest_handler.py | 145 ++++++++ delfin/drivers/pure/storage/storage.py | 332 ++++++++++++++++++ .../unit/drivers/pure/storage/test_storage.py | 304 ++++++++++++++++ 4 files changed, 781 insertions(+) create mode 100644 delfin/drivers/pure/storage/__init__.py create mode 100644 delfin/drivers/pure/storage/rest_handler.py create mode 100644 delfin/drivers/pure/storage/storage.py create mode 100644 delfin/tests/unit/drivers/pure/storage/test_storage.py diff --git a/delfin/drivers/pure/storage/__init__.py b/delfin/drivers/pure/storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/drivers/pure/storage/rest_handler.py b/delfin/drivers/pure/storage/rest_handler.py new file mode 100644 index 000000000..66dcb68c4 --- /dev/null +++ b/delfin/drivers/pure/storage/rest_handler.py @@ -0,0 +1,145 @@ +import six +from oslo_log import log as logging + +from delfin import exception, cryptor +from delfin.drivers.utils.rest_client import RestClient + +LOG = logging.getLogger(__name__) + + +class RestHandler(RestClient): + REST_STORAGE_URL = '/api/1.17/array?space=true' + REST_STORAGE_ID_URL = '/api/1.17/array' + REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=20&token=aWQgPSA5ODA1Mg==' + REST_VOLUME_TOKEN_URL = '/api/1.17/volume?space=true&limit=20&token=' + REST_VOLUME_ID_URL = '/api/1.17/volume/' + REST_POOLS_URL = '/api/1.17/vgroup' + REST_POOLS_CAPACITY_URL = '/api/1.17/vgroup?space=true' + REST_PORT_URL = '/api/1.17/port' + REST_NETWORK_URL = '/api/1.17/network' + REST_DISK_URL = '/api/1.17/drive' + REST_HARDWARE_URL = '/api/1.17/hardware' + REST_CONTROLLERS_URL = '/api/1.17/array?controllers=true' + REST_ALERTS_URL = '/api/1.17/message?flagged=true' + REST_AUTH_URL = '/api/1.17/auth/apitoken' + REST_SESSION_URL = '/api/1.17/auth/session' + HTTPS = 'https://' + + def __init__(self, **kwargs): + super(RestHandler, self).__init__(**kwargs) + rest_access = kwargs.get('rest') + if rest_access is None: + raise exception.InvalidInput('Input pure_rest_access is missing') + self.host = rest_access.get('host') + self.port = rest_access.get('port') + self.username = rest_access.get('username') + self.password = rest_access.get('password') + + def login(self): + try: + data = {'username': self.username, 'password': cryptor.decode(self.password)} + token = self.get_rest_login(RestHandler.REST_AUTH_URL, data, method='POST') + api_token = token.get('api_token') + if api_token is None and api_token == '': + raise exception.InvalidInput('api_token fail to get') + user = self.get_rest_login(RestHandler.REST_SESSION_URL, token, method='POST') + username = user.get('username') + if username is None and username == '': + raise exception.InvalidInput('session fail to get') + except Exception as e: + LOG.error("Login error: %s", six.text_type(e)) + raise exception.InvalidInput('login failure') + + def logout(self): + user = self.get_rest_login(RestHandler.REST_SESSION_URL, method='DELETE') + username = user.get('username') + if username is None and username == '': + raise exception.InvalidInput('delete fail to get') + + def get_all_rest_volumes(self): + url = '%s%s:%s%s' % (RestHandler.HTTPS, self.host, self.port, RestHandler.REST_VOLUME_URL) + result_json = self.get_rest_volumes_info(url) + return result_json + + def get_all_rest_volumes_id(self, name): + result_json = self.get_rest_info('{}{}'.format(RestHandler.REST_VOLUME_ID_URL, name)) + return result_json + + def get_storage(self): + result_json = self.get_rest_info(RestHandler.REST_STORAGE_URL) + return result_json + + def get_storage_ID(self): + result_json = self.get_rest_info(RestHandler.REST_STORAGE_ID_URL) + return result_json + + def get_all_pools(self): + result_json = self.get_rest_info(RestHandler.REST_POOLS_URL) + return result_json + + def get_capacity_pools(self): + result_json = self.get_rest_info(RestHandler.REST_POOLS_CAPACITY_URL) + return result_json + + def get_all_port(self): + result_json = self.get_rest_info(RestHandler.REST_PORT_URL) + return result_json + + def get_all_network(self): + result_json = self.get_rest_info(RestHandler.REST_NETWORK_URL) + return result_json + + def get_all_disk(self): + result_json = self.get_rest_info(RestHandler.REST_DISK_URL) + return result_json + + def get_all_hardware(self): + result_json = self.get_rest_info(RestHandler.REST_HARDWARE_URL) + return result_json + + def get_all_controllers(self): + result_json = self.get_rest_info(RestHandler.REST_CONTROLLERS_URL) + return result_json + + def get_all_alerts(self): + result_json = self.get_rest_info(RestHandler.REST_ALERTS_URL) + return result_json + + # 请求远程方法 + def get_rest_info(self, url, data=None, method='GET'): + result_json = None + # 请求远程接口 + res = self.do_call(url, data, method) + if res.status_code == 200: + result_json = res.json() + elif res.status_code == 401: + self.login() + self.get_rest_info(url, data, method) + return result_json + + def get_rest_login(self, url, data=None, method='GET'): + self.init_http_head() + # 请求远程接口 + res = self.do_call(url, data, method) + result_json = res.json() + return result_json + + def get_rest_volumes_info(self, url, data=None, volume_list=None): + if volume_list is None: + volume_list = [] + # 请求远程接口 + res = self.do_call(url, data, 'GET') + if res.status_code == 200: + result_json = res.json() + volume_list.extend(result_json) + next_token = res.headers._store.get('x-next-token') + if next_token: + token = next_token[1] + if token: + url = '%s%s:%s%s%s' % (RestHandler.HTTPS, self.host, self.port, + RestHandler.REST_VOLUME_TOKEN_URL, token) + self.get_rest_volumes_info(url, data, volume_list) + elif res.status_code == 401: + self.login() + self.get_rest_volumes_info(url, data, volume_list) + return volume_list diff --git a/delfin/drivers/pure/storage/storage.py b/delfin/drivers/pure/storage/storage.py new file mode 100644 index 000000000..915879388 --- /dev/null +++ b/delfin/drivers/pure/storage/storage.py @@ -0,0 +1,332 @@ +import hashlib +import json + +from oslo_log import log +from oslo_utils import units + +from delfin.common import constants +from delfin.drivers import driver +from delfin.drivers.pure.storage import rest_handler + +LOG = log.getLogger(__name__) + + +class StorageDriver(driver.StorageDriver): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.rest_handler = rest_handler.RestHandler(**kwargs) + self.rest_handler.login() + + def list_volumes(self, context): + LOG.info("进入list_volumes") + volume = [] + volumes_return = self.rest_handler.get_all_rest_volumes() + LOG.info("list_volumes--volumes_return：%s" % (json.dumps(volumes_return, ensure_ascii=False))) + if volumes_return: + volume = self.volume_handler(volumes_return) + return volume + + def volume_handler(self, volumes_return): + volume = [] + return_pool = self.rest_handler.get_all_pools() + for i in range(0, len(volumes_return)): + volume_dict = dict() + volume_object = volumes_return[i] + name = volume_object.get('name') + volume_dict['native_volume_id'] = name + volume_dict['name'] = name + total_capacity = int(int(volume_object.get('size')) / units.Ki) + volume_dict['total_capacity'] = total_capacity + used_capacity = int(int(volume_object.get('volumes')) / units.Ki) + volume_dict['used_capacity'] = used_capacity + volume_dict['free_capacity'] = total_capacity - used_capacity + volume_dict['storage_id'] = self.storage_id + volume_dict['description'] = "" + # status type无法获取，不写死页面显示错误 + volume_dict['status'] = constants.StorageStatus.NORMAL + volume_dict['type'] = constants.VolumeType.THICK + volume_dict['native_storage_pool_id'] = "" + if return_pool: + for j in return_pool: + volumes = j.get('volumes') + if name in volumes: + volume_dict['native_storage_pool_id'] = j.get('name') + break + volume.append(volume_dict) + LOG.info("卷：%s" % (json.dumps(volume, ensure_ascii=False))) + return volume + + def add_trap_config(self, context, trap_config): + pass + + def clear_alert(self, context, alert): + pass + + def get_storage(self, context): + LOG.info("进入get_storage") + storage_object = dict() + result_storage = self.rest_handler.get_storage() + storage_object['model'] = result_storage[0].get('hostname') + storage_object['vendor'] = 'PURE' + total_capacity = int(int(result_storage[0].get('provisioned')) / units.Ki) + storage_object['total_capacity'] = total_capacity + # 写死raw_capacity + storage_object['raw_capacity'] = total_capacity + used_capacity = int(int(result_storage[0].get('volumes')) / units.Ki) + storage_object['used_capacity'] = used_capacity + storage_object['free_capacity'] = total_capacity - used_capacity + + return_storage_id = self.rest_handler.get_storage_ID() + storage_object['name'] = return_storage_id.get('array_name') + storage_object['serial_number'] = return_storage_id.get('id') + storage_object['firmware_version'] = return_storage_id.get('version') + storage_object['description'] = "" + # status无法获取，不写死没法部署 + storage_object['status'] = constants.StorageStatus.NORMAL + storage_object['location'] = "" + LOG.info("系统：%s" % (json.dumps(storage_object, ensure_ascii=False))) + return storage_object + + def list_alerts(self, context, query_para=None): + LOG.info("进入list_alerts") + return_alerts = self.rest_handler.get_all_alerts() + alerts_list = [] + if return_alerts: + for alerts in return_alerts: + alerts_model = dict() + alerts_model['alert_id'] = alerts.get('id') + severity = alerts.get('current_severity') + if severity == 'fatal': + alerts_model['severity'] = constants.Severity.FATAL + elif severity == 'critical': + alerts_model['severity'] = constants.Severity.CRITICAL + elif severity == 'major': + alerts_model['severity'] = constants.Severity.MAJOR + elif severity == 'minor': + alerts_model['severity'] = constants.Severity.MINOR + elif severity == 'warning': + alerts_model['severity'] = constants.Severity.WARNING + elif severity == 'informational': + alerts_model['severity'] = constants.Severity.INFORMATIONAL + else: + alerts_model['severity'] = constants.Severity.NOT_SPECIFIED + + category = alerts.get('category') + if category == 'fault': + alerts_model['category'] = constants.Category.FAULT + elif category == 'event': + alerts_model['category'] = constants.Category.EVENT + elif category == 'recovery': + alerts_model['category'] = constants.Category.RECOVERY + else: + alerts_model['category'] = constants.Category.NOT_SPECIFIED + + alerts_model['occur_time'] = alerts.get('opened') + alerts_model['description'] = alerts.get('event') + alerts_model['location'] = alerts.get('component_name') + # 写死 + alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM + alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE + alerts_model['alert_name'] = alerts.get('id') + alerts_model['match_key'] = hashlib.md5(str(alerts.get('id')).encode()).hexdigest() + alerts_list.append(alerts_model) + LOG.info("list_alerts：%s" % (json.dumps(alerts_list, ensure_ascii=False))) + return alerts_list + + def list_controllers(self, context): + LOG.info("进入list_controllers") + list_controllers = [] + return_controllers = self.rest_handler.get_all_controllers() + if return_controllers: + for controllers in return_controllers: + controllers_Object = dict() + name = controllers.get('name') + controllers_Object['name'] = name + status = controllers.get('status') + if status == 'ready': + controllers_Object['status'] = constants.ControllerStatus.NORMAL + else: + controllers_Object['status'] = constants.ControllerStatus.OFFLINE + controllers_Object['soft_version'] = controllers.get('version') + controllers_Object['storage_id'] = self.storage_id + controllers_Object['id'] = name + controllers_Object['native_controller_id'] = name + controllers_Object['location'] = "" + controllers_Object['cpu_info'] = "" + controllers_Object['memory_size'] = "" + list_controllers.append(controllers_Object) + LOG.info("list_controllers：%s" % (json.dumps(list_controllers, ensure_ascii=False))) + return list_controllers + + def list_disks(self, context): + LOG.info("进入list_disks") + names = dict() + return_hardware = self.rest_handler.get_all_hardware() + if return_hardware: + for hardware in return_hardware: + hardware_name = dict() + hardware_name['speed'] = hardware.get('speed') + hardware_name['serial_number'] = hardware.get('serial') + hardware_name['model'] = hardware.get('model') + name = hardware.get('name') + names[name] = hardware_name + + list_disk = [] + return_disks = self.rest_handler.get_all_disk() + if return_disks: + for drive in return_disks: + disk = dict() + drive_name = drive.get('name') + disk['name'] = drive_name + physical_type = drive.get('type').lower() + if physical_type in constants.DiskPhysicalType.ALL: + disk['physical_type'] = physical_type + else: + disk['physical_type'] = constants.DiskPhysicalType.UNKNOWN + status = drive.get('status') + if status == 'healthy': + disk['status'] = constants.DiskStatus.NORMAL + elif status == 'unhealthy': + disk['status'] = constants.DiskStatus.ABNORMAL + else: + disk['status'] = constants.DiskStatus.OFFLINE + disk['storage_id'] = self.storage_id + disk['capacity'] = int(int(drive.get('capacity')) / units.Ki) + hardware_Object = names.get(drive_name) + if hardware_Object is not None and hardware_Object != "": + speed = hardware_Object.get('speed') + if speed is None: + disk['speed'] = '' + else: + disk['speed'] = int(speed) + model = hardware_Object.get('model') + if model is None: + disk['model'] = '' + else: + disk['model'] = model + serial_number = hardware_Object.get('serial_number') + if serial_number is None: + disk['serial_number'] = '' + else: + disk['serial_number'] = serial_number + else: + disk['speed'] = 0 + disk['model'] = '' + disk['serial_number'] = '' + disk['native_disk_id'] = drive_name + disk['id'] = drive_name + disk['location'] = drive_name + disk['logical_type'] = '' + disk['native_disk_group_id'] = "" + # 写死的 + disk['manufacturer'] = "pure" + disk['firmware'] = "" + disk['health_score'] = "" + list_disk.append(disk) + LOG.info("list_disk：%s" % (json.dumps(list_disk, ensure_ascii=False))) + return list_disk + + def list_ports(self, context): + LOG.info("进入list_ports") + networksObject = dict() + self.network_Object(networksObject) + list_port = [] + return_ports = self.rest_handler.get_all_port() + if return_ports: + for ports in return_ports: + port = dict() + name = ports.get('name') + wwn = ports.get('wwn') + if wwn: + port['type'] = constants.PortType.FC + port['name'] = wwn + port['id'] = wwn + port['wwn'] = wwn + else: + port['type'] = constants.PortType.ETH + iqn = ports.get('iqn') + if iqn is None: + LOG.info("数据来源有误") + port['name'] = iqn + port['id'] = iqn + port['wwn'] = '' + port['native_port_id'] = name + port['location'] = name + port['storage_id'] = self.storage_id + network = networksObject.get(name.lower()) + if network: + port['logical_type'] = network.get('logical_type') + port['speed'] = network.get('speed') + port['mac_address'] = network.get('address') + port['ipv4_mask'] = network.get('ipv4_mask') + else: + port['logical_type'] = '' + port['speed'] = '' + port['mac_address'] = '' + port['ipv4_mask'] = '' + # 写死 + port['connection_status '] = constants.PortConnectionStatus.CONNECTED + port['health_status'] = constants.PortHealthStatus.NORMAL + port['max_speed'] = '' + port['native_parent_id'] = "" + port['ipv4'] = "" + port['ipv6'] = "" + port['ipv6_mask'] = "" + list_port.append(port) + LOG.info("list_ports：%s" % (json.dumps(list_port, ensure_ascii=False))) + return list_port + + def network_Object(self, networksObject): + return_network = self.rest_handler.get_all_network() + if return_network: + for network in return_network: + network_Object = dict() + network_Object['address'] = network.get('address') + services = network.get('services')[0] + if services in constants.PortLogicalType.ALL: + network_Object['logical_type'] = services + else: + # network_Object['logical_type'] = constants.PortLogicalType.SERVICE + network_Object['logical_type'] = "" + if services in constants.PortType.ALL: + network_Object['type'] = services + else: + network_Object['type'] = constants.PortType.ISCSI + network_Object['speed'] = int(int(network.get('speed')) / units.Ki) + network_Object['ipv4_mask'] = network.get('netmask') + name = network.get('name') + networksObject[name] = network_Object + + def list_storage_pools(self, context): + LOG.info("进入list_storage_pools") + pool_list = [] + return_pools = self.rest_handler.get_capacity_pools() + LOG.info("list_storage_pools-return_pools：%s" % (json.dumps(return_pools, ensure_ascii=False))) + if return_pools: + for pools in return_pools: + pool_object = dict() + total_capacity = int(pools.get('size')) + pool_object['total_capacity'] = total_capacity + + used_capacity = int(pools.get('total_reduction')) + pool_object['used_capacity'] = used_capacity + + pool_object['free_capacity'] = total_capacity - used_capacity + pool_object['name'] = pools.get('name') + pool_object['native_storage_pool_id'] = pools.get('name') + pool_object['storage_id'] = self.storage_id + pool_object['description'] = "" + # status storage_type 获取不到写死 + pool_object['status'] = constants.StoragePoolStatus.NORMAL + pool_object['storage_type'] = constants.StorageType.BLOCK + pool_list.append(pool_object) + LOG.info("list_storage_pools：%s" % (json.dumps(pool_list, ensure_ascii=False))) + return pool_list + + def remove_trap_config(self, context, trap_config): + pass + + def reset_connection(self, context, **kwargs): + self.rest_handler.logout() + self.rest_handler.login() diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py new file mode 100644 index 000000000..ab6e2dfc6 --- /dev/null +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -0,0 +1,304 @@ +import sys +from unittest import TestCase, mock + +sys.modules['delfin.cryptor'] = mock.Mock() +from delfin import context +from delfin.drivers.pure.storage.rest_handler import RestHandler +from delfin.drivers.pure.storage.storage import StorageDriver + +ACCESS_INFO = { + "rest": { + "host": "10.0.0.1", + "port": 8443, + "username": "user", + "password": "pass" + } +} + +volumes_info = [ + { + "total": 116272464547, + "name": "oracl_ail", + "system": "", + "snapshots": 0, + "volumes": 116272464547, + "data_reduction": 1.82656654775252, + "size": 2156324555567, + "shared_space": "", + "thin_provisioning": 0.9225557589632, + "total_reduction": 18.92245232244555 + }, + { + "total": 0, + "name": "wxt1", + "system": "", + "snapshots": 0, + "volumes": 0, + "data_reduction": 1, + "size": 1073741824, + "shared_space": "", + "thin_provisioning": 1, + "total_reduction": 1 + } +] + +pool_info = [ + { + "name": "lktest", + "volumes": [ + "oracl_ail", + "wxt1", + "lktest/lk301", + "lktest/lk401", + "lktest/lk501", + ] + }, + { + "name": "ethanTestVG", + "volumes": [ + + ] + } +] +volume_info = { + "created": "2016-05-02T20:36:20Z", + "name": "oracl_ail", + "serial": "Fedd3455666y", + "size": 1073740124, + "source": "" +} +volume_info_two = { + "created": "2016-05-02T20:36:20Z", + "name": "wxt1", + "serial": "Fedd3475666y", + "size": 1073740124, + "source": "" +} +storage_info = [ + { + "parity": "0.996586544522471235", + "provisioned": "20869257625600", + "hostname": "FA-m20", + "system": 0, + "snapshots": 0, + "volumes": 227546215656, + "data_reduction": 1, + "capacity": 122276719419392, + "total": 324829845504, + "shared_space": 97544451659, + "thin_provisioning": 0.9526445631455244, + "total_reduction": 64.152236458789225 + } +] +storage_id_info = { + "array_name": "pure01", + "id": "dlmkk15xcfdf4v5", + "revision": "2016-20-29mfmkkk", + "version": "4.6.7" +} +alerts_info = [ + { + "category": "array", + "code": 42, + "actual": "", + "opened": "2018-05-12T10:55:21Z", + "component_type": "hardware", + "event": "failure", + "current_severity": "warning", + "details": "", + "expected": "", + "id": 135, + "component_name": "ct1.eth0" + }, + { + "category": "array", + "code": 13, + "actual": "", + "opened": "2018-05-12T10:55:21Z", + "component_type": "process", + "event": "server unreachable", + "current_severity": "warning", + "details": "", + "expected": "", + "id": 10088786, + "component_name": "ct1.ntpd" + } +] +controllers_info = [ + { + "status": "ready", + "name": "CT0", + "version": "5.3.0", + "mode": "primary", + "model": "FA-m20r2", + "type": "array_controller" + }, + { + "status": "ready", + "name": "CT1", + "version": "5.3.0", + "mode": "secondary", + "model": "FA-m20r2", + "type": "array_controller" + } +] +hardware_info = [ + { + "details": "", + "identify": "off", + "index": 0, + "name": "CH0.BAY1", + "slot": "", + "speed": "", + "status": "ok", + "temperature": "" + }, + { + "details": "", + "identify": "", + "index": 0, + "name": "CH0.BAY2", + "slot": 0, + "speed": 1000000, + "status": "ok", + "temperature": "" + } +] +drive_info = [ + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY1", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + }, + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY2", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + }, + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY3", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + } +] +port_info = [ + { + "name": "CTO.ETH14", + "failover": "", + "iqn": "iqn.2016-11-01.com.pure", + "portal": "100.12.253.23:4563", + "wwn": "43ddff45ggg4rty", + "nqn": "" + }, + { + "name": "CTO.ETH14", + "failover": "", + "iqn": "iqn.2016-11-01.com.pure", + "portal": "100.12.253.23:4563", + "wwn": "43ddff45gdcvrty", + "nqn": "" + } +] +pools_info = [ + { + "total": "", + "name": "lktest", + "snapshots": "", + "volumes": 0, + "data_reduction": 1, + "size": 5632155322, + "thin_provisioning": 1, + "total_reduction": 1 + }, + { + "total": "", + "name": "ethanTestVG", + "snapshots": "", + "volumes": 0, + "data_reduction": 1, + "size": 5632155322, + "thin_provisioning": 1, + "total_reduction": 1 + } +] +reset_connection_info = { + "username": "oksd" +} + + +def create_driver(): + RestHandler.login = mock.Mock( + return_value={""}) + return StorageDriver(**ACCESS_INFO) + + +class test_StorageDriver(TestCase): + driver = create_driver() + + def test_init(self): + RestHandler.login = mock.Mock( + return_value={""}) + StorageDriver(**ACCESS_INFO) + + def test_list_volumes(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[volumes_info, pool_info, volume_info, volume_info_two]) + volume = self.driver.list_volumes(context) + self.assertEqual(volume[0]['native_volume_id'], volume_info.get('serial')) + + def test_get_storage(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[storage_info, storage_id_info]) + storage_object = self.driver.get_storage(context) + self.assertEqual(storage_object.get('name'), storage_id_info.get('array_name')) + + def test_list_alerts(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[alerts_info]) + list_alerts = self.driver.list_alerts(context) + self.assertEqual(list_alerts[0].get('alert_id'), alerts_info[0].get('id')) + + def test_list_controllers(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[controllers_info]) + list_controllers = self.driver.list_controllers(context) + self.assertEqual(list_controllers[0].get('name'), controllers_info[0].get('name')) + + def test_list_disks(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[hardware_info, drive_info]) + list_disks = self.driver.list_disks(context) + self.assertEqual(list_disks[0].get('name'), hardware_info[0].get('name')) + + def test_list_ports(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[port_info]) + list_ports = self.driver.list_ports(context) + self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) + + def test_list_storage_pools(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[pools_info]) + list_storage_pools = self.driver.list_storage_pools(context) + self.assertEqual(list_storage_pools[0].get('native_storage_pool_id'), pools_info[0].get('name')) + + def test_reset_connection(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[reset_connection_info]) + reset_connection = self.driver.reset_connection(context) + From 2a14e4ba95baaa1f15bff1c8a9a4fddefec8bef9 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 14 Oct 2021 16:34:44 +0800 Subject: [PATCH 02/69] =?UTF-8?q?pure=E5=8E=BB=E6=8E=89=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- delfin/drivers/pure/storage/storage.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/delfin/drivers/pure/storage/storage.py b/delfin/drivers/pure/storage/storage.py index 915879388..b510026bb 100644 --- a/delfin/drivers/pure/storage/storage.py +++ b/delfin/drivers/pure/storage/storage.py @@ -1,5 +1,4 @@ import hashlib -import json from oslo_log import log from oslo_utils import units @@ -19,10 +18,8 @@ def __init__(self, **kwargs): self.rest_handler.login() def list_volumes(self, context): - LOG.info("进入list_volumes") volume = [] volumes_return = self.rest_handler.get_all_rest_volumes() - LOG.info("list_volumes--volumes_return：%s" % (json.dumps(volumes_return, ensure_ascii=False))) if volumes_return: volume = self.volume_handler(volumes_return) return volume @@ -54,7 +51,6 @@ def volume_handler(self, volumes_return): volume_dict['native_storage_pool_id'] = j.get('name') break volume.append(volume_dict) - LOG.info("卷：%s" % (json.dumps(volume, ensure_ascii=False))) return volume def add_trap_config(self, context, trap_config): @@ -64,7 +60,6 @@ def clear_alert(self, context, alert): pass def get_storage(self, context): - LOG.info("进入get_storage") storage_object = dict() result_storage = self.rest_handler.get_storage() storage_object['model'] = result_storage[0].get('hostname') @@ -85,11 +80,9 @@ def get_storage(self, context): # status无法获取，不写死没法部署 storage_object['status'] = constants.StorageStatus.NORMAL storage_object['location'] = "" - LOG.info("系统：%s" % (json.dumps(storage_object, ensure_ascii=False))) return storage_object def list_alerts(self, context, query_para=None): - LOG.info("进入list_alerts") return_alerts = self.rest_handler.get_all_alerts() alerts_list = [] if return_alerts: @@ -131,11 +124,9 @@ def list_alerts(self, context, query_para=None): alerts_model['alert_name'] = alerts.get('id') alerts_model['match_key'] = hashlib.md5(str(alerts.get('id')).encode()).hexdigest() alerts_list.append(alerts_model) - LOG.info("list_alerts：%s" % (json.dumps(alerts_list, ensure_ascii=False))) return alerts_list def list_controllers(self, context): - LOG.info("进入list_controllers") list_controllers = [] return_controllers = self.rest_handler.get_all_controllers() if return_controllers: @@ -156,11 +147,9 @@ def list_controllers(self, context): controllers_Object['cpu_info'] = "" controllers_Object['memory_size'] = "" list_controllers.append(controllers_Object) - LOG.info("list_controllers：%s" % (json.dumps(list_controllers, ensure_ascii=False))) return list_controllers def list_disks(self, context): - LOG.info("进入list_disks") names = dict() return_hardware = self.rest_handler.get_all_hardware() if return_hardware: @@ -224,11 +213,9 @@ def list_disks(self, context): disk['firmware'] = "" disk['health_score'] = "" list_disk.append(disk) - LOG.info("list_disk：%s" % (json.dumps(list_disk, ensure_ascii=False))) return list_disk def list_ports(self, context): - LOG.info("进入list_ports") networksObject = dict() self.network_Object(networksObject) list_port = [] @@ -274,7 +261,6 @@ def list_ports(self, context): port['ipv6'] = "" port['ipv6_mask'] = "" list_port.append(port) - LOG.info("list_ports：%s" % (json.dumps(list_port, ensure_ascii=False))) return list_port def network_Object(self, networksObject): @@ -299,10 +285,8 @@ def network_Object(self, networksObject): networksObject[name] = network_Object def list_storage_pools(self, context): - LOG.info("进入list_storage_pools") pool_list = [] return_pools = self.rest_handler.get_capacity_pools() - LOG.info("list_storage_pools-return_pools：%s" % (json.dumps(return_pools, ensure_ascii=False))) if return_pools: for pools in return_pools: pool_object = dict() @@ -321,7 +305,6 @@ def list_storage_pools(self, context): pool_object['status'] = constants.StoragePoolStatus.NORMAL pool_object['storage_type'] = constants.StorageType.BLOCK pool_list.append(pool_object) - LOG.info("list_storage_pools：%s" % (json.dumps(pool_list, ensure_ascii=False))) return pool_list def remove_trap_config(self, context, trap_config): From c5083b84609cbdcae9eb941b9209db2237eeeba8 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 14 Oct 2021 17:49:08 +0800 Subject: [PATCH 03/69] pure --- delfin/drivers/pure/storage/__init__.py | 0 delfin/drivers/pure/storage/rest_handler.py | 154 +++++++++ delfin/drivers/pure/storage/storage.py | 322 ++++++++++++++++++ .../unit/drivers/pure/storage/test_storage.py | 310 +++++++++++++++++ 4 files changed, 786 insertions(+) create mode 100644 delfin/drivers/pure/storage/__init__.py create mode 100644 delfin/drivers/pure/storage/rest_handler.py create mode 100644 delfin/drivers/pure/storage/storage.py create mode 100644 delfin/tests/unit/drivers/pure/storage/test_storage.py diff --git a/delfin/drivers/pure/storage/__init__.py b/delfin/drivers/pure/storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/drivers/pure/storage/rest_handler.py b/delfin/drivers/pure/storage/rest_handler.py new file mode 100644 index 000000000..b0969ce81 --- /dev/null +++ b/delfin/drivers/pure/storage/rest_handler.py @@ -0,0 +1,154 @@ +import six +from oslo_log import log as logging + +from delfin import exception, cryptor +from delfin.drivers.utils.rest_client import RestClient + +LOG = logging.getLogger(__name__) + + +class RestHandler(RestClient): + REST_STORAGE_URL = '/api/1.17/array?space=true' + REST_STORAGE_ID_URL = '/api/1.17/array' + REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=20&token=aWQgPSA5OD' \ + 'A1Mg==' + REST_VOLUME_TOKEN_URL = '/api/1.17/volume?space=true&limit=20&token=' + REST_VOLUME_ID_URL = '/api/1.17/volume/' + REST_POOLS_URL = '/api/1.17/vgroup' + REST_POOLS_CAPACITY_URL = '/api/1.17/vgroup?space=true' + REST_PORT_URL = '/api/1.17/port' + REST_NETWORK_URL = '/api/1.17/network' + REST_DISK_URL = '/api/1.17/drive' + REST_HARDWARE_URL = '/api/1.17/hardware' + REST_CONTROLLERS_URL = '/api/1.17/array?controllers=true' + REST_ALERTS_URL = '/api/1.17/message?flagged=true' + REST_AUTH_URL = '/api/1.17/auth/apitoken' + REST_SESSION_URL = '/api/1.17/auth/session' + HTTPS = 'https://' + + def __init__(self, **kwargs): + super(RestHandler, self).__init__(**kwargs) + rest_access = kwargs.get('rest') + if rest_access is None: + raise exception.InvalidInput('Input pure_rest_access is missing') + self.host = rest_access.get('host') + self.port = rest_access.get('port') + self.username = rest_access.get('username') + self.password = rest_access.get('password') + + def login(self): + try: + data = {'username': self.username, 'password': cryptor.decode( + self.password)} + token = self.get_rest_login(RestHandler.REST_AUTH_URL, data, + method='POST') + api_token = token.get('api_token') + if api_token is None and api_token == '': + raise exception.InvalidInput('api_token fail to get') + user = self.get_rest_login(RestHandler.REST_SESSION_URL, token, + method='POST') + username = user.get('username') + if username is None and username == '': + raise exception.InvalidInput('session fail to get') + except Exception as e: + LOG.error("Login error: %s", six.text_type(e)) + raise exception.InvalidInput('login failure') + + def logout(self): + user = self.get_rest_login(RestHandler.REST_SESSION_URL, + method='DELETE') + username = user.get('username') + if username is None and username == '': + raise exception.InvalidInput('delete fail to get') + + def get_all_rest_volumes(self): + url = '%s%s:%s%s' % ( + RestHandler.HTTPS, self.host, self.port, + RestHandler.REST_VOLUME_URL) + result_json = self.get_rest_volumes_info(url) + return result_json + + def get_all_rest_volumes_id(self, name): + result_json = self.get_rest_info( + '{}{}'.format(RestHandler.REST_VOLUME_ID_URL, name)) + return result_json + + def get_storage(self): + result_json = self.get_rest_info(RestHandler.REST_STORAGE_URL) + return result_json + + def get_storage_ID(self): + result_json = self.get_rest_info(RestHandler.REST_STORAGE_ID_URL) + return result_json + + def get_all_pools(self): + result_json = self.get_rest_info(RestHandler.REST_POOLS_URL) + return result_json + + def get_capacity_pools(self): + result_json = self.get_rest_info(RestHandler.REST_POOLS_CAPACITY_URL) + return result_json + + def get_all_port(self): + result_json = self.get_rest_info(RestHandler.REST_PORT_URL) + return result_json + + def get_all_network(self): + result_json = self.get_rest_info(RestHandler.REST_NETWORK_URL) + return result_json + + def get_all_disk(self): + result_json = self.get_rest_info(RestHandler.REST_DISK_URL) + return result_json + + def get_all_hardware(self): + result_json = self.get_rest_info(RestHandler.REST_HARDWARE_URL) + return result_json + + def get_all_controllers(self): + result_json = self.get_rest_info(RestHandler.REST_CONTROLLERS_URL) + return result_json + + def get_all_alerts(self): + result_json = self.get_rest_info(RestHandler.REST_ALERTS_URL) + return result_json + + # 请求远程方法 + def get_rest_info(self, url, data=None, method='GET'): + result_json = None + # 请求远程接口 + res = self.do_call(url, data, method) + if res.status_code == 200: + result_json = res.json() + elif res.status_code == 401: + self.login() + self.get_rest_info(url, data, method) + return result_json + + def get_rest_login(self, url, data=None, method='GET'): + self.init_http_head() + # 请求远程接口 + res = self.do_call(url, data, method) + result_json = res.json() + return result_json + + def get_rest_volumes_info(self, url, data=None, volume_list=None): + if volume_list is None: + volume_list = [] + # 请求远程接口 + res = self.do_call(url, data, 'GET') + if res.status_code == 200: + result_json = res.json() + volume_list.extend(result_json) + next_token = res.headers._store.get('x-next-token') + if next_token: + token = next_token[1] + if token: + url = '%s%s:%s%s%s' % ( + RestHandler.HTTPS, self.host, self.port, + RestHandler.REST_VOLUME_TOKEN_URL, token) + self.get_rest_volumes_info(url, data, volume_list) + elif res.status_code == 401: + self.login() + self.get_rest_volumes_info(url, data, volume_list) + return volume_list diff --git a/delfin/drivers/pure/storage/storage.py b/delfin/drivers/pure/storage/storage.py new file mode 100644 index 000000000..c4823c91e --- /dev/null +++ b/delfin/drivers/pure/storage/storage.py @@ -0,0 +1,322 @@ +import hashlib +import json + +from oslo_log import log +from oslo_utils import units + +from delfin.common import constants +from delfin.drivers import driver +from delfin.drivers.pure.storage import rest_handler + +LOG = log.getLogger(__name__) + + +class StorageDriver(driver.StorageDriver): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.rest_handler = rest_handler.RestHandler(**kwargs) + self.rest_handler.login() + + def list_volumes(self, context): + volume = [] + volumes_return = self.rest_handler.get_all_rest_volumes() + if volumes_return: + volume = self.volume_handler(volumes_return) + return volume + + def volume_handler(self, volumes_return): + volume = [] + return_pool = self.rest_handler.get_all_pools() + for i in range(0, len(volumes_return)): + volume_dict = dict() + volume_object = volumes_return[i] + name = volume_object.get('name') + volume_dict['native_volume_id'] = name + volume_dict['name'] = name + total_capacity = int(int(volume_object.get('size')) / units.Ki) + volume_dict['total_capacity'] = total_capacity + used_capacity = int(int(volume_object.get('volumes')) / units.Ki) + volume_dict['used_capacity'] = used_capacity + volume_dict['free_capacity'] = total_capacity - used_capacity + volume_dict['storage_id'] = self.storage_id + volume_dict['description'] = "" + # status type无法获取，不写死页面显示错误 + volume_dict['status'] = constants.StorageStatus.NORMAL + volume_dict['type'] = constants.VolumeType.THICK + volume_dict['native_storage_pool_id'] = "" + if return_pool: + for j in return_pool: + volumes = j.get('volumes') + if name in volumes: + volume_dict['native_storage_pool_id'] = j.get('name') + break + volume.append(volume_dict) + return volume + + def add_trap_config(self, context, trap_config): + pass + + def clear_alert(self, context, alert): + pass + + def get_storage(self, context): + storage_object = dict() + result_storage = self.rest_handler.get_storage() + storage_object['model'] = result_storage[0].get('hostname') + storage_object['vendor'] = 'PURE' + total_capacity = int(int(result_storage[0].get('provisioned')) + / units.Ki) + storage_object['total_capacity'] = total_capacity + # 写死raw_capacity + storage_object['raw_capacity'] = total_capacity + used_capacity = int(int(result_storage[0].get('volumes')) / units.Ki) + storage_object['used_capacity'] = used_capacity + storage_object['free_capacity'] = total_capacity - used_capacity + + return_storage_id = self.rest_handler.get_storage_ID() + storage_object['name'] = return_storage_id.get('array_name') + storage_object['serial_number'] = return_storage_id.get('id') + storage_object['firmware_version'] = return_storage_id.get('version') + storage_object['description'] = "" + # status无法获取，不写死没法部署 + storage_object['status'] = constants.StorageStatus.NORMAL + storage_object['location'] = "" + return storage_object + + def list_alerts(self, context, query_para=None): + return_alerts = self.rest_handler.get_all_alerts() + alerts_list = [] + if return_alerts: + for alerts in return_alerts: + alerts_model = dict() + alerts_model['alert_id'] = alerts.get('id') + severity = alerts.get('current_severity') + if severity == 'fatal': + alerts_model['severity'] = constants.Severity.FATAL + elif severity == 'critical': + alerts_model['severity'] = constants.Severity.CRITICAL + elif severity == 'major': + alerts_model['severity'] = constants.Severity.MAJOR + elif severity == 'minor': + alerts_model['severity'] = constants.Severity.MINOR + elif severity == 'warning': + alerts_model['severity'] = constants.Severity.WARNING + elif severity == 'informational': + alerts_model['severity'] = constants.Severity.INFORMATIONAL + else: + alerts_model['severity'] = constants.Severity.NOT_SPECIFIED + + category = alerts.get('category') + if category == 'fault': + alerts_model['category'] = constants.Category.FAULT + elif category == 'event': + alerts_model['category'] = constants.Category.EVENT + elif category == 'recovery': + alerts_model['category'] = constants.Category.RECOVERY + else: + alerts_model['category'] = constants.Category.NOT_SPECIFIED + + alerts_model['occur_time'] = alerts.get('opened') + alerts_model['description'] = alerts.get('event') + alerts_model['location'] = alerts.get('component_name') + # 写死 + alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM + alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE + alerts_model['alert_name'] = alerts.get('id') + alerts_model['match_key'] = hashlib.md5(str(alerts.get('id')). + encode()).hexdigest() + alerts_list.append(alerts_model) + return alerts_list + + def list_controllers(self, context): + list_controllers = [] + return_controllers = self.rest_handler.get_all_controllers() + if return_controllers: + for controllers in return_controllers: + controllers_object = dict() + name = controllers.get('name') + controllers_object['name'] = name + status = controllers.get('status') + if status == 'ready': + controllers_object['status'] = constants.ControllerStatus.\ + NORMAL + else: + controllers_object['status'] = constants.ControllerStatus.\ + OFFLINE + controllers_object['soft_version'] = controllers.get('version') + controllers_object['storage_id'] = self.storage_id + controllers_object['id'] = name + controllers_object['native_controller_id'] = name + controllers_object['location'] = "" + controllers_object['cpu_info'] = "" + controllers_object['memory_size'] = "" + list_controllers.append(controllers_object) + return list_controllers + + def list_disks(self, context): + names = dict() + return_hardware = self.rest_handler.get_all_hardware() + if return_hardware: + for hardware in return_hardware: + hardware_name = dict() + hardware_name['speed'] = hardware.get('speed') + hardware_name['serial_number'] = hardware.get('serial') + hardware_name['model'] = hardware.get('model') + name = hardware.get('name') + names[name] = hardware_name + + list_disk = [] + return_disks = self.rest_handler.get_all_disk() + if return_disks: + for drive in return_disks: + disk = dict() + drive_name = drive.get('name') + disk['name'] = drive_name + physical_type = drive.get('type').lower() + if physical_type in constants.DiskPhysicalType.ALL: + disk['physical_type'] = physical_type + else: + disk['physical_type'] = constants.DiskPhysicalType.UNKNOWN + status = drive.get('status') + if status == 'healthy': + disk['status'] = constants.DiskStatus.NORMAL + elif status == 'unhealthy': + disk['status'] = constants.DiskStatus.ABNORMAL + else: + disk['status'] = constants.DiskStatus.OFFLINE + disk['storage_id'] = self.storage_id + disk['capacity'] = int(int(drive.get('capacity')) / units.Ki) + hardware_Object = names.get(drive_name) + if hardware_Object is not None and hardware_Object != "": + speed = hardware_Object.get('speed') + if speed is None: + disk['speed'] = '' + else: + disk['speed'] = int(speed) + model = hardware_Object.get('model') + if model is None: + disk['model'] = '' + else: + disk['model'] = model + serial_number = hardware_Object.get('serial_number') + if serial_number is None: + disk['serial_number'] = '' + else: + disk['serial_number'] = serial_number + else: + disk['speed'] = 0 + disk['model'] = '' + disk['serial_number'] = '' + disk['native_disk_id'] = drive_name + disk['id'] = drive_name + disk['location'] = drive_name + disk['logical_type'] = '' + disk['native_disk_group_id'] = "" + # 写死的 + disk['manufacturer'] = "pure" + disk['firmware'] = "" + disk['health_score'] = "" + list_disk.append(disk) + LOG.info("list_disk：%s" % (json.dumps(list_disk, ensure_ascii=False))) + return list_disk + + def list_ports(self, context): + networks_object = dict() + self.network_object(networks_object) + list_port = [] + return_ports = self.rest_handler.get_all_port() + if return_ports: + for ports in return_ports: + port = dict() + name = ports.get('name') + wwn = ports.get('wwn') + if wwn: + port['type'] = constants.PortType.FC + port['name'] = wwn + port['id'] = wwn + port['wwn'] = wwn + else: + port['type'] = constants.PortType.ETH + iqn = ports.get('iqn') + if iqn is None: + LOG.info("数据来源有误") + port['name'] = iqn + port['id'] = iqn + port['wwn'] = '' + port['native_port_id'] = name + port['location'] = name + port['storage_id'] = self.storage_id + network = networks_object.get(name.lower()) + if network: + port['logical_type'] = network.get('logical_type') + port['speed'] = network.get('speed') + port['mac_address'] = network.get('address') + port['ipv4_mask'] = network.get('ipv4_mask') + else: + port['logical_type'] = '' + port['speed'] = '' + port['mac_address'] = '' + port['ipv4_mask'] = '' + # 写死 + port['connection_status '] = constants.PortConnectionStatus.\ + CONNECTED + port['health_status'] = constants.PortHealthStatus.NORMAL + port['max_speed'] = '' + port['native_parent_id'] = "" + port['ipv4'] = "" + port['ipv6'] = "" + port['ipv6_mask'] = "" + list_port.append(port) + return list_port + + def network_object(self, networks_object): + return_network = self.rest_handler.get_all_network() + if return_network: + for network in return_network: + network_object = dict() + network_object['address'] = network.get('address') + services = network.get('services')[0] + if services in constants.PortLogicalType.ALL: + network_object['logical_type'] = services + else: + network_object['logical_type'] = "" + if services in constants.PortType.ALL: + network_object['type'] = services + else: + network_object['type'] = constants.PortType.ISCSI + network_object['speed'] = int(int(network.get('speed')) / + units.Ki) + network_object['ipv4_mask'] = network.get('netmask') + name = network.get('name') + networks_object[name] = network_object + + def list_storage_pools(self, context): + pool_list = [] + return_pools = self.rest_handler.get_capacity_pools() + if return_pools: + for pools in return_pools: + pool_object = dict() + total_capacity = int(pools.get('size')) + pool_object['total_capacity'] = total_capacity + + used_capacity = int(pools.get('total_reduction')) + pool_object['used_capacity'] = used_capacity + + pool_object['free_capacity'] = total_capacity - used_capacity + pool_object['name'] = pools.get('name') + pool_object['native_storage_pool_id'] = pools.get('name') + pool_object['storage_id'] = self.storage_id + pool_object['description'] = "" + # status storage_type 获取不到写死 + pool_object['status'] = constants.StoragePoolStatus.NORMAL + pool_object['storage_type'] = constants.StorageType.BLOCK + pool_list.append(pool_object) + return pool_list + + def remove_trap_config(self, context, trap_config): + pass + + def reset_connection(self, context, **kwargs): + self.rest_handler.logout() + self.rest_handler.login() diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py new file mode 100644 index 000000000..423473d81 --- /dev/null +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -0,0 +1,310 @@ +import sys +from unittest import TestCase, mock + +sys.modules['delfin.cryptor'] = mock.Mock() +from delfin import context +from delfin.drivers.pure.storage.rest_handler import RestHandler +from delfin.drivers.pure.storage.storage import StorageDriver + +ACCESS_INFO = { + "rest": { + "host": "10.0.0.1", + "port": 8443, + "username": "user", + "password": "pass" + } +} + +volumes_info = [ + { + "total": 116272464547, + "name": "oracl_ail", + "system": "", + "snapshots": 0, + "volumes": 116272464547, + "data_reduction": 1.82656654775252, + "size": 2156324555567, + "shared_space": "", + "thin_provisioning": 0.9225557589632, + "total_reduction": 18.92245232244555 + }, + { + "total": 0, + "name": "wxt1", + "system": "", + "snapshots": 0, + "volumes": 0, + "data_reduction": 1, + "size": 1073741824, + "shared_space": "", + "thin_provisioning": 1, + "total_reduction": 1 + } +] + +pool_info = [ + { + "name": "lktest", + "volumes": [ + "oracl_ail", + "wxt1", + "lktest/lk301", + "lktest/lk401", + "lktest/lk501", + ] + }, + { + "name": "ethanTestVG", + "volumes": [ + + ] + } +] +volume_info = { + "created": "2016-05-02T20:36:20Z", + "name": "oracl_ail", + "serial": "Fedd3455666y", + "size": 1073740124, + "source": "" +} +volume_info_two = { + "created": "2016-05-02T20:36:20Z", + "name": "wxt1", + "serial": "Fedd3475666y", + "size": 1073740124, + "source": "" +} +storage_info = [ + { + "parity": "0.996586544522471235", + "provisioned": "20869257625600", + "hostname": "FA-m20", + "system": 0, + "snapshots": 0, + "volumes": 227546215656, + "data_reduction": 1, + "capacity": 122276719419392, + "total": 324829845504, + "shared_space": 97544451659, + "thin_provisioning": 0.9526445631455244, + "total_reduction": 64.152236458789225 + } +] +storage_id_info = { + "array_name": "pure01", + "id": "dlmkk15xcfdf4v5", + "revision": "2016-20-29mfmkkk", + "version": "4.6.7" +} +alerts_info = [ + { + "category": "array", + "code": 42, + "actual": "", + "opened": "2018-05-12T10:55:21Z", + "component_type": "hardware", + "event": "failure", + "current_severity": "warning", + "details": "", + "expected": "", + "id": 135, + "component_name": "ct1.eth0" + }, + { + "category": "array", + "code": 13, + "actual": "", + "opened": "2018-05-12T10:55:21Z", + "component_type": "process", + "event": "server unreachable", + "current_severity": "warning", + "details": "", + "expected": "", + "id": 10088786, + "component_name": "ct1.ntpd" + } +] +controllers_info = [ + { + "status": "ready", + "name": "CT0", + "version": "5.3.0", + "mode": "primary", + "model": "FA-m20r2", + "type": "array_controller" + }, + { + "status": "ready", + "name": "CT1", + "version": "5.3.0", + "mode": "secondary", + "model": "FA-m20r2", + "type": "array_controller" + } +] +hardware_info = [ + { + "details": "", + "identify": "off", + "index": 0, + "name": "CH0.BAY1", + "slot": "", + "speed": "", + "status": "ok", + "temperature": "" + }, + { + "details": "", + "identify": "", + "index": 0, + "name": "CH0.BAY2", + "slot": 0, + "speed": 1000000, + "status": "ok", + "temperature": "" + } +] +drive_info = [ + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY1", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + }, + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY2", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + }, + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY3", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + } +] +port_info = [ + { + "name": "CTO.ETH14", + "failover": "", + "iqn": "iqn.2016-11-01.com.pure", + "portal": "100.12.253.23:4563", + "wwn": "43ddff45ggg4rty", + "nqn": "" + }, + { + "name": "CTO.ETH14", + "failover": "", + "iqn": "iqn.2016-11-01.com.pure", + "portal": "100.12.253.23:4563", + "wwn": "43ddff45gdcvrty", + "nqn": "" + } +] +pools_info = [ + { + "total": "", + "name": "lktest", + "snapshots": "", + "volumes": 0, + "data_reduction": 1, + "size": 5632155322, + "thin_provisioning": 1, + "total_reduction": 1 + }, + { + "total": "", + "name": "ethanTestVG", + "snapshots": "", + "volumes": 0, + "data_reduction": 1, + "size": 5632155322, + "thin_provisioning": 1, + "total_reduction": 1 + } +] +reset_connection_info = { + "username": "oksd" +} + + +def create_driver(): + RestHandler.login = mock.Mock( + return_value={""}) + return StorageDriver(**ACCESS_INFO) + + +class test_StorageDriver(TestCase): + driver = create_driver() + + def test_init(self): + RestHandler.login = mock.Mock( + return_value={""}) + StorageDriver(**ACCESS_INFO) + + def test_list_volumes(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[volumes_info, pool_info, volume_info, + volume_info_two]) + volume = self.driver.list_volumes(context) + self.assertEqual(volume[0]['native_volume_id'], + volume_info.get('serial')) + + def test_get_storage(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[storage_info, storage_id_info]) + storage_object = self.driver.get_storage(context) + self.assertEqual(storage_object.get('name'), + storage_id_info.get('array_name')) + + def test_list_alerts(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[alerts_info]) + list_alerts = self.driver.list_alerts(context) + self.assertEqual(list_alerts[0].get('alert_id'), + alerts_info[0].get('id')) + + def test_list_controllers(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[controllers_info]) + list_controllers = self.driver.list_controllers(context) + self.assertEqual(list_controllers[0].get('name'), + controllers_info[0].get('name')) + + def test_list_disks(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[hardware_info, drive_info]) + list_disks = self.driver.list_disks(context) + self.assertEqual(list_disks[0].get('name'), + hardware_info[0].get('name')) + + def test_list_ports(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[port_info]) + list_ports = self.driver.list_ports(context) + self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) + + def test_list_storage_pools(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[pools_info]) + list_storage_pools = self.driver.list_storage_pools(context) + self.assertEqual(list_storage_pools[0].get('native_storage_pool_id'), + pools_info[0].get('name')) + + def test_reset_connection(self): + RestHandler.get_rest_info = mock.Mock( + side_effect=[reset_connection_info]) + self.driver.reset_connection(context) From ac42cf1ad0457ea870266d55fec33a6b61a6bb40 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 14 Oct 2021 17:57:52 +0800 Subject: [PATCH 04/69] =?UTF-8?q?pure=20=E8=B7=AF=E5=BE=84=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1e273d91e..bb58db6d9 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ 'huawei oceanstor = delfin.drivers.huawei.oceanstor.oceanstor:OceanStorDriver', 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver' + 'pure storage = delfin.drivers.pure.storage.storage:StorageDriver' ] }, ) From 9454fbb36581f561bd7e9d5730cdce25bea566c6 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 15 Oct 2021 16:39:29 +0800 Subject: [PATCH 05/69] =?UTF-8?q?pure=20=E4=BC=98=E5=8C=96=E5=91=BD?= =?UTF-8?q?=E5=90=8D=EF=BC=8C=E6=B5=8B=E8=AF=95=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pure/{storage => pure}/__init__.py | 0 .../storage.py => pure/pure_storage.py} | 47 ++++++++----------- .../pure/{storage => pure}/rest_handler.py | 0 .../unit/drivers/pure/storage/test_storage.py | 43 +++++++++++++---- setup.py | 2 +- 5 files changed, 55 insertions(+), 37 deletions(-) rename delfin/drivers/pure/{storage => pure}/__init__.py (100%) rename delfin/drivers/pure/{storage/storage.py => pure/pure_storage.py} (90%) rename delfin/drivers/pure/{storage => pure}/rest_handler.py (100%) diff --git a/delfin/drivers/pure/storage/__init__.py b/delfin/drivers/pure/pure/__init__.py similarity index 100% rename from delfin/drivers/pure/storage/__init__.py rename to delfin/drivers/pure/pure/__init__.py diff --git a/delfin/drivers/pure/storage/storage.py b/delfin/drivers/pure/pure/pure_storage.py similarity index 90% rename from delfin/drivers/pure/storage/storage.py rename to delfin/drivers/pure/pure/pure_storage.py index c4823c91e..8337ef40b 100644 --- a/delfin/drivers/pure/storage/storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -6,7 +6,7 @@ from delfin.common import constants from delfin.drivers import driver -from delfin.drivers.pure.storage import rest_handler +from delfin.drivers.pure.pure import rest_handler LOG = log.getLogger(__name__) @@ -41,7 +41,6 @@ def volume_handler(self, volumes_return): volume_dict['free_capacity'] = total_capacity - used_capacity volume_dict['storage_id'] = self.storage_id volume_dict['description'] = "" - # status type无法获取，不写死页面显示错误 volume_dict['status'] = constants.StorageStatus.NORMAL volume_dict['type'] = constants.VolumeType.THICK volume_dict['native_storage_pool_id'] = "" @@ -63,23 +62,23 @@ def clear_alert(self, context, alert): def get_storage(self, context): storage_object = dict() result_storage = self.rest_handler.get_storage() - storage_object['model'] = result_storage[0].get('hostname') - storage_object['vendor'] = 'PURE' - total_capacity = int(int(result_storage[0].get('provisioned')) - / units.Ki) - storage_object['total_capacity'] = total_capacity - # 写死raw_capacity - storage_object['raw_capacity'] = total_capacity - used_capacity = int(int(result_storage[0].get('volumes')) / units.Ki) - storage_object['used_capacity'] = used_capacity - storage_object['free_capacity'] = total_capacity - used_capacity + if result_storage: + storage_object['model'] = result_storage[0].get('hostname', "") + total_capacity = int(int(result_storage[0].get('provisioned', + 0)) / units.Ki) + storage_object['total_capacity'] = total_capacity + storage_object['raw_capacity'] = total_capacity + used_capacity = int(int(result_storage[0].get('volumes', 0)) / + units.Ki) + storage_object['used_capacity'] = used_capacity + storage_object['free_capacity'] = total_capacity - used_capacity return_storage_id = self.rest_handler.get_storage_ID() + storage_object['vendor'] = 'PURE' storage_object['name'] = return_storage_id.get('array_name') storage_object['serial_number'] = return_storage_id.get('id') storage_object['firmware_version'] = return_storage_id.get('version') storage_object['description'] = "" - # status无法获取，不写死没法部署 storage_object['status'] = constants.StorageStatus.NORMAL storage_object['location'] = "" return storage_object @@ -120,7 +119,6 @@ def list_alerts(self, context, query_para=None): alerts_model['occur_time'] = alerts.get('opened') alerts_model['description'] = alerts.get('event') alerts_model['location'] = alerts.get('component_name') - # 写死 alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE alerts_model['alert_name'] = alerts.get('id') @@ -213,7 +211,6 @@ def list_disks(self, context): disk['location'] = drive_name disk['logical_type'] = '' disk['native_disk_group_id'] = "" - # 写死的 disk['manufacturer'] = "pure" disk['firmware'] = "" disk['health_score'] = "" @@ -258,7 +255,6 @@ def list_ports(self, context): port['speed'] = '' port['mac_address'] = '' port['ipv4_mask'] = '' - # 写死 port['connection_status '] = constants.PortConnectionStatus.\ CONNECTED port['health_status'] = constants.PortHealthStatus.NORMAL @@ -276,16 +272,14 @@ def network_object(self, networks_object): for network in return_network: network_object = dict() network_object['address'] = network.get('address') - services = network.get('services')[0] - if services in constants.PortLogicalType.ALL: - network_object['logical_type'] = services - else: - network_object['logical_type'] = "" - if services in constants.PortType.ALL: - network_object['type'] = services - else: - network_object['type'] = constants.PortType.ISCSI - network_object['speed'] = int(int(network.get('speed')) / + services_list = network.get('services') + if services_list: + services = services_list[0] + if services in constants.PortLogicalType.ALL: + network_object['logical_type'] = services + else: + network_object['logical_type'] = "" + network_object['speed'] = int(int(network.get('speed', 0)) / units.Ki) network_object['ipv4_mask'] = network.get('netmask') name = network.get('name') @@ -308,7 +302,6 @@ def list_storage_pools(self, context): pool_object['native_storage_pool_id'] = pools.get('name') pool_object['storage_id'] = self.storage_id pool_object['description'] = "" - # status storage_type 获取不到写死 pool_object['status'] = constants.StoragePoolStatus.NORMAL pool_object['storage_type'] = constants.StorageType.BLOCK pool_list.append(pool_object) diff --git a/delfin/drivers/pure/storage/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py similarity index 100% rename from delfin/drivers/pure/storage/rest_handler.py rename to delfin/drivers/pure/pure/rest_handler.py diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py index 423473d81..e5541cfba 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -3,8 +3,8 @@ sys.modules['delfin.cryptor'] = mock.Mock() from delfin import context -from delfin.drivers.pure.storage.rest_handler import RestHandler -from delfin.drivers.pure.storage.storage import StorageDriver +from delfin.drivers.pure.pure.rest_handler import RestHandler +from delfin.drivers.pure.pure.pure_storage import StorageDriver ACCESS_INFO = { "rest": { @@ -149,7 +149,7 @@ "index": 0, "name": "CH0.BAY1", "slot": "", - "speed": "", + "speed": 0, "status": "ok", "temperature": "" }, @@ -206,7 +206,7 @@ "nqn": "" }, { - "name": "CTO.ETH14", + "name": "CTO.ETH15", "failover": "", "iqn": "iqn.2016-11-01.com.pure", "portal": "100.12.253.23:4563", @@ -214,6 +214,30 @@ "nqn": "" } ] +port_network_info = [ + { + "name": "CTO.ETH14", + "address": "45233662jksndj", + "speed": 12000, + "netmask": "100.12.253.23:4563", + "wwn": "43ddff45ggg4rty", + "nqn": "", + "services": [ + "management" + ] + }, + { + "name": "CTO.ETH15", + "address": "45233662jksndj", + "speed": 13000, + "netmask": "100.12.253.23:4563", + "wwn": "43ddff45ggg4rty", + "nqn": "", + "services": [ + "management" + ] + } +] pools_info = [ { "total": "", @@ -256,12 +280,13 @@ def test_init(self): StorageDriver(**ACCESS_INFO) def test_list_volumes(self): + RestHandler.get_rest_volumes_info = mock.Mock( + side_effect=[volumes_info]) RestHandler.get_rest_info = mock.Mock( - side_effect=[volumes_info, pool_info, volume_info, - volume_info_two]) + side_effect=[pool_info]) volume = self.driver.list_volumes(context) self.assertEqual(volume[0]['native_volume_id'], - volume_info.get('serial')) + pool_info[0].get('volumes')[0]) def test_get_storage(self): RestHandler.get_rest_info = mock.Mock( @@ -293,7 +318,7 @@ def test_list_disks(self): def test_list_ports(self): RestHandler.get_rest_info = mock.Mock( - side_effect=[port_info]) + side_effect=[port_network_info, port_info]) list_ports = self.driver.list_ports(context) self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) @@ -305,6 +330,6 @@ def test_list_storage_pools(self): pools_info[0].get('name')) def test_reset_connection(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_rest_login = mock.Mock( side_effect=[reset_connection_info]) self.driver.reset_connection(context) diff --git a/setup.py b/setup.py index bb58db6d9..fe94c7ec1 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'huawei oceanstor = delfin.drivers.huawei.oceanstor.oceanstor:OceanStorDriver', 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver' - 'pure storage = delfin.drivers.pure.storage.storage:StorageDriver' + 'pure pure = delfin.drivers.pure.pure.pure_storage:StorageDriver' ] }, ) From 0ba6ca7543013bede8a68b309c19f7f3e3ed33f0 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 15 Oct 2021 17:01:33 +0800 Subject: [PATCH 06/69] =?UTF-8?q?pure=20=E5=8E=BB=E6=8E=89=E4=B8=AD?= =?UTF-8?q?=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- delfin/drivers/pure/pure/pure_storage.py | 22 +++++++--------------- delfin/drivers/pure/pure/rest_handler.py | 4 ---- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index 8337ef40b..96ce8fdac 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -185,23 +185,17 @@ def list_disks(self, context): disk['status'] = constants.DiskStatus.OFFLINE disk['storage_id'] = self.storage_id disk['capacity'] = int(int(drive.get('capacity')) / units.Ki) - hardware_Object = names.get(drive_name) - if hardware_Object is not None and hardware_Object != "": - speed = hardware_Object.get('speed') + hardware_object = names.get(drive_name) + if hardware_object is not None and hardware_object != "": + speed = hardware_object.get('speed') if speed is None: disk['speed'] = '' else: disk['speed'] = int(speed) - model = hardware_Object.get('model') - if model is None: - disk['model'] = '' - else: - disk['model'] = model - serial_number = hardware_Object.get('serial_number') - if serial_number is None: - disk['serial_number'] = '' - else: - disk['serial_number'] = serial_number + model = hardware_object.get('model') + disk['model'] = model + serial_number = hardware_object.get('serial_number') + disk['serial_number'] = serial_number else: disk['speed'] = 0 disk['model'] = '' @@ -236,8 +230,6 @@ def list_ports(self, context): else: port['type'] = constants.PortType.ETH iqn = ports.get('iqn') - if iqn is None: - LOG.info("数据来源有误") port['name'] = iqn port['id'] = iqn port['wwn'] = '' diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index b0969ce81..409041b5f 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -113,10 +113,8 @@ def get_all_alerts(self): result_json = self.get_rest_info(RestHandler.REST_ALERTS_URL) return result_json - # 请求远程方法 def get_rest_info(self, url, data=None, method='GET'): result_json = None - # 请求远程接口 res = self.do_call(url, data, method) if res.status_code == 200: result_json = res.json() @@ -127,7 +125,6 @@ def get_rest_info(self, url, data=None, method='GET'): def get_rest_login(self, url, data=None, method='GET'): self.init_http_head() - # 请求远程接口 res = self.do_call(url, data, method) result_json = res.json() return result_json @@ -135,7 +132,6 @@ def get_rest_login(self, url, data=None, method='GET'): def get_rest_volumes_info(self, url, data=None, volume_list=None): if volume_list is None: volume_list = [] - # 请求远程接口 res = self.do_call(url, data, 'GET') if res.status_code == 200: result_json = res.json() From e95107556069a90a71294845bf37808d2fe39ebd Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 28 Oct 2021 17:58:18 +0800 Subject: [PATCH 07/69] =?UTF-8?q?pure=20codeview=20=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E7=AE=80=E6=B4=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- delfin/drivers/pure/pure/pure_storage.py | 107 ++++++++++------------- 1 file changed, 47 insertions(+), 60 deletions(-) diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index 96ce8fdac..e6c3aa9e8 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -12,6 +12,29 @@ class StorageDriver(driver.StorageDriver): + SEVERITY_MAP = {'fatal': constants.Severity.FATAL, + 'critical': constants.Severity.CRITICAL, + 'major': constants.Severity.MAJOR, + 'minor': constants.Severity.MINOR, + 'warning': constants.Severity.WARNING, + 'informational': constants.Severity.INFORMATIONAL, + 'NotSpecified': constants.Severity.NOT_SPECIFIED} + CATEGORY_MAP = {'fault': constants.Category.FAULT, + 'event': constants.Category.EVENT, + 'recovery': constants.Category.RECOVERY, + 'notSpecified': constants.Category.NOT_SPECIFIED} + CONTROLLER_STATUS_MAP = {'normal': constants.ControllerStatus.NORMAL, + 'ready': constants.ControllerStatus.NORMAL, + 'offline': constants.ControllerStatus.OFFLINE, + 'fault': constants.ControllerStatus.FAULT, + 'degraded': constants.ControllerStatus.DEGRADED, + 'unknown': constants.ControllerStatus.UNKNOWN, + 'unready': constants.ControllerStatus.UNKNOWN} + DISK_STATUS_MAP = {'normal': constants.DiskStatus.NORMAL, + 'healthy': constants.DiskStatus.NORMAL, + 'abnormal': constants.DiskStatus.ABNORMAL, + 'unhealthy': constants.DiskStatus.ABNORMAL, + 'offline': constants.DiskStatus.OFFLINE} def __init__(self, **kwargs): super().__init__(**kwargs) @@ -91,30 +114,12 @@ def list_alerts(self, context, query_para=None): alerts_model = dict() alerts_model['alert_id'] = alerts.get('id') severity = alerts.get('current_severity') - if severity == 'fatal': - alerts_model['severity'] = constants.Severity.FATAL - elif severity == 'critical': - alerts_model['severity'] = constants.Severity.CRITICAL - elif severity == 'major': - alerts_model['severity'] = constants.Severity.MAJOR - elif severity == 'minor': - alerts_model['severity'] = constants.Severity.MINOR - elif severity == 'warning': - alerts_model['severity'] = constants.Severity.WARNING - elif severity == 'informational': - alerts_model['severity'] = constants.Severity.INFORMATIONAL - else: - alerts_model['severity'] = constants.Severity.NOT_SPECIFIED + alerts_model['severity'] = StorageDriver.SEVERITY_MAP.get( + severity, constants.Severity.NOT_SPECIFIED) category = alerts.get('category') - if category == 'fault': - alerts_model['category'] = constants.Category.FAULT - elif category == 'event': - alerts_model['category'] = constants.Category.EVENT - elif category == 'recovery': - alerts_model['category'] = constants.Category.RECOVERY - else: - alerts_model['category'] = constants.Category.NOT_SPECIFIED + alerts_model['category'] = StorageDriver.CATEGORY_MAP.get( + category, constants.Category.NOT_SPECIFIED) alerts_model['occur_time'] = alerts.get('opened') alerts_model['description'] = alerts.get('event') @@ -136,12 +141,9 @@ def list_controllers(self, context): name = controllers.get('name') controllers_object['name'] = name status = controllers.get('status') - if status == 'ready': - controllers_object['status'] = constants.ControllerStatus.\ - NORMAL - else: - controllers_object['status'] = constants.ControllerStatus.\ - OFFLINE + controllers_object['status'] = StorageDriver. \ + CONTROLLER_STATUS_MAP.get(status, constants. + ControllerStatus.UNKNOWN) controllers_object['soft_version'] = controllers.get('version') controllers_object['storage_id'] = self.storage_id controllers_object['id'] = name @@ -177,29 +179,20 @@ def list_disks(self, context): else: disk['physical_type'] = constants.DiskPhysicalType.UNKNOWN status = drive.get('status') - if status == 'healthy': - disk['status'] = constants.DiskStatus.NORMAL - elif status == 'unhealthy': - disk['status'] = constants.DiskStatus.ABNORMAL - else: - disk['status'] = constants.DiskStatus.OFFLINE + disk['status'] = StorageDriver.DISK_STATUS_MAP.\ + get(status, constants.DiskStatus.OFFLINE) disk['storage_id'] = self.storage_id disk['capacity'] = int(int(drive.get('capacity')) / units.Ki) - hardware_object = names.get(drive_name) - if hardware_object is not None and hardware_object != "": - speed = hardware_object.get('speed') - if speed is None: - disk['speed'] = '' - else: - disk['speed'] = int(speed) - model = hardware_object.get('model') - disk['model'] = model - serial_number = hardware_object.get('serial_number') - disk['serial_number'] = serial_number + hardware_object = names.get(drive_name, {}) + + speed = hardware_object.get('speed') + if speed is None: + disk['speed'] = '' else: - disk['speed'] = 0 - disk['model'] = '' - disk['serial_number'] = '' + disk['speed'] = int(speed) + disk['model'] = hardware_object.get('model') + disk['serial_number'] = hardware_object.get('serial_number') + disk['native_disk_id'] = drive_name disk['id'] = drive_name disk['location'] = drive_name @@ -236,18 +229,12 @@ def list_ports(self, context): port['native_port_id'] = name port['location'] = name port['storage_id'] = self.storage_id - network = networks_object.get(name.lower()) - if network: - port['logical_type'] = network.get('logical_type') - port['speed'] = network.get('speed') - port['mac_address'] = network.get('address') - port['ipv4_mask'] = network.get('ipv4_mask') - else: - port['logical_type'] = '' - port['speed'] = '' - port['mac_address'] = '' - port['ipv4_mask'] = '' - port['connection_status '] = constants.PortConnectionStatus.\ + network = networks_object.get(name.lower(), {}) + port['logical_type'] = network.get('logical_type') + port['speed'] = network.get('speed') + port['mac_address'] = network.get('address') + port['ipv4_mask'] = network.get('ipv4_mask') + port['connection_status '] = constants.PortConnectionStatus. \ CONNECTED port['health_status'] = constants.PortHealthStatus.NORMAL port['max_speed'] = '' From 247cd5e4ec95a2821f02a33952d00892fbf51527 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 8 Nov 2021 10:54:02 +0800 Subject: [PATCH 08/69] pure codeview-Modify according to specifications --- delfin/drivers/pure/pure/pure_storage.py | 315 +++++++++--------- delfin/drivers/pure/pure/rest_handler.py | 44 ++- .../unit/drivers/pure/storage/test_storage.py | 25 +- 3 files changed, 192 insertions(+), 192 deletions(-) diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index e6c3aa9e8..c039e5120 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -1,5 +1,4 @@ import hashlib -import json from oslo_log import log from oslo_utils import units @@ -11,7 +10,7 @@ LOG = log.getLogger(__name__) -class StorageDriver(driver.StorageDriver): +class PureStorageDriver(driver.StorageDriver): SEVERITY_MAP = {'fatal': constants.Severity.FATAL, 'critical': constants.Severity.CRITICAL, 'major': constants.Severity.MAJOR, @@ -42,39 +41,34 @@ def __init__(self, **kwargs): self.rest_handler.login() def list_volumes(self, context): - volume = [] - volumes_return = self.rest_handler.get_all_rest_volumes() - if volumes_return: - volume = self.volume_handler(volumes_return) - return volume - - def volume_handler(self, volumes_return): - volume = [] - return_pool = self.rest_handler.get_all_pools() - for i in range(0, len(volumes_return)): - volume_dict = dict() - volume_object = volumes_return[i] - name = volume_object.get('name') - volume_dict['native_volume_id'] = name - volume_dict['name'] = name - total_capacity = int(int(volume_object.get('size')) / units.Ki) - volume_dict['total_capacity'] = total_capacity - used_capacity = int(int(volume_object.get('volumes')) / units.Ki) - volume_dict['used_capacity'] = used_capacity - volume_dict['free_capacity'] = total_capacity - used_capacity - volume_dict['storage_id'] = self.storage_id - volume_dict['description'] = "" - volume_dict['status'] = constants.StorageStatus.NORMAL - volume_dict['type'] = constants.VolumeType.THICK - volume_dict['native_storage_pool_id'] = "" - if return_pool: - for j in return_pool: - volumes = j.get('volumes') - if name in volumes: - volume_dict['native_storage_pool_id'] = j.get('name') - break - volume.append(volume_dict) - return volume + list_volumes = [] + volumes = self.rest_handler.get_volumes() + if volumes: + pools = self.rest_handler.get_pools() + for volume in volumes: + name = volume.get('name') + total_capacity = int(int(volume.get('size')) / units.Ki) + used_capacity = int(int(volume.get('volumes')) / units.Ki) + native_storage_pool_id = None + if pools: + for pool in pools: + pool_volumes = pool.get('volumes') + if name in pool_volumes: + native_storage_pool_id = pool.get('name') + break + volume_dict = { + 'native_volume_id': name, + 'name': name, + 'total_capacity': total_capacity, + 'used_capacity': used_capacity, + 'free_capacity': total_capacity - used_capacity, + 'storage_id': self.storage_id, + 'status': constants.StorageStatus.NORMAL, + 'type': constants.VolumeType.THICK, + 'native_storage_pool_id': native_storage_pool_id + } + list_volumes.append(volume_dict) + return list_volumes def add_trap_config(self, context, trap_config): pass @@ -83,43 +77,54 @@ def clear_alert(self, context, alert): pass def get_storage(self, context): - storage_object = dict() result_storage = self.rest_handler.get_storage() + model = None + total_capacity = None + used_capacity = None if result_storage: - storage_object['model'] = result_storage[0].get('hostname', "") - total_capacity = int(int(result_storage[0].get('provisioned', - 0)) / units.Ki) - storage_object['total_capacity'] = total_capacity - storage_object['raw_capacity'] = total_capacity - used_capacity = int(int(result_storage[0].get('volumes', 0)) / - units.Ki) - storage_object['used_capacity'] = used_capacity - storage_object['free_capacity'] = total_capacity - used_capacity + for storage in result_storage: + model = storage.get('hostname') + total_capacity = int(int(storage.get('provisioned', + 0)) / units.Ki) + used_capacity = int(int(storage.get('volumes', 0)) / + units.Ki) + break - return_storage_id = self.rest_handler.get_storage_ID() - storage_object['vendor'] = 'PURE' - storage_object['name'] = return_storage_id.get('array_name') - storage_object['serial_number'] = return_storage_id.get('id') - storage_object['firmware_version'] = return_storage_id.get('version') - storage_object['description'] = "" - storage_object['status'] = constants.StorageStatus.NORMAL - storage_object['location'] = "" - return storage_object + storage_id = self.rest_handler.get_storage_id() + name = None + serial_number = None + version = None + if storage_id: + name = storage_id.get('array_name') + serial_number = storage_id.get('id') + version = storage_id.get('version') + storage = { + 'model': model, + 'total_capacity': total_capacity, + 'raw_capacity': total_capacity, + 'used_capacity': used_capacity, + 'free_capacity': total_capacity - used_capacity, + 'vendor': 'PURE', + 'name': name, + 'serial_number': serial_number, + 'firmware_version': version, + 'status': constants.StorageStatus.NORMAL, + 'location': name + } + return storage def list_alerts(self, context, query_para=None): - return_alerts = self.rest_handler.get_all_alerts() + return_alerts = self.rest_handler.get_alerts() alerts_list = [] if return_alerts: for alerts in return_alerts: alerts_model = dict() alerts_model['alert_id'] = alerts.get('id') - severity = alerts.get('current_severity') - alerts_model['severity'] = StorageDriver.SEVERITY_MAP.get( - severity, constants.Severity.NOT_SPECIFIED) - - category = alerts.get('category') - alerts_model['category'] = StorageDriver.CATEGORY_MAP.get( - category, constants.Category.NOT_SPECIFIED) + alerts_model['severity'] = PureStorageDriver.SEVERITY_MAP.get( + alerts.get('current_severity'), + constants.Severity.NOT_SPECIFIED) + alerts_model['category'] = PureStorageDriver.CATEGORY_MAP.get( + alerts.get('category'), constants.Category.NOT_SPECIFIED) alerts_model['occur_time'] = alerts.get('opened') alerts_model['description'] = alerts.get('event') @@ -134,156 +139,142 @@ def list_alerts(self, context, query_para=None): def list_controllers(self, context): list_controllers = [] - return_controllers = self.rest_handler.get_all_controllers() - if return_controllers: - for controllers in return_controllers: + controllers = self.rest_handler.get_controllers() + if controllers: + for controller in controllers: controllers_object = dict() - name = controllers.get('name') + name = controller.get('name') controllers_object['name'] = name - status = controllers.get('status') - controllers_object['status'] = StorageDriver. \ - CONTROLLER_STATUS_MAP.get(status, constants. - ControllerStatus.UNKNOWN) - controllers_object['soft_version'] = controllers.get('version') + controllers_object['status'] = PureStorageDriver.\ + CONTROLLER_STATUS_MAP.get(controller.get('status'), + constants.ControllerStatus. + UNKNOWN) + controllers_object['soft_version'] = controller.get('version') controllers_object['storage_id'] = self.storage_id controllers_object['id'] = name controllers_object['native_controller_id'] = name - controllers_object['location'] = "" - controllers_object['cpu_info'] = "" - controllers_object['memory_size'] = "" + controllers_object['location'] = name list_controllers.append(controllers_object) return list_controllers def list_disks(self, context): names = dict() - return_hardware = self.rest_handler.get_all_hardware() - if return_hardware: - for hardware in return_hardware: + hardware = self.rest_handler.get_hardware() + if hardware: + for hardware_value in hardware: hardware_name = dict() - hardware_name['speed'] = hardware.get('speed') - hardware_name['serial_number'] = hardware.get('serial') - hardware_name['model'] = hardware.get('model') - name = hardware.get('name') + hardware_name['speed'] = hardware_value.get('speed') + hardware_name['serial_number'] = hardware_value.get('serial') + hardware_name['model'] = hardware_value.get('model') + name = hardware_value.get('name') names[name] = hardware_name - list_disk = [] - return_disks = self.rest_handler.get_all_disk() - if return_disks: - for drive in return_disks: + list_disks = [] + disks = self.rest_handler.get_disks() + if disks: + for drive in disks: disk = dict() drive_name = drive.get('name') disk['name'] = drive_name physical_type = drive.get('type').lower() - if physical_type in constants.DiskPhysicalType.ALL: - disk['physical_type'] = physical_type - else: - disk['physical_type'] = constants.DiskPhysicalType.UNKNOWN + disk['physical_type'] = physical_type \ + if physical_type in constants.DiskPhysicalType.ALL else\ + constants.DiskPhysicalType.UNKNOWN status = drive.get('status') - disk['status'] = StorageDriver.DISK_STATUS_MAP.\ + disk['status'] = PureStorageDriver.DISK_STATUS_MAP.\ get(status, constants.DiskStatus.OFFLINE) disk['storage_id'] = self.storage_id disk['capacity'] = int(int(drive.get('capacity')) / units.Ki) - hardware_object = names.get(drive_name, {}) + hardware_object = names.get(drive_name, {}) speed = hardware_object.get('speed') - if speed is None: - disk['speed'] = '' - else: - disk['speed'] = int(speed) + disk['speed'] = int(speed) if speed is not None else None disk['model'] = hardware_object.get('model') disk['serial_number'] = hardware_object.get('serial_number') disk['native_disk_id'] = drive_name disk['id'] = drive_name disk['location'] = drive_name - disk['logical_type'] = '' - disk['native_disk_group_id'] = "" disk['manufacturer'] = "pure" disk['firmware'] = "" - disk['health_score'] = "" - list_disk.append(disk) - LOG.info("list_disk：%s" % (json.dumps(list_disk, ensure_ascii=False))) - return list_disk + list_disks.append(disk) + return list_disks def list_ports(self, context): - networks_object = dict() - self.network_object(networks_object) - list_port = [] - return_ports = self.rest_handler.get_all_port() - if return_ports: - for ports in return_ports: - port = dict() - name = ports.get('name') - wwn = ports.get('wwn') + networks = self.get_network() + list_ports = [] + ports = self.rest_handler.get_ports() + if ports: + for port in ports: + port_result = dict() + name = port.get('name') + wwn = port.get('wwn') if wwn: - port['type'] = constants.PortType.FC - port['name'] = wwn - port['id'] = wwn - port['wwn'] = wwn + port_result['type'] = constants.PortType.FC + port_result['name'] = wwn + port_result['id'] = wwn + port_result['wwn'] = wwn else: - port['type'] = constants.PortType.ETH - iqn = ports.get('iqn') - port['name'] = iqn - port['id'] = iqn - port['wwn'] = '' - port['native_port_id'] = name - port['location'] = name - port['storage_id'] = self.storage_id - network = networks_object.get(name.lower(), {}) - port['logical_type'] = network.get('logical_type') - port['speed'] = network.get('speed') - port['mac_address'] = network.get('address') - port['ipv4_mask'] = network.get('ipv4_mask') - port['connection_status '] = constants.PortConnectionStatus. \ - CONNECTED - port['health_status'] = constants.PortHealthStatus.NORMAL - port['max_speed'] = '' - port['native_parent_id'] = "" - port['ipv4'] = "" - port['ipv6'] = "" - port['ipv6_mask'] = "" - list_port.append(port) - return list_port + port_result['type'] = constants.PortType.ETH + iqn = port.get('iqn') + port_result['name'] = iqn + port_result['id'] = iqn + port_result['wwn'] = '' + port_result['native_port_id'] = name + port_result['location'] = name + port_result['storage_id'] = self.storage_id + network = networks.get(name.lower(), {}) + port_result['logical_type'] = network.get('logical_type') + port_result['speed'] = network.get('speed') + port_result['mac_address'] = network.get('address') + port_result['ipv4_mask'] = network.get('ipv4_mask') + port_result['connection_status '] = constants.\ + PortConnectionStatus.CONNECTED + port_result['health_status'] = constants.PortHealthStatus.\ + NORMAL + list_ports.append(port_result) + return list_ports - def network_object(self, networks_object): - return_network = self.rest_handler.get_all_network() - if return_network: - for network in return_network: + def get_network(self): + networks_object = dict() + networks = self.rest_handler.get_networks() + if networks: + for network in networks: network_object = dict() network_object['address'] = network.get('address') services_list = network.get('services') if services_list: - services = services_list[0] - if services in constants.PortLogicalType.ALL: - network_object['logical_type'] = services - else: - network_object['logical_type'] = "" + for services in services_list: + network_object['logical_type'] = services if \ + services in constants.PortLogicalType.ALL else None + break network_object['speed'] = int(int(network.get('speed', 0)) / units.Ki) network_object['ipv4_mask'] = network.get('netmask') name = network.get('name') networks_object[name] = network_object + return networks_object def list_storage_pools(self, context): pool_list = [] - return_pools = self.rest_handler.get_capacity_pools() - if return_pools: - for pools in return_pools: - pool_object = dict() - total_capacity = int(pools.get('size')) - pool_object['total_capacity'] = total_capacity + pools = self.rest_handler.get_capacity_pools() + if pools: + for pool in pools: + pool_result = dict() + total_capacity = int(pool.get('size')) + pool_result['total_capacity'] = total_capacity - used_capacity = int(pools.get('total_reduction')) - pool_object['used_capacity'] = used_capacity + used_capacity = int(pool.get('total_reduction')) + pool_result['used_capacity'] = used_capacity - pool_object['free_capacity'] = total_capacity - used_capacity - pool_object['name'] = pools.get('name') - pool_object['native_storage_pool_id'] = pools.get('name') - pool_object['storage_id'] = self.storage_id - pool_object['description'] = "" - pool_object['status'] = constants.StoragePoolStatus.NORMAL - pool_object['storage_type'] = constants.StorageType.BLOCK - pool_list.append(pool_object) + pool_result['free_capacity'] = total_capacity - used_capacity + pool_result['name'] = pool.get('name') + pool_result['native_storage_pool_id'] = pool.get('name') + pool_result['storage_id'] = self.storage_id + pool_result['description'] = "" + pool_result['status'] = constants.StoragePoolStatus.NORMAL + pool_result['storage_type'] = constants.StorageType.BLOCK + pool_list.append(pool_result) return pool_list def remove_trap_config(self, context, trap_config): diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index 409041b5f..0f733bae3 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -3,6 +3,7 @@ from delfin import exception, cryptor from delfin.drivers.utils.rest_client import RestClient +from delfin.drivers.pure.pure.constants import c LOG = logging.getLogger(__name__) @@ -40,48 +41,43 @@ def login(self): try: data = {'username': self.username, 'password': cryptor.decode( self.password)} - token = self.get_rest_login(RestHandler.REST_AUTH_URL, data, + token = self.get_rest_token(RestHandler.REST_AUTH_URL, data, method='POST') api_token = token.get('api_token') if api_token is None and api_token == '': raise exception.InvalidInput('api_token fail to get') - user = self.get_rest_login(RestHandler.REST_SESSION_URL, token, + user = self.get_rest_token(RestHandler.REST_SESSION_URL, token, method='POST') username = user.get('username') - if username is None and username == '': + if username is None or username == '': raise exception.InvalidInput('session fail to get') except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise exception.InvalidInput('login failure') def logout(self): - user = self.get_rest_login(RestHandler.REST_SESSION_URL, + user = self.get_rest_token(RestHandler.REST_SESSION_URL, method='DELETE') username = user.get('username') - if username is None and username == '': + if username is None or username == '': raise exception.InvalidInput('delete fail to get') - def get_all_rest_volumes(self): + def get_volumes(self): url = '%s%s:%s%s' % ( RestHandler.HTTPS, self.host, self.port, RestHandler.REST_VOLUME_URL) result_json = self.get_rest_volumes_info(url) return result_json - def get_all_rest_volumes_id(self, name): - result_json = self.get_rest_info( - '{}{}'.format(RestHandler.REST_VOLUME_ID_URL, name)) - return result_json - def get_storage(self): result_json = self.get_rest_info(RestHandler.REST_STORAGE_URL) return result_json - def get_storage_ID(self): + def get_storage_id(self): result_json = self.get_rest_info(RestHandler.REST_STORAGE_ID_URL) return result_json - def get_all_pools(self): + def get_pools(self): result_json = self.get_rest_info(RestHandler.REST_POOLS_URL) return result_json @@ -89,27 +85,27 @@ def get_capacity_pools(self): result_json = self.get_rest_info(RestHandler.REST_POOLS_CAPACITY_URL) return result_json - def get_all_port(self): + def get_ports(self): result_json = self.get_rest_info(RestHandler.REST_PORT_URL) return result_json - def get_all_network(self): + def get_networks(self): result_json = self.get_rest_info(RestHandler.REST_NETWORK_URL) return result_json - def get_all_disk(self): + def get_disks(self): result_json = self.get_rest_info(RestHandler.REST_DISK_URL) return result_json - def get_all_hardware(self): + def get_hardware(self): result_json = self.get_rest_info(RestHandler.REST_HARDWARE_URL) return result_json - def get_all_controllers(self): + def get_controllers(self): result_json = self.get_rest_info(RestHandler.REST_CONTROLLERS_URL) return result_json - def get_all_alerts(self): + def get_alerts(self): result_json = self.get_rest_info(RestHandler.REST_ALERTS_URL) return result_json @@ -123,20 +119,20 @@ def get_rest_info(self, url, data=None, method='GET'): self.get_rest_info(url, data, method) return result_json - def get_rest_login(self, url, data=None, method='GET'): + def get_rest_token(self, url, data=None, method='GET'): self.init_http_head() res = self.do_call(url, data, method) result_json = res.json() return result_json - def get_rest_volumes_info(self, url, data=None, volume_list=None): + def get_rest_volumes_info(self, url, data=None, volume_list=None, count=0): if volume_list is None: volume_list = [] res = self.do_call(url, data, 'GET') if res.status_code == 200: result_json = res.json() volume_list.extend(result_json) - next_token = res.headers._store.get('x-next-token') + next_token = res.headers.get('x-next-token') if next_token: token = next_token[1] if token: @@ -146,5 +142,7 @@ def get_rest_volumes_info(self, url, data=None, volume_list=None): self.get_rest_volumes_info(url, data, volume_list) elif res.status_code == 401: self.login() - self.get_rest_volumes_info(url, data, volume_list) + if count < 3: + count = count + 1 + self.get_rest_volumes_info(url, data, volume_list, count) return volume_list diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py index e5541cfba..1c7200729 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -1,10 +1,14 @@ import sys from unittest import TestCase, mock +from oslo_log import log + +import six sys.modules['delfin.cryptor'] = mock.Mock() from delfin import context from delfin.drivers.pure.pure.rest_handler import RestHandler -from delfin.drivers.pure.pure.pure_storage import StorageDriver +from delfin.drivers.pure.pure.pure_storage import PureStorageDriver +LOG = log.getLogger(__name__) ACCESS_INFO = { "rest": { @@ -261,14 +265,14 @@ } ] reset_connection_info = { - "username": "oksd" + "username": "username" } def create_driver(): RestHandler.login = mock.Mock( - return_value={""}) - return StorageDriver(**ACCESS_INFO) + return_value={None}) + return PureStorageDriver(**ACCESS_INFO) class test_StorageDriver(TestCase): @@ -277,7 +281,7 @@ class test_StorageDriver(TestCase): def test_init(self): RestHandler.login = mock.Mock( return_value={""}) - StorageDriver(**ACCESS_INFO) + PureStorageDriver(**ACCESS_INFO) def test_list_volumes(self): RestHandler.get_rest_volumes_info = mock.Mock( @@ -330,6 +334,13 @@ def test_list_storage_pools(self): pools_info[0].get('name')) def test_reset_connection(self): - RestHandler.get_rest_login = mock.Mock( + RestHandler.get_rest_token = mock.Mock( side_effect=[reset_connection_info]) - self.driver.reset_connection(context) + name = None + try: + self.driver.reset_connection(context) + except Exception as e: + LOG.error("test_reset_connection error: %s", six.text_type(e)) + name = 'username' + self.assertNotEqual(reset_connection_info.get('username'), + name) From 5f639f21cc7aa83b1986873053d6a88d995e95cb Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 8 Nov 2021 17:16:04 +0800 Subject: [PATCH 09/69] pure codeview-Modify according to specifications --- delfin/drivers/pure/pure/consts.py | 65 ++++++ delfin/drivers/pure/pure/pure_storage.py | 216 ++++++++---------- delfin/drivers/pure/pure/rest_handler.py | 115 ++++++---- .../unit/drivers/pure/storage/test_storage.py | 33 +-- 4 files changed, 239 insertions(+), 190 deletions(-) create mode 100644 delfin/drivers/pure/pure/consts.py diff --git a/delfin/drivers/pure/pure/consts.py b/delfin/drivers/pure/pure/consts.py new file mode 100644 index 000000000..335582894 --- /dev/null +++ b/delfin/drivers/pure/pure/consts.py @@ -0,0 +1,65 @@ +# Copyright 2021 The SODA Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from delfin.common import constants + +# The default volume +DEFAULT_CAPACITY = 0 + +# The default speed +DEFAULT_SPEED = 0 + +# The default sequence for the get_volumes_info function +DEFAULT_LIST_GET_VOLUMES_INFO = 1 + +# The default count for the get_volumes_info function +DEFAULT_COUNT_GET_VOLUMES_INFO = 0 + +# Number of re-logins +RE_LOGIN_TIMES = 3 + +# Constant one +CONSTANT_ONE = 1 + +# Success status code +SUCCESS_STATUS_CODE = 200 + +# Status code of no permission +PERMISSION_DENIED_STATUS_CODE = 401 + +# Custom token of Pure +CUSTOM_TOKEN = 'x-next-token' + +SEVERITY_MAP = {'fatal': constants.Severity.FATAL, + 'critical': constants.Severity.CRITICAL, + 'major': constants.Severity.MAJOR, + 'minor': constants.Severity.MINOR, + 'warning': constants.Severity.WARNING, + 'informational': constants.Severity.INFORMATIONAL, + 'NotSpecified': constants.Severity.NOT_SPECIFIED} +CATEGORY_MAP = {'fault': constants.Category.FAULT, + 'event': constants.Category.EVENT, + 'recovery': constants.Category.RECOVERY, + 'notSpecified': constants.Category.NOT_SPECIFIED} +CONTROLLER_STATUS_MAP = {'normal': constants.ControllerStatus.NORMAL, + 'ready': constants.ControllerStatus.NORMAL, + 'offline': constants.ControllerStatus.OFFLINE, + 'fault': constants.ControllerStatus.FAULT, + 'degraded': constants.ControllerStatus.DEGRADED, + 'unknown': constants.ControllerStatus.UNKNOWN, + 'unready': constants.ControllerStatus.UNKNOWN} +DISK_STATUS_MAP = {'normal': constants.DiskStatus.NORMAL, + 'healthy': constants.DiskStatus.NORMAL, + 'abnormal': constants.DiskStatus.ABNORMAL, + 'unhealthy': constants.DiskStatus.ABNORMAL, + 'offline': constants.DiskStatus.OFFLINE} diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index c039e5120..5748b407b 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -5,35 +5,12 @@ from delfin.common import constants from delfin.drivers import driver -from delfin.drivers.pure.pure import rest_handler +from delfin.drivers.pure.pure import rest_handler, consts LOG = log.getLogger(__name__) class PureStorageDriver(driver.StorageDriver): - SEVERITY_MAP = {'fatal': constants.Severity.FATAL, - 'critical': constants.Severity.CRITICAL, - 'major': constants.Severity.MAJOR, - 'minor': constants.Severity.MINOR, - 'warning': constants.Severity.WARNING, - 'informational': constants.Severity.INFORMATIONAL, - 'NotSpecified': constants.Severity.NOT_SPECIFIED} - CATEGORY_MAP = {'fault': constants.Category.FAULT, - 'event': constants.Category.EVENT, - 'recovery': constants.Category.RECOVERY, - 'notSpecified': constants.Category.NOT_SPECIFIED} - CONTROLLER_STATUS_MAP = {'normal': constants.ControllerStatus.NORMAL, - 'ready': constants.ControllerStatus.NORMAL, - 'offline': constants.ControllerStatus.OFFLINE, - 'fault': constants.ControllerStatus.FAULT, - 'degraded': constants.ControllerStatus.DEGRADED, - 'unknown': constants.ControllerStatus.UNKNOWN, - 'unready': constants.ControllerStatus.UNKNOWN} - DISK_STATUS_MAP = {'normal': constants.DiskStatus.NORMAL, - 'healthy': constants.DiskStatus.NORMAL, - 'abnormal': constants.DiskStatus.ABNORMAL, - 'unhealthy': constants.DiskStatus.ABNORMAL, - 'offline': constants.DiskStatus.OFFLINE} def __init__(self, **kwargs): super().__init__(**kwargs) @@ -46,19 +23,19 @@ def list_volumes(self, context): if volumes: pools = self.rest_handler.get_pools() for volume in volumes: - name = volume.get('name') + volume_name = volume.get('name') total_capacity = int(int(volume.get('size')) / units.Ki) used_capacity = int(int(volume.get('volumes')) / units.Ki) native_storage_pool_id = None if pools: for pool in pools: pool_volumes = pool.get('volumes') - if name in pool_volumes: + if volume_name in pool_volumes: native_storage_pool_id = pool.get('name') break volume_dict = { - 'native_volume_id': name, - 'name': name, + 'native_volume_id': volume_name, + 'name': volume_name, 'total_capacity': total_capacity, 'used_capacity': used_capacity, 'free_capacity': total_capacity - used_capacity, @@ -77,62 +54,64 @@ def clear_alert(self, context, alert): pass def get_storage(self, context): - result_storage = self.rest_handler.get_storage() + storages = self.rest_handler.get_storages() model = None total_capacity = None used_capacity = None - if result_storage: - for storage in result_storage: + if storages: + for storage in storages: model = storage.get('hostname') total_capacity = int(int(storage.get('provisioned', - 0)) / units.Ki) - used_capacity = int(int(storage.get('volumes', 0)) / - units.Ki) + consts.DEFAULT_CAPACITY)) + / units.Ki) + used_capacity = int(int(storage.get('volumes', + consts.DEFAULT_CAPACITY)) + / units.Ki) break - storage_id = self.rest_handler.get_storage_id() - name = None + arrays = self.rest_handler.get_arrays() + storage_name = None serial_number = None version = None - if storage_id: - name = storage_id.get('array_name') - serial_number = storage_id.get('id') - version = storage_id.get('version') - storage = { + if arrays: + storage_name = arrays.get('array_name') + serial_number = arrays.get('id') + version = arrays.get('version') + storage_result = { 'model': model, 'total_capacity': total_capacity, 'raw_capacity': total_capacity, 'used_capacity': used_capacity, 'free_capacity': total_capacity - used_capacity, 'vendor': 'PURE', - 'name': name, + 'name': storage_name, 'serial_number': serial_number, 'firmware_version': version, 'status': constants.StorageStatus.NORMAL, - 'location': name + 'location': storage_name } - return storage + return storage_result def list_alerts(self, context, query_para=None): - return_alerts = self.rest_handler.get_alerts() + alerts = self.rest_handler.get_alerts() alerts_list = [] - if return_alerts: - for alerts in return_alerts: + if alerts: + for alert in alerts: alerts_model = dict() - alerts_model['alert_id'] = alerts.get('id') - alerts_model['severity'] = PureStorageDriver.SEVERITY_MAP.get( - alerts.get('current_severity'), + alerts_model['alert_id'] = alert.get('id') + alerts_model['severity'] = consts.SEVERITY_MAP.get( + alert.get('current_severity'), constants.Severity.NOT_SPECIFIED) - alerts_model['category'] = PureStorageDriver.CATEGORY_MAP.get( - alerts.get('category'), constants.Category.NOT_SPECIFIED) + alerts_model['category'] = consts.CATEGORY_MAP.get( + alert.get('category'), constants.Category.NOT_SPECIFIED) - alerts_model['occur_time'] = alerts.get('opened') - alerts_model['description'] = alerts.get('event') - alerts_model['location'] = alerts.get('component_name') + alerts_model['occur_time'] = alert.get('opened') + alerts_model['description'] = alert.get('event') + alerts_model['location'] = alert.get('component_name') alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE - alerts_model['alert_name'] = alerts.get('id') - alerts_model['match_key'] = hashlib.md5(str(alerts.get('id')). + alerts_model['alert_name'] = alert.get('id') + alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). encode()).hexdigest() alerts_list.append(alerts_model) return alerts_list @@ -142,23 +121,22 @@ def list_controllers(self, context): controllers = self.rest_handler.get_controllers() if controllers: for controller in controllers: - controllers_object = dict() - name = controller.get('name') - controllers_object['name'] = name - controllers_object['status'] = PureStorageDriver.\ - CONTROLLER_STATUS_MAP.get(controller.get('status'), - constants.ControllerStatus. - UNKNOWN) - controllers_object['soft_version'] = controller.get('version') - controllers_object['storage_id'] = self.storage_id - controllers_object['id'] = name - controllers_object['native_controller_id'] = name - controllers_object['location'] = name - list_controllers.append(controllers_object) + controllers_dict = dict() + controller_name = controller.get('name') + controllers_dict['name'] = controller_name + controllers_dict['status'] = consts.CONTROLLER_STATUS_MAP. \ + get(controller.get('status'), + constants.ControllerStatus.UNKNOWN) + controllers_dict['soft_version'] = controller.get('version') + controllers_dict['storage_id'] = self.storage_id + controllers_dict['id'] = controller_name + controllers_dict['native_controller_id'] = controller_name + controllers_dict['location'] = controller_name + list_controllers.append(controllers_dict) return list_controllers def list_disks(self, context): - names = dict() + hardware_dict = dict() hardware = self.rest_handler.get_hardware() if hardware: for hardware_value in hardware: @@ -166,38 +144,39 @@ def list_disks(self, context): hardware_name['speed'] = hardware_value.get('speed') hardware_name['serial_number'] = hardware_value.get('serial') hardware_name['model'] = hardware_value.get('model') - name = hardware_value.get('name') - names[name] = hardware_name + hardware_value_name = hardware_value.get('name') + hardware_dict[hardware_value_name] = hardware_name list_disks = [] disks = self.rest_handler.get_disks() if disks: - for drive in disks: - disk = dict() - drive_name = drive.get('name') - disk['name'] = drive_name - physical_type = drive.get('type').lower() - disk['physical_type'] = physical_type \ - if physical_type in constants.DiskPhysicalType.ALL else\ + for disk in disks: + disk_dict = dict() + drive_name = disk.get('name') + disk_dict['name'] = drive_name + physical_type = disk.get('type').lower() + disk_dict['physical_type'] = physical_type \ + if physical_type in constants.DiskPhysicalType.ALL else \ constants.DiskPhysicalType.UNKNOWN - status = drive.get('status') - disk['status'] = PureStorageDriver.DISK_STATUS_MAP.\ + status = disk.get('status') + disk_dict['status'] = consts.DISK_STATUS_MAP. \ get(status, constants.DiskStatus.OFFLINE) - disk['storage_id'] = self.storage_id - disk['capacity'] = int(int(drive.get('capacity')) / units.Ki) - - hardware_object = names.get(drive_name, {}) + disk_dict['storage_id'] = self.storage_id + disk_dict['capacity'] = int(int(disk.get('capacity')) / + units.Ki) + hardware_object = hardware_dict.get(drive_name, {}) speed = hardware_object.get('speed') - disk['speed'] = int(speed) if speed is not None else None - disk['model'] = hardware_object.get('model') - disk['serial_number'] = hardware_object.get('serial_number') + disk_dict['speed'] = int(speed) if speed is not None else None + disk_dict['model'] = hardware_object.get('model') + disk_dict['serial_number'] = hardware_object. \ + get('serial_number') - disk['native_disk_id'] = drive_name - disk['id'] = drive_name - disk['location'] = drive_name - disk['manufacturer'] = "pure" - disk['firmware'] = "" - list_disks.append(disk) + disk_dict['native_disk_id'] = drive_name + disk_dict['id'] = drive_name + disk_dict['location'] = drive_name + disk_dict['manufacturer'] = "pure" + disk_dict['firmware'] = "" + list_disks.append(disk_dict) return list_disks def list_ports(self, context): @@ -207,30 +186,26 @@ def list_ports(self, context): if ports: for port in ports: port_result = dict() - name = port.get('name') + port_name = port.get('name') wwn = port.get('wwn') if wwn: port_result['type'] = constants.PortType.FC - port_result['name'] = wwn - port_result['id'] = wwn - port_result['wwn'] = wwn else: port_result['type'] = constants.PortType.ETH - iqn = port.get('iqn') - port_result['name'] = iqn - port_result['id'] = iqn - port_result['wwn'] = '' - port_result['native_port_id'] = name - port_result['location'] = name + port_result['id'] = port_name + port_result['name'] = port_name + port_result['native_port_id'] = port_name + port_result['location'] = port_name port_result['storage_id'] = self.storage_id - network = networks.get(name.lower(), {}) + network = networks.get(port_name.lower(), {}) port_result['logical_type'] = network.get('logical_type') port_result['speed'] = network.get('speed') port_result['mac_address'] = network.get('address') port_result['ipv4_mask'] = network.get('ipv4_mask') - port_result['connection_status '] = constants.\ + port_result['wwn'] = wwn + port_result['connection_status '] = constants. \ PortConnectionStatus.CONNECTED - port_result['health_status'] = constants.PortHealthStatus.\ + port_result['health_status'] = constants.PortHealthStatus. \ NORMAL list_ports.append(port_result) return list_ports @@ -240,19 +215,20 @@ def get_network(self): networks = self.rest_handler.get_networks() if networks: for network in networks: - network_object = dict() - network_object['address'] = network.get('address') + network_dict = dict() + network_dict['address'] = network.get('address') services_list = network.get('services') if services_list: for services in services_list: - network_object['logical_type'] = services if \ + network_dict['logical_type'] = services if \ services in constants.PortLogicalType.ALL else None break - network_object['speed'] = int(int(network.get('speed', 0)) / - units.Ki) - network_object['ipv4_mask'] = network.get('netmask') - name = network.get('name') - networks_object[name] = network_object + network_dict['speed'] = int(int(network.get('speed', + consts. + DEFAULT_SPEED)) / units.Ki) + network_dict['ipv4_mask'] = network.get('netmask') + network_name = network.get('name') + networks_object[network_name] = network_dict return networks_object def list_storage_pools(self, context): @@ -261,17 +237,15 @@ def list_storage_pools(self, context): if pools: for pool in pools: pool_result = dict() - total_capacity = int(pool.get('size')) + total_capacity = int(pool.get('size', consts.DEFAULT_CAPACITY)) pool_result['total_capacity'] = total_capacity - - used_capacity = int(pool.get('total_reduction')) + used_capacity = int(pool.get('total_reduction', + consts.DEFAULT_CAPACITY)) pool_result['used_capacity'] = used_capacity - pool_result['free_capacity'] = total_capacity - used_capacity pool_result['name'] = pool.get('name') pool_result['native_storage_pool_id'] = pool.get('name') pool_result['storage_id'] = self.storage_id - pool_result['description'] = "" pool_result['status'] = constants.StoragePoolStatus.NORMAL pool_result['storage_type'] = constants.StorageType.BLOCK pool_list.append(pool_result) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index 0f733bae3..4abc7ef22 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -2,15 +2,15 @@ from oslo_log import log as logging from delfin import exception, cryptor +from delfin.drivers.pure.pure import consts from delfin.drivers.utils.rest_client import RestClient -from delfin.drivers.pure.pure.constants import c LOG = logging.getLogger(__name__) class RestHandler(RestClient): REST_STORAGE_URL = '/api/1.17/array?space=true' - REST_STORAGE_ID_URL = '/api/1.17/array' + REST_ARRAY_URL = '/api/1.17/array' REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=20&token=aWQgPSA5OD' \ 'A1Mg==' REST_VOLUME_TOKEN_URL = '/api/1.17/volume?space=true&limit=20&token=' @@ -41,108 +41,131 @@ def login(self): try: data = {'username': self.username, 'password': cryptor.decode( self.password)} - token = self.get_rest_token(RestHandler.REST_AUTH_URL, data, - method='POST') - api_token = token.get('api_token') - if api_token is None and api_token == '': - raise exception.InvalidInput('api_token fail to get') - user = self.get_rest_token(RestHandler.REST_SESSION_URL, token, + token_res = self.get_token(RestHandler.REST_AUTH_URL, data, method='POST') - username = user.get('username') - if username is None or username == '': - raise exception.InvalidInput('session fail to get') + if token_res.status_code != consts.SUCCESS_STATUS_CODE: + LOG.error("Login error.URL: %s, Reason: %s.", + RestHandler.REST_AUTH_URL, token_res.text) + raise exception.StorageBackendException(token_res.text) + else: + api_token = token_res.json().get('api_token') + if not api_token: + LOG.error("Login error. URL: %s, Reason: %s", + RestHandler.REST_AUTH_URL, + 'The API token does not exist') + raise exception.InvalidResults('The API token does not ' + 'exist') + session_res = self.get_token(RestHandler.REST_SESSION_URL, + api_token, method='POST') + if session_res.status_code == consts.SUCCESS_STATUS_CODE: + username = session_res.json().get('username') + if not username: + LOG.error("Login error. URL: %s, Reason: %s", + RestHandler.REST_SESSION_URL, + 'The API session does not exist') + raise exception.InvalidResults('The API session does ' + 'not exist') + else: + LOG.error("Login error.URL: %s, Reason: %s.", + RestHandler.REST_AUTH_URL, token_res.text) + raise exception.StorageBackendException(token_res.text) except Exception as e: LOG.error("Login error: %s", six.text_type(e)) - raise exception.InvalidInput('login failure') + raise e def logout(self): - user = self.get_rest_token(RestHandler.REST_SESSION_URL, - method='DELETE') - username = user.get('username') - if username is None or username == '': - raise exception.InvalidInput('delete fail to get') + res = self.get_token(RestHandler.REST_SESSION_URL, method='DELETE') + if res.status_code == consts.SUCCESS_STATUS_CODE: + username = res.json().get('username') + if not username: + raise exception.InvalidResults('The returned username' + ' is empty') + else: + LOG.error("Logout error.URL: %s, Reason: %s.", + RestHandler.REST_SESSION_URL, res.text) + raise exception.StorageBackendException(res.text) def get_volumes(self): url = '%s%s:%s%s' % ( RestHandler.HTTPS, self.host, self.port, RestHandler.REST_VOLUME_URL) - result_json = self.get_rest_volumes_info(url) + result_json = self.get_volumes_info(url) return result_json - def get_storage(self): - result_json = self.get_rest_info(RestHandler.REST_STORAGE_URL) + def get_storages(self): + result_json = self.get_info(RestHandler.REST_STORAGE_URL) return result_json - def get_storage_id(self): - result_json = self.get_rest_info(RestHandler.REST_STORAGE_ID_URL) + def get_arrays(self): + result_json = self.get_info(RestHandler.REST_ARRAY_URL) return result_json def get_pools(self): - result_json = self.get_rest_info(RestHandler.REST_POOLS_URL) + result_json = self.get_info(RestHandler.REST_POOLS_URL) return result_json def get_capacity_pools(self): - result_json = self.get_rest_info(RestHandler.REST_POOLS_CAPACITY_URL) + result_json = self.get_info(RestHandler.REST_POOLS_CAPACITY_URL) return result_json def get_ports(self): - result_json = self.get_rest_info(RestHandler.REST_PORT_URL) + result_json = self.get_info(RestHandler.REST_PORT_URL) return result_json def get_networks(self): - result_json = self.get_rest_info(RestHandler.REST_NETWORK_URL) + result_json = self.get_info(RestHandler.REST_NETWORK_URL) return result_json def get_disks(self): - result_json = self.get_rest_info(RestHandler.REST_DISK_URL) + result_json = self.get_info(RestHandler.REST_DISK_URL) return result_json def get_hardware(self): - result_json = self.get_rest_info(RestHandler.REST_HARDWARE_URL) + result_json = self.get_info(RestHandler.REST_HARDWARE_URL) return result_json def get_controllers(self): - result_json = self.get_rest_info(RestHandler.REST_CONTROLLERS_URL) + result_json = self.get_info(RestHandler.REST_CONTROLLERS_URL) return result_json def get_alerts(self): - result_json = self.get_rest_info(RestHandler.REST_ALERTS_URL) + result_json = self.get_info(RestHandler.REST_ALERTS_URL) return result_json - def get_rest_info(self, url, data=None, method='GET'): + def get_info(self, url, data=None, method='GET'): result_json = None res = self.do_call(url, data, method) - if res.status_code == 200: + if res.status_code == consts.SUCCESS_STATUS_CODE: result_json = res.json() - elif res.status_code == 401: + elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() - self.get_rest_info(url, data, method) + self.get_info(url, data, method) return result_json - def get_rest_token(self, url, data=None, method='GET'): + def get_token(self, url, data=None, method='GET'): self.init_http_head() res = self.do_call(url, data, method) - result_json = res.json() - return result_json + return res - def get_rest_volumes_info(self, url, data=None, volume_list=None, count=0): + def get_volumes_info(self, url, data=None, volume_list=None, + count=consts.DEFAULT_COUNT_GET_VOLUMES_INFO): if volume_list is None: volume_list = [] res = self.do_call(url, data, 'GET') - if res.status_code == 200: + if res.status_code == consts.SUCCESS_STATUS_CODE: result_json = res.json() volume_list.extend(result_json) - next_token = res.headers.get('x-next-token') + next_token = res.headers.get(consts.CUSTOM_TOKEN) if next_token: - token = next_token[1] + token = next_token[consts.DEFAULT_LIST_GET_VOLUMES_INFO] if token: url = '%s%s:%s%s%s' % ( RestHandler.HTTPS, self.host, self.port, RestHandler.REST_VOLUME_TOKEN_URL, token) - self.get_rest_volumes_info(url, data, volume_list) - elif res.status_code == 401: + self.get_volumes_info(url, data, volume_list) + elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() - if count < 3: - count = count + 1 - self.get_rest_volumes_info(url, data, volume_list, count) + if count < consts.RE_LOGIN_TIMES: + count = count + consts.CONSTANT_ONE + self.get_volumes_info(url, data, volume_list, count) return volume_list diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py index 1c7200729..98665fd8c 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -2,8 +2,6 @@ from unittest import TestCase, mock from oslo_log import log -import six - sys.modules['delfin.cryptor'] = mock.Mock() from delfin import context from delfin.drivers.pure.pure.rest_handler import RestHandler @@ -265,7 +263,8 @@ } ] reset_connection_info = { - "username": "username" + "username": "username", + "status_code": 200 } @@ -284,63 +283,51 @@ def test_init(self): PureStorageDriver(**ACCESS_INFO) def test_list_volumes(self): - RestHandler.get_rest_volumes_info = mock.Mock( + RestHandler.get_volumes_info = mock.Mock( side_effect=[volumes_info]) - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[pool_info]) volume = self.driver.list_volumes(context) self.assertEqual(volume[0]['native_volume_id'], pool_info[0].get('volumes')[0]) def test_get_storage(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[storage_info, storage_id_info]) storage_object = self.driver.get_storage(context) self.assertEqual(storage_object.get('name'), storage_id_info.get('array_name')) def test_list_alerts(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[alerts_info]) list_alerts = self.driver.list_alerts(context) self.assertEqual(list_alerts[0].get('alert_id'), alerts_info[0].get('id')) def test_list_controllers(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[controllers_info]) list_controllers = self.driver.list_controllers(context) self.assertEqual(list_controllers[0].get('name'), controllers_info[0].get('name')) def test_list_disks(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[hardware_info, drive_info]) list_disks = self.driver.list_disks(context) self.assertEqual(list_disks[0].get('name'), hardware_info[0].get('name')) def test_list_ports(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[port_network_info, port_info]) list_ports = self.driver.list_ports(context) self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) def test_list_storage_pools(self): - RestHandler.get_rest_info = mock.Mock( + RestHandler.get_info = mock.Mock( side_effect=[pools_info]) list_storage_pools = self.driver.list_storage_pools(context) self.assertEqual(list_storage_pools[0].get('native_storage_pool_id'), pools_info[0].get('name')) - - def test_reset_connection(self): - RestHandler.get_rest_token = mock.Mock( - side_effect=[reset_connection_info]) - name = None - try: - self.driver.reset_connection(context) - except Exception as e: - LOG.error("test_reset_connection error: %s", six.text_type(e)) - name = 'username' - self.assertNotEqual(reset_connection_info.get('username'), - name) From 952aaa07115f6ec0c676852691c4a87894d750e8 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 8 Nov 2021 18:18:55 +0800 Subject: [PATCH 10/69] pure codeview-recompose test_reset_connection --- delfin/drivers/pure/pure/pure_storage.py | 8 +++++-- .../unit/drivers/pure/storage/test_storage.py | 23 +++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index 5748b407b..b02cd49ec 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -24,8 +24,12 @@ def list_volumes(self, context): pools = self.rest_handler.get_pools() for volume in volumes: volume_name = volume.get('name') - total_capacity = int(int(volume.get('size')) / units.Ki) - used_capacity = int(int(volume.get('volumes')) / units.Ki) + total_capacity = int(int(volume.get('size', + consts.DEFAULT_CAPACITY)) / + units.Ki) + used_capacity = int(int(volume.get('volumes', + consts.DEFAULT_CAPACITY)) / + units.Ki) native_storage_pool_id = None if pools: for pool in pools: diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py index 98665fd8c..3a447a183 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -264,7 +264,7 @@ ] reset_connection_info = { "username": "username", - "status_code": 200 + "status": 200 } @@ -313,21 +313,30 @@ def test_list_controllers(self): controllers_info[0].get('name')) def test_list_disks(self): - RestHandler.get_info = mock.Mock( - side_effect=[hardware_info, drive_info]) + RestHandler.get_info = mock.Mock(side_effect= + [hardware_info, drive_info]) list_disks = self.driver.list_disks(context) self.assertEqual(list_disks[0].get('name'), hardware_info[0].get('name')) def test_list_ports(self): - RestHandler.get_info = mock.Mock( - side_effect=[port_network_info, port_info]) + RestHandler.get_info = mock.Mock(side_effect= + [port_network_info, port_info]) list_ports = self.driver.list_ports(context) self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) def test_list_storage_pools(self): - RestHandler.get_info = mock.Mock( - side_effect=[pools_info]) + RestHandler.get_info = mock.Mock(side_effect=[pools_info]) list_storage_pools = self.driver.list_storage_pools(context) self.assertEqual(list_storage_pools[0].get('native_storage_pool_id'), pools_info[0].get('name')) + + def test_reset_connection(self): + RestHandler.logout = mock.Mock(side_effect=None) + RestHandler.login = mock.Mock(side_effect=None) + username = None + try: + self.driver.reset_connection(context) + except Exception as e: + username = reset_connection_info.get('username') + self.assertEqual(username, None) From fff2e43e536b26a6c9196faf2747640cf58a576a Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 8 Nov 2021 18:27:51 +0800 Subject: [PATCH 11/69] pure codeview-recompose log test_reset_connection --- delfin/tests/unit/drivers/pure/storage/test_storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py index 3a447a183..b7e75c2f2 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -1,5 +1,7 @@ import sys from unittest import TestCase, mock + +import six from oslo_log import log sys.modules['delfin.cryptor'] = mock.Mock() @@ -338,5 +340,6 @@ def test_reset_connection(self): try: self.driver.reset_connection(context) except Exception as e: + LOG.error("test_reset_connection error: %s", six.text_type(e)) username = reset_connection_info.get('username') self.assertEqual(username, None) From 594c2baf0fe7465b1a347101d6c11d0d4a9d15b7 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 9 Nov 2021 09:45:56 +0800 Subject: [PATCH 12/69] pure codeview-test func format --- delfin/tests/unit/drivers/pure/storage/test_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/storage/test_storage.py index b7e75c2f2..38c556a0f 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/storage/test_storage.py @@ -315,15 +315,15 @@ def test_list_controllers(self): controllers_info[0].get('name')) def test_list_disks(self): - RestHandler.get_info = mock.Mock(side_effect= - [hardware_info, drive_info]) + RestHandler.get_info = mock.Mock( + side_effect=[hardware_info, drive_info]) list_disks = self.driver.list_disks(context) self.assertEqual(list_disks[0].get('name'), hardware_info[0].get('name')) def test_list_ports(self): - RestHandler.get_info = mock.Mock(side_effect= - [port_network_info, port_info]) + RestHandler.get_info = mock.Mock( + side_effect=[port_network_info, port_info]) list_ports = self.driver.list_ports(context) self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) From 7aaa25250e2da454853dd5062866af9ef029b76e Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 9 Nov 2021 10:08:50 +0800 Subject: [PATCH 13/69] pure codeview-test File name modification --- .../pure/{storage/test_storage.py => pure/test_pure_storage.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename delfin/tests/unit/drivers/pure/{storage/test_storage.py => pure/test_pure_storage.py} (99%) diff --git a/delfin/tests/unit/drivers/pure/storage/test_storage.py b/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py similarity index 99% rename from delfin/tests/unit/drivers/pure/storage/test_storage.py rename to delfin/tests/unit/drivers/pure/pure/test_pure_storage.py index 38c556a0f..b74d7f98b 100644 --- a/delfin/tests/unit/drivers/pure/storage/test_storage.py +++ b/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py @@ -276,7 +276,7 @@ def create_driver(): return PureStorageDriver(**ACCESS_INFO) -class test_StorageDriver(TestCase): +class test_PureStorageDriver(TestCase): driver = create_driver() def test_init(self): From cefca2cfdb0a87a8cdb1864742c64024e0d4374f Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 10 Nov 2021 11:08:24 +0800 Subject: [PATCH 14/69] pure codeview-Delete unnecessary Definitions --- delfin/drivers/pure/pure/pure_storage.py | 4 ++-- delfin/drivers/pure/pure/rest_handler.py | 21 ++++----------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index b02cd49ec..e42063290 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -158,7 +158,7 @@ def list_disks(self, context): disk_dict = dict() drive_name = disk.get('name') disk_dict['name'] = drive_name - physical_type = disk.get('type').lower() + physical_type = disk.get('type', "").lower() disk_dict['physical_type'] = physical_type \ if physical_type in constants.DiskPhysicalType.ALL else \ constants.DiskPhysicalType.UNKNOWN @@ -178,7 +178,7 @@ def list_disks(self, context): disk_dict['native_disk_id'] = drive_name disk_dict['id'] = drive_name disk_dict['location'] = drive_name - disk_dict['manufacturer'] = "pure" + disk_dict['manufacturer'] = "PURE" disk_dict['firmware'] = "" list_disks.append(disk_dict) return list_disks diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index 4abc7ef22..7d2786a00 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -25,22 +25,14 @@ class RestHandler(RestClient): REST_ALERTS_URL = '/api/1.17/message?flagged=true' REST_AUTH_URL = '/api/1.17/auth/apitoken' REST_SESSION_URL = '/api/1.17/auth/session' - HTTPS = 'https://' def __init__(self, **kwargs): super(RestHandler, self).__init__(**kwargs) - rest_access = kwargs.get('rest') - if rest_access is None: - raise exception.InvalidInput('Input pure_rest_access is missing') - self.host = rest_access.get('host') - self.port = rest_access.get('port') - self.username = rest_access.get('username') - self.password = rest_access.get('password') def login(self): try: - data = {'username': self.username, 'password': cryptor.decode( - self.password)} + data = {'username': self.rest_username, 'password': cryptor.decode( + self.rest_password)} token_res = self.get_token(RestHandler.REST_AUTH_URL, data, method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE: @@ -86,10 +78,7 @@ def logout(self): raise exception.StorageBackendException(res.text) def get_volumes(self): - url = '%s%s:%s%s' % ( - RestHandler.HTTPS, self.host, self.port, - RestHandler.REST_VOLUME_URL) - result_json = self.get_volumes_info(url) + result_json = self.get_volumes_info(RestHandler.REST_VOLUME_URL) return result_json def get_storages(self): @@ -159,9 +148,7 @@ def get_volumes_info(self, url, data=None, volume_list=None, if next_token: token = next_token[consts.DEFAULT_LIST_GET_VOLUMES_INFO] if token: - url = '%s%s:%s%s%s' % ( - RestHandler.HTTPS, self.host, self.port, - RestHandler.REST_VOLUME_TOKEN_URL, token) + url = '%s%s' % (RestHandler.REST_VOLUME_TOKEN_URL, token) self.get_volumes_info(url, data, volume_list) elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() From 11c61c979f9afa349689595c0d7d18f9b430a23c Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 11 Nov 2021 10:28:03 +0800 Subject: [PATCH 15/69] pure codeview-Modified the function name --- delfin/drivers/pure/pure/rest_handler.py | 36 +++++++++---------- .../drivers/pure/pure/test_pure_storage.py | 16 ++++----- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index 7d2786a00..33498611d 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -77,58 +77,54 @@ def logout(self): RestHandler.REST_SESSION_URL, res.text) raise exception.StorageBackendException(res.text) - def get_volumes(self): - result_json = self.get_volumes_info(RestHandler.REST_VOLUME_URL) - return result_json - def get_storages(self): - result_json = self.get_info(RestHandler.REST_STORAGE_URL) + result_json = self.rest_call(RestHandler.REST_STORAGE_URL) return result_json def get_arrays(self): - result_json = self.get_info(RestHandler.REST_ARRAY_URL) + result_json = self.rest_call(RestHandler.REST_ARRAY_URL) return result_json def get_pools(self): - result_json = self.get_info(RestHandler.REST_POOLS_URL) + result_json = self.rest_call(RestHandler.REST_POOLS_URL) return result_json def get_capacity_pools(self): - result_json = self.get_info(RestHandler.REST_POOLS_CAPACITY_URL) + result_json = self.rest_call(RestHandler.REST_POOLS_CAPACITY_URL) return result_json def get_ports(self): - result_json = self.get_info(RestHandler.REST_PORT_URL) + result_json = self.rest_call(RestHandler.REST_PORT_URL) return result_json def get_networks(self): - result_json = self.get_info(RestHandler.REST_NETWORK_URL) + result_json = self.rest_call(RestHandler.REST_NETWORK_URL) return result_json def get_disks(self): - result_json = self.get_info(RestHandler.REST_DISK_URL) + result_json = self.rest_call(RestHandler.REST_DISK_URL) return result_json def get_hardware(self): - result_json = self.get_info(RestHandler.REST_HARDWARE_URL) + result_json = self.rest_call(RestHandler.REST_HARDWARE_URL) return result_json def get_controllers(self): - result_json = self.get_info(RestHandler.REST_CONTROLLERS_URL) + result_json = self.rest_call(RestHandler.REST_CONTROLLERS_URL) return result_json def get_alerts(self): - result_json = self.get_info(RestHandler.REST_ALERTS_URL) + result_json = self.rest_call(RestHandler.REST_ALERTS_URL) return result_json - def get_info(self, url, data=None, method='GET'): + def rest_call(self, url, data=None, method='GET'): result_json = None res = self.do_call(url, data, method) if res.status_code == consts.SUCCESS_STATUS_CODE: result_json = res.json() elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() - self.get_info(url, data, method) + self.rest_call(url, data, method) return result_json def get_token(self, url, data=None, method='GET'): @@ -136,8 +132,8 @@ def get_token(self, url, data=None, method='GET'): res = self.do_call(url, data, method) return res - def get_volumes_info(self, url, data=None, volume_list=None, - count=consts.DEFAULT_COUNT_GET_VOLUMES_INFO): + def get_volumes(self, url=REST_VOLUME_URL, data=None, volume_list=None, + count=consts.DEFAULT_COUNT_GET_VOLUMES_INFO): if volume_list is None: volume_list = [] res = self.do_call(url, data, 'GET') @@ -149,10 +145,10 @@ def get_volumes_info(self, url, data=None, volume_list=None, token = next_token[consts.DEFAULT_LIST_GET_VOLUMES_INFO] if token: url = '%s%s' % (RestHandler.REST_VOLUME_TOKEN_URL, token) - self.get_volumes_info(url, data, volume_list) + self.get_volumes(url, data, volume_list) elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() if count < consts.RE_LOGIN_TIMES: count = count + consts.CONSTANT_ONE - self.get_volumes_info(url, data, volume_list, count) + self.get_volumes(url, data, volume_list, count) return volume_list diff --git a/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py b/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py index b74d7f98b..5053f563e 100644 --- a/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py +++ b/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py @@ -285,50 +285,50 @@ def test_init(self): PureStorageDriver(**ACCESS_INFO) def test_list_volumes(self): - RestHandler.get_volumes_info = mock.Mock( + RestHandler.get_volumes = mock.Mock( side_effect=[volumes_info]) - RestHandler.get_info = mock.Mock( + RestHandler.rest_call = mock.Mock( side_effect=[pool_info]) volume = self.driver.list_volumes(context) self.assertEqual(volume[0]['native_volume_id'], pool_info[0].get('volumes')[0]) def test_get_storage(self): - RestHandler.get_info = mock.Mock( + RestHandler.rest_call = mock.Mock( side_effect=[storage_info, storage_id_info]) storage_object = self.driver.get_storage(context) self.assertEqual(storage_object.get('name'), storage_id_info.get('array_name')) def test_list_alerts(self): - RestHandler.get_info = mock.Mock( + RestHandler.rest_call = mock.Mock( side_effect=[alerts_info]) list_alerts = self.driver.list_alerts(context) self.assertEqual(list_alerts[0].get('alert_id'), alerts_info[0].get('id')) def test_list_controllers(self): - RestHandler.get_info = mock.Mock( + RestHandler.rest_call = mock.Mock( side_effect=[controllers_info]) list_controllers = self.driver.list_controllers(context) self.assertEqual(list_controllers[0].get('name'), controllers_info[0].get('name')) def test_list_disks(self): - RestHandler.get_info = mock.Mock( + RestHandler.rest_call = mock.Mock( side_effect=[hardware_info, drive_info]) list_disks = self.driver.list_disks(context) self.assertEqual(list_disks[0].get('name'), hardware_info[0].get('name')) def test_list_ports(self): - RestHandler.get_info = mock.Mock( + RestHandler.rest_call = mock.Mock( side_effect=[port_network_info, port_info]) list_ports = self.driver.list_ports(context) self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) def test_list_storage_pools(self): - RestHandler.get_info = mock.Mock(side_effect=[pools_info]) + RestHandler.rest_call = mock.Mock(side_effect=[pools_info]) list_storage_pools = self.driver.list_storage_pools(context) self.assertEqual(list_storage_pools[0].get('native_storage_pool_id'), pools_info[0].get('name')) From 2c761fc960dc3252f1f8c597ae1f5b7e3f7ef9f2 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 11 Nov 2021 10:39:59 +0800 Subject: [PATCH 16/69] pure codeview-Modified the function name --- delfin/drivers/pure/pure/rest_handler.py | 40 ++++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index 33498611d..b69883f2a 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -78,44 +78,44 @@ def logout(self): raise exception.StorageBackendException(res.text) def get_storages(self): - result_json = self.rest_call(RestHandler.REST_STORAGE_URL) - return result_json + storages = self.rest_call(RestHandler.REST_STORAGE_URL) + return storages def get_arrays(self): - result_json = self.rest_call(RestHandler.REST_ARRAY_URL) - return result_json + arrays = self.rest_call(RestHandler.REST_ARRAY_URL) + return arrays def get_pools(self): - result_json = self.rest_call(RestHandler.REST_POOLS_URL) - return result_json + pools = self.rest_call(RestHandler.REST_POOLS_URL) + return pools def get_capacity_pools(self): - result_json = self.rest_call(RestHandler.REST_POOLS_CAPACITY_URL) - return result_json + capacity_pools = self.rest_call(RestHandler.REST_POOLS_CAPACITY_URL) + return capacity_pools def get_ports(self): - result_json = self.rest_call(RestHandler.REST_PORT_URL) - return result_json + ports = self.rest_call(RestHandler.REST_PORT_URL) + return ports def get_networks(self): - result_json = self.rest_call(RestHandler.REST_NETWORK_URL) - return result_json + networks = self.rest_call(RestHandler.REST_NETWORK_URL) + return networks def get_disks(self): - result_json = self.rest_call(RestHandler.REST_DISK_URL) - return result_json + disks = self.rest_call(RestHandler.REST_DISK_URL) + return disks def get_hardware(self): - result_json = self.rest_call(RestHandler.REST_HARDWARE_URL) - return result_json + hardware = self.rest_call(RestHandler.REST_HARDWARE_URL) + return hardware def get_controllers(self): - result_json = self.rest_call(RestHandler.REST_CONTROLLERS_URL) - return result_json + controllers = self.rest_call(RestHandler.REST_CONTROLLERS_URL) + return controllers def get_alerts(self): - result_json = self.rest_call(RestHandler.REST_ALERTS_URL) - return result_json + alerts = self.rest_call(RestHandler.REST_ALERTS_URL) + return alerts def rest_call(self, url, data=None, method='GET'): result_json = None From 19a7a81625171b6f2423f7f86b4e7fd6e8cf84b7 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 11 Nov 2021 11:29:23 +0800 Subject: [PATCH 17/69] pure codeview-Add files and change class names --- delfin/drivers/pure/__init__.py | 0 delfin/drivers/pure/pure/pure_storage.py | 3 +-- delfin/drivers/pure/pure/rest_handler.py | 2 +- delfin/tests/unit/drivers/pure/__init__.py | 0 delfin/tests/unit/drivers/pure/pure/__init__.py | 0 setup.py | 2 +- 6 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 delfin/drivers/pure/__init__.py create mode 100644 delfin/tests/unit/drivers/pure/__init__.py create mode 100644 delfin/tests/unit/drivers/pure/pure/__init__.py diff --git a/delfin/drivers/pure/__init__.py b/delfin/drivers/pure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index e42063290..1536e32bc 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -162,9 +162,8 @@ def list_disks(self, context): disk_dict['physical_type'] = physical_type \ if physical_type in constants.DiskPhysicalType.ALL else \ constants.DiskPhysicalType.UNKNOWN - status = disk.get('status') disk_dict['status'] = consts.DISK_STATUS_MAP. \ - get(status, constants.DiskStatus.OFFLINE) + get(disk.get('status'), constants.DiskStatus.OFFLINE) disk_dict['storage_id'] = self.storage_id disk_dict['capacity'] = int(int(disk.get('capacity')) / units.Ki) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index b69883f2a..a52421f33 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -33,6 +33,7 @@ def login(self): try: data = {'username': self.rest_username, 'password': cryptor.decode( self.rest_password)} + self.init_http_head() token_res = self.get_token(RestHandler.REST_AUTH_URL, data, method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE: @@ -128,7 +129,6 @@ def rest_call(self, url, data=None, method='GET'): return result_json def get_token(self, url, data=None, method='GET'): - self.init_http_head() res = self.do_call(url, data, method) return res diff --git a/delfin/tests/unit/drivers/pure/__init__.py b/delfin/tests/unit/drivers/pure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/tests/unit/drivers/pure/pure/__init__.py b/delfin/tests/unit/drivers/pure/pure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index fe94c7ec1..8fcddd13d 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'huawei oceanstor = delfin.drivers.huawei.oceanstor.oceanstor:OceanStorDriver', 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver' - 'pure pure = delfin.drivers.pure.pure.pure_storage:StorageDriver' + 'pure pure = delfin.drivers.pure.pure.pure_storage:PureStorageDriver' ] }, ) From 555bdc84a069b82bb8c57e97fc4fe8295827933f Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 11 Nov 2021 17:47:40 +0800 Subject: [PATCH 18/69] pure codeview-Remove unnecessary function wrapping --- delfin/drivers/pure/pure/pure_storage.py | 26 +++++++++------ delfin/drivers/pure/pure/rest_handler.py | 40 ------------------------ setup.py | 2 +- 3 files changed, 17 insertions(+), 51 deletions(-) diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/pure/pure_storage.py index 1536e32bc..2d8c81fa3 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/pure/pure_storage.py @@ -21,7 +21,8 @@ def list_volumes(self, context): list_volumes = [] volumes = self.rest_handler.get_volumes() if volumes: - pools = self.rest_handler.get_pools() + pools = self.rest_handler.rest_call( + self.rest_handler.REST_POOLS_URL) for volume in volumes: volume_name = volume.get('name') total_capacity = int(int(volume.get('size', @@ -58,7 +59,8 @@ def clear_alert(self, context, alert): pass def get_storage(self, context): - storages = self.rest_handler.get_storages() + storages = self.rest_handler.rest_call( + self.rest_handler.REST_STORAGE_URL) model = None total_capacity = None used_capacity = None @@ -73,7 +75,7 @@ def get_storage(self, context): / units.Ki) break - arrays = self.rest_handler.get_arrays() + arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) storage_name = None serial_number = None version = None @@ -97,7 +99,7 @@ def get_storage(self, context): return storage_result def list_alerts(self, context, query_para=None): - alerts = self.rest_handler.get_alerts() + alerts = self.rest_handler.rest_call(self.rest_handler.REST_ALERTS_URL) alerts_list = [] if alerts: for alert in alerts: @@ -122,7 +124,8 @@ def list_alerts(self, context, query_para=None): def list_controllers(self, context): list_controllers = [] - controllers = self.rest_handler.get_controllers() + controllers = self.rest_handler.rest_call( + self.rest_handler.REST_CONTROLLERS_URL) if controllers: for controller in controllers: controllers_dict = dict() @@ -141,7 +144,8 @@ def list_controllers(self, context): def list_disks(self, context): hardware_dict = dict() - hardware = self.rest_handler.get_hardware() + hardware = self.rest_handler.rest_call( + self.rest_handler.REST_HARDWARE_URL) if hardware: for hardware_value in hardware: hardware_name = dict() @@ -152,7 +156,7 @@ def list_disks(self, context): hardware_dict[hardware_value_name] = hardware_name list_disks = [] - disks = self.rest_handler.get_disks() + disks = self.rest_handler.rest_call(self.rest_handler.REST_DISK_URL) if disks: for disk in disks: disk_dict = dict() @@ -185,7 +189,7 @@ def list_disks(self, context): def list_ports(self, context): networks = self.get_network() list_ports = [] - ports = self.rest_handler.get_ports() + ports = self.rest_handler.rest_call(self.rest_handler.REST_PORT_URL) if ports: for port in ports: port_result = dict() @@ -215,7 +219,8 @@ def list_ports(self, context): def get_network(self): networks_object = dict() - networks = self.rest_handler.get_networks() + networks = self.rest_handler.rest_call( + self.rest_handler.REST_NETWORK_URL) if networks: for network in networks: network_dict = dict() @@ -236,7 +241,8 @@ def get_network(self): def list_storage_pools(self, context): pool_list = [] - pools = self.rest_handler.get_capacity_pools() + pools = self.rest_handler.rest_call( + self.rest_handler.REST_POOLS_CAPACITY_URL) if pools: for pool in pools: pool_result = dict() diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index a52421f33..55ced3c90 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -78,46 +78,6 @@ def logout(self): RestHandler.REST_SESSION_URL, res.text) raise exception.StorageBackendException(res.text) - def get_storages(self): - storages = self.rest_call(RestHandler.REST_STORAGE_URL) - return storages - - def get_arrays(self): - arrays = self.rest_call(RestHandler.REST_ARRAY_URL) - return arrays - - def get_pools(self): - pools = self.rest_call(RestHandler.REST_POOLS_URL) - return pools - - def get_capacity_pools(self): - capacity_pools = self.rest_call(RestHandler.REST_POOLS_CAPACITY_URL) - return capacity_pools - - def get_ports(self): - ports = self.rest_call(RestHandler.REST_PORT_URL) - return ports - - def get_networks(self): - networks = self.rest_call(RestHandler.REST_NETWORK_URL) - return networks - - def get_disks(self): - disks = self.rest_call(RestHandler.REST_DISK_URL) - return disks - - def get_hardware(self): - hardware = self.rest_call(RestHandler.REST_HARDWARE_URL) - return hardware - - def get_controllers(self): - controllers = self.rest_call(RestHandler.REST_CONTROLLERS_URL) - return controllers - - def get_alerts(self): - alerts = self.rest_call(RestHandler.REST_ALERTS_URL) - return alerts - def rest_call(self, url, data=None, method='GET'): result_json = None res = self.do_call(url, data, method) diff --git a/setup.py b/setup.py index 8fcddd13d..927aa041a 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'hpe 3par = delfin.drivers.hpe.hpe_3par.hpe_3parstor:Hpe3parStorDriver', 'huawei oceanstor = delfin.drivers.huawei.oceanstor.oceanstor:OceanStorDriver', 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', - 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver' + 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver', 'pure pure = delfin.drivers.pure.pure.pure_storage:PureStorageDriver' ] }, From 000fbef8c02b705f1a05753df5a62d54a3b5e8fa Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 12 Nov 2021 15:58:54 +0800 Subject: [PATCH 19/69] pure codeview-login amend --- delfin/drivers/pure/pure/rest_handler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index 55ced3c90..ae8f52f20 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -38,8 +38,8 @@ def login(self): method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE: LOG.error("Login error.URL: %s, Reason: %s.", - RestHandler.REST_AUTH_URL, token_res.text) - raise exception.StorageBackendException(token_res.text) + RestHandler.REST_AUTH_URL, token_res.status_code) + raise exception.StorageBackendException(token_res.status_code) else: api_token = token_res.json().get('api_token') if not api_token: @@ -60,8 +60,9 @@ def login(self): 'not exist') else: LOG.error("Login error.URL: %s, Reason: %s.", - RestHandler.REST_AUTH_URL, token_res.text) - raise exception.StorageBackendException(token_res.text) + RestHandler.REST_AUTH_URL, token_res.status_code) + raise exception.StorageBackendException( + token_res.status_code) except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e From aa261670759907d5667697f4452302f68cfddd77 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 12 Nov 2021 18:06:00 +0800 Subject: [PATCH 20/69] pure codeview-login amend --- delfin/drivers/pure/pure/rest_handler.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index ae8f52f20..c341eb5ed 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -37,32 +37,36 @@ def login(self): token_res = self.get_token(RestHandler.REST_AUTH_URL, data, method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE: - LOG.error("Login error.URL: %s, Reason: %s.", - RestHandler.REST_AUTH_URL, token_res.status_code) - raise exception.StorageBackendException(token_res.status_code) + LOG.error("Login error. status_code:%s, URL: %s, Reason: %s.", + token_res.status_code, RestHandler.REST_AUTH_URL, + token_res.text) + raise exception.StorageBackendException(token_res.text) else: api_token = token_res.json().get('api_token') if not api_token: - LOG.error("Login error. URL: %s, Reason: %s", + LOG.error("Login error. status_code:%s, URL: %s, Reason: " + "%s", token_res.status_code, RestHandler.REST_AUTH_URL, 'The API token does not exist') raise exception.InvalidResults('The API token does not ' 'exist') session_res = self.get_token(RestHandler.REST_SESSION_URL, - api_token, method='POST') + token_res.json(), method='POST') if session_res.status_code == consts.SUCCESS_STATUS_CODE: username = session_res.json().get('username') if not username: - LOG.error("Login error. URL: %s, Reason: %s", + LOG.error("Login error. status_code:%s, URL: %s, " + "Reason: %s", session_res.status_code, RestHandler.REST_SESSION_URL, 'The API session does not exist') raise exception.InvalidResults('The API session does ' 'not exist') else: - LOG.error("Login error.URL: %s, Reason: %s.", - RestHandler.REST_AUTH_URL, token_res.status_code) + LOG.error("Login error. status_code:%s," + " URL: %s, Reason: %s.", session_res.status_code, + RestHandler.REST_AUTH_URL, session_res.text) raise exception.StorageBackendException( - token_res.status_code) + token_res.text) except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e From 922cd7bbae99a0ea2ee28a71ae27f6b23e4dda1f Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 12 Nov 2021 18:12:35 +0800 Subject: [PATCH 21/69] pure codeview-The log print statement is modified --- delfin/drivers/pure/pure/rest_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/pure/rest_handler.py index c341eb5ed..2d4686739 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/pure/rest_handler.py @@ -79,8 +79,8 @@ def logout(self): raise exception.InvalidResults('The returned username' ' is empty') else: - LOG.error("Logout error.URL: %s, Reason: %s.", - RestHandler.REST_SESSION_URL, res.text) + LOG.error("Logout error.status_code:%s, URL: %s, Reason: %s.", + res.status_code, RestHandler.REST_SESSION_URL, res.text) raise exception.StorageBackendException(res.text) def rest_call(self, url, data=None, method='GET'): From 02dcc2e2dd47df6305af71a277663202db9723cb Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 14:44:33 +0800 Subject: [PATCH 22/69] pure codeview-file path --- delfin/drivers/pure/{pure => flasharray}/__init__.py | 0 delfin/drivers/pure/{pure => flasharray}/consts.py | 0 delfin/drivers/pure/{pure => flasharray}/pure_storage.py | 2 +- delfin/drivers/pure/{pure => flasharray}/rest_handler.py | 2 +- .../tests/unit/drivers/pure/{pure => flasharray}/__init__.py | 0 .../drivers/pure/{pure => flasharray}/test_pure_storage.py | 4 ++-- setup.py | 2 +- 7 files changed, 5 insertions(+), 5 deletions(-) rename delfin/drivers/pure/{pure => flasharray}/__init__.py (100%) rename delfin/drivers/pure/{pure => flasharray}/consts.py (100%) rename delfin/drivers/pure/{pure => flasharray}/pure_storage.py (99%) rename delfin/drivers/pure/{pure => flasharray}/rest_handler.py (99%) rename delfin/tests/unit/drivers/pure/{pure => flasharray}/__init__.py (100%) rename delfin/tests/unit/drivers/pure/{pure => flasharray}/test_pure_storage.py (98%) diff --git a/delfin/drivers/pure/pure/__init__.py b/delfin/drivers/pure/flasharray/__init__.py similarity index 100% rename from delfin/drivers/pure/pure/__init__.py rename to delfin/drivers/pure/flasharray/__init__.py diff --git a/delfin/drivers/pure/pure/consts.py b/delfin/drivers/pure/flasharray/consts.py similarity index 100% rename from delfin/drivers/pure/pure/consts.py rename to delfin/drivers/pure/flasharray/consts.py diff --git a/delfin/drivers/pure/pure/pure_storage.py b/delfin/drivers/pure/flasharray/pure_storage.py similarity index 99% rename from delfin/drivers/pure/pure/pure_storage.py rename to delfin/drivers/pure/flasharray/pure_storage.py index 2d8c81fa3..b1e9cf6fa 100644 --- a/delfin/drivers/pure/pure/pure_storage.py +++ b/delfin/drivers/pure/flasharray/pure_storage.py @@ -5,7 +5,7 @@ from delfin.common import constants from delfin.drivers import driver -from delfin.drivers.pure.pure import rest_handler, consts +from delfin.drivers.pure.flasharray import rest_handler, consts LOG = log.getLogger(__name__) diff --git a/delfin/drivers/pure/pure/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py similarity index 99% rename from delfin/drivers/pure/pure/rest_handler.py rename to delfin/drivers/pure/flasharray/rest_handler.py index 2d4686739..4ac96fa54 100644 --- a/delfin/drivers/pure/pure/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -2,7 +2,7 @@ from oslo_log import log as logging from delfin import exception, cryptor -from delfin.drivers.pure.pure import consts +from delfin.drivers.pure.flasharray import consts from delfin.drivers.utils.rest_client import RestClient LOG = logging.getLogger(__name__) diff --git a/delfin/tests/unit/drivers/pure/pure/__init__.py b/delfin/tests/unit/drivers/pure/flasharray/__init__.py similarity index 100% rename from delfin/tests/unit/drivers/pure/pure/__init__.py rename to delfin/tests/unit/drivers/pure/flasharray/__init__.py diff --git a/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_storage.py similarity index 98% rename from delfin/tests/unit/drivers/pure/pure/test_pure_storage.py rename to delfin/tests/unit/drivers/pure/flasharray/test_pure_storage.py index 5053f563e..d73965e01 100644 --- a/delfin/tests/unit/drivers/pure/pure/test_pure_storage.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_storage.py @@ -6,8 +6,8 @@ sys.modules['delfin.cryptor'] = mock.Mock() from delfin import context -from delfin.drivers.pure.pure.rest_handler import RestHandler -from delfin.drivers.pure.pure.pure_storage import PureStorageDriver +from delfin.drivers.pure.flasharray.rest_handler import RestHandler +from delfin.drivers.pure.flasharray.pure_storage import PureStorageDriver LOG = log.getLogger(__name__) ACCESS_INFO = { diff --git a/setup.py b/setup.py index 927aa041a..947f69e83 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'huawei oceanstor = delfin.drivers.huawei.oceanstor.oceanstor:OceanStorDriver', 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver', - 'pure pure = delfin.drivers.pure.pure.pure_storage:PureStorageDriver' + 'pure flasharray = delfin.drivers.pure.flasharray.pure_storage:PureStorageDriver' ] }, ) From 56d4b2d05aa4f8df00ef6b32844c576456e4b9c1 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 14:49:00 +0800 Subject: [PATCH 23/69] pure codeview-login log --- delfin/drivers/pure/flasharray/rest_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 4ac96fa54..d04d6e1b6 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -66,7 +66,7 @@ def login(self): " URL: %s, Reason: %s.", session_res.status_code, RestHandler.REST_AUTH_URL, session_res.text) raise exception.StorageBackendException( - token_res.text) + session_res.text) except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e From ee537b4a1c4ae23e3526a921991c6c4a8126b35e Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 15:01:29 +0800 Subject: [PATCH 24/69] pure codeview-login log --- delfin/drivers/pure/flasharray/rest_handler.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index d04d6e1b6..2fe6c0a9c 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -39,8 +39,9 @@ def login(self): if token_res.status_code != consts.SUCCESS_STATUS_CODE: LOG.error("Login error. status_code:%s, URL: %s, Reason: %s.", token_res.status_code, RestHandler.REST_AUTH_URL, - token_res.text) - raise exception.StorageBackendException(token_res.text) + 'Obtaining the token is abnormal. Procedure') + raise exception.StorageBackendException( + 'Obtaining the token is abnormal. Procedure') else: api_token = token_res.json().get('api_token') if not api_token: @@ -64,9 +65,10 @@ def login(self): else: LOG.error("Login error. status_code:%s," " URL: %s, Reason: %s.", session_res.status_code, - RestHandler.REST_AUTH_URL, session_res.text) + RestHandler.REST_AUTH_URL, + 'Obtaining the session is abnormal. Procedure') raise exception.StorageBackendException( - session_res.text) + 'Obtaining the session is abnormal. Procedure') except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e From 08981a5f5150b63aefc591ce80bcf7c2a1264726 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 15:13:57 +0800 Subject: [PATCH 25/69] pure codeview-file path and class name --- .../flasharray/{pure_storage.py => pure_flasharray.py} | 2 +- .../{test_pure_storage.py => test_pure_flasharray.py} | 8 ++++---- setup.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename delfin/drivers/pure/flasharray/{pure_storage.py => pure_flasharray.py} (99%) rename delfin/tests/unit/drivers/pure/flasharray/{test_pure_storage.py => test_pure_flasharray.py} (97%) diff --git a/delfin/drivers/pure/flasharray/pure_storage.py b/delfin/drivers/pure/flasharray/pure_flasharray.py similarity index 99% rename from delfin/drivers/pure/flasharray/pure_storage.py rename to delfin/drivers/pure/flasharray/pure_flasharray.py index b1e9cf6fa..f2d21460c 100644 --- a/delfin/drivers/pure/flasharray/pure_storage.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -10,7 +10,7 @@ LOG = log.getLogger(__name__) -class PureStorageDriver(driver.StorageDriver): +class PureFlashArrayDriver(driver.StorageDriver): def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_storage.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py similarity index 97% rename from delfin/tests/unit/drivers/pure/flasharray/test_pure_storage.py rename to delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index d73965e01..ce6ad5559 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_storage.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -7,7 +7,7 @@ sys.modules['delfin.cryptor'] = mock.Mock() from delfin import context from delfin.drivers.pure.flasharray.rest_handler import RestHandler -from delfin.drivers.pure.flasharray.pure_storage import PureStorageDriver +from delfin.drivers.pure.flasharray.pure_flasharray import PureFlashArrayDriver LOG = log.getLogger(__name__) ACCESS_INFO = { @@ -273,16 +273,16 @@ def create_driver(): RestHandler.login = mock.Mock( return_value={None}) - return PureStorageDriver(**ACCESS_INFO) + return PureFlashArrayDriver(**ACCESS_INFO) -class test_PureStorageDriver(TestCase): +class test_PureFlashArrayDriver(TestCase): driver = create_driver() def test_init(self): RestHandler.login = mock.Mock( return_value={""}) - PureStorageDriver(**ACCESS_INFO) + PureFlashArrayDriver(**ACCESS_INFO) def test_list_volumes(self): RestHandler.get_volumes = mock.Mock( diff --git a/setup.py b/setup.py index 947f69e83..a0bb95f6a 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'huawei oceanstor = delfin.drivers.huawei.oceanstor.oceanstor:OceanStorDriver', 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver', - 'pure flasharray = delfin.drivers.pure.flasharray.pure_storage:PureStorageDriver' + 'pure flasharray = delfin.drivers.pure.flasharray.pure_flasharray:PureFlashArrayDriver' ] }, ) From c2eed42115fefd7f35d14f886892939491ecf92c Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 15:23:15 +0800 Subject: [PATCH 26/69] pure codeview-token log --- delfin/drivers/pure/flasharray/rest_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 2fe6c0a9c..1bcffac48 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -39,9 +39,9 @@ def login(self): if token_res.status_code != consts.SUCCESS_STATUS_CODE: LOG.error("Login error. status_code:%s, URL: %s, Reason: %s.", token_res.status_code, RestHandler.REST_AUTH_URL, - 'Obtaining the token is abnormal. Procedure') + 'Obtaining the token is abnormal') raise exception.StorageBackendException( - 'Obtaining the token is abnormal. Procedure') + 'Obtaining the token is abnormal') else: api_token = token_res.json().get('api_token') if not api_token: @@ -66,9 +66,9 @@ def login(self): LOG.error("Login error. status_code:%s," " URL: %s, Reason: %s.", session_res.status_code, RestHandler.REST_AUTH_URL, - 'Obtaining the session is abnormal. Procedure') + 'Obtaining the session is abnormal') raise exception.StorageBackendException( - 'Obtaining the session is abnormal. Procedure') + 'Obtaining the session is abnormal') except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e From 7473811ee8909099830b643daeabd4cb0731a44b Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 15:34:44 +0800 Subject: [PATCH 27/69] pure codeview-token session log --- .../drivers/pure/flasharray/rest_handler.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 1bcffac48..f5a743711 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -37,38 +37,37 @@ def login(self): token_res = self.get_token(RestHandler.REST_AUTH_URL, data, method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE: - LOG.error("Login error. status_code:%s, URL: %s, Reason: %s.", - token_res.status_code, RestHandler.REST_AUTH_URL, - 'Obtaining the token is abnormal') + LOG.error("Login error, Obtaining the token is abnormal. " + "status_code:%s, URL: %s", + token_res.status_code, RestHandler.REST_AUTH_URL) raise exception.StorageBackendException( 'Obtaining the token is abnormal') else: api_token = token_res.json().get('api_token') if not api_token: - LOG.error("Login error. status_code:%s, URL: %s, Reason: " - "%s", token_res.status_code, - RestHandler.REST_AUTH_URL, - 'The API token does not exist') - raise exception.InvalidResults('The API token does not ' + LOG.error("Login error, the api token does not exist. " + "status_code:%s, URL: %s", token_res.status_code, + RestHandler.REST_AUTH_URL) + raise exception.InvalidResults('the api token does not ' 'exist') session_res = self.get_token(RestHandler.REST_SESSION_URL, token_res.json(), method='POST') if session_res.status_code == consts.SUCCESS_STATUS_CODE: username = session_res.json().get('username') if not username: - LOG.error("Login error. status_code:%s, URL: %s, " - "Reason: %s", session_res.status_code, - RestHandler.REST_SESSION_URL, - 'The API session does not exist') - raise exception.InvalidResults('The API session does ' + LOG.error("Login error, the api session does not exist" + ". status_code:%s, URL: %s", + session_res.status_code, + RestHandler.REST_SESSION_URL) + raise exception.InvalidResults('the api session does ' 'not exist') else: - LOG.error("Login error. status_code:%s," - " URL: %s, Reason: %s.", session_res.status_code, - RestHandler.REST_AUTH_URL, - 'Obtaining the session is abnormal') + LOG.error("Login error, obtaining the session is abnormal." + " status_code:%s, URL: %s", + session_res.status_code, + RestHandler.REST_AUTH_URL) raise exception.StorageBackendException( - 'Obtaining the session is abnormal') + 'obtaining the session is abnormal') except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e @@ -81,8 +80,9 @@ def logout(self): raise exception.InvalidResults('The returned username' ' is empty') else: - LOG.error("Logout error.status_code:%s, URL: %s, Reason: %s.", - res.status_code, RestHandler.REST_SESSION_URL, res.text) + LOG.error("Logout error, Deleting a Token Exception." + "status_code:%s, URL: %s", + res.status_code, RestHandler.REST_SESSION_URL) raise exception.StorageBackendException(res.text) def rest_call(self, url, data=None, method='GET'): From 3542fadc0cd78f25121b1021d97b01ac210434d4 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 15:53:04 +0800 Subject: [PATCH 28/69] pure codeview-storage model --- delfin/drivers/pure/flasharray/pure_flasharray.py | 11 +++++++++-- .../drivers/pure/flasharray/test_pure_flasharray.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index f2d21460c..f5d838d3e 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -61,12 +61,10 @@ def clear_alert(self, context, alert): def get_storage(self, context): storages = self.rest_handler.rest_call( self.rest_handler.REST_STORAGE_URL) - model = None total_capacity = None used_capacity = None if storages: for storage in storages: - model = storage.get('hostname') total_capacity = int(int(storage.get('provisioned', consts.DEFAULT_CAPACITY)) / units.Ki) @@ -83,6 +81,15 @@ def get_storage(self, context): storage_name = arrays.get('array_name') serial_number = arrays.get('id') version = arrays.get('version') + + model = None + controllers = self.rest_handler.rest_call( + self.rest_handler.REST_CONTROLLERS_URL) + if controllers: + for controller in controllers: + model = controller.get('mode') + break + storage_result = { 'model': model, 'total_capacity': total_capacity, diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index ce6ad5559..d505ef88d 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -295,7 +295,7 @@ def test_list_volumes(self): def test_get_storage(self): RestHandler.rest_call = mock.Mock( - side_effect=[storage_info, storage_id_info]) + side_effect=[storage_info, storage_id_info, controllers_info]) storage_object = self.driver.get_storage(context) self.assertEqual(storage_object.get('name'), storage_id_info.get('array_name')) From 6ec2515247ab31618033da3002f192e0ac12a2d5 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 16:49:26 +0800 Subject: [PATCH 29/69] pure codeview-get_volumes token --- delfin/drivers/pure/flasharray/rest_handler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index f5a743711..577b32136 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -109,10 +109,8 @@ def get_volumes(self, url=REST_VOLUME_URL, data=None, volume_list=None, volume_list.extend(result_json) next_token = res.headers.get(consts.CUSTOM_TOKEN) if next_token: - token = next_token[consts.DEFAULT_LIST_GET_VOLUMES_INFO] - if token: - url = '%s%s' % (RestHandler.REST_VOLUME_TOKEN_URL, token) - self.get_volumes(url, data, volume_list) + url = '%s%s' % (RestHandler.REST_VOLUME_TOKEN_URL, next_token) + self.get_volumes(url, data, volume_list) elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() if count < consts.RE_LOGIN_TIMES: From a6e19dc295ff60a9448f6076852797afa9424455 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 17:23:55 +0800 Subject: [PATCH 30/69] pure codeview-get_storage model --- delfin/drivers/pure/flasharray/pure_flasharray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index f5d838d3e..4b26bac3a 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -87,7 +87,7 @@ def get_storage(self, context): self.rest_handler.REST_CONTROLLERS_URL) if controllers: for controller in controllers: - model = controller.get('mode') + model = controller.get('model') break storage_result = { From b968959894fd6584ef333cbcfa5d4b39fec6f644 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 18:46:42 +0800 Subject: [PATCH 31/69] pure codeview-disk capacity and alerts Time conversion --- delfin/drivers/pure/flasharray/consts.py | 4 ++-- delfin/drivers/pure/flasharray/pure_flasharray.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 335582894..1c892bf99 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -19,8 +19,8 @@ # The default speed DEFAULT_SPEED = 0 -# The default sequence for the get_volumes_info function -DEFAULT_LIST_GET_VOLUMES_INFO = 1 +# The default list_alerts time conversion +DEFAULT_LIST_ALERTS_TIME_CONVERSION = 1000 # The default count for the get_volumes_info function DEFAULT_COUNT_GET_VOLUMES_INFO = 0 diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 4b26bac3a..1841cef92 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -1,3 +1,4 @@ +import datetime import hashlib from oslo_log import log @@ -117,8 +118,11 @@ def list_alerts(self, context, query_para=None): constants.Severity.NOT_SPECIFIED) alerts_model['category'] = consts.CATEGORY_MAP.get( alert.get('category'), constants.Category.NOT_SPECIFIED) - - alerts_model['occur_time'] = alert.get('opened') + time = alert.get('opened') + alerts_model['occur_time'] = datetime.datetime.strptime( + time, '%Y-%m-%dT%H:%M:%SZ').timestamp() * \ + consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION\ + if time is not None else None alerts_model['description'] = alert.get('event') alerts_model['location'] = alert.get('component_name') alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM @@ -176,8 +180,7 @@ def list_disks(self, context): disk_dict['status'] = consts.DISK_STATUS_MAP. \ get(disk.get('status'), constants.DiskStatus.OFFLINE) disk_dict['storage_id'] = self.storage_id - disk_dict['capacity'] = int(int(disk.get('capacity')) / - units.Ki) + disk_dict['capacity'] = int(int(disk.get('capacity'))) hardware_object = hardware_dict.get(drive_name, {}) speed = hardware_object.get('speed') disk_dict['speed'] = int(speed) if speed is not None else None From 3b96e440667e5bf23560a317b782ee05948fd9ec Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 15 Nov 2021 18:51:48 +0800 Subject: [PATCH 32/69] pure codeview-alerts Time conversion --- delfin/drivers/pure/flasharray/pure_flasharray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 1841cef92..7891bc055 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -119,9 +119,9 @@ def list_alerts(self, context, query_para=None): alerts_model['category'] = consts.CATEGORY_MAP.get( alert.get('category'), constants.Category.NOT_SPECIFIED) time = alert.get('opened') - alerts_model['occur_time'] = datetime.datetime.strptime( - time, '%Y-%m-%dT%H:%M:%SZ').timestamp() * \ - consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION\ + alerts_model['occur_time'] = int(datetime.datetime.strptime( + time, '%Y-%m-%dT%H:%M:%SZ').timestamp() + * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ if time is not None else None alerts_model['description'] = alert.get('event') alerts_model['location'] = alert.get('component_name') From ec6054fee8bb543d9d238bfa2f0e3bbf79924280 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 16 Nov 2021 16:07:56 +0800 Subject: [PATCH 33/69] pure codeview-Business correction --- delfin/drivers/pure/flasharray/consts.py | 9 ++ .../pure/flasharray/pure_flasharray.py | 88 +++++++++++-------- .../drivers/pure/flasharray/rest_handler.py | 17 ++-- 3 files changed, 68 insertions(+), 46 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 1c892bf99..cdd35be2a 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -30,6 +30,8 @@ # Constant one CONSTANT_ONE = 1 +# Constant zero +CONSTANT_ZERO = 0 # Success status code SUCCESS_STATUS_CODE = 200 @@ -40,6 +42,13 @@ # Custom token of Pure CUSTOM_TOKEN = 'x-next-token' +# The mode value of the controller +CONTROLLER_PRIMARY = 'primary' + +# list_port: Add ":" to the WWN every 2 sequences. +SPLICE_WWN_SERIAL = 2 +SPLICE_WWN_COLON = ':' + SEVERITY_MAP = {'fatal': constants.Severity.FATAL, 'critical': constants.Severity.CRITICAL, 'major': constants.Severity.MAJOR, diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 7891bc055..4cfc67644 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -26,12 +26,10 @@ def list_volumes(self, context): self.rest_handler.REST_POOLS_URL) for volume in volumes: volume_name = volume.get('name') - total_capacity = int(int(volume.get('size', - consts.DEFAULT_CAPACITY)) / - units.Ki) - used_capacity = int(int(volume.get('volumes', - consts.DEFAULT_CAPACITY)) / - units.Ki) + total_capacity = int(volume.get('size', + consts.DEFAULT_CAPACITY)) + used_capacity = int(volume.get('volumes', + consts.DEFAULT_CAPACITY)) native_storage_pool_id = None if pools: for pool in pools: @@ -66,12 +64,10 @@ def get_storage(self, context): used_capacity = None if storages: for storage in storages: - total_capacity = int(int(storage.get('provisioned', - consts.DEFAULT_CAPACITY)) - / units.Ki) - used_capacity = int(int(storage.get('volumes', - consts.DEFAULT_CAPACITY)) - / units.Ki) + total_capacity = int(storage.get('provisioned', + consts.DEFAULT_CAPACITY)) + used_capacity = int(storage.get('volumes', + consts.DEFAULT_CAPACITY)) break arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) @@ -88,8 +84,9 @@ def get_storage(self, context): self.rest_handler.REST_CONTROLLERS_URL) if controllers: for controller in controllers: - model = controller.get('model') - break + if controller.get('mode') == consts.CONTROLLER_PRIMARY: + model = controller.get('model') + break storage_result = { 'model': model, @@ -101,8 +98,7 @@ def get_storage(self, context): 'name': storage_name, 'serial_number': serial_number, 'firmware_version': version, - 'status': constants.StorageStatus.NORMAL, - 'location': storage_name + 'status': constants.StorageStatus.NORMAL } return storage_result @@ -154,18 +150,7 @@ def list_controllers(self, context): return list_controllers def list_disks(self, context): - hardware_dict = dict() - hardware = self.rest_handler.rest_call( - self.rest_handler.REST_HARDWARE_URL) - if hardware: - for hardware_value in hardware: - hardware_name = dict() - hardware_name['speed'] = hardware_value.get('speed') - hardware_name['serial_number'] = hardware_value.get('serial') - hardware_name['model'] = hardware_value.get('model') - hardware_value_name = hardware_value.get('name') - hardware_dict[hardware_value_name] = hardware_name - + hardware_dict = self.get_hardware() list_disks = [] disks = self.rest_handler.rest_call(self.rest_handler.REST_DISK_URL) if disks: @@ -180,7 +165,8 @@ def list_disks(self, context): disk_dict['status'] = consts.DISK_STATUS_MAP. \ get(disk.get('status'), constants.DiskStatus.OFFLINE) disk_dict['storage_id'] = self.storage_id - disk_dict['capacity'] = int(int(disk.get('capacity'))) + disk_dict['capacity'] = int(disk.get('capacity', + consts.DEFAULT_CAPACITY)) hardware_object = hardware_dict.get(drive_name, {}) speed = hardware_object.get('speed') disk_dict['speed'] = int(speed) if speed is not None else None @@ -196,8 +182,23 @@ def list_disks(self, context): list_disks.append(disk_dict) return list_disks + def get_hardware(self): + hardware_dict = dict() + hardware = self.rest_handler.rest_call( + self.rest_handler.REST_HARDWARE_URL) + if hardware: + for hardware_value in hardware: + hardware_name = dict() + hardware_name['speed'] = hardware_value.get('speed') + hardware_name['serial_number'] = hardware_value.get('serial') + hardware_name['model'] = hardware_value.get('model') + hardware_value_name = hardware_value.get('name') + hardware_dict[hardware_value_name] = hardware_name + return hardware_dict + def list_ports(self, context): networks = self.get_network() + hardware_dict = self.get_hardware() list_ports = [] ports = self.rest_handler.rest_call(self.rest_handler.REST_PORT_URL) if ports: @@ -209,17 +210,21 @@ def list_ports(self, context): port_result['type'] = constants.PortType.FC else: port_result['type'] = constants.PortType.ETH - port_result['id'] = port_name port_result['name'] = port_name port_result['native_port_id'] = port_name port_result['location'] = port_name port_result['storage_id'] = self.storage_id - network = networks.get(port_name.lower(), {}) + network = networks.get(port_name, {}) port_result['logical_type'] = network.get('logical_type') - port_result['speed'] = network.get('speed') port_result['mac_address'] = network.get('address') port_result['ipv4_mask'] = network.get('ipv4_mask') - port_result['wwn'] = wwn + + hardware = hardware_dict.get(port_name, {}) + speed = hardware.get('speed') + port_result['speed'] = int(speed)\ + if speed is not None else None + wwn_splice = self.get_splice_wwn(wwn) + port_result['wwn'] = wwn_splice port_result['connection_status '] = constants. \ PortConnectionStatus.CONNECTED port_result['health_status'] = constants.PortHealthStatus. \ @@ -227,6 +232,16 @@ def list_ports(self, context): list_ports.append(port_result) return list_ports + @staticmethod + def get_splice_wwn(wwn): + wwn_list = list(wwn) + wwn_splice = wwn_list[0] + for serial in range(1, len(wwn_list)): + if serial % consts.SPLICE_WWN_SERIAL == consts.CONSTANT_ZERO: + wwn_splice = '{}{}'.format(wwn_splice, consts.SPLICE_WWN_COLON) + wwn_splice = '{}{}'.format(wwn_splice, wwn_list[serial]) + return wwn_splice + def get_network(self): networks_object = dict() networks = self.rest_handler.rest_call( @@ -241,11 +256,8 @@ def get_network(self): network_dict['logical_type'] = services if \ services in constants.PortLogicalType.ALL else None break - network_dict['speed'] = int(int(network.get('speed', - consts. - DEFAULT_SPEED)) / units.Ki) network_dict['ipv4_mask'] = network.get('netmask') - network_name = network.get('name') + network_name = network.get('name').upper() networks_object[network_name] = network_dict return networks_object @@ -257,6 +269,8 @@ def list_storage_pools(self, context): for pool in pools: pool_result = dict() total_capacity = int(pool.get('size', consts.DEFAULT_CAPACITY)) + if total_capacity == consts.DEFAULT_CAPACITY: + continue pool_result['total_capacity'] = total_capacity used_capacity = int(pool.get('total_reduction', consts.DEFAULT_CAPACITY)) diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 577b32136..16ccb22b2 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -34,8 +34,8 @@ def login(self): data = {'username': self.rest_username, 'password': cryptor.decode( self.rest_password)} self.init_http_head() - token_res = self.get_token(RestHandler.REST_AUTH_URL, data, - method='POST') + token_res = self.do_call(RestHandler.REST_AUTH_URL, data, + method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE: LOG.error("Login error, Obtaining the token is abnormal. " "status_code:%s, URL: %s", @@ -50,8 +50,8 @@ def login(self): RestHandler.REST_AUTH_URL) raise exception.InvalidResults('the api token does not ' 'exist') - session_res = self.get_token(RestHandler.REST_SESSION_URL, - token_res.json(), method='POST') + session_res = self.do_call(RestHandler.REST_SESSION_URL, + token_res.json(), method='POST') if session_res.status_code == consts.SUCCESS_STATUS_CODE: username = session_res.json().get('username') if not username: @@ -71,9 +71,12 @@ def login(self): except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e + finally: + data = None + token_res = None def logout(self): - res = self.get_token(RestHandler.REST_SESSION_URL, method='DELETE') + res = self.do_call(RestHandler.REST_SESSION_URL, method='DELETE') if res.status_code == consts.SUCCESS_STATUS_CODE: username = res.json().get('username') if not username: @@ -95,10 +98,6 @@ def rest_call(self, url, data=None, method='GET'): self.rest_call(url, data, method) return result_json - def get_token(self, url, data=None, method='GET'): - res = self.do_call(url, data, method) - return res - def get_volumes(self, url=REST_VOLUME_URL, data=None, volume_list=None, count=consts.DEFAULT_COUNT_GET_VOLUMES_INFO): if volume_list is None: From d868ac0b68fe9386b7a686cf6162c2457d01723a Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 16 Nov 2021 18:49:52 +0800 Subject: [PATCH 34/69] pure codeview-list_port add hardware --- delfin/drivers/pure/flasharray/consts.py | 4 +- .../pure/flasharray/pure_flasharray.py | 100 +++++++++++------- .../pure/flasharray/test_pure_flasharray.py | 16 +-- 3 files changed, 73 insertions(+), 47 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index cdd35be2a..51c068a66 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -42,8 +42,8 @@ # Custom token of Pure CUSTOM_TOKEN = 'x-next-token' -# The mode value of the controller -CONTROLLER_PRIMARY = 'primary' +# The default get_storage model +DEFAULT_GET_STORAGE_MODEL = 'FlashArray' # list_port: Add ":" to the WWN every 2 sequences. SPLICE_WWN_SERIAL = 2 diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 4cfc67644..885f12a2d 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -45,7 +45,9 @@ def list_volumes(self, context): 'free_capacity': total_capacity - used_capacity, 'storage_id': self.storage_id, 'status': constants.StorageStatus.NORMAL, - 'type': constants.VolumeType.THICK, + 'type': constants.VolumeType.THIN if + volume.get('thin_provisioning') + else constants.VolumeType.THICK, 'native_storage_pool_id': native_storage_pool_id } list_volumes.append(volume_dict) @@ -79,17 +81,8 @@ def get_storage(self, context): serial_number = arrays.get('id') version = arrays.get('version') - model = None - controllers = self.rest_handler.rest_call( - self.rest_handler.REST_CONTROLLERS_URL) - if controllers: - for controller in controllers: - if controller.get('mode') == consts.CONTROLLER_PRIMARY: - model = controller.get('model') - break - storage_result = { - 'model': model, + 'model': consts.DEFAULT_GET_STORAGE_MODEL, 'total_capacity': total_capacity, 'raw_capacity': total_capacity, 'used_capacity': used_capacity, @@ -197,40 +190,73 @@ def get_hardware(self): return hardware_dict def list_ports(self, context): - networks = self.get_network() - hardware_dict = self.get_hardware() list_ports = [] + networks = self.get_network() + ports = self.get_ports() + hardware_dict = self.rest_handler.rest_call( + self.rest_handler.REST_HARDWARE_URL) + if not hardware_dict: + return list_ports + for hardware in hardware_dict: + hardware_result = dict() + hardware_name = hardware.get('name') + if 'FC' not in hardware_name and 'ETH' not in hardware_name and\ + 'SAS' not in hardware_name: + continue + hardware_result['name'] = hardware_name + hardware_result['native_port_id'] = hardware_name + hardware_result['storage_id'] = self.storage_id + hardware_result['location'] = hardware_name + speed = hardware.get('speed') + if speed is None: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.UNKNOWN + hardware_result['health_status'] = constants.PortHealthStatus.\ + UNKNOWN + elif speed == consts.CONSTANT_ZERO: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.DISCONNECTED + hardware_result['health_status'] = constants.PortHealthStatus.\ + ABNORMAL + hardware_result['speed'] = speed + else: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.CONNECTED + hardware_result['health_status'] = constants.PortHealthStatus.\ + NORMAL + hardware_result['speed'] = int(speed) + + port = ports.get(hardware_name) + if port: + hardware_result['type'] = port.get('type') + hardware_result['wwn'] = port.get('wwn') + else: + hardware_result['type'] = constants.PortType.SAS + + network = networks.get(hardware_name) + if network: + hardware_result['address'] = network.get('address') + hardware_result['logical_type'] = network.get('logical_type') + hardware_result['ipv4_mask'] = network.get('ipv4_mask') + list_ports.append(hardware_result) + return list_ports + + def get_ports(self): + ports_dict = dict() ports = self.rest_handler.rest_call(self.rest_handler.REST_PORT_URL) if ports: for port in ports: - port_result = dict() + port_dict = dict() port_name = port.get('name') wwn = port.get('wwn') if wwn: - port_result['type'] = constants.PortType.FC + port_dict['type'] = constants.PortType.FC + wwn_splice = self.get_splice_wwn(wwn) + port_dict['wwn'] = wwn_splice else: - port_result['type'] = constants.PortType.ETH - port_result['name'] = port_name - port_result['native_port_id'] = port_name - port_result['location'] = port_name - port_result['storage_id'] = self.storage_id - network = networks.get(port_name, {}) - port_result['logical_type'] = network.get('logical_type') - port_result['mac_address'] = network.get('address') - port_result['ipv4_mask'] = network.get('ipv4_mask') - - hardware = hardware_dict.get(port_name, {}) - speed = hardware.get('speed') - port_result['speed'] = int(speed)\ - if speed is not None else None - wwn_splice = self.get_splice_wwn(wwn) - port_result['wwn'] = wwn_splice - port_result['connection_status '] = constants. \ - PortConnectionStatus.CONNECTED - port_result['health_status'] = constants.PortHealthStatus. \ - NORMAL - list_ports.append(port_result) - return list_ports + port_dict['type'] = constants.PortType.ETH + ports_dict[port_name] = port_dict + return ports_dict @staticmethod def get_splice_wwn(wwn): diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index d505ef88d..2d49d0eff 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -151,7 +151,7 @@ "details": "", "identify": "off", "index": 0, - "name": "CH0.BAY1", + "name": "CTO.FC1", "slot": "", "speed": 0, "status": "ok", @@ -161,7 +161,7 @@ "details": "", "identify": "", "index": 0, - "name": "CH0.BAY2", + "name": "CTO.ETH15", "slot": 0, "speed": 1000000, "status": "ok", @@ -202,7 +202,7 @@ ] port_info = [ { - "name": "CTO.ETH14", + "name": "CTO.FC1", "failover": "", "iqn": "iqn.2016-11-01.com.pure", "portal": "100.12.253.23:4563", @@ -214,13 +214,13 @@ "failover": "", "iqn": "iqn.2016-11-01.com.pure", "portal": "100.12.253.23:4563", - "wwn": "43ddff45gdcvrty", + "wwn": "", "nqn": "" } ] port_network_info = [ { - "name": "CTO.ETH14", + "name": "CTO.FC1", "address": "45233662jksndj", "speed": 12000, "netmask": "100.12.253.23:4563", @@ -235,7 +235,7 @@ "address": "45233662jksndj", "speed": 13000, "netmask": "100.12.253.23:4563", - "wwn": "43ddff45ggg4rty", + "wwn": "", "nqn": "", "services": [ "management" @@ -323,9 +323,9 @@ def test_list_disks(self): def test_list_ports(self): RestHandler.rest_call = mock.Mock( - side_effect=[port_network_info, port_info]) + side_effect=[port_network_info, port_info, hardware_info]) list_ports = self.driver.list_ports(context) - self.assertEqual(list_ports[0].get('wwn'), port_info[0].get('wwn')) + self.assertEqual(list_ports[0].get('name'), port_info[0].get('name')) def test_list_storage_pools(self): RestHandler.rest_call = mock.Mock(side_effect=[pools_info]) From b39524d8b215096ef48b6fc71587827eadc78102 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 16 Nov 2021 19:37:03 +0800 Subject: [PATCH 35/69] pure codeview-list_port type --- .../pure/flasharray/pure_flasharray.py | 28 +++++++++---------- .../drivers/pure/flasharray/rest_handler.py | 8 +++--- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 885f12a2d..01e9d438b 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -46,7 +46,7 @@ def list_volumes(self, context): 'storage_id': self.storage_id, 'status': constants.StorageStatus.NORMAL, 'type': constants.VolumeType.THIN if - volume.get('thin_provisioning') + volume.get('thin_provisioning') is not None else constants.VolumeType.THICK, 'native_storage_pool_id': native_storage_pool_id } @@ -181,12 +181,11 @@ def get_hardware(self): self.rest_handler.REST_HARDWARE_URL) if hardware: for hardware_value in hardware: - hardware_name = dict() - hardware_name['speed'] = hardware_value.get('speed') - hardware_name['serial_number'] = hardware_value.get('serial') - hardware_name['model'] = hardware_value.get('model') - hardware_value_name = hardware_value.get('name') - hardware_dict[hardware_value_name] = hardware_name + hardware_Map = dict() + hardware_Map['speed'] = hardware_value.get('speed') + hardware_Map['serial_number'] = hardware_value.get('serial') + hardware_Map['model'] = hardware_value.get('model') + hardware_dict[hardware_value.get('name')] = hardware_Map return hardware_dict def list_ports(self, context): @@ -200,8 +199,13 @@ def list_ports(self, context): for hardware in hardware_dict: hardware_result = dict() hardware_name = hardware.get('name') - if 'FC' not in hardware_name and 'ETH' not in hardware_name and\ - 'SAS' not in hardware_name: + if 'FC' in hardware_name: + hardware_result['type'] = constants.PortType.FC + elif 'ETH' in hardware_name: + hardware_result['type'] = constants.PortType.ETH + elif 'SAS' in hardware_name: + hardware_result['type'] = constants.PortType.SAS + else: continue hardware_result['name'] = hardware_name hardware_result['native_port_id'] = hardware_name @@ -228,10 +232,7 @@ def list_ports(self, context): port = ports.get(hardware_name) if port: - hardware_result['type'] = port.get('type') hardware_result['wwn'] = port.get('wwn') - else: - hardware_result['type'] = constants.PortType.SAS network = networks.get(hardware_name) if network: @@ -250,11 +251,8 @@ def get_ports(self): port_name = port.get('name') wwn = port.get('wwn') if wwn: - port_dict['type'] = constants.PortType.FC wwn_splice = self.get_splice_wwn(wwn) port_dict['wwn'] = wwn_splice - else: - port_dict['type'] = constants.PortType.ETH ports_dict[port_name] = port_dict return ports_dict diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 16ccb22b2..97a3bdb3d 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -48,8 +48,8 @@ def login(self): LOG.error("Login error, the api token does not exist. " "status_code:%s, URL: %s", token_res.status_code, RestHandler.REST_AUTH_URL) - raise exception.InvalidResults('the api token does not ' - 'exist') + raise exception.StorageBackendException('the api token ' + 'does not exist') session_res = self.do_call(RestHandler.REST_SESSION_URL, token_res.json(), method='POST') if session_res.status_code == consts.SUCCESS_STATUS_CODE: @@ -59,8 +59,8 @@ def login(self): ". status_code:%s, URL: %s", session_res.status_code, RestHandler.REST_SESSION_URL) - raise exception.InvalidResults('the api session does ' - 'not exist') + raise exception.StorageBackendException( + 'the api session does not exist') else: LOG.error("Login error, obtaining the session is abnormal." " status_code:%s, URL: %s", From 5fc61669aa0eeea5f6785dca4345c95b8e6f912b Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 10:11:29 +0800 Subject: [PATCH 36/69] pure codeview-get_ports func iqn --- delfin/drivers/pure/flasharray/pure_flasharray.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 01e9d438b..9828cf71f 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -250,9 +250,8 @@ def get_ports(self): port_dict = dict() port_name = port.get('name') wwn = port.get('wwn') - if wwn: - wwn_splice = self.get_splice_wwn(wwn) - port_dict['wwn'] = wwn_splice + port_dict['wwn'] = self.get_splice_wwn(wwn)\ + if wwn is not None else port.get('iqn') ports_dict[port_name] = port_dict return ports_dict From 8cd821627d23767833dec35deb4deffba2d5d7cb Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 10:35:02 +0800 Subject: [PATCH 37/69] pure codeview-login,logout --- .../pure/flasharray/pure_flasharray.py | 10 ++-- .../drivers/pure/flasharray/rest_handler.py | 48 ++++++------------- 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 9828cf71f..7e6ddd2d7 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -181,11 +181,11 @@ def get_hardware(self): self.rest_handler.REST_HARDWARE_URL) if hardware: for hardware_value in hardware: - hardware_Map = dict() - hardware_Map['speed'] = hardware_value.get('speed') - hardware_Map['serial_number'] = hardware_value.get('serial') - hardware_Map['model'] = hardware_value.get('model') - hardware_dict[hardware_value.get('name')] = hardware_Map + hardware_map = dict() + hardware_map['speed'] = hardware_value.get('speed') + hardware_map['serial_number'] = hardware_value.get('serial') + hardware_map['model'] = hardware_value.get('model') + hardware_dict[hardware_value.get('name')] = hardware_map return hardware_dict def list_ports(self, context): diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 97a3bdb3d..cb03c2090 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -36,38 +36,24 @@ def login(self): self.init_http_head() token_res = self.do_call(RestHandler.REST_AUTH_URL, data, method='POST') - if token_res.status_code != consts.SUCCESS_STATUS_CODE: + if token_res.status_code != consts.SUCCESS_STATUS_CODE or not \ + token_res.json().get('api_token'): LOG.error("Login error, Obtaining the token is abnormal. " "status_code:%s, URL: %s", token_res.status_code, RestHandler.REST_AUTH_URL) raise exception.StorageBackendException( 'Obtaining the token is abnormal') - else: - api_token = token_res.json().get('api_token') - if not api_token: - LOG.error("Login error, the api token does not exist. " - "status_code:%s, URL: %s", token_res.status_code, - RestHandler.REST_AUTH_URL) - raise exception.StorageBackendException('the api token ' - 'does not exist') - session_res = self.do_call(RestHandler.REST_SESSION_URL, - token_res.json(), method='POST') - if session_res.status_code == consts.SUCCESS_STATUS_CODE: - username = session_res.json().get('username') - if not username: - LOG.error("Login error, the api session does not exist" - ". status_code:%s, URL: %s", - session_res.status_code, - RestHandler.REST_SESSION_URL) - raise exception.StorageBackendException( - 'the api session does not exist') - else: - LOG.error("Login error, obtaining the session is abnormal." - " status_code:%s, URL: %s", - session_res.status_code, - RestHandler.REST_AUTH_URL) - raise exception.StorageBackendException( - 'obtaining the session is abnormal') + session_res = self.do_call(RestHandler.REST_SESSION_URL, + token_res.json(), method='POST') + + if session_res.status_code != consts.SUCCESS_STATUS_CODE or not \ + session_res.json().get('username'): + LOG.error("Login error, Obtaining the session is abnormal." + "status_code:%s, URL: %s", + session_res.status_code, + RestHandler.REST_SESSION_URL) + raise exception.StorageBackendException( + 'Obtaining the session is abnormal.') except Exception as e: LOG.error("Login error: %s", six.text_type(e)) raise e @@ -77,12 +63,8 @@ def login(self): def logout(self): res = self.do_call(RestHandler.REST_SESSION_URL, method='DELETE') - if res.status_code == consts.SUCCESS_STATUS_CODE: - username = res.json().get('username') - if not username: - raise exception.InvalidResults('The returned username' - ' is empty') - else: + if res.status_code != consts.SUCCESS_STATUS_CODE\ + or not res.json().get('username'): LOG.error("Logout error, Deleting a Token Exception." "status_code:%s, URL: %s", res.status_code, RestHandler.REST_SESSION_URL) From ae66b86876fee598b6d47eb3e2ae2f6d9f0a0893 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 10:46:25 +0800 Subject: [PATCH 38/69] pure codeview-format --- delfin/drivers/pure/flasharray/pure_flasharray.py | 1 - delfin/drivers/pure/flasharray/rest_handler.py | 1 - 2 files changed, 2 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 7e6ddd2d7..39297d1b7 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -2,7 +2,6 @@ import hashlib from oslo_log import log -from oslo_utils import units from delfin.common import constants from delfin.drivers import driver diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index cb03c2090..9cd9f8b98 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -45,7 +45,6 @@ def login(self): 'Obtaining the token is abnormal') session_res = self.do_call(RestHandler.REST_SESSION_URL, token_res.json(), method='POST') - if session_res.status_code != consts.SUCCESS_STATUS_CODE or not \ session_res.json().get('username'): LOG.error("Login error, Obtaining the session is abnormal." From 6ce479bc2743e1e1e5c1eab4fc90288b22d638c3 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 11:30:26 +0800 Subject: [PATCH 39/69] pure codeview-get_storage model and Delete list_storage_pools --- delfin/drivers/pure/flasharray/consts.py | 5 +- .../pure/flasharray/pure_flasharray.py | 47 ++++++------------- .../drivers/pure/flasharray/rest_handler.py | 5 +- 3 files changed, 19 insertions(+), 38 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 51c068a66..4dd95333f 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -43,7 +43,10 @@ CUSTOM_TOKEN = 'x-next-token' # The default get_storage model -DEFAULT_GET_STORAGE_MODEL = 'FlashArray' +CONTROLLER_PRIMARY = 'primary' + +# Normal value of the controller status +NORMAL_CONTROLLER_STATUS = 'ready' # list_port: Add ":" to the WWN every 2 sequences. SPLICE_WWN_SERIAL = 2 diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 39297d1b7..e7abf0cc7 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -21,21 +21,12 @@ def list_volumes(self, context): list_volumes = [] volumes = self.rest_handler.get_volumes() if volumes: - pools = self.rest_handler.rest_call( - self.rest_handler.REST_POOLS_URL) for volume in volumes: volume_name = volume.get('name') total_capacity = int(volume.get('size', consts.DEFAULT_CAPACITY)) used_capacity = int(volume.get('volumes', consts.DEFAULT_CAPACITY)) - native_storage_pool_id = None - if pools: - for pool in pools: - pool_volumes = pool.get('volumes') - if volume_name in pool_volumes: - native_storage_pool_id = pool.get('name') - break volume_dict = { 'native_volume_id': volume_name, 'name': volume_name, @@ -47,7 +38,7 @@ def list_volumes(self, context): 'type': constants.VolumeType.THIN if volume.get('thin_provisioning') is not None else constants.VolumeType.THICK, - 'native_storage_pool_id': native_storage_pool_id + 'native_storage_pool_id': '' } list_volumes.append(volume_dict) return list_volumes @@ -80,8 +71,18 @@ def get_storage(self, context): serial_number = arrays.get('id') version = arrays.get('version') + model = None + status = constants.StorageStatus.NORMAL + controllers = self.rest_handler.rest_call( + self.rest_handler.REST_CONTROLLERS_URL) + if controllers: + for controller in controllers: + if controller.get('mode') == consts.CONTROLLER_PRIMARY: + model = controller.get('model') + if controller.get('status') != consts.NORMAL_CONTROLLER_STATUS: + status = constants.StorageStatus.ABNORMAL storage_result = { - 'model': consts.DEFAULT_GET_STORAGE_MODEL, + 'model': model, 'total_capacity': total_capacity, 'raw_capacity': total_capacity, 'used_capacity': used_capacity, @@ -90,7 +91,7 @@ def get_storage(self, context): 'name': storage_name, 'serial_number': serial_number, 'firmware_version': version, - 'status': constants.StorageStatus.NORMAL + 'status': status } return storage_result @@ -284,27 +285,7 @@ def get_network(self): return networks_object def list_storage_pools(self, context): - pool_list = [] - pools = self.rest_handler.rest_call( - self.rest_handler.REST_POOLS_CAPACITY_URL) - if pools: - for pool in pools: - pool_result = dict() - total_capacity = int(pool.get('size', consts.DEFAULT_CAPACITY)) - if total_capacity == consts.DEFAULT_CAPACITY: - continue - pool_result['total_capacity'] = total_capacity - used_capacity = int(pool.get('total_reduction', - consts.DEFAULT_CAPACITY)) - pool_result['used_capacity'] = used_capacity - pool_result['free_capacity'] = total_capacity - used_capacity - pool_result['name'] = pool.get('name') - pool_result['native_storage_pool_id'] = pool.get('name') - pool_result['storage_id'] = self.storage_id - pool_result['status'] = constants.StoragePoolStatus.NORMAL - pool_result['storage_type'] = constants.StorageType.BLOCK - pool_list.append(pool_result) - return pool_list + return [] def remove_trap_config(self, context, trap_config): pass diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 9cd9f8b98..c9b41a0f3 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -14,9 +14,6 @@ class RestHandler(RestClient): REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=20&token=aWQgPSA5OD' \ 'A1Mg==' REST_VOLUME_TOKEN_URL = '/api/1.17/volume?space=true&limit=20&token=' - REST_VOLUME_ID_URL = '/api/1.17/volume/' - REST_POOLS_URL = '/api/1.17/vgroup' - REST_POOLS_CAPACITY_URL = '/api/1.17/vgroup?space=true' REST_PORT_URL = '/api/1.17/port' REST_NETWORK_URL = '/api/1.17/network' REST_DISK_URL = '/api/1.17/drive' @@ -61,7 +58,7 @@ def login(self): token_res = None def logout(self): - res = self.do_call(RestHandler.REST_SESSION_URL, method='DELETE') + res = self.do_call(RestHandler.REST_SESSION_URL, None, method='DELETE') if res.status_code != consts.SUCCESS_STATUS_CODE\ or not res.json().get('username'): LOG.error("Logout error, Deleting a Token Exception." From 6cc51f21e86cda851a93035d2338ae343102f6cf Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 11:40:06 +0800 Subject: [PATCH 40/69] pure codeview-list_volumes Delete native_storage_pool_id --- delfin/drivers/pure/flasharray/pure_flasharray.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index e7abf0cc7..dda1c6e76 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -37,8 +37,7 @@ def list_volumes(self, context): 'status': constants.StorageStatus.NORMAL, 'type': constants.VolumeType.THIN if volume.get('thin_provisioning') is not None - else constants.VolumeType.THICK, - 'native_storage_pool_id': '' + else constants.VolumeType.THICK } list_volumes.append(volume_dict) return list_volumes From 08c873b666c47482ca776e18d3cf6a81b863e091 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 14:31:34 +0800 Subject: [PATCH 41/69] pure codeview-get_storage Not empty processing --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 ++++ delfin/drivers/pure/flasharray/rest_handler.py | 2 +- .../unit/drivers/pure/flasharray/test_pure_flasharray.py | 6 +----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index dda1c6e76..01a84ee24 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -3,6 +3,7 @@ from oslo_log import log +from delfin import exception from delfin.common import constants from delfin.drivers import driver from delfin.drivers.pure.flasharray import rest_handler, consts @@ -80,6 +81,9 @@ def get_storage(self, context): model = controller.get('model') if controller.get('status') != consts.NORMAL_CONTROLLER_STATUS: status = constants.StorageStatus.ABNORMAL + if storages is None and arrays is None and controllers is None: + LOG.error('get_storage error, Unable to obtain data.') + raise exception.StorageBackendException('Unable to obtain data') storage_result = { 'model': model, 'total_capacity': total_capacity, diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index c9b41a0f3..44998acf5 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -73,7 +73,7 @@ def rest_call(self, url, data=None, method='GET'): result_json = res.json() elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() - self.rest_call(url, data, method) + self.do_call(url, data, method) return result_json def get_volumes(self, url=REST_VOLUME_URL, data=None, volume_list=None, diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index 2d49d0eff..0b9dc0a17 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -287,8 +287,6 @@ def test_init(self): def test_list_volumes(self): RestHandler.get_volumes = mock.Mock( side_effect=[volumes_info]) - RestHandler.rest_call = mock.Mock( - side_effect=[pool_info]) volume = self.driver.list_volumes(context) self.assertEqual(volume[0]['native_volume_id'], pool_info[0].get('volumes')[0]) @@ -328,10 +326,8 @@ def test_list_ports(self): self.assertEqual(list_ports[0].get('name'), port_info[0].get('name')) def test_list_storage_pools(self): - RestHandler.rest_call = mock.Mock(side_effect=[pools_info]) list_storage_pools = self.driver.list_storage_pools(context) - self.assertEqual(list_storage_pools[0].get('native_storage_pool_id'), - pools_info[0].get('name')) + self.assertEqual(list_storage_pools, []) def test_reset_connection(self): RestHandler.logout = mock.Mock(side_effect=None) From 0628dbb795e9bf35ca8ed6d43e045575f3d2dbc2 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 14:55:33 +0800 Subject: [PATCH 42/69] pure codeview-rest_call --- delfin/drivers/pure/flasharray/pure_flasharray.py | 2 +- delfin/drivers/pure/flasharray/rest_handler.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 01a84ee24..55b0a22da 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -81,7 +81,7 @@ def get_storage(self, context): model = controller.get('model') if controller.get('status') != consts.NORMAL_CONTROLLER_STATUS: status = constants.StorageStatus.ABNORMAL - if storages is None and arrays is None and controllers is None: + if not all((storages, arrays, controllers)): LOG.error('get_storage error, Unable to obtain data.') raise exception.StorageBackendException('Unable to obtain data') storage_result = { diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 44998acf5..3b066a4c6 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -11,8 +11,8 @@ class RestHandler(RestClient): REST_STORAGE_URL = '/api/1.17/array?space=true' REST_ARRAY_URL = '/api/1.17/array' - REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=20&token=aWQgPSA5OD' \ - 'A1Mg==' + REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=500&token=' \ + 'aWQgPSA5ODA1Mg==' REST_VOLUME_TOKEN_URL = '/api/1.17/volume?space=true&limit=20&token=' REST_PORT_URL = '/api/1.17/port' REST_NETWORK_URL = '/api/1.17/network' @@ -73,7 +73,9 @@ def rest_call(self, url, data=None, method='GET'): result_json = res.json() elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: self.login() - self.do_call(url, data, method) + the_second_time_res = self.do_call(url, data, method) + if the_second_time_res.status_code == consts.SUCCESS_STATUS_CODE: + result_json = the_second_time_res.json() return result_json def get_volumes(self, url=REST_VOLUME_URL, data=None, volume_list=None, From 61347dd0dcb81ec2cb94d12116f0dc45af848885 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 16:04:04 +0800 Subject: [PATCH 43/69] pure codeview-test --- .../pure/flasharray/test_pure_flasharray.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index 0b9dc0a17..74fe4c8a4 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -214,8 +214,8 @@ "failover": "", "iqn": "iqn.2016-11-01.com.pure", "portal": "100.12.253.23:4563", - "wwn": "", - "nqn": "" + "wwn": None, + "nqn": None } ] port_network_info = [ @@ -225,7 +225,7 @@ "speed": 12000, "netmask": "100.12.253.23:4563", "wwn": "43ddff45ggg4rty", - "nqn": "", + "nqn": None, "services": [ "management" ] @@ -235,8 +235,8 @@ "address": "45233662jksndj", "speed": 13000, "netmask": "100.12.253.23:4563", - "wwn": "", - "nqn": "", + "wwn": None, + "nqn": None, "services": [ "management" ] @@ -317,13 +317,13 @@ def test_list_disks(self): side_effect=[hardware_info, drive_info]) list_disks = self.driver.list_disks(context) self.assertEqual(list_disks[0].get('name'), - hardware_info[0].get('name')) + drive_info[0].get('name')) def test_list_ports(self): RestHandler.rest_call = mock.Mock( side_effect=[port_network_info, port_info, hardware_info]) list_ports = self.driver.list_ports(context) - self.assertEqual(list_ports[0].get('name'), port_info[0].get('name')) + self.assertEqual(list_ports[0].get('name'), hardware_info[0].get('name')) def test_list_storage_pools(self): list_storage_pools = self.driver.list_storage_pools(context) From 5efe895de76888fd76814729e62190886e1b0a45 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 16:32:37 +0800 Subject: [PATCH 44/69] pure codeview-port mac_address --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 55b0a22da..3ba6d146c 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -239,7 +239,7 @@ def list_ports(self, context): network = networks.get(hardware_name) if network: - hardware_result['address'] = network.get('address') + hardware_result['mac_address'] = network.get('mac_address') hardware_result['logical_type'] = network.get('logical_type') hardware_result['ipv4_mask'] = network.get('ipv4_mask') list_ports.append(hardware_result) @@ -275,7 +275,7 @@ def get_network(self): if networks: for network in networks: network_dict = dict() - network_dict['address'] = network.get('address') + network_dict['mac_address'] = network.get('address') services_list = network.get('services') if services_list: for services in services_list: From d8d4c66fd6b01692f4653959249c0d436ae1fc1b Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 16:55:52 +0800 Subject: [PATCH 45/69] pure codeview-port mac_address ipv4 --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 3ba6d146c..48857365c 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -242,6 +242,7 @@ def list_ports(self, context): hardware_result['mac_address'] = network.get('mac_address') hardware_result['logical_type'] = network.get('logical_type') hardware_result['ipv4_mask'] = network.get('ipv4_mask') + hardware_result['ipv4'] = network.get('ipv4') list_ports.append(hardware_result) return list_ports @@ -275,7 +276,7 @@ def get_network(self): if networks: for network in networks: network_dict = dict() - network_dict['mac_address'] = network.get('address') + network_dict['mac_address'] = network.get('hwaddr') services_list = network.get('services') if services_list: for services in services_list: @@ -283,6 +284,7 @@ def get_network(self): services in constants.PortLogicalType.ALL else None break network_dict['ipv4_mask'] = network.get('netmask') + network_dict['ipv4'] = network.get('address') network_name = network.get('name').upper() networks_object[network_name] = network_dict return networks_object From f94bb22cd88eb2978d69d080c2a386bb6af10851 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 17 Nov 2021 18:43:45 +0800 Subject: [PATCH 46/69] pure codeview-list_alerts --- .../drivers/pure/flasharray/pure_flasharray.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 48857365c..f1516a62f 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -108,23 +108,27 @@ def list_alerts(self, context, query_para=None): alerts_model['severity'] = consts.SEVERITY_MAP.get( alert.get('current_severity'), constants.Severity.NOT_SPECIFIED) - alerts_model['category'] = consts.CATEGORY_MAP.get( - alert.get('category'), constants.Category.NOT_SPECIFIED) + alerts_model['category'] = constants.Category.FAULT time = alert.get('opened') alerts_model['occur_time'] = int(datetime.datetime.strptime( time, '%Y-%m-%dT%H:%M:%SZ').timestamp() * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ if time is not None else None - alerts_model['description'] = alert.get('event') + alerts_model['description'] = alert.get('details') alerts_model['location'] = alert.get('component_name') alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE - alerts_model['alert_name'] = alert.get('id') + alerts_model['alert_name'] = alert.get('event') + alerts_model['sequence_number'] = alert.get('id') alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). encode()).hexdigest() alerts_list.append(alerts_model) return alerts_list + @staticmethod + def parse_alert(context, alert): + return {} + def list_controllers(self, context): list_controllers = [] controllers = self.rest_handler.rest_call( @@ -298,3 +302,7 @@ def remove_trap_config(self, context, trap_config): def reset_connection(self, context, **kwargs): self.rest_handler.logout() self.rest_handler.login() + + @staticmethod + def get_access_url(): + return 'https://{ip}' From 11e98e564639054b234b1a3539e5da4bea63f205 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 18 Nov 2021 11:01:27 +0800 Subject: [PATCH 47/69] pure codeview-add parse_alert test --- .../pure/flasharray/pure_flasharray.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index f1516a62f..80ed1f8eb 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -1,5 +1,6 @@ import datetime import hashlib +import json from oslo_log import log @@ -127,7 +128,27 @@ def list_alerts(self, context, query_para=None): @staticmethod def parse_alert(context, alert): - return {} + LOG.info("list_disk：%s" % (json.dumps(alert, ensure_ascii=False))) + alert_model = dict() + alert_model['alert_id'] = alert.get('id') + alert_model['severity'] = consts.SEVERITY_MAP.get( + alert.get('current_severity'), + constants.Severity.NOT_SPECIFIED) + alert_model['category'] = constants.Category.FAULT + time = alert.get('opened') + alert_model['occur_time'] = int(datetime.datetime.strptime( + time, '%Y-%m-%dT%H:%M:%SZ').timestamp() + * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ + if time is not None else None + alert_model['description'] = alert.get('details') + alert_model['location'] = alert.get('component_name') + alert_model['type'] = constants.EventType.EQUIPMENT_ALARM + alert_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE + alert_model['alert_name'] = alert.get('event') + alert_model['sequence_number'] = alert.get('id') + alert_model['match_key'] = hashlib.md5(str(alert.get('id')). + encode()).hexdigest() + return alert_model def list_controllers(self, context): list_controllers = [] @@ -222,18 +243,18 @@ def list_ports(self, context): if speed is None: hardware_result['connection_status'] = \ constants.PortConnectionStatus.UNKNOWN - hardware_result['health_status'] = constants.PortHealthStatus.\ + hardware_result['health_status'] = constants.PortHealthStatus. \ UNKNOWN elif speed == consts.CONSTANT_ZERO: hardware_result['connection_status'] = \ constants.PortConnectionStatus.DISCONNECTED - hardware_result['health_status'] = constants.PortHealthStatus.\ + hardware_result['health_status'] = constants.PortHealthStatus. \ ABNORMAL hardware_result['speed'] = speed else: hardware_result['connection_status'] = \ constants.PortConnectionStatus.CONNECTED - hardware_result['health_status'] = constants.PortHealthStatus.\ + hardware_result['health_status'] = constants.PortHealthStatus. \ NORMAL hardware_result['speed'] = int(speed) @@ -258,7 +279,7 @@ def get_ports(self): port_dict = dict() port_name = port.get('name') wwn = port.get('wwn') - port_dict['wwn'] = self.get_splice_wwn(wwn)\ + port_dict['wwn'] = self.get_splice_wwn(wwn) \ if wwn is not None else port.get('iqn') ports_dict[port_name] = port_dict return ports_dict From 393fdb7a2bced016a6373c42011db2b0140e870d Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 18 Nov 2021 11:27:03 +0800 Subject: [PATCH 48/69] pure codeview-add list_quotas list_shares list_filesystems --- delfin/drivers/pure/flasharray/pure_flasharray.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 80ed1f8eb..cd93d0883 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -327,3 +327,12 @@ def reset_connection(self, context, **kwargs): @staticmethod def get_access_url(): return 'https://{ip}' + + def list_quotas(self, context): + return [] + + def list_shares(self, context): + return [] + + def list_filesystems(self, context): + return [] From cd39e881bebf22fcea304e2b42b5d19e5f8e0124 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 18 Nov 2021 14:20:39 +0800 Subject: [PATCH 49/69] pure codeview-delete list_quotas list_shares list_filesystems --- delfin/drivers/pure/flasharray/pure_flasharray.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index cd93d0883..b475cd6ad 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -128,7 +128,7 @@ def list_alerts(self, context, query_para=None): @staticmethod def parse_alert(context, alert): - LOG.info("list_disk：%s" % (json.dumps(alert, ensure_ascii=False))) + LOG.info("parse_alert：%s" % (json.dumps(alert, ensure_ascii=False))) alert_model = dict() alert_model['alert_id'] = alert.get('id') alert_model['severity'] = consts.SEVERITY_MAP.get( @@ -328,11 +328,3 @@ def reset_connection(self, context, **kwargs): def get_access_url(): return 'https://{ip}' - def list_quotas(self, context): - return [] - - def list_shares(self, context): - return [] - - def list_filesystems(self, context): - return [] From d59ad99907bf2dcd1d640a18731a97b7e0879dfe Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 18 Nov 2021 18:40:35 +0800 Subject: [PATCH 50/69] pure codeview-revamp parse_alert --- delfin/drivers/pure/flasharray/consts.py | 16 ++++++++++++ .../pure/flasharray/pure_flasharray.py | 26 ++++++++----------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 4dd95333f..a7497f33c 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -75,3 +75,19 @@ 'abnormal': constants.DiskStatus.ABNORMAL, 'unhealthy': constants.DiskStatus.ABNORMAL, 'offline': constants.DiskStatus.OFFLINE} + + +PARSE_ALERT_ALERT_ID = '1.3.6.1.2.1.1.3.0' +PARSE_ALERT_STORAGE_NAME = '1.3.6.1.4.1.40482.3.1' +PARSE_ALERT_CONTROLLER_NAME = '1.3.6.1.4.1.40482.3.3' +PARSE_ALERT_ALERT_NAME = '1.3.6.1.4.1.40482.3.5' +PARSE_ALERT_DESCRIPTION = '1.3.6.1.4.1.40482.3.6' +PARSE_ALERT_SEVERITY = '1.3.6.1.4.1.40482.3.7' + +PARSE_ALERT_SEVERITY_MAP = {'0': constants.Severity.FATAL, + '1': constants.Severity.CRITICAL, + '2': constants.Severity.MAJOR, + '3': constants.Severity.MINOR, + '4': constants.Severity.WARNING, + '5': constants.Severity.INFORMATIONAL, + '6': constants.Severity.NOT_SPECIFIED} diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index b475cd6ad..3e6a4ab1a 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -4,7 +4,7 @@ from oslo_log import log -from delfin import exception +from delfin import exception, utils from delfin.common import constants from delfin.drivers import driver from delfin.drivers.pure.flasharray import rest_handler, consts @@ -130,24 +130,20 @@ def list_alerts(self, context, query_para=None): def parse_alert(context, alert): LOG.info("parse_alert：%s" % (json.dumps(alert, ensure_ascii=False))) alert_model = dict() - alert_model['alert_id'] = alert.get('id') - alert_model['severity'] = consts.SEVERITY_MAP.get( - alert.get('current_severity'), + alert_model['alert_id'] = alert.get(consts.PARSE_ALERT_ALERT_ID) + alert_model['severity'] = consts.PARSE_ALERT_SEVERITY_MAP.get( + alert.get(consts.PARSE_ALERT_SEVERITY), constants.Severity.NOT_SPECIFIED) alert_model['category'] = constants.Category.FAULT - time = alert.get('opened') - alert_model['occur_time'] = int(datetime.datetime.strptime( - time, '%Y-%m-%dT%H:%M:%SZ').timestamp() - * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ - if time is not None else None - alert_model['description'] = alert.get('details') - alert_model['location'] = alert.get('component_name') + alert_model['occur_time'] = utils.utcnow_ms() + alert_model['description'] = alert.get(consts.PARSE_ALERT_DESCRIPTION) + alert_model['location'] = alert.get(consts.PARSE_ALERT_CONTROLLER_NAME) alert_model['type'] = constants.EventType.EQUIPMENT_ALARM alert_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE - alert_model['alert_name'] = alert.get('event') - alert_model['sequence_number'] = alert.get('id') - alert_model['match_key'] = hashlib.md5(str(alert.get('id')). - encode()).hexdigest() + alert_model['alert_name'] = alert.get(consts.PARSE_ALERT_ALERT_NAME) + alert_model['sequence_number'] = alert.get(consts.PARSE_ALERT_ALERT_ID) + alert_model['match_key'] = hashlib.md5(str(alert.get( + consts.PARSE_ALERT_ALERT_ID)).encode()).hexdigest() return alert_model def list_controllers(self, context): From 8bcc1f48e1489611adcba0a2b791998d924dd62b Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 18 Nov 2021 19:00:35 +0800 Subject: [PATCH 51/69] pure codeview-revamp parse_alert --- .../pure/flasharray/pure_flasharray.py | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 3e6a4ab1a..0ac9c8215 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -8,6 +8,7 @@ from delfin.common import constants from delfin.drivers import driver from delfin.drivers.pure.flasharray import rest_handler, consts +from delfin.i18n import _ LOG = log.getLogger(__name__) @@ -113,7 +114,7 @@ def list_alerts(self, context, query_para=None): time = alert.get('opened') alerts_model['occur_time'] = int(datetime.datetime.strptime( time, '%Y-%m-%dT%H:%M:%SZ').timestamp() - * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ + * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ if time is not None else None alerts_model['description'] = alert.get('details') alerts_model['location'] = alert.get('component_name') @@ -128,23 +129,33 @@ def list_alerts(self, context, query_para=None): @staticmethod def parse_alert(context, alert): - LOG.info("parse_alert：%s" % (json.dumps(alert, ensure_ascii=False))) - alert_model = dict() - alert_model['alert_id'] = alert.get(consts.PARSE_ALERT_ALERT_ID) - alert_model['severity'] = consts.PARSE_ALERT_SEVERITY_MAP.get( - alert.get(consts.PARSE_ALERT_SEVERITY), - constants.Severity.NOT_SPECIFIED) - alert_model['category'] = constants.Category.FAULT - alert_model['occur_time'] = utils.utcnow_ms() - alert_model['description'] = alert.get(consts.PARSE_ALERT_DESCRIPTION) - alert_model['location'] = alert.get(consts.PARSE_ALERT_CONTROLLER_NAME) - alert_model['type'] = constants.EventType.EQUIPMENT_ALARM - alert_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE - alert_model['alert_name'] = alert.get(consts.PARSE_ALERT_ALERT_NAME) - alert_model['sequence_number'] = alert.get(consts.PARSE_ALERT_ALERT_ID) - alert_model['match_key'] = hashlib.md5(str(alert.get( - consts.PARSE_ALERT_ALERT_ID)).encode()).hexdigest() - return alert_model + try: + LOG.info( + "parse_alert：%s" % (json.dumps(alert, ensure_ascii=False))) + alert_model = dict() + alert_model['alert_id'] = alert.get(consts.PARSE_ALERT_ALERT_ID) + alert_model['severity'] = consts.PARSE_ALERT_SEVERITY_MAP.get( + alert.get(consts.PARSE_ALERT_SEVERITY), + constants.Severity.NOT_SPECIFIED) + alert_model['category'] = constants.Category.FAULT + alert_model['occur_time'] = utils.utcnow_ms() + alert_model['description'] = alert.get( + consts.PARSE_ALERT_DESCRIPTION) + alert_model['location'] = alert.get( + consts.PARSE_ALERT_CONTROLLER_NAME) + alert_model['type'] = constants.EventType.EQUIPMENT_ALARM + alert_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE + alert_model['alert_name'] = alert.get( + consts.PARSE_ALERT_ALERT_NAME) + alert_model['sequence_number'] = alert.get( + consts.PARSE_ALERT_ALERT_ID) + alert_model['match_key'] = hashlib.md5(str(alert.get( + consts.PARSE_ALERT_ALERT_ID)).encode()).hexdigest() + return alert_model + except Exception as e: + LOG.error(e) + msg = (_("Failed to build alert model as some attributes missing")) + raise exception.InvalidResults(msg) def list_controllers(self, context): list_controllers = [] @@ -323,4 +334,3 @@ def reset_connection(self, context, **kwargs): @staticmethod def get_access_url(): return 'https://{ip}' - From d09a9dc1eb905539f20403159097277b36595419 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Mon, 22 Nov 2021 11:49:17 +0800 Subject: [PATCH 52/69] pure codeview-test parse_alert --- delfin/drivers/pure/flasharray/pure_flasharray.py | 11 ++++------- .../pure/flasharray/test_pure_flasharray.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 0ac9c8215..2b9ab2869 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -1,6 +1,5 @@ import datetime import hashlib -import json from oslo_log import log @@ -114,7 +113,7 @@ def list_alerts(self, context, query_para=None): time = alert.get('opened') alerts_model['occur_time'] = int(datetime.datetime.strptime( time, '%Y-%m-%dT%H:%M:%SZ').timestamp() - * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ + * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ if time is not None else None alerts_model['description'] = alert.get('details') alerts_model['location'] = alert.get('component_name') @@ -130,8 +129,6 @@ def list_alerts(self, context, query_para=None): @staticmethod def parse_alert(context, alert): try: - LOG.info( - "parse_alert：%s" % (json.dumps(alert, ensure_ascii=False))) alert_model = dict() alert_model['alert_id'] = alert.get(consts.PARSE_ALERT_ALERT_ID) alert_model['severity'] = consts.PARSE_ALERT_SEVERITY_MAP.get( @@ -250,18 +247,18 @@ def list_ports(self, context): if speed is None: hardware_result['connection_status'] = \ constants.PortConnectionStatus.UNKNOWN - hardware_result['health_status'] = constants.PortHealthStatus. \ + hardware_result['health_status'] = constants.PortHealthStatus.\ UNKNOWN elif speed == consts.CONSTANT_ZERO: hardware_result['connection_status'] = \ constants.PortConnectionStatus.DISCONNECTED - hardware_result['health_status'] = constants.PortHealthStatus. \ + hardware_result['health_status'] = constants.PortHealthStatus.\ ABNORMAL hardware_result['speed'] = speed else: hardware_result['connection_status'] = \ constants.PortConnectionStatus.CONNECTED - hardware_result['health_status'] = constants.PortHealthStatus. \ + hardware_result['health_status'] = constants.PortHealthStatus.\ NORMAL hardware_result['speed'] = int(speed) diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index 74fe4c8a4..7883a8189 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -128,6 +128,13 @@ "component_name": "ct1.ntpd" } ] +parse_alert_info = { + '1.3.6.1.2.1.1.3.0': '30007589', + '1.3.6.1.4.1.40482.3.7': '2', + '1.3.6.1.4.1.40482.3.6': 'server error', + '1.3.6.1.4.1.40482.3.3': 'cto', + '1.3.6.1.4.1.40482.3.5': 'cto.server error' +} controllers_info = [ { "status": "ready", @@ -305,6 +312,11 @@ def test_list_alerts(self): self.assertEqual(list_alerts[0].get('alert_id'), alerts_info[0].get('id')) + def test_parse_alert(self): + parse_alert = self.driver.parse_alert(context, parse_alert_info) + self.assertEqual(parse_alert.get('alert_id'), + parse_alert_info.get('1.3.6.1.2.1.1.3.0')) + def test_list_controllers(self): RestHandler.rest_call = mock.Mock( side_effect=[controllers_info]) @@ -323,7 +335,8 @@ def test_list_ports(self): RestHandler.rest_call = mock.Mock( side_effect=[port_network_info, port_info, hardware_info]) list_ports = self.driver.list_ports(context) - self.assertEqual(list_ports[0].get('name'), hardware_info[0].get('name')) + self.assertEqual(list_ports[0].get('name'), + hardware_info[0].get('name')) def test_list_storage_pools(self): list_storage_pools = self.driver.list_storage_pools(context) From f700dc909e574b11c686b7f368509a5fb8401da6 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 23 Nov 2021 17:05:07 +0800 Subject: [PATCH 53/69] pure codeview-get_storage Capacity field Modification --- delfin/drivers/pure/flasharray/pure_flasharray.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 2b9ab2869..37c69f0d6 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -55,12 +55,16 @@ def get_storage(self, context): self.rest_handler.REST_STORAGE_URL) total_capacity = None used_capacity = None + raw_capacity = None if storages: for storage in storages: - total_capacity = int(storage.get('provisioned', - consts.DEFAULT_CAPACITY)) used_capacity = int(storage.get('volumes', consts.DEFAULT_CAPACITY)) + raw_capacity = int(storage.get('capacity', + consts.DEFAULT_CAPACITY)) + shared_space = int(storage.get('shared_space', + consts.DEFAULT_CAPACITY)) + total_capacity = raw_capacity - shared_space break arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) @@ -88,7 +92,7 @@ def get_storage(self, context): storage_result = { 'model': model, 'total_capacity': total_capacity, - 'raw_capacity': total_capacity, + 'raw_capacity': raw_capacity, 'used_capacity': used_capacity, 'free_capacity': total_capacity - used_capacity, 'vendor': 'PURE', From 6acfaa189cb4104384ee95032c8001ef4794931e Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Tue, 23 Nov 2021 17:50:11 +0800 Subject: [PATCH 54/69] pure codeview-get_storage total_capacity field Modification --- delfin/drivers/pure/flasharray/pure_flasharray.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 37c69f0d6..8576e89c7 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -64,7 +64,11 @@ def get_storage(self, context): consts.DEFAULT_CAPACITY)) shared_space = int(storage.get('shared_space', consts.DEFAULT_CAPACITY)) - total_capacity = raw_capacity - shared_space + system = int(storage.get('system', consts.DEFAULT_CAPACITY)) + snapshots = int(storage.get('snapshots', + consts.DEFAULT_CAPACITY)) + total_capacity = raw_capacity - shared_space - system - \ + snapshots break arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) From 8c7815a3ed8519c5ffc6de0d3b88baef81f91b5f Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 24 Nov 2021 10:46:33 +0800 Subject: [PATCH 55/69] pure codeview-alert description --- delfin/drivers/pure/flasharray/consts.py | 9 ++------- delfin/drivers/pure/flasharray/pure_flasharray.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index a7497f33c..85608bea4 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -84,10 +84,5 @@ PARSE_ALERT_DESCRIPTION = '1.3.6.1.4.1.40482.3.6' PARSE_ALERT_SEVERITY = '1.3.6.1.4.1.40482.3.7' -PARSE_ALERT_SEVERITY_MAP = {'0': constants.Severity.FATAL, - '1': constants.Severity.CRITICAL, - '2': constants.Severity.MAJOR, - '3': constants.Severity.MINOR, - '4': constants.Severity.WARNING, - '5': constants.Severity.INFORMATIONAL, - '6': constants.Severity.NOT_SPECIFIED} +PARSE_ALERT_SEVERITY_MAP = {'1': constants.Severity.WARNING, + '2': constants.Severity.INFORMATIONAL} diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 8576e89c7..59b11667e 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -123,14 +123,17 @@ def list_alerts(self, context, query_para=None): time, '%Y-%m-%dT%H:%M:%SZ').timestamp() * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ if time is not None else None - alerts_model['description'] = alert.get('details') - alerts_model['location'] = alert.get('component_name') + component_name = alert.get('component_name') + alerts_model['location'] = component_name alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE - alerts_model['alert_name'] = alert.get('event') + event = alert.get('event') + alerts_model['alert_name'] = event alerts_model['sequence_number'] = alert.get('id') alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). encode()).hexdigest() + alerts_model['description'] = '：(<{}>:<{}>): <{}>'.\ + format(alert.get('component_type'), component_name, event) alerts_list.append(alerts_model) return alerts_list @@ -144,8 +147,10 @@ def parse_alert(context, alert): constants.Severity.NOT_SPECIFIED) alert_model['category'] = constants.Category.FAULT alert_model['occur_time'] = utils.utcnow_ms() - alert_model['description'] = alert.get( - consts.PARSE_ALERT_DESCRIPTION) + alert_model['description'] = '(<{}>:<{}>):<{}>'.format(alert.get( + consts.PARSE_ALERT_STORAGE_NAME), + alert.get(consts.PARSE_ALERT_CONTROLLER_NAME), + alert.get(consts.PARSE_ALERT_DESCRIPTION)) alert_model['location'] = alert.get( consts.PARSE_ALERT_CONTROLLER_NAME) alert_model['type'] = constants.EventType.EQUIPMENT_ALARM From 82505f37b4b1d838ca3e4bca0c2216d5d5f53bda Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 24 Nov 2021 11:33:04 +0800 Subject: [PATCH 56/69] pure codeview-alert description --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 59b11667e..c74b3a27f 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -132,7 +132,7 @@ def list_alerts(self, context, query_para=None): alerts_model['sequence_number'] = alert.get('id') alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). encode()).hexdigest() - alerts_model['description'] = '：(<{}>:<{}>): <{}>'.\ + alerts_model['description'] = '({}:{}): {}'.\ format(alert.get('component_type'), component_name, event) alerts_list.append(alerts_model) return alerts_list @@ -147,7 +147,7 @@ def parse_alert(context, alert): constants.Severity.NOT_SPECIFIED) alert_model['category'] = constants.Category.FAULT alert_model['occur_time'] = utils.utcnow_ms() - alert_model['description'] = '(<{}>:<{}>):<{}>'.format(alert.get( + alert_model['description'] = '({}:{}):{}'.format(alert.get( consts.PARSE_ALERT_STORAGE_NAME), alert.get(consts.PARSE_ALERT_CONTROLLER_NAME), alert.get(consts.PARSE_ALERT_DESCRIPTION)) From 6777fb3fa4427357b402f1e4ee78512be9a24717 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 24 Nov 2021 11:55:32 +0800 Subject: [PATCH 57/69] pure codeview-alert description --- delfin/drivers/pure/flasharray/pure_flasharray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index c74b3a27f..6d49227f4 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -147,7 +147,7 @@ def parse_alert(context, alert): constants.Severity.NOT_SPECIFIED) alert_model['category'] = constants.Category.FAULT alert_model['occur_time'] = utils.utcnow_ms() - alert_model['description'] = '({}:{}):{}'.format(alert.get( + alert_model['description'] = '({}:{}): {}'.format(alert.get( consts.PARSE_ALERT_STORAGE_NAME), alert.get(consts.PARSE_ALERT_CONTROLLER_NAME), alert.get(consts.PARSE_ALERT_DESCRIPTION)) From d32286b8b3dd41cfb8e617cd5b5605824a79e334 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 25 Nov 2021 15:31:56 +0800 Subject: [PATCH 58/69] pure codeview-Code indentation --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 6d49227f4..fb7718f90 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -67,8 +67,8 @@ def get_storage(self, context): system = int(storage.get('system', consts.DEFAULT_CAPACITY)) snapshots = int(storage.get('snapshots', consts.DEFAULT_CAPACITY)) - total_capacity = raw_capacity - shared_space - system - \ - snapshots + total_capacity =\ + raw_capacity - shared_space - system - snapshots break arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) From 0304cb29b275f43e5c21393de576a491a8f7495a Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 1 Dec 2021 17:00:38 +0800 Subject: [PATCH 59/69] pure amend-list_alert query_para;port eth;get_storage total; disk filter --- delfin/drivers/pure/flasharray/consts.py | 3 + .../pure/flasharray/pure_flasharray.py | 168 ++++++++++-------- .../drivers/pure/flasharray/rest_handler.py | 2 +- 3 files changed, 95 insertions(+), 78 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 85608bea4..77613163b 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -48,6 +48,9 @@ # Normal value of the controller status NORMAL_CONTROLLER_STATUS = 'ready' +# disk type +DISK_TYPE_NVRAM = 'NVRAM' + # list_port: Add ":" to the WWN every 2 sequences. SPLICE_WWN_SERIAL = 2 SPLICE_WWN_COLON = ':' diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index fb7718f90..5a260a84b 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -58,7 +58,7 @@ def get_storage(self, context): raw_capacity = None if storages: for storage in storages: - used_capacity = int(storage.get('volumes', + used_capacity = int(storage.get('total', consts.DEFAULT_CAPACITY)) raw_capacity = int(storage.get('capacity', consts.DEFAULT_CAPACITY)) @@ -113,16 +113,28 @@ def list_alerts(self, context, query_para=None): if alerts: for alert in alerts: alerts_model = dict() + time = alert.get('opened') + timestamp = int(datetime.datetime.strptime( + time, '%Y-%m-%dT%H:%M:%SZ').timestamp() + * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION)\ + if time is not None else None + if query_para is not None: + try: + start_time = int(query_para.get('start_time')) + end_time = int(query_para.get('end_time')) + except Exception as e: + LOG.error(e) + msg = (_("Time conversion error")) + raise exception.InvalidResults(msg) + if timestamp < start_time or timestamp > end_time or \ + timestamp is None: + continue + alerts_model['occur_time'] = timestamp alerts_model['alert_id'] = alert.get('id') alerts_model['severity'] = consts.SEVERITY_MAP.get( alert.get('current_severity'), constants.Severity.NOT_SPECIFIED) alerts_model['category'] = constants.Category.FAULT - time = alert.get('opened') - alerts_model['occur_time'] = int(datetime.datetime.strptime( - time, '%Y-%m-%dT%H:%M:%SZ').timestamp() - * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ - if time is not None else None component_name = alert.get('component_name') alerts_model['location'] = component_name alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM @@ -193,10 +205,14 @@ def list_disks(self, context): disks = self.rest_handler.rest_call(self.rest_handler.REST_DISK_URL) if disks: for disk in disks: + disk_type = disk.get('type') + if disk_type == consts.DISK_TYPE_NVRAM: + continue disk_dict = dict() drive_name = disk.get('name') disk_dict['name'] = drive_name - physical_type = disk.get('type', "").lower() + physical_type = disk_type.lower() if disk_type is not None \ + else None disk_dict['physical_type'] = physical_type \ if physical_type in constants.DiskPhysicalType.ALL else \ constants.DiskPhysicalType.UNKNOWN @@ -211,7 +227,6 @@ def list_disks(self, context): disk_dict['model'] = hardware_object.get('model') disk_dict['serial_number'] = hardware_object. \ get('serial_number') - disk_dict['native_disk_id'] = drive_name disk_dict['id'] = drive_name disk_dict['location'] = drive_name @@ -235,58 +250,77 @@ def get_hardware(self): def list_ports(self, context): list_ports = [] - networks = self.get_network() + hardware_dist = self.get_port_hardware() ports = self.get_ports() + networks = self.rest_handler.rest_call( + self.rest_handler.REST_NETWORK_URL) + if networks: + for network in networks: + network_dict = dict() + network_name = network.get('name').upper() + if 'FC' in network_name: + network_dict['type'] = constants.PortType.FC + elif 'ETH' in network_name: + network_dict['type'] = constants.PortType.ETH + elif 'SAS' in network_name: + network_dict['type'] = constants.PortType.SAS + else: + continue + network_dict['storage_id'] = self.storage_id + network_dict['name'] = network_name + network_dict['native_port_id'] = network_name + network_dict['location'] = network_name + network_dict['mac_address'] = network.get('hwaddr') + services_list = network.get('services') + if services_list: + for services in services_list: + network_dict['logical_type'] = services if \ + services in constants.PortLogicalType.ALL else None + break + network_dict['ipv4_mask'] = network.get('netmask') + network_dict['ipv4'] = network.get('address') + port = ports.get(network_name) + if port: + network_dict['wwn'] = port.get('wwn') + hardware = hardware_dist.get(network_name) + if hardware: + network_dict['connection_status'] = hardware.get( + 'connection_status') + network_dict['health_status'] = hardware.get( + 'health_status') + network_dict['speed'] = hardware.get('speed') + list_ports.append(network_dict) + return list_ports + + def get_port_hardware(self): + hardware_name_dict = dict() hardware_dict = self.rest_handler.rest_call( self.rest_handler.REST_HARDWARE_URL) - if not hardware_dict: - return list_ports - for hardware in hardware_dict: - hardware_result = dict() - hardware_name = hardware.get('name') - if 'FC' in hardware_name: - hardware_result['type'] = constants.PortType.FC - elif 'ETH' in hardware_name: - hardware_result['type'] = constants.PortType.ETH - elif 'SAS' in hardware_name: - hardware_result['type'] = constants.PortType.SAS - else: - continue - hardware_result['name'] = hardware_name - hardware_result['native_port_id'] = hardware_name - hardware_result['storage_id'] = self.storage_id - hardware_result['location'] = hardware_name - speed = hardware.get('speed') - if speed is None: - hardware_result['connection_status'] = \ - constants.PortConnectionStatus.UNKNOWN - hardware_result['health_status'] = constants.PortHealthStatus.\ - UNKNOWN - elif speed == consts.CONSTANT_ZERO: - hardware_result['connection_status'] = \ - constants.PortConnectionStatus.DISCONNECTED - hardware_result['health_status'] = constants.PortHealthStatus.\ - ABNORMAL - hardware_result['speed'] = speed - else: - hardware_result['connection_status'] = \ - constants.PortConnectionStatus.CONNECTED - hardware_result['health_status'] = constants.PortHealthStatus.\ - NORMAL - hardware_result['speed'] = int(speed) - - port = ports.get(hardware_name) - if port: - hardware_result['wwn'] = port.get('wwn') - - network = networks.get(hardware_name) - if network: - hardware_result['mac_address'] = network.get('mac_address') - hardware_result['logical_type'] = network.get('logical_type') - hardware_result['ipv4_mask'] = network.get('ipv4_mask') - hardware_result['ipv4'] = network.get('ipv4') - list_ports.append(hardware_result) - return list_ports + if hardware_dict: + for hardware in hardware_dict: + hardware_result = dict() + hardware_name = hardware.get('name') + hardware_result['name'] = hardware_name + speed = hardware.get('speed') + if speed is None: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.UNKNOWN + hardware_result['health_status'] = constants.\ + PortHealthStatus.UNKNOWN + elif speed == consts.CONSTANT_ZERO: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.DISCONNECTED + hardware_result['health_status'] = constants.\ + PortHealthStatus.ABNORMAL + hardware_result['speed'] = speed + else: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.CONNECTED + hardware_result['health_status'] = constants.\ + PortHealthStatus.NORMAL + hardware_result['speed'] = int(speed) + hardware_name_dict[hardware_name] = hardware_result + return hardware_name_dict def get_ports(self): ports_dict = dict() @@ -311,26 +345,6 @@ def get_splice_wwn(wwn): wwn_splice = '{}{}'.format(wwn_splice, wwn_list[serial]) return wwn_splice - def get_network(self): - networks_object = dict() - networks = self.rest_handler.rest_call( - self.rest_handler.REST_NETWORK_URL) - if networks: - for network in networks: - network_dict = dict() - network_dict['mac_address'] = network.get('hwaddr') - services_list = network.get('services') - if services_list: - for services in services_list: - network_dict['logical_type'] = services if \ - services in constants.PortLogicalType.ALL else None - break - network_dict['ipv4_mask'] = network.get('netmask') - network_dict['ipv4'] = network.get('address') - network_name = network.get('name').upper() - networks_object[network_name] = network_dict - return networks_object - def list_storage_pools(self, context): return [] diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 3b066a4c6..cb59f4809 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -38,7 +38,7 @@ def login(self): LOG.error("Login error, Obtaining the token is abnormal. " "status_code:%s, URL: %s", token_res.status_code, RestHandler.REST_AUTH_URL) - raise exception.StorageBackendException( + raise exception.InvalidUsernameOrPassword( 'Obtaining the token is abnormal') session_res = self.do_call(RestHandler.REST_SESSION_URL, token_res.json(), method='POST') From a02e506b75071185f21195f80779677f359cbf83 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Wed, 1 Dec 2021 18:31:49 +0800 Subject: [PATCH 60/69] pure amend-get_storage total disk --- delfin/drivers/pure/flasharray/consts.py | 3 +++ .../pure/flasharray/pure_flasharray.py | 20 ++++++++----------- .../drivers/pure/flasharray/rest_handler.py | 8 ++++---- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 77613163b..00bdca961 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -51,6 +51,9 @@ # disk type DISK_TYPE_NVRAM = 'NVRAM' +# The account password is incorrect during login. +LOGIN_PASSWORD_ERR = 'invalid credentials' + # list_port: Add ":" to the WWN every 2 sequences. SPLICE_WWN_SERIAL = 2 SPLICE_WWN_COLON = ':' diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 5a260a84b..5fa9c6ae2 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -55,22 +55,19 @@ def get_storage(self, context): self.rest_handler.REST_STORAGE_URL) total_capacity = None used_capacity = None - raw_capacity = None if storages: for storage in storages: used_capacity = int(storage.get('total', consts.DEFAULT_CAPACITY)) - raw_capacity = int(storage.get('capacity', - consts.DEFAULT_CAPACITY)) - shared_space = int(storage.get('shared_space', - consts.DEFAULT_CAPACITY)) - system = int(storage.get('system', consts.DEFAULT_CAPACITY)) - snapshots = int(storage.get('snapshots', - consts.DEFAULT_CAPACITY)) - total_capacity =\ - raw_capacity - shared_space - system - snapshots + total_capacity = int(storage.get('capacity', + consts.DEFAULT_CAPACITY)) break - + raw_capacity = consts.DEFAULT_CAPACITY + disks = self.rest_handler.rest_call(self.rest_handler.REST_DISK_URL) + if disks: + for disk in disks: + raw_capacity = raw_capacity + int(disk.get( + 'capacity', consts.DEFAULT_CAPACITY)) arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) storage_name = None serial_number = None @@ -79,7 +76,6 @@ def get_storage(self, context): storage_name = arrays.get('array_name') serial_number = arrays.get('id') version = arrays.get('version') - model = None status = constants.StorageStatus.NORMAL controllers = self.rest_handler.rest_call( diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index cb59f4809..7ddefd050 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -19,7 +19,7 @@ class RestHandler(RestClient): REST_DISK_URL = '/api/1.17/drive' REST_HARDWARE_URL = '/api/1.17/hardware' REST_CONTROLLERS_URL = '/api/1.17/array?controllers=true' - REST_ALERTS_URL = '/api/1.17/message?flagged=true' + REST_ALERTS_URL = '/api/1.17/message?flagged=true&open=true' REST_AUTH_URL = '/api/1.17/auth/apitoken' REST_SESSION_URL = '/api/1.17/auth/session' @@ -34,7 +34,8 @@ def login(self): token_res = self.do_call(RestHandler.REST_AUTH_URL, data, method='POST') if token_res.status_code != consts.SUCCESS_STATUS_CODE or not \ - token_res.json().get('api_token'): + token_res.json().get('api_token') or \ + token_res.json().get('msg') == consts.LOGIN_PASSWORD_ERR: LOG.error("Login error, Obtaining the token is abnormal. " "status_code:%s, URL: %s", token_res.status_code, RestHandler.REST_AUTH_URL) @@ -45,8 +46,7 @@ def login(self): if session_res.status_code != consts.SUCCESS_STATUS_CODE or not \ session_res.json().get('username'): LOG.error("Login error, Obtaining the session is abnormal." - "status_code:%s, URL: %s", - session_res.status_code, + "status_code:%s, URL: %s", session_res.status_code, RestHandler.REST_SESSION_URL) raise exception.StorageBackendException( 'Obtaining the session is abnormal.') From e044fa2b73fc0ce71a1752ff424a8e49df05e3cf Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 2 Dec 2021 10:06:03 +0800 Subject: [PATCH 61/69] pure amend-list_alerts timestamp --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 5fa9c6ae2..7bf0377cd 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -122,8 +122,8 @@ def list_alerts(self, context, query_para=None): LOG.error(e) msg = (_("Time conversion error")) raise exception.InvalidResults(msg) - if timestamp < start_time or timestamp > end_time or \ - timestamp is None: + if timestamp is None or timestamp < start_time or \ + timestamp > end_time: continue alerts_model['occur_time'] = timestamp alerts_model['alert_id'] = alert.get('id') From b7e8928d737ef5a94330d6e6d8cabd9e30bc19e3 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Thu, 2 Dec 2021 15:27:33 +0800 Subject: [PATCH 62/69] pure amend-login port list_alert --- delfin/drivers/pure/flasharray/consts.py | 4 +- .../pure/flasharray/pure_flasharray.py | 118 ++++++++---------- .../drivers/pure/flasharray/rest_handler.py | 11 +- 3 files changed, 66 insertions(+), 67 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 00bdca961..a608d81ca 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -81,7 +81,9 @@ 'abnormal': constants.DiskStatus.ABNORMAL, 'unhealthy': constants.DiskStatus.ABNORMAL, 'offline': constants.DiskStatus.OFFLINE} - +PORT_STATUS_MAP = {'ok': constants.PortHealthStatus.NORMAL, + 'not_installed': constants.PortHealthStatus.ABNORMAL + } PARSE_ALERT_ALERT_ID = '1.3.6.1.2.1.1.3.0' PARSE_ALERT_STORAGE_NAME = '1.3.6.1.4.1.40482.3.1' diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 7bf0377cd..852b789e4 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -116,15 +116,12 @@ def list_alerts(self, context, query_para=None): if time is not None else None if query_para is not None: try: - start_time = int(query_para.get('start_time')) - end_time = int(query_para.get('end_time')) + if timestamp is None or timestamp\ + < int(query_para.get('begin_time')) or\ + timestamp > int(query_para.get('end_time')): + continue except Exception as e: LOG.error(e) - msg = (_("Time conversion error")) - raise exception.InvalidResults(msg) - if timestamp is None or timestamp < start_time or \ - timestamp > end_time: - continue alerts_model['occur_time'] = timestamp alerts_model['alert_id'] = alert.get('id') alerts_model['severity'] = consts.SEVERITY_MAP.get( @@ -246,26 +243,60 @@ def get_hardware(self): def list_ports(self, context): list_ports = [] - hardware_dist = self.get_port_hardware() + networks = self.get_network() ports = self.get_ports() + hardware_dict = self.rest_handler.rest_call( + self.rest_handler.REST_HARDWARE_URL) + if not hardware_dict: + return list_ports + for hardware in hardware_dict: + hardware_result = dict() + hardware_name = hardware.get('name') + if 'FC' in hardware_name: + hardware_result['type'] = constants.PortType.FC + elif 'ETH' in hardware_name: + hardware_result['type'] = constants.PortType.ETH + elif 'SAS' in hardware_name: + hardware_result['type'] = constants.PortType.SAS + else: + continue + hardware_result['name'] = hardware_name + hardware_result['native_port_id'] = hardware_name + hardware_result['storage_id'] = self.storage_id + hardware_result['location'] = hardware_name + speed = hardware.get('speed') + if speed is None: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.UNKNOWN + elif speed == consts.CONSTANT_ZERO: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.DISCONNECTED + hardware_result['speed'] = speed + else: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.CONNECTED + hardware_result['speed'] = int(speed) + hardware_result['health_status'] = consts.PORT_STATUS_MAP.get( + hardware.get('status'), constants.PortHealthStatus.UNKNOWN) + port = ports.get(hardware_name) + if port: + hardware_result['wwn'] = port.get('wwn') + network = networks.get(hardware_name) + if network: + hardware_result['mac_address'] = network.get('mac_address') + hardware_result['logical_type'] = network.get('logical_type') + hardware_result['ipv4_mask'] = network.get('ipv4_mask') + hardware_result['ipv4'] = network.get('ipv4') + list_ports.append(hardware_result) + return list_ports + + def get_network(self): + networks_object = dict() networks = self.rest_handler.rest_call( self.rest_handler.REST_NETWORK_URL) if networks: for network in networks: network_dict = dict() - network_name = network.get('name').upper() - if 'FC' in network_name: - network_dict['type'] = constants.PortType.FC - elif 'ETH' in network_name: - network_dict['type'] = constants.PortType.ETH - elif 'SAS' in network_name: - network_dict['type'] = constants.PortType.SAS - else: - continue - network_dict['storage_id'] = self.storage_id - network_dict['name'] = network_name - network_dict['native_port_id'] = network_name - network_dict['location'] = network_name network_dict['mac_address'] = network.get('hwaddr') services_list = network.get('services') if services_list: @@ -275,48 +306,9 @@ def list_ports(self, context): break network_dict['ipv4_mask'] = network.get('netmask') network_dict['ipv4'] = network.get('address') - port = ports.get(network_name) - if port: - network_dict['wwn'] = port.get('wwn') - hardware = hardware_dist.get(network_name) - if hardware: - network_dict['connection_status'] = hardware.get( - 'connection_status') - network_dict['health_status'] = hardware.get( - 'health_status') - network_dict['speed'] = hardware.get('speed') - list_ports.append(network_dict) - return list_ports - - def get_port_hardware(self): - hardware_name_dict = dict() - hardware_dict = self.rest_handler.rest_call( - self.rest_handler.REST_HARDWARE_URL) - if hardware_dict: - for hardware in hardware_dict: - hardware_result = dict() - hardware_name = hardware.get('name') - hardware_result['name'] = hardware_name - speed = hardware.get('speed') - if speed is None: - hardware_result['connection_status'] = \ - constants.PortConnectionStatus.UNKNOWN - hardware_result['health_status'] = constants.\ - PortHealthStatus.UNKNOWN - elif speed == consts.CONSTANT_ZERO: - hardware_result['connection_status'] = \ - constants.PortConnectionStatus.DISCONNECTED - hardware_result['health_status'] = constants.\ - PortHealthStatus.ABNORMAL - hardware_result['speed'] = speed - else: - hardware_result['connection_status'] = \ - constants.PortConnectionStatus.CONNECTED - hardware_result['health_status'] = constants.\ - PortHealthStatus.NORMAL - hardware_result['speed'] = int(speed) - hardware_name_dict[hardware_name] = hardware_result - return hardware_name_dict + network_name = network.get('name').upper() + networks_object[network_name] = network_dict + return networks_object def get_ports(self): ports_dict = dict() diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 7ddefd050..462fd4b99 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -33,14 +33,19 @@ def login(self): self.init_http_head() token_res = self.do_call(RestHandler.REST_AUTH_URL, data, method='POST') - if token_res.status_code != consts.SUCCESS_STATUS_CODE or not \ - token_res.json().get('api_token') or \ - token_res.json().get('msg') == consts.LOGIN_PASSWORD_ERR: + if token_res.json().get('msg') == consts.LOGIN_PASSWORD_ERR: LOG.error("Login error, Obtaining the token is abnormal. " "status_code:%s, URL: %s", token_res.status_code, RestHandler.REST_AUTH_URL) raise exception.InvalidUsernameOrPassword( 'Obtaining the token is abnormal') + if token_res.status_code != consts.SUCCESS_STATUS_CODE or not \ + token_res.json().get('api_token'): + LOG.error("Login error, Obtaining the token is abnormal. " + "status_code:%s, URL: %s", + token_res.status_code, RestHandler.REST_AUTH_URL) + raise exception.StorageBackendException( + 'Obtaining the token is abnormal') session_res = self.do_call(RestHandler.REST_SESSION_URL, token_res.json(), method='POST') if session_res.status_code != consts.SUCCESS_STATUS_CODE or not \ From ce8d3c544b788017d21e4b9602748653eda60074 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 3 Dec 2021 10:39:01 +0800 Subject: [PATCH 63/69] pure amend - alert remove sequence_number --- delfin/drivers/pure/flasharray/pure_flasharray.py | 1 - 1 file changed, 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 852b789e4..51bc9f01b 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -134,7 +134,6 @@ def list_alerts(self, context, query_para=None): alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE event = alert.get('event') alerts_model['alert_name'] = event - alerts_model['sequence_number'] = alert.get('id') alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). encode()).hexdigest() alerts_model['description'] = '({}:{}): {}'.\ From ab9b3fec2b3a4dc45e5fc6ef1f31669f6b92691d Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Fri, 3 Dec 2021 16:40:58 +0800 Subject: [PATCH 64/69] pure amend-list_controllers status;list_disks type; --- delfin/drivers/pure/flasharray/consts.py | 4 +- .../pure/flasharray/pure_flasharray.py | 42 +++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index a608d81ca..65d6c5976 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -70,11 +70,11 @@ 'recovery': constants.Category.RECOVERY, 'notSpecified': constants.Category.NOT_SPECIFIED} CONTROLLER_STATUS_MAP = {'normal': constants.ControllerStatus.NORMAL, - 'ready': constants.ControllerStatus.NORMAL, + 'ok': constants.ControllerStatus.NORMAL, 'offline': constants.ControllerStatus.OFFLINE, + 'not_installed': constants.ControllerStatus.OFFLINE, 'fault': constants.ControllerStatus.FAULT, 'degraded': constants.ControllerStatus.DEGRADED, - 'unknown': constants.ControllerStatus.UNKNOWN, 'unready': constants.ControllerStatus.UNKNOWN} DISK_STATUS_MAP = {'normal': constants.DiskStatus.NORMAL, 'healthy': constants.DiskStatus.NORMAL, diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 51bc9f01b..96924c13b 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -1,5 +1,6 @@ import datetime import hashlib +import time from oslo_log import log @@ -63,11 +64,10 @@ def get_storage(self, context): consts.DEFAULT_CAPACITY)) break raw_capacity = consts.DEFAULT_CAPACITY - disks = self.rest_handler.rest_call(self.rest_handler.REST_DISK_URL) + disks = self.list_disks(context) if disks: for disk in disks: - raw_capacity = raw_capacity + int(disk.get( - 'capacity', consts.DEFAULT_CAPACITY)) + raw_capacity = raw_capacity + disk.get('capacity') arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) storage_name = None serial_number = None @@ -84,8 +84,9 @@ def get_storage(self, context): for controller in controllers: if controller.get('mode') == consts.CONTROLLER_PRIMARY: model = controller.get('model') - if controller.get('status') != consts.NORMAL_CONTROLLER_STATUS: - status = constants.StorageStatus.ABNORMAL + if controller.get('status') != \ + consts.NORMAL_CONTROLLER_STATUS: + status = constants.StorageStatus.ABNORMAL if not all((storages, arrays, controllers)): LOG.error('get_storage error, Unable to obtain data.') raise exception.StorageBackendException('Unable to obtain data') @@ -109,15 +110,18 @@ def list_alerts(self, context, query_para=None): if alerts: for alert in alerts: alerts_model = dict() - time = alert.get('opened') - timestamp = int(datetime.datetime.strptime( - time, '%Y-%m-%dT%H:%M:%SZ').timestamp() - * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION)\ - if time is not None else None + opened = alert.get('opened') + time_difference = time.mktime( + time.localtime()) - time.mktime(time.gmtime()) + timestamp = (int(datetime.datetime.strptime( + opened, '%Y-%m-%dT%H:%M:%SZ').timestamp() + + time_difference) + * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ + if opened is not None else None if query_para is not None: try: - if timestamp is None or timestamp\ - < int(query_para.get('begin_time')) or\ + if timestamp is None or timestamp \ + < int(query_para.get('begin_time')) or \ timestamp > int(query_para.get('end_time')): continue except Exception as e: @@ -136,7 +140,7 @@ def list_alerts(self, context, query_para=None): alerts_model['alert_name'] = event alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). encode()).hexdigest() - alerts_model['description'] = '({}:{}): {}'.\ + alerts_model['description'] = '({}:{}): {}'. \ format(alert.get('component_type'), component_name, event) alerts_list.append(alerts_model) return alerts_list @@ -175,14 +179,17 @@ def list_controllers(self, context): list_controllers = [] controllers = self.rest_handler.rest_call( self.rest_handler.REST_CONTROLLERS_URL) + hardware = self.get_hardware() if controllers: for controller in controllers: controllers_dict = dict() controller_name = controller.get('name') controllers_dict['name'] = controller_name - controllers_dict['status'] = consts.CONTROLLER_STATUS_MAP. \ - get(controller.get('status'), - constants.ControllerStatus.UNKNOWN) + controllers_dict['status'] = hardware.get( + controller_name, {}).get('status') + controllers_dict['status'] = consts.CONTROLLER_STATUS_MAP.get( + hardware.get(controller_name, {}).get('status'), + constants.ControllerStatus.UNKNOWN) controllers_dict['soft_version'] = controller.get('version') controllers_dict['storage_id'] = self.storage_id controllers_dict['id'] = controller_name @@ -198,7 +205,7 @@ def list_disks(self, context): if disks: for disk in disks: disk_type = disk.get('type') - if disk_type == consts.DISK_TYPE_NVRAM: + if disk_type == consts.DISK_TYPE_NVRAM or disk_type is None: continue disk_dict = dict() drive_name = disk.get('name') @@ -237,6 +244,7 @@ def get_hardware(self): hardware_map['speed'] = hardware_value.get('speed') hardware_map['serial_number'] = hardware_value.get('serial') hardware_map['model'] = hardware_value.get('model') + hardware_map['status'] = hardware_value.get('status') hardware_dict[hardware_value.get('name')] = hardware_map return hardware_dict From 991b859caabca7fec04ea071eb01d80a625ba8fe Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Sat, 4 Dec 2021 10:22:35 +0800 Subject: [PATCH 65/69] pure amend-test func --- .../unit/drivers/pure/flasharray/test_pure_flasharray.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index 7883a8189..ab0a332bb 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -300,7 +300,8 @@ def test_list_volumes(self): def test_get_storage(self): RestHandler.rest_call = mock.Mock( - side_effect=[storage_info, storage_id_info, controllers_info]) + side_effect=[storage_info, hardware_info, drive_info, + storage_id_info, controllers_info]) storage_object = self.driver.get_storage(context) self.assertEqual(storage_object.get('name'), storage_id_info.get('array_name')) @@ -319,7 +320,7 @@ def test_parse_alert(self): def test_list_controllers(self): RestHandler.rest_call = mock.Mock( - side_effect=[controllers_info]) + side_effect=[controllers_info, hardware_info]) list_controllers = self.driver.list_controllers(context) self.assertEqual(list_controllers[0].get('name'), controllers_info[0].get('name')) From 6fbbfe798d11997c4b8d299004a186893facab84 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Sat, 4 Dec 2021 10:48:24 +0800 Subject: [PATCH 66/69] pure amend-Code formatting --- delfin/drivers/pure/flasharray/pure_flasharray.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 96924c13b..58eede3c5 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -113,11 +113,11 @@ def list_alerts(self, context, query_para=None): opened = alert.get('opened') time_difference = time.mktime( time.localtime()) - time.mktime(time.gmtime()) - timestamp = (int(datetime.datetime.strptime( - opened, '%Y-%m-%dT%H:%M:%SZ').timestamp() + - time_difference) - * consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) \ - if opened is not None else None + timestamp = (int(datetime.datetime.strptime + (opened, '%Y-%m-%dT%H:%M:%SZ').timestamp() + + time_difference) * + consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) if\ + opened is not None else None if query_para is not None: try: if timestamp is None or timestamp \ From b55d1c1d49f5908b6fa89e9dd4f5827da03a4cf6 Mon Sep 17 00:00:00 2001 From: zhilong-xu Date: Sat, 4 Dec 2021 12:10:10 +0800 Subject: [PATCH 67/69] pure amend-remove disk id and controllers id --- delfin/drivers/pure/flasharray/pure_flasharray.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 58eede3c5..31db05b6b 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -185,14 +185,11 @@ def list_controllers(self, context): controllers_dict = dict() controller_name = controller.get('name') controllers_dict['name'] = controller_name - controllers_dict['status'] = hardware.get( - controller_name, {}).get('status') controllers_dict['status'] = consts.CONTROLLER_STATUS_MAP.get( hardware.get(controller_name, {}).get('status'), constants.ControllerStatus.UNKNOWN) controllers_dict['soft_version'] = controller.get('version') controllers_dict['storage_id'] = self.storage_id - controllers_dict['id'] = controller_name controllers_dict['native_controller_id'] = controller_name controllers_dict['location'] = controller_name list_controllers.append(controllers_dict) @@ -227,7 +224,6 @@ def list_disks(self, context): disk_dict['serial_number'] = hardware_object. \ get('serial_number') disk_dict['native_disk_id'] = drive_name - disk_dict['id'] = drive_name disk_dict['location'] = drive_name disk_dict['manufacturer'] = "PURE" disk_dict['firmware'] = "" From 7bcbbf2af960dc84cbb0aae28bca7ee4b7a39b89 Mon Sep 17 00:00:00 2001 From: Administrator Date: Wed, 5 Jan 2022 18:33:52 +0800 Subject: [PATCH 68/69] add pure --- delfin/drivers/pure/__init__.py | 0 delfin/drivers/pure/flasharray/__init__.py | 0 delfin/drivers/pure/flasharray/consts.py | 96 +++++ .../pure/flasharray/pure_flasharray.py | 351 +++++++++++++++++ .../drivers/pure/flasharray/rest_handler.py | 103 +++++ delfin/tests/unit/drivers/pure/__init__.py | 0 .../unit/drivers/pure/flasharray/__init__.py | 0 .../pure/flasharray/test_pure_flasharray.py | 355 ++++++++++++++++++ setup.py | 3 +- 9 files changed, 907 insertions(+), 1 deletion(-) create mode 100644 delfin/drivers/pure/__init__.py create mode 100644 delfin/drivers/pure/flasharray/__init__.py create mode 100644 delfin/drivers/pure/flasharray/consts.py create mode 100644 delfin/drivers/pure/flasharray/pure_flasharray.py create mode 100644 delfin/drivers/pure/flasharray/rest_handler.py create mode 100644 delfin/tests/unit/drivers/pure/__init__.py create mode 100644 delfin/tests/unit/drivers/pure/flasharray/__init__.py create mode 100644 delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py diff --git a/delfin/drivers/pure/__init__.py b/delfin/drivers/pure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/drivers/pure/flasharray/__init__.py b/delfin/drivers/pure/flasharray/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py new file mode 100644 index 000000000..65d6c5976 --- /dev/null +++ b/delfin/drivers/pure/flasharray/consts.py @@ -0,0 +1,96 @@ +# Copyright 2021 The SODA Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from delfin.common import constants + +# The default volume +DEFAULT_CAPACITY = 0 + +# The default speed +DEFAULT_SPEED = 0 + +# The default list_alerts time conversion +DEFAULT_LIST_ALERTS_TIME_CONVERSION = 1000 + +# The default count for the get_volumes_info function +DEFAULT_COUNT_GET_VOLUMES_INFO = 0 + +# Number of re-logins +RE_LOGIN_TIMES = 3 + +# Constant one +CONSTANT_ONE = 1 +# Constant zero +CONSTANT_ZERO = 0 + +# Success status code +SUCCESS_STATUS_CODE = 200 + +# Status code of no permission +PERMISSION_DENIED_STATUS_CODE = 401 + +# Custom token of Pure +CUSTOM_TOKEN = 'x-next-token' + +# The default get_storage model +CONTROLLER_PRIMARY = 'primary' + +# Normal value of the controller status +NORMAL_CONTROLLER_STATUS = 'ready' + +# disk type +DISK_TYPE_NVRAM = 'NVRAM' + +# The account password is incorrect during login. +LOGIN_PASSWORD_ERR = 'invalid credentials' + +# list_port: Add ":" to the WWN every 2 sequences. +SPLICE_WWN_SERIAL = 2 +SPLICE_WWN_COLON = ':' + +SEVERITY_MAP = {'fatal': constants.Severity.FATAL, + 'critical': constants.Severity.CRITICAL, + 'major': constants.Severity.MAJOR, + 'minor': constants.Severity.MINOR, + 'warning': constants.Severity.WARNING, + 'informational': constants.Severity.INFORMATIONAL, + 'NotSpecified': constants.Severity.NOT_SPECIFIED} +CATEGORY_MAP = {'fault': constants.Category.FAULT, + 'event': constants.Category.EVENT, + 'recovery': constants.Category.RECOVERY, + 'notSpecified': constants.Category.NOT_SPECIFIED} +CONTROLLER_STATUS_MAP = {'normal': constants.ControllerStatus.NORMAL, + 'ok': constants.ControllerStatus.NORMAL, + 'offline': constants.ControllerStatus.OFFLINE, + 'not_installed': constants.ControllerStatus.OFFLINE, + 'fault': constants.ControllerStatus.FAULT, + 'degraded': constants.ControllerStatus.DEGRADED, + 'unready': constants.ControllerStatus.UNKNOWN} +DISK_STATUS_MAP = {'normal': constants.DiskStatus.NORMAL, + 'healthy': constants.DiskStatus.NORMAL, + 'abnormal': constants.DiskStatus.ABNORMAL, + 'unhealthy': constants.DiskStatus.ABNORMAL, + 'offline': constants.DiskStatus.OFFLINE} +PORT_STATUS_MAP = {'ok': constants.PortHealthStatus.NORMAL, + 'not_installed': constants.PortHealthStatus.ABNORMAL + } + +PARSE_ALERT_ALERT_ID = '1.3.6.1.2.1.1.3.0' +PARSE_ALERT_STORAGE_NAME = '1.3.6.1.4.1.40482.3.1' +PARSE_ALERT_CONTROLLER_NAME = '1.3.6.1.4.1.40482.3.3' +PARSE_ALERT_ALERT_NAME = '1.3.6.1.4.1.40482.3.5' +PARSE_ALERT_DESCRIPTION = '1.3.6.1.4.1.40482.3.6' +PARSE_ALERT_SEVERITY = '1.3.6.1.4.1.40482.3.7' + +PARSE_ALERT_SEVERITY_MAP = {'1': constants.Severity.WARNING, + '2': constants.Severity.INFORMATIONAL} diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py new file mode 100644 index 000000000..31db05b6b --- /dev/null +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -0,0 +1,351 @@ +import datetime +import hashlib +import time + +from oslo_log import log + +from delfin import exception, utils +from delfin.common import constants +from delfin.drivers import driver +from delfin.drivers.pure.flasharray import rest_handler, consts +from delfin.i18n import _ + +LOG = log.getLogger(__name__) + + +class PureFlashArrayDriver(driver.StorageDriver): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.rest_handler = rest_handler.RestHandler(**kwargs) + self.rest_handler.login() + + def list_volumes(self, context): + list_volumes = [] + volumes = self.rest_handler.get_volumes() + if volumes: + for volume in volumes: + volume_name = volume.get('name') + total_capacity = int(volume.get('size', + consts.DEFAULT_CAPACITY)) + used_capacity = int(volume.get('volumes', + consts.DEFAULT_CAPACITY)) + volume_dict = { + 'native_volume_id': volume_name, + 'name': volume_name, + 'total_capacity': total_capacity, + 'used_capacity': used_capacity, + 'free_capacity': total_capacity - used_capacity, + 'storage_id': self.storage_id, + 'status': constants.StorageStatus.NORMAL, + 'type': constants.VolumeType.THIN if + volume.get('thin_provisioning') is not None + else constants.VolumeType.THICK + } + list_volumes.append(volume_dict) + return list_volumes + + def add_trap_config(self, context, trap_config): + pass + + def clear_alert(self, context, alert): + pass + + def get_storage(self, context): + storages = self.rest_handler.rest_call( + self.rest_handler.REST_STORAGE_URL) + total_capacity = None + used_capacity = None + if storages: + for storage in storages: + used_capacity = int(storage.get('total', + consts.DEFAULT_CAPACITY)) + total_capacity = int(storage.get('capacity', + consts.DEFAULT_CAPACITY)) + break + raw_capacity = consts.DEFAULT_CAPACITY + disks = self.list_disks(context) + if disks: + for disk in disks: + raw_capacity = raw_capacity + disk.get('capacity') + arrays = self.rest_handler.rest_call(self.rest_handler.REST_ARRAY_URL) + storage_name = None + serial_number = None + version = None + if arrays: + storage_name = arrays.get('array_name') + serial_number = arrays.get('id') + version = arrays.get('version') + model = None + status = constants.StorageStatus.NORMAL + controllers = self.rest_handler.rest_call( + self.rest_handler.REST_CONTROLLERS_URL) + if controllers: + for controller in controllers: + if controller.get('mode') == consts.CONTROLLER_PRIMARY: + model = controller.get('model') + if controller.get('status') != \ + consts.NORMAL_CONTROLLER_STATUS: + status = constants.StorageStatus.ABNORMAL + if not all((storages, arrays, controllers)): + LOG.error('get_storage error, Unable to obtain data.') + raise exception.StorageBackendException('Unable to obtain data') + storage_result = { + 'model': model, + 'total_capacity': total_capacity, + 'raw_capacity': raw_capacity, + 'used_capacity': used_capacity, + 'free_capacity': total_capacity - used_capacity, + 'vendor': 'PURE', + 'name': storage_name, + 'serial_number': serial_number, + 'firmware_version': version, + 'status': status + } + return storage_result + + def list_alerts(self, context, query_para=None): + alerts = self.rest_handler.rest_call(self.rest_handler.REST_ALERTS_URL) + alerts_list = [] + if alerts: + for alert in alerts: + alerts_model = dict() + opened = alert.get('opened') + time_difference = time.mktime( + time.localtime()) - time.mktime(time.gmtime()) + timestamp = (int(datetime.datetime.strptime + (opened, '%Y-%m-%dT%H:%M:%SZ').timestamp() + + time_difference) * + consts.DEFAULT_LIST_ALERTS_TIME_CONVERSION) if\ + opened is not None else None + if query_para is not None: + try: + if timestamp is None or timestamp \ + < int(query_para.get('begin_time')) or \ + timestamp > int(query_para.get('end_time')): + continue + except Exception as e: + LOG.error(e) + alerts_model['occur_time'] = timestamp + alerts_model['alert_id'] = alert.get('id') + alerts_model['severity'] = consts.SEVERITY_MAP.get( + alert.get('current_severity'), + constants.Severity.NOT_SPECIFIED) + alerts_model['category'] = constants.Category.FAULT + component_name = alert.get('component_name') + alerts_model['location'] = component_name + alerts_model['type'] = constants.EventType.EQUIPMENT_ALARM + alerts_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE + event = alert.get('event') + alerts_model['alert_name'] = event + alerts_model['match_key'] = hashlib.md5(str(alert.get('id')). + encode()).hexdigest() + alerts_model['description'] = '({}:{}): {}'. \ + format(alert.get('component_type'), component_name, event) + alerts_list.append(alerts_model) + return alerts_list + + @staticmethod + def parse_alert(context, alert): + try: + alert_model = dict() + alert_model['alert_id'] = alert.get(consts.PARSE_ALERT_ALERT_ID) + alert_model['severity'] = consts.PARSE_ALERT_SEVERITY_MAP.get( + alert.get(consts.PARSE_ALERT_SEVERITY), + constants.Severity.NOT_SPECIFIED) + alert_model['category'] = constants.Category.FAULT + alert_model['occur_time'] = utils.utcnow_ms() + alert_model['description'] = '({}:{}): {}'.format(alert.get( + consts.PARSE_ALERT_STORAGE_NAME), + alert.get(consts.PARSE_ALERT_CONTROLLER_NAME), + alert.get(consts.PARSE_ALERT_DESCRIPTION)) + alert_model['location'] = alert.get( + consts.PARSE_ALERT_CONTROLLER_NAME) + alert_model['type'] = constants.EventType.EQUIPMENT_ALARM + alert_model['resource_type'] = constants.DEFAULT_RESOURCE_TYPE + alert_model['alert_name'] = alert.get( + consts.PARSE_ALERT_ALERT_NAME) + alert_model['sequence_number'] = alert.get( + consts.PARSE_ALERT_ALERT_ID) + alert_model['match_key'] = hashlib.md5(str(alert.get( + consts.PARSE_ALERT_ALERT_ID)).encode()).hexdigest() + return alert_model + except Exception as e: + LOG.error(e) + msg = (_("Failed to build alert model as some attributes missing")) + raise exception.InvalidResults(msg) + + def list_controllers(self, context): + list_controllers = [] + controllers = self.rest_handler.rest_call( + self.rest_handler.REST_CONTROLLERS_URL) + hardware = self.get_hardware() + if controllers: + for controller in controllers: + controllers_dict = dict() + controller_name = controller.get('name') + controllers_dict['name'] = controller_name + controllers_dict['status'] = consts.CONTROLLER_STATUS_MAP.get( + hardware.get(controller_name, {}).get('status'), + constants.ControllerStatus.UNKNOWN) + controllers_dict['soft_version'] = controller.get('version') + controllers_dict['storage_id'] = self.storage_id + controllers_dict['native_controller_id'] = controller_name + controllers_dict['location'] = controller_name + list_controllers.append(controllers_dict) + return list_controllers + + def list_disks(self, context): + hardware_dict = self.get_hardware() + list_disks = [] + disks = self.rest_handler.rest_call(self.rest_handler.REST_DISK_URL) + if disks: + for disk in disks: + disk_type = disk.get('type') + if disk_type == consts.DISK_TYPE_NVRAM or disk_type is None: + continue + disk_dict = dict() + drive_name = disk.get('name') + disk_dict['name'] = drive_name + physical_type = disk_type.lower() if disk_type is not None \ + else None + disk_dict['physical_type'] = physical_type \ + if physical_type in constants.DiskPhysicalType.ALL else \ + constants.DiskPhysicalType.UNKNOWN + disk_dict['status'] = consts.DISK_STATUS_MAP. \ + get(disk.get('status'), constants.DiskStatus.OFFLINE) + disk_dict['storage_id'] = self.storage_id + disk_dict['capacity'] = int(disk.get('capacity', + consts.DEFAULT_CAPACITY)) + hardware_object = hardware_dict.get(drive_name, {}) + speed = hardware_object.get('speed') + disk_dict['speed'] = int(speed) if speed is not None else None + disk_dict['model'] = hardware_object.get('model') + disk_dict['serial_number'] = hardware_object. \ + get('serial_number') + disk_dict['native_disk_id'] = drive_name + disk_dict['location'] = drive_name + disk_dict['manufacturer'] = "PURE" + disk_dict['firmware'] = "" + list_disks.append(disk_dict) + return list_disks + + def get_hardware(self): + hardware_dict = dict() + hardware = self.rest_handler.rest_call( + self.rest_handler.REST_HARDWARE_URL) + if hardware: + for hardware_value in hardware: + hardware_map = dict() + hardware_map['speed'] = hardware_value.get('speed') + hardware_map['serial_number'] = hardware_value.get('serial') + hardware_map['model'] = hardware_value.get('model') + hardware_map['status'] = hardware_value.get('status') + hardware_dict[hardware_value.get('name')] = hardware_map + return hardware_dict + + def list_ports(self, context): + list_ports = [] + networks = self.get_network() + ports = self.get_ports() + hardware_dict = self.rest_handler.rest_call( + self.rest_handler.REST_HARDWARE_URL) + if not hardware_dict: + return list_ports + for hardware in hardware_dict: + hardware_result = dict() + hardware_name = hardware.get('name') + if 'FC' in hardware_name: + hardware_result['type'] = constants.PortType.FC + elif 'ETH' in hardware_name: + hardware_result['type'] = constants.PortType.ETH + elif 'SAS' in hardware_name: + hardware_result['type'] = constants.PortType.SAS + else: + continue + hardware_result['name'] = hardware_name + hardware_result['native_port_id'] = hardware_name + hardware_result['storage_id'] = self.storage_id + hardware_result['location'] = hardware_name + speed = hardware.get('speed') + if speed is None: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.UNKNOWN + elif speed == consts.CONSTANT_ZERO: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.DISCONNECTED + hardware_result['speed'] = speed + else: + hardware_result['connection_status'] = \ + constants.PortConnectionStatus.CONNECTED + hardware_result['speed'] = int(speed) + hardware_result['health_status'] = consts.PORT_STATUS_MAP.get( + hardware.get('status'), constants.PortHealthStatus.UNKNOWN) + port = ports.get(hardware_name) + if port: + hardware_result['wwn'] = port.get('wwn') + network = networks.get(hardware_name) + if network: + hardware_result['mac_address'] = network.get('mac_address') + hardware_result['logical_type'] = network.get('logical_type') + hardware_result['ipv4_mask'] = network.get('ipv4_mask') + hardware_result['ipv4'] = network.get('ipv4') + list_ports.append(hardware_result) + return list_ports + + def get_network(self): + networks_object = dict() + networks = self.rest_handler.rest_call( + self.rest_handler.REST_NETWORK_URL) + if networks: + for network in networks: + network_dict = dict() + network_dict['mac_address'] = network.get('hwaddr') + services_list = network.get('services') + if services_list: + for services in services_list: + network_dict['logical_type'] = services if \ + services in constants.PortLogicalType.ALL else None + break + network_dict['ipv4_mask'] = network.get('netmask') + network_dict['ipv4'] = network.get('address') + network_name = network.get('name').upper() + networks_object[network_name] = network_dict + return networks_object + + def get_ports(self): + ports_dict = dict() + ports = self.rest_handler.rest_call(self.rest_handler.REST_PORT_URL) + if ports: + for port in ports: + port_dict = dict() + port_name = port.get('name') + wwn = port.get('wwn') + port_dict['wwn'] = self.get_splice_wwn(wwn) \ + if wwn is not None else port.get('iqn') + ports_dict[port_name] = port_dict + return ports_dict + + @staticmethod + def get_splice_wwn(wwn): + wwn_list = list(wwn) + wwn_splice = wwn_list[0] + for serial in range(1, len(wwn_list)): + if serial % consts.SPLICE_WWN_SERIAL == consts.CONSTANT_ZERO: + wwn_splice = '{}{}'.format(wwn_splice, consts.SPLICE_WWN_COLON) + wwn_splice = '{}{}'.format(wwn_splice, wwn_list[serial]) + return wwn_splice + + def list_storage_pools(self, context): + return [] + + def remove_trap_config(self, context, trap_config): + pass + + def reset_connection(self, context, **kwargs): + self.rest_handler.logout() + self.rest_handler.login() + + @staticmethod + def get_access_url(): + return 'https://{ip}' diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py new file mode 100644 index 000000000..462fd4b99 --- /dev/null +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -0,0 +1,103 @@ +import six +from oslo_log import log as logging + +from delfin import exception, cryptor +from delfin.drivers.pure.flasharray import consts +from delfin.drivers.utils.rest_client import RestClient + +LOG = logging.getLogger(__name__) + + +class RestHandler(RestClient): + REST_STORAGE_URL = '/api/1.17/array?space=true' + REST_ARRAY_URL = '/api/1.17/array' + REST_VOLUME_URL = '/api/1.17/volume?space=true&limit=500&token=' \ + 'aWQgPSA5ODA1Mg==' + REST_VOLUME_TOKEN_URL = '/api/1.17/volume?space=true&limit=20&token=' + REST_PORT_URL = '/api/1.17/port' + REST_NETWORK_URL = '/api/1.17/network' + REST_DISK_URL = '/api/1.17/drive' + REST_HARDWARE_URL = '/api/1.17/hardware' + REST_CONTROLLERS_URL = '/api/1.17/array?controllers=true' + REST_ALERTS_URL = '/api/1.17/message?flagged=true&open=true' + REST_AUTH_URL = '/api/1.17/auth/apitoken' + REST_SESSION_URL = '/api/1.17/auth/session' + + def __init__(self, **kwargs): + super(RestHandler, self).__init__(**kwargs) + + def login(self): + try: + data = {'username': self.rest_username, 'password': cryptor.decode( + self.rest_password)} + self.init_http_head() + token_res = self.do_call(RestHandler.REST_AUTH_URL, data, + method='POST') + if token_res.json().get('msg') == consts.LOGIN_PASSWORD_ERR: + LOG.error("Login error, Obtaining the token is abnormal. " + "status_code:%s, URL: %s", + token_res.status_code, RestHandler.REST_AUTH_URL) + raise exception.InvalidUsernameOrPassword( + 'Obtaining the token is abnormal') + if token_res.status_code != consts.SUCCESS_STATUS_CODE or not \ + token_res.json().get('api_token'): + LOG.error("Login error, Obtaining the token is abnormal. " + "status_code:%s, URL: %s", + token_res.status_code, RestHandler.REST_AUTH_URL) + raise exception.StorageBackendException( + 'Obtaining the token is abnormal') + session_res = self.do_call(RestHandler.REST_SESSION_URL, + token_res.json(), method='POST') + if session_res.status_code != consts.SUCCESS_STATUS_CODE or not \ + session_res.json().get('username'): + LOG.error("Login error, Obtaining the session is abnormal." + "status_code:%s, URL: %s", session_res.status_code, + RestHandler.REST_SESSION_URL) + raise exception.StorageBackendException( + 'Obtaining the session is abnormal.') + except Exception as e: + LOG.error("Login error: %s", six.text_type(e)) + raise e + finally: + data = None + token_res = None + + def logout(self): + res = self.do_call(RestHandler.REST_SESSION_URL, None, method='DELETE') + if res.status_code != consts.SUCCESS_STATUS_CODE\ + or not res.json().get('username'): + LOG.error("Logout error, Deleting a Token Exception." + "status_code:%s, URL: %s", + res.status_code, RestHandler.REST_SESSION_URL) + raise exception.StorageBackendException(res.text) + + def rest_call(self, url, data=None, method='GET'): + result_json = None + res = self.do_call(url, data, method) + if res.status_code == consts.SUCCESS_STATUS_CODE: + result_json = res.json() + elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: + self.login() + the_second_time_res = self.do_call(url, data, method) + if the_second_time_res.status_code == consts.SUCCESS_STATUS_CODE: + result_json = the_second_time_res.json() + return result_json + + def get_volumes(self, url=REST_VOLUME_URL, data=None, volume_list=None, + count=consts.DEFAULT_COUNT_GET_VOLUMES_INFO): + if volume_list is None: + volume_list = [] + res = self.do_call(url, data, 'GET') + if res.status_code == consts.SUCCESS_STATUS_CODE: + result_json = res.json() + volume_list.extend(result_json) + next_token = res.headers.get(consts.CUSTOM_TOKEN) + if next_token: + url = '%s%s' % (RestHandler.REST_VOLUME_TOKEN_URL, next_token) + self.get_volumes(url, data, volume_list) + elif res.status_code == consts.PERMISSION_DENIED_STATUS_CODE: + self.login() + if count < consts.RE_LOGIN_TIMES: + count = count + consts.CONSTANT_ONE + self.get_volumes(url, data, volume_list, count) + return volume_list diff --git a/delfin/tests/unit/drivers/pure/__init__.py b/delfin/tests/unit/drivers/pure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/tests/unit/drivers/pure/flasharray/__init__.py b/delfin/tests/unit/drivers/pure/flasharray/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py new file mode 100644 index 000000000..ab0a332bb --- /dev/null +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -0,0 +1,355 @@ +import sys +from unittest import TestCase, mock + +import six +from oslo_log import log + +sys.modules['delfin.cryptor'] = mock.Mock() +from delfin import context +from delfin.drivers.pure.flasharray.rest_handler import RestHandler +from delfin.drivers.pure.flasharray.pure_flasharray import PureFlashArrayDriver +LOG = log.getLogger(__name__) + +ACCESS_INFO = { + "rest": { + "host": "10.0.0.1", + "port": 8443, + "username": "user", + "password": "pass" + } +} + +volumes_info = [ + { + "total": 116272464547, + "name": "oracl_ail", + "system": "", + "snapshots": 0, + "volumes": 116272464547, + "data_reduction": 1.82656654775252, + "size": 2156324555567, + "shared_space": "", + "thin_provisioning": 0.9225557589632, + "total_reduction": 18.92245232244555 + }, + { + "total": 0, + "name": "wxt1", + "system": "", + "snapshots": 0, + "volumes": 0, + "data_reduction": 1, + "size": 1073741824, + "shared_space": "", + "thin_provisioning": 1, + "total_reduction": 1 + } +] + +pool_info = [ + { + "name": "lktest", + "volumes": [ + "oracl_ail", + "wxt1", + "lktest/lk301", + "lktest/lk401", + "lktest/lk501", + ] + }, + { + "name": "ethanTestVG", + "volumes": [ + + ] + } +] +volume_info = { + "created": "2016-05-02T20:36:20Z", + "name": "oracl_ail", + "serial": "Fedd3455666y", + "size": 1073740124, + "source": "" +} +volume_info_two = { + "created": "2016-05-02T20:36:20Z", + "name": "wxt1", + "serial": "Fedd3475666y", + "size": 1073740124, + "source": "" +} +storage_info = [ + { + "parity": "0.996586544522471235", + "provisioned": "20869257625600", + "hostname": "FA-m20", + "system": 0, + "snapshots": 0, + "volumes": 227546215656, + "data_reduction": 1, + "capacity": 122276719419392, + "total": 324829845504, + "shared_space": 97544451659, + "thin_provisioning": 0.9526445631455244, + "total_reduction": 64.152236458789225 + } +] +storage_id_info = { + "array_name": "pure01", + "id": "dlmkk15xcfdf4v5", + "revision": "2016-20-29mfmkkk", + "version": "4.6.7" +} +alerts_info = [ + { + "category": "array", + "code": 42, + "actual": "", + "opened": "2018-05-12T10:55:21Z", + "component_type": "hardware", + "event": "failure", + "current_severity": "warning", + "details": "", + "expected": "", + "id": 135, + "component_name": "ct1.eth0" + }, + { + "category": "array", + "code": 13, + "actual": "", + "opened": "2018-05-12T10:55:21Z", + "component_type": "process", + "event": "server unreachable", + "current_severity": "warning", + "details": "", + "expected": "", + "id": 10088786, + "component_name": "ct1.ntpd" + } +] +parse_alert_info = { + '1.3.6.1.2.1.1.3.0': '30007589', + '1.3.6.1.4.1.40482.3.7': '2', + '1.3.6.1.4.1.40482.3.6': 'server error', + '1.3.6.1.4.1.40482.3.3': 'cto', + '1.3.6.1.4.1.40482.3.5': 'cto.server error' +} +controllers_info = [ + { + "status": "ready", + "name": "CT0", + "version": "5.3.0", + "mode": "primary", + "model": "FA-m20r2", + "type": "array_controller" + }, + { + "status": "ready", + "name": "CT1", + "version": "5.3.0", + "mode": "secondary", + "model": "FA-m20r2", + "type": "array_controller" + } +] +hardware_info = [ + { + "details": "", + "identify": "off", + "index": 0, + "name": "CTO.FC1", + "slot": "", + "speed": 0, + "status": "ok", + "temperature": "" + }, + { + "details": "", + "identify": "", + "index": 0, + "name": "CTO.ETH15", + "slot": 0, + "speed": 1000000, + "status": "ok", + "temperature": "" + } +] +drive_info = [ + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY1", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + }, + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY2", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + }, + { + "status": "healthy", + "protocol": "SAS", + "name": "CH0.BAY3", + "last_evac_completed": "1970-01-01T00:00:00Z", + "details": "", + "capacity": 1027895542547, + "type": "SSD", + "last_failure": "1970-01-01T00:00:00Z" + } +] +port_info = [ + { + "name": "CTO.FC1", + "failover": "", + "iqn": "iqn.2016-11-01.com.pure", + "portal": "100.12.253.23:4563", + "wwn": "43ddff45ggg4rty", + "nqn": "" + }, + { + "name": "CTO.ETH15", + "failover": "", + "iqn": "iqn.2016-11-01.com.pure", + "portal": "100.12.253.23:4563", + "wwn": None, + "nqn": None + } +] +port_network_info = [ + { + "name": "CTO.FC1", + "address": "45233662jksndj", + "speed": 12000, + "netmask": "100.12.253.23:4563", + "wwn": "43ddff45ggg4rty", + "nqn": None, + "services": [ + "management" + ] + }, + { + "name": "CTO.ETH15", + "address": "45233662jksndj", + "speed": 13000, + "netmask": "100.12.253.23:4563", + "wwn": None, + "nqn": None, + "services": [ + "management" + ] + } +] +pools_info = [ + { + "total": "", + "name": "lktest", + "snapshots": "", + "volumes": 0, + "data_reduction": 1, + "size": 5632155322, + "thin_provisioning": 1, + "total_reduction": 1 + }, + { + "total": "", + "name": "ethanTestVG", + "snapshots": "", + "volumes": 0, + "data_reduction": 1, + "size": 5632155322, + "thin_provisioning": 1, + "total_reduction": 1 + } +] +reset_connection_info = { + "username": "username", + "status": 200 +} + + +def create_driver(): + RestHandler.login = mock.Mock( + return_value={None}) + return PureFlashArrayDriver(**ACCESS_INFO) + + +class test_PureFlashArrayDriver(TestCase): + driver = create_driver() + + def test_init(self): + RestHandler.login = mock.Mock( + return_value={""}) + PureFlashArrayDriver(**ACCESS_INFO) + + def test_list_volumes(self): + RestHandler.get_volumes = mock.Mock( + side_effect=[volumes_info]) + volume = self.driver.list_volumes(context) + self.assertEqual(volume[0]['native_volume_id'], + pool_info[0].get('volumes')[0]) + + def test_get_storage(self): + RestHandler.rest_call = mock.Mock( + side_effect=[storage_info, hardware_info, drive_info, + storage_id_info, controllers_info]) + storage_object = self.driver.get_storage(context) + self.assertEqual(storage_object.get('name'), + storage_id_info.get('array_name')) + + def test_list_alerts(self): + RestHandler.rest_call = mock.Mock( + side_effect=[alerts_info]) + list_alerts = self.driver.list_alerts(context) + self.assertEqual(list_alerts[0].get('alert_id'), + alerts_info[0].get('id')) + + def test_parse_alert(self): + parse_alert = self.driver.parse_alert(context, parse_alert_info) + self.assertEqual(parse_alert.get('alert_id'), + parse_alert_info.get('1.3.6.1.2.1.1.3.0')) + + def test_list_controllers(self): + RestHandler.rest_call = mock.Mock( + side_effect=[controllers_info, hardware_info]) + list_controllers = self.driver.list_controllers(context) + self.assertEqual(list_controllers[0].get('name'), + controllers_info[0].get('name')) + + def test_list_disks(self): + RestHandler.rest_call = mock.Mock( + side_effect=[hardware_info, drive_info]) + list_disks = self.driver.list_disks(context) + self.assertEqual(list_disks[0].get('name'), + drive_info[0].get('name')) + + def test_list_ports(self): + RestHandler.rest_call = mock.Mock( + side_effect=[port_network_info, port_info, hardware_info]) + list_ports = self.driver.list_ports(context) + self.assertEqual(list_ports[0].get('name'), + hardware_info[0].get('name')) + + def test_list_storage_pools(self): + list_storage_pools = self.driver.list_storage_pools(context) + self.assertEqual(list_storage_pools, []) + + def test_reset_connection(self): + RestHandler.logout = mock.Mock(side_effect=None) + RestHandler.login = mock.Mock(side_effect=None) + username = None + try: + self.driver.reset_connection(context) + except Exception as e: + LOG.error("test_reset_connection error: %s", six.text_type(e)) + username = reset_connection_info.get('username') + self.assertEqual(username, None) diff --git a/setup.py b/setup.py index 630ca5258..ba8b5a571 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,8 @@ 'ibm storwize_svc = delfin.drivers.ibm.storwize_svc.storwize_svc:StorwizeSVCDriver', 'ibm ds8k = delfin.drivers.ibm.ds8k.ds8k:DS8KDriver', 'netapp cmode = delfin.drivers.netapp.dataontap.cluster_mode:NetAppCmodeDriver', - 'hitachi hnas = delfin.drivers.hitachi.hnas.hds_nas:HitachiHNasDriver' + 'hitachi hnas = delfin.drivers.hitachi.hnas.hds_nas:HitachiHNasDriver', + 'pure flasharray = delfin.drivers.pure.flasharray.pure_flasharray:PureFlashArrayDriver' ] }, ) From d7ea0652a85115c5a3538d9a24d53e77a2070ebe Mon Sep 17 00:00:00 2001 From: Administrator Date: Thu, 6 Jan 2022 14:07:02 +0800 Subject: [PATCH 69/69] copyright information --- delfin/drivers/pure/flasharray/consts.py | 2 +- delfin/drivers/pure/flasharray/pure_flasharray.py | 13 +++++++++++++ delfin/drivers/pure/flasharray/rest_handler.py | 13 +++++++++++++ .../drivers/pure/flasharray/test_pure_flasharray.py | 13 +++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/delfin/drivers/pure/flasharray/consts.py b/delfin/drivers/pure/flasharray/consts.py index 65d6c5976..7226a1c9f 100644 --- a/delfin/drivers/pure/flasharray/consts.py +++ b/delfin/drivers/pure/flasharray/consts.py @@ -1,4 +1,4 @@ -# Copyright 2021 The SODA Authors. +# Copyright 2022 The SODA Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/delfin/drivers/pure/flasharray/pure_flasharray.py b/delfin/drivers/pure/flasharray/pure_flasharray.py index 31db05b6b..92feb48cf 100644 --- a/delfin/drivers/pure/flasharray/pure_flasharray.py +++ b/delfin/drivers/pure/flasharray/pure_flasharray.py @@ -1,3 +1,16 @@ +# Copyright 2022 The SODA Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import datetime import hashlib import time diff --git a/delfin/drivers/pure/flasharray/rest_handler.py b/delfin/drivers/pure/flasharray/rest_handler.py index 462fd4b99..39570041a 100644 --- a/delfin/drivers/pure/flasharray/rest_handler.py +++ b/delfin/drivers/pure/flasharray/rest_handler.py @@ -1,3 +1,16 @@ +# Copyright 2022 The SODA Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import six from oslo_log import log as logging diff --git a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py index ab0a332bb..6c1046088 100644 --- a/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py +++ b/delfin/tests/unit/drivers/pure/flasharray/test_pure_flasharray.py @@ -1,3 +1,16 @@ +# Copyright 2022 The SODA Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import sys from unittest import TestCase, mock