From 05c79de08f4c867583d65c69c1e19b44d2e83a9c Mon Sep 17 00:00:00 2001 From: mprabhu-nokia <66807480+mprabhu-nokia@users.noreply.github.com> Date: Tue, 10 Nov 2020 19:23:07 -0500 Subject: [PATCH] PSUd changes to compute power-budget for Modular chassis (#104) PSUd changes to computer power-budget for Modular chassis HLD: Azure/SONiC#646 PSUd will introduce power requirements calculations. Platform APIs are introduced to provide consumers and total consumed power. Number of PSUs will help provide total supplied power **Output of STATE-DB:** ``` "CHASSIS_INFO|chassis_power_budget 1": { "expireat": 1603182970.639244, "ttl": -0.001, "type": "hash", "value": { "SUPERVISOR consumed_power": "80.0", "FABRIC-CARD consumed_power": "185.0", "FAN consumed_power": "999", "LINE-CARD consumed_power": "1000.0", "PSU supplied_power": "9000.0" } }, ``` --- .gitignore | 3 +- sonic-psud/pytest.ini | 2 + sonic-psud/scripts/psud | 154 +++++++++++++++++++++++- sonic-psud/setup.cfg | 2 + sonic-psud/setup.py | 10 ++ sonic-psud/tests/__init__.py | 0 sonic-psud/tests/mock_device_base.py | 11 ++ sonic-psud/tests/mock_platform.py | 116 +++++++++++++++++++ sonic-psud/tests/mock_swsscommon.py | 27 +++++ sonic-psud/tests/test_psud.py | 167 +++++++++++++++++++++++++++ 10 files changed, 488 insertions(+), 4 deletions(-) create mode 100644 sonic-psud/pytest.ini create mode 100644 sonic-psud/setup.cfg create mode 100644 sonic-psud/tests/__init__.py create mode 100644 sonic-psud/tests/mock_device_base.py create mode 100644 sonic-psud/tests/mock_platform.py create mode 100644 sonic-psud/tests/mock_swsscommon.py create mode 100644 sonic-psud/tests/test_psud.py diff --git a/.gitignore b/.gitignore index 2d9c861eb..aebbe29ba 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,9 @@ # Compiled code which doesn't end in '.pyc' sonic-thermalctld/scripts/thermalctldc sonic-chassisd/scripts/chassisdc +sonic-psud/scripts/psudc # Unit test / coverage reports coverage.xml .coverage -htmlcov/ +htmlcov/ \ No newline at end of file diff --git a/sonic-psud/pytest.ini b/sonic-psud/pytest.ini new file mode 100644 index 000000000..83b74d373 --- /dev/null +++ b/sonic-psud/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=scripts --cov-report html --cov-report term --cov-report xml diff --git a/sonic-psud/scripts/psud b/sonic-psud/scripts/psud index 0bc7526ce..8a41ff0fb 100644 --- a/sonic-psud/scripts/psud +++ b/sonic-psud/scripts/psud @@ -14,11 +14,21 @@ try: import sys import threading - from sonic_py_common import daemon_base - from swsscommon import swsscommon + from sonic_py_common import daemon_base, logger + except ImportError as e: raise ImportError (str(e) + " - required module not found") +try: + from swsscommon import swsscommon +except ImportError as e: + from tests import mock_swsscommon as swsscommon + +try: + from sonic_platform_base.device_base import DeviceBase +except ImportError as e: + from tests.mock_device_base import DeviceBase + # # Constants ==================================================================== # @@ -32,6 +42,12 @@ CHASSIS_INFO_TABLE = 'CHASSIS_INFO' CHASSIS_INFO_KEY_TEMPLATE = 'chassis {}' CHASSIS_INFO_PSU_NUM_FIELD = 'psu_num' +CHASSIS_INFO_POWER_CONSUMER_FIELD = 'Consumed Power {}' +CHASSIS_INFO_POWER_SUPPLIER_FIELD = 'Supplied Power {}' +CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD = 'Total Consumed Power' +CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD = 'Total Supplied Power' +CHASSIS_INFO_POWER_KEY_TEMPLATE = 'chassis_power_budget {}' + PSU_INFO_TABLE = 'PSU_INFO' PSU_INFO_KEY_TEMPLATE = 'PSU {}' PSU_INFO_PRESENCE_FIELD = 'presence' @@ -133,8 +149,111 @@ def log_on_status_changed(logger, normal_status, normal_log, abnormal_log): else: logger.log_warning(abnormal_log) - # +# PSU Chassis Info ========================================================== +# +class PsuChassisInfo(logger.Logger): + + def __init__(self, log_identifier, chassis): + """ + Constructor for PsuChassisInfo + :param chassis: Object representing a platform chassis + """ + super(PsuChassisInfo, self).__init__(log_identifier) + + self.chassis = chassis + self.master_status_good = True + self.total_consumed_power = 0.0 + self.total_supplied_power = 0.0 + + def run_power_budget(self, chassis_tbl): + self.total_supplied_power = 0.0 + self.total_consumed_power = 0.0 + total_supplied_power = 0.0 + total_fan_consumed_power = 0.0 + total_module_consumed_power = 0.0 + + dict_index = 0 + total_entries_len = 2 #For total supplied and consumed + dict_len = self.chassis.get_num_psus() +\ + self.chassis.get_num_fan_drawers() +\ + self.chassis.get_num_modules() + \ + total_entries_len + + fvs = swsscommon.FieldValuePairs(dict_len) + + for index, psu in enumerate(self.chassis.get_all_psus()): + presence = try_get(psu.get_presence) + if not presence: + continue + + power_good = try_get(psu.get_powergood_status) + if not power_good: + continue + + name = try_get(psu.get_name, 'PSU {}'.format(index + 1)) + supplied_power = try_get(psu.get_maximum_supplied_power, 0.0) + total_supplied_power = total_supplied_power + supplied_power + fvs[dict_index] = (CHASSIS_INFO_POWER_SUPPLIER_FIELD.format(name), str(supplied_power)) + dict_index += 1 + + for index, power_consumer in enumerate(self.chassis.get_all_fan_drawers()): + presence = try_get(power_consumer.get_presence) + if not presence: + continue + + name = try_get(power_consumer.get_name, 'FAN-DRAWER {}'.format(index)) + fan_drawer_power = try_get(power_consumer.get_maximum_consumed_power, 0.0) + total_fan_consumed_power = total_fan_consumed_power + fan_drawer_power + fvs[dict_index] = (CHASSIS_INFO_POWER_CONSUMER_FIELD.format(name), str(fan_drawer_power)) + dict_index += 1 + + for index, power_consumer in enumerate(self.chassis.get_all_modules()): + presence = try_get(power_consumer.get_presence) + if not presence: + continue + + name = try_get(power_consumer.get_name, 'MODULE {}'.format(index)) + module_power = try_get(power_consumer.get_maximum_consumed_power, 0.0) + total_module_consumed_power = total_module_consumed_power + module_power + fvs[dict_index] = (CHASSIS_INFO_POWER_CONSUMER_FIELD.format(name), str(module_power)) + dict_index += 1 + + #Record total supplied and consumed power + self.total_supplied_power = total_supplied_power + self.total_consumed_power = total_fan_consumed_power + total_module_consumed_power + + #Record in state DB in chassis table + fvs[dict_index] = (CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD, str(self.total_supplied_power)) + fvs[dict_index + 1] = (CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD, str(self.total_consumed_power)) + chassis_tbl.set(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1), fvs) + + def update_master_status(self): + if not self.total_supplied_power or not self.total_consumed_power: + if self.master_status_good is not True: + self.master_status_good = True + return False + + master_status_good = (self.total_consumed_power < self.total_supplied_power) + if master_status_good == self.master_status_good: + return False + + self.master_status_good = master_status_good + + return True + + def _set_psu_master_led(self, master_status): + try: + try: + from sonic_platform.psu import Psu + except ImportError as e: + from tests.mock_platform import MockPsu as Psu + + color = DeviceBase.STATUS_LED_COLOR_GREEN if master_status else DeviceBase.STATUS_LED_COLOR_RED + Psu.set_status_master_led(color) + except NotImplementedError as e: + pass + # PSU status =================================================================== # @@ -216,6 +335,7 @@ class DaemonPsud(daemon_base.DaemonBase): self.stop = threading.Event() self.psu_status_dict = {} self.fan_tbl = None + self.psu_chassis_info = None # Signal handler def signal_handler(self, sig, frame): @@ -263,6 +383,7 @@ class DaemonPsud(daemon_base.DaemonBase): fvs = swsscommon.FieldValuePairs([(CHASSIS_INFO_PSU_NUM_FIELD, str(psu_num))]) chassis_tbl.set(CHASSIS_INFO_KEY_TEMPLATE.format(1), fvs) + # Start main loop self.log_info("Start daemon main loop") @@ -271,6 +392,10 @@ class DaemonPsud(daemon_base.DaemonBase): self.update_psu_data(psu_tbl) self._update_led_color(psu_tbl) + if platform_chassis is not None and platform_chassis.is_modular_chassis(): + self.update_psu_chassis_info(chassis_tbl) + self.update_master_led_color(chassis_tbl) + self.log_info("Stop daemon main loop") # Delete all the information from DB and then exit @@ -278,6 +403,7 @@ class DaemonPsud(daemon_base.DaemonBase): psu_tbl._del(PSU_INFO_KEY_TEMPLATE.format(psu_index)) chassis_tbl._del(CHASSIS_INFO_KEY_TEMPLATE.format(1)) + chassis_tbl._del(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) self.log_info("Shutting down...") @@ -427,6 +553,28 @@ class DaemonPsud(daemon_base.DaemonBase): ]) self.fan_tbl.set(fan_name, fvs) + def update_psu_chassis_info(self, chassis_tbl): + if not platform_chassis: + return + + if not self.psu_chassis_info: + self.psu_chassis_info = PsuChassisInfo(SYSLOG_IDENTIFIER, platform_chassis) + + self.psu_chassis_info.run_power_budget(chassis_tbl) + + def update_master_led_color(self, chassis_tbl): + if not platform_chassis or not self.psu_chassis_info: + return + + psu_chassis_info = self.psu_chassis_info + if psu_chassis_info.update_master_status(): + log_on_status_changed(self, psu_chassis_info.master_status_good, + 'PSU supplied power warning cleared: supplied power is back to normal.', + 'PSU supplied power warning: {}W supplied-power less than {}W consumed-power'.format( + psu_chassis_info.total_supplied_power, psu_chassis_info.total_consumed_power) + ) + psu_chassis_info._set_psu_master_led(psu_chassis_info.master_status_good) + # # Main ========================================================================= diff --git a/sonic-psud/setup.cfg b/sonic-psud/setup.cfg new file mode 100644 index 000000000..b7e478982 --- /dev/null +++ b/sonic-psud/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/sonic-psud/setup.py b/sonic-psud/setup.py index 3a465b901..4f8b77216 100644 --- a/sonic-psud/setup.py +++ b/sonic-psud/setup.py @@ -10,12 +10,21 @@ url='https://github.com/Azure/sonic-platform-daemons', maintainer='Kevin Wang', maintainer_email='kevinw@mellanox.com', + packages=[ + 'tests' + ], scripts=[ 'scripts/psud', ], setup_requires= [ + 'pytest-runner', 'wheel' ], + tests_require = [ + 'pytest', + 'mock>=2.0.0', + 'pytest-cov' + ], classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: No Input/Output (Daemon)', @@ -29,4 +38,5 @@ 'Topic :: System :: Hardware', ], keywords='sonic SONiC psu PSU daemon psud PSUD', + test_suite='setup.get_test_suite' ) diff --git a/sonic-psud/tests/__init__.py b/sonic-psud/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sonic-psud/tests/mock_device_base.py b/sonic-psud/tests/mock_device_base.py new file mode 100644 index 000000000..272a99482 --- /dev/null +++ b/sonic-psud/tests/mock_device_base.py @@ -0,0 +1,11 @@ +class DeviceBase(): + #Device-types + DEVICE_TYPE_PSU = "PSU" + DEVICE_TYPE_FAN = "FAN" + DEVICE_TYPE_FANDRAWER = "FAN-DRAWER" + + #LED colors + STATUS_LED_COLOR_GREEN = "green" + STATUS_LED_COLOR_AMBER = "amber" + STATUS_LED_COLOR_RED = "red" + STATUS_LED_COLOR_OFF = "off" diff --git a/sonic-psud/tests/mock_platform.py b/sonic-psud/tests/mock_platform.py new file mode 100644 index 000000000..701bb1b70 --- /dev/null +++ b/sonic-psud/tests/mock_platform.py @@ -0,0 +1,116 @@ +from .mock_device_base import DeviceBase + +class MockDevice: + def __init__(self): + self.name = None + self.presence = True + self.model = 'Module Model' + self.serial = 'Module Serial' + + def get_name(self): + return self.name + + def set_presence(self, presence): + self.presence = presence + + def get_presence(self): + return self.presence + + def get_model(self): + return self.model + + def get_serial(self): + return self.serial + +class MockPsu(MockDevice): + + psu_master_led_color = DeviceBase.STATUS_LED_COLOR_OFF + + def __init__(self, psu_presence, psu_status, psu_name): + self.name = psu_name + self.presence = True + self.psu_status = psu_status + + def get_powergood_status(self): + return self.psu_status + + def set_status(self, status): + self.psu_status = status + + def set_maximum_supplied_power(self, supplied_power): + self.max_supplied_power = supplied_power + + def get_maximum_supplied_power(self): + return self.max_supplied_power + + @classmethod + def set_status_master_led(cls, color): + cls.psu_master_led_color = color + + @classmethod + def get_status_master_led(cls): + return cls.psu_master_led_color + +class MockFanDrawer(MockDevice): + def __init__(self, fan_drawer_presence, fan_drawer_status, fan_drawer_name): + self.name = fan_drawer_name + self.presence = True + self.fan_drawer_status = fan_drawer_status + + def get_status(self): + return self.fan_drawer_status + + def set_status(self, status): + self.fan_drawer_status = status + + def set_maximum_consumed_power(self, consumed_power): + self.max_consumed_power = consumed_power + + def get_maximum_consumed_power(self): + return self.max_consumed_power + +class MockModule(MockDevice): + def __init__(self, module_presence, module_status, module_name): + self.name = module_name + self.presence = True + self.module_status = module_status + + def get_status(self): + return self.module_status + + def set_status(self, status): + self.module_status = status + + def set_maximum_consumed_power(self, consumed_power): + self.max_consumed_power = consumed_power + + def get_maximum_consumed_power(self): + return self.max_consumed_power + +class MockChassis: + + def __init__(self): + self.psu_list = [] + self.fan_drawer_list = [] + self.module_list = [] + + def get_num_psus(self): + return len(self.psu_list) + + def get_all_psus(self): + return self.psu_list + + def get_psu(self, index): + return self.psu_list[index] + + def get_num_fan_drawers(self): + return len(self.fan_drawer_list) + + def get_all_fan_drawers(self): + return self.fan_drawer_list + + def get_num_modules(self): + return len(self.module_list) + + def get_all_modules(self): + return self.module_list diff --git a/sonic-psud/tests/mock_swsscommon.py b/sonic-psud/tests/mock_swsscommon.py new file mode 100644 index 000000000..bd16d7f7e --- /dev/null +++ b/sonic-psud/tests/mock_swsscommon.py @@ -0,0 +1,27 @@ +STATE_DB = '' + + +class Table: + def __init__(self, db, table_name): + self.table_name = table_name + self.mock_dict = {} + + def _del(self, key): + del self.mock_dict[key] + pass + + def set(self, key, fvs): + self.mock_dict[key] = fvs.fv_dict + pass + + def get(self, key): + if key in self.mock_dict: + return self.mock_dict[key] + return None + +class FieldValuePairs(dict): + def __init__(self, len): + self.fv_dict = {} + + def __setitem__(self, key, val_tuple): + self.fv_dict[val_tuple[0]] = val_tuple[1] diff --git a/sonic-psud/tests/test_psud.py b/sonic-psud/tests/test_psud.py new file mode 100644 index 000000000..2080d6fde --- /dev/null +++ b/sonic-psud/tests/test_psud.py @@ -0,0 +1,167 @@ +import os +import sys + +from mock import Mock, MagicMock, patch +from sonic_py_common import daemon_base + +from .mock_platform import MockChassis, MockPsu, MockFanDrawer, MockModule +from .mock_device_base import DeviceBase + +SYSLOG_IDENTIFIER = 'psud_test' +NOT_AVAILABLE = 'N/A' + +daemon_base.db_connect = MagicMock() + +test_path = os.path.dirname(os.path.abspath(__file__)) +modules_path = os.path.dirname(test_path) +scripts_path = os.path.join(modules_path, "scripts") +sys.path.insert(0, modules_path) + +from imp import load_source + +load_source('psud', scripts_path + '/psud') +from psud import * + +CHASSIS_INFO_TABLE = 'CHASSIS_INFO' +CHASSIS_INFO_KEY_TEMPLATE = 'chassis {}' + +CHASSIS_INFO_POWER_CONSUMER_FIELD = 'Consumed Power {}' +CHASSIS_INFO_POWER_SUPPLIER_FIELD = 'Supplied Power {}' +CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD = 'Total Consumed Power' +CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD = 'Total Supplied Power' +CHASSIS_INFO_POWER_KEY_TEMPLATE = 'chassis_power_budget {}' + +def setup_function(): + PsuChassisInfo.log_notice = MagicMock() + PsuChassisInfo.log_warning = MagicMock() + + +def teardown_function(): + PsuChassisInfo.log_notice.reset() + PsuChassisInfo.log_warning.reset() + +#Test cases to cover functionality in PsuChassisInfo class +def test_psuchassis_check_psu_supplied_power(): + chassis = MockChassis() + psu1 = MockPsu(True, True, "PSU 1") + psu1_power = 510.0 + psu1.set_maximum_supplied_power(psu1_power) + chassis.psu_list.append(psu1) + + psu2 = MockPsu(True, True, "PSU 2") + psu2_power = 800.0 + psu2.set_maximum_supplied_power(psu2_power) + chassis.psu_list.append(psu2) + + psu3 = MockPsu(True, True, "PSU 3") + psu3_power = 350.0 + psu3.set_maximum_supplied_power(psu3_power) + chassis.psu_list.append(psu3) + + total_power = psu1_power + psu2_power + psu3_power + state_db = daemon_base.db_connect("STATE_DB") + chassis_tbl = swsscommon.Table(state_db, CHASSIS_INFO_TABLE) + chassis_info = PsuChassisInfo(SYSLOG_IDENTIFIER, chassis) + chassis_info.run_power_budget(chassis_tbl) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + + #Check if supplied power is recorded in DB + assert total_power == float(fvs[CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD]) + + #Check if psu1 is not present + psu1.set_presence(False) + total_power = psu2_power + psu3_power + chassis_info.run_power_budget(chassis_tbl) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + assert total_power == float(fvs[CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD]) + + #Check if psu2 status is NOT_OK + psu2.set_status(False) + total_power = psu3_power + chassis_info.run_power_budget(chassis_tbl) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + assert total_power == float(fvs[CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD]) + +def test_psuchassis_check_consumed_power(): + chassis = MockChassis() + fan_drawer1 = MockFanDrawer(True, True, "FanDrawer 1") + fan_drawer1_power = 510.0 + fan_drawer1.set_maximum_consumed_power(fan_drawer1_power) + chassis.fan_drawer_list.append(fan_drawer1) + + module1 = MockFanDrawer(True, True, "Module 1") + module1_power = 700.0 + module1.set_maximum_consumed_power(module1_power) + chassis.module_list.append(module1) + + total_power = fan_drawer1_power + module1_power + state_db = daemon_base.db_connect("STATE_DB") + chassis_tbl = swsscommon.Table(state_db, CHASSIS_INFO_TABLE) + chassis_info = PsuChassisInfo(SYSLOG_IDENTIFIER, chassis) + chassis_info.run_power_budget(chassis_tbl) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + + #Check if supplied power is recorded in DB + assert total_power == float(fvs[CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD]) + + #Check if fan_drawer1 present + fan_drawer1.set_presence(False) + total_power = module1_power + chassis_info.run_power_budget(chassis_tbl) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + assert total_power == float(fvs[CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD]) + + #Check if module1 present + fan_drawer1.set_presence(True) + module1.set_presence(False) + total_power = fan_drawer1_power + chassis_info.run_power_budget(chassis_tbl) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + assert total_power == float(fvs[CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD]) + +def test_psuchassis_check_power_budget(): + chassis = MockChassis() + psu = MockPsu(True, True, "PSU 1") + psu1_power = 510.0 + psu.set_maximum_supplied_power(psu1_power) + chassis.psu_list.append(psu) + + fan_drawer1 = MockFanDrawer(True, True, "FanDrawer 1") + fan_drawer1_power = 510.0 + fan_drawer1.set_maximum_consumed_power(fan_drawer1_power) + chassis.fan_drawer_list.append(fan_drawer1) + + module1 = MockFanDrawer(True, True, "Module 1") + module1_power = 700.0 + module1.set_maximum_consumed_power(module1_power) + chassis.module_list.append(module1) + + state_db = daemon_base.db_connect("STATE_DB") + chassis_tbl = swsscommon.Table(state_db, CHASSIS_INFO_TABLE) + chassis_info = PsuChassisInfo(SYSLOG_IDENTIFIER, chassis) + + #Check if supplied_power < consumed_power + chassis_info.run_power_budget(chassis_tbl) + if chassis_info.update_master_status(): + chassis_info._set_psu_master_led(chassis_info.master_status_good) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + + assert float(fvs[CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD]) < float(fvs[CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD]) + assert chassis_info.master_status_good == False + assert MockPsu.get_status_master_led() == DeviceBase.STATUS_LED_COLOR_RED + + #Add a PSU + psu = MockPsu(True, True, "PSU 2") + psu2_power = 800.0 + psu.set_maximum_supplied_power(psu2_power) + chassis.psu_list.append(psu) + + #Check if supplied_power > consumed_power + chassis_info.run_power_budget(chassis_tbl) + if chassis_info.update_master_status(): + chassis_info._set_psu_master_led(chassis_info.master_status_good) + fvs = chassis_tbl.get(CHASSIS_INFO_POWER_KEY_TEMPLATE.format(1)) + + assert float(fvs[CHASSIS_INFO_TOTAL_POWER_SUPPLIED_FIELD]) > float(fvs[CHASSIS_INFO_TOTAL_POWER_CONSUMED_FIELD]) + assert chassis_info.master_status_good == True + assert MockPsu.get_status_master_led() == DeviceBase.STATUS_LED_COLOR_GREEN