From fceef2edda86e6b2f69ff5531ef0dd82c34a52ed Mon Sep 17 00:00:00 2001 From: jfeng-arista <98421150+jfeng-arista@users.noreply.github.com> Date: Thu, 15 Jun 2023 15:47:39 -0700 Subject: [PATCH] [chassis][voq] Clear fabric counters queue/port (#2789) Added two clear commands for fabric counters queue and fabric counters port. sonic-clear fabriccountersport sonic-clear fabriccountersqueue By issuing the command, the fabric counters are clear and saved. An example is as follow. # show fabric counters port ASIC PORT STATE IN_CELL IN_OCTET OUT_CELL OUT_OCTET CRC FEC_CORRECTABLE FEC_UNCORRECTABLE SYMBOL_ERR ------ ------ ------- --------- ---------- ---------- ----------- ----- ----------------- ------------------- ---------- ... 0 49 up 1 244 0 0 0 2 2,372,752,496 0 0 50 up 2 315 1 135 0 4 2,522,457,120 4 ... # sonic-clear fabriccountersport Clear and update saved counters port # show fabric counters port ASIC PORT STATE IN_CELL IN_OCTET OUT_CELL OUT_OCTET CRC FEC_CORRECTABLE FEC_UNCORRECTABLE SYMBOL_ERR ------ ------ ------- --------- ---------- ---------- ----------- ----- ----------------- ------------------- ------------ ... 0 49 up 0 0 0 0 0 0 0 0 0 50 up 0 0 0 0 0 0 0 Signed-off-by: Jie Feng --- clear/main.py | 12 +++ scripts/fabricstat | 134 +++++++++++++++++++++++++++++-- tests/fabricstat_test.py | 166 ++++++++++++++++++++++++++++++--------- 3 files changed, 268 insertions(+), 44 deletions(-) diff --git a/clear/main.py b/clear/main.py index 21c87d555b..eda542feb7 100755 --- a/clear/main.py +++ b/clear/main.py @@ -181,6 +181,18 @@ def queuecounters(): command = ["queuestat", "-c"] run_command(command) +@cli.command() +def fabriccountersqueue(): + """Clear fabric queue counters""" + command = "fabricstat -C -q" + run_command(command) + +@cli.command() +def fabriccountersport(): + """Clear fabric port counters""" + command = "fabricstat -C" + run_command(command) + @cli.command() def pfccounters(): """Clear pfc counters""" diff --git a/scripts/fabricstat b/scripts/fabricstat index fcc0983ade..d858c4cdc7 100755 --- a/scripts/fabricstat +++ b/scripts/fabricstat @@ -2,10 +2,13 @@ import argparse from collections import OrderedDict, namedtuple +import json import os import sys from utilities_common import constants +from utilities_common.cli import json_serial, UserCache +from utilities_common.netstat import format_number_with_comma, table_as_json, ns_diff, format_prate from natsort import natsorted from tabulate import tabulate from sonic_py_common import multi_asic @@ -32,6 +35,10 @@ FABRIC_PORT_STATUS_TABLE_PREFIX = APP_FABRIC_PORT_TABLE_NAME+"|" FABRIC_PORT_STATUS_FIELD = "STATUS" STATUS_NA = 'N/A' +cnstat_dir = 'N/A' +cnstat_fqn_file_port = 'N/A' +cnstat_fqn_file_queue = 'N/A' + class FabricStat(object): def __init__(self, namespace): self.db = None @@ -78,6 +85,12 @@ class FabricStat(object): """ assert False, 'Need to override this method' + def save_fresh_stats(self): + """ + Get stat for each port and save. + """ + assert False, 'Need to override this method' + def cnstat_print(self, cnstat_dict, errors_only=False): """ Print the counter stat. @@ -115,6 +128,24 @@ class FabricPortStat(FabricStat): cnstat_dict[port_name] = PortStat._make(cntr) return cnstat_dict + def save_fresh_stats(self): + # Get stat for each port and save + counter_port_name_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_FABRIC_PORT_NAME_MAP) + if counter_port_name_map is None: + return + cnstat_dict = self.get_cnstat() + asic_name = '0' + if self.namespace: + asic_name = multi_asic.get_asic_id_from_name(self.namespace) + try: + cnstat_fqn_file_port_name = cnstat_fqn_file_port + asic_name + json.dump(cnstat_dict, open(cnstat_fqn_file_port_name, 'w'), default=json_serial) + except IOError as e: + print(e.errno, e) + sys.exit(e.errno) + else: + print("Clear and update saved counters port") + def cnstat_print(self, cnstat_dict, errors_only=False): if len(cnstat_dict) == 0: print("Counters %s empty" % self.namespace) @@ -127,19 +158,44 @@ class FabricPortStat(FabricStat): asic_name = '0' if self.namespace: asic_name = multi_asic.get_asic_id_from_name(self.namespace) + + cnstat_fqn_file_port_name = cnstat_fqn_file_port + asic_name + cnstat_cached_dict = {} + if os.path.isfile(cnstat_fqn_file_port_name): + try: + cnstat_cached_dict = json.load(open(cnstat_fqn_file_port_name, 'r')) + except IOError as e: + print(e.errno, e) + for key, data in cnstat_dict.items(): port_id = key[len(PORT_NAME_PREFIX):] + port_name = "PORT" + port_id + # The content in the for each port: + # "IN_CELL, IN_OCTET, OUT_CELL, OUT_OCTET, CRC, FEC_CORRECTABLE, FEC_UNCORRECTABL, SYMBOL_ERR" + # e.g. PORT76 ['0', '0', '36', '6669', '0', '13', '302626', '3'] + # Now, set default saved values to 0 + diff_cached = ['0', '0', '0', '0', '0', '0', '0', '0'] + if port_name in cnstat_cached_dict: + diff_cached = cnstat_cached_dict.get(port_name) + if errors_only: header = portstat_header_errors_only table.append((asic_name, port_id, self.get_port_state(key), - data.crc, data.fec_correctable, data.fec_uncorrectable, - data.symbol_err)) + ns_diff(data.crc, diff_cached[4]), + ns_diff(data.fec_correctable, diff_cached[5]), + ns_diff(data.fec_uncorrectable, diff_cached[6]), + ns_diff(data.symbol_err, diff_cached[7]))) else: header = portstat_header_all table.append((asic_name, port_id, self.get_port_state(key), - data.in_cell, data.in_octet, data.out_cell, data.out_octet, - data.crc, data.fec_correctable, data.fec_uncorrectable, - data.symbol_err)) + ns_diff(data.in_cell, diff_cached[0]), + ns_diff(data.in_octet, diff_cached[1]), + ns_diff(data.out_cell, diff_cached[2]), + ns_diff(data.out_octet, diff_cached[3]), + ns_diff(data.crc, diff_cached[4]), + ns_diff(data.fec_correctable, diff_cached[5]), + ns_diff(data.fec_uncorrectable, diff_cached[6]), + ns_diff(data.symbol_err, diff_cached[7]))) print(tabulate(table, header, tablefmt='simple', stralign='right')) print() @@ -166,6 +222,24 @@ class FabricQueueStat(FabricStat): cnstat_dict[port_queue_name] = QueueStat._make(cntr) return cnstat_dict + def save_fresh_stats(self): + # Get stat for each port and save + counter_port_name_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_FABRIC_PORT_NAME_MAP) + if counter_port_name_map is None: + return + cnstat_dict = self.get_cnstat() + asic_name = '0' + if self.namespace: + asic_name = multi_asic.get_asic_id_from_name(self.namespace) + try: + cnstat_fqn_file_queue_name = cnstat_fqn_file_queue + asic_name + json.dump(cnstat_dict, open(cnstat_fqn_file_queue_name, 'w'), default=json_serial) + except IOError as e: + print(e.errno, e) + sys.exit(e.errno) + else: + print("Clear and update saved counters queue") + def cnstat_print(self, cnstat_dict, errors_only=False): if len(cnstat_dict) == 0: print("Counters %s empty" % self.namespace) @@ -177,11 +251,29 @@ class FabricQueueStat(FabricStat): asic_name = '0' if self.namespace: asic_name = multi_asic.get_asic_id_from_name(self.namespace) + + cnstat_fqn_file_queue_name = cnstat_fqn_file_queue + asic_name + cnstat_cached_dict={} + if os.path.isfile(cnstat_fqn_file_queue_name): + try: + cnstat_cached_dict = json.load(open(cnstat_fqn_file_queue_name, 'r')) + except IOError as e: + print(e.errno, e) + for key, data in cnstat_dict.items(): port_name, queue_id = key.split(':') + # The content of saved counters queue for each port: + # portName:queueId CURRENT_LEVEL, WATERMARK_LEVEL, CURRENT_BYTE + # e.g. PORT90:0 ['N/A', 'N/A', 'N/A'] + # Now, set default saved values to 0 + diff_cached = ['0', '0', '0'] + if key in cnstat_cached_dict: + diff_cached = cnstat_cached_dict.get(key) port_id = port_name[len(PORT_NAME_PREFIX):] table.append((asic_name, port_id, self.get_port_state(port_name), queue_id, - data.curbyte, data.curlevel, data.watermarklevel)) + ns_diff(data.curbyte, diff_cached[2]), + ns_diff(data.curlevel, diff_cached[0]), + ns_diff(data.watermarklevel, diff_cached[1]))) print(tabulate(table, queuestat_header, tablefmt='simple', stralign='right')) print() @@ -214,6 +306,10 @@ class FabricReachability(FabricStat): return def main(): + global cnstat_dir + global cnstat_fqn_file_port + global cnstat_fqn_file_queue + parser = argparse.ArgumentParser(description='Display the fabric port state and counters', formatter_class=argparse.RawTextHelpFormatter, epilog=""" @@ -223,12 +319,16 @@ Examples: fabricstat -p -n asic0 -e fabricstat -q fabricstat -q -n asic0 + fabricstat -C + fabricstat -D """) parser.add_argument('-q','--queue', action='store_true', help='Display fabric queue stat, otherwise port stat') parser.add_argument('-r','--reachability', action='store_true', help='Display reachability, otherwise port stat') parser.add_argument('-n','--namespace', default=None, help='Display fabric ports counters for specific namespace') parser.add_argument('-e', '--errors', action='store_true', help='Display errors') + parser.add_argument('-C','--clear', action='store_true', help='Copy & clear fabric counters') + parser.add_argument('-D','--delete', action='store_true', help='Delete saved stats') args = parser.parse_args() queue = args.queue @@ -236,6 +336,23 @@ Examples: namespace = args.namespace errors_only = args.errors + save_fresh_stats = args.clear + delete_stats = args.delete + + cache = UserCache() + + cnstat_dir = cache.get_directory() + + cnstat_file = "fabricstatport" + cnstat_fqn_file_port = os.path.join(cnstat_dir, cnstat_file) + + cnstat_file = "fabricstatqueue" + cnstat_fqn_file_queue = os.path.join(cnstat_dir, cnstat_file) + + if delete_stats: + cache.remove() + sys.exit(0) + def nsStat(ns, errors_only): if queue: stat = FabricQueueStat(ns) @@ -246,7 +363,10 @@ Examples: else: stat = FabricPortStat(ns) cnstat_dict = stat.get_cnstat_dict() - stat.cnstat_print(cnstat_dict, errors_only) + if save_fresh_stats: + stat.save_fresh_stats() + else: + stat.cnstat_print(cnstat_dict, errors_only) if namespace is None: # All asics or all fabric asics diff --git a/tests/fabricstat_test.py b/tests/fabricstat_test.py index 7e37e993fe..f5a4066d60 100644 --- a/tests/fabricstat_test.py +++ b/tests/fabricstat_test.py @@ -14,38 +14,55 @@ multi_asic_fabric_counters = """\ ASIC PORT STATE IN_CELL IN_OCTET OUT_CELL OUT_OCTET CRC FEC_CORRECTABLE FEC_UNCORRECTABLE SYMBOL_ERR ------ ------ ------- --------- ---------- ---------- ----------- ----- ----------------- ------------------- ------------ - 0 0 up 6 1113 0 0 0 5 1759692040 5 - 0 1 down 0 0 0 0 0 0 58977677898 0 - 0 2 up 2 371 0 0 0 0 1769448760 0 - 0 3 down 0 0 0 0 0 0 58976477608 0 - 0 4 up 10 1855 0 0 0 73 1763293100 73 - 0 5 down 0 0 0 0 0 44196 58975150569 0 - 0 6 up 4 742 0 0 0 10 1763174090 0 - 0 7 up 10 1855 0 0 0 187 1768439529 1331 + 0 0 up 6 1,113 0 0 0 5 1,759,692,040 5 + 0 1 down 0 0 0 0 0 0 58,977,677,898 0 + 0 2 up 2 371 0 0 0 0 1,769,448,760 0 + 0 3 down 0 0 0 0 0 0 58,976,477,608 0 + 0 4 up 10 1,855 0 0 0 73 1,763,293,100 73 + 0 5 down 0 0 0 0 0 44,196 58,975,150,569 0 + 0 6 up 4 742 0 0 0 10 1,763,174,090 0 + 0 7 up 10 1,855 0 0 0 187 1,768,439,529 1,331 + + ASIC PORT STATE IN_CELL IN_OCTET OUT_CELL OUT_OCTET CRC FEC_CORRECTABLE FEC_UNCORRECTABLE SYMBOL_ERR +------ ------ ------- --------- ---------- ---------- ----------- ----- ----------------- ------------------- -------------- + 1 0 up 16 2,968 0 0 0 0 1,763,890,500 0 + 1 1 down 0 0 0 0 0 0 105,269,481,425 0 + 1 2 down 0 0 0 0 0 0 105,268,895,944 0 + 1 3 down 0 0 0 0 0 0 105,268,290,607 0 + 1 4 up 14 2,597 0 0 0 0 1,762,188,940 0 + 1 5 down 0 0 0 0 0 968 105,267,020,477 0 + 1 6 down 0 0 0 0 0 53,192,703,023 1,422,986 41,913,682,074 + 1 7 down 0 0 0 0 0 0 105,264,567,398 0 +""" +multi_asic_fabric_counters_asic0 = """\ ASIC PORT STATE IN_CELL IN_OCTET OUT_CELL OUT_OCTET CRC FEC_CORRECTABLE FEC_UNCORRECTABLE SYMBOL_ERR ------ ------ ------- --------- ---------- ---------- ----------- ----- ----------------- ------------------- ------------ - 1 0 up 16 2968 0 0 0 0 1763890500 0 - 1 1 down 0 0 0 0 0 0 105269481425 0 - 1 2 down 0 0 0 0 0 0 105268895944 0 - 1 3 down 0 0 0 0 0 0 105268290607 0 - 1 4 up 14 2597 0 0 0 0 1762188940 0 - 1 5 down 0 0 0 0 0 968 105267020477 0 - 1 6 down 0 0 0 0 0 53192703023 1422986 41913682074 - 1 7 down 0 0 0 0 0 0 105264567398 0 + 0 0 up 6 1,113 0 0 0 5 1,759,692,040 5 + 0 1 down 0 0 0 0 0 0 58,977,677,898 0 + 0 2 up 2 371 0 0 0 0 1,769,448,760 0 + 0 3 down 0 0 0 0 0 0 58,976,477,608 0 + 0 4 up 10 1,855 0 0 0 73 1,763,293,100 73 + 0 5 down 0 0 0 0 0 44,196 58,975,150,569 0 + 0 6 up 4 742 0 0 0 10 1,763,174,090 0 + 0 7 up 10 1,855 0 0 0 187 1,768,439,529 1,331 """ -multi_asic_fabric_counters_asic0 = """\ + +clear_counter = """\ +Clear and update saved counters port""" + +multi_asic_fabric_counters_asic0_clear = """\ ASIC PORT STATE IN_CELL IN_OCTET OUT_CELL OUT_OCTET CRC FEC_CORRECTABLE FEC_UNCORRECTABLE SYMBOL_ERR ------ ------ ------- --------- ---------- ---------- ----------- ----- ----------------- ------------------- ------------ - 0 0 up 6 1113 0 0 0 5 1759692040 5 - 0 1 down 0 0 0 0 0 0 58977677898 0 - 0 2 up 2 371 0 0 0 0 1769448760 0 - 0 3 down 0 0 0 0 0 0 58976477608 0 - 0 4 up 10 1855 0 0 0 73 1763293100 73 - 0 5 down 0 0 0 0 0 44196 58975150569 0 - 0 6 up 4 742 0 0 0 10 1763174090 0 - 0 7 up 10 1855 0 0 0 187 1768439529 1331 + 0 0 up 0 0 0 0 0 0 0 0 + 0 1 down 0 0 0 0 0 0 0 0 + 0 2 up 0 0 0 0 0 0 0 0 + 0 3 down 0 0 0 0 0 0 0 0 + 0 4 up 0 0 0 0 0 0 0 0 + 0 5 down 0 0 0 0 0 0 0 0 + 0 6 up 0 0 0 0 0 0 0 0 + 0 7 up 0 0 0 0 0 0 0 0 """ @@ -58,18 +75,18 @@ 0 1 down 0 0 0 0 0 2 up 0 104 8 8 0 3 down 0 0 0 0 - 0 4 up 0 1147 14 22 + 0 4 up 0 1,147 14 22 0 5 down 0 0 0 0 0 6 up 0 527 8 10 - 0 7 up 0 1147 14 17 + 0 7 up 0 1,147 14 17 ASIC PORT STATE QUEUE_ID CURRENT_BYTE CURRENT_LEVEL WATERMARK_LEVEL ------ ------ ------- ---------- -------------- --------------- ----------------- - 1 0 up 0 1942 18 24 + 1 0 up 0 1,942 18 24 1 1 down 0 0 0 0 1 2 down 0 0 0 0 1 3 down 0 0 0 0 - 1 4 up 0 1362 15 24 + 1 4 up 0 1,362 15 24 1 5 down 0 0 0 0 1 6 down 0 0 0 0 1 7 down 0 0 0 0 @@ -83,10 +100,24 @@ 0 1 down 0 0 0 0 0 2 up 0 104 8 8 0 3 down 0 0 0 0 - 0 4 up 0 1147 14 22 + 0 4 up 0 1,147 14 22 0 5 down 0 0 0 0 0 6 up 0 527 8 10 - 0 7 up 0 1147 14 17 + 0 7 up 0 1,147 14 17 + +""" + +multi_asic_fabric_counters_queue_asic0_clear = """\ + ASIC PORT STATE QUEUE_ID CURRENT_BYTE CURRENT_LEVEL WATERMARK_LEVEL +------ ------ ------- ---------- -------------- --------------- ----------------- + 0 0 up 0 0 0 0 + 0 1 down 0 0 0 0 + 0 2 up 0 0 0 0 + 0 3 down 0 0 0 0 + 0 4 up 0 0 0 0 + 0 5 down 0 0 0 0 + 0 6 up 0 0 0 0 + 0 7 up 0 0 0 0 """ @@ -128,12 +159,8 @@ def setup_class(cls): os.environ["UTILITIES_UNIT_TESTING"] = "1" def test_single_show_fabric_counters(self): - from .mock_tables import mock_single_asic - import importlib - importlib.reload(mock_single_asic) - from .mock_tables import dbconnector - dbconnector.load_database_config - dbconnector.load_namespace_config() + return_code, result = get_result_and_return_code('fabricstat -D') + assert return_code == 0 return_code, result = get_result_and_return_code(['fabricstat']) print("return_code: {}".format(return_code)) @@ -141,6 +168,22 @@ def test_single_show_fabric_counters(self): assert return_code == 0 assert result == multi_asic_fabric_counters_asic0 + def test_single_clear_fabric_counters(self): + return_code, result = get_result_and_return_code('fabricstat -C') + print("return_code: {}".format(return_code)) + print("result = {}".format(result)) + assert return_code == 0 + assert result.rstrip() == clear_counter + + return_code, result = get_result_and_return_code('fabricstat') + print("return_code: {}".format(return_code)) + print("result = {}".format(result)) + assert return_code == 0 + assert result == multi_asic_fabric_counters_asic0_clear + + return_code, result = get_result_and_return_code('fabricstat -D') + assert return_code == 0 + @classmethod def teardown_class(cls): print("TEARDOWN") @@ -193,6 +236,27 @@ def test_multi_show_fabric_counters_queue_asic(self): assert return_code == 0 assert result == multi_asic_fabric_counters_queue_asic0 + def test_multi_show_fabric_counters_queue_clear(self): + return_code, result = get_result_and_return_code('fabricstat -C -q') + print("return_code: {}".format(return_code)) + print("result = {}".format(result)) + assert return_code == 0 + + return_code, result = get_result_and_return_code('fabricstat -q -n asic0') + print("return_code: {}".format(return_code)) + print("result = {}".format(result)) + assert return_code == 0 + assert result == multi_asic_fabric_counters_queue_asic0_clear + + return_code, result = get_result_and_return_code('fabricstat -D') + assert return_code == 0 + + return_code, result = get_result_and_return_code('fabricstat -q -n asic0') + print("return_code: {}".format(return_code)) + print("result = {}".format(result)) + assert return_code == 0 + assert result == multi_asic_fabric_counters_queue_asic0 + def test_multi_show_fabric_reachability(self): return_code, result = get_result_and_return_code(['fabricstat', '-r']) print("return_code: {}".format(return_code)) @@ -214,3 +278,31 @@ def teardown_class(cls): os.environ["PATH"].split(os.pathsep)[:-1]) os.environ["UTILITIES_UNIT_TESTING"] = "0" os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "" + + +class TestMultiAsicFabricStatCmd(object): + @classmethod + def setup_class(cls): + print("SETUP") + os.environ["PATH"] += os.pathsep + scripts_path + os.environ["UTILITIES_UNIT_TESTING"] = "2" + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "multi_asic" + + def test_clear_command(self): + runner = CliRunner() + result = runner.invoke(clear.cli.commands["fabriccountersqueue"], []) + assert result.exit_code == 0 + + result = runner.invoke(clear.cli.commands["fabriccountersport"], []) + assert result.exit_code == 0 + + return_code, result = get_result_and_return_code('fabricstat -D') + assert return_code == 0 + + @classmethod + def teardown_class(cls): + print("TEARDOWN") + os.environ["PATH"] = os.pathsep.join( + os.environ["PATH"].split(os.pathsep)[:-1]) + os.environ["UTILITIES_UNIT_TESTING"] = "0" + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = ""