From c6f9664b2e23770e72537e93a68f9189e6ea5e4c Mon Sep 17 00:00:00 2001 From: Neetha John Date: Tue, 12 Jul 2022 08:45:55 -0700 Subject: [PATCH] [202012] Minigraph parser changes to select mmu profiles based on SonicQosProfile attribute (#11383) Signed-off-by: Neetha John Why I did it There is a need to select different mmu profiles based on deployment type How I did it There will be separate subfolders (RDMA-CENTRIC, TCP-CENTRIC, BALANCED) in each hwsku folder which contains deployment specific mmu and qos settings. SonicQosProfile attribute in the minigraph will be used to determine which settings to use. If that attribute is not present, the default settings that exist in the hwsku folder will be used --- src/sonic-config-engine/minigraph.py | 34 ++++- .../tests/sample-dell-6100-t0-minigraph.xml | 5 + src/sonic-config-engine/tests/test_j2files.py | 118 +++--------------- 3 files changed, 57 insertions(+), 100 deletions(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 035fb0bcb558..5e8cf2966b64 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -853,6 +853,7 @@ def parse_meta(meta, hname): kube_data = {} redundancy_type = None downstream_redundancy_types = None + qos_profile = None device_metas = meta.find(str(QName(ns, "Devices"))) for device in device_metas.findall(str(QName(ns1, "DeviceMetadata"))): @@ -892,7 +893,9 @@ def parse_meta(meta, hname): downstream_redundancy_types = value elif name == "RedundancyType": redundancy_type = value - return syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, kube_data, downstream_redundancy_types, redundancy_type + elif name == "SonicQosProfile": + qos_profile = value + return syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, kube_data, downstream_redundancy_types, redundancy_type, qos_profile def parse_linkmeta(meta, hname): link = meta.find(str(QName(ns, "Link"))) @@ -1144,6 +1147,31 @@ def enable_internal_bgp_session(bgp_sessions, filename, asic_name): (local_sub_role == BACKEND_ASIC_SUB_ROLE and peer_sub_role == FRONTEND_ASIC_SUB_ROLE)): bgp_sessions[peer_ip].update({'admin_status': 'up'}) +def select_mmu_profiles(profile, platform, hwsku): + """ + Select MMU files based on the device metadata attribute - SonicQosProfile + if no QosProfile exists in the minigraph, then no action is needed. + if a profile exists in the minigraph, + - create a dir path to search 1 level down from the base path. + - if no such dir path exists, no action is needed. + - if a dir path exists, check for the presence of each file from + the copy list in the dir path and copy it over to the base path. + """ + if not profile: + return + + files_to_copy = ['pg_profile_lookup.ini', 'qos.json.j2', 'buffers_defaults_t0.j2', 'buffers_defaults_t1.j2'] + + path = os.path.join('/usr/share/sonic/device', platform, hwsku) + + dir_path = os.path.join(path, profile) + if os.path.exists(dir_path): + for file_item in files_to_copy: + file_in_dir = os.path.join(dir_path, file_item) + if os.path.isfile(file_in_dir): + base_file = os.path.join(path, file_item) + exec_cmd("sudo cp {} {}".format(file_in_dir, base_file)) + ############################################################################### # # Main functions @@ -1212,6 +1240,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw system_defaults = {} downstream_redundancy_types = None redundancy_type = None + qos_profile = None hwsku_qn = QName(ns, "HwSku") hostname_qn = QName(ns, "Hostname") @@ -1242,7 +1271,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw elif child.tag == str(QName(ns, "UngDec")): (u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname, None) elif child.tag == str(QName(ns, "MetadataDeclaration")): - (syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, kube_data, downstream_redundancy_types, redundancy_type) = parse_meta(child, hostname) + (syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, kube_data, downstream_redundancy_types, redundancy_type, qos_profile) = parse_meta(child, hostname) elif child.tag == str(QName(ns, "LinkMetadataDeclaration")): linkmetas = parse_linkmeta(child, hostname) elif child.tag == str(QName(ns, "DeviceInfos")): @@ -1262,6 +1291,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw elif child.tag == str(QName(ns, "DeviceInfos")): (port_speeds_default, port_descriptions) = parse_deviceinfo(child, hwsku) + select_mmu_profiles(qos_profile, platform, hwsku) # set the host device type in asic metadata also device_type = [devices[key]['type'] for key in devices if key.lower() == hostname.lower()][0] if asic_name is None: diff --git a/src/sonic-config-engine/tests/sample-dell-6100-t0-minigraph.xml b/src/sonic-config-engine/tests/sample-dell-6100-t0-minigraph.xml index 7cea7decfcc0..cb84ce744ed1 100644 --- a/src/sonic-config-engine/tests/sample-dell-6100-t0-minigraph.xml +++ b/src/sonic-config-engine/tests/sample-dell-6100-t0-minigraph.xml @@ -731,6 +731,11 @@ True + + SonicQosProfile + + RDMA-CENTRIC + ARISTA01T1:Ethernet1;s6100-dev-1:fortyGigE1/1/1 diff --git a/src/sonic-config-engine/tests/test_j2files.py b/src/sonic-config-engine/tests/test_j2files.py index b9cca3789f8e..9e840a154c19 100644 --- a/src/sonic-config-engine/tests/test_j2files.py +++ b/src/sonic-config-engine/tests/test_j2files.py @@ -21,14 +21,9 @@ def setUp(self): self.t0_7050cx3_port_config = os.path.join(self.test_dir, 't0_7050cx3_d48c8_port_config.ini') self.t1_mlnx_minigraph = os.path.join(self.test_dir, 't1-sample-graph-mlnx.xml') self.mlnx_port_config = os.path.join(self.test_dir, 'sample-port-config-mlnx.ini') - self.dell6100_t0_minigraph = os.path.join(self.test_dir, 'sample-dell-6100-t0-minigraph.xml') - self.mellanox2700_t0_minigraph = os.path.join(self.test_dir, 'sample-mellanox-2700-t0-minigraph.xml') - self.mellanox2410_t1_minigraph = os.path.join(self.test_dir, 'sample-mellanox-2410-t1-minigraph.xml') - self.arista7050_t0_minigraph = os.path.join(self.test_dir, 'sample-arista-7050-t0-minigraph.xml') self.multi_asic_minigraph = os.path.join(self.test_dir, 'multi_npu_data', 'sample-minigraph.xml') self.multi_asic_port_config = os.path.join(self.test_dir, 'multi_npu_data', 'sample_port_config-0.ini') self.radv_test_minigraph = os.path.join(self.test_dir, 'radv-test-sample-graph.xml') - self.dell9332_t1_minigraph = os.path.join(self.test_dir, 'sample-dell-9332-t1-minigraph.xml') self.output_file = os.path.join(self.test_dir, 'output') os.environ["CFGGEN_UNIT_TESTING"] = "2" @@ -235,120 +230,47 @@ def test_l2switch_template_dualtor(self): self.assertEqual(sample_output_json, output_json) def test_qos_arista7050_render_template(self): - arista_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'arista', 'x86_64-arista_7050_qx32s', 'Arista-7050-QX-32S') - qos_file = os.path.join(arista_dir_path, 'qos.json.j2') - port_config_ini_file = os.path.join(arista_dir_path, 'port_config.ini') - - # copy qos_config.j2 to the Arista 7050 directory to have all templates in one directory - qos_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'qos_config.j2') - shutil.copy2(qos_config_file, arista_dir_path) - - argument = '-m ' + self.arista7050_t0_minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file - self.run_script(argument) - - # cleanup - qos_config_file_new = os.path.join(arista_dir_path, 'qos_config.j2') - os.remove(qos_config_file_new) - - sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'qos-arista7050.json') - assert utils.cmp(sample_output_file, self.output_file) + self._test_qos_render_template('arista', 'x86_64-arista_7050_qx32s', 'Arista-7050-QX-32S', 'sample-arista-7050-t0-minigraph.xml', 'qos-arista7050.json') def test_qos_dell9332_render_template(self): - dell_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'dell', 'x86_64-dellemc_z9332f_d1508-r0', 'DellEMC-Z9332f-O32') - qos_file = os.path.join(dell_dir_path, 'qos.json.j2') - port_config_ini_file = os.path.join(dell_dir_path, 'port_config.ini') - - # copy qos_config.j2 to the Dell Z9332 directory to have all templates in one directory - qos_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'qos_config.j2') - shutil.copy2(qos_config_file, dell_dir_path) - - argument = '-m ' + self.dell9332_t1_minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file - self.run_script(argument) - - # cleanup - qos_config_file_new = os.path.join(dell_dir_path, 'qos_config.j2') - os.remove(qos_config_file_new) - - sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'qos-dell9332.json') - assert utils.cmp(sample_output_file, self.output_file) + self._test_qos_render_template('dell', 'x86_64-dellemc_z9332f_d1508-r0', 'DellEMC-Z9332f-O32', 'sample-dell-9332-t1-minigraph.xml', 'qos-dell9332.json') def test_qos_dell6100_render_template(self): - dell_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100') - qos_file = os.path.join(dell_dir_path, 'qos.json.j2') - port_config_ini_file = os.path.join(dell_dir_path, 'port_config.ini') + self._test_qos_render_template('dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100', 'sample-dell-6100-t0-minigraph.xml', 'qos-dell6100.json') - # copy qos_config.j2 to the Dell S6100 directory to have all templates in one directory + def _test_qos_render_template(self, vendor, platform, sku, minigraph, expected): + file_exist, dir_exist = self.create_machine_conf(platform, vendor) + dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', vendor, platform, sku) + qos_file = os.path.join(dir_path, 'qos.json.j2') + port_config_ini_file = os.path.join(dir_path, 'port_config.ini') + + # copy qos_config.j2 to the SKU directory to have all templates in one directory qos_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'qos_config.j2') - shutil.copy2(qos_config_file, dell_dir_path) + shutil.copy2(qos_config_file, dir_path) - argument = '-m ' + self.dell6100_t0_minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file + minigraph = os.path.join(self.test_dir, minigraph) + argument = '-m ' + minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file self.run_script(argument) # cleanup - qos_config_file_new = os.path.join(dell_dir_path, 'qos_config.j2') + qos_config_file_new = os.path.join(dir_path, 'qos_config.j2') os.remove(qos_config_file_new) - sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'qos-dell6100.json') + self.remove_machine_conf(file_exist, dir_exist) + + sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, expected) assert utils.cmp(sample_output_file, self.output_file) def test_buffers_dell6100_render_template(self): - dell_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100') - buffers_file = os.path.join(dell_dir_path, 'buffers.json.j2') - port_config_ini_file = os.path.join(dell_dir_path, 'port_config.ini') - - # copy buffers_config.j2 to the Dell S6100 directory to have all templates in one directory - buffers_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'buffers_config.j2') - shutil.copy2(buffers_config_file, dell_dir_path) - - argument = '-m ' + self.dell6100_t0_minigraph + ' -p ' + port_config_ini_file + ' -t ' + buffers_file + ' > ' + self.output_file - self.run_script(argument) - - # cleanup - buffers_config_file_new = os.path.join(dell_dir_path, 'buffers_config.j2') - os.remove(buffers_config_file_new) - - sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'buffers-dell6100.json') - assert utils.cmp(sample_output_file, self.output_file) + self._test_buffers_render_template('dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100', 'sample-dell-6100-t0-minigraph.xml', 'buffers.json.j2', 'buffers-dell6100.json') def test_buffers_mellanox2700_render_template(self): # Mellanox buffer template rendering for single ingress pool mode - mellanox_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'mellanox', 'x86_64-mlnx_msn2700-r0', 'Mellanox-SN2700-D48C8') - buffers_file = os.path.join(mellanox_dir_path, 'buffers.json.j2') - port_config_ini_file = os.path.join(mellanox_dir_path, 'port_config.ini') - - # copy buffers_config.j2 to the Mellanox 2700 directory to have all templates in one directory - buffers_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'buffers_config.j2') - shutil.copy2(buffers_config_file, mellanox_dir_path) - - argument = '-m ' + self.mellanox2700_t0_minigraph + ' -p ' + port_config_ini_file + ' -t ' + buffers_file + ' > ' + self.output_file - self.run_script(argument) - - # cleanup - buffers_config_file_new = os.path.join(mellanox_dir_path, 'buffers_config.j2') - os.remove(buffers_config_file_new) - - sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'buffers-mellanox2700.json') - assert utils.cmp(sample_output_file, self.output_file) + self._test_buffers_render_template('mellanox', 'x86_64-mlnx_msn2700-r0', 'Mellanox-SN2700-D48C8', 'sample-mellanox-2700-t0-minigraph.xml', 'buffers.json.j2', 'buffers-mellanox2700.json') def test_buffers_mellanox2410_render_template(self): # Mellanox buffer template rendering for double ingress pools mode - mellanox_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'mellanox', 'x86_64-mlnx_msn2410-r0', 'ACS-MSN2410') - buffers_file = os.path.join(mellanox_dir_path, 'buffers.json.j2') - port_config_ini_file = os.path.join(mellanox_dir_path, 'port_config.ini') - - # copy buffers_config.j2 to the Mellanox 2410 directory to have all templates in one directory - buffers_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'buffers_config.j2') - shutil.copy2(buffers_config_file, mellanox_dir_path) - - argument = '-m ' + self.mellanox2410_t1_minigraph + ' -p ' + port_config_ini_file + ' -t ' + buffers_file + ' > ' + self.output_file - self.run_script(argument) - - # cleanup - buffers_config_file_new = os.path.join(mellanox_dir_path, 'buffers_config.j2') - os.remove(buffers_config_file_new) - - sample_output_file = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'buffers-mellanox2410.json') - assert utils.cmp(sample_output_file, self.output_file) + self._test_buffers_render_template('mellanox', 'x86_64-mlnx_msn2410-r0', 'ACS-MSN2410', 'sample-mellanox-2410-t1-minigraph.xml', 'buffers.json.j2', 'buffers-mellanox2410.json') def test_config_brcm_render_template(self): if utils.PYvX_DIR != 'py3':