From d117172b239bb3c46bd5e2c2661c306c15b01629 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Tue, 24 Apr 2018 16:01:17 -0700 Subject: [PATCH] sonic-cfggen supports hwsku parameter (#1631) --- sonic-slave/Dockerfile | 18 +-- src/sonic-config-engine/data/l2switch.j2 | 21 ++++ src/sonic-config-engine/setup.py | 6 +- src/sonic-config-engine/sonic-cfggen | 29 +++-- .../tests/sample_output/l2switch.json | 107 ++++++++++++++++++ src/sonic-config-engine/tests/test_j2files.py | 8 ++ 6 files changed, 171 insertions(+), 18 deletions(-) create mode 100644 src/sonic-config-engine/data/l2switch.j2 create mode 100644 src/sonic-config-engine/tests/sample_output/l2switch.json diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index ed85b0ca436a..213daf1a8e47 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -184,16 +184,12 @@ RUN apt-get update && apt-get install -y \ libjs-sphinxdoc \ libjs-underscore \ python-docutils \ - python-jinja2 \ python-markupsafe \ python-pygments \ python-roman \ - python-sphinx \ sphinx-common \ - python3-sphinx \ # For sonic config engine testing python-lxml \ - python-jinja2 \ python-netaddr \ python-ipaddr \ python-yaml \ @@ -229,16 +225,22 @@ RUN export VERSION=1.8.3 \ && echo 'export GOROOT=/usr/local/go' >> /etc/bash.bashrc \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc +# Upgrade pip2 +# Note: use pip2 specific version so jinja2 2.10 will install +RUN python2 -m pip install -U pip==9.0.3 + # For p4 build RUN pip install \ ctypesgen \ crc16 -# For templating -RUN pip install j2cli - # For sonic config engine testing -RUN pip install pyangbind==0.5.10 +RUN pip install pyangbind==0.6.0 +# Note: force upgrade debian packaged jinja2, if installed +RUN pip install --force-reinstall --upgrade jinja2>=2.10 + +# For templating (requiring jinja2) +RUN pip install j2cli # For sonic utilities testing RUN pip install click-default-group click natsort tabulate diff --git a/src/sonic-config-engine/data/l2switch.j2 b/src/sonic-config-engine/data/l2switch.j2 new file mode 100644 index 000000000000..ee24f5ebd19d --- /dev/null +++ b/src/sonic-config-engine/data/l2switch.j2 @@ -0,0 +1,21 @@ +{ + "DEVICE_METADATA": {{ DEVICE_METADATA | tojson }}, + "PORT": {{ PORT | tojson }}, + "VLAN": { + "Vlan1000": { + "vlanid": "1000" + } + }, + {% set ns = {'firstPrinted': False} -%} + "VLAN_MEMBER": { + {%- for port in PORT -%} + {%- if ns.firstPrinted %},{% endif %} + + "Vlan1000|{{ port }}": { + "tagging_mode": "untagged" + } + {%- if ns.update({'firstPrinted': True}) %}{% endif -%} + {%- endfor %} + + } +} diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index e1530d0d3d32..1d0c7f99fa41 100755 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -3,6 +3,7 @@ from setuptools import setup import os.path import unittest +import glob def get_test_suite(): test_loader = unittest.TestLoader() @@ -17,6 +18,9 @@ def get_test_suite(): url='https://github.com/Azure/sonic-buildimage', py_modules=['portconfig', 'minigraph', 'openconfig_acl', 'sonic_platform'], scripts=['sonic-cfggen'], - install_requires=['lxml', 'jinja2', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'], + install_requires=['lxml', 'jinja2>=2.10', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'], test_suite='setup.get_test_suite', + data_files=[ + ('/usr/share/sonic/templates', glob.glob('data/*')), + ], ) diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 2495bc89fe06..bdb3c804fc73 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -15,6 +15,7 @@ Examples: See usage string for detail description for arguments. """ +from __future__ import print_function import sys import os.path import argparse @@ -26,6 +27,7 @@ from functools import partial from minigraph import minigraph_encoder from minigraph import parse_xml from minigraph import parse_device_desc_xml +from portconfig import get_port_config from sonic_platform import get_machine_info from sonic_platform import get_platform_info from sonic_platform import get_system_mac @@ -140,7 +142,8 @@ def main(): group = parser.add_mutually_exclusive_group() group.add_argument("-m", "--minigraph", help="minigraph xml file", nargs='?', const='/etc/sonic/minigraph.xml') group.add_argument("-M", "--device-description", help="device description xml file") - parser.add_argument("-p", "--port-config", help="port config file, used with -m") + group.add_argument("-k", "--hwsku", help="HwSKU") + parser.add_argument("-p", "--port-config", help="port config file, used with -m or -k", nargs='?', const=None) parser.add_argument("-y", "--yaml", help="yaml file that contains additional variables", action='append', default=[]) parser.add_argument("-j", "--json", help="json file that contains additional variables", action='append', default=[]) parser.add_argument("-a", "--additional-data", help="addition data, in json string") @@ -162,7 +165,18 @@ def main(): db_kwargs['unix_socket_path'] = args.redis_unix_sock_file data = {} + hwsku = args.hwsku + if hwsku is not None: + hardware_data = {'DEVICE_METADATA': {'localhost': { + 'hwsku': hwsku + }}} + deep_update(data, hardware_data) + (ports, _) = get_port_config(hwsku, platform, args.port_config) + if not ports: + print('Failed to get port config', file=sys.stderr) + sys.exit(1) + deep_update(data, {'PORT': ports}) if args.minigraph != None: minigraph = args.minigraph @@ -172,10 +186,7 @@ def main(): else: deep_update(data, parse_xml(minigraph, platform)) else: - if args.port_config != None: - deep_update(data, parse_xml(minigraph, port_config_file=args.port_config)) - else: - deep_update(data, parse_xml(minigraph)) + deep_update(data, parse_xml(minigraph, port_config_file=args.port_config)) if args.device_description != None: deep_update(data, parse_device_desc_xml(args.device_description)) @@ -216,14 +227,14 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) - print template.render(sort_data(data)) + print(template.render(sort_data(data))) if args.var != None: template = jinja2.Template('{{' + args.var + '}}') - print template.render(data) + print(template.render(data)) if args.var_json != None: - print json.dumps(FormatConverter.to_serialized(data[args.var_json]), indent=4, cls=minigraph_encoder) + print(json.dumps(FormatConverter.to_serialized(data[args.var_json]), indent=4, cls=minigraph_encoder)) if args.write_to_db: configdb = ConfigDBConnector(**db_kwargs) @@ -231,7 +242,7 @@ def main(): configdb.mod_config(FormatConverter.output_to_db(data)) if args.print_data: - print json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder) + print(json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder)) if __name__ == "__main__": diff --git a/src/sonic-config-engine/tests/sample_output/l2switch.json b/src/sonic-config-engine/tests/sample_output/l2switch.json new file mode 100644 index 000000000000..e8af54869e02 --- /dev/null +++ b/src/sonic-config-engine/tests/sample_output/l2switch.json @@ -0,0 +1,107 @@ +{ + "DEVICE_METADATA": {"localhost": {"hwsku": "Mellanox-SN2700"}}, + "PORT": {"Ethernet0": {"alias": "fortyGigE0/0", "lanes": "29,30,31,32"}, "Ethernet100": {"alias": "fortyGigE0/100", "lanes": "125,126,127,128"}, "Ethernet104": {"alias": "fortyGigE0/104", "lanes": "85,86,87,88"}, "Ethernet108": {"alias": "fortyGigE0/108", "lanes": "81,82,83,84"}, "Ethernet112": {"alias": "fortyGigE0/112", "lanes": "89,90,91,92"}, "Ethernet116": {"alias": "fortyGigE0/116", "lanes": "93,94,95,96"}, "Ethernet12": {"alias": "fortyGigE0/12", "lanes": "33,34,35,36"}, "Ethernet120": {"alias": "fortyGigE0/120", "lanes": "97,98,99,100"}, "Ethernet124": {"alias": "fortyGigE0/124", "lanes": "101,102,103,104"}, "Ethernet16": {"alias": "fortyGigE0/16", "lanes": "41,42,43,44"}, "Ethernet20": {"alias": "fortyGigE0/20", "lanes": "45,46,47,48"}, "Ethernet24": {"alias": "fortyGigE0/24", "lanes": "5,6,7,8"}, "Ethernet28": {"alias": "fortyGigE0/28", "lanes": "1,2,3,4"}, "Ethernet32": {"alias": "fortyGigE0/32", "lanes": "9,10,11,12"}, "Ethernet36": {"alias": "fortyGigE0/36", "lanes": "13,14,15,16"}, "Ethernet4": {"alias": "fortyGigE0/4", "lanes": "25,26,27,28"}, "Ethernet40": {"alias": "fortyGigE0/40", "lanes": "21,22,23,24"}, "Ethernet44": {"alias": "fortyGigE0/44", "lanes": "17,18,19,20"}, "Ethernet48": {"alias": "fortyGigE0/48", "lanes": "49,50,51,52"}, "Ethernet52": {"alias": "fortyGigE0/52", "lanes": "53,54,55,56"}, "Ethernet56": {"alias": "fortyGigE0/56", "lanes": "61,62,63,64"}, "Ethernet60": {"alias": "fortyGigE0/60", "lanes": "57,58,59,60"}, "Ethernet64": {"alias": "fortyGigE0/64", "lanes": "65,66,67,68"}, "Ethernet68": {"alias": "fortyGigE0/68", "lanes": "69,70,71,72"}, "Ethernet72": {"alias": "fortyGigE0/72", "lanes": "77,78,79,80"}, "Ethernet76": {"alias": "fortyGigE0/76", "lanes": "73,74,75,76"}, "Ethernet8": {"alias": "fortyGigE0/8", "lanes": "37,38,39,40"}, "Ethernet80": {"alias": "fortyGigE0/80", "lanes": "105,106,107,108"}, "Ethernet84": {"alias": "fortyGigE0/84", "lanes": "109,110,111,112"}, "Ethernet88": {"alias": "fortyGigE0/88", "lanes": "117,118,119,120"}, "Ethernet92": {"alias": "fortyGigE0/92", "lanes": "113,114,115,116"}, "Ethernet96": {"alias": "fortyGigE0/96", "lanes": "121,122,123,124"}}, + "VLAN": { + "Vlan1000": { + "vlanid": "1000" + } + }, + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet12": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet16": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet20": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet24": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet28": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet32": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet36": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet40": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet44": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet48": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet52": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet56": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet60": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet64": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet68": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet72": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet76": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet80": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet84": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet88": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet92": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet96": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet100": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet104": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet108": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet112": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet116": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet120": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet124": { + "tagging_mode": "untagged" + } + } +} diff --git a/src/sonic-config-engine/tests/test_j2files.py b/src/sonic-config-engine/tests/test_j2files.py index 129c3973d7b5..8ff974c1eb10 100644 --- a/src/sonic-config-engine/tests/test_j2files.py +++ b/src/sonic-config-engine/tests/test_j2files.py @@ -106,6 +106,14 @@ def test_msn27xx_32ports_buffers(self): self.assertTrue(filecmp.cmp(sample_output_file, self.output_file)) + def test_sku_render_template(self): + argument = '-k Mellanox-SN2700 -t ' + os.path.join(self.test_dir, '../data/l2switch.j2') + ' -p ' + self.t0_port_config + ' > ' + self.output_file + self.run_script(argument) + + sample_output_file = os.path.join(self.test_dir, 'sample_output', 'l2switch.json') + + self.assertTrue(filecmp.cmp(sample_output_file, self.output_file)) + def tearDown(self): try: os.remove(self.output_file)