From 4444249f2de72668de4386b99fa4f81ff4065a3c Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Thu, 19 Apr 2018 03:05:22 +0000 Subject: [PATCH 01/10] sonic-cfggen supports hwsku parameter Signed-off-by: Qi Luo --- src/sonic-config-engine/sonic-cfggen | 29 +++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 2495bc89fe06..259a9ce3b88c 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -15,6 +15,7 @@ Examples: See usage string for detail description for arguments. """ +from __future__ import print_function import sys import os.path import argparse @@ -26,6 +27,7 @@ from functools import partial from minigraph import minigraph_encoder from minigraph import parse_xml from minigraph import parse_device_desc_xml +from portconfig import get_port_config from sonic_platform import get_machine_info from sonic_platform import get_platform_info from sonic_platform import get_system_mac @@ -140,7 +142,8 @@ def main(): group = parser.add_mutually_exclusive_group() group.add_argument("-m", "--minigraph", help="minigraph xml file", nargs='?', const='/etc/sonic/minigraph.xml') group.add_argument("-M", "--device-description", help="device description xml file") - parser.add_argument("-p", "--port-config", help="port config file, used with -m") + group.add_argument("-k", "--hwsku", help="HwSKU") + parser.add_argument("-p", "--port-config", help="port config file, used with -m or -k", nargs='?', const=None) parser.add_argument("-y", "--yaml", help="yaml file that contains additional variables", action='append', default=[]) parser.add_argument("-j", "--json", help="json file that contains additional variables", action='append', default=[]) parser.add_argument("-a", "--additional-data", help="addition data, in json string") @@ -162,7 +165,18 @@ def main(): db_kwargs['unix_socket_path'] = args.redis_unix_sock_file data = {} + hwsku = args.hwsku + if hwsku is not None: + hardware_data = {'DEVICE_METADATA': {'localhost': { + 'hwsku': hwsku + }}} + deep_update(data, hardware_data) + (ports, _) = get_port_config(hwsku, platform, args.port_config) + if not bool(ports): + print('Failed to get port config', file=sys.stderr) + sys.exit(1) + deep_update(data, {'PORT': ports}) if args.minigraph != None: minigraph = args.minigraph @@ -172,10 +186,7 @@ def main(): else: deep_update(data, parse_xml(minigraph, platform)) else: - if args.port_config != None: - deep_update(data, parse_xml(minigraph, port_config_file=args.port_config)) - else: - deep_update(data, parse_xml(minigraph)) + deep_update(data, parse_xml(minigraph, port_config_file=args.port_config)) if args.device_description != None: deep_update(data, parse_device_desc_xml(args.device_description)) @@ -216,14 +227,14 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) - print template.render(sort_data(data)) + print(template.render(sort_data(data))) if args.var != None: template = jinja2.Template('{{' + args.var + '}}') - print template.render(data) + print(template.render(data)) if args.var_json != None: - print json.dumps(FormatConverter.to_serialized(data[args.var_json]), indent=4, cls=minigraph_encoder) + print(json.dumps(FormatConverter.to_serialized(data[args.var_json]), indent=4, cls=minigraph_encoder)) if args.write_to_db: configdb = ConfigDBConnector(**db_kwargs) @@ -231,7 +242,7 @@ def main(): configdb.mod_config(FormatConverter.output_to_db(data)) if args.print_data: - print json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder) + print(json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder)) if __name__ == "__main__": From a41ad2258fd656c7366c787f0177a95065bed156 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Fri, 20 Apr 2018 01:26:18 +0000 Subject: [PATCH 02/10] Install l2switch.j2 into templates directory Signed-off-by: Qi Luo --- src/sonic-config-engine/data/l2switch.j2 | 21 +++++++++++++++++++++ src/sonic-config-engine/setup.py | 4 ++++ 2 files changed, 25 insertions(+) create mode 100644 src/sonic-config-engine/data/l2switch.j2 diff --git a/src/sonic-config-engine/data/l2switch.j2 b/src/sonic-config-engine/data/l2switch.j2 new file mode 100644 index 000000000000..ee24f5ebd19d --- /dev/null +++ b/src/sonic-config-engine/data/l2switch.j2 @@ -0,0 +1,21 @@ +{ + "DEVICE_METADATA": {{ DEVICE_METADATA | tojson }}, + "PORT": {{ PORT | tojson }}, + "VLAN": { + "Vlan1000": { + "vlanid": "1000" + } + }, + {% set ns = {'firstPrinted': False} -%} + "VLAN_MEMBER": { + {%- for port in PORT -%} + {%- if ns.firstPrinted %},{% endif %} + + "Vlan1000|{{ port }}": { + "tagging_mode": "untagged" + } + {%- if ns.update({'firstPrinted': True}) %}{% endif -%} + {%- endfor %} + + } +} diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index e1530d0d3d32..0ffb0bd10102 100755 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -3,6 +3,7 @@ from setuptools import setup import os.path import unittest +import glob def get_test_suite(): test_loader = unittest.TestLoader() @@ -19,4 +20,7 @@ def get_test_suite(): scripts=['sonic-cfggen'], install_requires=['lxml', 'jinja2', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'], test_suite='setup.get_test_suite', + data_files=[ + ('/usr/share/sonic/templates', glob.glob('data/*')), + ], ) From 95353c8dc605187618008ec963dd578545c0abd5 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 21 Apr 2018 00:18:38 +0000 Subject: [PATCH 03/10] Add test case Signed-off-by: Qi Luo --- .../tests/sample_output/l2switch.json | 107 ++++++++++++++++++ src/sonic-config-engine/tests/test_j2files.py | 8 ++ 2 files changed, 115 insertions(+) create mode 100644 src/sonic-config-engine/tests/sample_output/l2switch.json diff --git a/src/sonic-config-engine/tests/sample_output/l2switch.json b/src/sonic-config-engine/tests/sample_output/l2switch.json new file mode 100644 index 000000000000..e8af54869e02 --- /dev/null +++ b/src/sonic-config-engine/tests/sample_output/l2switch.json @@ -0,0 +1,107 @@ +{ + "DEVICE_METADATA": {"localhost": {"hwsku": "Mellanox-SN2700"}}, + "PORT": {"Ethernet0": {"alias": "fortyGigE0/0", "lanes": "29,30,31,32"}, "Ethernet100": {"alias": "fortyGigE0/100", "lanes": "125,126,127,128"}, "Ethernet104": {"alias": "fortyGigE0/104", "lanes": "85,86,87,88"}, "Ethernet108": {"alias": "fortyGigE0/108", "lanes": "81,82,83,84"}, "Ethernet112": {"alias": "fortyGigE0/112", "lanes": "89,90,91,92"}, "Ethernet116": {"alias": "fortyGigE0/116", "lanes": "93,94,95,96"}, "Ethernet12": {"alias": "fortyGigE0/12", "lanes": "33,34,35,36"}, "Ethernet120": {"alias": "fortyGigE0/120", "lanes": "97,98,99,100"}, "Ethernet124": {"alias": "fortyGigE0/124", "lanes": "101,102,103,104"}, "Ethernet16": {"alias": "fortyGigE0/16", "lanes": "41,42,43,44"}, "Ethernet20": {"alias": "fortyGigE0/20", "lanes": "45,46,47,48"}, "Ethernet24": {"alias": "fortyGigE0/24", "lanes": "5,6,7,8"}, "Ethernet28": {"alias": "fortyGigE0/28", "lanes": "1,2,3,4"}, "Ethernet32": {"alias": "fortyGigE0/32", "lanes": "9,10,11,12"}, "Ethernet36": {"alias": "fortyGigE0/36", "lanes": "13,14,15,16"}, "Ethernet4": {"alias": "fortyGigE0/4", "lanes": "25,26,27,28"}, "Ethernet40": {"alias": "fortyGigE0/40", "lanes": "21,22,23,24"}, "Ethernet44": {"alias": "fortyGigE0/44", "lanes": "17,18,19,20"}, "Ethernet48": {"alias": "fortyGigE0/48", "lanes": "49,50,51,52"}, "Ethernet52": {"alias": "fortyGigE0/52", "lanes": "53,54,55,56"}, "Ethernet56": {"alias": "fortyGigE0/56", "lanes": "61,62,63,64"}, "Ethernet60": {"alias": "fortyGigE0/60", "lanes": "57,58,59,60"}, "Ethernet64": {"alias": "fortyGigE0/64", "lanes": "65,66,67,68"}, "Ethernet68": {"alias": "fortyGigE0/68", "lanes": "69,70,71,72"}, "Ethernet72": {"alias": "fortyGigE0/72", "lanes": "77,78,79,80"}, "Ethernet76": {"alias": "fortyGigE0/76", "lanes": "73,74,75,76"}, "Ethernet8": {"alias": "fortyGigE0/8", "lanes": "37,38,39,40"}, "Ethernet80": {"alias": "fortyGigE0/80", "lanes": "105,106,107,108"}, "Ethernet84": {"alias": "fortyGigE0/84", "lanes": "109,110,111,112"}, "Ethernet88": {"alias": "fortyGigE0/88", "lanes": "117,118,119,120"}, "Ethernet92": {"alias": "fortyGigE0/92", "lanes": "113,114,115,116"}, "Ethernet96": {"alias": "fortyGigE0/96", "lanes": "121,122,123,124"}}, + "VLAN": { + "Vlan1000": { + "vlanid": "1000" + } + }, + "VLAN_MEMBER": { + "Vlan1000|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet4": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet12": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet16": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet20": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet24": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet28": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet32": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet36": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet40": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet44": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet48": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet52": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet56": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet60": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet64": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet68": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet72": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet76": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet80": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet84": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet88": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet92": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet96": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet100": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet104": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet108": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet112": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet116": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet120": { + "tagging_mode": "untagged" + }, + "Vlan1000|Ethernet124": { + "tagging_mode": "untagged" + } + } +} diff --git a/src/sonic-config-engine/tests/test_j2files.py b/src/sonic-config-engine/tests/test_j2files.py index 129c3973d7b5..8ff974c1eb10 100644 --- a/src/sonic-config-engine/tests/test_j2files.py +++ b/src/sonic-config-engine/tests/test_j2files.py @@ -106,6 +106,14 @@ def test_msn27xx_32ports_buffers(self): self.assertTrue(filecmp.cmp(sample_output_file, self.output_file)) + def test_sku_render_template(self): + argument = '-k Mellanox-SN2700 -t ' + os.path.join(self.test_dir, '../data/l2switch.j2') + ' -p ' + self.t0_port_config + ' > ' + self.output_file + self.run_script(argument) + + sample_output_file = os.path.join(self.test_dir, 'sample_output', 'l2switch.json') + + self.assertTrue(filecmp.cmp(sample_output_file, self.output_file)) + def tearDown(self): try: os.remove(self.output_file) From 775aea95b0331e7bf01f02fbb3bcbc854d0837d3 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 21 Apr 2018 01:05:41 +0000 Subject: [PATCH 04/10] Specify jinja2 required version Signed-off-by: Qi Luo --- src/sonic-config-engine/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index 0ffb0bd10102..1d0c7f99fa41 100755 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -18,7 +18,7 @@ def get_test_suite(): url='https://github.com/Azure/sonic-buildimage', py_modules=['portconfig', 'minigraph', 'openconfig_acl', 'sonic_platform'], scripts=['sonic-cfggen'], - install_requires=['lxml', 'jinja2', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'], + install_requires=['lxml', 'jinja2>=2.10', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'], test_suite='setup.get_test_suite', data_files=[ ('/usr/share/sonic/templates', glob.glob('data/*')), From 798424fa19db6f383b7fd69468189a4288afb998 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 21 Apr 2018 01:15:58 +0000 Subject: [PATCH 05/10] Install new version of jinja2 when building --- sonic-slave/Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index ed85b0ca436a..fc0a80a6f45f 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -184,16 +184,12 @@ RUN apt-get update && apt-get install -y \ libjs-sphinxdoc \ libjs-underscore \ python-docutils \ - python-jinja2 \ python-markupsafe \ python-pygments \ python-roman \ - python-sphinx \ sphinx-common \ - python3-sphinx \ # For sonic config engine testing python-lxml \ - python-jinja2 \ python-netaddr \ python-ipaddr \ python-yaml \ @@ -234,11 +230,13 @@ RUN pip install \ ctypesgen \ crc16 -# For templating -RUN pip install j2cli - # For sonic config engine testing RUN pip install pyangbind==0.5.10 +# Note: force upgrade debian packaged jinja2, if installed +RUN pip install --force-reinstall jinja2>=2.10 + +# For templating (requiring jinja2) +RUN pip install j2cli # For sonic utilities testing RUN pip install click-default-group click natsort tabulate From 6da1febb937cc98647b584b11fd1a6fa2301c175 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 21 Apr 2018 02:12:13 +0000 Subject: [PATCH 06/10] Install latest version of pip when building, so we could install jinja2 Signed-off-by: Qi Luo --- sonic-slave/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index fc0a80a6f45f..7ff61fa3eba8 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -30,11 +30,9 @@ RUN apt-get update && apt-get install -y \ libboost-program-options1.55-dev \ libtinyxml2-dev \ python \ - python-pip \ libncurses5-dev \ texinfo \ dh-autoreconf \ - python3-pip \ doxygen \ devscripts \ git-buildpackage \ @@ -225,6 +223,11 @@ RUN export VERSION=1.8.3 \ && echo 'export GOROOT=/usr/local/go' >> /etc/bash.bashrc \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc +# For latest version pip +RUN curl https://bootstrap.pypa.io/get-pip.py | python3 +# Note: install pip2 later, so pip will be pip2 by default +RUN curl https://bootstrap.pypa.io/get-pip.py | python2 + # For p4 build RUN pip install \ ctypesgen \ From 46ceddac3e44608a4a9f41fd907ed2f3ebe3cb49 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 21 Apr 2018 02:36:47 +0000 Subject: [PATCH 07/10] Refine code Signed-off-by: Qi Luo --- src/sonic-config-engine/sonic-cfggen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 259a9ce3b88c..bdb3c804fc73 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -173,7 +173,7 @@ def main(): }}} deep_update(data, hardware_data) (ports, _) = get_port_config(hwsku, platform, args.port_config) - if not bool(ports): + if not ports: print('Failed to get port config', file=sys.stderr) sys.exit(1) deep_update(data, {'PORT': ports}) From 6aa03c5302bd1045988a3dd67bc194fad36c6734 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Mon, 23 Apr 2018 16:15:54 +0000 Subject: [PATCH 08/10] Fix pip2 version for pyangbind Signed-off-by: Qi Luo --- sonic-slave/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index 7ff61fa3eba8..47b57c177ac1 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -227,6 +227,8 @@ RUN export VERSION=1.8.3 \ RUN curl https://bootstrap.pypa.io/get-pip.py | python3 # Note: install pip2 later, so pip will be pip2 by default RUN curl https://bootstrap.pypa.io/get-pip.py | python2 +# Note: downgrade pip2 so pyangbind 0.6.0 will work +RUN python2 -m pip install --upgrade pip==9.0.3 # For p4 build RUN pip install \ @@ -234,7 +236,7 @@ RUN pip install \ crc16 # For sonic config engine testing -RUN pip install pyangbind==0.5.10 +RUN pip install pyangbind==0.6.0 # Note: force upgrade debian packaged jinja2, if installed RUN pip install --force-reinstall jinja2>=2.10 From fd2998a67176d7d8d86999af90426780eb12fe92 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Mon, 23 Apr 2018 20:44:13 +0000 Subject: [PATCH 09/10] Fix pip jinja Signed-off-by: Qi Luo --- sonic-slave/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index 47b57c177ac1..af699d748a49 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -238,7 +238,7 @@ RUN pip install \ # For sonic config engine testing RUN pip install pyangbind==0.6.0 # Note: force upgrade debian packaged jinja2, if installed -RUN pip install --force-reinstall jinja2>=2.10 +RUN pip install --force-reinstall --upgrade jinja2>=2.10 # For templating (requiring jinja2) RUN pip install j2cli From 2df0f3ad8201748c4b16fe8109b6e9b52775f5db Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Tue, 24 Apr 2018 19:14:47 +0000 Subject: [PATCH 10/10] Refine pip installation Signed-off-by: Qi Luo --- sonic-slave/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sonic-slave/Dockerfile b/sonic-slave/Dockerfile index af699d748a49..213daf1a8e47 100644 --- a/sonic-slave/Dockerfile +++ b/sonic-slave/Dockerfile @@ -30,9 +30,11 @@ RUN apt-get update && apt-get install -y \ libboost-program-options1.55-dev \ libtinyxml2-dev \ python \ + python-pip \ libncurses5-dev \ texinfo \ dh-autoreconf \ + python3-pip \ doxygen \ devscripts \ git-buildpackage \ @@ -223,12 +225,9 @@ RUN export VERSION=1.8.3 \ && echo 'export GOROOT=/usr/local/go' >> /etc/bash.bashrc \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc -# For latest version pip -RUN curl https://bootstrap.pypa.io/get-pip.py | python3 -# Note: install pip2 later, so pip will be pip2 by default -RUN curl https://bootstrap.pypa.io/get-pip.py | python2 -# Note: downgrade pip2 so pyangbind 0.6.0 will work -RUN python2 -m pip install --upgrade pip==9.0.3 +# Upgrade pip2 +# Note: use pip2 specific version so jinja2 2.10 will install +RUN python2 -m pip install -U pip==9.0.3 # For p4 build RUN pip install \