[sonic-yang-mgmt]: Move yang models to src/sonic-yang-models and bui…

…ld infra changes. (#54)

* [sonic-yang-mgmt]: Move yang models to src/sonic-yang-models and build infra changes.

1.) Move yang models to src/sonic-yang-models.
2.) New build infra for src/sonic-yang-models.
3.) Change in build infra for src/sonic-yang-mgmt.
4.) Makefile changes to make sonic-yang-mgmt depend on sonic-yang-models.
5.) Test files changes.

* [sonic-yang-mgmt/setup.py]: Find path of dependencies of os environment.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-yang-mgmt/setup.py]: Change Python tag from Py2 to Py.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-extension.yang]: Re-add sonic-extension file.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-slave-jessie/Dockerfile.j2]: Remove ijson from jessie.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [LICENSE]: Changing sonic-yang-mgmt LICENSE.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-yang-models/LICENSE]: Changing license to APACHE.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [rules/sonic-yang-mgmt-py2.mk]: Reverting back to PY2.

Changes:
-- Reverting back to PY2.
-- Keeping package environment varible name to SONIC_YANG_MGMT_PY

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-yang-models/setup.py]: Address LGTM issues and rebasing.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

files/build_templates/sonic_debian_extension.j2

@@ -116,14 +116,18 @@ sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
 
-# Install sonic-yang-mgmt Python 2 package, install dependencies
+# Install sonic-yang-models py3 and sonic-yang-mgmt Python 2 package, install dependencies
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb
-SONIC_YANG_MGMT_PY2_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py2_wheel_path}})
-sudo cp {{sonic_yang_mgmt_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY2_WHEEL_NAME
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_YANG_MGMT_PY2_WHEEL_NAME
-sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY2_WHEEL_NAME
+SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}})
+sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
+SONIC_YANG_MGMT_PY_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py_wheel_path}})
+sudo cp {{sonic_yang_mgmt_py_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_YANG_MGMT_PY_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY_WHEEL_NAME
 
 # Install sonic-platform-common Python 2 package
 PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}})

rules/sonic-yang-mgmt-py2.mk

@@ -1,8 +1,9 @@
 # sonic-yang-mgmt python2 wheel
 
-SONIC_YANG_MGMT_PY2 = sonic_yang_mgmt-1.0-py2-none-any.whl
-$(SONIC_YANG_MGMT_PY2)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
-$(SONIC_YANG_MGMT_PY2)_PYTHON_VERSION = 2
-$(SONIC_YANG_MGMT_PY2)_DEBS_DEPENDS = $(LIBYANG)
+SONIC_YANG_MGMT_PY = sonic_yang_mgmt-1.0-py2-none-any.whl
+$(SONIC_YANG_MGMT_PY)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
+$(SONIC_YANG_MGMT_PY)_PYTHON_VERSION = 2
+$(SONIC_YANG_MGMT_PY)_DEBS_DEPENDS = $(LIBYANG)
+$(SONIC_YANG_MGMT_PY)_DEPENDS = $(SONIC_YANG_MODELS_PY3)
 
-SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY2)
+SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY)

rules/sonic-yang-models-py3.mk

@@ -0,0 +1,7 @@
+SONIC_YANG_MODELS_PY3 = sonic_yang_models-1.0-py3-none-any.whl
+$(SONIC_YANG_MODELS_PY3)_SRC_PATH = $(SRC_PATH)/sonic-yang-models
+$(SONIC_YANG_MODELS_PY3)_PYTHON_VERSION = 3
+$(SONIC_YANG_MODELS_PY3)_DEBS_DEPENDS = $(LIBYANG)
+
+SONIC_PYTHON_WHEELS += $(SONIC_YANG_MODELS_PY3)
+export SONIC_YANG_MODELS_PY3

slave.mk

@@ -49,6 +49,8 @@ export BUILD_NUMBER
 export BUILD_TIMESTAMP
 export CONFIGURED_PLATFORM
 export CONFIGURED_ARCH
+export STRETCH_DEBS_PATH
+export PYTHON_WHEELS_PATH
 
 ###############################################################################
 ## Utility rules
@@ -639,7 +641,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
-        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2))
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))\
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))
 	$(HEADER)
 	# Pass initramfs and linux kernel explicitly. They are used for all platforms
 	export debs_path="$(STRETCH_DEBS_PATH)"
@@ -668,7 +671,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
 	export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))"
 	export install_debug_image="$(INSTALL_DEBUG_TOOLS)"
-	export sonic_yang_mgmt_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2))"
+	export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))"
+	export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))"
 
 	$(foreach docker, $($*_DOCKERS),\
 		export docker_image="$(docker)"

sonic-slave-jessie/Dockerfile.j2

@@ -322,9 +322,6 @@ RUN pip install mockredispy==2.9.3
 RUN pip install pytest-runner==4.4
 RUN pip install setuptools==40.8.0
 
-# For sonic_yang_mgmt build
-RUN pip install ijson
-
 # Install dependencies for isc-dhcp-relay build
 RUN apt-get -y build-dep isc-dhcp
 

sonic-slave-stretch/Dockerfile.j2

@@ -372,6 +372,7 @@ RUN pip install setuptools==40.8.0
 
 # For sonic_yang_mgmt build
 RUN pip install ijson==2.6.1
+RUN pip3 install ijson==2.6.1
 RUN pip install jsondiff==1.2.0
 RUN pip install xmltodict==0.12.0
 RUN pip install pyang==2.1.1

src/sonic-yang-mgmt/LICENSE

@@ -1,32 +1,13 @@
-GNU GENERAL PUBLIC LICENSE
-    Version 3, 29 June 2007
+Copyright 2019 Microsoft, Inc
 
-    Copyright (C) 2019  Praveen Chaudhary
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+    http://www.apache.org/licenses/LICENSE-2.0
 
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

src/sonic-yang-mgmt/README.rst

@@ -1,7 +1,6 @@
 "
-This Package contains YANG models for sonic which are written with guidelines mentioned in
-https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md.
-
-In future this package will include python yang libraries which will be used with sonic utilities
-pacakge to validate the config.
+This package includes python yang libraries which will be used with sonic utilities
+pacakge to validate the config. This python libraries are written on top of libyang
+and also provides functionality to translate the config from SONiC ConfigDB to SONiC
+YANG and vice-versa.
 "

src/sonic-yang-mgmt/setup.py

@@ -10,10 +10,20 @@
 import pytest
 import os
 
+# find path of pkgs from os environment vars
+prefix = '/sonic'; debs = os.environ["STRETCH_DEBS_PATH"]
+wheels = os.environ["PYTHON_WHEELS_PATH"]
+wheels_path = '{}/{}'.format(prefix, wheels)
+deps_path = '{}/{}'.format(prefix, debs)
+# dependencies
+libyang = '{}/{}'.format(deps_path, os.environ["LIBYANG"])
+libyangCpp = '{}/{}'.format(deps_path, os.environ["LIBYANG_CPP"])
+libyangPy2 = '{}/{}'.format(deps_path, os.environ["LIBYANG_PY2"])
+libyangPy3 = '{}/{}'.format(deps_path, os.environ["LIBYANG_PY3"])
+sonicYangModels = '{}/{}'.format(wheels_path, os.environ["SONIC_YANG_MODELS_PY3"])
+
 # important reuirements parameters
-build_requirements = ['../../target/debs/stretch/libyang_1.0.73_amd64.deb',
-                        '../../target/debs/stretch/libyang-cpp_1.0.73_amd64.deb',
-                        '../../target/debs/stretch/python2-yang_1.0.73_amd64.deb',]
+build_requirements = [libyang, libyangCpp, libyangPy2, libyangPy3, sonicYangModels,]
 
 setup_requirements = ['pytest-runner']
 
@@ -28,37 +38,22 @@ class pkgBuild(build_py):
     """Custom Build PLY"""
 
     def run (self):
-        # json file for YANG model test cases.
-        test_yangJson_file = './tests/yang-model-tests/yangTest.json'
-        # YANG models are in below dir
-        yang_model_dir = './yang-models/'
-        # yang model tester python module
-        yang_test_py = './tests/yang-model-tests/yangModelTesting.py'
-        #  install libyang
+        #  install libyang and sonic_yang_models
         for req in build_requirements:
-            if 'target/debs'in req:
+            if '.deb' in req:
                 pkg_install_cmd = "sudo dpkg -i {}".format(req)
                 if (system(pkg_install_cmd)):
                     print("{} installation failed".format(req))
                     exit(1)
                 else:
                     print("{} installed".format(req))
-
-        #  run tests for yang models
-        test_yang_cmd = "python {} -f {} -y {}".format(yang_test_py, test_yangJson_file, yang_model_dir)
-        if (system(test_yang_cmd)):
-            print("YANG Tests failed\n")
-            # below line will be uncommented after libyang python support PR #
-            exit(1)
-        else:
-            print("YANG Tests passed\n")
-
-        # Generate YANG Tree
-        pyang_tree_cmd = "pyang -f tree ./yang-models/*.yang > ./yang-models/sonic_yang_tree"
-        if (system(pyang_tree_cmd)):
-            print("Failed: {}".format(pyang_tree_cmd))
-        else:
-            print("Passed: {}".format(pyang_tree_cmd))
+            elif '.whl' in req:
+                pkg_install_cmd = "pip3 install {}".format(req)
+                if (system(pkg_install_cmd)):
+                    print("{} installation failed".format(req))
+                    exit(1)
+                else:
+                    print("{} installed".format(req))
 
         # run pytest for libyang python APIs
         self.pytest_args = []
@@ -89,7 +84,7 @@ def run (self):
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
     ],
-    description="Package contains YANG models for sonic.",
+    description="Package contains Python Library for YANG for sonic.",
     tests_require = test_requirements,
     license="GNU General Public License v3",
     long_description=readme + '\n\n',
@@ -100,14 +95,5 @@ def run (self):
     packages=find_packages(),
     setup_requires=setup_requirements,
     version='1.0',
-    data_files=[
-        ('yang-models', ['./yang-models/sonic-head.yang',
-                         './yang-models/sonic-acl.yang',
-                         './yang-models/sonic-interface.yang',
-                         './yang-models/sonic-port.yang',
-                         './yang-models/sonic-portchannel.yang',
-                         './yang-models/sonic-vlan.yang',
-                         './yang-models/sonic_yang_tree']),
-    ],
     zip_safe=False,
 )

src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py

@@ -21,7 +21,6 @@
 
 class Test_SonicYang(object):
     # class vars
-    yang_test_file = "/sonic/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json"
 
     @pytest.fixture(autouse=True, scope='class')
     def data(self):
@@ -48,13 +47,13 @@ def jsonTestParser(self, file):
         Get the JSON input based on func name
         and return jsonInput
     """
-    def readIjsonInput(self, test):
+    def readIjsonInput(self, yang_test_file, test):
         try:
             # load test specific Dictionary, using Key = func
             # this is to avoid loading very large JSON in memory
             print(" Read JSON Section: " + test)
             jInput = ""
-            with open(self.yang_test_file, 'rb') as f:
+            with open(yang_test_file, 'rb') as f:
                 jInst = ijson_itmes(f, test)
                 for it in jInst:
                     jInput = jInput + json.dumps(it)
@@ -261,10 +260,12 @@ def test_get_leafref_type_schema(self, yang_s, data):
             assert expected_type == data_type
 
     def test_xlate_rev_xlate(self):
-        # This Test is with Sonic YANG model, so create class from start
-        # read the config
-        yang_dir = "/sonic/src/sonic-yang-mgmt/yang-models/"
-        jIn = self.readIjsonInput('SAMPLE_CONFIG_DB_JSON')
+        # In this test, xlation and revXlation is tested with latest Sonic
+        # YANG model.
+
+        yang_dir = "/sonic/src/sonic-yang-models/yang-models/"
+        yang_test_file = "/sonic/src/sonic-yang-models/tests/yang_model_tests/yangTest.json"
+        jIn = self.readIjsonInput(yang_test_file, 'SAMPLE_CONFIG_DB_JSON')
         # load yang models
         syc = sy.sonic_yang(yang_dir)
 

src/sonic-yang-models/AUTHORS.rst

@@ -0,0 +1,14 @@
+=======
+Credits
+=======
+
+Development Lead
+----------------
+
+LNOS-CODERS    <lnos-coders@linkedin.com>
+MSFT-LINUX-DEV <linuxnetdev@microsoft.com>
+
+Contributors
+------------
+
+Praveen Chaudhary      <pchaudhary@linkedin.com>

src/sonic-yang-models/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2019 Microsoft, Inc
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

src/sonic-yang-models/README.rst

@@ -0,0 +1,4 @@
+"
+This Package contains YANG models for sonic which are written with guidelines mentioned in
+https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md.
+"