yang models for device_metadata, device_neighbor, flex_counters, crm,…

… versions. Processing code and tests. (#87)

* [sonic-yang-models/yang-models]: Changing sonic-head to sonic-types.

Changes:
1.) Changing sonic-head to sonic-types.
2.) change prefix from head to stypes.
Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

 Conflicts:
	src/sonic-yang-models/yang-models/sonic-acl.yang
	src/sonic-yang-models/yang-models/sonic-extension.yang
	src/sonic-yang-models/yang-models/sonic-types.yang

* [yang_models]: yang models for device_metadata, device_neighbor, flex_counters, crm, versions.

Changes:
1.) yang models for device_metadata, device_neighbor, flex_counters, crm, versions.
Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com:

* [sonic_yang_models]: Includes files in setup.py and fix pyang warnings.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-yang-models/yang-models/sonic-versions.yang]: Version Pattern Correction.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [yangTest.json]: Tests for CRM, BRK CFG, FLEX, DEV_META, DEV_NEIGH and VERSION Yang Models.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [_sonic_yang_ext.py]: Process container and leaves in container.

Changes:
Support container(s), list(s) and leaves inside a container.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [test_sonic_yang.py]: Update test to make sure yang models exist for all tables in input.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic_yang_ext.py]: Minor change because of sonic-head.yang->sonic-types.yang

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic_yang_ext.py]: Minor changes in comments.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -76,7 +76,7 @@ def createDBTableToModuleMap(self):
             # get module name
             moduleName = j['module']['@name']
             # topLevelContainer does not exist in sonic-head and sonic-extension.
-            if "sonic-head" in moduleName or "sonic-extension" in moduleName:
+            if "sonic-types" in moduleName or "sonic-extension" in moduleName:
                 continue;
             # get all top level container
             topLevelContainer = j['module']['container']
@@ -297,41 +297,86 @@ def xlateList(self, model, yang, config, table):
 
         return
 
+    """
+    Process list inside a Container.
+    This function will call xlateList based on list(s) present in Container.
+    """
+    def xlateListInContainer(self, model, yang, configC, table):
+        clist = model
+        #print(clist['@name'])
+        yang[clist['@name']] = list()
+        self.sysLog(msg="xlateProcessListOfContainer: {}".format(clist['@name']))
+        self.xlateList(clist, yang[clist['@name']], configC, table)
+        # clean empty lists
+        if len(yang[clist['@name']]) == 0:
+            del yang[clist['@name']]
+
+        return
+
+    """
+    Process container inside a Container.
+    This function will call xlateContainer based on Container(s) present
+    in outer Container.
+    """
+    def xlateContainerInContainer(self, model, yang, configC, table):
+        ccontainer = model
+        #print(ccontainer['@name'])
+        yang[ccontainer['@name']] = dict()
+        if not configC.get(ccontainer['@name']):
+            return
+        self.sysLog(msg="xlateProcessListOfContainer: {}".format(ccontainer['@name']))
+        self.xlateContainer(ccontainer, yang[ccontainer['@name']], \
+        configC[ccontainer['@name']], table)
+        # clean empty container
+        if len(yang[ccontainer['@name']]) == 0:
+            del yang[ccontainer['@name']]
+        # remove copy after processing
+        del configC[ccontainer['@name']]
+
+        return
+
     """
     Xlate a container
     This function will xlate from a dict in config DB to a Yang JSON container
     using yang model. Output will be stored in self.xlateJson
     """
     def xlateContainer(self, model, yang, config, table):
 
-        # To Handle multiple List, Make a copy of config, because we delete keys
+        # To Handle multiple Lists, Make a copy of config, because we delete keys
         # from config after each match. This is done to match one pkey with one list.
         configC = config.copy()
 
         clist = model.get('list')
         # If single list exists in container,
         if clist and isinstance(clist, dict) and \
            clist['@name'] == model['@name']+"_LIST" and bool(configC):
-                #print(clist['@name'])
-                yang[clist['@name']] = list()
-                self.sysLog(msg="xlateContainer listD {}".format(clist['@name']))
-                self.xlateList(clist, yang[clist['@name']], \
-                               configC, table)
-                # clean empty lists
-                if len(yang[clist['@name']]) == 0:
-                    del yang[clist['@name']]
-                #print(yang[clist['@name']])
-
+                self.xlateListInContainer(clist, yang, configC, table)
         # If multi-list exists in container,
         elif clist and isinstance(clist, list) and bool(configC):
             for modelList in clist:
-                yang[modelList['@name']] = list()
-                self.sysLog(msg="xlateContainer listL {}".format(modelList['@name']))
-                self.xlateList(modelList, yang[modelList['@name']], configC, table)
-                # clean empty lists
-                if len(yang[modelList['@name']]) == 0:
-                    del yang[modelList['@name']]
-
+                self.xlateListInContainer(modelList, yang, configC, table)
+
+        # Handle container(s) in container
+        ccontainer = model.get('container')
+        # If single container exists in container,
+        if ccontainer and isinstance(ccontainer, dict) and bool(configC):
+            self.xlateContainerInContainer(ccontainer, yang, configC, table)
+        # If multi-containers in container,
+        elif ccontainer and isinstance(ccontainer, list) and bool(configC):
+            for modelContainer in ccontainer:
+                self.xlateContainerInContainer(modelContainer, yang, configC, table)
+
+        ## Handle other leaves in container,
+        leafDict = self.createLeafDict(model)
+        for vKey in configC.keys():
+            #vkey must be a leaf\leaf-list\choice in container
+            if leafDict.get(vKey):
+                self.sysLog(syslog.LOG_DEBUG, "xlateContainer vkey {}".format(vKey))
+                yang[vKey] = self.findYangTypedValue(vKey, configC[vKey], leafDict)
+                # delete entry from copy of config
+                del configC[vKey]
+
+        # All entries in copy of config must have been parsed.
         if len(configC):
             self.sysLog(syslog.LOG_ERR, "Alert: Remaining keys in Config")
             raise(Exception("All Keys are not parsed in {}".format(table)))
@@ -446,23 +491,60 @@ def revXlateList(self, model, yang, config, table):
 
         return
 
+    """
+    Rev xlate a list inside a yang container
+    """
+    def revXlateListInContainer(self, model, yang, config, table):
+        modelList = model
+        # Pass matching list from Yang Json if exist
+        if yang.get(modelList['@name']):
+            self.sysLog(msg="revXlateListInContainer {}".format(modelList['@name']))
+            self.revXlateList(modelList, yang[modelList['@name']], config, table)
+        return
+
+    """
+    Rev xlate a container inside a yang container
+    """
+    def revXlateContainerInContainer(self, model, yang, config, table):
+        modelContainer = model
+        # Pass matching list from Yang Json if exist
+        if yang.get(modelContainer['@name']):
+            config[modelContainer['@name']] = dict()
+            self.sysLog(msg="revXlateContainerInContainer {}".format(modelContainer['@name']))
+            self.revXlateContainer(modelContainer, yang[modelContainer['@name']], \
+                config[modelContainer['@name']], table)
+        return
+
     """
     Rev xlate from yang container to table in config DB
     """
     def revXlateContainer(self, model, yang, config, table):
 
-        # Note: right now containers has only LISTs.
         # IF container has only one list
-        if isinstance(model['list'], dict):
-            modelList = model['list']
-            # Pass matching list from Yang Json
-            self.sysLog(msg="revXlateContainer {}".format(modelList['@name']))
-            self.revXlateList(modelList, yang[modelList['@name']], config, table)
-
-        elif isinstance(model['list'], list):
-            for modelList in model['list']:
-                self.sysLog(msg="revXlateContainer {}".format(modelList['@name']))
-                self.revXlateList(modelList, yang[modelList['@name']], config, table)
+        clist = model.get('list')
+        if isinstance(clist, dict):
+            self.revXlateListInContainer(clist, yang, config, table)
+        # IF container has lists
+        elif isinstance(clist, list):
+            for modelList in clist:
+                self.revXlateListInContainer(modelList, yang, config, table)
+
+        ccontainer = model.get('container')
+        # IF container has only one inner container
+        if isinstance(ccontainer, dict):
+            self.revXlateContainerInContainer(ccontainer, yang, config, table)
+        # IF container has many inner containers
+        elif isinstance(ccontainer, list):
+            for modelContainer in ccontainer:
+                self.revXlateContainerInContainer(modelContainer, yang, config, table)
+
+        ## Handle other leaves in container,
+        leafDict = self.createLeafDict(model)
+        for vKey in yang:
+            #vkey must be a leaf\leaf-list\choice in container
+            if leafDict.get(vKey):
+                self.sysLog(syslog.LOG_DEBUG, "revXlateContainer vkey {}".format(vKey))
+                config[vKey] = self.revFindYangTypedValue(vKey, yang[vKey], leafDict)
 
         return
 

src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py

@@ -288,16 +288,17 @@ def test_xlate_rev_xlate(self, sonic_yang_data):
         syc = sonic_yang_data['syc']
 
         jIn = self.readIjsonInput(test_file, 'SAMPLE_CONFIG_DB_JSON')
+        jIn = json.loads(jIn)
+        numTables = len(jIn)
 
-        syc.load_data(json.loads(jIn))
-
-        syc.load_data(json.loads(jIn))
-
-        # TODO: Make sure no extra table is loaded
+        syc.load_data(jIn)
+        # check all tables are loaded and no tables is without Yang Models
+        assert len(syc.jIn) == numTables
+        assert len(syc.tablesWithOutYang) == 0
 
         syc.get_data()
 
-        if syc.jIn and syc.jIn == syc.revXlateJson:
+        if len(syc.jIn) and syc.jIn == syc.revXlateJson:
             print("Xlate and Rev Xlate Passed")
         else:
             print("Xlate and Rev Xlate failed")

src/sonic-yang-models/setup.py

@@ -100,13 +100,19 @@ def run (self):
     setup_requires=setup_requirements,
     version='1.0',
     data_files=[
-        ('yang-models', ['./yang-models/sonic-head.yang',
+        ('yang-models', ['./yang-models/sonic-acl.yang',
+                         './yang-models/sonic-breakout_cfg.yang',
+                         './yang-models/sonic-crm.yang',
+                         './yang-models/sonic-device_metadata.yang',
+                         './yang-models/sonic-device_neighbor.yang',
                          './yang-models/sonic-extension.yang',
-                         './yang-models/sonic-acl.yang',
+                         './yang-models/sonic-flex_counter.yang',
                          './yang-models/sonic-interface.yang',
                          './yang-models/sonic-loopback-interface.yang',
                          './yang-models/sonic-port.yang',
                          './yang-models/sonic-portchannel.yang',
+                         './yang-models/sonic-types.yang',
+                         './yang-models/sonic-versions.yang',
                          './yang-models/sonic-vlan.yang',
                          './yang-models/sonic_yang_tree']),
     ],

src/sonic-yang-models/tests/yang_model_tests/yangModelTesting.py

@@ -132,6 +132,14 @@ def __init__(self, tests, yangDir, jsonFile):
                     'key': 'sonic-acl:stage',
                     'value': 'INGRESS'
                 }
+            },
+            'CRM_BRK_CFG_FLEX_TABLE': {
+                'desc': 'CRM BREAKOUT CFG FLEX COUNTER TABLE.',
+                'eStr': self.defaultYANGFailure['None']
+            },
+            'DEV_META_DEV_NEIGH_VERSION_TABLE': {
+                'desc': 'DEVICE_METADATA DEVICE_NEIGHBOR VERSION TABLE.',
+                'eStr': self.defaultYANGFailure['None']
             }
         }
 

src/sonic-yang-models/tests/yang_model_tests/yangTest.json

@@ -601,6 +601,103 @@
 		}
 	},
 
+	"DEV_META_DEV_NEIGH_VERSION_TABLE": {
+		"sonic-device_metadata:sonic-device_metadata": {
+			"sonic-device_metadata:DEVICE_METADATA": {
+				"localhost": {
+					"bgp_asn": "64850",
+					"mac": "00:11:22:33:44:55",
+					"hostname": "asw.dc",
+					"type": "ToR",
+					"hwsku": "Stone"
+				}
+			}
+		},
+		"sonic-device_neighbor:sonic-device_neighbor": {
+			"sonic-device_neighbor:DEVICE_NEIGHBOR": {
+				"DEVICE_NEIGHBOR_LIST": [
+					{
+						"port": "Eth18",
+						"name": "dccsw03.nw",
+						"peer_name": "Ethernet116"
+					},
+					{
+						"port": "Eth18",
+						"name": "dccsw02.nw",
+						"peer_name": "Ethernet114"
+					},
+					{
+						"port": "Eth18",
+						"name": "dccsw01.nw",
+						"peer_name": "Ethernet112"
+					},
+					{
+						"port": "Eth18",
+						"name": "dccsw04.nw",
+						"peer_name": "Ethernet118"
+					}
+				]
+			}
+		},
+		"sonic-versions:sonic-versions": {
+			"sonic-versions:VERSIONS": {
+				"DATABASE": {
+					"VERSION": "version_2_10_31"
+				}
+			}
+		}
+	},
+
+	"CRM_BRK_CFG_FLEX_TABLE": {
+		"sonic-crm:sonic-crm": {
+			"sonic-crm:CRM": {
+				"Config": {
+					"acl_counter_low_threshold": "70",
+					"acl_counter_high_threshold": "85",
+					"acl_counter_threshold_type": "percentage",
+					"polling_interval": "0"
+				}
+			}
+		},
+		"sonic-breakout_cfg:sonic-breakout_cfg": {
+			"sonic-breakout_cfg:BREAKOUT_CFG": {
+				"BREAKOUT_CFG_LIST": [
+					{
+						"brkout_mode": "1x100G[40G]",
+						"port": "Ethernet0"
+					},
+					{
+						"brkout_mode": "1x100G[40G]",
+						"port": "Ethernet8"
+					},
+					{
+						"brkout_mode": "4x25G",
+						"port": "Ethernet4"
+					}
+				]
+			}
+		},
+		"sonic-flex_counter:sonic-flex_counter": {
+			"sonic-flex_counter:FLEX_COUNTER_TABLE": {
+				"QUEUE": {
+					"FLEX_COUNTER_STATUS": "enable"
+				},
+				"PG_WATERMARK": {
+					"FLEX_COUNTER_STATUS": "enable"
+				},
+				"QUEUE_WATERMARK": {
+					"FLEX_COUNTER_STATUS": "enable"
+				},
+				"PFCWD": {
+					"FLEX_COUNTER_STATUS": "enable"
+				},
+				"PORT": {
+					"FLEX_COUNTER_STATUS": "enable"
+				}
+			}
+		}
+	},
+
 	"ACL_RULE_WRONG_INNER_ETHER_TYPE": {
 		"sonic-acl:sonic-acl": {
 			"sonic-acl:ACL_RULE": {

src/sonic-yang-models/yang-models/sonic-acl.yang

@@ -5,16 +5,12 @@ module sonic-acl {
 	namespace "http://github.com/Azure/sonic-acl";
 	prefix acl;
 
-	import ietf-yang-types {
-		prefix yang;
-	}
-
 	import ietf-inet-types {
 		prefix inet;
 	}
 
-	import sonic-head {
-		prefix head;
+	import sonic-types {
+		prefix stypes;
 		revision-date 2019-07-01;
 	}
 
@@ -70,11 +66,11 @@ module sonic-acl {
 				}
 
 				leaf PACKET_ACTION {
-					type head:packet_action;
+					type stypes:packet_action;
 				}
 
 				leaf IP_TYPE {
-					type head:ip_type;
+					type stypes:ip_type;
 				}
 
 				leaf PRIORITY {
@@ -257,7 +253,7 @@ module sonic-acl {
 
 				leaf type {
 					mandatory true;
-					type head:acl_table_type;
+					type stypes:acl_table_type;
 				}
 
 				leaf stage {