Skip to content

Commit

Permalink
[YANG] Fix issue: Non compliant leaf list in config_db schema (#10291)
Browse files Browse the repository at this point in the history 
#### Why I did it

Fix issue: Non compliant leaf list in config_db schema: sonic-net/sonic-buildimage#9801

#### How I did it

The basic flow of DPB is like:
1.	Transfer config db json value to YANG json value, name it “yangIn”
2.	Validate “yangIn” by libyang
3.	Generate a YANG json value to represent the target configuration, name it “yangTarget”
4.	Do diff between “yangIn” and “yangTarget”
5.	Apply the diff to CONFIG DB json and save it back to DB
 
The fix:
•	For step #1, If value of a leaf-list field string type, transfer it to a list by splitting it with “,” the purpose here is to make step#2 happy. We also need to save <table_name>.<key>.<field_name> to a set named “leaf_list_with_string_value_set”.
•	For step#5, loop “leaf_list_with_string_value_set” and change those fields back to a string.


#### How to verify it

1. Manual test
2. Changed sample config DB and unit test passed
  • Loading branch information
@Junchao-Mellanox
Junchao-Mellanox authored May 5, 2022
1 parent 7c4ee43 commit f850122
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 14 deletions.
4 changes: 3 additions & 1 deletion src/sonic-yang-mgmt/sonic_yang.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_opt
# below dict will store preProcessed yang objects, which may be needed by
# all yang modules, such as grouping.
self.preProcessedYang = dict()

# element path for CONFIG DB. An example for this list could be:
# ['PORT', 'Ethernet0', 'speed']
self.elementPath = []
try:
self.ctx = ly.Context(yang_dir, sonic_yang_options)
except Exception as e:
Expand Down
58 changes: 53 additions & 5 deletions src/sonic-yang-mgmt/sonic_yang_ext.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,19 @@
'CABLE_LENGTH_LIST'
]

# Workaround for those fields who is defined as leaf-list in YANG model but have string value in config DB.
# Dictinary structure key = (<table_name>, <field_name>), value = seperator
LEAF_LIST_WITH_STRING_VALUE_DICT = {
('MIRROR_SESSION', 'src_ip'): ',',
('NTP', 'src_intf'): ';',
('BGP_ALLOWED_PREFIXES', 'prefixes_v4'): ',',
('BGP_ALLOWED_PREFIXES', 'prefixes_v6'): ',',
('BUFFER_PORT_EGRESS_PROFILE_LIST', 'profile_list'): ',',
('BUFFER_PORT_INGRESS_PROFILE_LIST', 'profile_list'): ',',
('PORT', 'adv_speeds'): ',',
('PORT', 'adv_interface_types'): ',',
}

"""
This is the Exception thrown out of all public function of this class.
"""
Expand Down Expand Up @@ -407,6 +420,11 @@ def _yangConvert(val):
# if it is a leaf-list do it for each element
if leafDict[key]['__isleafList']:
vValue = list()
if isinstance(value, str) and (self.elementPath[0], self.elementPath[-1]) in LEAF_LIST_WITH_STRING_VALUE_DICT:
# For field defined as leaf-list but has string value in CONFIG DB, need do special handling here. For exampe:
# port.adv_speeds in CONFIG DB has value "100,1000,10000", it shall be transferred to [100,1000,10000] as YANG value here to
# make it align with its YANG definition.
value = (x.strip() for x in value.split(LEAF_LIST_WITH_STRING_VALUE_DICT[(self.elementPath[0], self.elementPath[-1])]))
for v in value:
vValue.append(_yangConvert(v))
else:
Expand Down Expand Up @@ -545,16 +563,21 @@ def _xlateList(self, model, yang, config, table, exceptionList):
primaryKeys = list(config.keys())
for pkey in primaryKeys:
try:
self.elementPath.append(pkey)
vKey = None
self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\
format(pkey))
# Find and extracts key from each dict in config
keyDict = self._extractKey(pkey, listKeys)
# fill rest of the values in keyDict
for vKey in config[pkey]:
self.elementPath.append(vKey)
self.sysLog(syslog.LOG_DEBUG, "xlateList vkey {}".format(vKey))
keyDict[vKey] = self._findYangTypedValue(vKey, \
config[pkey][vKey], leafDict)
try:
keyDict[vKey] = self._findYangTypedValue(vKey, \
config[pkey][vKey], leafDict)
finally:
self.elementPath.pop()
yang.append(keyDict)
# delete pkey from config, done to match one key with one list
del config[pkey]
Expand All @@ -566,6 +589,8 @@ def _xlateList(self, model, yang, config, table, exceptionList):
exceptionList.append(str(e))
# with multilist, we continue matching other keys.
continue
finally:
self.elementPath.pop()

return

Expand Down Expand Up @@ -596,13 +621,17 @@ def _xlateContainerInContainer(self, model, yang, configC, table):
if ccName not in configC:
# Inner container doesn't exist in config
return

if len(configC[ccName]) == 0:
# Empty container, clean config and return
del configC[ccName]
return
self.sysLog(msg="xlateProcessListOfContainer: {}".format(ccName))
self.elementPath.append(ccName)
self._xlateContainer(ccontainer, yang[ccName], \
configC[ccName], table)
self.elementPath.pop()

# clean empty container
if len(yang[ccName]) == 0:
del yang[ccName]
Expand Down Expand Up @@ -650,8 +679,10 @@ def _xlateContainer(self, model, yang, config, table):
for vKey in vKeys:
#vkey must be a leaf\leaf-list\choice in container
if leafDict.get(vKey):
self.elementPath.append(vKey)
self.sysLog(syslog.LOG_DEBUG, "xlateContainer vkey {}".format(vKey))
yang[vKey] = self._findYangTypedValue(vKey, configC[vKey], leafDict)
self.elementPath.pop()
# delete entry from copy of config
del configC[vKey]

Expand Down Expand Up @@ -681,8 +712,10 @@ def _xlateConfigDBtoYang(self, jIn, yangJ):
yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key]
yangJ[key][subkey] = dict()
self.sysLog(msg="xlateConfigDBtoYang {}:{}".format(key, subkey))
self.elementPath.append(table)
self._xlateContainer(cmap['container'], yangJ[key][subkey], \
jIn[table], table)
self.elementPath = []

return

Expand Down Expand Up @@ -739,9 +772,14 @@ def _revYangConvert(val):

# if it is a leaf-list do it for each element
if leafDict[key]['__isleafList']:
vValue = list()
for v in value:
vValue.append(_revYangConvert(v))
if isinstance(value, list) and (self.elementPath[0], self.elementPath[-1]) in LEAF_LIST_WITH_STRING_VALUE_DICT:
# For field defined as leaf-list but has string value in CONFIG DB, we need do special handling here:
# e.g. port.adv_speeds is [10,100,1000] in YANG, need to convert it into a string for CONFIG DB: "10,100,1000"
vValue = LEAF_LIST_WITH_STRING_VALUE_DICT[(self.elementPath[0], self.elementPath[-1])].join((_revYangConvert(x) for x in value))
else:
vValue = list()
for v in value:
vValue.append(_revYangConvert(v))
elif leafDict[key]['type']['@name'] == 'boolean':
vValue = 'true' if value else 'false'
else:
Expand Down Expand Up @@ -850,12 +888,16 @@ def _revXlateList(self, model, yang, config, table):
# create key of config DB table
pkey, pkeydict = self._createKey(entry, listKeys)
self.sysLog(syslog.LOG_DEBUG, "revXlateList pkey:{}".format(pkey))
self.elementPath.append(pkey)
config[pkey]= dict()
# fill rest of the entries
for key in entry:
if key not in pkeydict:
self.elementPath.append(key)
config[pkey][key] = self._revFindYangTypedValue(key, \
entry[key], leafDict)
self.elementPath.pop()
self.elementPath.pop()

return

Expand All @@ -879,8 +921,10 @@ def _revXlateContainerInContainer(self, model, yang, config, table):
if yang.get(modelContainer['@name']):
config[modelContainer['@name']] = dict()
self.sysLog(msg="revXlateContainerInContainer {}".format(modelContainer['@name']))
self.elementPath.append(modelContainer['@name'])
self._revXlateContainer(modelContainer, yang[modelContainer['@name']], \
config[modelContainer['@name']], table)
self.elementPath.pop()
return

"""
Expand Down Expand Up @@ -912,7 +956,9 @@ def _revXlateContainer(self, model, yang, config, table):
#vkey must be a leaf\leaf-list\choice in container
if leafDict.get(vKey):
self.sysLog(syslog.LOG_DEBUG, "revXlateContainer vkey {}".format(vKey))
self.elementPath.append(vKey)
config[vKey] = self._revFindYangTypedValue(vKey, yang[vKey], leafDict)
self.elementPath.pop()

return

Expand Down Expand Up @@ -940,8 +986,10 @@ def _revXlateYangtoConfigDB(self, yangJ, cDbJson):
cDbJson[table] = dict()
#print(key + "--" + subkey)
self.sysLog(msg="revXlateYangtoConfigDB {}".format(table))
self.elementPath.append(table)
self._revXlateContainer(cmap['container'], yangJ[module_top][container], \
cDbJson[table], table)
self.elementPath = []

return

Expand Down
19 changes: 11 additions & 8 deletions src/sonic-yang-models/tests/files/sample_config_db.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@
},
"BUFFER_PORT_INGRESS_PROFILE_LIST": {
"Ethernet9": {
"profile_list": ["ingress_lossy_profile"]
"profile_list": "ingress_lossy_profile"
}
},
"BUFFER_PORT_EGRESS_PROFILE_LIST": {
"Ethernet9": {
"profile_list": ["egress_lossless_profile", "egress_lossy_profile"]
"profile_list": "egress_lossless_profile,egress_lossy_profile"
}
},
"PORTCHANNEL": {
Expand Down Expand Up @@ -408,10 +408,7 @@
"NTP": {
"global": {
"vrf": "mgmt",
"src_intf": [
"eth0",
"Loopback0"
]
"src_intf": "eth0;Loopback0"
}
},
"NTP_SERVER": {
Expand Down Expand Up @@ -449,15 +446,21 @@
"description": "",
"speed": "11100",
"tpid": "0x8100",
"admin_status": "up"
"admin_status": "up",
"autoneg": "on",
"adv_speeds": "100000,50000",
"adv_interface_types": "CR,CR4"
},
"Ethernet2": {
"alias": "Eth1/3",
"lanes": "67",
"description": "",
"speed": "11100",
"tpid": "0x8100",
"admin_status": "up"
"admin_status": "up",
"autoneg": "on",
"adv_speeds": "all",
"adv_interface_types": "all"
},
"Ethernet3": {
"alias": "Eth1/4",
Expand Down

0 comments on commit f850122

Please sign in to comment.