From f3fc40f5bc394e6825fddb40c795a78cce5620c8 Mon Sep 17 00:00:00 2001 From: Partha Dutta <51353699+dutta-partha@users.noreply.github.com> Date: Fri, 31 Jul 2020 04:50:41 +0530 Subject: [PATCH] CVL Changes #1: Initial CVL code reorganization and common infra changes (#18) - Why I did it As part of CVL code update and new features addition, the existing CVL code has to be reorganized and common infra needs to be changed. - How I did it N/A. Please refer to change log for more details. - How to verify it No specific test cases to be executed. Run cvl go test cases using - 'make cvl-test'. - Description for the changelog Modify Makefiles Modify CVL Makefile for cleanall rule Modify Makefile for CVL YIN schema generation Modify Makefile for CVL test YIN schema generation Moved cache, syntax and semantic related code from cvl.go to cvl_cache.go, cvl_syntax.go and cvl_semantics.go respectively . Added new API signatures for integration. Implementation will be added in subsequent PRs. Copied sonic-port.yang and sonic-acl.yang to CVL own's test folder. --- cvl/Makefile | 2 + cvl/cvl.go | 460 ------------------ cvl/cvl_api.go | 52 ++ cvl/cvl_cache.go | 234 +++++++++ cvl/cvl_semantics.go | 123 +++++ cvl/cvl_syntax.go | 203 ++++++++ cvl/cvl_test.go | 17 - cvl/schema/Makefile | 44 +- cvl/testdata/schema/Makefile | 42 +- cvl/testdata/schema/sonic-acl.yang | 241 +++++++++ cvl/testdata/schema/sonic-port.yang | 124 +++++ models/yang/sonic/common/sonic-common.yang | 8 + models/yang/sonic/common/sonic-extension.yang | 11 +- 13 files changed, 1042 insertions(+), 519 deletions(-) create mode 100644 cvl/cvl_cache.go create mode 100644 cvl/cvl_semantics.go create mode 100644 cvl/cvl_syntax.go create mode 100644 cvl/testdata/schema/sonic-acl.yang create mode 100644 cvl/testdata/schema/sonic-port.yang diff --git a/cvl/Makefile b/cvl/Makefile index 28f0e240676e..278d97e6acfc 100644 --- a/cvl/Makefile +++ b/cvl/Makefile @@ -70,3 +70,5 @@ clean: $(RM) -r $(CVL_PKG) $(RM) -r $(CVL_TEST_DIR) +cleanall:clean + diff --git a/cvl/cvl.go b/cvl/cvl.go index 858aa08fafab..98922aa4b0ef 100644 --- a/cvl/cvl.go +++ b/cvl/cvl.go @@ -25,7 +25,6 @@ import ( "regexp" "time" log "github.com/golang/glog" - "encoding/json" "github.com/go-redis/redis" "github.com/antchfx/xmlquery" "github.com/antchfx/jsonquery" @@ -33,7 +32,6 @@ import ( . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "sync" "flag" - "runtime" ) //DB number @@ -557,14 +555,6 @@ func getRedisToYangKeys(tableName string, redisKey string)[]keyValuePairStruct{ return mkeys } - -//Add child node to a parent node -func(c *CVL) addChildNode(tableName string, parent *yparser.YParserNode, name string) *yparser.YParserNode { - - //return C.lyd_new(parent, modelInfo.tableInfo[tableName].module, C.CString(name)) - return c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, parent, name) -} - //Check for path resolution func (c *CVL) checkPathForTableEntry(tableName string, currentValue string, cfgData *CVLEditConfigData, mustExpStk []string, token string) ([]string, string, CVLRetCode) { @@ -929,54 +919,6 @@ func (c *CVL) addTableEntryToCache(tableName string, redisKey string) { } } -//Check delete constraint for leafref if key/field is deleted -func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData, - tableName, keyVal, field string) CVLRetCode { - var leafRefs []tblFieldPair - if (field != "") { - //Leaf or field is getting deleted - leafRefs = c.findUsedAsLeafRef(tableName, field) - } else { - //Entire entry is getting deleted - leafRefs = c.findUsedAsLeafRef(tableName, modelInfo.tableInfo[tableName].keys[0]) - } - - //The entry getting deleted might have been referred from multiple tables - //Return failure if at-least one table is using this entry - for _, leafRef := range leafRefs { - TRACE_LOG(INFO_API, (TRACE_DELETE | TRACE_SEMANTIC), "Checking delete constraint for leafRef %s/%s", leafRef.tableName, leafRef.field) - //Check in dependent data first, if the referred entry is already deleted - leafRefDeleted := false - for _, cfgDataItem := range cfgData { - if (cfgDataItem.VType == VALIDATE_NONE) && - (cfgDataItem.VOp == OP_DELETE ) && - (strings.HasPrefix(cfgDataItem.Key, (leafRef.tableName + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim + keyVal + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim))) { - //Currently, checking for one entry is being deleted in same session - //We should check for all entries - leafRefDeleted = true - break - } - } - - if (leafRefDeleted == true) { - continue //check next leafref - } - - //Else, check if any referred enrty is present in DB - var nokey []string - refKeyVal, err := luaScripts["find_key"].Run(redisClient, nokey, leafRef.tableName, - modelInfo.tableInfo[leafRef.tableName].redisKeyDelim, leafRef.field, keyVal).Result() - if (err == nil && refKeyVal != "") { - CVL_LOG(ERROR, "Delete will violate the constraint as entry %s is referred in %s", tableName, refKeyVal) - - return CVL_SEMANTIC_ERROR - } - } - - - return CVL_SUCCESS -} - //Add the data which are referring this key func (c *CVL) updateDeleteDataToCache(tableName string, redisKey string) { if _, existing := c.tmpDbCache[tableName]; existing == false { @@ -1088,24 +1030,6 @@ func (c *CVL) addLeafRef(config bool, tableName string, name string, value strin } } - -func (c *CVL) addChildLeaf(config bool, tableName string, parent *yparser.YParserNode, name string, value string) { - - /* If there is no value then assign default space string. */ - if len(value) == 0 { - value = " " - } - - //Batch leaf creation - c.batchLeaf = c.batchLeaf + name + "#" + value + "#" - //Check if this leaf has leafref, - //If so add the add redis key to its table so that those - // details can be fetched for dependency validation - - c.addLeafRef(config, tableName, name, value) -} - - func (c *CVL) checkFieldMap(fieldMap *map[string]string) map[string]interface{} { fieldMapNew := map[string]interface{}{} @@ -1138,351 +1062,6 @@ func mergeMap(dest map[string]string, src map[string]string) { } } -// Fetch dependent data from validated data cache, -// Returns the data and flag to indicate that if requested data -// is found in update request, the data should be merged with Redis data -func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (map[string]string, bool) { - cfgDataArr := c.requestCache[tableName][key] - if (cfgDataArr != nil) { - for _, cfgReqData := range cfgDataArr { - //Delete request doesn't have depedent data - if (cfgReqData.VOp == OP_CREATE) { - return cfgReqData.Data, false - } else if (cfgReqData.VOp == OP_UPDATE) { - return cfgReqData.Data, true - } - } - } - - return nil, false -} - -//Fetch given table entries using pipeline -func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]interface{}) int { - - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now()) - - totalCount := len(dbKeys) - if (totalCount == 0) { - //No entry to be fetched - return 0 - } - - entryFetched := 0 - bulkCount := 0 - bulkKeys := []string{} - for dbKey, val := range dbKeys { //for all keys - - if (val != nil) { //skip entry already fetched - mapTable := c.tmpDbCache[tableName] - delete(mapTable.(map[string]interface{}), dbKey) //delete entry already fetched - totalCount = totalCount - 1 - if(bulkCount != totalCount) { - //If some entries are remaining go back to 'for' loop - continue - } - } else { - //Accumulate entries to be fetched - bulkKeys = append(bulkKeys, dbKey) - bulkCount = bulkCount + 1 - } - - if(bulkCount != totalCount) && ((bulkCount % MAX_BULK_ENTRIES_IN_PIPELINE) != 0) { - //If some entries are remaining and bulk bucket is not filled, - //go back to 'for' loop - continue - } - - mCmd := map[string]*redis.StringStringMapCmd{} - - pipe := redisClient.Pipeline() - - for _, dbKey := range bulkKeys { - - redisKey := tableName + modelInfo.tableInfo[tableName].redisKeyDelim + dbKey - //Check in validated cache first and add as dependent data - if entry, mergeNeeded := c.fetchDataFromRequestCache(tableName, dbKey); (entry != nil) { - c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry - entryFetched = entryFetched + 1 - //Entry found in validated cache, so skip fetching from Redis - //if merging is not required with Redis DB - if (mergeNeeded == false) { - continue - } - } - - //Otherwise fetch it from Redis - mCmd[dbKey] = pipe.HGetAll(redisKey) //write into pipeline - if mCmd[dbKey] == nil { - CVL_LOG(ERROR, "Failed pipe.HGetAll('%s')", redisKey) - } - } - - _, err := pipe.Exec() - if err != nil { - CVL_LOG(ERROR, "Failed to fetch details for table %s", tableName) - return 0 - } - pipe.Close() - bulkKeys = nil - - mapTable := c.tmpDbCache[tableName] - - for key, val := range mCmd { - res, err := val.Result() - if (err != nil || len(res) == 0) { - //no data found, don't keep blank entry - delete(mapTable.(map[string]interface{}), key) - continue - } - //exclude table name and delim - keyOnly := key - - if (mapTable.(map[string]interface{})[keyOnly] != nil) { - tmpFieldMap := (mapTable.(map[string]interface{})[keyOnly]).(map[string]string) - //merge with validated cache data - mergeMap(res, tmpFieldMap) - fieldMap := c.checkFieldMap(&res) - mapTable.(map[string]interface{})[keyOnly] = fieldMap - } else { - fieldMap := c.checkFieldMap(&res) - mapTable.(map[string]interface{})[keyOnly] = fieldMap - } - - entryFetched = entryFetched + 1 - } - - runtime.Gosched() - } - - TRACE_LOG(INFO_API, TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now()) - - return entryFetched -} - -//populate redis data to cache -func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now()) - - entryToFetch := 0 - var root *yparser.YParserNode = nil - var errObj yparser.YParserError - - for entryToFetch = 1; entryToFetch > 0; { //Force to enter the loop for first time - //Repeat until all entries are fetched - entryToFetch = 0 - for tableName, dbKeys := range c.tmpDbCache { //for each table - entryToFetch = entryToFetch + c.fetchTableDataToTmpCache(tableName, dbKeys.(map[string]interface{})) - } //for each table - - //If no table entry delete the table itself - for tableName, dbKeys := range c.tmpDbCache { //for each table - if (len(dbKeys.(map[string]interface{})) == 0) { - delete(c.tmpDbCache, tableName) - continue - } - } - - if (entryToFetch == 0) { - //No more entry to fetch - break - } - - if (Tracing == true) { - jsonDataBytes, _ := json.Marshal(c.tmpDbCache) - jsonData := string(jsonDataBytes) - TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonData) - } - - data, err := jsonquery.ParseJsonMap(&c.tmpDbCache) - - if (err != nil) { - return nil - } - - //Build yang tree for each table and cache it - for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { - TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonNode.Data) - //Visit each top level list in a loop for creating table data - topNode, _ := c.generateTableData(true, jsonNode) - if (root == nil) { - root = topNode - } else { - if root, errObj = c.yp.MergeSubtree(root, topNode); errObj.ErrCode != yparser.YP_SUCCESS { - return nil - } - } - } - } // until all dependent data is fetched - - if root != nil && Tracing == true { - dumpStr := c.yp.NodeDump(root) - TRACE_LOG(INFO_DETAIL, TRACE_CACHE, "Dependent Data = %v\n", dumpStr) - } - - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now()) - return root -} - - -func (c *CVL) clearTmpDbCache() { - for key, _ := range c.tmpDbCache { - delete(c.tmpDbCache, key) - } -} - -func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *jsonquery.Node, -parent *yparser.YParserNode) CVLRetCode { - - //Traverse fields - for jsonFieldNode := jsonNode.FirstChild; jsonFieldNode!= nil; - jsonFieldNode = jsonFieldNode.NextSibling { - //Add fields as leaf to the list - if (jsonFieldNode.Type == jsonquery.ElementNode && - jsonFieldNode.FirstChild != nil && - jsonFieldNode.FirstChild.Type == jsonquery.TextNode) { - - if (len(modelInfo.tableInfo[tableName].mapLeaf) == 2) {//mapping should have two leaf always - //Values should be stored inside another list as map table - listNode := c.addChildNode(tableName, parent, tableName) //Add the list to the top node - c.addChildLeaf(config, tableName, - listNode, modelInfo.tableInfo[tableName].mapLeaf[0], - jsonFieldNode.Data) - - c.addChildLeaf(config, tableName, - listNode, modelInfo.tableInfo[tableName].mapLeaf[1], - jsonFieldNode.FirstChild.Data) - - } else { - //check if it is hash-ref, then need to add only key from "TABLE|k1" - hashRefMatch := reHashRef.FindStringSubmatch(jsonFieldNode.FirstChild.Data) - - if (hashRefMatch != nil && len(hashRefMatch) == 3) { - /*if (strings.HasPrefix(jsonFieldNode.FirstChild.Data, "[")) && - (strings.HasSuffix(jsonFieldNode.FirstChild.Data, "]")) && - (strings.Index(jsonFieldNode.FirstChild.Data, "|") > 0) {*/ - - c.addChildLeaf(config, tableName, - parent, jsonFieldNode.Data, - hashRefMatch[2]) //take hashref key value - } else { - c.addChildLeaf(config, tableName, - parent, jsonFieldNode.Data, - jsonFieldNode.FirstChild.Data) - } - } - - } else if (jsonFieldNode.Type == jsonquery.ElementNode && - jsonFieldNode.FirstChild != nil && - jsonFieldNode.FirstChild.Type == jsonquery.ElementNode) { - //Array data e.g. VLAN members - for arrayNode:=jsonFieldNode.FirstChild; arrayNode != nil; - - arrayNode = arrayNode.NextSibling { - c.addChildLeaf(config, tableName, - parent, jsonFieldNode.Data, - arrayNode.FirstChild.Data) - } - } - } - - return CVL_SUCCESS -} - -func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node)(*yparser.YParserNode, CVLErrorInfo) { - var cvlErrObj CVLErrorInfo - - tableName := fmt.Sprintf("%s",jsonNode.Data) - c.batchLeaf = "" - - //Every Redis table is mapped as list within a container, - //E.g. ACL_RULE is mapped as - // container ACL_RULE { list ACL_RULE_LIST {} } - var topNode *yparser.YParserNode - - // Add top most conatiner e.g. 'container sonic-acl {...}' - if _, exists := modelInfo.tableInfo[tableName]; exists == false { - return nil, cvlErrObj - } - topNode = c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, - nil, modelInfo.tableInfo[tableName].modelName) - - //Add the container node for each list - //e.g. 'container ACL_TABLE { list ACL_TABLE_LIST ...} - listConatinerNode := c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, - topNode, tableName) - - //Traverse each key instance - for jsonNode = jsonNode.FirstChild; jsonNode != nil; jsonNode = jsonNode.NextSibling { - - //For each field check if is key - //If it is key, create list as child of top container - // Get all key name/value pairs - if yangListName := getRedisKeyToYangList(tableName, jsonNode.Data); yangListName!= "" { - tableName = yangListName - } - keyValuePair := getRedisToYangKeys(tableName, jsonNode.Data) - keyCompCount := len(keyValuePair) - totalKeyComb := 1 - var keyIndices []int - - //Find number of all key combinations - //Each key can have one or more key values, which results in nk1 * nk2 * nk2 combinations - idx := 0 - for i,_ := range keyValuePair { - totalKeyComb = totalKeyComb * len(keyValuePair[i].values) - keyIndices = append(keyIndices, 0) - } - - for ; totalKeyComb > 0 ; totalKeyComb-- { - //Get the YANG list name from Redis table name - //Ideally they are same except when one Redis table is split - //into multiple YANG lists - - //Add table i.e. create list element - listNode := c.addChildNode(tableName, listConatinerNode, tableName + "_LIST") //Add the list to the top node - - //For each key combination - //Add keys as leaf to the list - for idx = 0; idx < keyCompCount; idx++ { - c.addChildLeaf(config, tableName, - listNode, keyValuePair[idx].key, - keyValuePair[idx].values[keyIndices[idx]]) - } - - //Get all fields under the key field and add them as children of the list - c.generateTableFieldsData(config, tableName, jsonNode, listNode) - - //Check which key elements left after current key element - var next int = keyCompCount - 1 - for ((next > 0) && ((keyIndices[next] +1) >= len(keyValuePair[next].values))) { - next-- - } - //No more combination possible - if (next < 0) { - break - } - - keyIndices[next]++ - - //Reset indices for all other key elements - for idx = next+1; idx < keyCompCount; idx++ { - keyIndices[idx] = 0 - } - - TRACE_LOG(INFO_API, TRACE_CACHE, "Starting batch leaf creation - %s\n", c.batchLeaf) - //process batch leaf creation - if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, c.batchLeaf); errObj.ErrCode != yparser.YP_SUCCESS { - cvlErrObj = CreateCVLErrObj(errObj) - return nil, cvlErrObj - } - c.batchLeaf = "" - } - } - - return topNode, cvlErrObj -} - func (c *CVL) translateToYang(jsonMap *map[string]interface{}) (*yparser.YParserNode, CVLErrorInfo) { var cvlErrObj CVLErrorInfo @@ -1587,45 +1166,6 @@ func (c *CVL) validateSyntax(data *yparser.YParserNode) (CVLErrorInfo, CVLRetCod return cvlErrObj, CVL_SUCCESS } -//Perform semantic checks -func (c *CVL) validateSemantics(data *yparser.YParserNode, appDepData *yparser.YParserNode) (CVLErrorInfo, CVLRetCode) { - var cvlErrObj CVLErrorInfo - - if (SkipSemanticValidation() == true) { - return cvlErrObj, CVL_SUCCESS - } - - //Get dependent data from - depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation - - if (Tracing == true) { - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "Validating semantics data=%s\n depData =%s\n, appDepData=%s\n....", c.yp.NodeDump(data), c.yp.NodeDump(depData), c.yp.NodeDump(appDepData)) - } - - if errObj := c.yp.ValidateSemantics(data, depData, appDepData); errObj.ErrCode != yparser.YP_SUCCESS { - - retCode := CVLRetCode(errObj.ErrCode) - - cvlErrObj = CVLErrorInfo { - TableName : errObj.TableName, - ErrCode : CVLRetCode(errObj.ErrCode), - CVLErrDetails : cvlErrorMap[retCode], - Keys : errObj.Keys, - Value : errObj.Value, - Field : errObj.Field, - Msg : errObj.Msg, - ConstraintErrMsg : errObj.ErrTxt, - ErrAppTag : errObj.ErrAppTag, - } - - - - return cvlErrObj, retCode - } - - return cvlErrObj ,CVL_SUCCESS -} - //Add config data item to accumulate per table func (c *CVL) addCfgDataItem(configData *map[string]interface{}, cfgDataItem CVLEditConfigData) (string, string){ diff --git a/cvl/cvl_api.go b/cvl/cvl_api.go index 9e42d190a488..719467f31736 100644 --- a/cvl/cvl_api.go +++ b/cvl/cvl_api.go @@ -26,6 +26,7 @@ import ( "path/filepath" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" + "time" ) type CVLValidateType uint @@ -108,6 +109,26 @@ type CVLEditConfigData struct { Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down} } +// ValidationTimeStats CVL validations stats +//Maintain time stats for call to ValidateEditConfig(). +//Hits : Total number of times ValidateEditConfig() called +//Time : Total time spent in ValidateEditConfig() +//Peak : Highest time spent in ValidateEditConfig() +type ValidationTimeStats struct { + Hits uint + Time time.Duration + Peak time.Duration +} + +//CVLDepDataForDelete Structure for dependent entry to be deleted +type CVLDepDataForDelete struct { + RefKey string //Ref Key which is getting deleted + Entry map[string]map[string]string //Entry or field which should be deleted as a result +} + +//Global data structure for maintaining validation stats +var cfgValidationStats ValidationTimeStats + func Initialize() CVLRetCode { if (cvlInitialized == true) { //CVL has already been initialized @@ -512,3 +533,34 @@ func (c *CVL) ValidateKeyData(key string, data string) CVLRetCode { func (c *CVL) ValidateFields(key string, field string, value string) CVLRetCode { return CVL_NOT_IMPLEMENTED } + +//SortDepTables Sort list of given tables as per their dependency +func (c *CVL) SortDepTables(inTableList []string) ([]string, CVLRetCode) { + return []string{}, CVL_NOT_IMPLEMENTED +} + +//GetOrderedTables Get the order list(parent then child) of tables in a given YANG module +//within a single model this is obtained using leafref relation +func (c *CVL) GetOrderedTables(yangModule string) ([]string, CVLRetCode) { + return []string{}, CVL_NOT_IMPLEMENTED +} + +//GetDepTables Get the list of dependent tables for a given table in a YANG module +func (c *CVL) GetDepTables(yangModule string, tableName string) ([]string, CVLRetCode) { + return []string{}, CVL_NOT_IMPLEMENTED +} + +//GetDepDataForDelete Get the dependent (Redis keys) to be deleted or modified +//for a given entry getting deleted +func (c *CVL) GetDepDataForDelete(redisKey string) ([]CVLDepDataForDelete) { + return []CVLDepDataForDelete{} +} + +//GetValidationTimeStats Retrieve global stats +func GetValidationTimeStats() ValidationTimeStats { + return cfgValidationStats +} + +//ClearValidationTimeStats Clear global stats +func ClearValidationTimeStats() { +} diff --git a/cvl/cvl_cache.go b/cvl/cvl_cache.go new file mode 100644 index 000000000000..66cc22bc7928 --- /dev/null +++ b/cvl/cvl_cache.go @@ -0,0 +1,234 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package cvl +import ( + "encoding/json" + "github.com/go-redis/redis" + //lint:ignore ST1001 This is safe to dot import for util package + . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" + "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + "time" + "runtime" + "github.com/antchfx/jsonquery" +) + +// Fetch dependent data from validated data cache, +// Returns the data and flag to indicate that if requested data +// is found in update request, the data should be merged with Redis data +func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (d map[string]string, m bool) { + defer func() { + pd := &d + pm := &m + + TRACE_LOG(INFO_API, TRACE_CACHE, + "Returning data from request cache, data = %v, merge needed = %v", + *pd, *pm) + }() + + cfgDataArr := c.requestCache[tableName][key] + if (cfgDataArr != nil) { + for _, cfgReqData := range cfgDataArr { + //Delete request doesn't have depedent data + if (cfgReqData.VOp == OP_CREATE) { + return cfgReqData.Data, false + } else if (cfgReqData.VOp == OP_UPDATE) { + return cfgReqData.Data, true + } + } + } + + return nil, false +} + +//Fetch given table entries using pipeline +func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]interface{}) int { + + TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now()) + + totalCount := len(dbKeys) + if (totalCount == 0) { + //No entry to be fetched + return 0 + } + + entryFetched := 0 + bulkCount := 0 + bulkKeys := []string{} + for dbKey, val := range dbKeys { //for all keys + + if (val != nil) { //skip entry already fetched + mapTable := c.tmpDbCache[tableName] + delete(mapTable.(map[string]interface{}), dbKey) //delete entry already fetched + totalCount = totalCount - 1 + if(bulkCount != totalCount) { + //If some entries are remaining go back to 'for' loop + continue + } + } else { + //Accumulate entries to be fetched + bulkKeys = append(bulkKeys, dbKey) + bulkCount = bulkCount + 1 + } + + if(bulkCount != totalCount) && ((bulkCount % MAX_BULK_ENTRIES_IN_PIPELINE) != 0) { + //If some entries are remaining and bulk bucket is not filled, + //go back to 'for' loop + continue + } + + mCmd := map[string]*redis.StringStringMapCmd{} + + pipe := redisClient.Pipeline() + + for _, dbKey := range bulkKeys { + + redisKey := tableName + modelInfo.tableInfo[tableName].redisKeyDelim + dbKey + //Check in validated cache first and add as dependent data + if entry, mergeNeeded := c.fetchDataFromRequestCache(tableName, dbKey); (entry != nil) { + c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry + entryFetched = entryFetched + 1 + //Entry found in validated cache, so skip fetching from Redis + //if merging is not required with Redis DB + if (mergeNeeded == false) { + continue + } + } + + //Otherwise fetch it from Redis + mCmd[dbKey] = pipe.HGetAll(redisKey) //write into pipeline + if mCmd[dbKey] == nil { + CVL_LOG(ERROR, "Failed pipe.HGetAll('%s')", redisKey) + } + } + + _, err := pipe.Exec() + if err != nil { + CVL_LOG(ERROR, "Failed to fetch details for table %s", tableName) + return 0 + } + pipe.Close() + bulkKeys = nil + + mapTable := c.tmpDbCache[tableName] + + for key, val := range mCmd { + res, err := val.Result() + if (err != nil || len(res) == 0) { + //no data found, don't keep blank entry + delete(mapTable.(map[string]interface{}), key) + continue + } + //exclude table name and delim + keyOnly := key + + if (len(mapTable.(map[string]interface{})) > 0) && (mapTable.(map[string]interface{})[keyOnly] != nil) { + tmpFieldMap := (mapTable.(map[string]interface{})[keyOnly]).(map[string]string) + //merge with validated cache data + mergeMap(res, tmpFieldMap) + fieldMap := c.checkFieldMap(&res) + mapTable.(map[string]interface{})[keyOnly] = fieldMap + } else { + fieldMap := c.checkFieldMap(&res) + mapTable.(map[string]interface{})[keyOnly] = fieldMap + } + + entryFetched = entryFetched + 1 + } + + runtime.Gosched() + } + + TRACE_LOG(INFO_API, TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now()) + + return entryFetched +} + +//populate redis data to cache +func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { + TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now()) + + entryToFetch := 0 + var root *yparser.YParserNode = nil + var errObj yparser.YParserError + + for entryToFetch = 1; entryToFetch > 0; { //Force to enter the loop for first time + //Repeat until all entries are fetched + entryToFetch = 0 + for tableName, dbKeys := range c.tmpDbCache { //for each table + entryToFetch = entryToFetch + c.fetchTableDataToTmpCache(tableName, dbKeys.(map[string]interface{})) + } //for each table + + //If no table entry delete the table itself + for tableName, dbKeys := range c.tmpDbCache { //for each table + if (len(dbKeys.(map[string]interface{})) == 0) { + delete(c.tmpDbCache, tableName) + continue + } + } + + if (entryToFetch == 0) { + //No more entry to fetch + break + } + + if Tracing { + jsonDataBytes, _ := json.Marshal(c.tmpDbCache) + jsonData := string(jsonDataBytes) + TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonData) + } + + data, err := jsonquery.ParseJsonMap(&c.tmpDbCache) + + if (err != nil) { + return nil + } + + //Build yang tree for each table and cache it + for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { + TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonNode.Data) + //Visit each top level list in a loop for creating table data + topNode, _ := c.generateTableData(true, jsonNode) + if (root == nil) { + root = topNode + } else { + if root, errObj = c.yp.MergeSubtree(root, topNode); errObj.ErrCode != yparser.YP_SUCCESS { + return nil + } + } + } + } // until all dependent data is fetched + + if root != nil && Tracing { + dumpStr := c.yp.NodeDump(root) + TRACE_LOG(INFO_API, TRACE_CACHE, "Dependent Data = %v\n", dumpStr) + } + + TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now()) + return root +} + + +func (c *CVL) clearTmpDbCache() { + for key := range c.tmpDbCache { + delete(c.tmpDbCache, key) + } +} + + diff --git a/cvl/cvl_semantics.go b/cvl/cvl_semantics.go new file mode 100644 index 000000000000..b7eab1f539e5 --- /dev/null +++ b/cvl/cvl_semantics.go @@ -0,0 +1,123 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package cvl + +import ( + "strings" + "github.com/antchfx/xmlquery" + "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + //lint:ignore ST1001 This is safe to dot import for util package + . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" +) + +//YValidator YANG Validator used for external semantic +//validation including custom/platform validation +type YValidator struct { + root *xmlquery.Node //Top evel root for data + current *xmlquery.Node //Current position +} + +//Check delete constraint for leafref if key/field is deleted +func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData, + tableName, keyVal, field string) CVLRetCode { + var leafRefs []tblFieldPair + if (field != "") { + //Leaf or field is getting deleted + leafRefs = c.findUsedAsLeafRef(tableName, field) + } else { + //Entire entry is getting deleted + leafRefs = c.findUsedAsLeafRef(tableName, modelInfo.tableInfo[tableName].keys[0]) + } + + //The entry getting deleted might have been referred from multiple tables + //Return failure if at-least one table is using this entry + for _, leafRef := range leafRefs { + TRACE_LOG(INFO_API, (TRACE_DELETE | TRACE_SEMANTIC), "Checking delete constraint for leafRef %s/%s", leafRef.tableName, leafRef.field) + //Check in dependent data first, if the referred entry is already deleted + leafRefDeleted := false + for _, cfgDataItem := range cfgData { + if (cfgDataItem.VType == VALIDATE_NONE) && + (cfgDataItem.VOp == OP_DELETE ) && + (strings.HasPrefix(cfgDataItem.Key, (leafRef.tableName + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim + keyVal + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim))) { + //Currently, checking for one entry is being deleted in same session + //We should check for all entries + leafRefDeleted = true + break + } + } + + if (leafRefDeleted == true) { + continue //check next leafref + } + + //Else, check if any referred enrty is present in DB + var nokey []string + refKeyVal, err := luaScripts["find_key"].Run(redisClient, nokey, leafRef.tableName, + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim, leafRef.field, keyVal).Result() + if (err == nil && refKeyVal != "") { + CVL_LOG(ERROR, "Delete will violate the constraint as entry %s is referred in %s", tableName, refKeyVal) + + return CVL_SEMANTIC_ERROR + } + } + + + return CVL_SUCCESS +} + +//Perform semantic checks +func (c *CVL) validateSemantics(data *yparser.YParserNode, appDepData *yparser.YParserNode) (CVLErrorInfo, CVLRetCode) { + var cvlErrObj CVLErrorInfo + + if (SkipSemanticValidation() == true) { + return cvlErrObj, CVL_SUCCESS + } + + //Get dependent data from + depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation + + if (Tracing == true) { + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "Validating semantics data=%s\n depData =%s\n, appDepData=%s\n....", c.yp.NodeDump(data), c.yp.NodeDump(depData), c.yp.NodeDump(appDepData)) + } + + if errObj := c.yp.ValidateSemantics(data, depData, appDepData); errObj.ErrCode != yparser.YP_SUCCESS { + + retCode := CVLRetCode(errObj.ErrCode) + + cvlErrObj = CVLErrorInfo { + TableName : errObj.TableName, + ErrCode : CVLRetCode(errObj.ErrCode), + CVLErrDetails : cvlErrorMap[retCode], + Keys : errObj.Keys, + Value : errObj.Value, + Field : errObj.Field, + Msg : errObj.Msg, + ConstraintErrMsg : errObj.ErrTxt, + ErrAppTag : errObj.ErrAppTag, + } + + + + return cvlErrObj, retCode + } + + return cvlErrObj ,CVL_SUCCESS +} + diff --git a/cvl/cvl_syntax.go b/cvl/cvl_syntax.go new file mode 100644 index 000000000000..7cc812e0374a --- /dev/null +++ b/cvl/cvl_syntax.go @@ -0,0 +1,203 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package cvl +import ( + "fmt" + "github.com/antchfx/jsonquery" + "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + //lint:ignore ST1001 This is safe to dot import for util package + . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" +) + +//Add child node to a parent node +func(c *CVL) addChildNode(tableName string, parent *yparser.YParserNode, name string) *yparser.YParserNode { + + //return C.lyd_new(parent, modelInfo.tableInfo[tableName].module, C.CString(name)) + return c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, parent, name) +} + +func (c *CVL) addChildLeaf(config bool, tableName string, parent *yparser.YParserNode, name string, value string) { + + /* If there is no value then assign default space string. */ + if len(value) == 0 { + value = " " + } + + //Batch leaf creation + c.batchLeaf = c.batchLeaf + name + "#" + value + "#" + //Check if this leaf has leafref, + //If so add the add redis key to its table so that those + // details can be fetched for dependency validation + + c.addLeafRef(config, tableName, name, value) +} + +func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *jsonquery.Node, +parent *yparser.YParserNode) CVLRetCode { + + //Traverse fields + for jsonFieldNode := jsonNode.FirstChild; jsonFieldNode!= nil; + jsonFieldNode = jsonFieldNode.NextSibling { + //Add fields as leaf to the list + if (jsonFieldNode.Type == jsonquery.ElementNode && + jsonFieldNode.FirstChild != nil && + jsonFieldNode.FirstChild.Type == jsonquery.TextNode) { + + if (len(modelInfo.tableInfo[tableName].mapLeaf) == 2) {//mapping should have two leaf always + //Values should be stored inside another list as map table + listNode := c.addChildNode(tableName, parent, tableName) //Add the list to the top node + c.addChildLeaf(config, tableName, + listNode, modelInfo.tableInfo[tableName].mapLeaf[0], + jsonFieldNode.Data) + + c.addChildLeaf(config, tableName, + listNode, modelInfo.tableInfo[tableName].mapLeaf[1], + jsonFieldNode.FirstChild.Data) + + } else { + //check if it is hash-ref, then need to add only key from "TABLE|k1" + hashRefMatch := reHashRef.FindStringSubmatch(jsonFieldNode.FirstChild.Data) + + if (hashRefMatch != nil && len(hashRefMatch) == 3) { + /*if (strings.HasPrefix(jsonFieldNode.FirstChild.Data, "[")) && + (strings.HasSuffix(jsonFieldNode.FirstChild.Data, "]")) && + (strings.Index(jsonFieldNode.FirstChild.Data, "|") > 0) {*/ + + c.addChildLeaf(config, tableName, + parent, jsonFieldNode.Data, + hashRefMatch[2]) //take hashref key value + } else { + c.addChildLeaf(config, tableName, + parent, jsonFieldNode.Data, + jsonFieldNode.FirstChild.Data) + } + } + + } else if (jsonFieldNode.Type == jsonquery.ElementNode && + jsonFieldNode.FirstChild != nil && + jsonFieldNode.FirstChild.Type == jsonquery.ElementNode) { + //Array data e.g. VLAN members + for arrayNode:=jsonFieldNode.FirstChild; arrayNode != nil; + + arrayNode = arrayNode.NextSibling { + c.addChildLeaf(config, tableName, + parent, jsonFieldNode.Data, + arrayNode.FirstChild.Data) + } + } + } + + return CVL_SUCCESS +} + +func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node)(*yparser.YParserNode, CVLErrorInfo) { + var cvlErrObj CVLErrorInfo + + tableName := fmt.Sprintf("%s",jsonNode.Data) + c.batchLeaf = "" + + //Every Redis table is mapped as list within a container, + //E.g. ACL_RULE is mapped as + // container ACL_RULE { list ACL_RULE_LIST {} } + var topNode *yparser.YParserNode + + // Add top most conatiner e.g. 'container sonic-acl {...}' + if _, exists := modelInfo.tableInfo[tableName]; exists == false { + return nil, cvlErrObj + } + topNode = c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, + nil, modelInfo.tableInfo[tableName].modelName) + + //Add the container node for each list + //e.g. 'container ACL_TABLE { list ACL_TABLE_LIST ...} + listConatinerNode := c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, + topNode, tableName) + + //Traverse each key instance + for jsonNode = jsonNode.FirstChild; jsonNode != nil; jsonNode = jsonNode.NextSibling { + + //For each field check if is key + //If it is key, create list as child of top container + // Get all key name/value pairs + if yangListName := getRedisKeyToYangList(tableName, jsonNode.Data); yangListName!= "" { + tableName = yangListName + } + keyValuePair := getRedisToYangKeys(tableName, jsonNode.Data) + keyCompCount := len(keyValuePair) + totalKeyComb := 1 + var keyIndices []int + + //Find number of all key combinations + //Each key can have one or more key values, which results in nk1 * nk2 * nk2 combinations + idx := 0 + for i,_ := range keyValuePair { + totalKeyComb = totalKeyComb * len(keyValuePair[i].values) + keyIndices = append(keyIndices, 0) + } + + for ; totalKeyComb > 0 ; totalKeyComb-- { + //Get the YANG list name from Redis table name + //Ideally they are same except when one Redis table is split + //into multiple YANG lists + + //Add table i.e. create list element + listNode := c.addChildNode(tableName, listConatinerNode, tableName + "_LIST") //Add the list to the top node + + //For each key combination + //Add keys as leaf to the list + for idx = 0; idx < keyCompCount; idx++ { + c.addChildLeaf(config, tableName, + listNode, keyValuePair[idx].key, + keyValuePair[idx].values[keyIndices[idx]]) + } + + //Get all fields under the key field and add them as children of the list + c.generateTableFieldsData(config, tableName, jsonNode, listNode) + + //Check which key elements left after current key element + var next int = keyCompCount - 1 + for ((next > 0) && ((keyIndices[next] +1) >= len(keyValuePair[next].values))) { + next-- + } + //No more combination possible + if (next < 0) { + break + } + + keyIndices[next]++ + + //Reset indices for all other key elements + for idx = next+1; idx < keyCompCount; idx++ { + keyIndices[idx] = 0 + } + + TRACE_LOG(INFO_API, TRACE_CACHE, "Starting batch leaf creation - %s\n", c.batchLeaf) + //process batch leaf creation + if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, c.batchLeaf); errObj.ErrCode != yparser.YP_SUCCESS { + cvlErrObj = CreateCVLErrObj(errObj) + return nil, cvlErrObj + } + c.batchLeaf = "" + } + } + + return topNode, cvlErrObj +} + diff --git a/cvl/cvl_test.go b/cvl/cvl_test.go index 3731311b63ed..655bf1eafc50 100644 --- a/cvl/cvl_test.go +++ b/cvl/cvl_test.go @@ -32,7 +32,6 @@ import ( "runtime" "github.com/Azure/sonic-mgmt-common/cvl" . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" - "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" ) type testEditCfgData struct { @@ -2861,22 +2860,6 @@ func TestValidateEditConfig_Delete_Entry_Then_Dep_Leafref_Positive(t *testing.T) unloadConfigDB(rclient, depDataMap) } -func TestBadSchema(t *testing.T) { - badSchema, err := ioutil.TempFile("", "sonic-test-*.yin") - if err != nil { - t.Fatalf("could not create temp file") - } - - // write incomplete module data to temporary schema file - badSchema.WriteString("") - badSchema.Close() - - if module, _ := yparser.ParseSchemaFile(badSchema.Name()); module != nil { //should fail - t.Errorf("Bad schema parsing should fail.") - } -} - - func TestServicability_Debug_Trace(t *testing.T) { cvl.Debug(false) diff --git a/cvl/schema/Makefile b/cvl/schema/Makefile index e16280601b99..2642d1d96466 100644 --- a/cvl/schema/Makefile +++ b/cvl/schema/Makefile @@ -28,19 +28,27 @@ out_dir=$(TOPDIR)/build/cvl/schema src_files=$(wildcard $(sonic_yang)/*.yang) src_files += $(wildcard $(sonic_yang_common)/*.yang) -out=$(patsubst %.yang, $(out_dir)/%.yin, $(notdir $(src_files))) -search_path=$(std_yang_common):$(sonic_yang):$(sonic_yang_common) +out=$(patsubst %.yang, $(out_dir)/%.yin, $(shell ls -1 $(sonic_yang)/*.yang | cut -d'/' -f6)) +out_common=$(patsubst %.yang, $(out_dir)/%.yin, $(shell ls -1 $(sonic_yang_common)/*.yang | cut -d'/' -f7)) +out_platform=$(patsubst %.yang, $(out_dir)/%.yin, $(shell find $(sonic_yang_platform) -name '*.yang' | cut -d'/' -f6-8)) +out_platform_dep=$(shell find $(sonic_yang_platform) -name '*.yang') +out_tree=$(patsubst %.yang, $(out_dir)/%.tree, $(src_files)) -all: schema +search_path=$(sonic_yang):$(sonic_yang_common):$(sonic_yang_common)/ietf -schema: $(out) -.PRECIOUS: %/. -%/.: - mkdir -p $(@D) +all: precheck schema + +precheck: + mkdir -p $(out_dir) + +schema: $(out) $(out_common) $(out_platform) +# @$(call install_cvl_schema) + +schema-tree: $(out_tree) #Build YANG models -$(out_dir)/%.yin:$(sonic_yang)/%.yang | $(out_dir)/. +$(out_dir)/%.yin:$(sonic_yang)/%.yang @echo "Generating `basename $@` ..." @devFile="`echo $@ | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ @@ -49,12 +57,28 @@ $(out_dir)/%.yin:$(sonic_yang)/%.yang | $(out_dir)/. #Build common YANG models -$(out_dir)/%.yin:$(sonic_yang_common)/%.yang | $(out_dir)/. +$(out_dir)/%.yin:$(sonic_yang_common)/%.yang @echo "Generating `basename $@` ..." @devFile="`echo $@ | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ pyang -p $(search_path) --plugindir $(pyang_plugin_dir) \ -f yin-cvl $$devOpt $< -o $@ +#Build platform specific YANG models +$(out_platform):$(out_platform_dep) + @mkdir -p `dirname $@` + @echo "Generating $@ ..." + @pyang -p $(search_path) --plugindir $(pyang_plugin_dir) \ + -f yin-cvl "$(sonic_yang)/`echo $@ | grep -o 'platform/.*\.'`yang" -o $@ + +$(out_dir)/%.tree:%.yang + @echo "Generating `basename $@` ..." + @devFile="`echo $< | cut -d . -f1`-dev.yang"; \ + if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ + pyang -p $(search_path) -f tree $$devOpt $< -o `basename $@` + clean: - $(RM) -r $(out_dir) + @echo "Removing files ..." + rm -rf $(out_dir) + rm -rf platform/ + diff --git a/cvl/testdata/schema/Makefile b/cvl/testdata/schema/Makefile index 93976f79f515..4bd09da99640 100644 --- a/cvl/testdata/schema/Makefile +++ b/cvl/testdata/schema/Makefile @@ -17,44 +17,38 @@ # # ################################################################################ -TOPDIR := ../../.. -YANGDIR=$(TOPDIR)/models/yang -sonic_yang=$(YANGDIR)/sonic -std_yang_common=$(YANGDIR)/common/ -sonic_yang_common=$(sonic_yang)/common -pyang_plugin_dir=$(TOPDIR)/tools/pyang/pyang_plugins +TOPDIR?=$(abspath ../../../) +sonic_yang=../../../models/yang/sonic +pyang_plugin_dir=../../../tools/pyang/pyang_plugins src_files=$(wildcard *.yang) +out_dir=$(TOPDIR)/build/tests/cvl/testdata/schema/ +out=$(patsubst %.yang, $(out_dir)/%.yin, $(src_files)) +out_ext=$(patsubst %.yang, $(out_dir)/%.tree, $(src_files)) -out_dir=$(TOPDIR)/build/tests/cvl/testdata/schema -out=$(patsubst %.yang, $(out_dir)/%.yin, $(notdir $(src_files)) ) -out_ext=$(patsubst %.yang, %.tree, $(src_files)) -search_path=$(std_yang_common):$(sonic_yang):$(sonic_yang_common) +all:precheck schema -all:schema +precheck: + mkdir -p $(out_dir) schema: $(out) schema-tree: $(out_ext) -.PRECIOUS: %/. -%/.: - mkdir -p $(@D) - -$(out_dir)/%.yin:%.yang | $(out_dir)/. +$(out_dir)/%.yin:%.yang @echo "Generating `basename $@` ..." - @devFile="`echo $< | cut -d . -f1`-dev.yang"; \ + @devFile="`echo $< | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ - pyang -p $(search_path) \ + pyang -p $(sonic_yang)/common:$(sonic_yang)/common/ietf \ --plugindir $(pyang_plugin_dir) -f yin-cvl $$devOpt $< -o $@ -%.tree:%.yang +$(out_dir)/%.tree:%.yang @echo "Generating `basename $@` ..." - @devFile="`echo $< | cut -d . -f1`-dev.yang"; \ + @devFile="`echo $< | cut -d . -f1`-deviation.yang"; \ if [ -f $$devFile ] ; then devOpt="--deviation-module $$devFile"; fi; \ - pyang -p $(search_path) \ - -f tree $$devOpt $< -o `basename $@` + pyang -p $(sonic_yang)/common:$(sonic_yang)/common/ietf \ + -f tree $$devOpt $< -o $@ clean: - $(RM) -r $(out_dir) - $(RM) *.tree + @echo "Removing files ..." + rm -rf platform diff --git a/cvl/testdata/schema/sonic-acl.yang b/cvl/testdata/schema/sonic-acl.yang new file mode 100644 index 000000000000..34b416940991 --- /dev/null +++ b/cvl/testdata/schema/sonic-acl.yang @@ -0,0 +1,241 @@ +module sonic-acl { + namespace "http://github.com/Azure/sonic-acl"; + prefix sacl; + yang-version 1.1; + + import ietf-inet-types { + prefix inet; + } + + import sonic-common { + prefix cmn; + } + + import sonic-extension { + prefix sonic-ext; + } + + import sonic-port { + prefix prt; + } + + import sonic-portchannel { + prefix po; + } + + import sonic-mirror-session { + prefix sms; + } + + organization + "SONiC"; + + contact + "SONiC"; + + description + "SONIC ACL"; + + revision 2019-05-15 { + description + "Initial revision."; + } + + container sonic-acl { + + container ACL_TABLE { + + list ACL_TABLE_LIST { + key "aclname"; + max-elements 1024; // Max 1K ACL tables for all platforms + + leaf aclname { + type string { + pattern '[a-zA-Z0-9]{1}([-a-zA-Z0-9_]{0,71})'; + length 1..72; + } + } + + leaf policy_desc { + type string { + length 1..255 { + error-app-tag policy-desc-invalid-length; + } + } + } + + leaf stage { + type enumeration { + enum INGRESS; + enum EGRESS; + } + } + + leaf type { + type enumeration { + enum MIRROR; + enum L2; + enum L3; + enum L3V6; + } + } + + leaf-list ports { + type union { + type leafref { + path "/prt:sonic-port/prt:PORT/prt:PORT_LIST/prt:ifname"; + } + type leafref { + path "/po:sonic-portchannel/po:PORTCHANNEL/po:PORTCHANNEL_LIST/po:name"; + } + } + } + } + } + + container ACL_RULE { + + list ACL_RULE_LIST { + key "aclname rulename"; + sonic-ext:custom-validation ValidateMaxAclRule; // Max 64K ACL rules for all platforms + + leaf aclname { + type leafref { + path "../../../ACL_TABLE/ACL_TABLE_LIST/aclname"; + } + must "(/cmn:operation/cmn:operation != 'DELETE') or " + + "count(current()/../../../ACL_TABLE/ACL_TABLE_LIST[aclname=current()]/ports) = 0" { + error-message "Ports are already bound to this rule."; + } + } + + leaf rulename { + type string; + } + + leaf PRIORITY { + type uint16 { + range "1..65535"{ + error-message "Invalid ACL rule priority."; + } + } + } + + leaf RULE_DESCRIPTION { + type string; + } + + leaf PACKET_ACTION { + type enumeration { + enum FORWARD; + enum DROP; + enum REDIRECT; + enum INT_INSERT; + enum INT_DELETE; + } + } + + leaf MIRROR_ACTION { + type leafref { + path "/sms:sonic-mirror-session/sms:MIRROR_SESSION/sms:MIRROR_SESSION_LIST/sms:name"; + } + } + + leaf IP_TYPE { + sonic-ext:custom-validation ValidateAclRuleIPAddress; + type enumeration { + enum ANY; + enum IP; + enum IPV4; + enum IPV4ANY; + enum NON_IPV4; + enum IPV6ANY; + enum NON_IPV6; + } + + default IPV4; + } + + leaf IP_PROTOCOL { + type uint8 { + range "1|2|6|17|46|47|51|103|115"; + } + } + + leaf ETHER_TYPE { + type string { + pattern "(0x88CC)|(0x8100)|(0x8915)|(0x0806)|(0x0800)|(0x86DD)|(0x8847)" { + error-message "Invalid ACL Rule Ether Type"; + error-app-tag ether-type-invalid; + } + } + } + + choice ip_src_dst { + case ipv4_src_dst { + when "boolean(IP_TYPE[.='ANY' or .='IP' or .='IPV4' or .='IPV4ANY'])"; + leaf SRC_IP { + mandatory true; + type inet:ipv4-prefix; + } + leaf DST_IP { + mandatory true; + type inet:ipv4-prefix; + } + } + case ipv6_src_dst { + when "boolean(IP_TYPE[.='ANY' or .='IP' or .='IPV6' or .='IPV6ANY'])"; + leaf SRC_IPV6 { + mandatory true; + type inet:ipv6-prefix; + } + leaf DST_IPV6 { + mandatory true; + type inet:ipv6-prefix; + } + } + } + + choice src_port { + case l4_src_port { + leaf L4_SRC_PORT { + type uint16; + } + } + case l4_src_port_range { + leaf L4_SRC_PORT_RANGE { + type string { + pattern "[0-9]{1,5}(-)[0-9]{1,5}"; + } + } + } + } + + choice dst_port { + case l4_dst_port { + leaf L4_DST_PORT { + type uint16; + } + } + case l4_dst_port_range { + leaf L4_DST_PORT_RANGE { + type string { + pattern "[0-9]{1,5}(-)[0-9]{1,5}"; + } + } + } + } + + leaf TCP_FLAGS { + type string { + pattern "0[xX][0-9a-fA-F]{2}[/]0[xX][0-9a-fA-F]{2}"; + } + } + + leaf DSCP { + type uint8; + } + } + } + } +} diff --git a/cvl/testdata/schema/sonic-port.yang b/cvl/testdata/schema/sonic-port.yang new file mode 100644 index 000000000000..44eb6eddd541 --- /dev/null +++ b/cvl/testdata/schema/sonic-port.yang @@ -0,0 +1,124 @@ +module sonic-port { + namespace "http://github.com/Azure/sonic-port"; + prefix prt; + + import sonic-common { + prefix scommon; + } + + organization + "SONiC"; + + contact + "SONiC"; + + description + "SONIC PORT"; + + revision 2019-05-15 { + description + "Initial revision."; + } + + + container sonic-port { + + container PORT { + + list PORT_LIST { + key "ifname"; + + leaf ifname { + type string { + pattern "Ethernet([1-3][0-9]{3}|[1-9][0-9]{2}|[1-9][0-9]|[0-9])" { + error-message "Invalid interface name"; + error-app-tag interface-name-invalid; + } + } + } + + leaf index { + type uint16; + } + + leaf speed { + type uint64; + } + + leaf valid_speeds { + type string; + } + + leaf alias { + type string; + } + + leaf description { + type string; + } + + leaf mtu{ + type uint32 { + range "1312..9216" { + error-message "Invalid MTU value"; + error-app-tag mtu-invalid; + } + } + } + + leaf lanes { + type string; + } + + leaf admin_status { + type scommon:admin-status; + } + } + } + container PORT_TABLE { + config false; + + list PORT_TABLE_LIST { + key "ifname"; + + leaf ifname { + type string { + pattern "Ethernet([1-3][0-9]{3}|[1-9][0-9]{2}|[1-9][0-9]|[0-9])"{ + error-message "Invalid interface name"; + error-app-tag interface-name-invalid; + } + } + } + + leaf index { + type uint16; + } + + leaf lanes { + type string; + } + + leaf mtu { + type uint32 { + range "1312..9216" { + error-message "Invalid MTU value"; + error-app-tag mtu-invalid; + } + } + } + + leaf valid_speeds { + type string; + } + + leaf alias { + type string; + } + + leaf oper_status { + type scommon:oper-status; + } + } + } + } +} diff --git a/models/yang/sonic/common/sonic-common.yang b/models/yang/sonic/common/sonic-common.yang index 45d608b684f8..42147e09757c 100644 --- a/models/yang/sonic/common/sonic-common.yang +++ b/models/yang/sonic/common/sonic-common.yang @@ -32,6 +32,14 @@ module sonic-common { } } + typedef oper-status { + type enumeration { + enum up; + enum down; + } + } + + container operation { description "This definition is used internally by CVL and is not exposed in NBI. Leaf 'operation' allows diff --git a/models/yang/sonic/common/sonic-extension.yang b/models/yang/sonic/common/sonic-extension.yang index f4888095863e..937c7adb3431 100644 --- a/models/yang/sonic/common/sonic-extension.yang +++ b/models/yang/sonic/common/sonic-extension.yang @@ -17,12 +17,6 @@ module sonic-extension { "Initial revision."; } - extension custom-handler { - description - "Node should be handled by custom handler"; - argument "name"; - } - extension db-name { description "DB name, e.g. APPL_DB, CONFIG_DB"; @@ -53,9 +47,10 @@ module sonic-extension { argument "value"; } - extension pf-check { + extension custom-validation { description - "Platform specific validation"; + "Extension for custom validation. + Platform specific validation can be implemented using custom validation."; argument "handler"; } }