From 4aa0899720ed11241d012a149edbd033ec777860 Mon Sep 17 00:00:00 2001 From: Partha Dutta <51353699+dutta-partha@users.noreply.github.com> Date: Tue, 12 Jan 2021 23:48:32 +0530 Subject: [PATCH] CVL Changes #9: Clean up CVL code and integrate custom xpath engine for leafref, when, must expression evaluation (#33) Clean up CVL code YIN schema reading using libyang API Integrate with xpath custom engine for leafref, when, must expression evaluation Add delete constraint check Add max-element constraint Add CVL trace log to /tmp/cvl.log based on config file --- cvl/cvl.go | 1307 +++++++-------------------- cvl/cvl_api.go | 376 ++++---- cvl/cvl_cache.go | 105 ++- cvl/cvl_luascript.go | 163 +++- cvl/cvl_semantics.go | 307 +++++-- cvl/cvl_syntax.go | 84 +- cvl/cvl_test.go | 52 +- cvl/internal/util/luascript_util.go | 83 ++ cvl/internal/util/util.go | 296 +++++- cvl/internal/yparser/yparser.go | 238 +---- 10 files changed, 1493 insertions(+), 1518 deletions(-) create mode 100644 cvl/internal/util/luascript_util.go diff --git a/cvl/cvl.go b/cvl/cvl.go index 69d92ac81f94..cc6a1c678950 100644 --- a/cvl/cvl.go +++ b/cvl/cvl.go @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // // its subsidiaries. // // // // Licensed under the Apache License, Version 2.0 (the "License"); // @@ -24,15 +24,16 @@ import ( "strings" "regexp" "time" - log "github.com/golang/glog" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" "github.com/antchfx/xmlquery" "github.com/antchfx/xpath" "github.com/antchfx/jsonquery" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + //lint:ignore ST1001 This is safe to dot import for util package . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "sync" - "flag" + "io/ioutil" + "path/filepath" ) //DB number @@ -51,11 +52,10 @@ const ( const DEFAULT_CACHE_DURATION uint16 = 300 /* 300 sec */ const MAX_BULK_ENTRIES_IN_PIPELINE int = 50 +const MAX_DEVICE_METADATA_FETCH_RETRY = 60 +const PLATFORM_SCHEMA_PATH = "platform/" -var reLeafRef *regexp.Regexp = nil var reHashRef *regexp.Regexp = nil -var reSelKeyVal *regexp.Regexp = nil -var reLeafInXpath *regexp.Regexp = nil var cvlInitialized bool var dbNameToDbNum map[string]uint8 @@ -63,11 +63,16 @@ var dbNameToDbNum map[string]uint8 //map of lua script loaded var luaScripts map[string]*redis.Script -type whenInfo struct { - expr string //when expression +type tblFieldPair struct { + tableName string + field string +} + +type mustInfo struct { + expr string //must expression exprTree *xpath.Expr //compiled expression tree - nodeNames []string //list of nodes under when condition - yangListNames []string //all yang list in expression + errCode string //err-app-tag + errStr string //error message } type leafRefInfo struct { @@ -77,11 +82,11 @@ type leafRefInfo struct { targetNodeName string //target node name } -type mustInfo struct { - expr string //must expression +type whenInfo struct { + expr string //when expression exprTree *xpath.Expr //compiled expression tree - errCode string //err-app-tag - errStr string //error message + nodeNames []string //list of nodes under when condition + yangListNames []string //all yang list in expression } //Important schema information to be loaded at bootup time @@ -93,6 +98,7 @@ type modelTableInfo struct { keys []string redisKeyDelim string redisKeyPattern string + redisTableSize int mapLeaf []string //for 'mapping list' leafRef map[string][]*leafRefInfo //for storing all leafrefs for a leaf in a table, //multiple leafref possible for union @@ -101,14 +107,15 @@ type modelTableInfo struct { tablesForMustExp map[string]CVLOperation refFromTables []tblFieldPair //list of table or table/field referring to this table dfltLeafVal map[string]string //map of leaf names and default value + mandatoryNodes map[string]bool //map of leaf names and mandatory flag } -/* CVL Error Structure. */ +// CVLErrorInfo Struct for CVL Error Info type CVLErrorInfo struct { TableName string /* Table having error */ ErrCode CVLRetCode /* CVL Error return Code. */ - CVLErrDetails string /* CVL Error Message details. */ + CVLErrDetails string /* CVL Error Message details. */ Keys []string /* Keys of the Table having error. */ Value string /* Field Value throwing error */ Field string /* Field Name throwing error . */ @@ -123,44 +130,45 @@ type requestCacheType struct { yangData *xmlquery.Node } +// CVL Struct for CVL session type CVL struct { - redisClient *redis.Client + //redisClient *redis.Client yp *yparser.YParser tmpDbCache map[string]interface{} //map of table storing map of key-value pair requestCache map[string]map[string][]*requestCacheType//Cache of validated data, - //per table, per key. Can be used as dependent data in next request + //per table, per key. Can be used as dependent data in next request + maxTableElem map[string]int //max element count per table batchLeaf []*yparser.YParserLeafValue //field name and value - chkLeafRefWithOthCache bool yv *YValidator //Custom YANG validator for validating external dependencies } +// Struct for model namepsace and prefix type modelNamespace struct { prefix string ns string } +// Struct for storing all YANG list schema info type modelDataInfo struct { - modelNs map[string]modelNamespace //model namespace + modelNs map[string]*modelNamespace //model namespace tableInfo map[string]*modelTableInfo //redis table to model name and keys redisTableToYangList map[string][]string //Redis table to all YANG lists when it is not 1:1 mapping allKeyDelims map[string]bool } -//Struct for storing global DB cache to store DB which are needed frequently like PORT -type dbCachedData struct { - root *yparser.YParserNode //Root of the cached data - startTime time.Time //When cache started - expiry uint16 //How long cache should be maintained in sec -} - //Global data cache for redis table type cvlGlobalSessionType struct { - db map[string]dbCachedData - pubsub *redis.PubSub stopChan chan int //stop channel to stop notification listener cv *CVL mutex *sync.Mutex } + +// Struct for storing key and value pair +type keyValuePairStruct struct { + key string + values []string +} + var cvg cvlGlobalSessionType //Single redis client for validation @@ -169,13 +177,8 @@ var redisClient *redis.Client //Stores important model info var modelInfo modelDataInfo -type keyValuePairStruct struct { - key string - values []string -} - -func TRACE_LOG(level log.Level, tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { - TRACE_LEVEL_LOG(level, tracelevel, fmtStr, args...) +func TRACE_LOG(tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { + TRACE_LEVEL_LOG(tracelevel, fmtStr, args...) } func CVL_LOG(level CVLLogLevel, fmtStr string, args ...interface{}) { @@ -206,24 +209,17 @@ func init() { cvlCfgMap := ReadConfFile() if (cvlCfgMap != nil) { - if (strings.Compare(cvlCfgMap["LOGTOSTDERR"], "true") == 0) { - flag.Set("logtostderr", "true") - flag.Set("stderrthreshold", cvlCfgMap["STDERRTHRESHOLD"]) - flag.Set("v", cvlCfgMap["VERBOSITY"]) - } - CVL_LOG(INFO ,"Current Values of CVL Configuration File %v", cvlCfgMap) } //regular expression for leafref and hashref finding - reLeafRef = regexp.MustCompile(`.*[/]([a-zA-Z]*:)?(.*)[/]([a-zA-Z]*:)?(.*)`) reHashRef = regexp.MustCompile(`\[(.*)\|(.*)\]`) - reSelKeyVal = regexp.MustCompile("=[ ]*['\"]?([0-9_a-zA-Z]+)['\"]?|(current[(][)])") - reLeafInXpath = regexp.MustCompile("(.*[:/]{1})([a-zA-Z0-9_-]+)([^a-zA-Z0-9_-]*)") + //Regular expression to select key value + //Regular expression to find leafref in xpath - Initialize() - - cvg.db = make(map[string]dbCachedData) + if Initialize() != CVL_SUCCESS { + CVL_LOG(FATAL, "CVL initialization failed") + } //Global session keeps the global cache cvg.cv, _ = ValidationSessOpen() @@ -231,20 +227,20 @@ func init() { cvg.stopChan = make(chan int, 1) //Initialize mutex cvg.mutex = &sync.Mutex{} + //Intialize mutex for stats + statsMutex = &sync.Mutex{} _, err := redisClient.ConfigSet("notify-keyspace-events", "AKE").Result() if err != nil { - CVL_LOG(ERROR ,"Could not enable notification error %s", err) + CVL_LOG(WARNING ,"Could not enable notification error %s", err) } - dbCacheSet(false, "PORT", 0) - xpath.SetLogCallback(func(fmt string, args ...interface{}) { if !IsTraceLevelSet(TRACE_SEMANTIC) { return } - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "XPATH: " + fmt, args...) + TRACE_LOG(TRACE_SEMANTIC, "XPATH: " + fmt, args...) }) } @@ -257,17 +253,6 @@ func isLeafListNode(node *xmlquery.Node) bool { return len(node.Attr) != 0 && node.Attr[0].Name.Local == "leaf-list" } -//Get attribute value of xml node -func getXmlNodeAttr(node *xmlquery.Node, attrName string) string { - for _, attr := range node.Attr { - if (attrName == attr.Name.Local) { - return attr.Value - } - } - - return "" -} - // getNodeName returns database field name for the xml node. func getNodeName(node *xmlquery.Node) string { if isLeafListNode(node) { @@ -276,6 +261,126 @@ func getNodeName(node *xmlquery.Node) string { return node.Data } +// Load all YIN schema files, apply deviation files +func loadSchemaFiles() CVLRetCode { + + platformName := "" + // Wait to check if CONFIG_DB is populated with DEVICE_METADATA. + // This is needed to apply deviation file + retryCnt := 0 + for ; (retryCnt < MAX_DEVICE_METADATA_FETCH_RETRY); retryCnt++ { + deviceMetaDataKey, err := redisClient.Keys("DEVICE_METADATA|localhost").Result() + if (err != nil) || (len(deviceMetaDataKey) == 0) { + //Retry for 1 min + time.Sleep(100 * time.Millisecond) //sleep for 1 sec and then retry + continue + } + + //Redis is populated with DEVICE_METADATA + break + } + + //Now try to fetch the platform details + if (retryCnt < MAX_DEVICE_METADATA_FETCH_RETRY) { + deviceMetaData, err := redisClient.HGetAll("DEVICE_METADATA|localhost").Result() + var exists bool + platformName, exists = deviceMetaData["platform"] + if !exists || (err != nil) || (platformName == "") { + CVL_LOG(WARNING, "Could not fetch 'platform' details from CONFIG_DB") + } + } + + //Scan schema directory to get all schema files + modelFiles, err := filepath.Glob(CVL_SCHEMA + "/*.yin") + if err != nil { + CVL_LOG(FATAL ,"Could not read schema files %v", err) + } + + moduleMap := map[string]*yparser.YParserModule{} + // Load all common schema files + for _, modelFilePath := range modelFiles { + _, modelFile := filepath.Split(modelFilePath) + + TRACE_LOG(TRACE_LIBYANG, "Parsing schema file %s ...", + modelFilePath) + + // Now parse each schema file + var module *yparser.YParserModule + if module, _ = yparser.ParseSchemaFile(modelFilePath); module == nil { + + CVL_LOG(FATAL,fmt.Sprintf("Unable to parse schema file %s", modelFile)) + return CVL_ERROR + } + + moduleMap[modelFile] = module + } + + // Load all platform specific schema files based on platform details + // present in DEVICE_METADATA + for { + if (platformName == "") { + CVL_LOG(INFO, "Skipping parsing of any platform specific YIN schema " + + "files as platform name can't be determined") + break + } + + // Read directory under 'platform' directory + allDirs, errDir := ioutil.ReadDir(CVL_SCHEMA + "/" + PLATFORM_SCHEMA_PATH) + if (errDir != nil) || (len(allDirs) == 0) { + CVL_LOG(INFO, "Could not read platform schema location or no platform " + + "specific schema exists. %v", err) + break + } + + // For matched platform directory parse all schema files + for _, sDir := range allDirs { + + sDirName := sDir.Name() + //Check which directory matches + if !(strings.Contains(platformName, sDirName)) { + continue + } + + //Get all platform specific YIN schema file names + modelFiles, err := filepath.Glob(CVL_SCHEMA + "/" + + PLATFORM_SCHEMA_PATH + "/" + sDirName + "/*.yin") + if err != nil { + CVL_LOG(WARNING,"Could not read platform schema directory %v", err) + break + } + + + //Now parse platform schema files + for _, modelFilePath := range modelFiles { + _, modelFile := filepath.Split(modelFilePath) + + TRACE_LOG(TRACE_YPARSER, "Parsing platform specific schema" + "file %s ...\n", modelFilePath) + + var module *yparser.YParserModule + if module, _ = yparser.ParseSchemaFile(modelFilePath); module == nil { + + CVL_LOG(ERROR, "Unable to parse platform specific schema file %s", modelFile) + return CVL_ERROR + } + + moduleMap[modelFile] = module + } + + //platform found + break + } + + break + } + + for modelFile, parsedModule := range moduleMap { + //store schema related info to use in validation + storeModelInfo(modelFile, parsedModule) + } + + return CVL_SUCCESS +} + //Get list of YANG list names used in xpath expression func getYangListNamesInExpr(expr string) []string { tbl := []string{} @@ -313,185 +418,89 @@ func getLeafRefTargetInfo(path string) ([]string, string) { return tbl, target } -//Store useful schema data during initialization -func storeModelInfo(modelFile string, module *yparser.YParserModule) { //such model info can be maintained in C code and fetched from there - f, err := os.Open(CVL_SCHEMA + modelFile) - root, err := xmlquery.Parse(f) - - if err != nil { - return - } - f.Close() - +func storeModelInfo(modelFile string, module *yparser.YParserModule) { //model is derived from file name tokens := strings.Split(modelFile, ".") modelName := tokens[0] - //Store namespace - modelNs := modelNamespace{} - - nodes := xmlquery.Find(root, "//module/namespace") - if (nodes != nil) { - modelNs.ns = nodes[0].Attr[0].Value - } - - nodes = xmlquery.Find(root, "//module/prefix") - if (nodes != nil) { - modelNs.prefix = nodes[0].Attr[0].Value - } + //Store namespace and prefix + ns, prefix := yparser.GetModelNs(module) + modelInfo.modelNs[modelName] = &modelNamespace{ns:ns, prefix:prefix} - modelInfo.modelNs[modelName] = modelNs + list := yparser.GetModelListInfo(module) - //Store metadata present in each list. - //Each list represent one Redis table in general. - //However when one Redis table is mapped to multiple - //YANG lists need to store the information in redisTableToYangList map - nodes = xmlquery.Find(root, "//module/container/container/list") - if (nodes == nil) { + if (list == nil) { + CVL_LOG(WARNING, "Unable to get schema details for %s", modelFile) return } - //number list under one table container i.e. ACL_TABLE container - //has only one ACL_TABLE_LIST list - for _, node := range nodes { - //for each list, remove "_LIST" suffix - tableName := node.Attr[0].Value - if (strings.HasSuffix(tableName, "_LIST")) { - tableName = tableName[0:len(tableName) - len("_LIST")] - } - tableInfo := modelTableInfo{modelName: modelName} - //Store Redis table name - tableInfo.redisTableName = node.Parent.Attr[0].Value - //Store the reference for list node to be used later - listNode := node - node = node.FirstChild - //Default database is CONFIG_DB since CVL works with config db mainly - tableInfo.module = module - tableInfo.dbNum = CONFIG_DB - //default delim '|' - tableInfo.redisKeyDelim = "|" - modelInfo.allKeyDelims[tableInfo.redisKeyDelim] = true - - fieldCount := 0 - - //Check for meta data in schema - for node != nil { - switch node.Data { - case "db-name": - tableInfo.dbNum = dbNameToDbNum[node.Attr[0].Value] - fieldCount++ - case "key": - tableInfo.keys = strings.Split(node.Attr[0].Value," ") - fieldCount++ - keypattern := []string{tableName} - - // Create the default key pattern of the form Table Name|{key1}|{key2}. - for _ , key := range tableInfo.keys { - keypattern = append(keypattern, fmt.Sprintf("{%s}",key)) - } - - tableInfo.redisKeyPattern = strings.Join(keypattern, tableInfo.redisKeyDelim) - - case "key-delim": - tableInfo.redisKeyDelim = node.Attr[0].Value - fieldCount++ - //store all possible key delims - modelInfo.allKeyDelims[tableInfo.redisKeyDelim] = true - case "key-pattern": - tableInfo.redisKeyPattern = node.Attr[0].Value - fieldCount++ - case "map-leaf": - tableInfo.mapLeaf = strings.Split(node.Attr[0].Value," ") - fieldCount++ - } - node = node.NextSibling - } - //Find and store all leafref under each table - /* - if (listNode == nil) { - //Store the tableInfo in global data - modelInfo.tableInfo[tableName] = tableInfo + for _, lInfo := range list { + TRACE_LOG(TRACE_YPARSER, + "Storing schema details for list %s", lInfo.ListName) - continue - } - */ - - //If container has more than one list, it means one Redis table is mapped to - //multiple lists, store the info in redisTableToYangList - allLists := xmlquery.Find(listNode.Parent, "/list") - if len(allLists) > 1 { - yangList := modelInfo.redisTableToYangList[tableInfo.redisTableName] - yangList = append(yangList, tableName) - //Update the map - modelInfo.redisTableToYangList[tableInfo.redisTableName] = yangList - } + tInfo := modelTableInfo{modelName: modelName} - leafRefNodes := xmlquery.Find(listNode, "//type[@name='leafref']") - if (leafRefNodes == nil) { - //Store the tableInfo in global data - modelInfo.tableInfo[tableName] = &tableInfo + tInfo.dbNum = dbNameToDbNum[lInfo.DbName] + tInfo.redisTableName = lInfo.RedisTableName + tInfo.module = module + tInfo.redisKeyDelim = lInfo.RedisKeyDelim + tInfo.redisKeyPattern = lInfo.RedisKeyPattern + tInfo.redisTableSize = lInfo.RedisTableSize + tInfo.keys = lInfo.Keys + tInfo.mapLeaf = lInfo.MapLeaf + tInfo.mandatoryNodes = lInfo.MandatoryNodes - continue + //store default values used in must and when exp + tInfo.dfltLeafVal = make(map[string]string, len(lInfo.DfltLeafVal)) + for nodeName, val := range lInfo.DfltLeafVal { + tInfo.dfltLeafVal[nodeName] = val } - tableInfo.leafRef = make(map[string][]*leafRefInfo) - - for _, leafRefNode := range leafRefNodes { - if (leafRefNode.Parent == nil || leafRefNode.FirstChild == nil) { - continue - } - - //Get the leaf/leaf-list name holding this leafref - //Note that leaf can have union of leafrefs - leafName := "" - for node := leafRefNode.Parent; node != nil; node = node.Parent { - if (node.Data == "leaf" || node.Data == "leaf-list") { - leafName = getXmlNodeAttr(node, "name") - break - } - } - - //Store the leafref path - if (leafName != "") { - tableInfo.leafRef[leafName] = append(tableInfo.leafRef[leafName], - &leafRefInfo{path: getXmlNodeAttr(leafRefNode.FirstChild, "value")}) + //Store leafref details + tInfo.leafRef = make(map[string][]*leafRefInfo, len(lInfo.LeafRef)) + for nodeName, lpathArr := range lInfo.LeafRef { //for each leaf or leaf-list + leafRefInfoArr := []*leafRefInfo{} + for _, lpath := range lpathArr { + //just store the leafref path + leafRefData := leafRefInfo{path: lpath} + leafRefInfoArr = append(leafRefInfoArr, &leafRefData) } + tInfo.leafRef[nodeName] = leafRefInfoArr } - //Find all 'must' expression and store the against its parent node - mustExps := xmlquery.Find(listNode, "//must") - if (mustExps == nil) { - //Update the tableInfo in global data - modelInfo.tableInfo[tableName] = &tableInfo - continue + //Store must expression details + tInfo.mustExpr = make(map[string][]*mustInfo, len(lInfo.XpathExpr)) + for nodeName, xprArr := range lInfo.XpathExpr { + for _, xpr := range xprArr { + tInfo.mustExpr[nodeName] = append(tInfo.mustExpr[nodeName], + &mustInfo{ + expr: xpr.Expr, + errCode: xpr.ErrCode, + errStr: xpr.ErrStr, + }) + } } - tableInfo.mustExpr = make(map[string][]*mustInfo) - for _, mustExp := range mustExps { - if (mustExp.Parent == nil) { - continue - } - parentName := "" - for node := mustExp.Parent; node != nil; node = node.Parent { - //assuming must exp is at leaf or list level - if (node.Data == "leaf" || node.Data == "leaf-list" || - node.Data == "list") { - parentName = getXmlNodeAttr(node, "name") - break - } - } - if (parentName != "") { - tableInfo.mustExpr[parentName] = append(tableInfo.mustExpr[parentName], - &mustInfo{ - expr: getXmlNodeAttr(mustExp, "condition"), + //Store when expression details + tInfo.whenExpr = make(map[string][]*whenInfo, len(lInfo.WhenExpr)) + for nodeName, whenExprArr := range lInfo.WhenExpr { + for _, whenExpr := range whenExprArr { + tInfo.whenExpr[nodeName] = append(tInfo.whenExpr[nodeName], + &whenInfo { + expr: whenExpr.Expr, + nodeNames: whenExpr.NodeNames, }) } } - //Update the tableInfo in global data - modelInfo.tableInfo[tableName] = &tableInfo + modelInfo.allKeyDelims[tInfo.redisKeyDelim] = true + yangList := modelInfo.redisTableToYangList[tInfo.redisTableName] + yangList = append(yangList, lInfo.ListName) + //Update the map + modelInfo.redisTableToYangList[tInfo.redisTableName] = yangList + modelInfo.tableInfo[lInfo.ListName] = &tInfo } } @@ -577,43 +586,40 @@ func buildRefTableInfo() { func addTableNamesForMustExp() { for tblName, tblInfo := range modelInfo.tableInfo { - if (tblInfo.mustExpr == nil) { + if (len(tblInfo.mustExpr) == 0) { continue } tblInfo.tablesForMustExp = make(map[string]CVLOperation) - for _, mustExp := range tblInfo.mustExpr { - var op CVLOperation = OP_NONE - //Check if 'must' expression should be executed for a particular operation - if (strings.Contains(mustExp[0].expr, - "/scommon:operation/scommon:operation != CREATE") == true) { - op = op | OP_CREATE - } else if (strings.Contains(mustExp[0].expr, - "/scommon:operation/scommon:operation != UPDATE") == true) { - op = op | OP_UPDATE - } else if (strings.Contains(mustExp[0].expr, - "/scommon:operation/scommon:operation != DELETE") == true) { - op = op | OP_DELETE - } - - //store the current table if aggregate function like count() is used - /*if (strings.Contains(mustExp, "count") == true) { - tblInfo.tablesForMustExp[tblName] = op - }*/ - - //check which table name is present in the must expression - for tblNameSrch, _ := range modelInfo.tableInfo { - if (tblNameSrch == tblName) { - continue + for _, mustExpArr := range tblInfo.mustExpr { + for _, mustExp := range mustExpArr { + var op CVLOperation = OP_NONE + //Check if 'must' expression should be executed for a particular operation + if strings.Contains(mustExp.expr, ":operation != 'CREATE'") { + op = op | OP_CREATE + } + if strings.Contains(mustExp.expr, ":operation != 'UPDATE'") { + op = op | OP_UPDATE } - //Table name should appear like "../VLAN_MEMBER/tagging_mode' or ' - // "/prt:PORT/prt:ifname" - re := regexp.MustCompile(fmt.Sprintf(".*[/]([a-zA-Z]*:)?%s[\\[/]", tblNameSrch)) - matches := re.FindStringSubmatch(mustExp[0].expr) - if (len(matches) > 0) { - //stores the table name - tblInfo.tablesForMustExp[tblNameSrch] = op + if strings.Contains(mustExp.expr, ":operation != 'DELETE'") { + op = op | OP_DELETE + } + + //store the current table if aggregate function like count() is used + //check which table name is present in the must expression + for tblNameSrch := range modelInfo.tableInfo { + if (tblNameSrch == tblName) { + continue + } + //Table name should appear like "../VLAN_MEMBER_LIST/tagging_mode' or ' + // "/prt:PORT/prt:ifname" + re := regexp.MustCompile(fmt.Sprintf(".*[/]([-_a-zA-Z]*:)?%s_LIST[\\[/]?", tblNameSrch)) + matches := re.FindStringSubmatch(mustExp.expr) + if (len(matches) > 0) { + //stores the table name + tblInfo.tablesForMustExp[tblNameSrch] = op + } } } } @@ -628,7 +634,7 @@ func splitRedisKey(key string) (string, string) { var foundIdx int = -1 //Check with all key delim - for keyDelim, _ := range modelInfo.allKeyDelims { + for keyDelim := range modelInfo.allKeyDelims { foundIdx = strings.Index(key, keyDelim) if (foundIdx >= 0) { //Matched with key delim @@ -638,18 +644,25 @@ func splitRedisKey(key string) (string, string) { if (foundIdx < 0) { //No matches + CVL_LOG(WARNING, "Could not find any of key delimeter %v in key '%s'", + modelInfo.allKeyDelims, key) return "", "" } tblName := key[:foundIdx] - if _, exists := modelInfo.tableInfo[tblName]; exists == false { + if _, exists := modelInfo.tableInfo[tblName]; !exists { //Wrong table name + CVL_LOG(WARNING, "Could not find table '%s' in schema", tblName) return "", "" } prefixLen := foundIdx + 1 + + TRACE_LOG(TRACE_SYNTAX, "Split Redis Key %s into (%s, %s)", + key, tblName, key[prefixLen:]) + return tblName, key[prefixLen:] } @@ -662,7 +675,7 @@ func splitRedisKey(key string) (string, string) { func getRedisTblToYangList(tableName, key string) (yangList string) { defer func() { pYangList := &yangList - TRACE_LOG(INFO_API, TRACE_SYNTAX, "Got YANG list '%s' " + + TRACE_LOG(TRACE_SYNTAX, "Got YANG list '%s' " + "from Redis Table '%s', Key '%s'", *pYangList, tableName, key) }() @@ -725,495 +738,22 @@ func getRedisToYangKeys(tableName string, redisKey string)[]keyValuePairStruct{ for idx, keyName := range keyNames { //check if key-pattern contains specific key pattern - if (keyPatterns[idx+1] == fmt.Sprintf("({%s},)*", keyName)) { // key pattern is "({key},)*" i.e. repeating keys seperated by ',' - repeatedKeys := strings.Split(keyVals[idx], ",") - mkeys = append(mkeys, keyValuePairStruct{keyName, repeatedKeys}) - - } else if (keyPatterns[idx+1] == fmt.Sprintf("{%s}", keyName)) { //no specific key pattern - just "{key}" - + if (keyPatterns[idx+1] == ("{" + keyName + "}")) { //no specific key pattern - just "{key}" //Store key/value mapping mkeys = append(mkeys, keyValuePairStruct{keyName, []string{keyVals[idx]}}) + } else if (keyPatterns[idx+1] == ("({" + keyName + "},)*")) { // key pattern is "({key},)*" i.e. repeating keys seperated by ',' + repeatedKeys := strings.Split(keyVals[idx], ",") + mkeys = append(mkeys, keyValuePairStruct{keyName, repeatedKeys}) } } - return mkeys -} - -//Check for path resolution -func (c *CVL) checkPathForTableEntry(tableName string, currentValue string, cfgData *CVLEditConfigData, mustExpStk []string, token string) ([]string, string, CVLRetCode) { - - n := len(mustExpStk) - 1 - xpath := "" - - if (token == ")") { - for n = len(mustExpStk) - 1; mustExpStk[n] != "("; n = len(mustExpStk) - 1 { - //pop until "(" - xpath = mustExpStk[n] + xpath - mustExpStk = mustExpStk[:n] - } - } else if (token == "]") { - //pop until "[" - for n = len(mustExpStk) - 1; mustExpStk[n] != "["; n = len(mustExpStk) - 1 { - xpath = mustExpStk[n] + xpath - mustExpStk = mustExpStk[:n] - } - } - - mustExpStk = mustExpStk[:n] - targetTbl := "" - //Search the table name in xpath - for tblNameSrch, _ := range modelInfo.tableInfo { - if (tblNameSrch == tableName) { - continue - } - //Table name should appear like "../VLAN_MEMBER/tagging_mode' or ' - // "/prt:PORT/prt:ifname" - //re := regexp.MustCompile(fmt.Sprintf(".*[/]([a-zA-Z]*:)?%s[\\[/]", tblNameSrch)) - tblSrchIdx := strings.Index(xpath, fmt.Sprintf("/%s_LIST", tblNameSrch)) //no preifx - if (tblSrchIdx < 0) { - tblSrchIdx = strings.Index(xpath, fmt.Sprintf(":%s_LIST", tblNameSrch)) //with prefix - } - if (tblSrchIdx < 0) { - continue - } - - tblSrchIdxEnd := strings.Index(xpath[tblSrchIdx+len(tblNameSrch)+1:], "[") - if (tblSrchIdxEnd < 0) { - tblSrchIdxEnd = strings.Index(xpath[tblSrchIdx+len(tblNameSrch)+1:], "/") - } - - if (tblSrchIdxEnd >= 0) { //match found - targetTbl = tblNameSrch - break - } - } - - //No match with table found, could be just keys like 'aclname='TestACL1' - //just return the same - if (targetTbl == "") { - return mustExpStk, xpath, CVL_SUCCESS - } - - tableKey := targetTbl - //Add the keys - keyNames := modelInfo.tableInfo[tableKey].keys - - //Form the Redis Key to fetch the entry - for idx, keyName := range keyNames { - //Key value is string/numeric literal, extract the same - keySrchIdx := strings.Index(xpath, keyName) - if (keySrchIdx < 0 ) { - continue - } - - matches := reSelKeyVal.FindStringSubmatch(xpath[keySrchIdx+len(keyName):]) - if (len(matches) > 1) { - if (matches[1] == "current()") { - //replace with current field value - tableKey = tableKey + "*" + modelInfo.tableInfo[tableName].redisKeyDelim + currentValue - } else { - //Use literal - tableKey = tableKey + "*" + modelInfo.tableInfo[tableName].redisKeyDelim + matches[1] - } - - if (idx != len(keyNames) - 1) { - tableKey = tableKey + "|*" - } - } - } - - //Fetch the entries - redisTableKeys, err:= redisClient.Keys(tableKey).Result() - - if (err !=nil || len (redisTableKeys) > 1) { //more than one entry is returned, can't proceed further - //Just add all the entries for caching - for _, redisTableKey := range redisTableKeys { - c.addTableEntryToCache(splitRedisKey(redisTableKey)) - } - - return mustExpStk, "", CVL_SUCCESS - } - - for _, redisTableKey := range redisTableKeys { - - var entry map[string]string - - if tmpEntry, mergeNeeded := c.fetchDataFromRequestCache(splitRedisKey(redisTableKey)); (tmpEntry == nil || mergeNeeded == true) { - //If data is not available in validated cache fetch from Redis DB - entry, err = redisClient.HGetAll(redisTableKey).Result() - - if (mergeNeeded) { - mergeMap(entry, tmpEntry) - } - } - - //Get the entry fields from Redis - if (entry != nil) { - //Just add all the entries for caching - c.addTableEntryToCache(splitRedisKey(redisTableKey)) - - leafInPath := "" - index := strings.LastIndex(xpath, "/") - if (index >= 0) { - - matches := reLeafInXpath.FindStringSubmatch(xpath[index:]) - if (len(matches) > 2) { //should return atleasts two subgroup and entire match - leafInPath = matches[2] - } else { - //No leaf requested in xpath selection - return mustExpStk, "", CVL_SUCCESS - } - - index = strings.Index(xpath, "=") - tblIndex := strings.Index(xpath, targetTbl) - if (index >= 0 && tblIndex >=0) { - - if leafVal, existing := entry[leafInPath + "@"]; existing == true { - //Get the field value referred in the xpath - if (index < tblIndex) { - // case like - [ifname=../../ACL_TABLE[aclname=current()] - return mustExpStk, xpath[:index+1] + leafVal, CVL_SUCCESS - } else { - // case like - [ifname=current()] - return mustExpStk, leafVal, CVL_SUCCESS - } - } else { - - if (index < tblIndex) { - return mustExpStk, xpath[:index+1] + entry[leafInPath], CVL_SUCCESS - } else { - return mustExpStk, entry[leafInPath], CVL_SUCCESS - } - } - } - } - } - } - - return mustExpStk, "", CVL_FAILURE -} - -//Add specific entries by looking at must expression -//Must expression may need single or multiple entries -//It can be within same table or across multiple tables -//Node-set function such count() can be quite expensive and -//should be avoided through this function -func (c *CVL) addTableEntryForMustExp(cfgData *CVLEditConfigData, tableName string) CVLRetCode { - if (modelInfo.tableInfo[tableName].mustExpr == nil) { - return CVL_SUCCESS - } - - for fieldName, mustExp := range modelInfo.tableInfo[tableName].mustExpr { - - currentValue := "" // Current value for current() function - - //Get the current() field value from the entry being created/updated/deleted - keyValuePair := getRedisToYangKeys(tableName, cfgData.Key[len(tableName)+1:]) - - //Try to get the current() from the 'key' provided - if (keyValuePair != nil) { - for _, keyValItem := range keyValuePair { - if (keyValItem.key == fieldName) { - currentValue = keyValItem.values[0] - } - } - } - - //current() value needs to be fetched from other field - if (currentValue == "") { - if (cfgData.VOp == OP_CREATE) { - if (tableName != fieldName) { //must expression is not at list level - currentValue = cfgData.Data[fieldName] - if (currentValue == "") { - currentValue = cfgData.Data[fieldName + "@"] - } - } - } else if (cfgData.VOp == OP_UPDATE || cfgData.VOp == OP_DELETE) { - //fetch the entry to get current() value - c.clearTmpDbCache() - entryKey := cfgData.Key[len(tableName)+1:] - c.tmpDbCache[tableName] = map[string]interface{}{entryKey: nil} - - if (c.fetchTableDataToTmpCache(tableName, - map[string]interface{}{entryKey: nil}) > 0) { - mapTable := c.tmpDbCache[tableName] - if fields, existing := mapTable.(map[string]interface{})[entryKey]; existing == true { - currentValue = fmt.Sprintf("%v", fields.(map[string]interface{})[fieldName]) - } - } - } - } - - mustExpStk := []string{} //Use the string slice as stack - mustExpStr := "(" + mustExp[0].expr + ")" - strLen := len(mustExpStr) - strTmp := "" - //Parse the xpath expression and fetch Redis entry by looking at xpath, - // any xpath function call is ignored except current(). - for i := 0; i < strLen; i++ { - switch mustExpStr[i] { - case '(': - if (mustExpStr[i+1] == ')') { - strTmp = strTmp + "()" - if index := strings.Index(strTmp, "current()"); index >= 0 { - strTmp = strTmp[:index] + currentValue - } - i = i + 1 - continue - } - if (strTmp != "") { - mustExpStk = append(mustExpStk, strTmp) - } - mustExpStk = append(mustExpStk, "(") - strTmp = "" - case ')': - if (strTmp != "") { - mustExpStk = append(mustExpStk, strTmp) - } - strTmp = "" - //Check Path - pop until ')' - mustExpStk, evalPath,_ := c.checkPathForTableEntry(tableName, currentValue, cfgData, - mustExpStk, ")") - if (evalPath != "") { - mustExpStk = append(mustExpStk, evalPath) - } - mustExpStk = append(mustExpStk, ")") - case '[': - if (strTmp != "") { - mustExpStk = append(mustExpStk, strTmp) - } - mustExpStk = append(mustExpStk, "[") - strTmp = "" - case ']': - if (strTmp != "") { - mustExpStk = append(mustExpStk, strTmp) - } - //Check Path - pop until = or '[' - mustExpStk, evalPath,_ := c.checkPathForTableEntry(tableName, currentValue, cfgData, - mustExpStk, "]") - if (evalPath != "") { - mustExpStk = append(mustExpStk, "[" + evalPath + "]") - } - strTmp = "" - default: - strTmp = fmt.Sprintf("%s%c", strTmp, mustExpStr[i]) - } - } - - //Get the redis data for accumulated keys and add them to session cache - depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation - - if (depData != nil) { - if (Tracing == true) { - TRACE_LOG(INFO_API, TRACE_CACHE, "Adding entries for 'must' expression : %s", c.yp.NodeDump(depData)) - } - } else { - //Could not fetch any entry from Redis after xpath evaluation - return CVL_FAILURE - } - - if errObj := c.yp.CacheSubtree(false, depData); errObj.ErrCode != yparser.YP_SUCCESS { - return CVL_FAILURE - } - - } //for each must expression - - return CVL_SUCCESS -} - -//Add all other table data for validating all 'must' exp for tableName -func (c *CVL) addTableDataForMustExp(op CVLOperation, tableName string) CVLRetCode { - if (modelInfo.tableInfo[tableName].mustExpr == nil) { - return CVL_SUCCESS - } - - for mustTblName, mustOp := range modelInfo.tableInfo[tableName].tablesForMustExp { - //First check if must expression should be executed for the given operation - if (mustOp != OP_NONE) && ((mustOp & op) == OP_NONE) { - //must to be excuted for particular operation, but current operation - //is not the same one - continue - } - - //Check in global cache first and merge to session cache - if topNode, _ := dbCacheGet(mustTblName); topNode != nil { - var errObj yparser.YParserError - //If global cache has the table, add to the session validation - TRACE_LOG(INFO_API, TRACE_CACHE, "Adding global cache to session cache for table %s", tableName) - if errObj = c.yp.CacheSubtree(true, topNode); errObj.ErrCode != yparser.YP_SUCCESS { - return CVL_SYNTAX_ERROR - } - } else { //Put the must table in global table and add to session cache - cvg.cv.chkLeafRefWithOthCache = true - dbCacheSet(false, mustTblName, 100*DEFAULT_CACHE_DURATION) //Keep the cache for default duration - cvg.cv.chkLeafRefWithOthCache = false - - if topNode, ret := dbCacheGet(mustTblName); topNode != nil { - var errObj yparser.YParserError - //If global cache has the table, add to the session validation - TRACE_LOG(INFO_API, TRACE_CACHE, "Global cache created, add the data to session cache for table %s", tableName) - if errObj = c.yp.CacheSubtree(true, topNode); errObj.ErrCode != yparser.YP_SUCCESS { - return CVL_SYNTAX_ERROR - } - } else if (ret == CVL_SUCCESS) { - TRACE_LOG(INFO_API, TRACE_CACHE, "Global cache empty, no data in Redis for table %s", tableName) - return CVL_SUCCESS - } else { - CVL_LOG(ERROR ,"Could not create global cache for table %s", mustTblName) - return CVL_ERROR - } - - - /* - tableKeys, err:= redisClient.Keys(mustTblName + - modelInfo.tableInfo[mustTblName].redisKeyDelim + "*").Result() - - if (err != nil) { - continue - } - - for _, tableKey := range tableKeys { - tableKey = tableKey[len(mustTblName+ modelInfo.tableInfo[mustTblName].redisKeyDelim):] //remove table prefix - if (c.tmpDbCache[mustTblName] == nil) { - c.tmpDbCache[mustTblName] = map[string]interface{}{tableKey: nil} - } else { - tblMap := c.tmpDbCache[mustTblName] - tblMap.(map[string]interface{})[tableKey] =nil - c.tmpDbCache[mustTblName] = tblMap - } - } - */ - } - } - - return CVL_SUCCESS -} - -func (c *CVL) addTableEntryToCache(tableName string, redisKey string) { - if (tableName == "" || redisKey == "") { - return - } - - if (c.tmpDbCache[tableName] == nil) { - c.tmpDbCache[tableName] = map[string]interface{}{redisKey: nil} - } else { - tblMap := c.tmpDbCache[tableName] - tblMap.(map[string]interface{})[redisKey] =nil - c.tmpDbCache[tableName] = tblMap - } -} - -//Add the data which are referring this key -func (c *CVL) updateDeleteDataToCache(tableName string, redisKey string) { - if _, existing := c.tmpDbCache[tableName]; existing == false { - return - } else { - tblMap := c.tmpDbCache[tableName] - if _, existing := tblMap.(map[string]interface{})[redisKey]; existing == true { - delete(tblMap.(map[string]interface{}), redisKey) - c.tmpDbCache[tableName] = tblMap - } - } -} - -//Find which all tables (and which field) is using given (tableName/field) -// as leafref -//Use LUA script to find if table has any entry for this leafref - -type tblFieldPair struct { - tableName string - field string -} - -func (c *CVL) findUsedAsLeafRef(tableName, field string) []tblFieldPair { - - var tblFieldPairArr []tblFieldPair - - for tblName, tblInfo := range modelInfo.tableInfo { - if (tableName == tblName) { - continue - } - if (len(tblInfo.leafRef) == 0) { - continue - } - - for fieldName, leafRefs := range tblInfo.leafRef { - found := false - //Find leafref by searching table and field name - for _, leafRef := range leafRefs { - if ((strings.Contains(leafRef.path, tableName) == true) && - (strings.Contains(leafRef.path, field) == true)) { - tblFieldPairArr = append(tblFieldPairArr, - tblFieldPair{tblName, fieldName}) - //Found as leafref, no need to search further - found = true - break - } - } - - if (found == true) { - break - } - } - } - - return tblFieldPairArr -} - -//Add leafref entry for caching -//It has to be recursive in nature, as there can be chained leafref -func (c *CVL) addLeafRef(config bool, tableName string, name string, value string) { - - if (config == false) { - return - } - - //Check if leafRef entry is there for this field - if (len(modelInfo.tableInfo[tableName].leafRef[name]) > 0) { //array of leafrefs for a leaf - for _, leafRef := range modelInfo.tableInfo[tableName].leafRef[name] { - - //Get reference table name from the path and the leaf name - matches := reLeafRef.FindStringSubmatch(leafRef.path) - - //We have the leafref table name and the leaf name as well - if (matches != nil && len(matches) == 5) { //whole + 4 sub matches - refTableName := matches[2] - redisKey := value - - //Check if leafref dependency can also be met from 'must' table - if (c.chkLeafRefWithOthCache == true) { - found := false - for mustTbl, _ := range modelInfo.tableInfo[tableName].tablesForMustExp { - if mustTbl == refTableName { - found = true - break - } - } - if (found == true) { - //Leafref data will be available from must table dep data, skip this leafref entry - continue - } - } + TRACE_LOG(TRACE_SYNTAX, "getRedisToYangKeys() returns %v " + + "from Redis Table '%s', Key '%s'", mkeys, tableName, redisKey) - //only key is there, value wil be fetched and stored here, - //if value can't fetched this entry will be deleted that time - //Strip "_LIST" suffix - refRedisTableName := refTableName[0:len(refTableName) - len("_LIST")] - if (c.tmpDbCache[refRedisTableName] == nil) { - c.tmpDbCache[refRedisTableName] = map[string]interface{}{redisKey: nil} - } else { - tblMap := c.tmpDbCache[refRedisTableName] - _, exist := tblMap.(map[string]interface{})[redisKey] - if (exist == false) { - tblMap.(map[string]interface{})[redisKey] = nil - c.tmpDbCache[refRedisTableName] = tblMap - } - } - } - } - } + return mkeys } +//Checks field map values and removes "NULL" entry, create array for leaf-list func (c *CVL) checkFieldMap(fieldMap *map[string]string) map[string]interface{} { fieldMapNew := map[string]interface{}{} @@ -1241,6 +781,9 @@ func (c *CVL) checkFieldMap(fieldMap *map[string]string) map[string]interface{} //Merge 'src' map to 'dest' map of map[string]string type func mergeMap(dest map[string]string, src map[string]string) { + TRACE_LOG(TRACE_SEMANTIC, + "Merging map %v into %v", src, dest) + for key, data := range src { dest[key] = data } @@ -1256,22 +799,57 @@ func (c *CVL) translateToYang(jsonMap *map[string]interface{}) (*yparser.YParser var errObj yparser.YParserError for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { - TRACE_LOG(INFO_API, TRACE_LIBYANG, "Top Node=%v\n", jsonNode.Data) + TRACE_LOG(TRACE_LIBYANG, "Translating, Top Node=%v\n", jsonNode.Data) //Visit each top level list in a loop for creating table data topNode, cvlErrObj := c.generateTableData(true, jsonNode) + //Generate YANG data for Yang Validator + topYangNode, cvlYErrObj := c.generateYangListData(jsonNode, true) + if topNode == nil { cvlErrObj.ErrCode = CVL_SYNTAX_ERROR + CVL_LOG(WARNING, "Unable to translate request data to YANG format") return nil, cvlErrObj } + if topYangNode == nil { + cvlYErrObj.ErrCode = CVL_SYNTAX_ERROR + CVL_LOG(WARNING, "Unable to translate request data to YANG format") + return nil, cvlYErrObj + } + if (root == nil) { root = topNode } else { if root, errObj = c.yp.MergeSubtree(root, topNode); errObj.ErrCode != yparser.YP_SUCCESS { + CVL_LOG(WARNING, "Unable to merge translated YANG data(libyang) " + + "while translating from request data to YANG format") return nil, cvlErrObj } } + + //Create a full document and merge with main YANG data + doc := &xmlquery.Node{Type: xmlquery.DocumentNode} + doc.FirstChild = topYangNode + doc.LastChild = topYangNode + topYangNode.Parent = doc + + if (IsTraceLevelSet(TRACE_CACHE)) { + TRACE_LOG(TRACE_CACHE, "Before merge, YANG data tree = %s, source = %s", + c.yv.root.OutputXML(false), + doc.OutputXML(false)) + } + + if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS { + CVL_LOG(WARNING, "Unable to merge translated YANG data while " + + "translating from request data to YANG format") + cvlYErrObj.ErrCode = CVL_SYNTAX_ERROR + return nil, cvlErrObj + } + if (IsTraceLevelSet(TRACE_CACHE)) { + TRACE_LOG(TRACE_CACHE, "After merge, YANG data tree = %s", + c.yv.root.OutputXML(false)) + } } return root, cvlErrObj @@ -1281,24 +859,18 @@ func (c *CVL) translateToYang(jsonMap *map[string]interface{}) (*yparser.YParser func (c *CVL) validate (data *yparser.YParserNode) CVLRetCode { depData := c.fetchDataToTmpCache() - /* - if (depData != nil) { - if (0 != C.lyd_merge_to_ctx(&data, depData, C.LYD_OPT_DESTRUCT, ctx)) { - TRACE_LOG(1, "Failed to merge status data\n") - } - } - if (0 != C.lyd_data_validate(&data, C.LYD_OPT_CONFIG, ctx)) { - fmt.Println("Validation failed\n") - return CVL_SYNTAX_ERROR - }*/ - - TRACE_LOG(INFO_DATA, TRACE_LIBYANG, "\nValidate1 data=%v\n", c.yp.NodeDump(data)) - errObj := c.yp.ValidateData(data, depData) + TRACE_LOG(TRACE_LIBYANG, "\nValidate1 data=%v\n", c.yp.NodeDump(data)) + errObj := c.yp.ValidateSyntax(data, depData) if yparser.YP_SUCCESS != errObj.ErrCode { return CVL_FAILURE } + cvlErrObj := c.validateCfgSemantics(c.yv.root) + if CVL_SUCCESS != cvlErrObj.ErrCode { + return cvlErrObj.ErrCode + } + return CVL_SUCCESS } @@ -1324,16 +896,19 @@ func createCVLErrObj(errObj yparser.YParserError) CVLErrorInfo { //Perform syntax checks func (c *CVL) validateSyntax(data *yparser.YParserNode) (CVLErrorInfo, CVLRetCode) { var cvlErrObj CVLErrorInfo - TRACE_LOG(INFO_DATA, TRACE_LIBYANG, "Validating syntax \n....") + TRACE_LOG(TRACE_YPARSER, "Validating syntax ....") - if errObj := c.yp.ValidateSyntax(data); errObj.ErrCode != yparser.YP_SUCCESS { + //Get dependent data from Redis + depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation + + if errObj := c.yp.ValidateSyntax(data, depData); errObj.ErrCode != yparser.YP_SUCCESS { retCode := CVLRetCode(errObj.ErrCode) cvlErrObj = CVLErrorInfo { TableName : errObj.TableName, - ErrCode : CVLRetCode(errObj.ErrCode), - CVLErrDetails : cvlErrorMap[retCode], + ErrCode : CVLRetCode(errObj.ErrCode), + CVLErrDetails : cvlErrorMap[retCode], Keys : errObj.Keys, Value : errObj.Value, Field : errObj.Field, @@ -1342,7 +917,7 @@ func (c *CVL) validateSyntax(data *yparser.YParserNode) (CVLErrorInfo, CVLRetCod ErrAppTag : errObj.ErrAppTag, } - + CVL_LOG(WARNING,"Syntax validation failed. Error - %v", cvlErrObj) return cvlErrObj, retCode } @@ -1353,8 +928,7 @@ func (c *CVL) validateSyntax(data *yparser.YParserNode) (CVLErrorInfo, CVLRetCod //Add config data item to accumulate per table func (c *CVL) addCfgDataItem(configData *map[string]interface{}, cfgDataItem CVLEditConfigData) (string, string){ - var cfgData map[string]interface{} - cfgData = *configData + var cfgData map[string]interface{} = *configData tblName, key := splitRedisKey(cfgDataItem.Key) if (tblName == "" || key == "") { @@ -1362,227 +936,46 @@ func (c *CVL) addCfgDataItem(configData *map[string]interface{}, return "", "" } - if (cfgDataItem.VOp == OP_DELETE) { - //Don't add data it is delete operation - return tblName, key - } - if _, existing := cfgData[tblName]; existing { fieldsMap := cfgData[tblName].(map[string]interface{}) + if (cfgDataItem.VOp == OP_DELETE) { + return tblName, key + } fieldsMap[key] = c.checkFieldMap(&cfgDataItem.Data) } else { fieldsMap := make(map[string]interface{}) - fieldsMap[key] = c.checkFieldMap(&cfgDataItem.Data) + if (cfgDataItem.VOp == OP_DELETE) { + fieldsMap[key] = nil + } else { + fieldsMap[key] = c.checkFieldMap(&cfgDataItem.Data) + } cfgData[tblName] = fieldsMap } return tblName, key } -//Get table entry from cache for redis key -func dbCacheEntryGet(tableName, key string) (*yparser.YParserNode, CVLRetCode) { - //First check if the table is cached - topNode, _ := dbCacheGet(tableName) - - - if (topNode != nil) { - //Convert to Yang keys - keyValuePair := getRedisToYangKeys(tableName, key) - - //Find if the entry is cached - keyCompStr := "" - for _, keyValItem := range keyValuePair { - keyCompStr = keyCompStr + fmt.Sprintf("[%s='%s']", - keyValItem.key, keyValItem.values[0]) - } - - entryNode := yparser.FindNode(topNode, fmt.Sprintf("//%s:%s/%s%s", - modelInfo.tableInfo[tableName].modelName, - modelInfo.tableInfo[tableName].modelName, - tableName, keyCompStr)) - - if (entryNode != nil) { - return entryNode, CVL_SUCCESS +// getLeafRefInfo This function returns leafrefInfo structure based on table name, +// target table name and leaf node name where leafRef is present +func getLeafRefInfo(tblName, fldName, targetTblName string) *leafRefInfo { + for _, refTblLeafRef := range modelInfo.tableInfo[tblName].leafRef[fldName] { + if (refTblLeafRef.path == "non-leafref") { + continue } - } - return nil, CVL_ERROR -} - -//Get the data from global cache -func dbCacheGet(tableName string) (*yparser.YParserNode, CVLRetCode) { - - TRACE_LOG(INFO_ALL, TRACE_CACHE, "Updating global cache for table %s", tableName) - dbCacheTmp, existing := cvg.db[tableName] - - if (existing == false) { - return nil, CVL_FAILURE //not even empty cache present - } - - if (dbCacheTmp.root != nil) { - if (dbCacheTmp.expiry != 0) { - //If cache is destroyable (i.e. expiry != 0), check if it has already expired. - //If not expired update the time stamp - if (time.Now().After(dbCacheTmp.startTime.Add(time.Second * time.Duration(dbCacheTmp.expiry)))) { - //Cache expired, clear the cache - dbCacheClear(tableName) - - return nil, CVL_ERROR + for k := range refTblLeafRef.yangListNames { + if refTblLeafRef.yangListNames[k] == targetTblName { + return refTblLeafRef } - - //Since the cache is used actively, update the timestamp - dbCacheTmp.startTime = time.Now() - cvg.db[tableName] = dbCacheTmp - } - - return dbCacheTmp.root, CVL_SUCCESS - } else { - return nil, CVL_SUCCESS // return success for no entry in Redis db and hencec empty cache - } -} - -//Get the table data from redis and cache it in yang node format -//expiry =0 never expire the cache -func dbCacheSet(update bool, tableName string, expiry uint16) CVLRetCode { - - cvg.mutex.Lock() - - //Get the data from redis and save it - tableKeys, err:= redisClient.Keys(tableName + - modelInfo.tableInfo[tableName].redisKeyDelim + "*").Result() - - if (err != nil) { - cvg.mutex.Unlock() - return CVL_FAILURE - } - - TRACE_LOG(INFO_ALL, TRACE_CACHE, "Building global cache for table %s", tableName) - - tablePrefixLen := len(tableName + modelInfo.tableInfo[tableName].redisKeyDelim) - for _, tableKey := range tableKeys { - tableKey = tableKey[tablePrefixLen:] //remove table prefix - if (cvg.cv.tmpDbCache[tableName] == nil) { - cvg.cv.tmpDbCache[tableName] = map[string]interface{}{tableKey: nil} - } else { - tblMap := cvg.cv.tmpDbCache[tableName] - tblMap.(map[string]interface{})[tableKey] =nil - cvg.cv.tmpDbCache[tableName] = tblMap } } - - cvg.db[tableName] = dbCachedData{startTime:time.Now(), expiry: expiry, - root: cvg.cv.fetchDataToTmpCache()} - - if (Tracing == true) { - TRACE_LOG(INFO_ALL, TRACE_CACHE, "Cached Data = %v\n", cvg.cv.yp.NodeDump(cvg.db[tableName].root)) - } - - cvg.mutex.Unlock() - - //install keyspace notification for updating the cache - if (update == false) { - installDbChgNotif() - } - - - return CVL_SUCCESS + return nil } -//Receive all updates for all tables on a single channel -func installDbChgNotif() { - if (len(cvg.db) > 1) { //notif running for at least one table added previously - cvg.stopChan <- 1 //stop active notification +func isMandatoryTrueNode(tblName, field string) bool { + if flag, exists := modelInfo.tableInfo[tblName].mandatoryNodes[field]; exists { + return flag } - subList := make([]string, 0) - for tableName, _ := range cvg.db { - subList = append(subList, - fmt.Sprintf("__keyspace@%d__:%s%s*", modelInfo.tableInfo[tableName].dbNum, - tableName, modelInfo.tableInfo[tableName].redisKeyDelim)) - - } - - //Listen on multiple channels - cvg.pubsub = redisClient.PSubscribe(subList...) - - go func() { - keySpacePrefixLen := len("__keyspace@4__:") - - notifCh := cvg.pubsub.Channel() - for { - select { - case <-cvg.stopChan: - //stop this routine - return - case msg:= <-notifCh: - //Handle update - tbl, key := splitRedisKey(msg.Channel[keySpacePrefixLen:]) - if (tbl != "" && key != "") { - dbCacheUpdate(tbl, key, msg.Payload) - } - } - } - }() + return false } - -func dbCacheUpdate(tableName, key, op string) CVLRetCode { - TRACE_LOG(INFO_ALL, TRACE_CACHE, "Updating global cache for table %s with key %s", tableName, key) - - //Find the node - //Delete the entry in yang tree - - cvg.mutex.Lock() - - node, _:= dbCacheEntryGet(tableName, key) - if (node != nil) { - //unlink and free the node - cvg.cv.yp.FreeNode(node) - } - - //Clear json map cache if any - cvg.cv.clearTmpDbCache() - - tableKeys := []string {key} - switch op { - case "hset", "hmset", "hdel": - //Get the entry from DB - for _, tableKey := range tableKeys { - cvg.cv.tmpDbCache[tableName] = map[string]interface{}{tableKey: nil} - } - - //Get the translated Yang tree - topNode := cvg.cv.fetchDataToTmpCache() - - //Merge the subtree with existing yang tree - var errObj yparser.YParserError - if (cvg.db[tableName].root != nil) { - if topNode, errObj = cvg.cv.yp.MergeSubtree(cvg.db[tableName].root, topNode); errObj.ErrCode != yparser.YP_SUCCESS { - cvg.mutex.Unlock() - return CVL_ERROR - } - } - - //Update DB map - db := cvg.db[tableName] - db.root = topNode - cvg.db[tableName] = db - - case "del": - //NOP, already deleted the entry - } - - cvg.mutex.Unlock() - - return CVL_SUCCESS -} - -//Clear cache data for given table -func dbCacheClear(tableName string) CVLRetCode { - cvg.cv.yp.FreeNode(cvg.db[tableName].root) - delete(cvg.db, tableName) - - TRACE_LOG(INFO_ALL, TRACE_CACHE, "Clearing global cache for table %s", tableName) - - return CVL_SUCCESS -} - diff --git a/cvl/cvl_api.go b/cvl/cvl_api.go index 64af1bc76852..de5e84bbc3b9 100644 --- a/cvl/cvl_api.go +++ b/cvl/cvl_api.go @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // // its subsidiaries. // // // // Licensed under the Apache License, Version 2.0 (the "License"); // @@ -22,12 +22,14 @@ package cvl import ( "fmt" "encoding/json" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" toposort "github.com/philopon/go-toposort" - "path/filepath" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" + //lint:ignore ST1001 This is safe to dot import for util package . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "strings" + "github.com/antchfx/xmlquery" + "runtime" "time" "sync" ) @@ -66,7 +68,7 @@ var cvlErrorMap = map[CVLRetCode]string { CVL_SEMANTIC_DEPENDENT_DATA_MISSING : "Dependent Data is missing", CVL_SEMANTIC_MANDATORY_DATA_MISSING : "Mandatory Data is missing", CVL_SEMANTIC_KEY_ALREADY_EXIST : "Key already existing.", - CVL_SEMANTIC_KEY_NOT_EXIST : "Key does not exist.", + CVL_SEMANTIC_KEY_NOT_EXIST : "Key is missing.", CVL_SEMANTIC_KEY_DUPLICATE : "Duplicate key received", CVL_SEMANTIC_KEY_INVALID : "Invalid Key Received", CVL_INTERNAL_UNKNOWN : "Internal Unknown Error", @@ -75,7 +77,7 @@ var cvlErrorMap = map[CVLRetCode]string { CVL_FAILURE : "Generic Failure", } -//Error code +// CVLRetCode CVL Error codes type CVLRetCode int const ( CVL_SUCCESS CVLRetCode = iota @@ -101,10 +103,10 @@ const ( CVL_SEMANTIC_KEY_ALREADY_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_ALREADY_EXIST) /* Key already existing. */ CVL_SEMANTIC_KEY_NOT_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_NOT_EXIST) /* Key is missing. */ CVL_SEMANTIC_KEY_DUPLICATE = CVLRetCode(yparser.YP_SEMANTIC_KEY_DUPLICATE) /* Duplicate key. */ - CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID) + CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID) ) -//Strcture for key and data in API +// CVLEditConfigData Strcture for key and data in API type CVLEditConfigData struct { VType CVLValidateType //Validation type VOp CVLOperation //Operation type @@ -134,7 +136,7 @@ var cfgValidationStats ValidationTimeStats var statsMutex *sync.Mutex func Initialize() CVLRetCode { - if (cvlInitialized == true) { + if cvlInitialized { //CVL has already been initialized return CVL_SUCCESS } @@ -146,43 +148,38 @@ func Initialize() CVLRetCode { CVL_LOG(FATAL, "Unable to connect to Redis Config DB Server") return CVL_ERROR } - //Scan schema directory to get all schema files - modelFiles, err := filepath.Glob(CVL_SCHEMA + "/*.yin") - if err != nil { - CVL_LOG(FATAL ,"Could not read schema %v", err) - } + + //Load lua script into redis + luaScripts = make(map[string]*redis.Script) + loadLuaScript(luaScripts) yparser.Initialize() - modelInfo.modelNs = make(map[string]modelNamespace) //redis table to model name + modelInfo.modelNs = make(map[string]*modelNamespace) //redis table to model name modelInfo.tableInfo = make(map[string]*modelTableInfo) //model namespace modelInfo.allKeyDelims = make(map[string]bool) //all key delimiter modelInfo.redisTableToYangList = make(map[string][]string) //Redis table to Yang list map dbNameToDbNum = map[string]uint8{"APPL_DB": APPL_DB, "CONFIG_DB": CONFIG_DB} - /* schema */ - for _, modelFilePath := range modelFiles { - _, modelFile := filepath.Split(modelFilePath) + // Load all YIN schema files + if retCode := loadSchemaFiles(); retCode != CVL_SUCCESS { + return retCode + } - TRACE_LOG(INFO_DEBUG, TRACE_LIBYANG, "Parsing schema file %s ...\n", modelFilePath) - var module *yparser.YParserModule - if module, _ = yparser.ParseSchemaFile(modelFilePath); module == nil { + //Compile leafref path + compileLeafRefPath() - CVL_LOG(FATAL,fmt.Sprintf("Unable to parse schema file %s", modelFile)) - return CVL_ERROR - } + //Compile all must exps + compileMustExps() - storeModelInfo(modelFile, module) - } + //Compile all when exps + compileWhenExps() //Add all table names to be fetched to validate 'must' expression addTableNamesForMustExp() - //Initialize redis Client - - - //Load lua script into redis - loadLuaScript() + //Build reverse leafref info i.e. which table/field uses one table through leafref + buildRefTableInfo() cvlInitialized = true @@ -197,7 +194,10 @@ func ValidationSessOpen() (*CVL, CVLRetCode) { cvl := &CVL{} cvl.tmpDbCache = make(map[string]interface{}) cvl.requestCache = make(map[string]map[string][]*requestCacheType) + cvl.maxTableElem = make(map[string]int) cvl.yp = &yparser.YParser{} + cvl.yv = &YValidator{} + cvl.yv.root = &xmlquery.Node{Type: xmlquery.DocumentNode} if (cvl == nil || cvl.yp == nil) { return nil, CVL_FAILURE @@ -219,11 +219,12 @@ func (c *CVL) ValidateStartupConfig(jsonData string) CVLRetCode { return CVL_NOT_IMPLEMENTED } +//ValidateIncrementalConfig Steps: +// Check config data syntax +// Fetch the depedent data +// Merge config and dependent data +// Finally validate func (c *CVL) ValidateIncrementalConfig(jsonData string) CVLRetCode { - //Check config data syntax - //Fetch the depedent data - //Merge config and dependent data - //Finally validate c.clearTmpDbCache() var v interface{} @@ -240,9 +241,14 @@ func (c *CVL) ValidateIncrementalConfig(jsonData string) CVLRetCode { } + errObj := c.yp.ValidateSyntax(root, nil) + if yparser.YP_SUCCESS != errObj.ErrCode { + return CVL_FAILURE + } + //Add and fetch entries if already exists in Redis for tableName, data := range dataMap { - for key, _ := range data.(map[string]interface{}) { + for key := range data.(map[string]interface{}) { c.addTableEntryToCache(tableName, key) } } @@ -250,9 +256,8 @@ func (c *CVL) ValidateIncrementalConfig(jsonData string) CVLRetCode { existingData := c.fetchDataToTmpCache() //Merge existing data for update syntax or checking duplicate entries - var errObj yparser.YParserError if (existingData != nil) { - if root, errObj = c.yp.MergeSubtree(root, existingData); + if _, errObj = c.yp.MergeSubtree(root, existingData); errObj.ErrCode != yparser.YP_SUCCESS { return CVL_ERROR } @@ -261,20 +266,15 @@ func (c *CVL) ValidateIncrementalConfig(jsonData string) CVLRetCode { //Clear cache c.clearTmpDbCache() - //Add tables for 'must' expression - for tableName, _ := range dataMap { - c.addTableDataForMustExp(OP_NONE, tableName) - } - //Perform validation - if _, cvlRetCode := c.validateSemantics(root, nil); cvlRetCode != CVL_SUCCESS { - return cvlRetCode + if cvlErrObj := c.validateCfgSemantics(c.yv.root); cvlErrObj.ErrCode != CVL_SUCCESS { + return cvlErrObj.ErrCode } return CVL_SUCCESS } -//Validate data for operation +//ValidateConfig Validate data for operation func (c *CVL) ValidateConfig(jsonData string) CVLRetCode { c.clearTmpDbCache() var v interface{} @@ -283,7 +283,7 @@ func (c *CVL) ValidateConfig(jsonData string) CVLRetCode { if err := json.Unmarshal(b, &v); err == nil { var value map[string]interface{} = v.(map[string]interface{}) root, _ := c.translateToYang(&value) - //if ret == CVL_SUCCESS && root != nil { + if root == nil { return CVL_FAILURE @@ -298,98 +298,142 @@ func (c *CVL) ValidateConfig(jsonData string) CVLRetCode { return CVL_SUCCESS } -//Validate config data based on edit operation - no marshalling in between -func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (CVLErrorInfo, CVLRetCode) { +//ValidateEditConfig Validate config data based on edit operation +func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorInfo, ret CVLRetCode) { + + ts := time.Now() + + defer func() { + if (cvlErr.ErrCode != CVL_SUCCESS) { + CVL_LOG(WARNING, "ValidateEditConfig() failed: %+v", cvlErr) + } + //Update validation time stats + updateValidationTimeStats(time.Since(ts)) + }() + var cvlErrObj CVLErrorInfo - if (SkipValidation() == true) { + caller := "" + if (IsTraceSet()) { + pc := make([]uintptr, 10) + runtime.Callers(2, pc) + f := runtime.FuncForPC(pc[0]) + caller = f.Name() + } + CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData) + + if SkipValidation() { + CVL_LOG(INFO_TRACE, "Skipping CVL validation.") return cvlErrObj, CVL_SUCCESS } + //Type cast to custom validation cfg data + c.clearTmpDbCache() + //c.yv.root.FirstChild = nil + //c.yv.root.LastChild = nil - //Step 1: Get requested dat first + + //Step 1: Get requested data first //add all dependent data to be fetched from Redis requestedData := make(map[string]interface{}) - for _, cfgDataItem := range cfgData { - if (VALIDATE_ALL != cfgDataItem.VType) { + cfgDataLen := len(cfgData) + for i := 0; i < cfgDataLen; i++ { + if (VALIDATE_ALL != cfgData[i].VType) { continue } //Add config data item to be validated - tbl,key := c.addCfgDataItem(&requestedData, cfgDataItem) + tbl,key := c.addCfgDataItem(&requestedData, cfgData[i]) //Add to request cache reqTbl, exists := c.requestCache[tbl] - if (exists == false) { + if !exists { //Create new table key data reqTbl = make(map[string][]*requestCacheType) } - cfgDataItemArr, _ := reqTbl[key] - cfgDataItemArr = append(cfgDataItemArr, &requestCacheType{cfgDataItem, nil}) + cfgDataItemArr := reqTbl[key] + cfgDataItemArr = append(cfgDataItemArr, &requestCacheType{cfgData[i], nil}) reqTbl[key] = cfgDataItemArr c.requestCache[tbl] = reqTbl //Invalid table name or invalid key separator if key == "" { cvlErrObj.ErrCode = CVL_SYNTAX_ERROR + cvlErrObj.Msg = "Invalid table or key for " + cfgData[i].Key cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] return cvlErrObj, CVL_SYNTAX_ERROR } - switch cfgDataItem.VOp { + switch cfgData[i].VOp { case OP_CREATE: - if (c.addTableEntryForMustExp(&cfgDataItem, tbl) != CVL_SUCCESS) { - c.addTableDataForMustExp(cfgDataItem.VOp, tbl) + //Check max-element constraint + if ret := c.checkMaxElemConstraint(OP_CREATE, tbl); ret != CVL_SUCCESS { + cvlErrObj.ErrCode = CVL_SYNTAX_ERROR + cvlErrObj.ErrAppTag = "too-many-elements" + cvlErrObj.Msg = "Max elements limit reached" + cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] + cvlErrObj.ConstraintErrMsg = fmt.Sprintf("Max elements limit %v reached", + modelInfo.tableInfo[tbl].redisTableSize) + + return cvlErrObj, CVL_SYNTAX_ERROR } case OP_UPDATE: - //Get the existing data from Redis to cache, so that final validation can be done after merging this dependent data - if (c.addTableEntryForMustExp(&cfgDataItem, tbl) != CVL_SUCCESS) { - c.addTableDataForMustExp(cfgDataItem.VOp, tbl) - } + //Get the existing data from Redis to cache, so that final + //validation can be done after merging this dependent data c.addTableEntryToCache(tbl, key) case OP_DELETE: - if (len(cfgDataItem.Data) > 0) { - //Delete a single field - if (len(cfgDataItem.Data) != 1) { - CVL_LOG(ERROR, "Only single field is allowed for field deletion") - } else { - for field, _ := range cfgDataItem.Data { - if (c.checkDeleteConstraint(cfgData, tbl, key, field) != CVL_SUCCESS) { - cvlErrObj.ErrCode = CVL_SEMANTIC_ERROR - cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] - return cvlErrObj, CVL_SEMANTIC_ERROR - } - break //only one field there + if (len(cfgData[i].Data) > 0) { + //Check constraints for deleting field(s) + for field := range cfgData[i].Data { + if (c.checkDeleteConstraint(cfgData, tbl, key, field) != CVL_SUCCESS) { + cvlErrObj.ErrCode = CVL_SEMANTIC_ERROR + cvlErrObj.Msg = "Validation failed for Delete operation, given instance is in use" + cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] + cvlErrObj.ErrAppTag = "instance-in-use" + cvlErrObj.ConstraintErrMsg = cvlErrObj.Msg + return cvlErrObj, CVL_SEMANTIC_ERROR + } + + //Check mandatory node deletion + if len(field) != 0 && isMandatoryTrueNode(tbl, field) { + cvlErrObj.ErrCode = CVL_SEMANTIC_ERROR + cvlErrObj.Msg = "Mandatory field getting deleted" + cvlErrObj.TableName = tbl + cvlErrObj.Field = field + cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] + cvlErrObj.ErrAppTag = "mandatory-field-delete" + cvlErrObj.ConstraintErrMsg = cvlErrObj.Msg + return cvlErrObj, CVL_SEMANTIC_ERROR } } } else { //Entire entry to be deleted + + //Update max-elements count + c.checkMaxElemConstraint(OP_DELETE, tbl) + + //Now check delete constraints if (c.checkDeleteConstraint(cfgData, tbl, key, "") != CVL_SUCCESS) { cvlErrObj.ErrCode = CVL_SEMANTIC_ERROR + cvlErrObj.Msg = "Validation failed for Delete operation, given instance is in use" cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] - return cvlErrObj, CVL_SEMANTIC_ERROR + cvlErrObj.ErrAppTag = "instance-in-use" + cvlErrObj.ConstraintErrMsg = cvlErrObj.Msg + return cvlErrObj, CVL_SEMANTIC_ERROR } - //TBD : Can we do this ? - //No entry has depedency on this key, - //remove from requestCache, we don't need any more as depedent data - //delete(c.requestCache[tbl], key) - } - - if (c.addTableEntryForMustExp(&cfgDataItem, tbl) != CVL_SUCCESS) { - c.addTableDataForMustExp(cfgDataItem.VOp, tbl) } c.addTableEntryToCache(tbl, key) } } - //Only for tracing if (IsTraceSet()) { + //Only for tracing jsonData := "" jsonDataBytes, err := json.Marshal(requestedData) @@ -398,116 +442,114 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (CVLErrorInfo, CVL } else { cvlErrObj.ErrCode = CVL_SYNTAX_ERROR cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] - return cvlErrObj, CVL_SYNTAX_ERROR + return cvlErrObj, CVL_SYNTAX_ERROR } - TRACE_LOG(INFO_DATA, TRACE_LIBYANG, "Requested JSON Data = [%s]\n", jsonData) + TRACE_LOG(TRACE_LIBYANG, "Requested JSON Data = [%s]\n", jsonData) } //Step 2 : Perform syntax validation only yang, errN := c.translateToYang(&requestedData) if (errN.ErrCode == CVL_SUCCESS) { if cvlErrObj, cvlRetCode := c.validateSyntax(yang); cvlRetCode != CVL_SUCCESS { - return cvlErrObj, cvlRetCode + return cvlErrObj, cvlRetCode } } else { - return errN,errN.ErrCode + return errN,errN.ErrCode } - //Step 3 : Check keys and update dependent data - dependentData := make(map[string]interface{}) - - for _, cfgDataItem := range cfgData { - - if (cfgDataItem.VType == VALIDATE_ALL || cfgDataItem.VType == VALIDATE_SEMANTICS) { - //Step 3.1 : Check keys - switch cfgDataItem.VOp { - case OP_CREATE: - //Check key should not already exist - n, err1 := redisClient.Exists(cfgDataItem.Key).Result() - if (err1 == nil && n > 0) { - //Check if key deleted and CREATE done in same session, - //allow to create the entry - tbl, key := splitRedisKey(cfgDataItem.Key) - deletedInSameSession := false - if tbl != "" && key != "" { - for _, cachedCfgData := range c.requestCache[tbl][key] { - if cachedCfgData.reqData.VOp == OP_DELETE { - deletedInSameSession = true - break - } - } - } + //Step 3 : Check keys and perform semantics validation + for i := 0; i < cfgDataLen; i++ { - if deletedInSameSession == false { - CVL_LOG(ERROR, "\nValidateEditConfig(): Key = %s already exists", cfgDataItem.Key) - cvlErrObj.ErrCode = CVL_SEMANTIC_KEY_ALREADY_EXIST - cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] - return cvlErrObj, CVL_SEMANTIC_KEY_ALREADY_EXIST + if (cfgData[i].VType != VALIDATE_ALL && cfgData[i].VType != VALIDATE_SEMANTICS) { + continue + } + + tbl, key := splitRedisKey(cfgData[i].Key) - } else { - TRACE_LOG(INFO_API, TRACE_CREATE, "\nKey %s is deleted in same session, skipping key existence check for OP_CREATE operation", cfgDataItem.Key) + //Step 3.1 : Check keys + switch cfgData[i].VOp { + case OP_CREATE: + //Check key should not already exist + n, err1 := redisClient.Exists(cfgData[i].Key).Result() + if (err1 == nil && n > 0) { + //Check if key deleted and CREATE done in same session, + //allow to create the entry + deletedInSameSession := false + if tbl != "" && key != "" { + for _, cachedCfgData := range c.requestCache[tbl][key] { + if cachedCfgData.reqData.VOp == OP_DELETE { + deletedInSameSession = true + break + } } } - c.yp.SetOperation("CREATE") - - case OP_UPDATE: - n, err1 := redisClient.Exists(cfgDataItem.Key).Result() - if (err1 != nil || n == 0) { //key must exists - CVL_LOG(ERROR, "\nValidateEditConfig(): Key = %s does not exist", cfgDataItem.Key) + if !deletedInSameSession { + CVL_LOG(WARNING, "\nValidateEditConfig(): Key = %s already exists", cfgData[i].Key) cvlErrObj.ErrCode = CVL_SEMANTIC_KEY_ALREADY_EXIST cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] - return cvlErrObj, CVL_SEMANTIC_KEY_NOT_EXIST + return cvlErrObj, CVL_SEMANTIC_KEY_ALREADY_EXIST + + } else { + TRACE_LOG(TRACE_CREATE, "\nKey %s is deleted in same session, " + + "skipping key existence check for OP_CREATE operation", cfgData[i].Key) } + } - c.yp.SetOperation("UPDATE") + c.yp.SetOperation("CREATE") - case OP_DELETE: - n, err1 := redisClient.Exists(cfgDataItem.Key).Result() - if (err1 != nil || n == 0) { //key must exists - CVL_LOG(ERROR, "\nValidateEditConfig(): Key = %s does not exist", cfgDataItem.Key) - cvlErrObj.ErrCode = CVL_SEMANTIC_KEY_NOT_EXIST - cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] - return cvlErrObj, CVL_SEMANTIC_KEY_NOT_EXIST - } + case OP_UPDATE: + n, err1 := redisClient.Exists(cfgData[i].Key).Result() + if (err1 != nil || n == 0) { //key must exists + CVL_LOG(WARNING, "\nValidateEditConfig(): Key = %s does not exist", cfgData[i].Key) + cvlErrObj.ErrCode = CVL_SEMANTIC_KEY_NOT_EXIST + cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] + return cvlErrObj, CVL_SEMANTIC_KEY_NOT_EXIST + } - c.yp.SetOperation("DELETE") - //store deleted keys + // Skip validation if UPDATE is received with only NULL field + if _, exists := cfgData[i].Data["NULL"]; exists && len(cfgData[i].Data) == 1 { + continue; } - }/* else if (cfgDataItem.VType == VALIDATE_NONE) { - //Step 3.2 : Get dependent data - - switch cfgDataItem.VOp { - case OP_CREATE: - //NOP - case OP_UPDATE: - //NOP - case OP_DELETE: - tbl,key := c.addCfgDataItem(&dependentData, cfgDataItem) - //update cache by removing deleted entry - c.updateDeleteDataToCache(tbl, key) - //store deleted keys + c.yp.SetOperation("UPDATE") + + case OP_DELETE: + n, err1 := redisClient.Exists(cfgData[i].Key).Result() + if (err1 != nil || n == 0) { //key must exists + CVL_LOG(WARNING, "\nValidateEditConfig(): Key = %s does not exist", cfgData[i].Key) + cvlErrObj.ErrCode = CVL_SEMANTIC_KEY_NOT_EXIST + cvlErrObj.CVLErrDetails = cvlErrorMap[cvlErrObj.ErrCode] + return cvlErrObj, CVL_SEMANTIC_KEY_NOT_EXIST } - }*/ - } - var depYang *yparser.YParserNode = nil - if (len(dependentData) > 0) { - depYang, errN = c.translateToYang(&dependentData) - } - //Step 4 : Perform validation - if cvlErrObj, cvlRetCode1 := c.validateSemantics(yang, depYang); cvlRetCode1 != CVL_SUCCESS { - return cvlErrObj, cvlRetCode1 - } + c.yp.SetOperation("DELETE") + } - //Cache validated data - /* - if errObj := c.yp.CacheSubtree(false, yang); errObj.ErrCode != yparser.YP_SUCCESS { - TRACE_LOG(INFO_API, TRACE_CACHE, "Could not cache validated data") + yangListName := getRedisTblToYangList(tbl, key) + + //Get the YANG validator node + var node *xmlquery.Node = nil + if (c.requestCache[tbl][key][0].yangData != nil) { //get the node for CREATE/UPDATE or DELETE operation + node = c.requestCache[tbl][key][0].yangData + } else { + //Find the node from YANG tree + node = c.moveToYangList(yangListName, key) + } + + if (node == nil) { + CVL_LOG(WARNING, "Could not find data for semantic validation, " + + "table %s , key %s", tbl, key) + continue + } + + //Step 3.3 : Perform semantic validation + if cvlErrObj = c.validateSemantics(node, yangListName, key, &cfgData[i]); + cvlErrObj.ErrCode != CVL_SUCCESS { + return cvlErrObj,cvlErrObj.ErrCode + } } - */ c.yp.DestroyCache() return cvlErrObj, CVL_SUCCESS @@ -965,7 +1007,7 @@ func (c *CVL) GetDepDataForDelete(redisKey string) ([]CVLDepDataForDelete) { } } - TRACE_LOG(INFO_API, INFO_TRACE, "GetDepDataForDelete() : input key %s, " + + TRACE_LOG(INFO_TRACE, "GetDepDataForDelete() : input key %s, " + "entries to be deleted : %v", redisKey, depEntries) //For each key, find dependent data for delete recursively diff --git a/cvl/cvl_cache.go b/cvl/cvl_cache.go index ab2792df679c..d9619a0c05a4 100644 --- a/cvl/cvl_cache.go +++ b/cvl/cvl_cache.go @@ -20,15 +20,30 @@ package cvl import ( "encoding/json" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" //lint:ignore ST1001 This is safe to dot import for util package . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" "time" "runtime" "github.com/antchfx/jsonquery" + "github.com/antchfx/xmlquery" ) +func (c *CVL) addTableEntryToCache(tableName string, redisKey string) { + if (tableName == "" || redisKey == "") { + return + } + + if (c.tmpDbCache[tableName] == nil) { + c.tmpDbCache[tableName] = map[string]interface{}{redisKey: nil} + } else { + tblMap := c.tmpDbCache[tableName] + tblMap.(map[string]interface{})[redisKey] =nil + c.tmpDbCache[tableName] = tblMap + } +} + // Fetch dependent data from validated data cache, // Returns the data and flag to indicate that if requested data // is found in update request, the data should be merged with Redis data @@ -37,20 +52,18 @@ func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (d map[str pd := &d pm := &m - TRACE_LOG(INFO_API, TRACE_CACHE, + TRACE_LOG(TRACE_CACHE, "Returning data from request cache, data = %v, merge needed = %v", *pd, *pm) }() cfgDataArr := c.requestCache[tableName][key] - if (cfgDataArr != nil) { - for _, cfgReqData := range cfgDataArr { - //Delete request doesn't have depedent data - if (cfgReqData.reqData.VOp == OP_CREATE) { - return cfgReqData.reqData.Data, false - } else if (cfgReqData.reqData.VOp == OP_UPDATE) { - return cfgReqData.reqData.Data, true - } + for _, cfgReqData := range cfgDataArr { + //Delete request doesn't have depedent data + if (cfgReqData.reqData.VOp == OP_CREATE) { + return cfgReqData.reqData.Data, false + } else if (cfgReqData.reqData.VOp == OP_UPDATE) { + return cfgReqData.reqData.Data, true } } @@ -60,7 +73,7 @@ func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (d map[str //Fetch given table entries using pipeline func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]interface{}) int { - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now()) + TRACE_LOG(TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now()) totalCount := len(dbKeys) if (totalCount == 0) { @@ -102,25 +115,27 @@ func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]inter redisKey := tableName + modelInfo.tableInfo[tableName].redisKeyDelim + dbKey //Check in validated cache first and add as dependent data if entry, mergeNeeded := c.fetchDataFromRequestCache(tableName, dbKey); (entry != nil) { - c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry - entryFetched = entryFetched + 1 - //Entry found in validated cache, so skip fetching from Redis - //if merging is not required with Redis DB - if (mergeNeeded == false) { - continue - } + entryFetched = entryFetched + 1 + //Entry found in validated cache, so skip fetching from Redis + //if merging is not required with Redis DB + if !mergeNeeded { + fieldMap := c.checkFieldMap(&entry) + c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = fieldMap + continue + } + c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry } //Otherwise fetch it from Redis mCmd[dbKey] = pipe.HGetAll(redisKey) //write into pipeline if mCmd[dbKey] == nil { - CVL_LOG(ERROR, "Failed pipe.HGetAll('%s')", redisKey) + CVL_LOG(WARNING, "Failed pipe.HGetAll('%s')", redisKey) } } _, err := pipe.Exec() if err != nil { - CVL_LOG(ERROR, "Failed to fetch details for table %s", tableName) + CVL_LOG(WARNING, "Failed to fetch details for table %s", tableName) return 0 } pipe.Close() @@ -130,6 +145,11 @@ func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]inter for key, val := range mCmd { res, err := val.Result() + + if (mapTable == nil) { + break + } + if (err != nil || len(res) == 0) { //no data found, don't keep blank entry delete(mapTable.(map[string]interface{}), key) @@ -155,14 +175,14 @@ func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]inter runtime.Gosched() } - TRACE_LOG(INFO_API, TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now()) + TRACE_LOG(TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now()) return entryFetched } //populate redis data to cache func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now()) + TRACE_LOG(TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now()) entryToFetch := 0 var root *yparser.YParserNode = nil @@ -191,7 +211,7 @@ func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { if Tracing { jsonDataBytes, _ := json.Marshal(c.tmpDbCache) jsonData := string(jsonDataBytes) - TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonData) + TRACE_LOG(TRACE_CACHE, "Top Node=%v\n", jsonData) } data, err := jsonquery.ParseJsonMap(&c.tmpDbCache) @@ -202,7 +222,7 @@ func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { //Build yang tree for each table and cache it for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { - TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonNode.Data) + TRACE_LOG(TRACE_CACHE, "Top Node=%v\n", jsonNode.Data) //Visit each top level list in a loop for creating table data topNode, _ := c.generateTableData(true, jsonNode) if (root == nil) { @@ -212,15 +232,46 @@ func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode { return nil } } + + //Generate YANG data for Yang Validator + topYangNode, cvlYErrObj := c.generateYangListData(jsonNode, true) + if topYangNode == nil { + cvlYErrObj.ErrCode = CVL_SYNTAX_ERROR + CVL_LOG(WARNING, "Unable to translate cache data to YANG format") + return nil + } + + //Create a full document and merge with main YANG data + doc := &xmlquery.Node{Type: xmlquery.DocumentNode} + doc.FirstChild = topYangNode + doc.LastChild = topYangNode + topYangNode.Parent = doc + + if (IsTraceLevelSet(TRACE_CACHE)) { + TRACE_LOG(TRACE_CACHE, "Before cache merge = %s, source = %s", + c.yv.root.OutputXML(false), + doc.OutputXML(false)) + } + + if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS { + CVL_LOG(WARNING, "Unable to merge translated YANG data while " + + "translating from cache data to YANG format") + cvlYErrObj.ErrCode = CVL_SYNTAX_ERROR + return nil + } + if (IsTraceLevelSet(TRACE_CACHE)) { + TRACE_LOG(TRACE_CACHE, "After cache merge = %s", + c.yv.root.OutputXML(false)) + } } } // until all dependent data is fetched if root != nil && Tracing { dumpStr := c.yp.NodeDump(root) - TRACE_LOG(INFO_API, TRACE_CACHE, "Dependent Data = %v\n", dumpStr) + TRACE_LOG(TRACE_CACHE, "Dependent Data = %v\n", dumpStr) } - TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now()) + TRACE_LOG(TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now()) return root } @@ -230,5 +281,3 @@ func (c *CVL) clearTmpDbCache() { delete(c.tmpDbCache, key) } } - - diff --git a/cvl/cvl_luascript.go b/cvl/cvl_luascript.go index fd4c863ad1e7..0950c43bf79b 100644 --- a/cvl/cvl_luascript.go +++ b/cvl/cvl_luascript.go @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // // its subsidiaries. // // // // Licensed under the Apache License, Version 2.0 (the "License"); // @@ -19,22 +19,35 @@ package cvl import ( - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" + "github.com/Azure/sonic-mgmt-common/cvl/internal/util" ) //Redis server side script -func loadLuaScript() { - luaScripts = make(map[string]*redis.Script) +func loadLuaScript(luaScripts map[string]*redis.Script) { // Find entry which has given fieldName and value luaScripts["find_key"] = redis.NewScript(` local tableName=ARGV[1] local sep=ARGV[2] - local fieldName=ARGV[3] - local fieldValue=ARGV[4] + local keySetNames = {} + ARGV[3]:gsub("([^|]+)",function(c) table.insert(keySetNames, c) end) + local fieldName=ARGV[4] + local fieldValue=ARGV[5] + + local entries = {} -- Check if field value is part of key - local entries=redis.call('KEYS', tableName..sep.."*"..fieldValue.."*") + if (#keySetNames == 1) then + -- field is only key + entries=redis.call('KEYS', tableName..sep..fieldValue) + elseif (keySetNames[#keySetNames] == fieldName) then + -- field is the last key + entries=redis.call('KEYS', tableName..sep.."*"..sep..fieldValue) + else + -- field is not the last key + entries=redis.call('KEYS', tableName.."*"..sep..fieldValue..sep.."*") + end if (entries[1] ~= nil) then @@ -48,7 +61,8 @@ func loadLuaScript() { while(entries[idx] ~= nil) do local val = redis.call("HGET", entries[idx], fieldName) - if (val == fieldValue) + local valArrFld = redis.call("HGET", entries[idx], fieldName.."@") + if ((val == fieldValue) or (valArrFld == fieldValue)) then -- Return the key return entries[idx] @@ -62,4 +76,137 @@ func loadLuaScript() { return "" `) + //Find current number of entries in a table + luaScripts["count_entries"] = redis.NewScript(` + --ARGV[1] => Key patterns + --ARGV[2] => Key names separated by '|' + --ARGV[3] => predicate patterns + --ARGV[4] => Field + + if (#ARGV == 1) then + --count of all table entries + return #redis.call('KEYS', ARGV[1]) + end + + -- count with filter + local keys = redis.call('KEYS', ARGV[1]) + if #keys == 0 then return 0 end + + local cnt = 0 + + local sepStart = string.find(keys[1], "|") + if sepStart == nil then return ; end + + -- Function to load lua predicate code + local function loadPredicateScript(str) + if (str == nil or str == "") then return nil; end + + local f, err = loadstring("return function (k,h) " .. str .. " end") + if f then return f(); else return nil;end + end + + local keySetNames = {} + ARGV[2]:gsub("([^|]+)",function(c) table.insert(keySetNames, c) end) + + local predicate = loadPredicateScript(ARGV[3]) + + local field = "" + if (ARGV[4] ~= nil) then field = ARGV[4]; end + + for _, key in ipairs(keys) do + local hash = redis.call('HGETALL', key) + local row = {}; local keySet = {}; local keyVal = {} + local keyOnly = string.sub(key, sepStart+1) + + for index = 1, #hash, 2 do + row[hash[index]] = hash[index + 1] + end + + local incFlag = false + if (predicate == nil) then + incFlag = true + else + --Split key values + keyOnly:gsub("([^|]+)", function(c) table.insert(keyVal, c) end) + + for idx = 1, #keySetNames, 1 do + keySet[keySetNames[idx]] = keyVal[idx] + end + + if (predicate(keySet, row) == true) then + incFlag = true + end + end + + if (incFlag == true) then + if (field ~= "") then + if (row[field] ~= nil) then + cnt = cnt + 1 + elseif (row[field.."@"] ~= nil) then + row[field.."@"]:gsub("([^,]+)", function(c) cnt = cnt + 1 end) + elseif (string.match(ARGV[2], field.."[|]?") ~= nil) then + cnt = cnt + 1 + end + else + cnt = cnt + 1 + end + end + end + + return cnt + `) + + // Find entry which has given fieldName and value + luaScripts["filter_keys"] = redis.NewScript(` + --ARGV[1] => Key patterns + --ARGV[2] => Key names separated by '|' + --ARGV[3] => predicate patterns + local filterKeys = "" + + local keys = redis.call('KEYS', ARGV[1]) + if #keys == 0 then return end + + local sepStart = string.find(keys[1], "|") + if sepStart == nil then return ; end + + -- Function to load lua predicate code + local function loadPredicateScript(str) + if (str == nil or str == "") then return nil; end + + local f, err = loadstring("return function (k,h) " .. str .. " end") + if f then return f(); else return nil;end + end + + local keySetNames = {} + ARGV[2]:gsub("([^|]+)",function(c) table.insert(keySetNames, c) end) + + local predicate = loadPredicateScript(ARGV[3]) + + for _, key in ipairs(keys) do + local hash = redis.call('HGETALL', key) + local row = {}; local keySet = {}; local keyVal = {} + local keyOnly = string.sub(key, sepStart+1) + + for index = 1, #hash, 2 do + row[hash[index]] = hash[index + 1] + end + + --Split key values + keyOnly:gsub("([^|]+)", function(c) table.insert(keyVal, c) end) + + for idx = 1, #keySetNames, 1 do + keySet[keySetNames[idx]] = keyVal[idx] + end + + if (predicate == nil) or (predicate(keySet, row) == true) then + filterKeys = filterKeys .. key .. "," + end + + end + + return string.sub(filterKeys, 1, #filterKeys - 1) + `) + + //Get filtered entries as per given key filters and predicate + luaScripts["filter_entries"] = util.FILTER_ENTRIES_LUASCRIPT } diff --git a/cvl/cvl_semantics.go b/cvl/cvl_semantics.go index 0f9ff2b82483..2a7c4684d78b 100644 --- a/cvl/cvl_semantics.go +++ b/cvl/cvl_semantics.go @@ -38,6 +38,7 @@ import ( type YValidator struct { root *xmlquery.Node //Top evel root for data current *xmlquery.Node //Current position + //operation string //Edit operation } //Generate leaf/leaf-list YANG data @@ -415,7 +416,7 @@ func (c *CVL) detachNode(node *xmlquery.Node) CVLRetCode { //destination func (c *CVL) deleteDestLeafList(dest *xmlquery.Node) { - TRACE_LOG(INFO_API, TRACE_CACHE, "Updating leaf-list by " + + TRACE_LOG(TRACE_CACHE, "Updating leaf-list by " + "removing and then adding leaf-list") //find start and end of dest leaf list @@ -490,7 +491,7 @@ func (c *CVL) mergeYangData(dest, src *xmlquery.Node) CVLRetCode { return CVL_FAILURE } - TRACE_LOG(INFO_API, (TRACE_SYNTAX | TRACE_SEMANTIC), + TRACE_LOG((TRACE_SYNTAX | TRACE_SEMANTIC), "Merging YANG data %s %s", dest.Data, src.Data) if (dest.Type == xmlquery.TextNode) && (src.Type == xmlquery.TextNode) { @@ -625,7 +626,7 @@ func (c *CVL) moveToYangList(tableName string, redisKey string) *xmlquery.Node { } if (nodeTbl == nil) { - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "YANG data for table %s, key %s is not present in YANG tree", + TRACE_LOG(TRACE_SEMANTIC, "YANG data for table %s, key %s is not present in YANG tree", tableName, redisKey) return nil } @@ -720,7 +721,7 @@ func (c *CVL) addDepYangData(redisKeys []string, redisKeyFilter, singleLeaf := "" //leaf data for single leaf - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "addDepYangData() with redisKeyFilter=%s, " + + TRACE_LOG(TRACE_SEMANTIC, "addDepYangData() with redisKeyFilter=%s, " + "predicate=%s, fields=%s, returned cfgData = %s, err=%v", redisKeyFilter, predicate, fields, cfgData, err) @@ -847,7 +848,7 @@ func (c *CVL) addYangDataForMustExp(op CVLOperation, tableName string, oneEntry } //Load only one entry if oneEntry { - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "addYangDataForMustExp(): Adding one entry table %s, key %s", + TRACE_LOG(TRACE_SEMANTIC, "addYangDataForMustExp(): Adding one entry table %s, key %s", redisTblName, tableKey) break } @@ -992,7 +993,7 @@ func (c *CVL) validateMustExp(node *xmlquery.Node, redisKeyFilter, keyNames, pred, fields, count string) string { c := ctxt.(*CVL) - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateMustExp(): calling addDepYangData()") + TRACE_LOG(TRACE_SEMANTIC, "validateMustExp(): calling addDepYangData()") return c.addDepYangData(redisKeys, redisKeyFilter, keyNames, pred, fields, "") }) @@ -1013,7 +1014,7 @@ func (c *CVL) validateMustExp(node *xmlquery.Node, count = float64(redisEntries.(int64)) } - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateMustExp(): depDataCntClbk() with redisKeyFilter=%s, " + + TRACE_LOG(TRACE_SEMANTIC, "validateMustExp(): depDataCntClbk() with redisKeyFilter=%s, " + "keyNames= %s, predicate=%s, fields=%s, returned = %v", redisKeyFilter, keyNames, pred, field, count) @@ -1094,7 +1095,7 @@ func (c *CVL) validateWhenExp(node *xmlquery.Node, defer func() { ret := &r - CVL_LOG(INFO_API, "validateWhenExp(): table name = %s, " + + CVL_LOG(INFO_API, "validateWhenExp(): table name = %s, " + "return value = %v", tableName, *ret) }() @@ -1107,7 +1108,7 @@ func (c *CVL) validateWhenExp(node *xmlquery.Node, xpath.SetDepDataClbk(c, func(ctxt interface{}, redisKeys []string, redisKeyFilter, keyNames, pred, fields, count string) string { c := ctxt.(*CVL) - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateWhenExp(): calling addDepYangData()") + TRACE_LOG(TRACE_SEMANTIC, "validateWhenExp(): calling addDepYangData()") return c.addDepYangData(redisKeys, redisKeyFilter, keyNames, pred, fields, "") }) @@ -1210,7 +1211,7 @@ func (c *CVL) validateLeafRef(node *xmlquery.Node, xpath.SetDepDataClbk(c, func(ctxt interface{}, redisKeys []string, redisKeyFilter, keyNames, pred, fields, count string) string { c := ctxt.(*CVL) - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateLeafRef(): calling addDepYangData()") + TRACE_LOG(TRACE_SEMANTIC, "validateLeafRef(): calling addDepYangData()") return c.addDepYangData(redisKeys, redisKeyFilter, keyNames, pred, fields, "") }) @@ -1289,7 +1290,7 @@ func (c *CVL) validateLeafRef(node *xmlquery.Node, if (err != nil) || (len(tableKeys) == 0) { //There must be at least one entry in the ref table - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "Leafref dependent data " + + TRACE_LOG(TRACE_SEMANTIC, "Leafref dependent data " + "table %s, key %s not found in Redis", refRedisTableName, ctxtVal) @@ -1353,7 +1354,7 @@ func (c *CVL) validateLeafRef(node *xmlquery.Node, ConstraintErrMsg: "No instance found for '" + ctxtVal + "'", } } else if !leafRefSuccess { - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateLeafRef(): " + + TRACE_LOG(TRACE_SEMANTIC, "validateLeafRef(): " + "Leafref dependent data not found but leaf has " + "other data type in union, returning success.") } @@ -1363,90 +1364,256 @@ func (c *CVL) validateLeafRef(node *xmlquery.Node, return CVLErrorInfo{ErrCode:CVL_SUCCESS} } +//This function returns true if any entry +//in request cache is using the given entry +//getting deleted. The given entry can be found +//either in key or in hash-field. +//Example : If T1|K1 is getting deleted, +//check if T2*|K1 or T2|*:{H1: K1} +//was using T1|K1 and getting deleted +//in same session also. +func (c *CVL) checkDeleteInRequestCache(cfgData []CVLEditConfigData, + leafRef *tblFieldPair, depDataKey, keyVal string) bool { + + for _, cfgDataItem := range cfgData { + // All cfgDataItems which have VType as VALIDATE_NONE should be + // checked in cache + if (cfgDataItem.VType != VALIDATE_NONE) { + continue + } + + //If any entry is using the given entry + //getting deleted, break immediately + + //Find in request key, case - T2*|K1 + if cfgDataItem.Key == depDataKey && + (cfgDataItem.VOp != OP_DELETE || (cfgDataItem.VOp == OP_DELETE && len(cfgDataItem.Data) == 0)) { + return true + } + + //Find in request hash-field, case - T2*|K2:{H1: K1} + val, exists := cfgDataItem.Data[leafRef.field] + if !exists { + // Leaf-lists field names are suffixed by "@". + val, exists = cfgDataItem.Data[leafRef.field+"@"] + } + // For delete cases, val sent is empty. + if exists && ((val == keyVal) || (val == "")) { + return true + } + } + + return false +} + +// checkDepDataCompatible This function evaluates relationship between two table +// keys based on leafref +func (c *CVL) checkDepDataCompatible(tblName, key, reftblName, refTblKey, leafRefField string, depEntry map[string]string) bool { + CVL_LOG(INFO_DEBUG, "checkDepDataCompatible--> TargetTbl: %s[%s] referred by refTbl: %s[%s] through field: %s", tblName, key, reftblName, refTblKey, leafRefField) + + // Key compatibility to be checked only if no. of keys are more than 1 + if len(modelInfo.tableInfo[tblName].keys) <= 1 { + return true + } + + // Fetch key value pair for both current table and ref table + tblKeysKVP := getRedisToYangKeys(tblName, key) + refTblKeysKVP := getRedisToYangKeys(reftblName, refTblKey) + + // Determine the leafref + leafRef := getLeafRefInfo(reftblName, leafRefField, tblName) + + // If depEntry map is Empty, it means leafRefField is one of key of RefTable. + // So RefTblKey should equal to targetTbl key plus additional 1 or more key + // For ex. BGP_NEIGHBOR|Vrf1|2114::2 referred by BGP_NEIGHBOR_AF|Vrf1|2114::2|ipv4_unicast + if len(depEntry) == 0 { + // TODO Need to revisit. For now assumed that orderof keys are same. + return strings.Contains(refTblKey, key) + } else { + // leafRefField is NOT key but a normal leaf node + if _, exists := depEntry[leafRefField]; exists { + // Compare keys of table and refTable + //LeafRef: /sonic-bgp-peergroup:sonic-bgp-peergroup/sonic-bgp-peergroup:BGP_PEER_GROUP/sonic-bgp-peergroup:BGP_PEER_GROUP_LIST[sonic-bgp-peergroup:vrf_name=current()/../vrf_name]/sonic-bgp-peergroup:peer_group_name + if leafRef != nil && leafRef.exprTree != nil { + leafrefExpr := leafRef.exprTree.String() + if len(leafrefExpr) > 0 { + // TODO Assumed that predicate will have one expression without any 'and'/'or' operators + // predicate looks like [sonic-bgp-peergroup:vrf_name=current()/../vrf_name] + predicate := leafrefExpr[strings.Index(leafrefExpr, "[")+1 : strings.Index(leafrefExpr, "]")] + if strings.Contains(predicate, "=") { + // target tbl key is left of '=' + leafrefTargetTblkey := predicate[:strings.Index(predicate, "=")] + if strings.Contains(leafrefTargetTblkey, ":") { + leafrefTargetTblkey = leafrefTargetTblkey[strings.Index(leafrefTargetTblkey, ":")+1 :] + } + // current tbl key is right of '=' after last '/'('current()/../vrf_name") + leafrefCurrentTblkey := predicate[strings.LastIndex(predicate, "/")+1:] + + var leafrefTargetKeyVal, leafrefCurrentKeyVal string + for _, kvp := range refTblKeysKVP { + if kvp.key == leafrefTargetTblkey { + leafrefTargetKeyVal = kvp.values[0] + } + } + + for _, kvp := range tblKeysKVP { + if kvp.key == leafrefCurrentTblkey { + leafrefCurrentKeyVal = kvp.values[0] + } + } + + return leafrefTargetKeyVal == leafrefCurrentKeyVal + } + } + } + } + } + + return true +} + //Check delete constraint for leafref if key/field is deleted func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData, tableName, keyVal, field string) CVLRetCode { - var leafRefs []tblFieldPair - if (field != "") { - //Leaf or field is getting deleted - leafRefs = c.findUsedAsLeafRef(tableName, field) - } else { - //Entire entry is getting deleted - leafRefs = c.findUsedAsLeafRef(tableName, modelInfo.tableInfo[tableName].keys[0]) + + // Creating a map of leaf-ref referred tableName and associated fields array + refTableFieldsMap := map[string][]string{} + for _, leafRef := range modelInfo.tableInfo[tableName].refFromTables { + // If field is getting deleted, then collect only those leaf-refs that + // refers that field + if (field != "") && ((field != leafRef.field) || (field != leafRef.field + "@")) { + continue + } + + if _, ok := refTableFieldsMap[leafRef.tableName]; !ok { + refTableFieldsMap[leafRef.tableName] = make([]string, 0) + } + refFieldsArr := refTableFieldsMap[leafRef.tableName] + refFieldsArr = append(refFieldsArr, leafRef.field) + refTableFieldsMap[leafRef.tableName] = refFieldsArr + } + + if len(refTableFieldsMap) == 0 { + return CVL_SUCCESS } //The entry getting deleted might have been referred from multiple tables //Return failure if at-least one table is using this entry - for _, leafRef := range leafRefs { - TRACE_LOG(INFO_API, (TRACE_DELETE | TRACE_SEMANTIC), "Checking delete constraint for leafRef %s/%s", leafRef.tableName, leafRef.field) - //Check in dependent data first, if the referred entry is already deleted - leafRefDeleted := false - for _, cfgDataItem := range cfgData { - if (cfgDataItem.VType == VALIDATE_NONE) && - (cfgDataItem.VOp == OP_DELETE ) && - (strings.HasPrefix(cfgDataItem.Key, (leafRef.tableName + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim + keyVal + modelInfo.tableInfo[leafRef.tableName].redisKeyDelim))) { - //Currently, checking for one entry is being deleted in same session - //We should check for all entries - leafRefDeleted = true - break - } - } - if (leafRefDeleted == true) { - continue //check next leafref - } + // Retrieve all dependent DB entries referring the given entry tableName and keyVal + redisKeyForDepData := tableName + "|" + keyVal + depDataArr := c.GetDepDataForDelete(redisKeyForDepData) + TRACE_LOG(TRACE_SEMANTIC, "checkDeleteConstraint--> All Data for deletion: %v", depDataArr) + + // Iterate through dependent DB entries and check if it already present in delete request cache + for _, depData := range depDataArr { + if len(depData.Entry) > 0 && depData.RefKey == redisKeyForDepData { + TRACE_LOG(TRACE_SEMANTIC, "checkDeleteConstraint--> DepData: %v", depData.Entry) + for depEntkey := range depData.Entry { + depEntkeyList := strings.SplitN(depEntkey, "|", 2) + refTblName := depEntkeyList[0] + refTblKey := depEntkeyList[1] + + var isRefTblKeyNotCompatible bool + var isEntryInRequestCache bool + leafRefFieldsArr := refTableFieldsMap[refTblName] + // Verify each dependent data with help of its associated leaf-ref + for _, leafRefField := range leafRefFieldsArr { + TRACE_LOG(TRACE_SEMANTIC, "checkDeleteConstraint--> Checking delete constraint for leafRef %s/%s", refTblName, leafRefField) + // Key compatibility to be checked only if no. of keys are more than 1 + // because dep data for key like "BGP_PEER_GROUP|Vrf1|PG1" can be returned as + // BGP_NEIGHBOR|Vrf1|11.1.1.1 or BGP_NEIGHBOR|default|11.1.1.1 or BGP_NEIGHBOR|Vrf2|11.1.1.1 + // So we have to discard imcompatible dep data + if !c.checkDepDataCompatible(tableName, keyVal, refTblName, refTblKey, leafRefField, depData.Entry[depEntkey]) { + isRefTblKeyNotCompatible = true + TRACE_LOG(TRACE_SEMANTIC, "checkDeleteConstraint--> %s is NOT compatible with %s", redisKeyForDepData, depEntkey) + break + } + tempLeafRef := tblFieldPair{refTblName, leafRefField} + if c.checkDeleteInRequestCache(cfgData, &tempLeafRef, depEntkey, keyVal) { + isEntryInRequestCache = true + break + } + } - //Else, check if any referred enrty is present in DB - var nokey []string - refKeyVal, err := luaScripts["find_key"].Run(redisClient, nokey, leafRef.tableName, - modelInfo.tableInfo[leafRef.tableName].redisKeyDelim, leafRef.field, keyVal).Result() - if (err == nil && refKeyVal != "") { - CVL_LOG(ERROR, "Delete will violate the constraint as entry %s is referred in %s", tableName, refKeyVal) + if isRefTblKeyNotCompatible { + continue + } - return CVL_SEMANTIC_ERROR + if isEntryInRequestCache { + // Entry already in delete request cache, proceed for next dep data + continue + } else { + CVL_LOG(WARNING, "Delete will violate the constraint as entry %s is referred in %v", redisKeyForDepData, depEntkey) + return CVL_SEMANTIC_ERROR + } + } } } - return CVL_SUCCESS } -//Perform semantic checks -func (c *CVL) validateSemantics(data *yparser.YParserNode, appDepData *yparser.YParserNode) (CVLErrorInfo, CVLRetCode) { - var cvlErrObj CVLErrorInfo - - if (SkipSemanticValidation() == true) { - return cvlErrObj, CVL_SUCCESS - } +//Validate external dependency using leafref, must and when expression +func (c *CVL) validateSemantics(node *xmlquery.Node, + yangListName, key string, + cfgData *CVLEditConfigData) (r CVLErrorInfo) { - //Get dependent data from - depData := c.fetchDataToTmpCache() //fetch data to temp cache for temporary validation - - if (Tracing == true) { - TRACE_LOG(INFO_API, TRACE_SEMANTIC, "Validating semantics data=%s\n depData =%s\n, appDepData=%s\n....", c.yp.NodeDump(data), c.yp.NodeDump(depData), c.yp.NodeDump(appDepData)) + //Mark the list entries from DB if OP_DELETE operation when complete list delete requested + if (node != nil) && (cfgData.VOp == OP_DELETE) && (len(cfgData.Data) == 0) { + addAttrNode(node, "db", "") } - if errObj := c.yp.ValidateSemantics(data, depData, appDepData); errObj.ErrCode != yparser.YP_SUCCESS { + //Check all leafref + if errObj := c.validateLeafRef(node, yangListName, key, cfgData.VOp) ; + errObj.ErrCode != CVL_SUCCESS { + return errObj + } - retCode := CVLRetCode(errObj.ErrCode) + //Validate when expression + if errObj := c.validateWhenExp(node, yangListName, key, cfgData.VOp) ; + errObj.ErrCode != CVL_SUCCESS { + return errObj + } - cvlErrObj = CVLErrorInfo { - TableName : errObj.TableName, - ErrCode : CVLRetCode(errObj.ErrCode), - CVLErrDetails : cvlErrorMap[retCode], - Keys : errObj.Keys, - Value : errObj.Value, - Field : errObj.Field, - Msg : errObj.Msg, - ConstraintErrMsg : errObj.ErrTxt, - ErrAppTag : errObj.ErrAppTag, + //Validate must expression + if (cfgData.VOp == OP_DELETE) { + if (len(cfgData.Data) > 0) { + // Delete leaf nodes from tree. This ensures validateMustExp will + // skip all must expressions defined for deleted nodes; and other + // must expressions get correct context. + c.deleteLeafNodes(node, cfgData.Data) } + } + if errObj := c.validateMustExp(node, yangListName, key, cfgData.VOp) ; + errObj.ErrCode != CVL_SUCCESS { + return errObj + } + + return CVLErrorInfo{ErrCode:CVL_SUCCESS} +} +//Validate external dependency using leafref, must and when expression +func (c *CVL) validateCfgSemantics(root *xmlquery.Node) (r CVLErrorInfo) { + if (strings.HasSuffix(root.Data, "_LIST")) { + if (len(root.Attr) == 0) { + return CVLErrorInfo{ErrCode:CVL_SUCCESS} + } + yangListName := root.Data[:len(root.Data) - 5] + return c.validateSemantics(root, yangListName, root.Attr[0].Value, + &CVLEditConfigData{VType: VALIDATE_NONE, VOp: OP_NONE}) + } - return cvlErrObj, retCode + //Traverse through all list instances and validate + ret := CVLErrorInfo{} + for node := root.FirstChild; node != nil ; node = node.NextSibling { + ret = c.validateCfgSemantics(node) + if (ret.ErrCode != CVL_SUCCESS) { + break + } } - return cvlErrObj ,CVL_SUCCESS + return ret } diff --git a/cvl/cvl_syntax.go b/cvl/cvl_syntax.go index 81009bd18132..397dba147a19 100644 --- a/cvl/cvl_syntax.go +++ b/cvl/cvl_syntax.go @@ -25,6 +25,75 @@ import ( . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" ) +//This function should be called before adding any new entry +//Checks max-elements defined with (current number of entries +//getting added + entries already added and present in request +//cache + entries present in Redis DB) +func (c *CVL) checkMaxElemConstraint(op CVLOperation, tableName string) CVLRetCode { + var nokey []string + + if (op != OP_CREATE) && (op != OP_DELETE) { + //Nothing to do, just return + return CVL_SUCCESS + } + + if modelInfo.tableInfo[tableName].redisTableSize == -1 { + //No limit for table size + return CVL_SUCCESS + } + + curSize, exists := c.maxTableElem[tableName] + + if !exists { //fetch from Redis first time in the session + redisEntries, err := luaScripts["count_entries"].Run(redisClient, nokey, tableName + "|*").Result() + if err != nil { + CVL_LOG(WARNING,"Unable to fetch current size of table %s from Redis, err= %v", + tableName, err) + return CVL_FAILURE + } + + curSize = int(redisEntries.(int64)) + + //Store the current table size + c.maxTableElem[tableName] = curSize + } + + if (op == OP_DELETE) { + //For delete operation we need to reduce the count. + //Because same table can be deleted and added back + //in same session. + + if (curSize > 0) { + c.maxTableElem[tableName] = (curSize - 1) + } + return CVL_SUCCESS + } + + //Otherwise CREATE case + if curSize >= modelInfo.tableInfo[tableName].redisTableSize { + CVL_LOG(WARNING, "%s table size has already reached to max-elements %d", + tableName, modelInfo.tableInfo[tableName].redisTableSize) + + return CVL_SYNTAX_ERROR + } + + curSize = curSize + 1 + if (curSize > modelInfo.tableInfo[tableName].redisTableSize) { + //Does not meet the constraint + CVL_LOG(WARNING, "Max-elements check failed for table '%s'," + + " current size = %v, size in schema = %v", + tableName, curSize, modelInfo.tableInfo[tableName].redisTableSize) + + return CVL_SYNTAX_ERROR + } + + //Update current size + c.maxTableElem[tableName] = curSize + + return CVL_SUCCESS +} + + //Add child node to a parent node func(c *CVL) addChildNode(tableName string, parent *yparser.YParserNode, name string) *yparser.YParserNode { @@ -41,12 +110,6 @@ func (c *CVL) addChildLeaf(config bool, tableName string, parent *yparser.YParse //Batch leaf creation *multileaf = append(*multileaf, &yparser.YParserLeafValue{Name: name, Value: value}) - - //Check if this leaf has leafref, - //If so add the add redis key to its table so that those - // details can be fetched for dependency validation - - c.addLeafRef(config, tableName, name, value) } func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *jsonquery.Node, @@ -75,7 +138,7 @@ parent *yparser.YParserNode, multileaf *[]*yparser.YParserLeafValue) CVLErrorInf if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, batchInnerListLeaf); errObj.ErrCode != yparser.YP_SUCCESS { cvlErrObj = createCVLErrObj(errObj) - CVL_LOG(ERROR, "Failed to create innner list leaf nodes, data = %v", batchInnerListLeaf) + CVL_LOG(WARNING, "Failed to create innner list leaf nodes, data = %v", batchInnerListLeaf) return cvlErrObj } } else { @@ -126,7 +189,7 @@ func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node)(*yparser. // Add top most conatiner e.g. 'container sonic-acl {...}' if _, exists := modelInfo.tableInfo[tableName]; !exists { - CVL_LOG(ERROR, "Schema details not found for %s", tableName) + CVL_LOG(WARNING, "Schema details not found for %s", tableName) cvlErrObj.ErrCode = CVL_SYNTAX_ERROR cvlErrObj.TableName = tableName cvlErrObj.Msg ="Schema details not found" @@ -200,12 +263,11 @@ func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node)(*yparser. keyIndices[idx] = 0 } - TRACE_LOG(INFO_API, TRACE_CACHE, "Starting batch leaf creation - %v\n", c.batchLeaf) + TRACE_LOG(TRACE_CACHE, "Starting batch leaf creation - %v\n", c.batchLeaf) //process batch leaf creation if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, c.batchLeaf); errObj.ErrCode != yparser.YP_SUCCESS { cvlErrObj = createCVLErrObj(errObj) - CVL_LOG(ERROR, "Failed to create leaf nodes, data = %v", - c.batchLeaf) + CVL_LOG(WARNING, "Failed to create leaf nodes, data = %v", c.batchLeaf) return nil, cvlErrObj } diff --git a/cvl/cvl_test.go b/cvl/cvl_test.go index 0af107d4525f..c352fd098d82 100644 --- a/cvl/cvl_test.go +++ b/cvl/cvl_test.go @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // // its subsidiaries. // // // // Licensed under the Apache License, Version 2.0 (the "License"); // @@ -22,7 +22,7 @@ package cvl_test import ( "encoding/json" "fmt" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" "io/ioutil" "os" "os/exec" @@ -2299,7 +2299,7 @@ func TestValidateEditConfig_Create_ErrAppTag_In_Must_Negative(t *testing.T) { WriteToFile(fmt.Sprintf("\nCVL Error Info is %v\n", cvlErrInfo)) /* Compare expected error details and error tag. */ - if compareErrorDetails(cvlErrInfo, cvl.CVL_SEMANTIC_DEPENDENT_DATA_MISSING ,"vlan-invalid", "") != true { + if compareErrorDetails(cvlErrInfo, cvl.CVL_SEMANTIC_ERROR ,"vlan-invalid", "") != true { t.Errorf("Config Validation failed -- error details %v %v", cvlErrInfo, retCode) } @@ -2550,19 +2550,22 @@ func TestValidateEditConfig_Create_Chained_Leafref_DepData_Negative(t *testing.T "index": "5", }, }, - "ACL_TABLE" : map[string]interface{} { - "TestACL1": map[string] interface{} { - "stage": "INGRESS", - "type": "L3", - "ports@":"Ethernet2", - }, - }, } //Prepare data in Redis loadConfigDB(rclient, depDataMap) - cfgDataAclRule := []cvl.CVLEditConfigData { + cfgDataAclRule := []cvl.CVLEditConfigData { + cvl.CVLEditConfigData { + cvl.VALIDATE_ALL, + cvl.OP_CREATE, + "ACL_TABLE|TestACL1", + map[string]string { + "stage": "INGRESS", + "type": "L3", + "ports@":"Ethernet2", + }, + }, cvl.CVLEditConfigData { cvl.VALIDATE_ALL, cvl.OP_CREATE, @@ -3214,33 +3217,6 @@ func TestValidateEditConfig_Create_Syntax_DependentData_NegativePortChannelEther } } -func TestValidateEditConfig_Create_Syntax_DependentData_NegativePortChannelNew(t *testing.T) { - - cfgData := []cvl.CVLEditConfigData{ - cvl.CVLEditConfigData{ - cvl.VALIDATE_ALL, - cvl.OP_CREATE, - "VLAN|Vlan1001", - map[string]string{ - "vlanid": "1001", - "members@": "Ethernet12,PortChannel001", - }, - }, - } - - cvSess, _ := cvl.ValidationSessOpen() - - cvlErrInfo, _ := cvSess.ValidateEditConfig(cfgData) - - cvl.ValidationSessClose(cvSess) - - WriteToFile(fmt.Sprintf("\nCVL Error Info is %v\n", cvlErrInfo)) - - if cvlErrInfo.ErrCode == cvl.CVL_SUCCESS { - t.Errorf("Config Validation failed -- error details %v", cvlErrInfo) - } -} - func TestValidateEditConfig_Use_Updated_Data_As_Create_DependentData_Positive(t *testing.T) { depDataMap := map[string]interface{} { "VLAN" : map[string]interface{} { diff --git a/cvl/internal/util/luascript_util.go b/cvl/internal/util/luascript_util.go new file mode 100644 index 000000000000..272566b5b6af --- /dev/null +++ b/cvl/internal/util/luascript_util.go @@ -0,0 +1,83 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package util +import ( + "github.com/go-redis/redis/v7" +) + +var FILTER_ENTRIES_LUASCRIPT *redis.Script = redis.NewScript(` + --ARGV[1] => Key patterns + --ARGV[2] => Key names separated by '|' + --ARGV[3] => predicate patterns + --ARGV[4] => Fields to return + --ARGV[5] => Count of entries to return + + local tableData = {} ; local tbl = {} + + local keys = redis.call('KEYS', ARGV[1]) + if #keys == 0 then return end + + local count = -1 + if (ARGV[5] ~= nil and ARGV[5] ~= "") then count=tonumber(ARGV[5]) end + + local sepStart = string.find(keys[1], "|") + if sepStart == nil then return ; end + + -- Function to load lua predicate code + local function loadPredicateScript(str) + if (str == nil or str == "") then return nil; end + + local f, err = loadstring("return function (k,h) " .. str .. " end") + if f then return f(); else return nil;end + end + + local keySetNames = {} + ARGV[2]:gsub("([^|]+)",function(c) table.insert(keySetNames, c) end) + + local predicate = loadPredicateScript(ARGV[3]) + + for _, key in ipairs(keys) do + local hash = redis.call('HGETALL', key) + local row = {}; local keySet = {}; local keyVal = {} + local keyOnly = string.sub(key, sepStart+1) + + for index = 1, #hash, 2 do + row[hash[index]] = hash[index + 1] + end + + --Split key values + keyOnly:gsub("([^|]+)", function(c) table.insert(keyVal, c) end) + + for idx = 1, #keySetNames, 1 do + keySet[keySetNames[idx]] = keyVal[idx] + end + + if (predicate == nil) or (predicate(keySet, row) == true) then + tbl[keyOnly] = row + end + + if (count ~= -1 and #tbl == count) then break end + + end + + tableData[string.sub(keys[1], 0, sepStart-1)] = tbl + + return cjson.encode(tableData) +`) diff --git a/cvl/internal/util/util.go b/cvl/internal/util/util.go index aee1666e2b8b..209ca2127522 100644 --- a/cvl/internal/util/util.go +++ b/cvl/internal/util/util.go @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // // its subsidiaries. // // // // Licensed under the Apache License, Version 2.0 (the "License"); // @@ -19,22 +19,48 @@ package util +/* +#cgo LDFLAGS: -lyang +#include + +extern void customLogCallback(LY_LOG_LEVEL, char* msg, char* path); + +static void customLogCb(LY_LOG_LEVEL level, const char* msg, const char* path) { + customLogCallback(level, (char*)msg, (char*)path); +} + +static void ly_set_log_callback(int enable) { + ly_set_log_clb(customLogCb, 0); + if (enable == 1) { + ly_verb(LY_LLDBG); + } else { + ly_verb(LY_LLERR); + } +} + +*/ +import "C" import ( "os" "fmt" + "io" "runtime" - "encoding/json" - "io/ioutil" - "os/signal" - "syscall" + "encoding/json" + "io/ioutil" + "os/signal" + "syscall" "strings" - "flag" - "github.com/go-redis/redis" + "strconv" + "github.com/go-redis/redis/v7" log "github.com/golang/glog" + set "github.com/Workiva/go-datastructures/set" + fileLog "log" + "sync" ) -var CVL_SCHEMA string = "/usr/sbin/schema/" +var CVL_SCHEMA string = "schema/" var CVL_CFG_FILE string = "/usr/sbin/cvl_cfg.json" +const CVL_LOG_FILE = "/tmp/cvl.log" const SONIC_DB_CONFIG_FILE string = "/var/run/redis/sonic-db/database_config.json" const ENV_VAR_SONIC_DB_CONFIG_FILE = "DB_CONFIG_PATH" var sonic_db_config = make(map[string]interface{}) @@ -49,44 +75,50 @@ func init() { CVL_CFG_FILE = os.Getenv("CVL_CFG_FILE") } + //Initialize mutex + logFileMutex = &sync.Mutex{} + //Initialize DB settings dbCfgInit() } var cvlCfgMap map[string]string +var isLogToFile bool +var logFileSize int +var pLogFile *os.File +var logFileMutex *sync.Mutex -/* Logging Level for CVL global logging. */ -type CVLLogLevel uint8 +//CVLLogLevel Logging Level for CVL global logging +type CVLLogLevel uint8 const ( - INFO = 0 + iota - WARNING - ERROR - FATAL - INFO_API - INFO_TRACE + INFO = 0 + iota + WARNING + ERROR + FATAL INFO_DEBUG + INFO_API INFO_DATA INFO_DETAIL + INFO_TRACE INFO_ALL ) var cvlTraceFlags uint32 -/* Logging levels for CVL Tracing. */ +//CVLTraceLevel Logging levels for CVL Tracing type CVLTraceLevel uint32 const ( TRACE_MIN = 0 TRACE_MAX = 8 - TRACE_CACHE = 1 << TRACE_MIN - TRACE_LIBYANG = 1 << 1 - TRACE_YPARSER = 1 << 2 - TRACE_CREATE = 1 << 3 - TRACE_UPDATE = 1 << 4 - TRACE_DELETE = 1 << 5 - TRACE_SEMANTIC = 1 << 6 - TRACE_ONERROR = 1 << 7 - TRACE_SYNTAX = 1 << TRACE_MAX - + TRACE_CACHE = 1 << TRACE_MIN + TRACE_LIBYANG = 1 << 1 + TRACE_YPARSER = 1 << 2 + TRACE_CREATE = 1 << 3 + TRACE_UPDATE = 1 << 4 + TRACE_DELETE = 1 << 5 + TRACE_SEMANTIC = 1 << 6 + TRACE_ONERROR = 1 << 7 + TRACE_SYNTAX = 1 << TRACE_MAX ) @@ -116,7 +148,7 @@ var Tracing bool = false var traceFlags uint16 = 0 func SetTrace(on bool) { - if (on == true) { + if on { Tracing = true traceFlags = 1 } else { @@ -133,40 +165,137 @@ func IsTraceSet() bool { } } +/* The following function enbles the libyang logging by +changing libyang's global log setting */ + +//export customLogCallback +func customLogCallback(level C.LY_LOG_LEVEL, msg *C.char, path *C.char) { + if level == C.LY_LLERR { + CVL_LEVEL_LOG(WARNING, "[libyang Error] %s (path: %s)", C.GoString(msg), C.GoString(path)) + } else { + TRACE_LEVEL_LOG(TRACE_YPARSER, "[libyang] %s (path: %s)", C.GoString(msg), C.GoString(path)) + } +} + func IsTraceLevelSet(tracelevel CVLTraceLevel) bool { return (cvlTraceFlags & (uint32)(tracelevel)) != 0 } -func TRACE_LEVEL_LOG(level log.Level, tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { +func TRACE_LEVEL_LOG(tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { + /* if (IsTraceSet() == false) { return } level = (level - INFO_API) + 1; + */ traceEnabled := false - if ((cvlTraceFlags & (uint32)(tracelevel)) != 0) { - traceEnabled = true - } + if ((cvlTraceFlags & (uint32)(tracelevel)) != 0) { + traceEnabled = true + } + if traceEnabled && isLogToFile { + logToCvlFile(fmtStr, args...) + return + } - if IsTraceSet() == true && traceEnabled == true { + if IsTraceSet() && traceEnabled { pc := make([]uintptr, 10) runtime.Callers(2, pc) f := runtime.FuncForPC(pc[0]) file, line := f.FileLine(pc[0]) - fmt.Printf("%s:%d %s(): ", file, line, f.Name()) + fmt.Printf("%s:%d [CVL] : %s(): ", file, line, f.Name()) fmt.Printf(fmtStr+"\n", args...) } else { - if (traceEnabled == true) { - log.V(level).Infof(fmtStr, args...) + if traceEnabled { + fmtStr = "[CVL] : " + fmtStr + //Trace logs has verbose level INFO_TRACE + log.V(INFO_TRACE).Infof(fmtStr, args...) + } + } +} + +//Logs to /tmp/cvl.log file +func logToCvlFile(format string, args ...interface{}) { + if (pLogFile == nil) { + return + } + + logFileMutex.Lock() + if (logFileSize == 0) { + fileLog.Printf(format, args...) + logFileMutex.Unlock() + return + } + + fStat, err := pLogFile.Stat() + + var curSize int64 = 0 + if (err == nil) && (fStat != nil) { + curSize = fStat.Size() + } + + // Roll over the file contents if size execeeds max defined limit + if (curSize >= int64(logFileSize)) { + //Write 70% contents from bottom and write to top + //Truncate 30% of bottom + + //close the file first + pLogFile.Close() + + pFile, err := os.OpenFile(CVL_LOG_FILE, + os.O_RDONLY, 0666) + pFileOut, errOut := os.OpenFile(CVL_LOG_FILE + ".tmp", + os.O_WRONLY | os.O_CREATE, 0666) + + + if (err != nil) && (errOut != nil) { + fileLog.Printf("Failed to roll over the file, current size %v", curSize) + } else { + pFile.Seek(int64(logFileSize * 30/100), io.SeekStart) + _, err := io.Copy(pFileOut, pFile) + if err == nil { + os.Rename(CVL_LOG_FILE + ".tmp", CVL_LOG_FILE) + } + } + + if (pFile != nil) { + pFile.Close() + } + if (pFileOut != nil) { + pFileOut.Close() + } + + // Reopen the file + pLogFile, err := os.OpenFile(CVL_LOG_FILE, + os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666) + if err != nil { + fmt.Printf("Error in opening log file %s, %v", CVL_LOG_FILE, err) + } else { + fileLog.SetOutput(pLogFile) } } + + + fileLog.Printf(format, args...) + + logFileMutex.Unlock() } func CVL_LEVEL_LOG(level CVLLogLevel, format string, args ...interface{}) { + if isLogToFile { + logToCvlFile(format, args...) + if level == FATAL { + log.Fatalf("[CVL] : " + format, args...) + } + return + } + + format = "[CVL] : " + format + switch level { case INFO: log.Infof(format, args...) @@ -188,10 +317,48 @@ func CVL_LEVEL_LOG(level CVLLogLevel, format string, args ...interface{}) { log.V(5).Infof(format, args...) case INFO_ALL: log.V(6).Infof(format, args...) - } + } } +// Function to check CVL log file related settings +func applyCvlLogFileConfig() { + + if (pLogFile != nil) { + pLogFile.Close() + pLogFile = nil + } + + //Disable libyang trace log + C.ly_set_log_callback(0) + isLogToFile = false + logFileSize = 0 + + enabled, exists := cvlCfgMap["LOG_TO_FILE"] + if !exists { + return + } + + if fileSize, sizeExists := cvlCfgMap["LOG_FILE_SIZE"]; sizeExists { + logFileSize, _ = strconv.Atoi(fileSize) + } + + if (enabled == "true") { + pFile, err := os.OpenFile(CVL_LOG_FILE, + os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666) + + if err != nil { + fmt.Printf("Error in opening log file %s, %v", CVL_LOG_FILE, err) + } else { + pLogFile = pFile + fileLog.SetOutput(pLogFile) + isLogToFile = true + } + + //Enable libyang trace log + C.ly_set_log_callback(1) + } +} func ConfigFileSyncHandler() { sigs := make(chan os.Signal, 1) @@ -207,12 +374,8 @@ func ConfigFileSyncHandler() { CVL_LEVEL_LOG(INFO ,"Received SIGUSR2. Changed configuration values are %v", cvlCfgMap) - if (strings.Compare(cvlCfgMap["LOGTOSTDERR"], "true") == 0) { SetTrace(true) - flag.Set("logtostderr", "true") - flag.Set("stderrthreshold", cvlCfgMap["STDERRTHRESHOLD"]) - flag.Set("v", cvlCfgMap["VERBOSITY"]) } } }() @@ -227,9 +390,12 @@ func ReadConfFile() map[string]string{ } data, err := ioutil.ReadFile(CVL_CFG_FILE) + if err != nil { + CVL_LEVEL_LOG(INFO ,"Error in reading cvl configuration file %v", err) + return nil + } err = json.Unmarshal(data, &cvlCfgMap) - if err != nil { CVL_LEVEL_LOG(INFO ,"Error in reading cvl configuration file %v", err) return nil @@ -238,18 +404,20 @@ func ReadConfFile() map[string]string{ CVL_LEVEL_LOG(INFO ,"Current Values of CVL Configuration File %v", cvlCfgMap) var index uint32 - for index = TRACE_MIN ; index < TRACE_MAX ; index++ { + for index = TRACE_MIN ; index <= TRACE_MAX ; index++ { if (strings.Compare(cvlCfgMap[traceLevelMap[1 << index]], "true") == 0) { cvlTraceFlags = cvlTraceFlags | (1 << index) } } + applyCvlLogFileConfig() + return cvlCfgMap } func SkipValidation() bool { val, existing := cvlCfgMap["SKIP_VALIDATION"] - if (existing == true) && (val == "true") { + if existing && (val == "true") { return true } @@ -258,7 +426,7 @@ func SkipValidation() bool { func SkipSemanticValidation() bool { val, existing := cvlCfgMap["SKIP_SEMANTIC_VALIDATION"] - if (existing == true) && (val == "true") { + if existing && (val == "true") { return true } @@ -445,3 +613,41 @@ func NewDbClient(dbName string) *redis.Client { return redisClient } + +// createStringSet This will create Set data-structure from a list of items +func createStringSet(arr []string) *set.Set { + s := set.New() + for i := range arr { + s.Add(arr[i]) + } + return s +} + +// GetDifference This will determine items which are +// missing in a-list if size of b-list is greater +// missing in b-list if size of a-list is greater +// missing in a-list if size of a-list, b-list are equal +func GetDifference(a, b []string) []string { + var res []string + + aSet := createStringSet(a) + bSet := createStringSet(b) + + if aSet != nil && bSet != nil { + if aSet.Len() < bSet.Len() { + for _, item := range bSet.Flatten() { + if !aSet.Exists(item) { + res = append(res, item.(string)) + } + } + } else { + for _, item := range aSet.Flatten() { + if !bSet.Exists(item) { + res = append(res, item.(string)) + } + } + } + } + + return res +} diff --git a/cvl/internal/yparser/yparser.go b/cvl/internal/yparser/yparser.go index 3a7eb5f66858..727cccb23722 100644 --- a/cvl/internal/yparser/yparser.go +++ b/cvl/internal/yparser/yparser.go @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // // its subsidiaries. // // // // Licensed under the Apache License, Version 2.0 (the "License"); // @@ -25,7 +25,6 @@ import ( "os" "fmt" "strings" - log "github.com/golang/glog" //lint:ignore ST1001 This is safe to dot import for util package . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "unsafe" @@ -39,6 +38,8 @@ import ( #include #include +//extern int lyd_check_mandatory_tree(struct lyd_node *root, struct ly_ctx *ctx, const struct lys_module **modules, int mod_count, int options); + struct lyd_node* lyd_parse_data_path(struct ly_ctx *ctx, const char *path, LYD_FORMAT format, int options) { return lyd_parse_path(ctx, path, format, options); } @@ -50,49 +51,18 @@ struct lyd_node *lyd_parse_data_mem(struct ly_ctx *ctx, const char *data, LYD_FO int lyd_data_validate(struct lyd_node **node, int options, struct ly_ctx *ctx) { - return lyd_validate(node, options, ctx); -} + int ret = -1; -int lyd_data_validate_all(const char *data, const char *depData, const char *othDepData, int options, struct ly_ctx *ctx) -{ - struct lyd_node *pData; - struct lyd_node *pDepData; - struct lyd_node *pOthDepData; + //Check mandatory elements as it is skipped for LYD_OPT_EDIT + //ret = lyd_check_mandatory_tree(*node, ctx, NULL, 0, LYD_OPT_CONFIG | LYD_OPT_NOEXTDEPS); + ret = 0; - if ((data == NULL) || (data[0] == '\0')) + if (ret != 0) { - return -1; + return ret; } - pData = lyd_parse_mem(ctx, data, LYD_XML, LYD_OPT_EDIT | LYD_OPT_NOEXTDEPS); - if (pData == NULL) - { - return -1; - } - - if ((depData != NULL) && (depData[0] != '\0')) - { - if (NULL != (pDepData = lyd_parse_mem(ctx, depData, LYD_XML, LYD_OPT_EDIT | LYD_OPT_NOEXTDEPS))) - { - if (0 != lyd_merge_to_ctx(&pData, pDepData, LYD_OPT_DESTRUCT, ctx)) - { - return -1; - } - } - } - - if ((othDepData != NULL) && (othDepData[0] != '\0')) - { - if (NULL != (pOthDepData = lyd_parse_mem(ctx, othDepData, LYD_XML, LYD_OPT_EDIT | LYD_OPT_NOEXTDEPS))) - { - if (0 != lyd_merge_to_ctx(&pData, pOthDepData, LYD_OPT_DESTRUCT, ctx)) - { - return -1; - } - } - } - - return lyd_validate(&pData, LYD_OPT_CONFIG, ctx); + return lyd_validate(node, options, ctx); } struct leaf_value { @@ -103,7 +73,7 @@ struct leaf_value { int lyd_multi_new_leaf(struct lyd_node *parent, const struct lys_module *module, struct leaf_value *leafValArr, int size) { - const char *name, *val; + const char *name, *val; struct lyd_node *leaf; struct lys_type *type = NULL; int has_ptr_type = 0; @@ -305,6 +275,7 @@ type YParserListInfo struct { XpathExpr map[string][]*XpathExpression CustValidation map[string]string WhenExpr map[string][]*WhenExpression //multiple when expression for choice/case etc + MandatoryNodes map[string]bool } type YParserLeafValue struct { @@ -313,7 +284,6 @@ type YParserLeafValue struct { } type YParser struct { - ctx *YParserCtx //Parser context root *YParserNode //Top evel root for validation operation string //Edit operation } @@ -364,8 +334,8 @@ const ( var yparserInitialized bool = false -func TRACE_LOG(level log.Level, tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { - TRACE_LEVEL_LOG(level, tracelevel , fmtStr, args...) +func TRACE_LOG(tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { + TRACE_LEVEL_LOG(tracelevel , fmtStr, args...) } func CVL_LOG(level CVLLogLevel, fmtStr string, args ...interface{}) { @@ -404,9 +374,6 @@ func Finish() { //ParseSchemaFile Parse YIN schema file func ParseSchemaFile(modelFile string) (*YParserModule, YParserError) { - /* schema */ - TRACE_LOG(INFO_DEBUG, TRACE_YPARSER, "Parsing schema file %s ...\n", modelFile) - module := C.lys_parse_path((*C.struct_ly_ctx)(ypCtx), C.CString(modelFile), C.LYS_IN_YIN) if module == nil { return nil, getErrorDetails() @@ -426,7 +393,7 @@ func(yp *YParser) AddChildNode(module *YParserModule, parent *YParserNode, name ret := (*YParserNode)(C.lyd_new((*C.struct_lyd_node)(parent), (*C.struct_lys_module)(module), C.CString(name))) if (ret == nil) { - TRACE_LOG(INFO_DEBUG, TRACE_YPARSER, "Failed parsing node %s\n", name) + TRACE_LOG(TRACE_YPARSER, "Failed parsing node %s", name) } return ret @@ -438,7 +405,7 @@ func (yp *YParser) IsLeafrefMatchedInUnion(module *YParserModule, xpath, value s return C.lyd_node_leafref_match_in_union((*C.struct_lys_module)(module), C.CString(xpath), C.CString(value)) == 0 } -//AddMultiLeafNodes Add child node to a parent node +//AddMultiLeafNodes dd child node to a parent node func(yp *YParser) AddMultiLeafNodes(module *YParserModule, parent *YParserNode, multiLeaf []*YParserLeafValue) YParserError { leafValArr := make([]C.struct_leaf_value, len(multiLeaf)) @@ -457,7 +424,7 @@ func(yp *YParser) AddMultiLeafNodes(module *YParserModule, parent *YParserNode, if C.lyd_multi_new_leaf((*C.struct_lyd_node)(parent), (*C.struct_lys_module)(module), (*C.struct_leaf_value)(unsafe.Pointer(&leafValArr[0])), size) != 0 { if Tracing { - TRACE_LOG(INFO_API, TRACE_ONERROR, "Failed to create Multi Leaf Data = %v", multiLeaf) + TRACE_LOG(TRACE_ONERROR, "Failed to create Multi Leaf Data = %v", multiLeaf) } return getErrorDetails() } @@ -465,7 +432,6 @@ func(yp *YParser) AddMultiLeafNodes(module *YParserModule, parent *YParserNode, return YParserError {ErrCode : YP_SUCCESS,} } - //NodeDump Return entire subtree in XML format in string func (yp *YParser) NodeDump(root *YParserNode) string { if (root == nil) { @@ -487,7 +453,7 @@ func (yp *YParser) MergeSubtree(root, node *YParserNode) (*YParserNode, YParserE if Tracing { rootdumpStr := yp.NodeDump((*YParserNode)(rootTmp)) - TRACE_LOG(INFO_API, TRACE_YPARSER, "Root subtree = %v\n", rootdumpStr) + TRACE_LOG(TRACE_YPARSER, "Root subtree = %v\n", rootdumpStr) } if C.lyd_merge_to_ctx(&rootTmp, (*C.struct_lyd_node)(node), C.LYD_OPT_DESTRUCT, (*C.struct_ly_ctx)(ypCtx)) != 0 { @@ -496,45 +462,12 @@ func (yp *YParser) MergeSubtree(root, node *YParserNode) (*YParserNode, YParserE if Tracing { dumpStr := yp.NodeDump((*YParserNode)(rootTmp)) - TRACE_LOG(INFO_API, TRACE_YPARSER, "Merged subtree = %v\n", dumpStr) + TRACE_LOG(TRACE_YPARSER, "Merged subtree = %v\n", dumpStr) } return (*YParserNode)(rootTmp), YParserError {ErrCode : YP_SUCCESS,} } -//Cache subtree -func (yp *YParser) CacheSubtree(dupSrc bool, node *YParserNode) YParserError { - rootTmp := (*C.struct_lyd_node)(yp.root) - var dup *C.struct_lyd_node - - if (node == nil) { - //nothing to merge - return YParserError {ErrCode : YP_SUCCESS,} - } - - if (dupSrc == true) { - dup = C.lyd_dup_withsiblings((*C.struct_lyd_node)(node), C.LYD_DUP_OPT_RECURSIVE | C.LYD_DUP_OPT_NO_ATTR) - } else { - dup = (*C.struct_lyd_node)(node) - } - - if (yp.root != nil) { - if (0 != C.lyd_merge_to_ctx(&rootTmp, (*C.struct_lyd_node)(dup), C.LYD_OPT_DESTRUCT, - (*C.struct_ly_ctx)(ypCtx))) { - return getErrorDetails() - } - } else { - yp.root = (*YParserNode)(dup) - } - - if (Tracing == true) { - dumpStr := yp.NodeDump((*YParserNode)(rootTmp)) - TRACE_LOG(INFO_API, TRACE_YPARSER, "Cached subtree = %v\n", dumpStr) - } - - return YParserError {ErrCode : YP_SUCCESS,} -} - func (yp *YParser) DestroyCache() YParserError { if (yp.root != nil) { @@ -551,7 +484,7 @@ func (yp *YParser) SetOperation(op string) YParserError { return YParserError {ErrCode : YP_INTERNAL_UNKNOWN,} } - if (0 != C.lyd_change_leaf_data((*C.struct_lyd_node)(ypOpNode), C.CString(op))) { + if C.lyd_change_leaf_data((*C.struct_lyd_node)(ypOpNode), C.CString(op)) != 0 { return YParserError {ErrCode : YP_INTERNAL_UNKNOWN,} } @@ -559,122 +492,26 @@ func (yp *YParser) SetOperation(op string) YParserError { return YParserError {ErrCode : YP_SUCCESS,} } -//Validate config - syntax and semantics -func (yp *YParser) ValidateData(data, depData *YParserNode) YParserError { - - var dataRoot *YParserNode +//ValidateSyntax Perform syntax checks +func (yp *YParser) ValidateSyntax(data, depData *YParserNode) YParserError { + dataTmp := (*C.struct_lyd_node)(data) - if (depData != nil) { - if dataRoot, _ = yp.MergeSubtree(data, depData); dataRoot == nil { - CVL_LOG(ERROR, "Failed to merge dependent data\n") + if (data != nil && depData != nil) { + //merge ependent data for synatx validation - Update/Delete case + if C.lyd_merge_to_ctx(&dataTmp, (*C.struct_lyd_node)(depData), C.LYD_OPT_DESTRUCT, (*C.struct_ly_ctx)(ypCtx)) != 0 { + TRACE_LOG((TRACE_SYNTAX | TRACE_LIBYANG), "Unable to merge dependent data\n") return getErrorDetails() } } - dataRootTmp := (*C.struct_lyd_node)(dataRoot) - - if (0 != C.lyd_data_validate(&dataRootTmp, C.LYD_OPT_CONFIG, (*C.struct_ly_ctx)(ypCtx))) { - if (Tracing == true) { - strData := yp.NodeDump((*YParserNode)(dataRootTmp)) - TRACE_LOG(INFO_API, TRACE_ONERROR, "Failed to validate data = %v", strData) - } - - CVL_LOG(ERROR, "Validation failed\n") - return getErrorDetails() - } - - return YParserError {ErrCode : YP_SUCCESS,} -} - -//Perform syntax checks -func (yp *YParser) ValidateSyntax(data *YParserNode) YParserError { - dataTmp := (*C.struct_lyd_node)(data) - //Just validate syntax - if (0 != C.lyd_data_validate(&dataTmp, C.LYD_OPT_EDIT | C.LYD_OPT_NOEXTDEPS, - (*C.struct_ly_ctx)(ypCtx))) { - if (Tracing == true) { + if C.lyd_data_validate(&dataTmp, C.LYD_OPT_EDIT | C.LYD_OPT_NOEXTDEPS, (*C.struct_ly_ctx)(ypCtx)) != 0 { + if Tracing { strData := yp.NodeDump((*YParserNode)(dataTmp)) - TRACE_LOG(INFO_API, TRACE_ONERROR, "Failed to validate Syntax, data = %v", strData) + TRACE_LOG(TRACE_ONERROR, "Failed to validate Syntax, data = %v", strData) } return getErrorDetails() } - //fmt.Printf("Error Code from libyang is %d\n", C.ly_errno) - - return YParserError {ErrCode : YP_SUCCESS,} -} - -//Perform semantic checks -func (yp *YParser) ValidateSemantics(data, depData, appDepData *YParserNode) YParserError { - - var dataTmp *C.struct_lyd_node - - if (data != nil) { - dataTmp = (*C.struct_lyd_node)(data) - } else if (depData != nil) { - dataTmp = (*C.struct_lyd_node)(depData) - } else if (yp.root != nil) { - dataTmp = (*C.struct_lyd_node)(yp.root) - } else { - if (yp.operation == "CREATE") || (yp.operation == "UPDATE") { - return YParserError {ErrCode : YP_INTERNAL_UNKNOWN,} - } else { - return YParserError {ErrCode : YP_SUCCESS,} - } - } - - //parse dependent data - if (data != nil && depData != nil) { - - //merge input data and dependent data for semantic validation - if (0 != C.lyd_merge_to_ctx(&dataTmp, (*C.struct_lyd_node)(depData), - C.LYD_OPT_DESTRUCT, (*C.struct_ly_ctx)(ypCtx))) { - TRACE_LOG(INFO_API, (TRACE_SEMANTIC | TRACE_LIBYANG), "Unable to merge dependent data\n") - return getErrorDetails() - } - } - - //Merge cached data - if ((data != nil || depData != nil) && yp.root != nil) { - if (0 != C.lyd_merge_to_ctx(&dataTmp, (*C.struct_lyd_node)(yp.root), - 0, (*C.struct_ly_ctx)(ypCtx))) { - TRACE_LOG(INFO_API, (TRACE_SEMANTIC | TRACE_LIBYANG), "Unable to merge cached dependent data\n") - return getErrorDetails() - } - } - - //Merge appDepData - if (appDepData != nil) { - if (0 != C.lyd_merge_to_ctx(&dataTmp, (*C.struct_lyd_node)(appDepData), - C.LYD_OPT_DESTRUCT, (*C.struct_ly_ctx)(ypCtx))) { - TRACE_LOG(INFO_API, (TRACE_SEMANTIC | TRACE_LIBYANG), "Unable to merge other dependent data\n") - return getErrorDetails() - } - } - - //Add operation for constraint check - if (ypOpRoot != nil) { - //if (0 != C.lyd_insert_sibling(&dataTmp, (*C.struct_lyd_node)(ypOpRoot))) { - if (0 != C.lyd_merge_to_ctx(&dataTmp, (*C.struct_lyd_node)(ypOpRoot), - 0, (*C.struct_ly_ctx)(ypCtx))) { - TRACE_LOG(INFO_API, (TRACE_SEMANTIC | TRACE_LIBYANG), "Unable to insert operation node") - return getErrorDetails() - } - } - - if (Tracing == true) { - strData := yp.NodeDump((*YParserNode)(dataTmp)) - TRACE_LOG(INFO_API, TRACE_YPARSER, "Semantics data = %v", strData) - } - - //Check semantic validation - if (0 != C.lyd_data_validate(&dataTmp, C.LYD_OPT_CONFIG, (*C.struct_ly_ctx)(ypCtx))) { - if (Tracing == true) { - strData1 := yp.NodeDump((*YParserNode)(dataTmp)) - TRACE_LOG(INFO_API, TRACE_ONERROR, "Failed to validate Semantics, data = %v", strData1) - } - return getErrorDetails() - } return YParserError {ErrCode : YP_SUCCESS,} } @@ -796,6 +633,7 @@ func getErrorDetails() YParserError { } /* Fetch the invalid field name. */ + // errMsg is like: Invalid value "Port1" in "dst_port" element. result := strings.Split(errMsg, "\"") if (len(result) > 1) { for i := range result { @@ -803,6 +641,10 @@ func getErrorDetails() YParserError { (strings.Contains(result[i], "Value")) { ElemVal = result[i+1] } + + if (strings.Contains(result[i], "element") || strings.Contains(result[i], "Element")) && (i > 0) { + ElemName = result[i-1] + } } } else if (len(result) == 1) { /* Custom contraint error message like in must statement. @@ -871,7 +713,7 @@ func getErrorDetails() YParserError { ErrAppTag: errAppTag, } - TRACE_LOG(INFO_API, TRACE_YPARSER, "YParser error details: %v...", errObj) + TRACE_LOG(TRACE_YPARSER, "YParser error details: %v...", errObj) return errObj } @@ -1022,6 +864,13 @@ func getModelChildInfo(l *YParserListInfo, node *C.struct_lys_node, } } } + + // check for mandatory flag + if (sChild.flags & C.LYS_MAND_MASK) == C.LYS_MAND_TRUE { + l.MandatoryNodes[leafName] = true + } else if (sChild.flags & C.LYS_MAND_MASK) == C.LYS_MAND_FALSE { + l.MandatoryNodes[leafName] = false + } } } } @@ -1072,6 +921,7 @@ func GetModelListInfo(module *YParserModule) []*YParserListInfo { l.CustValidation = make(map[string]string) l.WhenExpr = make(map[string][]*WhenExpression) l.DfltLeafVal = make(map[string]string) + l.MandatoryNodes = make(map[string]bool) //Add keys keys := (*[10]*C.struct_lys_node_leaf)(unsafe.Pointer(slist.keys))