From 3c6cdb344b6ab54b65104c6f2fa3d6c763621282 Mon Sep 17 00:00:00 2001 From: Partha Dutta <51353699+dutta-partha@users.noreply.github.com> Date: Fri, 23 Oct 2020 20:24:37 +0530 Subject: [PATCH] CVL Changes #8: 'must' and 'when' expression evaluation (#31) Adding support for evaluating 'must' and 'when' expression based on customized xpath engine. --- cvl/cvl.go | 48 ++++-- cvl/cvl_semantics.go | 366 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 399 insertions(+), 15 deletions(-) diff --git a/cvl/cvl.go b/cvl/cvl.go index 56697ac1a06f..69d92ac81f94 100644 --- a/cvl/cvl.go +++ b/cvl/cvl.go @@ -63,6 +63,13 @@ var dbNameToDbNum map[string]uint8 //map of lua script loaded var luaScripts map[string]*redis.Script +type whenInfo struct { + expr string //when expression + exprTree *xpath.Expr //compiled expression tree + nodeNames []string //list of nodes under when condition + yangListNames []string //all yang list in expression +} + type leafRefInfo struct { path string //leafref path exprTree *xpath.Expr //compiled expression tree @@ -70,6 +77,13 @@ type leafRefInfo struct { targetNodeName string //target node name } +type mustInfo struct { + expr string //must expression + exprTree *xpath.Expr //compiled expression tree + errCode string //err-app-tag + errStr string //error message +} + //Important schema information to be loaded at bootup time type modelTableInfo struct { dbNum uint8 @@ -82,7 +96,8 @@ type modelTableInfo struct { mapLeaf []string //for 'mapping list' leafRef map[string][]*leafRefInfo //for storing all leafrefs for a leaf in a table, //multiple leafref possible for union - mustExp map[string]string + mustExpr map[string][]*mustInfo + whenExpr map[string][]*whenInfo tablesForMustExp map[string]CVLOperation refFromTables []tblFieldPair //list of table or table/field referring to this table dfltLeafVal map[string]string //map of leaf names and default value @@ -370,7 +385,7 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) { //such mo fieldCount++ keypattern := []string{tableName} - /* Create the default key pattern of the form Table Name|{key1}|{key2}. */ + // Create the default key pattern of the form Table Name|{key1}|{key2}. for _ , key := range tableInfo.keys { keypattern = append(keypattern, fmt.Sprintf("{%s}",key)) } @@ -420,7 +435,7 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) { //such mo continue } - tableInfo.leafRef = make(map[string][]*leafRefInfo) + tableInfo.leafRef = make(map[string][]*leafRefInfo) for _, leafRefNode := range leafRefNodes { if (leafRefNode.Parent == nil || leafRefNode.FirstChild == nil) { @@ -452,7 +467,7 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) { //such mo continue } - tableInfo.mustExp = make(map[string]string) + tableInfo.mustExpr = make(map[string][]*mustInfo) for _, mustExp := range mustExps { if (mustExp.Parent == nil) { continue @@ -467,7 +482,10 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) { //such mo } } if (parentName != "") { - tableInfo.mustExp[parentName] = getXmlNodeAttr(mustExp, "condition") + tableInfo.mustExpr[parentName] = append(tableInfo.mustExpr[parentName], + &mustInfo{ + expr: getXmlNodeAttr(mustExp, "condition"), + }) } } @@ -559,22 +577,22 @@ func buildRefTableInfo() { func addTableNamesForMustExp() { for tblName, tblInfo := range modelInfo.tableInfo { - if (tblInfo.mustExp == nil) { + if (tblInfo.mustExpr == nil) { continue } tblInfo.tablesForMustExp = make(map[string]CVLOperation) - for _, mustExp := range tblInfo.mustExp { + for _, mustExp := range tblInfo.mustExpr { var op CVLOperation = OP_NONE //Check if 'must' expression should be executed for a particular operation - if (strings.Contains(mustExp, + if (strings.Contains(mustExp[0].expr, "/scommon:operation/scommon:operation != CREATE") == true) { op = op | OP_CREATE - } else if (strings.Contains(mustExp, + } else if (strings.Contains(mustExp[0].expr, "/scommon:operation/scommon:operation != UPDATE") == true) { op = op | OP_UPDATE - } else if (strings.Contains(mustExp, + } else if (strings.Contains(mustExp[0].expr, "/scommon:operation/scommon:operation != DELETE") == true) { op = op | OP_DELETE } @@ -592,7 +610,7 @@ func addTableNamesForMustExp() { //Table name should appear like "../VLAN_MEMBER/tagging_mode' or ' // "/prt:PORT/prt:ifname" re := regexp.MustCompile(fmt.Sprintf(".*[/]([a-zA-Z]*:)?%s[\\[/]", tblNameSrch)) - matches := re.FindStringSubmatch(mustExp) + matches := re.FindStringSubmatch(mustExp[0].expr) if (len(matches) > 0) { //stores the table name tblInfo.tablesForMustExp[tblNameSrch] = op @@ -881,11 +899,11 @@ func (c *CVL) checkPathForTableEntry(tableName string, currentValue string, cfgD //Node-set function such count() can be quite expensive and //should be avoided through this function func (c *CVL) addTableEntryForMustExp(cfgData *CVLEditConfigData, tableName string) CVLRetCode { - if (modelInfo.tableInfo[tableName].mustExp == nil) { + if (modelInfo.tableInfo[tableName].mustExpr == nil) { return CVL_SUCCESS } - for fieldName, mustExp := range modelInfo.tableInfo[tableName].mustExp { + for fieldName, mustExp := range modelInfo.tableInfo[tableName].mustExpr { currentValue := "" // Current value for current() function @@ -927,7 +945,7 @@ func (c *CVL) addTableEntryForMustExp(cfgData *CVLEditConfigData, tableName stri } mustExpStk := []string{} //Use the string slice as stack - mustExpStr := "(" + mustExp + ")" + mustExpStr := "(" + mustExp[0].expr + ")" strLen := len(mustExpStr) strTmp := "" //Parse the xpath expression and fetch Redis entry by looking at xpath, @@ -1005,7 +1023,7 @@ func (c *CVL) addTableEntryForMustExp(cfgData *CVLEditConfigData, tableName stri //Add all other table data for validating all 'must' exp for tableName func (c *CVL) addTableDataForMustExp(op CVLOperation, tableName string) CVLRetCode { - if (modelInfo.tableInfo[tableName].mustExp == nil) { + if (modelInfo.tableInfo[tableName].mustExpr == nil) { return CVL_SUCCESS } diff --git a/cvl/cvl_semantics.go b/cvl/cvl_semantics.go index 30176057cbed..0f9ff2b82483 100644 --- a/cvl/cvl_semantics.go +++ b/cvl/cvl_semantics.go @@ -786,6 +786,159 @@ func (c *CVL) addDepYangData(redisKeys []string, redisKeyFilter, return "" } +//Add all other table data for validating all 'must' exp for tableName +//One entry is needed for incremental loading of must tables +func (c *CVL) addYangDataForMustExp(op CVLOperation, tableName string, oneEntry bool) CVLRetCode { + if (modelInfo.tableInfo[tableName].mustExpr == nil) { + return CVL_SUCCESS + } + + for mustTblName, mustOp := range modelInfo.tableInfo[tableName].tablesForMustExp { + //First check if must expression should be executed for the given operation + if (mustOp != OP_NONE) && ((mustOp & op) == OP_NONE) { + //must to be excuted for particular operation, but current operation + //is not the same one + continue + } + + //If one entry is needed and it is already availale in c.yv.root cache + //just ignore and continue + if oneEntry { + node := c.moveToYangList(mustTblName, "") + if node != nil { + //One entry exists, continue + continue + } + } + + redisTblName := getYangListToRedisTbl(mustTblName) //1 yang to N Redis table case + tableKeys, err:= redisClient.Keys(redisTblName + + modelInfo.tableInfo[mustTblName].redisKeyDelim + "*").Result() + + if (err != nil) { + return CVL_FAILURE + } + + if (len(tableKeys) == 0) { + //No dependent data for mustTable available + continue + } + + cvg.cv.clearTmpDbCache() + + //fill all keys; TBD Optimize based on predicate in Xpath + tablePrefixLen := len(redisTblName + modelInfo.tableInfo[mustTblName].redisKeyDelim) + for _, tableKey := range tableKeys { + tableKey = tableKey[tablePrefixLen:] //remove table prefix + + tmpKeyArr := strings.Split(tableKey, modelInfo.tableInfo[mustTblName].redisKeyDelim) + if (len(tmpKeyArr) != len(modelInfo.tableInfo[mustTblName].keys)) { + //Number of keys should be same as in YANG list keys + //Need to check this for one Redis table to many YANG list case + continue + } + + if (cvg.cv.tmpDbCache[redisTblName] == nil) { + cvg.cv.tmpDbCache[redisTblName] = map[string]interface{}{tableKey: nil} + } else { + tblMap := cvg.cv.tmpDbCache[redisTblName] + tblMap.(map[string]interface{})[tableKey] =nil + cvg.cv.tmpDbCache[redisTblName] = tblMap + } + //Load only one entry + if oneEntry { + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "addYangDataForMustExp(): Adding one entry table %s, key %s", + redisTblName, tableKey) + break + } + } + + if (cvg.cv.tmpDbCache[redisTblName] == nil) { + //No entry present in DB + continue + } + + //fetch using pipeline + cvg.cv.fetchTableDataToTmpCache(redisTblName, cvg.cv.tmpDbCache[redisTblName].(map[string]interface{})) + data, err := jsonquery.ParseJsonMap(&cvg.cv.tmpDbCache) + + if (err != nil) { + return CVL_FAILURE + } + + //Build yang tree for each table and cache it + for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { + //Visit each top level list in a loop for creating table data + topYangNode, _ := c.generateYangListData(jsonNode, false) + + if (topYangNode == nil) { + //No entry found, check next entry + continue + } + + if (topYangNode.FirstChild != nil) && + (topYangNode.FirstChild.FirstChild != nil) { + //Add attribute mentioning that data is from db + addAttrNode(topYangNode.FirstChild.FirstChild, "db", "") + } + + //Create full document by adding document node + doc := &xmlquery.Node{Type: xmlquery.DocumentNode} + doc.FirstChild = topYangNode + doc.LastChild = topYangNode + topYangNode.Parent = doc + if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS { + return CVL_INTERNAL_UNKNOWN + } + } + + } + + return CVL_SUCCESS +} + +//Compile all must expression and save the expression tree +func compileMustExps() { + reMultiPred := regexp.MustCompile(`\][ ]*\[`) + + for _, tInfo := range modelInfo.tableInfo { + if (tInfo.mustExpr == nil) { + continue + } + + // Replace multiple predicate using 'and' expressiona + // xpath engine not accepting multiple predicates + for _, mustExprArr := range tInfo.mustExpr { + for _, mustExpr := range mustExprArr { + mustExpr.exprTree = xpath.MustCompile( + reMultiPred.ReplaceAllString(mustExpr.expr, " and ")) + } + } + } +} + +//Compile all when expression and save the expression tree +func compileWhenExps() { + reMultiPred := regexp.MustCompile(`\][ ]*\[`) + + for _, tInfo := range modelInfo.tableInfo { + if (tInfo.whenExpr == nil) { + continue + } + + // Replace multiple predicate using 'and' expressiona + // xpath engine not accepting multiple predicates + for _, whenExprArr := range tInfo.whenExpr { + for _, whenExpr := range whenExprArr { + whenExpr.exprTree = xpath.MustCompile( + reMultiPred.ReplaceAllString(whenExpr.expr, " and ")) + //Store all YANG list used in the expression + whenExpr.yangListNames = getYangListNamesInExpr(whenExpr.expr) + } + } + } +} + func compileLeafRefPath() { reMultiPred := regexp.MustCompile(`\][ ]*\[`) @@ -823,6 +976,219 @@ func compileLeafRefPath() { } } +//Validate must expression +func (c *CVL) validateMustExp(node *xmlquery.Node, + tableName, key string, op CVLOperation) (r CVLErrorInfo) { + defer func() { + ret := &r + CVL_LOG(INFO_API, "validateMustExp(): table name = %s, " + + "return value = %v", tableName, *ret) + }() + + c.setOperation(op) + + //Set xpath callback for retreiving dependent data + xpath.SetDepDataClbk(c, func(ctxt interface{}, redisKeys []string, + redisKeyFilter, keyNames, pred, fields, count string) string { + c := ctxt.(*CVL) + + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateMustExp(): calling addDepYangData()") + return c.addDepYangData(redisKeys, redisKeyFilter, keyNames, pred, fields, "") + }) + + //Set xpath callback for retriving dependent data count + xpath.SetDepDataCntClbk(c, func(ctxt interface{}, + redisKeyFilter, keyNames, pred, field string) float64 { + + if (pred != "") { + pred = "return (" + pred + ")" + } + + redisEntries, err := luaScripts["count_entries"].Run(redisClient, + []string{}, redisKeyFilter, keyNames, pred, field).Result() + + count := float64(0) + + if (err == nil) && (redisEntries.(int64) > 0) { + count = float64(redisEntries.(int64)) + } + + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateMustExp(): depDataCntClbk() with redisKeyFilter=%s, " + + "keyNames= %s, predicate=%s, fields=%s, returned = %v", + redisKeyFilter, keyNames, pred, field, count) + + return count + }) + + if (node == nil || node.FirstChild == nil) { + return CVLErrorInfo{ + TableName: tableName, + ErrCode: CVL_SEMANTIC_ERROR, + CVLErrDetails: cvlErrorMap[CVL_SEMANTIC_ERROR], + Msg: "Failed to find YANG data for must expression validation", + } + } + + //Load all table's any one entry for 'must' expression execution. + //This helps building full YANG tree for tables needed. + //Other instances/entries would be fetched as per xpath predicate execution + //during expression evaluation + c.addYangDataForMustExp(op, tableName, true) + + //Find the node where must expression is attached + for nodeName, mustExpArr := range modelInfo.tableInfo[tableName].mustExpr { + for _, mustExp := range mustExpArr { + ctxNode := node + if (ctxNode.Data != nodeName) { //must expression at list level + ctxNode = ctxNode.FirstChild + for (ctxNode !=nil) && (ctxNode.Data != nodeName) { + ctxNode = ctxNode.NextSibling //must expression at leaf level + } + if ctxNode != nil && op == OP_UPDATE { + addAttrNode(ctxNode, "db", "") + } + } + + //Check leafref for each leaf-list node + /*for ;(ctxNode != nil) && (ctxNode.Data == nodeName); + ctxNode = ctxNode.NextSibling { + //Load first data for each referred table. + //c.yv.root has all requested data merged and any depdendent + //data needed for leafref validation should be available from this. + + leafRefSuccess := false*/ + + if (ctxNode != nil) { + CVL_LOG(INFO_DEBUG, "Eval must \"%s\"; ctxNode=%s", + mustExp.expr, ctxNode.Data) + + if (!xmlquery.Eval(c.yv.root, ctxNode, mustExp.exprTree)) { + keys := []string{} + if (len(ctxNode.Parent.Attr) > 0) { + keys = strings.Split(ctxNode.Parent.Attr[0].Value, + modelInfo.tableInfo[tableName].redisKeyDelim) + } + + return CVLErrorInfo{ + TableName: tableName, + ErrCode: CVL_SEMANTIC_ERROR, + CVLErrDetails: cvlErrorMap[CVL_SEMANTIC_ERROR], + Keys: keys, + Value: ctxNode.FirstChild.Data, + Field: nodeName, + Msg: "Must expression validation failed", + ConstraintErrMsg: mustExp.errStr, + ErrAppTag: mustExp.errCode, + } + } + } + } //for each must exp + } //all must exp under one node + + return CVLErrorInfo{ErrCode:CVL_SUCCESS} +} + +//Currently supports when expression with current table only +func (c *CVL) validateWhenExp(node *xmlquery.Node, + tableName, key string, op CVLOperation) (r CVLErrorInfo) { + + defer func() { + ret := &r + CVL_LOG(INFO_API, "validateWhenExp(): table name = %s, " + + "return value = %v", tableName, *ret) + }() + + if (op == OP_DELETE) { + //No new node getting added so skip when validation + return CVLErrorInfo{ErrCode:CVL_SUCCESS} + } + + //Set xpath callback for retreiving dependent data + xpath.SetDepDataClbk(c, func(ctxt interface{}, redisKeys []string, + redisKeyFilter, keyNames, pred, fields, count string) string { + c := ctxt.(*CVL) + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "validateWhenExp(): calling addDepYangData()") + return c.addDepYangData(redisKeys, redisKeyFilter, keyNames, pred, fields, "") + }) + + if (node == nil || node.FirstChild == nil) { + return CVLErrorInfo{ + TableName: tableName, + ErrCode: CVL_SEMANTIC_ERROR, + CVLErrDetails: cvlErrorMap[CVL_SEMANTIC_ERROR], + Msg: "Failed to find YANG data for must expression validation", + } + } + + //Find the node where when expression is attached + for nodeName, whenExpArr := range modelInfo.tableInfo[tableName].whenExpr { + for _, whenExp := range whenExpArr { //for each when expression + ctxNode := node + if (ctxNode.Data != nodeName) { //when expression not at list level + ctxNode = ctxNode.FirstChild + for (ctxNode !=nil) && (ctxNode.Data != nodeName) { + ctxNode = ctxNode.NextSibling //whent expression at leaf level + } + } + + //Add data for dependent table in when expression + //Add one entry only + for _, refListName := range whenExp.yangListNames { + refRedisTableName := getYangListToRedisTbl(refListName) + + filter := refRedisTableName + + modelInfo.tableInfo[refListName].redisKeyDelim + "*" + + c.addDepYangData([]string{}, filter, + strings.Join(modelInfo.tableInfo[refListName].keys, "|"), + "true", "", "1") //fetch one entry only + } + + //Validate the when expression + if (ctxNode != nil) && !(xmlquery.Eval(c.yv.root, ctxNode, whenExp.exprTree)) { + keys := []string{} + if (len(ctxNode.Parent.Attr) > 0) { + keys = strings.Split(ctxNode.Parent.Attr[0].Value, + modelInfo.tableInfo[tableName].redisKeyDelim) + } + + if (len(whenExp.nodeNames) == 1) && //when in leaf + (nodeName == whenExp.nodeNames[0]) { + return CVLErrorInfo{ + TableName: tableName, + ErrCode: CVL_SEMANTIC_ERROR, + CVLErrDetails: cvlErrorMap[CVL_SEMANTIC_ERROR], + Keys: keys, + Value: ctxNode.FirstChild.Data, + Field: nodeName, + Msg: "When expression validation failed", + } + } else { + //check if any nodes in whenExp.nodeNames + //present in request data, when at list level + whenNodeList := strings.Join(whenExp.nodeNames, ",") + "," + for cNode := node.FirstChild; cNode !=nil; + cNode = cNode.NextSibling { + if strings.Contains(whenNodeList, (cNode.Data + ",")) { + return CVLErrorInfo{ + TableName: tableName, + ErrCode: CVL_SEMANTIC_ERROR, + CVLErrDetails: cvlErrorMap[CVL_SEMANTIC_ERROR], + Keys: keys, + Value: cNode.FirstChild.Data, + Field: cNode.Data, + Msg: "When expression validation failed", + } + } + } + } + } + } + } + + return CVLErrorInfo{ErrCode:CVL_SUCCESS} +} + //Validate leafref //Convert leafref to must expression //type leafref { path "../../../ACL_TABLE/ACL_TABLE_LIST/aclname";} converts to