CVL Changes sonic-net#6: Customized Xpath Engine integration (sonic-n…

…et#27)

Adding support for evaluating xpath expression (leafref, must, when expression) using customized open source xpath, xmlquery and jsonquery.

cvl/cvl.go

@@ -27,6 +27,7 @@ import (
 	 log "github.com/golang/glog"
 	"github.com/go-redis/redis"
 	"github.com/antchfx/xmlquery"
+	"github.com/antchfx/xpath"
 	"github.com/antchfx/jsonquery"
 	"github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
 	. "github.com/Azure/sonic-mgmt-common/cvl/internal/util"
@@ -177,6 +178,15 @@ func init() {
 		SetTrace(true)
 	}
 
+	xpath.SetKeyGetClbk(func(listName string) []string {
+		if modelInfo.tableInfo[listName] != nil {
+			return modelInfo.tableInfo[listName].keys
+		}
+
+		return nil
+	})
+
+
 	ConfigFileSyncHandler()
 
 	cvlCfgMap := ReadConfFile()
@@ -214,6 +224,14 @@ func init() {
 	}
 
 	dbCacheSet(false, "PORT", 0)
+
+	xpath.SetLogCallback(func(fmt string, args ...interface{}) {
+		if !IsTraceLevelSet(TRACE_SEMANTIC) {
+			return
+		}
+
+		TRACE_LOG(INFO_API, TRACE_SEMANTIC, "XPATH: " + fmt, args...)
+	})
 }
 
 func Debug(on bool) {

cvl/cvl_semantics.go

@@ -22,6 +22,7 @@ package cvl
 import (
 	"strings"
 	"encoding/xml"
+	"encoding/json"
 	"github.com/antchfx/xmlquery"
 	"github.com/antchfx/jsonquery"
 	"github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
@@ -691,6 +692,97 @@ func (c *CVL) setOperation(op CVLOperation) {
 	}
 }
 
+//Add given YANG data buffer to Yang Validator
+//redisKeys - Set of redis keys
+//redisKeyFilter - Redis key filter in glob style pattern
+//keyNames - Names of all keys separated by "|"
+//predicate - Condition on keys/fields
+//fields - Fields to retrieve, separated by "|"
+//Return "," separated list of leaf nodes if only one leaf is requested
+//One leaf is used as xpath query result in other nested xpath
+func (c *CVL) addDepYangData(redisKeys []string, redisKeyFilter,
+		keyNames, predicate, fields, count string) string {
+
+	var v interface{}
+	tmpPredicate := ""
+
+	//Get filtered Redis data based on lua script 
+	//filter derived from Xpath predicate
+	if (predicate != "") {
+		tmpPredicate = "return (" + predicate + ")"
+	}
+
+	cfgData, err := luaScripts["filter_entries"].Run(redisClient, []string{},
+	redisKeyFilter, keyNames, tmpPredicate, fields, count).Result()
+
+	singleLeaf := "" //leaf data for single leaf
+
+	TRACE_LOG(INFO_API, TRACE_SEMANTIC, "addDepYangData() with redisKeyFilter=%s, " +
+	"predicate=%s, fields=%s, returned cfgData = %s, err=%v",
+	redisKeyFilter, predicate, fields, cfgData, err)
+
+	if (cfgData == nil) {
+		return ""
+	}
+
+	//Parse the JSON map received from lua script
+	b := []byte(cfgData.(string))
+	if err := json.Unmarshal(b, &v); err != nil {
+		return ""
+	}
+
+	var dataMap map[string]interface{} = v.(map[string]interface{})
+
+	dataTop, _ := jsonquery.ParseJsonMap(&dataMap)
+
+	for jsonNode := dataTop.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling {
+		//Generate YANG data for Yang Validator from Redis JSON
+		topYangNode, _ := c.generateYangListData(jsonNode, false)
+
+		if  topYangNode == nil {
+			continue
+		}
+
+		if (topYangNode.FirstChild != nil) &&
+		(topYangNode.FirstChild.FirstChild != nil) {
+			//Add attribute mentioning that data is from db
+			addAttrNode(topYangNode.FirstChild.FirstChild, "db", "")
+		}
+
+		//Build single leaf data requested
+		singleLeaf = ""
+		for redisKey := topYangNode.FirstChild.FirstChild;
+		redisKey != nil; redisKey = redisKey.NextSibling {
+
+			for field := redisKey.FirstChild; field != nil;
+			field = field.NextSibling {
+				if (field.Data == fields) {
+					//Single field requested
+					singleLeaf = singleLeaf + field.FirstChild.Data + ","
+					break
+				}
+			}
+		}
+
+		//Merge with main YANG data cache
+		doc := &xmlquery.Node{Type: xmlquery.DocumentNode}
+		doc.FirstChild = topYangNode
+		doc.LastChild = topYangNode
+		topYangNode.Parent = doc
+		if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS {
+			continue
+		}
+	}
+
+
+	//remove last comma in case mulitple values returned
+	if (singleLeaf != "") {
+		return singleLeaf[:len(singleLeaf) - 1]
+	}
+
+	return ""
+}
+
 //Check delete constraint for leafref if key/field is deleted
 func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData,
 			tableName, keyVal, field string) CVLRetCode {

cvl/internal/util/util.go

@@ -133,6 +133,10 @@ func IsTraceSet() bool {
 	}
 }
 
+func IsTraceLevelSet(tracelevel CVLTraceLevel) bool {
+	return (cvlTraceFlags & (uint32)(tracelevel)) != 0
+}
+
 func TRACE_LEVEL_LOG(level log.Level, tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) {
 
 	if (IsTraceSet() == false) {

patches/apply.sh

@@ -43,6 +43,8 @@ patch -d ${DEST_DIR}/github.com/openconfig -p1 < ${PATCH_DIR}/ygot/ygot.patch
 patch -d ${DEST_DIR}/github.com/openconfig/goyang -p1 < ${PATCH_DIR}/goyang/goyang.patch
 
 patch -d ${DEST_DIR}/github.com/antchfx/jsonquery -p1 < ${PATCH_DIR}/jsonquery.patch
+patch -d ${DEST_DIR}/github.com/antchfx/xmlquery -p1 < ${PATCH_DIR}/xmlquery.patch
+patch -d ${DEST_DIR}/github.com/antchfx/xpath -p1 < ${PATCH_DIR}/xpath.patch
 
 patch -d ${DEST_DIR}/github.com/golang/glog  -p1 < ${PATCH_DIR}/glog.patch
 

patches/jsonquery.patch

@@ -1,8 +1,70 @@
 diff --git a/node.go b/node.go
-index 76032bb..db73a1e 100644
+index 76032bb..f6103d9 100644
 --- a/node.go
 +++ b/node.go
-@@ -155,3 +155,9 @@ func Parse(r io.Reader) (*Node, error) {
+@@ -8,6 +8,7 @@ import (
+ 	"net/http"
+ 	"sort"
+ 	"strconv"
++	"strings"
+ )
+
+ // A NodeType is the type of a Node.
+@@ -110,6 +111,29 @@ func parseValue(x interface{}, top *Node, level int) {
+ 			addNode(n)
+ 			parseValue(vv, n, level+1)
+ 		}
++	case map[string]string:
++		var keys []string
++		for key := range v {
++			keys = append(keys, key)
++		}
++		sort.Strings(keys)
++		for _, key := range keys {
++			tmpKey := key
++			var tmpVal interface{}
++			tmpVal = v[key]
++			if (strings.HasSuffix(key, "@")) {
++				tmpKey = key[:len(key) - 1]
++				tmpValArr := []interface{}{}
++				for _, val := range strings.Split(v[key], ",") {
++					tmpValArr = append(tmpValArr, val)
++				}
++				tmpVal = tmpValArr
++			}
++
++			n := &Node{Data: tmpKey, Type: ElementNode, level: level}
++			addNode(n)
++			parseValue(tmpVal, n, level+1)
++		}
+ 	case map[string]interface{}:
+ 		// The Go’s map iteration order is random.
+ 		// (https://blog.golang.org/go-maps-in-action#Iteration-order)
+@@ -119,9 +143,21 @@ func parseValue(x interface{}, top *Node, level int) {
+ 		}
+ 		sort.Strings(keys)
+ 		for _, key := range keys {
+-			n := &Node{Data: key, Type: ElementNode, level: level}
++			tmpKey := key
++			var tmpVal interface{}
++			tmpVal = v[key]
++			if (strings.HasSuffix(key, "@")) {
++				tmpKey = key[:len(key) - 1]
++				tmpValArr := []interface{}{}
++				for _, val := range strings.Split(v[key].(string), ",") {
++					tmpValArr = append(tmpValArr, val)
++				}
++				tmpVal = tmpValArr
++			}
++
++			n := &Node{Data: tmpKey, Type: ElementNode, level: level}
+ 			addNode(n)
+-			parseValue(v[key], n, level+1)
++			parseValue(tmpVal, n, level+1)
+ 		}
+ 	case string:
+ 		n := &Node{Data: v, Type: TextNode, level: level}
+@@ -155,3 +191,9 @@ func Parse(r io.Reader) (*Node, error) {
  	}
  	return parse(b)
  }
@@ -12,3 +74,22 @@ index 76032bb..db73a1e 100644
 +	parseValue(*jsonMap, doc, 1)
 +	return doc, nil
 +}
+diff --git a/query.go b/query.go
+index d105962..e8db1d6 100644
+--- a/query.go
++++ b/query.go
+@@ -120,6 +120,14 @@ func (a *NodeNavigator) MoveToRoot() {
+ 	a.cur = a.root
+ }
+
++func (a *NodeNavigator) MoveToContext() {
++	return
++}
++
++func (a *NodeNavigator) CurrentPrefix() string {
++	return ""
++}
++
+ func (a *NodeNavigator) MoveToParent() bool {
+ 	if n := a.cur.Parent; n != nil {
+ 		a.cur = n

patches/xmlquery.patch

@@ -0,0 +1,94 @@
+diff --git a/node.go b/node.go
+index e86c0c3..028867c 100644
+--- a/node.go
++++ b/node.go
+@@ -48,7 +48,7 @@ type Node struct {
+
+ // InnerText returns the text between the start and end tags of the object.
+ func (n *Node) InnerText() string {
+-	var output func(*bytes.Buffer, *Node)
++	/*var output func(*bytes.Buffer, *Node)
+ 	output = func(buf *bytes.Buffer, n *Node) {
+ 		switch n.Type {
+ 		case TextNode:
+@@ -64,7 +64,18 @@ func (n *Node) InnerText() string {
+
+ 	var buf bytes.Buffer
+ 	output(&buf, n)
+-	return buf.String()
++	return buf.String()*/
++
++	if (n.Type == TextNode) {
++		return n.Data
++	} else if (n.Type == ElementNode) &&
++	(n.FirstChild != nil) &&
++	(n.FirstChild.Type == TextNode) {
++		return n.FirstChild.Data
++	}
++
++
++	return ""
+ }
+
+ func (n *Node) sanitizedData(preserveSpaces bool) string {
+diff --git a/query.go b/query.go
+index 146c2a4..f21b61b 100644
+--- a/query.go
++++ b/query.go
+@@ -49,6 +49,29 @@ func CreateXPathNavigator(top *Node) *NodeNavigator {
+ 	return &NodeNavigator{curr: top, root: top, attr: -1}
+ }
+
++//Evaluate XPath expression, the expression should evaluate to true or false
++func Eval(top, ctx *Node, exp *xpath.Expr) bool {
++	if exp == nil {
++		return false
++	}
++
++	v := exp.Evaluate(&NodeNavigator{curr: ctx, ctxt: ctx, root: top, attr: -1})
++
++	switch val := v.(type) {
++	case bool:
++		return val
++	case string:
++		return (val != "")
++	case float64:
++		return (val != 0)
++	case *xpath.NodeIterator:
++		return (val != nil)
++	}
++
++	//return v.(bool)
++	return false
++}
++
+ func getCurrentNode(it *xpath.NodeIterator) *Node {
+ 	n := it.Current().(*NodeNavigator)
+ 	if n.NodeType() == xpath.AttributeNode {
+@@ -145,7 +168,7 @@ func FindEachWithBreak(top *Node, expr string, cb func(int, *Node) bool) {
+ }
+
+ type NodeNavigator struct {
+-	root, curr *Node
++	root, curr, ctxt *Node
+ 	attr       int
+ }
+
+@@ -212,6 +235,17 @@ func (x *NodeNavigator) MoveToRoot() {
+ 	x.curr = x.root
+ }
+
++func (x *NodeNavigator) MoveToContext() {
++	x.curr = x.ctxt
++}
++
++func (x *NodeNavigator) CurrentPrefix() string {
++	if (x.ctxt != nil) {
++		return x.ctxt.Prefix
++	}
++	return ""
++}
++
+ func (x *NodeNavigator) MoveToParent() bool {
+ 	if x.attr != -1 {
+ 		x.attr = -1