diff --git a/cvl/cvl.go b/cvl/cvl.go index 012d9f5ea072..30d1ee18534d 100644 --- a/cvl/cvl.go +++ b/cvl/cvl.go @@ -27,6 +27,7 @@ import ( log "github.com/golang/glog" "github.com/go-redis/redis" "github.com/antchfx/xmlquery" + "github.com/antchfx/xpath" "github.com/antchfx/jsonquery" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" . "github.com/Azure/sonic-mgmt-common/cvl/internal/util" @@ -177,6 +178,15 @@ func init() { SetTrace(true) } + xpath.SetKeyGetClbk(func(listName string) []string { + if modelInfo.tableInfo[listName] != nil { + return modelInfo.tableInfo[listName].keys + } + + return nil + }) + + ConfigFileSyncHandler() cvlCfgMap := ReadConfFile() @@ -214,6 +224,14 @@ func init() { } dbCacheSet(false, "PORT", 0) + + xpath.SetLogCallback(func(fmt string, args ...interface{}) { + if !IsTraceLevelSet(TRACE_SEMANTIC) { + return + } + + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "XPATH: " + fmt, args...) + }) } func Debug(on bool) { diff --git a/cvl/cvl_semantics.go b/cvl/cvl_semantics.go index 94270a16c111..345be7f75278 100644 --- a/cvl/cvl_semantics.go +++ b/cvl/cvl_semantics.go @@ -22,6 +22,7 @@ package cvl import ( "strings" "encoding/xml" + "encoding/json" "github.com/antchfx/xmlquery" "github.com/antchfx/jsonquery" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" @@ -691,6 +692,97 @@ func (c *CVL) setOperation(op CVLOperation) { } } +//Add given YANG data buffer to Yang Validator +//redisKeys - Set of redis keys +//redisKeyFilter - Redis key filter in glob style pattern +//keyNames - Names of all keys separated by "|" +//predicate - Condition on keys/fields +//fields - Fields to retrieve, separated by "|" +//Return "," separated list of leaf nodes if only one leaf is requested +//One leaf is used as xpath query result in other nested xpath +func (c *CVL) addDepYangData(redisKeys []string, redisKeyFilter, + keyNames, predicate, fields, count string) string { + + var v interface{} + tmpPredicate := "" + + //Get filtered Redis data based on lua script + //filter derived from Xpath predicate + if (predicate != "") { + tmpPredicate = "return (" + predicate + ")" + } + + cfgData, err := luaScripts["filter_entries"].Run(redisClient, []string{}, + redisKeyFilter, keyNames, tmpPredicate, fields, count).Result() + + singleLeaf := "" //leaf data for single leaf + + TRACE_LOG(INFO_API, TRACE_SEMANTIC, "addDepYangData() with redisKeyFilter=%s, " + + "predicate=%s, fields=%s, returned cfgData = %s, err=%v", + redisKeyFilter, predicate, fields, cfgData, err) + + if (cfgData == nil) { + return "" + } + + //Parse the JSON map received from lua script + b := []byte(cfgData.(string)) + if err := json.Unmarshal(b, &v); err != nil { + return "" + } + + var dataMap map[string]interface{} = v.(map[string]interface{}) + + dataTop, _ := jsonquery.ParseJsonMap(&dataMap) + + for jsonNode := dataTop.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling { + //Generate YANG data for Yang Validator from Redis JSON + topYangNode, _ := c.generateYangListData(jsonNode, false) + + if topYangNode == nil { + continue + } + + if (topYangNode.FirstChild != nil) && + (topYangNode.FirstChild.FirstChild != nil) { + //Add attribute mentioning that data is from db + addAttrNode(topYangNode.FirstChild.FirstChild, "db", "") + } + + //Build single leaf data requested + singleLeaf = "" + for redisKey := topYangNode.FirstChild.FirstChild; + redisKey != nil; redisKey = redisKey.NextSibling { + + for field := redisKey.FirstChild; field != nil; + field = field.NextSibling { + if (field.Data == fields) { + //Single field requested + singleLeaf = singleLeaf + field.FirstChild.Data + "," + break + } + } + } + + //Merge with main YANG data cache + doc := &xmlquery.Node{Type: xmlquery.DocumentNode} + doc.FirstChild = topYangNode + doc.LastChild = topYangNode + topYangNode.Parent = doc + if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS { + continue + } + } + + + //remove last comma in case mulitple values returned + if (singleLeaf != "") { + return singleLeaf[:len(singleLeaf) - 1] + } + + return "" +} + //Check delete constraint for leafref if key/field is deleted func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData, tableName, keyVal, field string) CVLRetCode { diff --git a/cvl/internal/util/util.go b/cvl/internal/util/util.go index 52c81763552c..aee1666e2b8b 100644 --- a/cvl/internal/util/util.go +++ b/cvl/internal/util/util.go @@ -133,6 +133,10 @@ func IsTraceSet() bool { } } +func IsTraceLevelSet(tracelevel CVLTraceLevel) bool { + return (cvlTraceFlags & (uint32)(tracelevel)) != 0 +} + func TRACE_LEVEL_LOG(level log.Level, tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) { if (IsTraceSet() == false) { diff --git a/patches/apply.sh b/patches/apply.sh index 37e382f002ec..8d76c6329ae7 100755 --- a/patches/apply.sh +++ b/patches/apply.sh @@ -43,6 +43,8 @@ patch -d ${DEST_DIR}/github.com/openconfig -p1 < ${PATCH_DIR}/ygot/ygot.patch patch -d ${DEST_DIR}/github.com/openconfig/goyang -p1 < ${PATCH_DIR}/goyang/goyang.patch patch -d ${DEST_DIR}/github.com/antchfx/jsonquery -p1 < ${PATCH_DIR}/jsonquery.patch +patch -d ${DEST_DIR}/github.com/antchfx/xmlquery -p1 < ${PATCH_DIR}/xmlquery.patch +patch -d ${DEST_DIR}/github.com/antchfx/xpath -p1 < ${PATCH_DIR}/xpath.patch patch -d ${DEST_DIR}/github.com/golang/glog -p1 < ${PATCH_DIR}/glog.patch diff --git a/patches/jsonquery.patch b/patches/jsonquery.patch index 122ef8e01416..cca6140eccc8 100644 --- a/patches/jsonquery.patch +++ b/patches/jsonquery.patch @@ -1,8 +1,70 @@ diff --git a/node.go b/node.go -index 76032bb..db73a1e 100644 +index 76032bb..f6103d9 100644 --- a/node.go +++ b/node.go -@@ -155,3 +155,9 @@ func Parse(r io.Reader) (*Node, error) { +@@ -8,6 +8,7 @@ import ( + "net/http" + "sort" + "strconv" ++ "strings" + ) + + // A NodeType is the type of a Node. +@@ -110,6 +111,29 @@ func parseValue(x interface{}, top *Node, level int) { + addNode(n) + parseValue(vv, n, level+1) + } ++ case map[string]string: ++ var keys []string ++ for key := range v { ++ keys = append(keys, key) ++ } ++ sort.Strings(keys) ++ for _, key := range keys { ++ tmpKey := key ++ var tmpVal interface{} ++ tmpVal = v[key] ++ if (strings.HasSuffix(key, "@")) { ++ tmpKey = key[:len(key) - 1] ++ tmpValArr := []interface{}{} ++ for _, val := range strings.Split(v[key], ",") { ++ tmpValArr = append(tmpValArr, val) ++ } ++ tmpVal = tmpValArr ++ } ++ ++ n := &Node{Data: tmpKey, Type: ElementNode, level: level} ++ addNode(n) ++ parseValue(tmpVal, n, level+1) ++ } + case map[string]interface{}: + // The Go’s map iteration order is random. + // (https://blog.golang.org/go-maps-in-action#Iteration-order) +@@ -119,9 +143,21 @@ func parseValue(x interface{}, top *Node, level int) { + } + sort.Strings(keys) + for _, key := range keys { +- n := &Node{Data: key, Type: ElementNode, level: level} ++ tmpKey := key ++ var tmpVal interface{} ++ tmpVal = v[key] ++ if (strings.HasSuffix(key, "@")) { ++ tmpKey = key[:len(key) - 1] ++ tmpValArr := []interface{}{} ++ for _, val := range strings.Split(v[key].(string), ",") { ++ tmpValArr = append(tmpValArr, val) ++ } ++ tmpVal = tmpValArr ++ } ++ ++ n := &Node{Data: tmpKey, Type: ElementNode, level: level} + addNode(n) +- parseValue(v[key], n, level+1) ++ parseValue(tmpVal, n, level+1) + } + case string: + n := &Node{Data: v, Type: TextNode, level: level} +@@ -155,3 +191,9 @@ func Parse(r io.Reader) (*Node, error) { } return parse(b) } @@ -12,3 +74,22 @@ index 76032bb..db73a1e 100644 + parseValue(*jsonMap, doc, 1) + return doc, nil +} +diff --git a/query.go b/query.go +index d105962..e8db1d6 100644 +--- a/query.go ++++ b/query.go +@@ -120,6 +120,14 @@ func (a *NodeNavigator) MoveToRoot() { + a.cur = a.root + } + ++func (a *NodeNavigator) MoveToContext() { ++ return ++} ++ ++func (a *NodeNavigator) CurrentPrefix() string { ++ return "" ++} ++ + func (a *NodeNavigator) MoveToParent() bool { + if n := a.cur.Parent; n != nil { + a.cur = n diff --git a/patches/xmlquery.patch b/patches/xmlquery.patch new file mode 100644 index 000000000000..fbb218e8492f --- /dev/null +++ b/patches/xmlquery.patch @@ -0,0 +1,94 @@ +diff --git a/node.go b/node.go +index e86c0c3..028867c 100644 +--- a/node.go ++++ b/node.go +@@ -48,7 +48,7 @@ type Node struct { + + // InnerText returns the text between the start and end tags of the object. + func (n *Node) InnerText() string { +- var output func(*bytes.Buffer, *Node) ++ /*var output func(*bytes.Buffer, *Node) + output = func(buf *bytes.Buffer, n *Node) { + switch n.Type { + case TextNode: +@@ -64,7 +64,18 @@ func (n *Node) InnerText() string { + + var buf bytes.Buffer + output(&buf, n) +- return buf.String() ++ return buf.String()*/ ++ ++ if (n.Type == TextNode) { ++ return n.Data ++ } else if (n.Type == ElementNode) && ++ (n.FirstChild != nil) && ++ (n.FirstChild.Type == TextNode) { ++ return n.FirstChild.Data ++ } ++ ++ ++ return "" + } + + func (n *Node) sanitizedData(preserveSpaces bool) string { +diff --git a/query.go b/query.go +index 146c2a4..f21b61b 100644 +--- a/query.go ++++ b/query.go +@@ -49,6 +49,29 @@ func CreateXPathNavigator(top *Node) *NodeNavigator { + return &NodeNavigator{curr: top, root: top, attr: -1} + } + ++//Evaluate XPath expression, the expression should evaluate to true or false ++func Eval(top, ctx *Node, exp *xpath.Expr) bool { ++ if exp == nil { ++ return false ++ } ++ ++ v := exp.Evaluate(&NodeNavigator{curr: ctx, ctxt: ctx, root: top, attr: -1}) ++ ++ switch val := v.(type) { ++ case bool: ++ return val ++ case string: ++ return (val != "") ++ case float64: ++ return (val != 0) ++ case *xpath.NodeIterator: ++ return (val != nil) ++ } ++ ++ //return v.(bool) ++ return false ++} ++ + func getCurrentNode(it *xpath.NodeIterator) *Node { + n := it.Current().(*NodeNavigator) + if n.NodeType() == xpath.AttributeNode { +@@ -145,7 +168,7 @@ func FindEachWithBreak(top *Node, expr string, cb func(int, *Node) bool) { + } + + type NodeNavigator struct { +- root, curr *Node ++ root, curr, ctxt *Node + attr int + } + +@@ -212,6 +235,17 @@ func (x *NodeNavigator) MoveToRoot() { + x.curr = x.root + } + ++func (x *NodeNavigator) MoveToContext() { ++ x.curr = x.ctxt ++} ++ ++func (x *NodeNavigator) CurrentPrefix() string { ++ if (x.ctxt != nil) { ++ return x.ctxt.Prefix ++ } ++ return "" ++} ++ + func (x *NodeNavigator) MoveToParent() bool { + if x.attr != -1 { + x.attr = -1 diff --git a/patches/xpath.patch b/patches/xpath.patch new file mode 100644 index 000000000000..704d60fd1601 --- /dev/null +++ b/patches/xpath.patch @@ -0,0 +1,1159 @@ +diff --git a/build.go b/build.go +index 74f266b..d7d3aab 100644 +--- a/build.go ++++ b/build.go +@@ -44,7 +44,9 @@ func axisPredicate(root *axisNode) func(NodeNavigator) bool { + predicate := func(n NodeNavigator) bool { + if typ == n.NodeType() || typ == allNode || typ == TextNode { + if nametest { +- if root.LocalName == n.LocalName() && root.Prefix == n.Prefix() { ++ prefix := n.Prefix() ++ if root.LocalName == n.LocalName() && ++ ((root.Prefix == prefix) || (prefix == n.CurrentPrefix())) { + return true + } + } else { +@@ -107,7 +109,7 @@ func (b *builder) processAxisNode(root *axisNode) (query, error) { + } + return v + } +- qyOutput = &childQuery{Input: qyInput, Predicate: filter} ++ qyOutput = &childQuery{Name: &root.LocalName, Prefix: &root.Prefix, Input: qyInput, Predicate: filter} + case "descendant": + qyOutput = &descendantQuery{Input: qyInput, Predicate: predicate} + case "descendant-or-self": +@@ -130,9 +132,66 @@ func (b *builder) processAxisNode(root *axisNode) (query, error) { + err = fmt.Errorf("unknown axe type: %s", root.AxeType) + return nil, err + } ++ ++ b.setCaller(qyInput, qyOutput) ++ + return qyOutput, nil + } + ++func isKey(keys []string, name string) int { ++ for idx := 0; idx < len(keys); idx++ { ++ if keys[idx] == name { ++ return idx ++ } ++ } ++ ++ return -1 ++} ++ ++func (b *builder) setCaller(callee, caller query) { ++ ++ isCallerFilterQ := false ++ ++ switch typ := caller.(type) { ++ case *parentQuery: ++ isCallerFilterQ = typ.SFilter.UnderFilter ++ case *childQuery: ++ isCallerFilterQ = typ.SFilter.UnderFilter ++ case *booleanQuery: ++ isCallerFilterQ = typ.SFilter.UnderFilter ++ case *functionQuery: ++ isCallerFilterQ = typ.SFilter.UnderFilter ++ case *logicalQuery: ++ isCallerFilterQ = typ.SFilter.UnderFilter ++ case *filterQuery: ++ isCallerFilterQ = true ++ } ++ ++ switch typ := callee.(type) { ++ case *parentQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ case *childQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ case *booleanQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ case *functionQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ case *logicalQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ case *filterQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ case *currentQuery: ++ typ.Caller = caller ++ typ.SFilter.UnderFilter = isCallerFilterQ ++ } ++} ++ + // processFilterNode builds query for the XPath filter predicate. + func (b *builder) processFilterNode(root *filterNode) (query, error) { + b.flag |= filterFlag +@@ -146,6 +205,10 @@ func (b *builder) processFilterNode(root *filterNode) (query, error) { + return nil, err + } + qyOutput := &filterQuery{Input: qyInput, Predicate: qyCond} ++ ++ b.setCaller(qyCond, qyOutput) ++ b.setCaller(qyInput, qyOutput) ++ + return qyOutput, nil + } + +@@ -339,6 +402,9 @@ func (b *builder) processFunctionNode(root *functionNode) (query, error) { + return nil, err + } + qyOutput = &functionQuery{Input: argQuery, Func: countFunc} ++ ++ b.setCaller(argQuery, qyOutput) ++ + case "sum": + if len(root.Args) == 0 { + return nil, fmt.Errorf("xpath: sum(node-sets) function must with have parameters node-sets") +@@ -379,6 +445,10 @@ func (b *builder) processFunctionNode(root *functionNode) (query, error) { + args = append(args, q) + } + qyOutput = &functionQuery{Input: b.firstInput, Func: concatFunc(args...)} ++ case "current": ++ qyOutput = &functionQuery{Input: ¤tQuery{}, Func: currentFunc} ++ b.setCaller(qyOutput.(*functionQuery).Input, qyOutput) ++ + default: + return nil, fmt.Errorf("not yet support this function %s()", root.FuncName) + } +@@ -396,13 +466,15 @@ func (b *builder) processOperatorNode(root *operatorNode) (query, error) { + } + var qyOutput query + switch root.Op { +- case "+", "-", "div", "mod": // Numeric operator ++ case "+", "-", "*", "div", "mod": // Numeric operator + var exprFunc func(interface{}, interface{}) interface{} + switch root.Op { + case "+": + exprFunc = plusFunc + case "-": + exprFunc = minusFunc ++ case "*": ++ exprFunc = mulFunc + case "div": + exprFunc = divFunc + case "mod": +@@ -435,6 +507,10 @@ func (b *builder) processOperatorNode(root *operatorNode) (query, error) { + case "|": + qyOutput = &unionQuery{Left: left, Right: right} + } ++ ++ b.setCaller(left, qyOutput) ++ b.setCaller(right, qyOutput) ++ + return qyOutput, nil + } + +diff --git a/func.go b/func.go +index a2f0dce..4abde27 100644 +--- a/func.go ++++ b/func.go +@@ -21,6 +21,11 @@ func predicate(q query) func(NodeNavigator) bool { + return func(NodeNavigator) bool { return true } + } + ++// currentFunc is a XPath Node Set functions current(). ++func currentFunc(q query, t iterator) interface{} { ++ return q.Evaluate(t) ++} ++ + // positionFunc is a XPath Node Set functions position(). + func positionFunc(q query, t iterator) interface{} { + var ( +@@ -58,16 +63,59 @@ func lastFunc(q query, t iterator) interface{} { + // countFunc is a XPath Node Set functions count(node-set). + func countFunc(q query, t iterator) interface{} { + var count = 0 ++ var fQuery *functionQuery = nil ++ ++ switch qtyp := q.(type) { ++ case *childQuery: ++ switch pQtyp := qtyp.Caller.(type) { ++ case *functionQuery: ++ fQuery = pQtyp ++ } ++ case *filterQuery: ++ switch pQtyp := qtyp.Caller.(type) { ++ case *functionQuery: ++ fQuery = pQtyp ++ } ++ } ++ ++ //Reset count first ++ if (fQuery != nil) { ++ fQuery.CountFuncVal = 0 ++ } ++ + test := predicate(q) + switch typ := q.Evaluate(t).(type) { + case query: + for node := typ.Select(t); node != nil; node = typ.Select(t) { ++ ++ tmpNode := node.Copy() ++ if (strings.HasSuffix(node.LocalName(), "_LIST") == false) { ++ //Go to leaf's parent i.e. list ++ tmpNode.MoveToParent() ++ } ++ //Go to 2nd attribute ++ if tmpNode.MoveToNextAttribute() && tmpNode.MoveToNextAttribute() { ++ if (tmpNode.LocalName() == "db") { ++ //Don't count list entry brought from db ++ //which is already counted during path evaluation ++ continue ++ } ++ ++ } ++ + if test(node) { + count++ + } + } + } +- return float64(count) ++ if (fQuery != nil) && (fQuery.CountFuncVal > 0) { ++ // -1 since first data always gets added before ++ //starting xpath evaluation, then count is fetched from Redis ++ return (float64(count) + (fQuery.CountFuncVal)) ++ ++ } else { ++ return float64(count) ++ } + } + + // sumFunc is a XPath Node Set functions sum(node-set). +diff --git a/operator.go b/operator.go +index 308d3cb..e40b4b7 100644 +--- a/operator.go ++++ b/operator.go +@@ -45,7 +45,13 @@ var logicalFuncs = [][]logical{ + } + + // number vs number +-func cmpNumberNumberF(op string, a, b float64) bool { ++func cmpNumberNumberF(op string, a, b float64) (r bool) { ++ defer func() { ++ res := &r ++ Log("cmpNumberNumberF(): (%f %s %f) = %t", ++ a, op, b, *res) ++ }() ++ + switch op { + case "=": + return a == b +@@ -64,7 +70,13 @@ func cmpNumberNumberF(op string, a, b float64) bool { + } + + // string vs string +-func cmpStringStringF(op string, a, b string) bool { ++func cmpStringStringF(op string, a, b string)(r bool) { ++ defer func(){ ++ res := &r ++ Log("cmpStringStringF(): ('%v' %s '%v') == %v", ++ a, op, b, *res) ++ }() ++ + switch op { + case "=": + return a == b +@@ -82,7 +94,13 @@ func cmpStringStringF(op string, a, b string) bool { + return false + } + +-func cmpBooleanBooleanF(op string, a, b bool) bool { ++func cmpBooleanBooleanF(op string, a, b bool) (r bool) { ++ defer func(){ ++ res := &r ++ Log("cmpBooleanBooleanF(): (%v %s %v) = %v", ++ a, op, b, *res) ++ }() ++ + switch op { + case "or": + return a || b +@@ -163,6 +181,48 @@ func cmpNodeSetString(t iterator, op string, m, n interface{}) bool { + } + + func cmpNodeSetNodeSet(t iterator, op string, m, n interface{}) bool { ++ a := m.(query) ++ b := n.(query) ++ ++ for { ++ node := a.Select(t) ++ if node == nil { ++ break ++ } ++ ++ b.Evaluate(t) ++ for { ++ node1 := b.Select(t) ++ if node1 == nil { ++ break ++ } ++ opnd1 := node.Value() ++ opnd2 := node1.Value() ++ ++ Log("cmpNodeSetNodeSet(): Comparing (%v %v %v)", opnd1, op, opnd2) ++ ++ //Check if both are number, then use cmpNumericNumeric ++ num1, err1 := strconv.ParseFloat(opnd1, 64) ++ if err1 == nil { ++ num2, err2 := strconv.ParseFloat(opnd2, 64) ++ if err2 == nil { ++ if cmpNumberNumberF(op, num1, num2) { ++ return true ++ } ++ } else { ++ if cmpStringStringF(op, opnd1, opnd2) { ++ return true ++ } ++ } ++ } else { ++ // ++ if cmpStringStringF(op, opnd1, opnd2) { ++ return true ++ } ++ } ++ } ++ } ++ + return false + } + +diff --git a/query.go b/query.go +index afeb890..d3322cf 100644 +--- a/query.go ++++ b/query.go +@@ -1,10 +1,12 @@ + package xpath +- + import ( + "bytes" + "fmt" + "hash/fnv" + "reflect" ++ "runtime" ++ "strings" ++ "regexp" + ) + + type iterator interface { +@@ -33,10 +35,251 @@ func (nopQuery) Evaluate(iterator) interface{} { return nil } + + func (nopQuery) Clone() query { return nopQuery{} } + ++type currentQuery struct { ++ posit int ++ iterator func() NodeNavigator ++ Caller query ++ SFilter scriptFilter ++} ++ ++//Set lua script in query ++func setScriptFilter(q query, val interface{}) (sFilter *scriptFilter) { ++ var sf *scriptFilter = nil ++ ++ defer func() { ++ if (sFilter != nil) && (sFilter.Predicate != "") { ++ Log("setScriptFilter() : Filter : %v", sFilter.Predicate) ++ } ++ }() ++ ++ switch typ := q.(type) { ++ case *currentQuery: ++ if (val != nil) { ++ typ.SFilter.Predicate = fmt.Sprintf("%v", val) ++ } ++ ++ sf = &typ.SFilter ++ case *childQuery: ++ ++ //Reach to the parent list or first filter and then parent list ++ var caller query = typ.Caller ++ fQuery := false ++ ++ for (caller != nil) { ++ var fieldVal reflect.Value ++ if (reflect.TypeOf(caller) == reflect.TypeOf(&filterQuery{})) { ++ fieldVal = reflect.ValueOf(caller).Elem().FieldByName("Input") ++ fQuery = true ++ } else { ++ //Once filter query is reached, look for child query only ++ if (fQuery == true) && ++ (reflect.TypeOf(caller) != reflect.TypeOf(&childQuery{})) { ++ caller = nil ++ break ++ } ++ ++ fieldVal = reflect.ValueOf(caller).Elem().FieldByName("Caller") ++ ++ } ++ if fieldVal.IsValid() && fieldVal.IsNil() == false { ++ //caller = caller.Caller ... previous caller ++ caller = fieldVal.Interface().(query) ++ } else { ++ caller = nil ++ break ++ } ++ ++ nameVal := reflect.ValueOf(caller).Elem().FieldByName("Name") ++ if nameVal.IsValid() && ++ strings.HasSuffix(nameVal.Elem().String(), "_LIST") { ++ //List element found ++ break ++ } else if (fQuery == true) { ++ //If filter query is found but list is not found break ++ caller = nil ++ break ++ } ++ } ++ ++ if (caller != nil) { ++ switch typ1 := caller.(type) { ++ case *childQuery: ++ listName := *typ1.Name ++ if (strings.HasSuffix(listName, "_LIST") == false) { ++ break ++ } ++ ++ keyNames := getKeysClbk(listName[:len(listName)-5]) ++ if (typ1.SFilter.Key == nil) { ++ typ1.SFilter.Key = make([]string, len(keyNames)) ++ } ++ ++ isChildAKey := false ++ for idx:=0; idx < len(keyNames); idx++ { ++ typ1.SFilter.Key[idx] = "*" ++ if (keyNames[idx] == *typ.Name) { //check with child name ++ //typ1.SFilter.Key[idx] = typ.SFilter.Predicate ++ isChildAKey = true ++ }/* else { ++ typ1.SFilter.Key[idx] = "*" ++ }*/ ++ } ++ ++ if *(typ1.Prefix) == *(typ.Prefix) { ++ //Check parent caller is a parent query ++ switch typ.Input.(type) { ++ case *functionQuery: // current()/ ++ break ++ case *parentQuery: // current()/../ ++ //Just break, since the value should be used for ++ //filter script ++ break ++ default: ++ if (isChildAKey == true) { ++ //match with keys ++ typ.SFilter.Predicate = "k['" + *(typ.Name) + "']" ++ } else { ++ //match with hash-field ++ typ.SFilter.Predicate = "h['" + *(typ.Name) + "']" ++ } ++ } ++ } ++ } ++ } else { ++ typ.SFilter.Predicate = *(typ.Name) ++ } ++ ++ sf = &typ.SFilter ++ case *constantQuery: ++ if (val != nil) { ++ typ.SFilter.Predicate = fmt.Sprintf("%v", val) ++ } ++ ++ sf = &typ.SFilter ++ case *logicalQuery: ++ sfl := setScriptFilter(typ.Left, nil) ++ sfr := setScriptFilter(typ.Right, nil) ++ op := "" ++ ++ lFunc := runtime.FuncForPC(reflect.ValueOf(typ.Do).Pointer()).Name() ++ ++ //Check which logical operator ++ switch lFunc[len(lFunc) - 6:] { //take out 'xpath:' prefix ++ case "eqFunc": ++ op = " == " ++ } ++ if (sfl != nil && sfr != nil) { ++ if (strings.Contains(sfr.Predicate, ",")) { //multi value ++ //typ.SFilter.Predicate = fmt.Sprintf( ++ // "(string.match('%s', %s..'[,]*') ~= nil)", ++ // sfr.Predicate, sfl.Predicate) ++ typ.SFilter.Predicate = "(string.match('" + ++ sfr.Predicate + "', " + sfl.Predicate + "..'[,]*') ~= nil)" ++ } else { ++ //typ.SFilter.Predicate = fmt.Sprintf("(%s %s '%s')", ++ //sfl.Predicate, op, sfr.Predicate) ++ typ.SFilter.Predicate = "(" + sfl.Predicate + op + "'" + ++ sfr.Predicate + "')" ++ } ++ } ++ ++ sf = &typ.SFilter ++ case *booleanQuery: ++ sfl := setScriptFilter(typ.Left, nil) ++ sfr := setScriptFilter(typ.Right, nil) ++ op := "and" ++ ++ if (typ.IsOr == true) { ++ op = "or" ++ } ++ ++ if (sfl != nil && sfr != nil) { ++ typ.SFilter.Predicate = fmt.Sprintf("(%s %s %s)", ++ sfl.Predicate, op, sfr.Predicate) ++ } ++ ++ sf = &typ.SFilter ++ ++ case *functionQuery: ++ sfi := setScriptFilter(typ.Input, nil) ++ if (sfi != nil) { ++ switch typ.Input.(type) { ++ case *currentQuery: ++ typ.SFilter.Predicate = sfi.Predicate ++ } ++ } ++ ++ sf = &typ.SFilter ++ } ++ ++ return sf ++} ++ ++// currentQuery returns the current context node under which the xpath query is invoked ++func (c *currentQuery) Select(t iterator) (node NodeNavigator) { ++ if c.iterator == nil { ++ c.posit = 0 ++ node = t.Current().Copy() ++ if node == nil { ++ return nil ++ } ++ ++ //Current node is the root node i.e. under which the xpath query is invoked ++ node.MoveToContext() ++ first := true ++ rootLocalName := node.LocalName() ++ rootPrefix := node.Prefix() ++ ++ c.iterator = func() NodeNavigator { ++ for { ++ if (first && node == nil) || (!first && !node.MoveToNext()) { ++ return nil ++ } ++ ++ first = false ++ ++ nodeLocalName := node.LocalName() ++ ++ if (rootLocalName == nodeLocalName) && ++ (rootPrefix == node.Prefix()) { ++ return node ++ } ++ ++ //Other node started ++ if (nodeLocalName[0:1] != "\n") && (nodeLocalName[0:1] != " ") { ++ return nil ++ } ++ } ++ } ++ } ++ ++ if n := c.iterator(); n != nil { ++ c.posit++ ++ setScriptFilter(c, n.Value()) ++ ++ return n ++ } ++ ++ c.iterator = nil ++ ++ return nil ++} ++ ++func (c *currentQuery) Evaluate(iterator) interface{} { ++ c.posit = 0 ++ c.iterator = nil ++ return c ++} ++ ++func (c *currentQuery) Clone() query { ++ return ¤tQuery{posit: c.posit} ++} ++ + // contextQuery is returns current node on the iterator object query. + type contextQuery struct { + count int + Root bool // Moving to root-level node in the current context iterator. ++ Caller query + } + + func (c *contextQuery) Select(t iterator) (n NodeNavigator) { +@@ -164,13 +407,103 @@ func (a *attributeQuery) Clone() query { + return &attributeQuery{Input: a.Input.Clone(), Predicate: a.Predicate} + } + ++type scriptFilter struct { ++ Key []string //all keys in slice |*|Key2|* ++ Predicate string //Keys and fields as lua condition in filter ++ // - (h.Key1 == 'Test' and h.Field1 == 'test') ++ Fields string //Fields to retrieve |Field1|Field2|Field3| ++ UnderFilter bool //If the query is within filter query ++} ++ + // childQuery is an XPath child node query.(child::*) + type childQuery struct { ++ Name *string ++ Prefix *string + posit int + iterator func() NodeNavigator +- + Input query + Predicate func(NodeNavigator) bool ++ Caller query ++ SFilter scriptFilter ++ execFilter bool // Execute filter only once during expression ++} ++ ++//Check for count("../TABLE_LIST"), count("../TABLE_LIST/field"), ++//count("../TABLE_LIST[key='val']"), count ("../TABLE_LIST[key='val']/field") ++func checkIfCountFunc(q query) (*functionQuery, string) { ++ if (q == nil) { ++ return nil, "" ++ } ++ ++ switch typ := q.(type) { ++ case *childQuery: ++ if (strings.HasSuffix(*typ.Name, "_LIST")) { ++ return checkIfCountFunc(typ.Caller) ++ } else { ++ funcQ, _ := checkIfCountFunc(typ.Caller) ++ return funcQ, *typ.Name ++ } ++ case *filterQuery: ++ return checkIfCountFunc(typ.Caller) ++ case *functionQuery: ++ funcName := runtime.FuncForPC(reflect.ValueOf(typ.Func).Pointer()).Name() ++ ++ if (strings.HasSuffix(funcName, ".countFunc")) { //take out 'xpath:' prefix ++ return typ, "" ++ } ++ return nil, "" ++ } ++ ++ return nil, "" ++} ++ ++//Execute filter if no predicate is provided ++func executeFilterWithoutPred(c *childQuery) { ++ ++ //Should be a list and filter should have not excuted already ++ if ((strings.HasSuffix(*c.Name, "_LIST")) == false) || ++ (c.execFilter == true) { ++ return ++ } ++ ++ //Should not be a filterQuery i.e. no predicate ++ if (c.Caller == nil) || ++ (reflect.TypeOf(c.Caller) == reflect.TypeOf(&filterQuery{})) { ++ return ++ } ++ ++ funcQ, fieldName := checkIfCountFunc(c) ++ ++ listName := *c.Name ++ listName = listName[:len(listName) - len("_LIST")] ++ ++ redisTblname := "" ++ switch typContainer := c.Input.(type) { ++ case *childQuery: ++ //Get table name from container ++ redisTblname = *typContainer.Name ++ default: ++ //Get table name from list name ++ redisTblname = listName ++ } ++ ++ keys := getKeysClbk(listName) ++ keyNames := "" ++ ++ if (keys != nil) { ++ keyNames = strings.Join(keys, "|") ++ } ++ ++ if (funcQ != nil) { //Within a count function ++ funcQ.CountFuncVal = getDepDataCntClbk(depDataCtxt, ++ redisTblname + "|*", keyNames, "", fieldName) ++ } else { ++ getDepDataClbk(depDataCtxt, []string{}, ++ redisTblname + "|*", ++ keyNames, "true", "", "") ++ } ++ ++ c.execFilter = true + } + + func (c *childQuery) Select(t iterator) NodeNavigator { +@@ -181,6 +514,10 @@ func (c *childQuery) Select(t iterator) NodeNavigator { + if node == nil { + return nil + } ++ ++ //Execute filter without any filterQuery ++ executeFilterWithoutPred(c) ++ + node = node.Copy() + first := true + c.iterator = func() NodeNavigator { +@@ -198,6 +535,10 @@ func (c *childQuery) Select(t iterator) NodeNavigator { + + if node := c.iterator(); node != nil { + c.posit++ ++ if (c.SFilter.Fields != *c.Name) { ++ //If fields already retrieved from db don't overwrite ++ c.SFilter.Predicate = node.Value() ++ } + return node + } + c.iterator = nil +@@ -207,6 +548,12 @@ func (c *childQuery) Select(t iterator) NodeNavigator { + func (c *childQuery) Evaluate(t iterator) interface{} { + c.Input.Evaluate(t) + c.iterator = nil ++ ++ //Reset execFilter flag in LIST node ++ if (strings.HasSuffix(*c.Name, "_LIST")) { ++ c.execFilter = false ++ } ++ + return c + } + +@@ -215,7 +562,7 @@ func (c *childQuery) Test(n NodeNavigator) bool { + } + + func (c *childQuery) Clone() query { +- return &childQuery{Input: c.Input.Clone(), Predicate: c.Predicate} ++ return &childQuery{Name: c.Name, Prefix: c.Prefix, Input: c.Input.Clone(), Predicate: c.Predicate} + } + + // position returns a position of current NodeNavigator. +@@ -473,6 +820,8 @@ func (p *precedingQuery) position() int { + type parentQuery struct { + Input query + Predicate func(NodeNavigator) bool ++ Caller query ++ SFilter scriptFilter + } + + func (p *parentQuery) Select(t iterator) NodeNavigator { +@@ -505,6 +854,8 @@ func (p *parentQuery) Test(n NodeNavigator) bool { + type selfQuery struct { + Input query + Predicate func(NodeNavigator) bool ++ Caller query ++ SFilter scriptFilter + } + + func (s *selfQuery) Select(t iterator) NodeNavigator { +@@ -538,10 +889,14 @@ type filterQuery struct { + Input query + Predicate query + posit int ++ //execFilter bool // Execute filter only once during expression ++ Caller query ++ SFilter scriptFilter + } + + func (f *filterQuery) do(t iterator) bool { + val := reflect.ValueOf(f.Predicate.Evaluate(t)) ++ + switch val.Kind() { + case reflect.Bool: + return val.Bool() +@@ -562,21 +917,198 @@ func (f *filterQuery) position() int { + return f.posit + } + ++var reSanitizer *regexp.Regexp = nil ++var reKeyPred *regexp.Regexp = nil ++func init() { ++ reSanitizer = regexp.MustCompile(`(\()|(\))|(')`) // remove (, ) and ' ++ //Regexp to check (k.k1 == 'val1' and k.k2 == 'val2') like pattern ++ reKeyPred = regexp.MustCompile(`(k\.([a-zA-Z0-9-_]+)( == )(.*)( and )?)+`) ++} ++ ++func checkForKeyFilter(keys []string, predicate string) string { ++ ++ if (predicate == "") { ++ return "" ++ } ++ ++ tmpPredicate := reSanitizer.ReplaceAllLiteralString(predicate, "") ++ ++ if (reKeyPred.MatchString(tmpPredicate) == false) { ++ return "" ++ } ++ ++ keyFilter := make([]string, len(keys)) ++ for idx :=0 ; idx < len(keys); idx++ { ++ keyFilter[idx] = "*" //fill with default pattern ++ } ++ ++ //All are keys in predicate, form the key filter and ++ //remove generic predicate ++ ++ //first split by 'and' ++ for _, keyEqExpr := range strings.Split(tmpPredicate, " and ") { ++ //Then split by '==' ++ keyValPair := strings.Split(keyEqExpr, " == ") ++ ++ if (len(keyValPair) != 2) { ++ return "" ++ } ++ ++ //Check with which key it does match ++ for idx, key := range keys { ++ tmpKey := "k." + key ++ if (tmpKey == keyValPair[0]) { //matching with left side of == ++ keyFilter[idx] = keyValPair[1] ++ break ++ } else if (tmpKey == keyValPair[1]) { //matching with right side of == ++ keyFilter[idx] = keyValPair[0] ++ break ++ } ++ } ++ } ++ ++ return strings.Join(keyFilter, "|") ++} ++ ++//Excute filter to get dependent data ++func executeFilter(f *filterQuery) { ++ switch typ := f.Predicate.(type) { ++ case *logicalQuery: ++ f.SFilter = typ.SFilter ++ Log("executeFilter(): getting filtered data, predicate=%s", f.SFilter.Predicate) ++ case *booleanQuery: ++ f.SFilter = typ.SFilter ++ Log("executeFilter(): getting filtered data, predicate=%s", f.SFilter.Predicate) ++ } ++ //All filter must be executed only once ++ //Need to excute filter script and then collect leaf data ++ //for multiple data it should be separated by "," ++ //Filter script should have - strings.match("Eth1,Eth2,Eth4", "Eth1[,]*)) ++ ++ //Get leaf data ++ var cq *childQuery = nil ++ if (f.Caller != nil) { ++ switch typ := f.Caller.(type) { ++ case *childQuery: ++ cq = typ ++ } ++ } ++ ++ if (f.Input != nil) { ++ switch typ := f.Input.(type) { ++ case *childQuery: ++ ++ listName := *typ.Name ++ listName = listName[:len(listName) - len("_LIST")] ++ ++ redisTblname := "" ++ switch typContainer := typ.Input.(type) { ++ case *childQuery: ++ //Get table name from container ++ redisTblname = *typContainer.Name ++ default: ++ //Get table name from list name ++ redisTblname = listName ++ } ++ ++ keys := getKeysClbk(listName) ++ keyNames := "" ++ ++ if (keys != nil) { ++ keyNames = strings.Join(keys, "|") ++ } ++ ++ //Check the predicate, if all fields are ++ //key and having equality check with 'and', ++ //should use as key filter ++ keyFilter := checkForKeyFilter(keys, f.SFilter.Predicate) ++ if (keyFilter != "") { ++ f.SFilter.Predicate = "" ++ keyFilter = redisTblname + "|" + keyFilter ++ } else { ++ keyFilter = redisTblname + "|*" ++ } ++ ++ ++ ++ //Check if count(), just store the count from ++ //Redis without fetching data ++ funcQ, fieldName := checkIfCountFunc(f) ++ if (funcQ != nil) { ++ funcQ.CountFuncVal = getDepDataCntClbk(depDataCtxt, ++ keyFilter, keyNames, ++ f.SFilter.Predicate, fieldName) ++ ++ return ++ } ++ ++ if (cq != nil) { ++ //Child query through filter ++ cq.SFilter.Fields = *cq.Name ++ ++ data := getDepDataClbk(depDataCtxt, []string{}, ++ keyFilter, ++ keyNames, ++ f.SFilter.Predicate, ++ cq.SFilter.Fields, "") ++ ++ //Set single field data to child ++ cq.SFilter.Predicate = data ++ Log("executeFilter(): value returned for '%s' is '%s' " + ++ "inside predicate", cq.SFilter.Fields, data) ++ } else { ++ //Just filter query ++ getDepDataClbk(depDataCtxt, []string{}, ++ keyFilter, ++ keyNames, ++ f.SFilter.Predicate, "", "") ++ } ++ } ++ } ++ ++} ++ + func (f *filterQuery) Select(t iterator) NodeNavigator { + ++ success := false + for { + + node := f.Input.Select(t) + if node == nil { + return node + } ++ + node = node.Copy() + + t.Current().MoveTo(node) ++ + if f.do(t) { + f.posit++ ++ success = true ++ } ++ ++ /* ++ if (f.execFilter == false) { ++ executeFilter(f) ++ f.execFilter = true ++ }*/ ++ ++ if (f.Input != nil) { ++ //check the filter flag in list node ++ switch typ := f.Input.(type) { ++ case *childQuery: ++ if strings.HasSuffix(*typ.Name, "_LIST") && ++ (typ.execFilter == false) { ++ executeFilter(f) ++ typ.execFilter = true ++ } ++ } ++ } ++ ++ if (success == true) { + return node + } ++ + f.posit = 0 + } + } +@@ -595,9 +1127,19 @@ func (f *filterQuery) Clone() query { + type functionQuery struct { + Input query // Node Set + Func func(query, iterator) interface{} // The xpath function. ++ //CVL specific callback for getting Redis count directly ++ //Handles count (//table), count(//table[k1=a][k2=b]) ++ CountFuncVal float64 //count of elements in query ++ Caller query ++ SFilter scriptFilter + } + + func (f *functionQuery) Select(t iterator) NodeNavigator { ++ _, ok := f.Input.(*currentQuery) ++ if ok == true { ++ f.SFilter = f.Input.(*currentQuery).SFilter ++ return f.Input.(*currentQuery).Select(t) ++ } + return nil + } + +@@ -614,6 +1156,8 @@ func (f *functionQuery) Clone() query { + // constantQuery is an XPath constant operand. + type constantQuery struct { + Val interface{} ++ Caller query ++ SFilter scriptFilter + } + + func (c *constantQuery) Select(t iterator) NodeNavigator { +@@ -621,6 +1165,7 @@ func (c *constantQuery) Select(t iterator) NodeNavigator { + } + + func (c *constantQuery) Evaluate(t iterator) interface{} { ++ setScriptFilter(c, c.Val) + return c.Val + } + +@@ -633,6 +1178,8 @@ type logicalQuery struct { + Left, Right query + + Do func(iterator, interface{}, interface{}) interface{} ++ Caller query ++ SFilter scriptFilter + } + + func (l *logicalQuery) Select(t iterator) NodeNavigator { +@@ -649,9 +1196,16 @@ func (l *logicalQuery) Select(t iterator) NodeNavigator { + } + + func (l *logicalQuery) Evaluate(t iterator) interface{} { ++ + m := l.Left.Evaluate(t) + n := l.Right.Evaluate(t) +- return l.Do(t, m, n) ++ v := l.Do(t, m, n) ++ ++ if (l.SFilter.UnderFilter == true) { ++ setScriptFilter(l, nil) ++ } ++ ++ return v + } + + func (l *logicalQuery) Clone() query { +@@ -672,7 +1226,7 @@ func (n *numericQuery) Select(t iterator) NodeNavigator { + func (n *numericQuery) Evaluate(t iterator) interface{} { + m := n.Left.Evaluate(t) + k := n.Right.Evaluate(t) +- return n.Do(m, k) ++ return n.Do(asNumber(t, m), asNumber(t, k)) + } + + func (n *numericQuery) Clone() query { +@@ -683,6 +1237,8 @@ type booleanQuery struct { + IsOr bool + Left, Right query + iterator func() NodeNavigator ++ Caller query ++ SFilter scriptFilter + } + + func (b *booleanQuery) Select(t iterator) NodeNavigator { +@@ -750,14 +1306,26 @@ func (b *booleanQuery) Select(t iterator) NodeNavigator { + } + + func (b *booleanQuery) Evaluate(t iterator) interface{} { ++ + m := b.Left.Evaluate(t) ++ + left := asBool(t, m) ++ ++ if (b.SFilter.UnderFilter == true) { ++ m = b.Right.Evaluate(t) ++ setScriptFilter(b, nil) ++ } ++ + if b.IsOr && left { + return true + } else if !b.IsOr && !left { + return false + } +- m = b.Right.Evaluate(t) ++ ++ if (b.SFilter.UnderFilter == false) { ++ m = b.Right.Evaluate(t) ++ } ++ + return asBool(t, m) + } + +diff --git a/xpath.go b/xpath.go +index d6c9912..2319860 100644 +--- a/xpath.go ++++ b/xpath.go +@@ -44,6 +44,10 @@ type NodeNavigator interface { + // Copy does a deep copy of the NodeNavigator and all its components. + Copy() NodeNavigator + ++ MoveToContext() ++ ++ CurrentPrefix() string ++ + // MoveToRoot moves the NodeNavigator to the root node of the current node. + MoveToRoot() + +@@ -117,6 +121,7 @@ func (f iteratorFunc) Current() NodeNavigator { + // Evaluate returns the result of the expression. + // The result type of the expression is one of the follow: bool,float64,string,NodeIterator). + func (expr *Expr) Evaluate(root NodeNavigator) interface{} { ++ + val := expr.q.Evaluate(iteratorFunc(func() NodeNavigator { return root })) + switch val.(type) { + case query: +@@ -147,11 +152,48 @@ func Compile(expr string) (*Expr, error) { + return &Expr{s: expr, q: qy}, nil + } + ++var getKeysClbk func(string) []string ++var getDepDataClbk func(interface{}, []string, string, string, string, string, string) string ++var getDepDataCntClbk func(interface{}, string, string, string, string) float64 ++var logClbk func(string, ...interface{}) ++var depDataCtxt interface{} ++ + // MustCompile compiles an XPath expression string and ignored error. + func MustCompile(expr string) *Expr { + exp, err := Compile(expr) + if err != nil { + return &Expr{s: expr, q: nopQuery{}} + } ++ + return exp + } ++ ++func SetKeyGetClbk(keyFetchCb func(string) []string) { ++ getKeysClbk = keyFetchCb ++} ++ ++func SetDepDataClbk(ctxt interface{}, ++ depDataCb func(interface{}, []string, string, string, string, string, string) string) { ++ ++ depDataCtxt = ctxt ++ getDepDataClbk = depDataCb ++} ++ ++func SetDepDataCntClbk(ctxt interface{}, ++ depDataCntCb func(interface{}, string, string, string, string) float64) { ++ ++ depDataCtxt = ctxt ++ getDepDataCntClbk = depDataCntCb ++} ++ ++ ++func SetLogCallback(clbk func(string, ...interface{})) { ++ logClbk = clbk ++} ++ ++func Log(fmt string, args...interface{}) { ++ if (logClbk != nil) { ++ logClbk(fmt, args...) ++ } ++} ++