fix bugs and enhance Transformer (sonic-net#35)

Optimize transformer for GET performance
For any request URI and its children, table-name & db-key are queried for the first time and cached internally by the transformer infra. This cache block will be subsequently referred by the infra to avoid redundant callback invocations while the yang model tree is being traversed for GET operation. The cache block is local & valid for a single transaction.
Data from Redis DB is queried only once for a table & cached in the infra for further reference. This cache is local & valid for a single transaction
On-change, sample based subscription support for gNMI
On receipt of subscription request uri, the transformer infra translates it to corresponding Redis Db-table and key, using YANG extensions and provides this info along with other subscription parameters to the translib, which takes care of sending notifications on-change of table/key. YANG extensions introduced : "subscribe-preference", "subscribe-on-change", "subscribe-min-interval".
rpc callback support
RPC statements defined in both OpenConfig and SONiC yang can be annotated with YANG extension “rpc-callback” to invoke the callback to take an action for the given rpc node
YANG default value support
Infra has been enhanced to fill the oc yang defaults by referring internal yang-metadata for terminal nodes, if it is not present in the request payload. These defaults will be pushed along with other payload data when a resource is created.
RFC compliance support - resource check etc.
Parent resource is checked for CRUD and GET requests for the incoming URI, before processing the request. If the parent resource is not available in DB, the transaction is rejected with 404 / "Resource not found" error.
Cascade delete to de-configure all related nodes
Cascade delete enables dependent config clean up for a DELETE request, based on SONiC yang table dependency.
YANG extension "cascade-delete" can be annotated to a node that can be deleted along with all dependent nodes.
value xfmr callback
The value-xfmr callback supports data conversion before writing into the DB for CRUD request and after reading from DB for GET request. Value-xfmr annotated for a leaf will also be invoked, for all other leaves having a leaf-ref to it.
The value-xfmr callback is valid only for the SONIC yang.
Operation and tables ordering, dependent tables watch list
For a set of DB table/instances, grouped by operation, resulting from processing a request URI, infra supports the following :
CRUD Operation ordering,
sequencing tables before operating based on sonic-yang/Redis dependencies
providing translib, with a list of Redis-tables to watch for changes, when performing operations on dependent tables.
Yang model capability
Provide yang module name, organization and version information to translib in order to serve model capability request.
Miscellaneous bug fixes

translib/tlerr/tlerr.go

@@ -103,12 +103,19 @@ func (e TranslibSyntaxValidationError) Error() string {
 }
 
 type TranslibUnsupportedClientVersion struct {
-    ClientVersion string
-    ServerVersion string
-    ServerBaseVersion string
+	ClientVersion string
+	ServerVersion string
+	ServerBaseVersion string
 }
 
 func (e TranslibUnsupportedClientVersion) Error() string {
-    return p.Sprintf("Unsupported client version %s", e.ClientVersion)
+	return p.Sprintf("Unsupported client version %s", e.ClientVersion)
 }
 
+type TranslibXfmrRetError struct {
+     XlateFailDelReq bool
+}
+
+func (e TranslibXfmrRetError) Error() string {
+     return p.Sprintf("Translib transformer return %s", e.XlateFailDelReq)
+}

translib/transformer/transformer.go

@@ -21,7 +21,6 @@ package transformer
 import (
 	"fmt"
 	"github.com/openconfig/goyang/pkg/yang"
-	"github.com/openconfig/ygot/ygot"
 	"os"
 	"strings"
         "bufio"
@@ -30,15 +29,8 @@ import (
 )
 
 var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models path
-
-var entries = map[string]*yang.Entry{}
-
-//Interface for xfmr methods
-type xfmrInterface interface {
-	tableXfmr(s *ygot.GoStruct, t *interface{}) (string, error)
-	keyXfmr(s *ygot.GoStruct, t *interface{}) (string, error)
-	fieldXfmr(s *ygot.GoStruct, t *interface{}) (string, error)
-}
+var ModelsListFile  = "models_list"
+var TblInfoJsonFile = "sonic_table_info.json"
 
 func reportIfError(errs []error) {
 	if len(errs) > 0 {
@@ -50,15 +42,15 @@ func reportIfError(errs []error) {
 
 func getOcModelsList () ([]string) {
     var fileList []string
-    file, err := os.Open(YangPath + "models_list")
+    file, err := os.Open(YangPath + ModelsListFile)
     if err != nil {
         return fileList
     }
     defer file.Close()
     scanner := bufio.NewScanner(file)
     for scanner.Scan() {
         fileEntry := scanner.Text()
-        if strings.HasPrefix(fileEntry, "#") != true {
+        if !strings.HasPrefix(fileEntry, "#") {
             _, err := os.Stat(YangPath + fileEntry)
             if err != nil {
                 continue
@@ -86,9 +78,9 @@ func getDefaultModelsList () ([]string) {
 
 func init() {
 	initYangModelsPath()
-	yangFiles := []string{}
+	initRegex()
         ocList := getOcModelsList()
-        yangFiles = getDefaultModelsList()
+	yangFiles := getDefaultModelsList()
         yangFiles = append(yangFiles, ocList...)
         fmt.Println("Yang model List:", yangFiles)
 	err := loadYangModules(yangFiles...)
@@ -105,7 +97,7 @@ func initYangModelsPath() {
 		YangPath = path
 	}
 
-	fmt.Println("Yang models path:", YangPath)
+	fmt.Println("Yang modles path:", YangPath)
 }
 
 func loadYangModules(files ...string) error {
@@ -147,15 +139,19 @@ func loadYangModules(files ...string) error {
 		}
 	}
 
-	sonic_entries       := make([]*yang.Entry, len(names))
+	sonic_entries       := make([]*yang.Entry, 0)
 	oc_entries          := make(map[string]*yang.Entry)
-	oc_annot_entries    := make([]*yang.Entry, len(names))
-	sonic_annot_entries := make([]*yang.Entry, len(names))
+	oc_annot_entries    := make([]*yang.Entry, 0)
+	sonic_annot_entries := make([]*yang.Entry, 0)
 
 	for _, n := range names {
 		if strings.Contains(n, "annot") && strings.Contains(n, "sonic") {
 			sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(mods[n]))
 		} else if strings.Contains(n, "annot") {
+			yangMdlNmDt := strings.Split(n, "-annot")
+			if len(yangMdlNmDt) > 0 {
+				addMdlCpbltEntry(yangMdlNmDt[0])
+			}
 			oc_annot_entries = append(oc_annot_entries, yang.ToEntry(mods[n]))
 		} else if strings.Contains(n, "sonic") {
 			sonic_entries = append(sonic_entries, yang.ToEntry(mods[n]))
@@ -164,6 +160,37 @@ func loadYangModules(files ...string) error {
 		}
 	}
 
+	// populate model capabilities data
+	for yngMdlNm := range(xMdlCpbltMap) {
+		org := ""
+		ver := ""
+		ocVerSet := false
+		yngEntry := oc_entries[yngMdlNm]
+		if (yngEntry != nil) {
+			// OC yang has version in standard extension oc-ext:openconfig-version
+			if strings.HasPrefix(yngMdlNm, "openconfig-") {
+				for _, ext := range yngEntry.Exts {
+					dataTagArr := strings.Split(ext.Keyword, ":")
+					tagType := dataTagArr[len(dataTagArr)-1]
+					if tagType == "openconfig-version" {
+						ver = ext.NName()
+						fmt.Printf("Found version %v for yang module %v", ver, yngMdlNm)
+						if len(strings.TrimSpace(ver)) > 0 {
+							ocVerSet = true
+						}
+					break
+					}
+
+				}
+			}
+		}
+		if ((strings.HasPrefix(yngMdlNm, "ietf-")) || (!ocVerSet)) {
+			// as per RFC7895 revision date to be used as version
+			ver = mods[yngMdlNm].Current() //gives the most recent revision date for yang module
+		}
+		org = mods[yngMdlNm].Organization.Name
+		addMdlCpbltData(yngMdlNm, ver, org)
+	}
 	dbMapBuild(sonic_entries)
 	annotDbSpecMap(sonic_annot_entries)
 	annotToDbMapBuild(oc_annot_entries)

translib/transformer/xconst.go

@@ -24,6 +24,8 @@ const (
 	YANG_CONTAINER = "container"
 	YANG_LEAF      = "leaf"
 	YANG_LEAF_LIST = "leaf-list"
+	YANG_CHOICE    = "choice"
+	YANG_CASE      = "case"
 
 	YANG_ANNOT_DB_NAME    = "db-name"
 	YANG_ANNOT_TABLE_NAME = "table-name"
@@ -46,6 +48,43 @@ const (
 
 	XPATH_SEP_FWD_SLASH = "/"
 	XFMR_EMPTY_STRING   = ""
+	XFMR_NONE_STRING    = "NONE"
 	SONIC_TABLE_INDEX = 2
+	SONIC_LIST_INDEX = 3
+	SONIC_FIELD_INDEX = 4
+	SONIC_MDL_PFX = "sonic"
+	OC_MDL_PFX = "openconfig-"
+	IETF_MDL_PFX = "ietf-"
+	IANA_MDL_PFX = "iana-"
+	YTDB_KEY_XFMR_RET_ARGS = 2
+	YTDB_KEY_XFMR_RET_VAL_INDX = 0
+	YTDB_KEY_XFMR_RET_ERR_INDX = 1
+	YTDB_SBT_XFMR_RET_ARGS = 2
+	YTDB_SBT_XFMR_RET_VAL_INDX = 0
+	YTDB_SBT_XFMR_RET_ERR_INDX = 1
+	YTDB_FLD_XFMR_RET_ARGS = 2
+	YTDB_FLD_XFMR_RET_VAL_INDX = 0
+	YTDB_FLD_XFMR_RET_ERR_INDX = 1
+	DBTY_KEY_XFMR_RET_ARGS = 2
+	DBTY_KEY_XFMR_RET_VAL_INDX = 0
+	DBTY_KEY_XFMR_RET_ERR_INDX = 1
+	DBTY_FLD_XFMR_RET_ARGS = 2
+	DBTY_FLD_XFMR_RET_VAL_INDX = 0
+	DBTY_FLD_XFMR_RET_ERR_INDX = 1
+	SUBSC_SBT_XFMR_RET_ARGS = 2
+	SUBSC_SBT_XFMR_RET_VAL_INDX = 0
+	SUBSC_SBT_XFMR_RET_ERR_INDX = 1
+	DBTY_SBT_XFMR_RET_ERR_INDX = 0
+	TBL_XFMR_RET_ARGS = 2
+	TBL_XFMR_RET_VAL_INDX = 0
+	TBL_XFMR_RET_ERR_INDX = 1
+	POST_XFMR_RET_ARGS = 2
+	POST_XFMR_RET_VAL_INDX = 0
+	POST_XFMR_RET_ERR_INDX = 1
+	PRE_XFMR_RET_ARGS = 1
+	PRE_XFMR_RET_ERR_INDX = 0
 
+	XFMR_INVALID = -1
+	XFMR_DISABLE = 0
+	XFMR_ENABLE  = 1
 )