From 1d730dfba39f5dbb3d568d88eaef76e53058dd7e Mon Sep 17 00:00:00 2001
From: Sachin Holla <51310506+sachinholla@users.noreply.github.com>
Date: Fri, 31 Jul 2020 04:30:44 +0530
Subject: [PATCH] RFC7895 yang module library implementation (#15)
Added new yang ietf-yang-library.yang, which defines data models for
RFC7895 yang library discovery feature.
New app module yanglib_app.go to handle ietf-yang-library.yang APIs.
REST and gNMI requests for '/ietf-yang-library:modules-state' and its
child paths will be serviced by this app module.
Parses all NB yangs using goyang parser and builds the module info data
tree as per ietf-yang-library.yang. Yangs are parsed upon first request
and cached thereafter.
Uses translib.GetModel() API to identify the yang modules that are
implemented by apps. Other modules are marked as 'import' in yang
library response data. Transformer annotation files (*_annot.yang) are
ignored -- they do not define any data model.
Uses hardcoded module-set-id value for now. Should be changed to use
"yang bundle version" number when yang versioning feature is in.
Yang schema URL is included in the response only if main program set the
root URL through translib.SetSchemaRootURL() API. Schema URL is prepared
by appending the yang file name to the root URL.
---
Makefile | 4 +-
go.sum | 2 +
models/yang/ietf-yang-library.yang | 245 ++++++++++++++
translib/path_utils.go | 25 +-
translib/path_utils_test.go | 65 ++++
translib/yanglib_app.go | 512 +++++++++++++++++++++++++++++
translib/yanglib_app_test.go | 152 +++++++++
7 files changed, 1000 insertions(+), 5 deletions(-)
create mode 100644 models/yang/ietf-yang-library.yang
create mode 100644 translib/yanglib_app.go
create mode 100644 translib/yanglib_app_test.go
diff --git a/Makefile b/Makefile
index d1b0bf18b385..8d8bb8420597 100644
--- a/Makefile
+++ b/Makefile
@@ -81,9 +81,9 @@ $(GOYANG_BIN): $(GO_DEPS)
cd vendor/github.com/openconfig/goyang && \
$(GO) build -o $@ *.go
-clean: models-clean translib-clean cvl-clean
+clean: models-clean translib-clean cvl-clean go-deps-clean
git check-ignore debian/* | xargs -r $(RM) -r
$(RM) -r $(BUILD_DIR)
-cleanall: clean go-deps-clean
+cleanall: clean
git clean -fdX tools
diff --git a/go.sum b/go.sum
index 77b875b4d8cf..4f0831d843fe 100644
--- a/go.sum
+++ b/go.sum
@@ -35,6 +35,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4 h1:+EOh4OY6tjM6ZueeUKinl1f0U2820HzQOuf1iqMnsks=
github.com/golang/protobuf v1.4.0-rc.4/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0 h1:aRz0NBceriICVtjhCgKkDvl+RudKu1CT6h0ZvUTrNfE=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -121,6 +122,7 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.20.1 h1:ESRXHgpUBG5D2I5mmsQIyYxB/tQIZfSZ8wLyFDf/N/U=
google.golang.org/protobuf v1.20.1/go.mod h1:KqelGeouBkcbcuB3HCk4/YH2tmNLk6YSWA5LIWeI/lY=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
diff --git a/models/yang/ietf-yang-library.yang b/models/yang/ietf-yang-library.yang
new file mode 100644
index 000000000000..7d84e64ed880
--- /dev/null
+++ b/models/yang/ietf-yang-library.yang
@@ -0,0 +1,245 @@
+module ietf-yang-library {
+ namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library";
+ prefix "yanglib";
+
+ import ietf-yang-types {
+ prefix yang;
+ }
+ import ietf-inet-types {
+ prefix inet;
+ }
+
+ organization
+ "IETF NETCONF (Network Configuration) Working Group";
+
+ contact
+ "WG Web:
+ WG List:
+
+ WG Chair: Mehmet Ersue
+
+
+ WG Chair: Mahesh Jethanandani
+
+
+ Editor: Andy Bierman
+
+
+ Editor: Martin Bjorklund
+
+
+ Editor: Kent Watsen
+ ";
+
+ description
+ "This module contains monitoring information about the YANG
+ modules and submodules that are used within a YANG-based
+ server.
+
+ Copyright (c) 2016 IETF Trust and the persons identified as
+ authors of the code. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or
+ without modification, is permitted pursuant to, and subject
+ to the license terms contained in, the Simplified BSD License
+ set forth in Section 4.c of the IETF Trust's Legal Provisions
+ Relating to IETF Documents
+ (http://trustee.ietf.org/license-info).
+
+ This version of this YANG module is part of RFC 7895; see
+ the RFC itself for full legal notices.";
+
+ revision 2016-06-21 {
+ description
+ "Initial revision.";
+ reference
+ "RFC 7895: YANG Module Library.";
+ }
+
+ /*
+ * Typedefs
+ */
+
+ typedef revision-identifier {
+ type string {
+ pattern '\d{4}-\d{2}-\d{2}';
+ }
+ description
+ "Represents a specific date in YYYY-MM-DD format.";
+ }
+
+ /*
+ * Groupings
+ */
+
+ grouping module-list {
+ description
+ "The module data structure is represented as a grouping
+ so it can be reused in configuration or another monitoring
+ data structure.";
+
+ grouping common-leafs {
+ description
+ "Common parameters for YANG modules and submodules.";
+
+ leaf name {
+ type yang:yang-identifier;
+ description
+ "The YANG module or submodule name.";
+ }
+ leaf revision {
+ type union {
+ type revision-identifier;
+ type string { length 0; }
+ }
+ description
+ "The YANG module or submodule revision date.
+ A zero-length string is used if no revision statement
+ is present in the YANG module or submodule.";
+ }
+ }
+
+ grouping schema-leaf {
+ description
+ "Common schema leaf parameter for modules and submodules.";
+
+ leaf schema {
+ type inet:uri;
+ description
+ "Contains a URL that represents the YANG schema
+ resource for this module or submodule.
+
+ This leaf will only be present if there is a URL
+ available for retrieval of the schema for this entry.";
+ }
+ }
+
+ list module {
+ key "name revision";
+ description
+ "Each entry represents one revision of one module
+ currently supported by the server.";
+
+ uses common-leafs;
+ uses schema-leaf;
+
+ leaf namespace {
+ type inet:uri;
+ mandatory true;
+ description
+ "The XML namespace identifier for this module.";
+ }
+ leaf-list feature {
+ type yang:yang-identifier;
+ description
+ "List of YANG feature names from this module that are
+ supported by the server, regardless of whether they are
+ defined in the module or any included submodule.";
+ }
+ list deviation {
+ key "name revision";
+ description
+ "List of YANG deviation module names and revisions
+ used by this server to modify the conformance of
+ the module associated with this entry. Note that
+ the same module can be used for deviations for
+ multiple modules, so the same entry MAY appear
+ within multiple 'module' entries.
+
+ The deviation module MUST be present in the 'module'
+ list, with the same name and revision values.
+ The 'conformance-type' value will be 'implement' for
+ the deviation module.";
+ uses common-leafs;
+ }
+ leaf conformance-type {
+ type enumeration {
+ enum implement {
+ description
+ "Indicates that the server implements one or more
+ protocol-accessible objects defined in the YANG module
+ identified in this entry. This includes deviation
+ statements defined in the module.
+
+ For YANG version 1.1 modules, there is at most one
+ module entry with conformance type 'implement' for a
+ particular module name, since YANG 1.1 requires that,
+ at most, one revision of a module is implemented.
+
+ For YANG version 1 modules, there SHOULD NOT be more
+ than one module entry for a particular module name.";
+ }
+ enum import {
+ description
+ "Indicates that the server imports reusable definitions
+ from the specified revision of the module but does
+ not implement any protocol-accessible objects from
+ this revision.
+
+ Multiple module entries for the same module name MAY
+ exist. This can occur if multiple modules import the
+ same module but specify different revision dates in
+ the import statements.";
+ }
+ }
+ mandatory true;
+ description
+ "Indicates the type of conformance the server is claiming
+ for the YANG module identified by this entry.";
+ }
+ list submodule {
+ key "name revision";
+ description
+ "Each entry represents one submodule within the
+ parent module.";
+ uses common-leafs;
+ uses schema-leaf;
+ }
+ }
+ }
+
+ /*
+ * Operational state data nodes
+ */
+
+ container modules-state {
+ config false;
+ description
+ "Contains YANG module monitoring information.";
+
+ leaf module-set-id {
+ type string;
+ mandatory true;
+ description
+ "Contains a server-specific identifier representing
+ the current set of modules and submodules. The
+ server MUST change the value of this leaf if the
+ information represented by the 'module' list instances
+ has changed.";
+ }
+
+ uses module-list;
+ }
+
+ /*
+ * Notifications
+ */
+
+ notification yang-library-change {
+ description
+ "Generated when the set of modules and submodules supported
+ by the server has changed.";
+ leaf module-set-id {
+ type leafref {
+ path "/yanglib:modules-state/yanglib:module-set-id";
+ }
+ mandatory true;
+ description
+ "Contains the module-set-id value representing the
+ set of modules and submodules supported at the server at
+ the time the notification is generated.";
+ }
+ }
+
+}
+
diff --git a/translib/path_utils.go b/translib/path_utils.go
index 1083f1054336..dc724b6723d5 100644
--- a/translib/path_utils.go
+++ b/translib/path_utils.go
@@ -42,6 +42,12 @@ type PathInfo struct {
Vars map[string]string
}
+// HasVar checks if the PathInfo contains given variable.
+func (p *PathInfo) HasVar(name string) bool {
+ _, exists := p.Vars[name]
+ return exists
+}
+
// Var returns the string value for a path variable. Returns
// empty string if no such variable exists.
func (p *PathInfo) Var(name string) string {
@@ -91,6 +97,14 @@ func NewPathInfo(path string) *PathInfo {
name := readUntil(r, '=')
value := readUntil(r, ']')
+
+ // Handle duplicate parameter names by suffixing "#N" to it.
+ // N is the number of occurance of that parameter name from left.
+ namePrefix := name
+ for k := 2; info.HasVar(name); k++ {
+ name = fmt.Sprintf("%s#%d", namePrefix, k)
+ }
+
if len(name) != 0 {
fmt.Fprintf(&template, "{}")
info.Vars[name] = value
@@ -104,12 +118,17 @@ func NewPathInfo(path string) *PathInfo {
func readUntil(r *strings.Reader, delim byte) string {
var buff strings.Builder
+ var escaped bool
+
for {
c, err := r.ReadByte()
- if err == nil && c != delim {
- buff.WriteByte(c)
- } else {
+ if err != nil || (c == delim && !escaped) {
break
+ } else if c == '\\' && !escaped {
+ escaped = true
+ } else {
+ escaped = false
+ buff.WriteByte(c)
}
}
diff --git a/translib/path_utils_test.go b/translib/path_utils_test.go
index 1d108086d46e..33b8194aa5a2 100644
--- a/translib/path_utils_test.go
+++ b/translib/path_utils_test.go
@@ -162,3 +162,68 @@ func TestGetObjectFieldName(t *testing.T) {
}
}
}
+
+func TestNewPathInfo_empty(t *testing.T) {
+ testPathInfo(t, "", "", mkmap())
+}
+
+func TestNewPathInfo_novar(t *testing.T) {
+ testPathInfo(t, "/test/simple", "/test/simple", mkmap())
+}
+
+func TestNewPathInfo_var1(t *testing.T) {
+ testPathInfo(t, "/test/xx[one=1]", "/test/xx{}", mkmap("one", "1"))
+}
+
+func TestNewPathInfo_vars(t *testing.T) {
+ testPathInfo(t, "/test/xx[one=1][two=2]/new[three=3]", "/test/xx{}{}/new{}",
+ mkmap("one", "1", "two", "2", "three", "3"))
+}
+
+func TestNewPathInfo_dup1(t *testing.T) {
+ testPathInfo(t, "/test/xx[one=1][two=2]/new[one=0001]", "/test/xx{}{}/new{}",
+ mkmap("one", "1", "two", "2", "one#2", "0001"))
+}
+
+func TestNewPathInfo_dups(t *testing.T) {
+ testPathInfo(t, "/test/one[xx=1]/two[yy=2]/three[xx=3]/four[zz=4]/five[yy=5]/six[xx=6]",
+ "/test/one{}/two{}/three{}/four{}/five{}/six{}",
+ mkmap("xx", "1", "yy", "2", "xx#2", "3", "zz", "4", "yy#2", "5", "xx#3", "6"))
+}
+
+func TestNewPathInfo_escaped_name(t *testing.T) {
+ testPathInfo(t, "/test/xx[one\\==1][two[\\]=2]", "/test/xx{}{}",
+ mkmap("one=", "1", "two[]", "2"))
+}
+
+func TestNewPathInfo_escaped_valu(t *testing.T) {
+ testPathInfo(t, "/test/xx[one=[1\\]][two=\\0\\02 [\\.\\D]", "/test/xx{}{}",
+ mkmap("one", "[1]", "two", "002 [.D"))
+}
+
+func testPathInfo(t *testing.T, path, expTemplate string, expVars map[string]string) {
+ info := NewPathInfo(path)
+ if info == nil {
+ t.Errorf("NewPathInfo() returned null!")
+ } else if info.Path != path {
+ t.Errorf("Expected info.Path = %s", path)
+ t.Errorf("Actual info.Path = %s", info.Path)
+ } else if info.Template != expTemplate {
+ t.Errorf("Expected info.Template = %s", expTemplate)
+ t.Errorf("Actual info.Template = %s", info.Template)
+ } else if reflect.DeepEqual(info.Vars, expVars) == false {
+ t.Errorf("Expected info.Vars = %v", expVars)
+ t.Errorf("Actual info.Vars = %v", info.Vars)
+ }
+ if t.Failed() {
+ t.Fatalf("NewPathInfo() failed to parse \"%s\"", path)
+ }
+}
+
+func mkmap(args ...string) map[string]string {
+ m := make(map[string]string)
+ for i := 0; (i + 1) < len(args); i += 2 {
+ m[args[i]] = args[i+1]
+ }
+ return m
+}
diff --git a/translib/yanglib_app.go b/translib/yanglib_app.go
new file mode 100644
index 000000000000..c235a97677e0
--- /dev/null
+++ b/translib/yanglib_app.go
@@ -0,0 +1,512 @@
+////////////////////////////////////////////////////////////////////////////////
+// //
+// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
+// its subsidiaries. //
+// //
+// Licensed under the Apache License, Version 2.0 (the "License"); //
+// you may not use this file except in compliance with the License. //
+// You may obtain a copy of the License at //
+// //
+// http://www.apache.org/licenses/LICENSE-2.0 //
+// //
+// Unless required by applicable law or agreed to in writing, software //
+// distributed under the License is distributed on an "AS IS" BASIS, //
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
+// See the License for the specific language governing permissions and //
+// limitations under the License. //
+// //
+////////////////////////////////////////////////////////////////////////////////
+
+package translib
+
+import (
+ "path/filepath"
+ "reflect"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Azure/sonic-mgmt-common/translib/db"
+ "github.com/Azure/sonic-mgmt-common/translib/ocbinds"
+ errors "github.com/Azure/sonic-mgmt-common/translib/tlerr"
+ "github.com/Azure/sonic-mgmt-common/translib/transformer"
+
+ "github.com/golang/glog"
+ "github.com/openconfig/goyang/pkg/yang"
+)
+
+// yanglibApp implements app interface for the
+// ietf-yang-library module
+type yanglibApp struct {
+ pathInfo *PathInfo
+ ygotRoot *ocbinds.Device
+ ygotTarget *interface{}
+}
+
+// theYanglibMutex synchronizes all cache loads
+var theYanglibMutex sync.Mutex
+
+// theYanglibCache holds parsed yanglib info. Populated on first
+// request.
+var theYanglibCache *ocbinds.IETFYangLibrary_ModulesState
+
+// theSchemaRootURL is the base URL for the yang file download URL.
+// Main program must set the value through SetSchemaRootURL() API.
+// Individual file URL is obtained by appending file name to it.
+var theSchemaRootURL string
+
+func init() {
+ err := register("/ietf-yang-library:modules-state",
+ &appInfo{
+ appType: reflect.TypeOf(yanglibApp{}),
+ ygotRootType: reflect.TypeOf(ocbinds.IETFYangLibrary_ModulesState{}),
+ isNative: false,
+ })
+ if err != nil {
+ glog.Fatal("register() failed for yanglibApp;", err)
+ }
+
+ err = addModel(&ModelData{
+ Name: "ietf-yang-library",
+ Org: "IETF NETCONF (Network Configuration) Working Group",
+ Ver: "2016-06-21",
+ })
+ if err != nil {
+ glog.Fatal("addModel() failed for yanglibApp;", err)
+ }
+}
+
+/*
+ * App interface functions
+ */
+
+func (app *yanglibApp) initialize(data appData) {
+ app.pathInfo = NewPathInfo(data.path)
+ app.ygotRoot = (*data.ygotRoot).(*ocbinds.Device)
+ app.ygotTarget = data.ygotTarget
+}
+
+func (app *yanglibApp) translateCreate(d *db.DB) ([]db.WatchKeys, error) {
+ return nil, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) translateUpdate(d *db.DB) ([]db.WatchKeys, error) {
+ return nil, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) translateReplace(d *db.DB) ([]db.WatchKeys, error) {
+ return nil, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) translateDelete(d *db.DB) ([]db.WatchKeys, error) {
+ return nil, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) translateGet(dbs [db.MaxDB]*db.DB) error {
+ return nil // NOOP! everyting is in processGet
+}
+
+func (app *yanglibApp) translateAction(dbs [db.MaxDB]*db.DB) error {
+ return errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) translateSubscribe(dbs [db.MaxDB]*db.DB, path string) (*notificationOpts, *notificationInfo, error) {
+ return nil, nil, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) processCreate(d *db.DB) (SetResponse, error) {
+ return SetResponse{}, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) processUpdate(d *db.DB) (SetResponse, error) {
+ return SetResponse{}, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) processReplace(d *db.DB) (SetResponse, error) {
+ return SetResponse{}, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) processDelete(d *db.DB) (SetResponse, error) {
+ return SetResponse{}, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) processAction(dbs [db.MaxDB]*db.DB) (ActionResponse, error) {
+ return ActionResponse{}, errors.NotSupported("Unsupported")
+}
+
+func (app *yanglibApp) processGet(dbs [db.MaxDB]*db.DB) (GetResponse, error) {
+ glog.Infof("path = %s", app.pathInfo.Template)
+ glog.Infof("vars = %s", app.pathInfo.Vars)
+
+ var resp GetResponse
+ ylib, err := getYanglibInfo()
+ if err != nil {
+ return resp, err
+ }
+
+ switch {
+ case app.pathInfo.HasSuffix("/module-set-id"): // only module-set-id
+ app.ygotRoot.ModulesState.ModuleSetId = ylib.ModuleSetId
+
+ case app.pathInfo.HasVar("name"): // only one module
+ err = app.copyOneModuleInfo(ylib)
+
+ default: // all modules
+ app.ygotRoot.ModulesState = ylib
+ }
+
+ if err == nil {
+ resp.Payload, err = generateGetResponsePayload(
+ app.pathInfo.Path, app.ygotRoot, app.ygotTarget)
+ }
+
+ return resp, err
+}
+
+// copyOneModuleInfo fills one module from given ygot IETFYangLibrary_ModulesState
+// object into app.ygotRoot.
+func (app *yanglibApp) copyOneModuleInfo(fromMods *ocbinds.IETFYangLibrary_ModulesState) error {
+ key := ocbinds.IETFYangLibrary_ModulesState_Module_Key{
+ Name: app.pathInfo.Var("name"), Revision: app.pathInfo.Var("revision")}
+
+ glog.Infof("Copying module %s@%s", key.Name, key.Revision)
+
+ to := app.ygotRoot.ModulesState.Module[key]
+ from := fromMods.Module[key]
+ if from == nil {
+ glog.Errorf("No module %s in yanglib", key)
+ return errors.NotFound("Module %s@%s not found", key.Name, key.Revision)
+ }
+
+ switch pt := app.pathInfo.Template; {
+ case strings.HasSuffix(pt, "/deviation"):
+ // Copy only deviations.
+ if len(from.Deviation) != 0 {
+ to.Deviation = from.Deviation
+ } else {
+ return errors.NotFound("Module %s@%s has no deviations", key.Name, key.Revision)
+ }
+
+ case strings.Contains(pt, "/deviation{}{}"):
+ // Copy only one deviation info
+ devkey := ocbinds.IETFYangLibrary_ModulesState_Module_Deviation_Key{
+ Name: app.pathInfo.Var("name#2"), Revision: app.pathInfo.Var("revision#2")}
+
+ if devmod := from.Deviation[devkey]; devmod != nil {
+ *to.Deviation[devkey] = *devmod
+ } else {
+ return errors.NotFound("Module %s@%s has no deviation %s@%s",
+ key.Name, key.Revision, devkey.Name, devkey.Revision)
+ }
+
+ case strings.HasSuffix(pt, "/submodule"):
+ // Copy only submodules..
+ if len(from.Submodule) != 0 {
+ to.Submodule = from.Submodule
+ } else {
+ return errors.NotFound("Module %s@%s has no submodules", key.Name, key.Revision)
+ }
+
+ case strings.Contains(pt, "/submodule{}{}"):
+ // Copy only one submodule info
+ subkey := ocbinds.IETFYangLibrary_ModulesState_Module_Submodule_Key{
+ Name: app.pathInfo.Var("name#2"), Revision: app.pathInfo.Var("revision#2")}
+
+ if submod := from.Submodule[subkey]; submod != nil {
+ *to.Submodule[subkey] = *submod
+ } else {
+ return errors.NotFound("Module %s@%s has no submodule %s@%s",
+ key.Name, key.Revision, subkey.Name, subkey.Revision)
+ }
+
+ default:
+ // Copy full module
+ app.ygotRoot.ModulesState.Module[key] = from
+ }
+
+ return nil
+}
+
+/*
+ * Yang parsing utilities
+ */
+
+// yanglibBuilder is the utility for parsing and loading yang files into
+// ygot IETFYangLibrary_ModulesState object.
+type yanglibBuilder struct {
+ // yangDir is the directory with all yang files
+ yangDir string
+
+ // implModules contains top level yang module names implemented
+ // by this system. Values are discovered from translib.getModels() API
+ implModules map[string]bool
+
+ // yangModules is the temporary cache of all parsed yang modules.
+ // Populated by loadYangs() function.
+ yangModules *yang.Modules
+
+ // ygotModules is the output ygot object tree containing all
+ // yang module info
+ ygotModules *ocbinds.IETFYangLibrary_ModulesState
+}
+
+// getYanglibInfo returns the ygot IETFYangLibrary_ModulesState object
+// with all yang library information.
+func getYanglibInfo() (ylib *ocbinds.IETFYangLibrary_ModulesState, err error) {
+ theYanglibMutex.Lock()
+ if theYanglibCache == nil {
+ glog.Infof("Building yanglib cache")
+ theYanglibCache, err = newYanglibInfo()
+ glog.Infof("Yanglib cache ready; err=%v", err)
+ }
+
+ ylib = theYanglibCache
+ theYanglibMutex.Unlock()
+ return
+}
+
+// newYanglibInfo loads all eligible yangs and fills yanglib info into the
+// ygot IETFYangLibrary_ModulesState object
+func newYanglibInfo() (*ocbinds.IETFYangLibrary_ModulesState, error) {
+ var yb yanglibBuilder
+ if err := yb.prepare(); err != nil {
+ return nil, err
+ }
+ if err := yb.loadYangs(); err != nil {
+ return nil, err
+ }
+ if err := yb.translate(); err != nil {
+ return nil, err
+ }
+
+ return yb.ygotModules, nil
+}
+
+// prepare function initializes the yanglibBuilder object for
+// parsing yangs and translating into ygot.
+func (yb *yanglibBuilder) prepare() error {
+ yb.yangDir = GetYangPath()
+ glog.Infof("yanglibBuilder.prepare: yangDir = %s", yb.yangDir)
+ glog.Infof("yanglibBuilder.prepare: baseURL = %s", theSchemaRootURL)
+
+ // Load supported model information
+ yb.implModules = make(map[string]bool)
+ for _, m := range getModels() {
+ yb.implModules[m.Name] = true
+ }
+
+ yb.ygotModules = &ocbinds.IETFYangLibrary_ModulesState{}
+ return nil
+}
+
+// loadYangs reads eligible yang files into yang.Modules object.
+// Skips transformer annotation yangs.
+func (yb *yanglibBuilder) loadYangs() error {
+ glog.Infof("Loading yangs from %s directory", yb.yangDir)
+ var parsed, ignored uint32
+ mods := yang.NewModules()
+ start := time.Now()
+
+ files, _ := filepath.Glob(filepath.Join(yb.yangDir, "*.yang"))
+ for _, f := range files {
+ // ignore transformer annotation yangs
+ if strings.HasSuffix(filepath.Base(f), "-annot.yang") {
+ ignored++
+ continue
+ }
+ if err := mods.Read(f); err != nil {
+ glog.Errorf("Failed to parse %s; err=%v", f, err)
+ return errors.New("System error")
+ }
+ parsed++
+ }
+
+ glog.Infof("%d yang files loaded in %s; %d ignored", parsed, time.Since(start), ignored)
+ yb.yangModules = mods
+ return nil
+}
+
+// translate function fills parsed yang.Modules info into the
+// ygot IETFYangLibrary_ModulesState object.
+func (yb *yanglibBuilder) translate() error {
+ var modsWithDeviation []*yang.Module
+
+ // First iteration -- create ygot module entry for each yang.Module
+ for _, mod := range yb.yangModules.Modules {
+ m, _ := yb.ygotModules.NewModule(mod.Name, mod.Current())
+ if m == nil {
+ // ignore; yang.Modules map contains dupicate entries - one for name and
+ // other for name@rev. NewModule() will return nil if entry exists.
+ continue
+ }
+
+ // Fill basic properties into ygot module
+ yb.fillModuleInfo(m, mod)
+
+ // Mark the yang.Module with "deviation" statements for 2nd iteration. We need reverse
+ // mapping of deviation target -> current module in ygot. Hence 2nd iteration..
+ if len(mod.Deviation) != 0 {
+ modsWithDeviation = append(modsWithDeviation, mod)
+ }
+ }
+
+ // 2nd iteration -- fill deviations.
+ for _, mod := range modsWithDeviation {
+ yb.translateDeviations(mod)
+ }
+
+ // 3rd iteration -- fill conformance type
+ for _, m := range yb.ygotModules.Module {
+ if yb.implModules[*m.Name] {
+ m.ConformanceType = ocbinds.IETFYangLibrary_ModulesState_Module_ConformanceType_implement
+ } else {
+ m.ConformanceType = ocbinds.IETFYangLibrary_ModulesState_Module_ConformanceType_import
+ }
+ }
+
+ // Use yang bundle version as module-set-id
+ msetID := GetYangModuleSetID()
+ yb.ygotModules.ModuleSetId = &msetID
+
+ return nil
+}
+
+// fillModuleInfo yang module info from yang.Module to ygot IETFYangLibrary_ModulesState_Module
+// object.. Deviation information is not filled.
+func (yb *yanglibBuilder) fillModuleInfo(to *ocbinds.IETFYangLibrary_ModulesState_Module, from *yang.Module) {
+ to.Namespace = &from.Namespace.Name
+ to.Schema = yb.getSchemaURL(from)
+
+ // Fill the "feature" info from yang even though we dont have full
+ // support for yang features.
+ for _, f := range from.Feature {
+ to.Feature = append(to.Feature, f.Name)
+ }
+
+ // Iterate thru "include" statements to resolve submodules
+ for _, inc := range from.Include {
+ submod := yb.yangModules.FindModule(inc)
+ if submod == nil { // should not happen
+ glog.Errorf("No sub-module %s; @%s", inc.Name, inc.Statement().Location())
+ continue
+ }
+
+ // NewSubmodule() returns nil if submodule entry already exists.. Ignore it.
+ if sm, _ := to.NewSubmodule(submod.Name, submod.Current()); sm != nil {
+ sm.Schema = yb.getSchemaURL(submod)
+ }
+ }
+}
+
+// fillModuleDeviation creates a deviation module info in the ygot structure
+// for a given main module.
+func (yb *yanglibBuilder) fillModuleDeviation(main *yang.Module, deviation *yang.Module) {
+ key := ocbinds.IETFYangLibrary_ModulesState_Module_Key{
+ Name: main.Name, Revision: main.Current()}
+
+ if m, ok := yb.ygotModules.Module[key]; ok {
+ m.NewDeviation(deviation.Name, deviation.Current())
+
+ // Mark the deviation module as "implemented" if main module is also "implemented"
+ if yb.implModules[main.Name] {
+ yb.implModules[deviation.Name] = true
+ }
+ } else {
+ glog.Errorf("Ygot module entry %s not found", key)
+ }
+}
+
+// translateDeviations function will process all "devaiation" statements of
+// a yang.Module and fill deviation info into corresponding ygot module objects.
+func (yb *yanglibBuilder) translateDeviations(mod *yang.Module) error {
+ deviationTargets := make(map[string]bool)
+
+ // Loop thru deviation statements and find modules deviated by current module
+ for _, d := range mod.Deviation {
+ if !strings.HasPrefix(d.Name, "/") {
+ glog.Errorf("Deviation path \"%s\" is not absolute! @%s", d.Name, d.Statement().Location())
+ continue
+ }
+
+ // Get prefix of root node from the deviation path. First split the path
+ // by "/" char and then split 1st part by ":".
+ // Eg, find "acl" from "/acl:scl-sets/config/something"
+ root := strings.SplitN(strings.SplitN(d.Name, "/", 3)[1], ":", 2)
+ if len(root) != 2 {
+ glog.Errorf("Deviation path \"%s\" has no prefix for root element! @%s",
+ d.Name, d.Statement().Location())
+ } else {
+ deviationTargets[root[0]] = true
+ }
+ }
+
+ glog.V(2).Infof("Module %s has deviations for %d modules", mod.FullName(), len(deviationTargets))
+
+ // Deviation target prefixes must be in the import list.. Find the target
+ // modules by matching the prefix in imports.
+ for _, imp := range mod.Import {
+ prefix := imp.Name
+ if imp.Prefix != nil {
+ prefix = imp.Prefix.Name
+ }
+ if !deviationTargets[prefix] {
+ continue
+ }
+
+ if m := yb.yangModules.FindModule(imp); m != nil {
+ yb.fillModuleDeviation(m, mod)
+ } else {
+ glog.Errorf("No module for prefix \"%s\"", prefix)
+ }
+ }
+
+ return nil
+}
+
+// getSchemaURL resolves the URL for downloading yang file from current
+// device. Returns nil if yang URL could not be prepared.
+func (yb *yanglibBuilder) getSchemaURL(m *yang.Module) *string {
+ if len(theSchemaRootURL) == 0 {
+ return nil // Base URL not resolved; hence no yang URL
+ }
+
+ // Ugly hack to get source file name from yang.Module. See implementation
+ // of yang.Statement.Location() function.
+ // TODO: any better way to get source file path from yang.Module??
+ toks := strings.Split(m.Source.Location(), ":")
+ if len(toks) != 1 && len(toks) != 3 {
+ glog.Warningf("Could not resolve file path for module %s; location=%s",
+ m.FullName(), m.Source.Location())
+ return nil
+ }
+
+ uri := theSchemaRootURL + filepath.Base(toks[0])
+ return &uri
+}
+
+// SetSchemaRootURL sets root URL for yang file download URLs.
+func SetSchemaRootURL(url string) {
+ theYanglibMutex.Lock()
+ defer theYanglibMutex.Unlock()
+
+ newURL := url
+ if len(url) != 0 && !strings.HasSuffix(url, "/") {
+ newURL += "/"
+ }
+
+ if theSchemaRootURL != newURL {
+ theSchemaRootURL = newURL
+ theYanglibCache = nil // reset cache
+ }
+}
+
+// GetYangPath returns directory containing yang files. Use
+// transformer.YangPath for now.
+func GetYangPath() string {
+ return transformer.YangPath
+}
+
+// GetYangModuleSetID returns the ietf-yang-library's module-set-id value.
+func GetYangModuleSetID() string {
+ return "0.1.0" //FIXME use YangBundleVersion when API versioning is available.
+}
diff --git a/translib/yanglib_app_test.go b/translib/yanglib_app_test.go
new file mode 100644
index 000000000000..111d808e22cf
--- /dev/null
+++ b/translib/yanglib_app_test.go
@@ -0,0 +1,152 @@
+////////////////////////////////////////////////////////////////////////////////
+// //
+// Copyright 2020 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
+// its subsidiaries. //
+// //
+// Licensed under the Apache License, Version 2.0 (the "License"); //
+// you may not use this file except in compliance with the License. //
+// You may obtain a copy of the License at //
+// //
+// http://www.apache.org/licenses/LICENSE-2.0 //
+// //
+// Unless required by applicable law or agreed to in writing, software //
+// distributed under the License is distributed on an "AS IS" BASIS, //
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
+// See the License for the specific language governing permissions and //
+// limitations under the License. //
+// //
+////////////////////////////////////////////////////////////////////////////////
+
+package translib
+
+import (
+ "encoding/json"
+ "fmt"
+ "testing"
+
+ "github.com/Azure/sonic-mgmt-common/translib/tlerr"
+)
+
+func TestYanglibGetAll(t *testing.T) {
+ data := getYanglibDataT(t, "", "", "")
+ v := data["ietf-yang-library:modules-state"]
+ if v1, ok := v.(map[string]interface{}); ok {
+ validateMSetID(t, v1["module-set-id"])
+ v = v1["module"]
+ }
+ if v1, ok := v.([]interface{}); !ok || len(v1) == 0 {
+ t.Fatalf("App returned incorrect info.. %v", data)
+ }
+}
+
+func TestYanglibGetMsetID(t *testing.T) {
+ data := getYanglibDataT(t, "", "", "module-set-id")
+ if len(data) != 1 {
+ t.Fatalf("App returned incorrect info.. %v", data)
+ }
+ validateMSetID(t, data["ietf-yang-library:module-set-id"])
+}
+
+func validateMSetID(t *testing.T, msetID interface{}) {
+ if m, ok := msetID.(string); !ok || m != GetYangModuleSetID() {
+ t.Fatalf("App returned incorrect module-set-id \"%s\"; expected \"%s\"",
+ msetID, GetYangModuleSetID())
+ }
+}
+
+func TestYanglibGetOne(t *testing.T) {
+ data := getYanglibDataT(t, "ietf-yang-library", "2016-06-21", "")
+
+ var m map[string]interface{}
+ if v, ok := data["ietf-yang-library:module"].([]interface{}); ok && len(v) == 1 {
+ m, ok = v[0].(map[string]interface{})
+ }
+
+ if m["name"] != "ietf-yang-library" ||
+ m["revision"] != "2016-06-21" ||
+ m["namespace"] != "urn:ietf:params:xml:ns:yang:ietf-yang-library" ||
+ m["conformance-type"] != "implement" {
+ t.Fatalf("App returned incorrect info.. %v", data)
+ }
+}
+
+func TestYanglibGetOneAttr(t *testing.T) {
+ data := getYanglibDataT(t, "ietf-yang-library", "2016-06-21", "namespace")
+ if data["ietf-yang-library:namespace"] != "urn:ietf:params:xml:ns:yang:ietf-yang-library" {
+ t.Fatalf("App returned incorrect info.. %v", data)
+ }
+}
+
+func TestYanglibSchemaURL(t *testing.T) {
+ defer SetSchemaRootURL("")
+
+ t.Run("default", testYlibSchema(nil))
+
+ SetSchemaRootURL("https://localhost/schema1")
+ t.Run("no_slash", testYlibSchema("https://localhost/schema1/ietf-yang-library.yang"))
+
+ SetSchemaRootURL("https://localhost/schema2/")
+ t.Run("with_slash", testYlibSchema("https://localhost/schema2/ietf-yang-library.yang"))
+
+ SetSchemaRootURL("")
+ t.Run("reset", testYlibSchema(nil))
+}
+
+func testYlibSchema(expURL interface{}) func(*testing.T) {
+ return func(t *testing.T) {
+ data := getYanglibDataT(t, "ietf-yang-library", "2016-06-21", "schema")
+ if data["ietf-yang-library:schema"] != expURL {
+ t.Fatalf("Expected schema url '%s', found '%s'",
+ expURL, data["ietf-yang-library:schema"])
+ }
+ }
+}
+
+func TestYanglibConformance(t *testing.T) {
+ t.Run("ietf-yang-library", testConfType("ietf-yang-library", "2016-06-21", "implement"))
+ t.Run("ietf-yang-types", testConfType("ietf-yang-types", "2013-07-15", "import"))
+ t.Run("ietf-inet-types", testConfType("ietf-inet-types", "2013-07-15", "import"))
+}
+
+func testConfType(mod, rev, exp string) func(*testing.T) {
+ return func(t *testing.T) {
+ data := getYanglibDataT(t, mod, rev, "conformance-type")
+ if data["ietf-yang-library:conformance-type"] != exp {
+ t.Fatalf("App returned unexpected conformance-type for %s@%s; found=%s, exp=%s",
+ mod, rev, data["ietf-yang-library:conformance-type"], exp)
+ }
+ }
+}
+
+func TestYanglibGetUnknown(t *testing.T) {
+ _, err := getYanglibData("unknown", "0000-00-00", "")
+ if _, ok := err.(tlerr.NotFoundError); !ok {
+ t.Fatalf("Expected NotFoundError, got %T", err)
+ }
+}
+
+func getYanglibData(name, rev, attr string) (map[string]interface{}, error) {
+ u := "/ietf-yang-library:modules-state"
+ if name != "" || rev != "" {
+ u += fmt.Sprintf("/module[name=%s][revision=%s]", name, rev)
+ }
+ if attr != "" {
+ u += ("/" + attr)
+ }
+
+ data := make(map[string]interface{})
+ response, err := Get(GetRequest{Path: u})
+ if err == nil {
+ err = json.Unmarshal(response.Payload, &data)
+ }
+
+ return data, err
+}
+
+func getYanglibDataT(t *testing.T, name, rev, attr string) map[string]interface{} {
+ data, err := getYanglibData(name, rev, attr)
+ if err != nil {
+ t.Fatalf("Unexpected erorr: %v", err)
+ }
+ return data
+}