Query parameters enhancements in rest-server. (sonic-net#119)

* Add content and fields query parameter support in rest server

* Changes to prepareRequest in test files to send correct request to rest-server

* Download only required artifacts during pipeline build

* Modify Failing test case

azure-pipelines.yml

@@ -52,6 +52,8 @@ stages:
         artifact: sonic-buildimage.vs
         runVersion: 'latestFromBranch'
         runBranch: 'refs/heads/master'
+        patterns: |
+          target/debs/buster/libyang*.deb
       displayName: "Download sonic buildimage"
 
     - script: |

rest/server/handler.go

@@ -260,6 +260,8 @@ type translibArgs struct {
 	data        []byte           // payload
 	version     translib.Version // client version
 	depth       uint             // RESTCONF depth, for Get API only
+	content     string           // RESTCONF content, for Get API only
+	fields      []string         // RESTCONF fields, for Get API only
 	deleteEmpty bool             // Delete empty entry during field delete
 }
 
@@ -304,8 +306,12 @@ func invokeTranslib(args *translibArgs, rc *RequestContext) (int, []byte, error)
 	case "GET", "HEAD":
 		req := translib.GetRequest{
 			Path:          args.path,
-			Depth:         args.depth,
 			ClientVersion: args.version,
+			QueryParams: translib.QueryParameters{
+				Depth:   args.depth,
+				Content: args.content,
+				Fields:  args.fields,
+			},
 		}
 		resp, err1 := translib.Get(req)
 		if err1 == nil {

rest/server/handler_test.go

@@ -561,10 +561,10 @@ func verifyParseVersion(t *testing.T, r *http.Request, expSuccess bool, expVer t
 
 func TestPanic(t *testing.T) {
 	s := newEmptyRouter()
-	s.addRoute("panic", "GET", "/panic",
+	s.addRoute("panic", "GET", "/restconf/panic",
 		func(w http.ResponseWriter, r *http.Request) { panic("testing 123") })
 	w := httptest.NewRecorder()
-	s.ServeHTTP(w, prepareRequest(t, "GET", "/panic", ""))
+	s.ServeHTTP(w, prepareRequest(t, "GET", "/restconf/panic", ""))
 	verifyResponse(t, w, 500)
 }
 
@@ -574,6 +574,60 @@ func TestProcessGET(t *testing.T) {
 	verifyResponseData(t, w, 200, jsonObj{"path": "/api-tests:sample", "depth": 0})
 }
 
+func TestProcessGET_query_depth(t *testing.T) {
+	w := httptest.NewRecorder()
+	Process(w, prepareRequest(t, "GET", "/api-tests:sample?depth=10", ""))
+	if restconfCapabilities.depth {
+		verifyResponseData(t, w, 200, jsonObj{"depth": 10})
+	} else {
+		verifyResponse(t, w, 400)
+	}
+}
+
+func TestProcessGET_query_depth_error(t *testing.T) {
+	w := httptest.NewRecorder()
+	Process(w, prepareRequest(t, "GET", "/api-tests:sample?depth=none", ""))
+	verifyResponse(t, w, 400)
+}
+
+func TestProcessGET_query_depth_content_field_capability_support(t *testing.T) {
+	if !restconfCapabilities.depth || !restconfCapabilities.content || !restconfCapabilities.fields {
+		t.Fatalf("depth/content/fields capability is expected to be supported in rest-server")
+	}
+}
+
+func TestProcessGET_query_content(t *testing.T) {
+	w := httptest.NewRecorder()
+	Process(w, prepareRequest(t, "GET", "/api-tests:sample?content=all", ""))
+	if restconfCapabilities.content {
+		verifyResponseData(t, w, 200, jsonObj{"content": "all"})
+	} else {
+		verifyResponse(t, w, 400)
+	}
+}
+
+func TestProcessGET_query_content_error(t *testing.T) {
+	w := httptest.NewRecorder()
+	Process(w, prepareRequest(t, "GET", "/api-tests:sample?content=getall", ""))
+	verifyResponse(t, w, 400)
+}
+
+func TestProcessGET_query_fields(t *testing.T) {
+	w := httptest.NewRecorder()
+	Process(w, prepareRequest(t, "GET", "/api-tests:sample?fields=home/name", ""))
+	if restconfCapabilities.fields {
+		verifyResponseData(t, w, 200, jsonObj{"fields": "[home/name]"})
+	} else {
+		verifyResponse(t, w, 400)
+	}
+}
+
+func TestProcessGET_query_fields_error(t *testing.T) {
+	w := httptest.NewRecorder()
+	Process(w, prepareRequest(t, "GET", "/api-tests:sample?fields=home&content=all", ""))
+	verifyResponse(t, w, 400)
+}
+
 func TestProcessGET_error(t *testing.T) {
 	w := httptest.NewRecorder()
 	Process(w, prepareRequest(t, "GET", "/api-tests:sample/error/not-found", ""))
@@ -690,6 +744,10 @@ func TestProcessReadError(t *testing.T) {
 }
 
 func prepareRequest(t *testing.T, method, path, data string) *http.Request {
+	if !strings.Contains(path, "/restconf/") {
+		path = "/restconf/data" + path
+	}
+
 	r := httptest.NewRequest(method, path, strings.NewReader(data))
 	rc, r := GetContext(r)
 	rc.ID = t.Name()

rest/server/query.go

@@ -30,6 +30,9 @@ import (
 // parseQueryParams parses the http request's query parameters
 // into a translibArgs args.
 func (args *translibArgs) parseQueryParams(r *http.Request) error {
+	if r.URL.RawQuery == "" {
+		return nil
+	}
 	if strings.Contains(r.URL.Path, restconfDataPathPrefix) {
 		return args.parseRestconfQueryParams(r)
 	}
@@ -42,12 +45,16 @@ func (args *translibArgs) parseQueryParams(r *http.Request) error {
 // if any parameter is unsupported or has invalid value.
 func (args *translibArgs) parseRestconfQueryParams(r *http.Request) error {
 	var err error
-	qParams := r.URL.Query()
+	qParams := extractQuery(r.URL.RawQuery)
 
 	for name, vals := range qParams {
 		switch name {
 		case "depth":
 			args.depth, err = parseDepthParam(vals, r)
+		case "content":
+			args.content, err = parseContentParam(vals, r)
+		case "fields":
+			args.fields, err = parseFieldsParam(vals, r)
 		case "deleteEmptyEntry":
 			args.deleteEmpty, err = parseDeleteEmptyEntryParam(vals, r)
 		default:
@@ -57,10 +64,41 @@ func (args *translibArgs) parseRestconfQueryParams(r *http.Request) error {
 			return err
 		}
 	}
+	if len(args.fields) > 0 {
+		if len(args.content) > 0 || args.depth > 0 {
+			return httpError(http.StatusBadRequest, "Fields query parameter is not supported along with other query parameters")
+		}
+	}
 
 	return nil
 }
 
+func extractQuery(rawQuery string) map[string][]string {
+	queryParamsMap := make(map[string][]string)
+	if len(rawQuery) == 0 {
+		return queryParamsMap
+	}
+	// The query parameters are seperated by &
+	qpList := strings.Split(rawQuery, "&")
+	for _, each := range qpList {
+		var valList []string
+		if strings.Contains(each, "=") {
+			eqIdx := strings.Index(each, "=")
+			key := each[:eqIdx]
+			val := each[eqIdx+1:]
+			if _, ok := queryParamsMap[key]; ok {
+				queryParamsMap[key] = append(queryParamsMap[key], val)
+			} else {
+				valList = append(valList, val)
+				queryParamsMap[key] = valList
+			}
+		} else {
+			queryParamsMap[each] = valList
+		}
+	}
+	return queryParamsMap
+}
+
 func newUnsupportedParamError(name string, r *http.Request) error {
 	return httpError(http.StatusBadRequest, "query parameter '%s' not supported", name)
 }
@@ -79,7 +117,7 @@ func parseDepthParam(v []string, r *http.Request) (uint, error) {
 
 	if r.Method != "GET" && r.Method != "HEAD" {
 		glog.V(1).Infof("[%s] 'depth' not supported for %s", getRequestID(r), r.Method)
-		return 0, newUnsupportedParamError("depth", r)
+		return 0, newUnsupportedParamError("depth supported only for GET/HEAD requests", r)
 	}
 
 	if len(v) != 1 {
@@ -100,6 +138,95 @@ func parseDepthParam(v []string, r *http.Request) (uint, error) {
 	return uint(d), nil
 }
 
+// parseContentParam parses query parameter value for "content" parameter.
+// See https://tools.ietf.org/html/rfc8040#section-4.8.1
+func parseContentParam(v []string, r *http.Request) (string, error) {
+	if !restconfCapabilities.content {
+		glog.V(1).Infof("'content' support disabled")
+		return "", newUnsupportedParamError("content", r)
+	}
+
+	if r.Method != "GET" && r.Method != "HEAD" {
+		glog.V(1).Infof("'content' not supported for %s", r.Method)
+		return "", newUnsupportedParamError("content", r)
+	}
+
+	if len(v) != 1 {
+		glog.V(1).Infof("Expecting only 1 content param; found %d", len(v))
+		return "", newInvalidParamError("content", r)
+	}
+
+	if v[0] == "all" || v[0] == "config" || v[0] == "nonconfig" {
+		return v[0], nil
+	} else {
+		glog.V(1).Infof("Bad content value '%s'", v[0])
+		return "", newInvalidParamError("content", r)
+	}
+
+	return v[0], nil
+}
+
+func extractFields(s string) []string {
+	prefix := ""
+	cur := ""
+	res := make([]string, 0)
+	for i, c := range s {
+		if c == '(' {
+			prefix = cur
+			cur = ""
+		} else if c == ')' {
+			res = append(res, prefix+"/"+cur)
+			prefix = ""
+			cur = ""
+		} else if c == ';' {
+			fullpath := prefix
+			if len(prefix) > 0 {
+				fullpath += "/"
+			}
+			if len(fullpath+cur) > 0 {
+				res = append(res, fullpath+cur)
+			}
+			cur = ""
+		} else if c == ' ' {
+			continue
+		} else {
+			cur += string(c)
+		}
+		if i == (len(s) - 1) {
+			fullpath := prefix
+			if len(prefix) > 0 {
+				fullpath += "/"
+			}
+			if len(fullpath+cur) > 0 {
+				res = append(res, fullpath+cur)
+			}
+		}
+	}
+	return res
+}
+
+// parseFieldsParam parses query parameter value for "fields" parameter.
+// See https://tools.ietf.org/html/rfc8040#section-4.8.3
+func parseFieldsParam(v []string, r *http.Request) ([]string, error) {
+	if !restconfCapabilities.fields {
+		glog.V(1).Infof("'fields' support disabled")
+		return v, newUnsupportedParamError("fields", r)
+	}
+
+	if r.Method != "GET" && r.Method != "HEAD" {
+		glog.V(1).Infof("'fields' not supported for %s", r.Method)
+		return v, newUnsupportedParamError("fields supported only for GET/HEAD query", r)
+	}
+
+	if len(v) != 1 {
+		glog.V(1).Infof("Expecting atleast 1 fields param; found %d", len(v))
+		return v, newInvalidParamError("fields", r)
+	}
+
+	res := extractFields(v[0])
+	return res, nil
+}
+
 // parseDeleteEmptyEntryParam parses the custom "deleteEmptyEntry" query parameter.
 func parseDeleteEmptyEntryParam(v []string, r *http.Request) (bool, error) {
 	if r.Method != "DELETE" {