From d772c107a7a5cd4b39888fb88c9cd2953b7412e6 Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Mon, 25 Mar 2024 16:47:26 -0400 Subject: [PATCH 1/7] 267: Add http json functionality Signed-off-by: Paul Grant --- README.md | 61 +++++++++ go.mod | 2 + go.sum | 4 + pkg/providers/httpjson/httpjson.go | 138 +++++++++++++++++++++ pkg/stringmapprovider/stringmapprovider.go | 3 + pkg/stringprovider/stringprovider.go | 3 + vals.go | 19 +++ vals_httpjson_test.go | 116 +++++++++++++++++ 8 files changed, 346 insertions(+) create mode 100644 pkg/providers/httpjson/httpjson.go create mode 100644 vals_httpjson_test.go diff --git a/README.md b/README.md index 690e371..8a17814 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ It supports various backends including: - Conjur - HCP Vault Secrets - Bitwarden +- HTTP JSON - Use `vals eval -f refs.yaml` to replace all the `ref`s in the file to actual values and secrets. - Use `vals exec -f env.yaml -- ` to populate envvars and execute the command. @@ -225,6 +226,7 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/ - [Kubernetes](#kubernetes) - [Conjur](#conjur) - [HCP Vault Secrets](#hcp-vault-secrets) +- [HTTP JSON](#http-json) - [Bitwarden](#bitwarden) Please see [pkg/providers](https://github.com/helmfile/vals/tree/master/pkg/providers) for the implementations of all the providers. The package names corresponds to the URI schemes. @@ -823,6 +825,65 @@ Examples: - `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/{username,password,uri,notes,item}` gets username, password, uri, notes or the whole item of the given item id - `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/notes#/key1` gets the *key1* from the yaml stored as note in the item +### HTTP JSON + +This provider retrieves values stored in JSON hosted by an HTTP frontend. + +This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages. + +Given the diverse array of JSON structures that can be encountered, utilizing jsonquery with XPath presents a more effective approach for handling this variability in data structures. + +This provider requires an xpath to be provided in singleparam mode (as last argument). + +Do not include the protocol scheme i.e. http/https. Provider defaults to scheme https + +Examples: + +#### Fetch string value + +`ref+httpjson:///?[insecure=false&floatAsInt=false]mode=singleparam#/` + +Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos: +```json +[ + { + "name": "chartify" + }, + { + "name": "go-yaml" + } +] +``` +``` +# To get name="chartify" using https protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/name +# To get name="go-yaml" using https protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[2]/name +# To get name="go-yaml" using http protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?insecure=true&mode=singleparam#///*[2]/name +``` + +#### Fetch integer value + +`ref+httpjson:///?[insecure=false&floatAsInt=false]mode=singleparam#/` + +Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos: +```json +[ + { + "id": 251296379 + } +] +``` +``` +# Running the following will return: 2.51296379e+08 +ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/id +# Running the following will return: 251296379 +ref+httpjson://api.github.com/users/helmfile/repos?floatAsInt=true&mode=singleparam#///*[1]/id +``` + + + ## Advanced Usages ### Discriminating config and secrets diff --git a/go.mod b/go.mod index c4cfd15..b480bd0 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,8 @@ require ( ) require ( + github.com/antchfx/jsonquery v1.3.3 // indirect + github.com/antchfx/xpath v1.2.3 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect diff --git a/go.sum b/go.sum index 001ce71..e6bf185 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,10 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg= github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/antchfx/jsonquery v1.3.3 h1:zjZpbnZhYng3uOAbIfdNq81A9mMEeuDJeYIpeKpZ4es= +github.com/antchfx/jsonquery v1.3.3/go.mod h1:1JG4DqRlRCHgVYDPY1ioYFAGSXGfWHzNgrbiGQHsWck= +github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= diff --git a/pkg/providers/httpjson/httpjson.go b/pkg/providers/httpjson/httpjson.go new file mode 100644 index 0000000..ccfb7d5 --- /dev/null +++ b/pkg/providers/httpjson/httpjson.go @@ -0,0 +1,138 @@ +package httpjson + +import ( + "fmt" + "strconv" + "strings" + + "github.com/antchfx/jsonquery" + "github.com/helmfile/vals/pkg/api" + "github.com/helmfile/vals/pkg/log" +) + +type provider struct { + // Keeping track of httpjson services since we need a service per url + protocol string + log *log.Logger + docs map[string]*jsonquery.Node + floatAsInt bool +} + +func New(l *log.Logger, cfg api.StaticConfig) *provider { + p := &provider{ + log: l, + } + + // Should the protocol be insecure i.e. http + insecureArg := cfg.String("insecure") + p.protocol = "https" + if insecureArg == "true" { + p.protocol = "http" + } + + // By default JSON will return large integers as float64 + floatAsIntArg := cfg.String("floatAsInt") + p.floatAsInt = false + if floatAsIntArg == "true" { + p.floatAsInt = true + } + + // Initialize docs map to store the json object for use multiple times + if len(p.docs) == 0 { + p.docs = make(map[string]*jsonquery.Node) + } + + return p +} + +func GetXpathFromUri(uri string) (xpath string, err error) { + found := strings.Split(uri, "mode=singleparam#")[1] + found = strings.Split(found, "&")[0] + xpath = strings.TrimPrefix(found, "/") + + return xpath, nil +} + +func getUrlFromUri(uri string, protocol string) (string, error) { + // Grab url from uri + uriParts := strings.Split(uri, "?") + if len(uriParts) < 2 { + return "", fmt.Errorf("error getting url from uri: %v, ensure xpath singleparam is set as a query parameter", uri) + } + url := strings.Replace(uriParts[0], "httpjson", protocol, 1) + + return url, nil +} + +func (p *provider) GetJsonDoc(url string) error { + if _, ok := p.docs[url]; !ok { + doc, err := jsonquery.LoadURL(url) + if err != nil { + return fmt.Errorf("error fetching json document at %v: %v", url, err) + } + p.docs[url] = doc + } + + return nil +} + +func (p *provider) GetString(uri string) (string, error) { + url, err := getUrlFromUri(uri, p.protocol) + if err != nil { + return "", err + } + err = p.GetJsonDoc(url) + if err != nil { + return "", err + } + xpathQuery, err := GetXpathFromUri(uri) + if err != nil { + return "", err + } + + returnValue := "" + var values []string + node, err := jsonquery.Query(p.docs[url], xpathQuery) + if err != nil || node == nil { + return "", fmt.Errorf("unable to query doc for value with xpath query using %v", uri) + } + + if node.FirstChild.Data != node.LastChild.Data { + return "", fmt.Errorf("location %v has child nodes at %v, please use a more granular query", xpathQuery, url) + } + + childNodesLength := countChildNodes(node) + + if childNodesLength > 1 { + for child := node.FirstChild; child != nil; child = child.NextSibling { + values = append(values, child.Value().(string)) + } + returnValue = strings.Join(values, ",") + + } else { + returnValue = node.FirstChild.Value().(string) + } + + if p.floatAsInt { + intValue, err := strconv.ParseFloat(returnValue, 64) + if err != nil { + return "", fmt.Errorf("unable to convert possible float to int for value: %v", returnValue) + } + returnValue = fmt.Sprintf("%.0f", intValue) + } + + return returnValue, nil +} + +func countChildNodes(node *jsonquery.Node) int { + // Check if there are more child nodes i.e. keys under this json key + count := 0 + for child := node.FirstChild; child != nil; child = child.NextSibling { + count++ + } + return count +} + +func (p *provider) GetStringMap(key string) (map[string]interface{}, error) { + return nil, fmt.Errorf("we should not be in the GetStringMap method") +} diff --git a/pkg/stringmapprovider/stringmapprovider.go b/pkg/stringmapprovider/stringmapprovider.go index 28e2a39..cc8bc4d 100644 --- a/pkg/stringmapprovider/stringmapprovider.go +++ b/pkg/stringmapprovider/stringmapprovider.go @@ -11,6 +11,7 @@ import ( "github.com/helmfile/vals/pkg/providers/doppler" "github.com/helmfile/vals/pkg/providers/gcpsecrets" "github.com/helmfile/vals/pkg/providers/gkms" + "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/sops" @@ -46,6 +47,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringMapProvi return gkms.New(l, provider), nil case "k8s": return k8s.New(l, provider) + case "httpjson": + return httpjson.New(l, provider), nil } return nil, fmt.Errorf("failed initializing string-map provider from config: %v", provider) diff --git a/pkg/stringprovider/stringprovider.go b/pkg/stringprovider/stringprovider.go index 52e8b42..c00b9b0 100644 --- a/pkg/stringprovider/stringprovider.go +++ b/pkg/stringprovider/stringprovider.go @@ -15,6 +15,7 @@ import ( "github.com/helmfile/vals/pkg/providers/gitlab" "github.com/helmfile/vals/pkg/providers/gkms" "github.com/helmfile/vals/pkg/providers/hcpvaultsecrets" + "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/pulumi" @@ -73,6 +74,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringProvider return conjur.New(l, provider), nil case "hcpvaultsecrets": return hcpvaultsecrets.New(l, provider), nil + case "httpjson": + return httpjson.New(l, provider), nil } return nil, fmt.Errorf("failed initializing string provider from config: %v", provider) diff --git a/vals.go b/vals.go index 5c58720..12294a7 100644 --- a/vals.go +++ b/vals.go @@ -35,6 +35,7 @@ import ( "github.com/helmfile/vals/pkg/providers/gkms" "github.com/helmfile/vals/pkg/providers/googlesheets" "github.com/helmfile/vals/pkg/providers/hcpvaultsecrets" + "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/pulumi" @@ -96,6 +97,7 @@ const ( ProviderK8s = "k8s" ProviderConjur = "conjur" ProviderHCPVaultSecrets = "hcpvaultsecrets" + ProviderHttpJsonManager = "httpjson" ProviderBitwarden = "bw" ) @@ -264,6 +266,9 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { case ProviderHCPVaultSecrets: p := hcpvaultsecrets.New(r.logger, conf) return p, nil + case ProviderHttpJsonManager: + p := httpjson.New(r.logger, conf) + return p, nil case ProviderBitwarden: p := bitwarden.New(r.logger, conf) return p, nil @@ -367,6 +372,7 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { } return str, nil + } else { mapRequestURI := key[:strings.LastIndex(key, uri.Fragment)-1] var obj map[string]interface{} @@ -375,6 +381,19 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { if !ok { return "", fmt.Errorf("error reading map from cache: unsupported value type %T", cachedMap) } + } else if uri.Scheme == "httpjson" { + // Due to the unpredictability in the structure of the JSON object, + // an alternative parsing method is used here. + // The standard approach couldn't be applied because the JSON object + // may vary in its key-value pairs and nesting depth, making it difficult + // to reliably parse using conventional methods. + // This alternative approach allows for flexible handling of the JSON + // object, accommodating different configurations and variations. + value, err := p.GetString(uri.String()) + if err != nil { + return "", err + } + return value, nil } else { obj, err = p.GetStringMap(path) if err != nil { diff --git a/vals_httpjson_test.go b/vals_httpjson_test.go new file mode 100644 index 0000000..ebaf455 --- /dev/null +++ b/vals_httpjson_test.go @@ -0,0 +1,116 @@ +package vals + +import ( + "os" + "testing" + + config2 "github.com/helmfile/vals/pkg/config" +) + +func createProvider(providerPath string, inlineValue string, floatAsInt string) config2.MapConfig { + config := map[string]interface{}{ + "provider": map[string]interface{}{ + "name": "httpjson", + "path": providerPath, + "floatAsInt": floatAsInt, + }, + "inline": map[string]interface{}{ + "value": inlineValue, + }, + } + return config2.Map(config) +} + +// nolint +func Test_HttpJson(t *testing.T) { + if os.Getenv("SKIP_TESTS") != "" { + t.Skip("Skipping tests") + } + + t.Run("Get name from first array item", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/name", "false") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "chartify" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) + + t.Run("Get name from second array item", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[2]/name", "false") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "go-yaml" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) + + t.Run("Error getting document from location jsonquery.LoadURL", func(t *testing.T) { + config := createProvider("httpjson://boom.github.com/users/helmfile/repos?mode=singleparam#", "//owner", "false") + _, err := Load(config) + if err != nil { + expected := "error fetching json document at https://boom.github.com/users/helmfile/repos: invalid character '<' looking for beginning of value" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Error running json.Query", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "/boom", "false") + _, err := Load(config) + if err != nil { + expected := "unable to query doc for value with xpath query using httpjson://api.github.com/users/helmfile/repos?mode=singleparam#//boom" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Get Avatar URL with child nodes causing error", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//owner", "false") + _, err := Load(config) + if err != nil { + expected := "location //owner has child nodes at https://api.github.com/users/helmfile/repos, please use a more granular query" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Test floatAsInt Success", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/id", "true") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "251296379" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) + + t.Run("Test floatAsInt failure", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/name", "true") + _, err := Load(config) + if err != nil { + expected := "unable to convert possible float to int for value: chartify" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) +} From bbfdac5b5712aeebd9d5803bf2ad4c511ad18e51 Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Tue, 26 Mar 2024 09:25:00 -0400 Subject: [PATCH 2/7] 267: Add http json functionality - resolve linting issues - Create mock json server to run testing against Signed-off-by: Paul Grant --- pkg/providers/httpjson/httpjson.go | 2 +- vals_httpjson_test.go | 131 ++++++++++++++++++++++++++--- 2 files changed, 122 insertions(+), 11 deletions(-) diff --git a/pkg/providers/httpjson/httpjson.go b/pkg/providers/httpjson/httpjson.go index ccfb7d5..067800b 100644 --- a/pkg/providers/httpjson/httpjson.go +++ b/pkg/providers/httpjson/httpjson.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" + // nolint "github.com/antchfx/jsonquery" "github.com/helmfile/vals/pkg/api" "github.com/helmfile/vals/pkg/log" @@ -108,7 +109,6 @@ func (p *provider) GetString(uri string) (string, error) { values = append(values, child.Value().(string)) } returnValue = strings.Join(values, ",") - } else { returnValue = node.FirstChild.Value().(string) } diff --git a/vals_httpjson_test.go b/vals_httpjson_test.go index ebaf455..f6036de 100644 --- a/vals_httpjson_test.go +++ b/vals_httpjson_test.go @@ -1,18 +1,93 @@ package vals import ( + "encoding/json" + "net/http" + "net/http/httptest" "os" + "strings" "testing" config2 "github.com/helmfile/vals/pkg/config" ) +var server *httptest.Server + +func setup() { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Define the JSON data + data := []map[string]interface{}{ + { + "name": "chartify", + "id": 251296379, + "status": map[string]interface{}{ + "database": map[string]interface{}{ + "DBNodes": []string{ + "chartify.database1.io", + "chartify.database2.io", + "chartify.database3.io", + "chartify.database4.io", + "chartify.database5.io", + }, + }, + }, + "owner": map[string]interface{}{ + "login": "helmfile", + "id": 8319146, + }, + }, + { + "name": "go-yaml", + "id": 597918420, + "status": map[string]interface{}{ + "database": map[string]interface{}{ + "DBNodes": []string{ + "go-yaml.database1.io", + "go-yaml.database2.io", + "go-yaml.database3.io", + "go-yaml.database4.io", + "go-yaml.database5.io", + }, + }, + }, + "owner": map[string]interface{}{ + "login": "helmfile", + "id": 83191469, + }, + }, + } + + // Encode the JSON data + jsonData, err := json.Marshal(data) + if err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + // Set the Content-Type header + w.Header().Set("Content-Type", "application/json") + + // Write the JSON response + w.Write(jsonData) + }) + + // Create a test server (using any free port) + server = httptest.NewServer(handler) +} + +func teardown() { + // Close the test server + server.Close() +} + func createProvider(providerPath string, inlineValue string, floatAsInt string) config2.MapConfig { + // Construct the configuration map with the provided values config := map[string]interface{}{ "provider": map[string]interface{}{ "name": "httpjson", "path": providerPath, "floatAsInt": floatAsInt, + "insecure": "true", }, "inline": map[string]interface{}{ "value": inlineValue, @@ -27,8 +102,17 @@ func Test_HttpJson(t *testing.T) { t.Skip("Skipping tests") } + // Initialize a web server to serve JSON data for testing purposes + setup() + + // Teardown web server once testing is complete + defer teardown() + + // Get the server URL without the protocol + serverURLWithoutProtocol := strings.TrimPrefix(server.URL, "http://") + t.Run("Get name from first array item", func(t *testing.T) { - config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/name", "false") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[1]/name", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -41,7 +125,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Get name from second array item", func(t *testing.T) { - config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[2]/name", "false") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[2]/name", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -54,10 +138,10 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Error getting document from location jsonquery.LoadURL", func(t *testing.T) { - config := createProvider("httpjson://boom.github.com/users/helmfile/repos?mode=singleparam#", "//owner", "false") + config := createProvider("httpjson://boom.github.com/users/helmfile/repos?insecure=true&mode=singleparam#", "//owner", "false") _, err := Load(config) if err != nil { - expected := "error fetching json document at https://boom.github.com/users/helmfile/repos: invalid character '<' looking for beginning of value" + expected := "error fetching json document at http://boom.github.com/users/helmfile/repos: invalid character '<' looking for beginning of value" actual := err.Error() if actual != expected { t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) @@ -66,10 +150,24 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Error running json.Query", func(t *testing.T) { - config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "/boom", "false") + uri := "httpjson://" + serverURLWithoutProtocol + "?insecure=true&mode=singleparam#" + config := createProvider(uri, "/boom", "false") _, err := Load(config) if err != nil { - expected := "unable to query doc for value with xpath query using httpjson://api.github.com/users/helmfile/repos?mode=singleparam#//boom" + expected := "unable to query doc for value with xpath query using " + uri + "//boom" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Query list for comma separated string", func(t *testing.T) { + uri := "httpjson://" + serverURLWithoutProtocol + "?insecure=true&mode=singleparam#" + config := createProvider(uri, "/boom", "false") + _, err := Load(config) + if err != nil { + expected := "unable to query doc for value with xpath query using " + uri + "//boom" actual := err.Error() if actual != expected { t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) @@ -78,10 +176,10 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Get Avatar URL with child nodes causing error", func(t *testing.T) { - config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//owner", "false") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//owner", "false") _, err := Load(config) if err != nil { - expected := "location //owner has child nodes at https://api.github.com/users/helmfile/repos, please use a more granular query" + expected := "location //owner has child nodes at " + server.URL + ", please use a more granular query" actual := err.Error() if actual != expected { t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) @@ -90,7 +188,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test floatAsInt Success", func(t *testing.T) { - config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/id", "true") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&floatAsInt=true&mode=singleparam#", "//*[1]/id", "true") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -103,7 +201,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test floatAsInt failure", func(t *testing.T) { - config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/name", "true") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[1]/name", "false") _, err := Load(config) if err != nil { expected := "unable to convert possible float to int for value: chartify" @@ -113,4 +211,17 @@ func Test_HttpJson(t *testing.T) { } } }) + + t.Run("Test list returned as string", func(t *testing.T) { + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[1]/DBNodes", "false") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "chartify.database1.io,chartify.database2.io,chartify.database3.io,chartify.database4.io,chartify.database5.io" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) } From 3c00eb735acf0f355ae836ab378329fc52450eae Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Wed, 27 Mar 2024 10:20:32 -0400 Subject: [PATCH 3/7] 267: Add http json functionality - review comments - nolint,extra line Signed-off-by: Paul Grant --- .gitignore | 1 + pkg/providers/httpjson/httpjson.go | 2 +- vals.go | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 744604a..e27d184 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea *~ bin/vals +.vscode diff --git a/pkg/providers/httpjson/httpjson.go b/pkg/providers/httpjson/httpjson.go index 067800b..f2c3584 100644 --- a/pkg/providers/httpjson/httpjson.go +++ b/pkg/providers/httpjson/httpjson.go @@ -5,8 +5,8 @@ import ( "strconv" "strings" - // nolint "github.com/antchfx/jsonquery" + "github.com/helmfile/vals/pkg/api" "github.com/helmfile/vals/pkg/log" ) diff --git a/vals.go b/vals.go index 12294a7..ed9540c 100644 --- a/vals.go +++ b/vals.go @@ -372,7 +372,6 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { } return str, nil - } else { mapRequestURI := key[:strings.LastIndex(key, uri.Fragment)-1] var obj map[string]interface{} From 81aab50fc247451c0345da3297c344f37cc8e8c3 Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Mon, 1 Apr 2024 20:43:10 -0400 Subject: [PATCH 4/7] 267: Add http json functionality - review comments - unit testing and various fixes Signed-off-by: Paul Grant --- README.md | 30 +++--- pkg/providers/httpjson/httpjson.go | 47 ++++++--- vals.go | 3 +- vals_httpjson_test.go | 153 +++++++++++++++++++++++++++-- 4 files changed, 199 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 8a17814..5362db3 100644 --- a/README.md +++ b/README.md @@ -827,21 +827,21 @@ Examples: ### HTTP JSON -This provider retrieves values stored in JSON hosted by an HTTP frontend. +This provider retrieves values stored in JSON hosted by a HTTP frontend. This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages. Given the diverse array of JSON structures that can be encountered, utilizing jsonquery with XPath presents a more effective approach for handling this variability in data structures. -This provider requires an xpath to be provided in singleparam mode (as last argument). +This provider requires an xpath to be provided. -Do not include the protocol scheme i.e. http/https. Provider defaults to scheme https +Do not include the protocol scheme i.e. http/https. Provider defaults to scheme https (http is available, see below) Examples: #### Fetch string value -`ref+httpjson:///?[insecure=false&floatAsInt=false]mode=singleparam#/` +`ref+httpjson:///?[insecure=false&floatAsInt=false]#/` Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos: ```json @@ -855,17 +855,19 @@ Let's say you want to fetch the below JSON object from https://api.github.com/us ] ``` ``` -# To get name="chartify" using https protocol you would use: -ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/name -# To get name="go-yaml" using https protocol you would use: -ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[2]/name -# To get name="go-yaml" using http protocol you would use: -ref+httpjson://api.github.com/users/helmfile/repos?insecure=true&mode=singleparam#///*[2]/name +# To get name="chartify" using https protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos#///*[1]/name + +# To get name="go-yaml" using https protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos#///*[2]/name + +# To get name="go-yaml" using http protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?insecure=true#///*[2]/ ``` #### Fetch integer value -`ref+httpjson:///?[insecure=false&floatAsInt=false]mode=singleparam#/` +`ref+httpjson:///?[insecure=false&floatAsInt=false]#/` Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos: ```json @@ -877,13 +879,13 @@ Let's say you want to fetch the below JSON object from https://api.github.com/us ``` ``` # Running the following will return: 2.51296379e+08 -ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/id +ref+httpjson://api.github.com/users/helmfile/repos#///*[1]/id + # Running the following will return: 251296379 -ref+httpjson://api.github.com/users/helmfile/repos?floatAsInt=true&mode=singleparam#///*[1]/id +ref+httpjson://api.github.com/users/helmfile/repos?floatAsInt=true#///*[1]/id ``` - ## Advanced Usages ### Discriminating config and secrets diff --git a/pkg/providers/httpjson/httpjson.go b/pkg/providers/httpjson/httpjson.go index f2c3584..51413e8 100644 --- a/pkg/providers/httpjson/httpjson.go +++ b/pkg/providers/httpjson/httpjson.go @@ -2,10 +2,12 @@ package httpjson import ( "fmt" + "net/url" "strconv" "strings" "github.com/antchfx/jsonquery" + "github.com/antchfx/xpath" "github.com/helmfile/vals/pkg/api" "github.com/helmfile/vals/pkg/log" @@ -46,23 +48,42 @@ func New(l *log.Logger, cfg api.StaticConfig) *provider { return p } -func GetXpathFromUri(uri string) (xpath string, err error) { - found := strings.Split(uri, "mode=singleparam#")[1] - found = strings.Split(found, "&")[0] - xpath = strings.TrimPrefix(found, "/") +func GetXpathFromUri(uri string) (xpathExpression string, err error) { + paths := strings.Split(uri, "#/") + if len(paths) == 1 { + return "", fmt.Errorf("no xpath expression found in uri: %s", uri) + } + _, err = xpath.Compile(paths[1]) + if err != nil { + return "", fmt.Errorf("unable to compile xpath expression '%s' from uri: %s", xpathExpression, uri) + } + xpathExpression = paths[1] - return xpath, nil + return xpathExpression, nil } -func getUrlFromUri(uri string, protocol string) (string, error) { - // Grab url from uri - uriParts := strings.Split(uri, "?") - if len(uriParts) < 2 { - return "", fmt.Errorf("error getting url from uri: %v, ensure xpath singleparam is set as a query parameter", uri) +func GetUrlFromUri(uri string, protocol string) (string, error) { + // Remove httpjson:// prefix + trimmedStr := strings.TrimPrefix(uri, "httpjson://") + // Attempt to split uri on argument + uriParts := strings.Split(trimmedStr, "?") + urlDomain := "" + if len(uriParts) == 1 { + // Attempt to split uri on parameter + urlDomain = strings.Split(trimmedStr, "#")[0] + } else { + urlDomain = uriParts[0] + } + if urlDomain == "" { + return "", fmt.Errorf("no domain found in uri: %s", uri) + } + fullURL := fmt.Sprintf("%s://%s", protocol, urlDomain) + _, err := url.Parse(fullURL) + if err != nil { + return "", fmt.Errorf("invalid domain: %s", err.Error()) } - url := strings.Replace(uriParts[0], "httpjson", protocol, 1) - return url, nil + return fullURL, nil } func (p *provider) GetJsonDoc(url string) error { @@ -78,7 +99,7 @@ func (p *provider) GetJsonDoc(url string) error { } func (p *provider) GetString(uri string) (string, error) { - url, err := getUrlFromUri(uri, p.protocol) + url, err := GetUrlFromUri(uri, p.protocol) if err != nil { return "", err } diff --git a/vals.go b/vals.go index ed9540c..b2abba3 100644 --- a/vals.go +++ b/vals.go @@ -388,7 +388,8 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { // to reliably parse using conventional methods. // This alternative approach allows for flexible handling of the JSON // object, accommodating different configurations and variations. - value, err := p.GetString(uri.String()) + //value, err := p.GetString(uri.String()) + value, err := p.GetString(key) if err != nil { return "", err } diff --git a/vals_httpjson_test.go b/vals_httpjson_test.go index f6036de..13a0252 100644 --- a/vals_httpjson_test.go +++ b/vals_httpjson_test.go @@ -9,6 +9,7 @@ import ( "testing" config2 "github.com/helmfile/vals/pkg/config" + "github.com/helmfile/vals/pkg/providers/httpjson" ) var server *httptest.Server @@ -112,7 +113,7 @@ func Test_HttpJson(t *testing.T) { serverURLWithoutProtocol := strings.TrimPrefix(server.URL, "http://") t.Run("Get name from first array item", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[1]/name", "false") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true#", "//*[1]/name", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -125,7 +126,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Get name from second array item", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[2]/name", "false") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true#", "//*[2]/name", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -138,7 +139,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Error getting document from location jsonquery.LoadURL", func(t *testing.T) { - config := createProvider("httpjson://boom.github.com/users/helmfile/repos?insecure=true&mode=singleparam#", "//owner", "false") + config := createProvider("httpjson://boom.github.com/users/helmfile/repos?insecure=true#", "//owner", "false") _, err := Load(config) if err != nil { expected := "error fetching json document at http://boom.github.com/users/helmfile/repos: invalid character '<' looking for beginning of value" @@ -150,7 +151,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Error running json.Query", func(t *testing.T) { - uri := "httpjson://" + serverURLWithoutProtocol + "?insecure=true&mode=singleparam#" + uri := "httpjson://" + serverURLWithoutProtocol + "?insecure=true#" config := createProvider(uri, "/boom", "false") _, err := Load(config) if err != nil { @@ -188,7 +189,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test floatAsInt Success", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&floatAsInt=true&mode=singleparam#", "//*[1]/id", "true") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&floatAsInt=true#", "//*[1]/id", "true") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -201,7 +202,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test floatAsInt failure", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[1]/name", "false") + config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true#", "//*[1]/name", "false") _, err := Load(config) if err != nil { expected := "unable to convert possible float to int for value: chartify" @@ -225,3 +226,143 @@ func Test_HttpJson(t *testing.T) { } }) } + +func Test_HttpJson_UnitTests(t *testing.T) { + if os.Getenv("SKIP_TESTS") != "" { + t.Skip("Skipping tests") + } + + // GetUrlFromUri + t.Run("GetUrlFromUri: valid (http)", func(t *testing.T) { + returnValue, err := httpjson.GetUrlFromUri("httpjson://boom.com/path?insecure=true#///*[1]/name", "http") + if err != nil { + t.Fatalf("%v", err) + } + expected := "http://boom.com/path" + if returnValue != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue) + } + }) + t.Run("GetUrlFromUri: valid (https)", func(t *testing.T) { + returnValue, err := httpjson.GetUrlFromUri("httpjson://boom.com/path#///*[1]/name", "https") + if err != nil { + t.Fatalf("%v", err) + } + expected := "https://boom.com/path" + if returnValue != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue) + } + }) + t.Run("GetUrlFromUri: invalid character in host name (https)", func(t *testing.T) { + _, err := httpjson.GetUrlFromUri("httpjson://supsupsup^boom#///*[1]/name", "https") + if err != nil { + expected := "invalid domain: parse \"https://supsupsup^boom\": invalid character \"^\" in host name" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetUrlFromUri: no domain provided (http)", func(t *testing.T) { + _, err := httpjson.GetUrlFromUri("httpjson://?insecure=true#///*[1]/name", "http") + if err != nil { + expected := "no domain found in uri: httpjson://?insecure=true#///*[1]/name" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetUrlFromUri: no domain provided (https)", func(t *testing.T) { + _, err := httpjson.GetUrlFromUri("httpjson://#///*[1]/name", "https") + if err != nil { + expected := "no domain found in uri: httpjson://#///*[1]/name" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + // GetXpathFromUri + t.Run("GetXpathFromUri: valid (http)", func(t *testing.T) { + returnValue, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah?insecure=true#///*[1]/name") + if err != nil { + t.Fatalf("%v", err) + } + expected := "//*[1]/name" + if returnValue != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue) + } + }) + t.Run("GetXpathFromUri: valid (https)", func(t *testing.T) { + returnValue, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah#///*[1]/name") + if err != nil { + t.Fatalf("%v", err) + } + expected := "//*[1]/name" + if returnValue != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, returnValue) + } + }) + t.Run("GetXpathFromUri: no xpath provided (http)", func(t *testing.T) { + _, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah?insecure=true") + if err != nil { + expected := "no xpath expression found in uri: httpjson://blah.blah/blah?insecure=true" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetXpathFromUri: no xpath provided (https)", func(t *testing.T) { + _, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah") + if err != nil { + expected := "no xpath expression found in uri: httpjson://blah.blah/blah" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetXpathFromUri: invalid xpath 1 (http)", func(t *testing.T) { + _, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah?insecure=true#/") + if err != nil { + expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah?insecure=true#/" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetXpathFromUri: invalid xpath 1 (https)", func(t *testing.T) { + _, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah#/") + if err != nil { + expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah#/" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetXpathFromUri: invalid xpath 2 (http)", func(t *testing.T) { + _, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah?insecure=true#/hello^sup") + if err != nil { + expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah?insecure=true#/hello^sup" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + t.Run("GetXpathFromUri: invalid xpath 2 (https)", func(t *testing.T) { + _, err := httpjson.GetXpathFromUri("httpjson://blah.blah/blah#/hello^sup") + if err != nil { + expected := "unable to compile xpath expression '' from uri: httpjson://blah.blah/blah#/hello^sup" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) +} From 7ea832d0681d0341d76c14636c05a58c79b64b7c Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Mon, 1 Apr 2024 21:02:15 -0400 Subject: [PATCH 5/7] 267: Add http json functionality - review comments - unit testing and various fixes Signed-off-by: Paul Grant --- vals.go | 1 - 1 file changed, 1 deletion(-) diff --git a/vals.go b/vals.go index b2abba3..2ccb825 100644 --- a/vals.go +++ b/vals.go @@ -388,7 +388,6 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { // to reliably parse using conventional methods. // This alternative approach allows for flexible handling of the JSON // object, accommodating different configurations and variations. - //value, err := p.GetString(uri.String()) value, err := p.GetString(key) if err != nil { return "", err From 5c172e4f11c1f76e31b222ddbf62fb6e39e10d9b Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Wed, 3 Apr 2024 11:49:20 -0400 Subject: [PATCH 6/7] 267: Add http json functionality - review comments - remove nolint Signed-off-by: Paul Grant --- vals_httpjson_test.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/vals_httpjson_test.go b/vals_httpjson_test.go index 13a0252..772966f 100644 --- a/vals_httpjson_test.go +++ b/vals_httpjson_test.go @@ -2,6 +2,7 @@ package vals import ( "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -12,6 +13,8 @@ import ( "github.com/helmfile/vals/pkg/providers/httpjson" ) +const HttpJsonPrefix = "httpjson://" + var server *httptest.Server func setup() { @@ -97,7 +100,6 @@ func createProvider(providerPath string, inlineValue string, floatAsInt string) return config2.Map(config) } -// nolint func Test_HttpJson(t *testing.T) { if os.Getenv("SKIP_TESTS") != "" { t.Skip("Skipping tests") @@ -111,9 +113,10 @@ func Test_HttpJson(t *testing.T) { // Get the server URL without the protocol serverURLWithoutProtocol := strings.TrimPrefix(server.URL, "http://") + prefixAndPath := fmt.Sprintf("httpjson://%v", serverURLWithoutProtocol) t.Run("Get name from first array item", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true#", "//*[1]/name", "false") + config := createProvider(prefixAndPath+"?insecure=true#", "//*[1]/name", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -126,7 +129,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Get name from second array item", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true#", "//*[2]/name", "false") + config := createProvider(prefixAndPath+"?insecure=true#", "//*[2]/name", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -151,7 +154,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Error running json.Query", func(t *testing.T) { - uri := "httpjson://" + serverURLWithoutProtocol + "?insecure=true#" + uri := prefixAndPath + "?insecure=true#" config := createProvider(uri, "/boom", "false") _, err := Load(config) if err != nil { @@ -164,7 +167,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Query list for comma separated string", func(t *testing.T) { - uri := "httpjson://" + serverURLWithoutProtocol + "?insecure=true&mode=singleparam#" + uri := prefixAndPath + "?insecure=true&mode=singleparam#" config := createProvider(uri, "/boom", "false") _, err := Load(config) if err != nil { @@ -177,7 +180,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Get Avatar URL with child nodes causing error", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//owner", "false") + config := createProvider(prefixAndPath+"?insecure=true&mode=singleparam#", "//owner", "false") _, err := Load(config) if err != nil { expected := "location //owner has child nodes at " + server.URL + ", please use a more granular query" @@ -189,7 +192,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test floatAsInt Success", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&floatAsInt=true#", "//*[1]/id", "true") + config := createProvider(prefixAndPath+"?insecure=true&floatAsInt=true#", "//*[1]/id", "true") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) @@ -202,7 +205,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test floatAsInt failure", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true#", "//*[1]/name", "false") + config := createProvider(prefixAndPath+"?insecure=true#", "//*[1]/name", "false") _, err := Load(config) if err != nil { expected := "unable to convert possible float to int for value: chartify" @@ -214,7 +217,7 @@ func Test_HttpJson(t *testing.T) { }) t.Run("Test list returned as string", func(t *testing.T) { - config := createProvider("httpjson://"+serverURLWithoutProtocol+"?insecure=true&mode=singleparam#", "//*[1]/DBNodes", "false") + config := createProvider(prefixAndPath+"?insecure=true&mode=singleparam#", "//*[1]/DBNodes", "false") vals, err := Load(config) if err != nil { t.Fatalf("%v", err) From f3d91ea0d209db5542d26d25898b8625d1752fe9 Mon Sep 17 00:00:00 2001 From: Paul Grant Date: Wed, 3 Apr 2024 20:39:51 -0400 Subject: [PATCH 7/7] 267: Add http json functionality - review comments - added debug message Signed-off-by: Paul Grant --- pkg/providers/httpjson/httpjson.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/providers/httpjson/httpjson.go b/pkg/providers/httpjson/httpjson.go index 51413e8..cad278b 100644 --- a/pkg/providers/httpjson/httpjson.go +++ b/pkg/providers/httpjson/httpjson.go @@ -92,6 +92,7 @@ func (p *provider) GetJsonDoc(url string) error { if err != nil { return fmt.Errorf("error fetching json document at %v: %v", url, err) } + p.log.Debugf("httpjson: successfully retrieved JSON data from: %s", url) p.docs[url] = doc }