diff --git a/README.md b/README.md index 690e3710..8a178147 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ It supports various backends including: - Conjur - HCP Vault Secrets - Bitwarden +- HTTP JSON - Use `vals eval -f refs.yaml` to replace all the `ref`s in the file to actual values and secrets. - Use `vals exec -f env.yaml -- ` to populate envvars and execute the command. @@ -225,6 +226,7 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/ - [Kubernetes](#kubernetes) - [Conjur](#conjur) - [HCP Vault Secrets](#hcp-vault-secrets) +- [HTTP JSON](#http-json) - [Bitwarden](#bitwarden) Please see [pkg/providers](https://github.com/helmfile/vals/tree/master/pkg/providers) for the implementations of all the providers. The package names corresponds to the URI schemes. @@ -823,6 +825,65 @@ Examples: - `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/{username,password,uri,notes,item}` gets username, password, uri, notes or the whole item of the given item id - `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/notes#/key1` gets the *key1* from the yaml stored as note in the item +### HTTP JSON + +This provider retrieves values stored in JSON hosted by an HTTP frontend. + +This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages. + +Given the diverse array of JSON structures that can be encountered, utilizing jsonquery with XPath presents a more effective approach for handling this variability in data structures. + +This provider requires an xpath to be provided in singleparam mode (as last argument). + +Do not include the protocol scheme i.e. http/https. Provider defaults to scheme https + +Examples: + +#### Fetch string value + +`ref+httpjson:///?[insecure=false&floatAsInt=false]mode=singleparam#/` + +Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos: +```json +[ + { + "name": "chartify" + }, + { + "name": "go-yaml" + } +] +``` +``` +# To get name="chartify" using https protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/name +# To get name="go-yaml" using https protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[2]/name +# To get name="go-yaml" using http protocol you would use: +ref+httpjson://api.github.com/users/helmfile/repos?insecure=true&mode=singleparam#///*[2]/name +``` + +#### Fetch integer value + +`ref+httpjson:///?[insecure=false&floatAsInt=false]mode=singleparam#/` + +Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos: +```json +[ + { + "id": 251296379 + } +] +``` +``` +# Running the following will return: 2.51296379e+08 +ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/id +# Running the following will return: 251296379 +ref+httpjson://api.github.com/users/helmfile/repos?floatAsInt=true&mode=singleparam#///*[1]/id +``` + + + ## Advanced Usages ### Discriminating config and secrets diff --git a/go.mod b/go.mod index c4cfd153..b480bd02 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,8 @@ require ( ) require ( + github.com/antchfx/jsonquery v1.3.3 // indirect + github.com/antchfx/xpath v1.2.3 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect diff --git a/go.sum b/go.sum index 001ce719..e6bf1857 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,10 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg= github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/antchfx/jsonquery v1.3.3 h1:zjZpbnZhYng3uOAbIfdNq81A9mMEeuDJeYIpeKpZ4es= +github.com/antchfx/jsonquery v1.3.3/go.mod h1:1JG4DqRlRCHgVYDPY1ioYFAGSXGfWHzNgrbiGQHsWck= +github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= diff --git a/pkg/providers/httpjson/httpjson.go b/pkg/providers/httpjson/httpjson.go new file mode 100644 index 00000000..c3c2e77a --- /dev/null +++ b/pkg/providers/httpjson/httpjson.go @@ -0,0 +1,137 @@ +package httpjson + +import ( + "fmt" + "strconv" + "strings" + + "github.com/antchfx/jsonquery" + "github.com/helmfile/vals/pkg/api" + "github.com/helmfile/vals/pkg/log" +) + +type provider struct { + // Keeping track of httpjson services since we need a service per url + protocol string + log *log.Logger + docs map[string]*jsonquery.Node + floatAsInt bool +} + +func New(l *log.Logger, cfg api.StaticConfig) *provider { + p := &provider{ + log: l, + } + + // Should the protocol be insecure i.e. http + insecureArg := cfg.String("insecure") + p.protocol = "https" + if insecureArg == "true" { + p.protocol = "http" + } + + // By default JSON will return large integers as float64 + floatAsIntArg := cfg.String("floatAsInt") + p.floatAsInt = false + if floatAsIntArg == "true" { + p.floatAsInt = true + } + + // Initialize docs map to store the json object for use multiple times + if len(p.docs) == 0 { + p.docs = make(map[string]*jsonquery.Node) + } + + return p +} + +func GetXpathFromUri(uri string) (xpath string, err error) { + found := strings.Split(uri, "mode=singleparam#")[1] + found = strings.Split(found, "&")[0] + xpath = strings.TrimPrefix(found, "/") + + return xpath, nil +} + +func getUrlFromUri(uri string, protocol string) (string, error) { + // Grab url from uri + uriParts := strings.Split(uri, "?") + if len(uriParts) < 2 { + return "", fmt.Errorf("error getting url from uri: %v, ensure xpath singleparam is set as a query parameter", uri) + } + url := strings.Replace(uriParts[0], "httpjson", protocol, 1) + + return url, nil +} + +func (p *provider) GetJsonDoc(url string) error { + if _, ok := p.docs[url]; !ok { + doc, err := jsonquery.LoadURL(url) + if err != nil { + return fmt.Errorf("error fetching json document at %v: %v", url, err) + } + p.docs[url] = doc + } + + return nil +} + +func (p *provider) GetString(uri string) (string, error) { + url, err := getUrlFromUri(uri, p.protocol) + if err != nil { + return "", err + } + err = p.GetJsonDoc(url) + if err != nil { + return "", err + } + xpathQuery, err := GetXpathFromUri(uri) + if err != nil { + return "", err + } + + returnValue := "" + var values []string + node, err := jsonquery.Query(p.docs[url], xpathQuery) + if err != nil || node == nil { + return "", fmt.Errorf("unable to query doc for value with xpath query using %v", uri) + } + + if node.FirstChild.Data != node.LastChild.Data { + return "", fmt.Errorf("location %v has child nodes at %v, please use a more granular query", xpathQuery, url) + } + + childNodesLength := countChildNodes(node) + + if childNodesLength > 1 { + for child := node.FirstChild; child != nil; child = child.NextSibling { + values = append(values, child.Value().(string)) + } + returnValue = strings.Join(values, ",") + + } else { + returnValue = node.FirstChild.Value().(string) + } + + if p.floatAsInt { + intValue, err := strconv.ParseFloat(returnValue, 64) + if err != nil { + return "", fmt.Errorf("unable to convert possible float to int for value: %v", returnValue) + } + returnValue = fmt.Sprintf("%.0f", intValue) + } + + return returnValue, nil +} + +func countChildNodes(node *jsonquery.Node) int { + count := 0 + for child := node.FirstChild; child != nil; child = child.NextSibling { + count++ + } + return count +} + +func (p *provider) GetStringMap(key string) (map[string]interface{}, error) { + return nil, fmt.Errorf("we should not be in the GetStringMap method") +} diff --git a/pkg/stringmapprovider/stringmapprovider.go b/pkg/stringmapprovider/stringmapprovider.go index 28e2a39d..cc8bc4d7 100644 --- a/pkg/stringmapprovider/stringmapprovider.go +++ b/pkg/stringmapprovider/stringmapprovider.go @@ -11,6 +11,7 @@ import ( "github.com/helmfile/vals/pkg/providers/doppler" "github.com/helmfile/vals/pkg/providers/gcpsecrets" "github.com/helmfile/vals/pkg/providers/gkms" + "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/sops" @@ -46,6 +47,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringMapProvi return gkms.New(l, provider), nil case "k8s": return k8s.New(l, provider) + case "httpjson": + return httpjson.New(l, provider), nil } return nil, fmt.Errorf("failed initializing string-map provider from config: %v", provider) diff --git a/pkg/stringprovider/stringprovider.go b/pkg/stringprovider/stringprovider.go index 52e8b421..c00b9b0a 100644 --- a/pkg/stringprovider/stringprovider.go +++ b/pkg/stringprovider/stringprovider.go @@ -15,6 +15,7 @@ import ( "github.com/helmfile/vals/pkg/providers/gitlab" "github.com/helmfile/vals/pkg/providers/gkms" "github.com/helmfile/vals/pkg/providers/hcpvaultsecrets" + "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/pulumi" @@ -73,6 +74,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringProvider return conjur.New(l, provider), nil case "hcpvaultsecrets": return hcpvaultsecrets.New(l, provider), nil + case "httpjson": + return httpjson.New(l, provider), nil } return nil, fmt.Errorf("failed initializing string provider from config: %v", provider) diff --git a/vals.go b/vals.go index 5c58720c..12294a7b 100644 --- a/vals.go +++ b/vals.go @@ -35,6 +35,7 @@ import ( "github.com/helmfile/vals/pkg/providers/gkms" "github.com/helmfile/vals/pkg/providers/googlesheets" "github.com/helmfile/vals/pkg/providers/hcpvaultsecrets" + "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/pulumi" @@ -96,6 +97,7 @@ const ( ProviderK8s = "k8s" ProviderConjur = "conjur" ProviderHCPVaultSecrets = "hcpvaultsecrets" + ProviderHttpJsonManager = "httpjson" ProviderBitwarden = "bw" ) @@ -264,6 +266,9 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { case ProviderHCPVaultSecrets: p := hcpvaultsecrets.New(r.logger, conf) return p, nil + case ProviderHttpJsonManager: + p := httpjson.New(r.logger, conf) + return p, nil case ProviderBitwarden: p := bitwarden.New(r.logger, conf) return p, nil @@ -367,6 +372,7 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { } return str, nil + } else { mapRequestURI := key[:strings.LastIndex(key, uri.Fragment)-1] var obj map[string]interface{} @@ -375,6 +381,19 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { if !ok { return "", fmt.Errorf("error reading map from cache: unsupported value type %T", cachedMap) } + } else if uri.Scheme == "httpjson" { + // Due to the unpredictability in the structure of the JSON object, + // an alternative parsing method is used here. + // The standard approach couldn't be applied because the JSON object + // may vary in its key-value pairs and nesting depth, making it difficult + // to reliably parse using conventional methods. + // This alternative approach allows for flexible handling of the JSON + // object, accommodating different configurations and variations. + value, err := p.GetString(uri.String()) + if err != nil { + return "", err + } + return value, nil } else { obj, err = p.GetStringMap(path) if err != nil { diff --git a/vals_httpjson_test.go b/vals_httpjson_test.go new file mode 100644 index 00000000..ebaf4555 --- /dev/null +++ b/vals_httpjson_test.go @@ -0,0 +1,116 @@ +package vals + +import ( + "os" + "testing" + + config2 "github.com/helmfile/vals/pkg/config" +) + +func createProvider(providerPath string, inlineValue string, floatAsInt string) config2.MapConfig { + config := map[string]interface{}{ + "provider": map[string]interface{}{ + "name": "httpjson", + "path": providerPath, + "floatAsInt": floatAsInt, + }, + "inline": map[string]interface{}{ + "value": inlineValue, + }, + } + return config2.Map(config) +} + +// nolint +func Test_HttpJson(t *testing.T) { + if os.Getenv("SKIP_TESTS") != "" { + t.Skip("Skipping tests") + } + + t.Run("Get name from first array item", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/name", "false") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "chartify" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) + + t.Run("Get name from second array item", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[2]/name", "false") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "go-yaml" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) + + t.Run("Error getting document from location jsonquery.LoadURL", func(t *testing.T) { + config := createProvider("httpjson://boom.github.com/users/helmfile/repos?mode=singleparam#", "//owner", "false") + _, err := Load(config) + if err != nil { + expected := "error fetching json document at https://boom.github.com/users/helmfile/repos: invalid character '<' looking for beginning of value" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Error running json.Query", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "/boom", "false") + _, err := Load(config) + if err != nil { + expected := "unable to query doc for value with xpath query using httpjson://api.github.com/users/helmfile/repos?mode=singleparam#//boom" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Get Avatar URL with child nodes causing error", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//owner", "false") + _, err := Load(config) + if err != nil { + expected := "location //owner has child nodes at https://api.github.com/users/helmfile/repos, please use a more granular query" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) + + t.Run("Test floatAsInt Success", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/id", "true") + vals, err := Load(config) + if err != nil { + t.Fatalf("%v", err) + } + expected := "251296379" + actual := vals["value"] + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + }) + + t.Run("Test floatAsInt failure", func(t *testing.T) { + config := createProvider("httpjson://api.github.com/users/helmfile/repos?mode=singleparam#", "//*[1]/name", "true") + _, err := Load(config) + if err != nil { + expected := "unable to convert possible float to int for value: chartify" + actual := err.Error() + if actual != expected { + t.Errorf("unepected value for key %q: expected=%q, got=%q", "value", expected, actual) + } + } + }) +}