267: Add http json functionality

.gitignore

@@ -1,3 +1,4 @@
 .idea
 *~
 bin/vals
+.vscode

README.md

@@ -21,6 +21,7 @@ It supports various backends including:
 - Conjur
 - HCP Vault Secrets
 - Bitwarden
+- HTTP JSON
 
 - Use `vals eval -f refs.yaml` to replace all the `ref`s in the file to actual values and secrets.
 - Use `vals exec -f env.yaml -- <COMMAND>` to populate envvars and execute the command.
@@ -225,6 +226,7 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/
 - [Kubernetes](#kubernetes)
 - [Conjur](#conjur)
 - [HCP Vault Secrets](#hcp-vault-secrets)
+- [HTTP JSON](#http-json)
 - [Bitwarden](#bitwarden)
 
 Please see [pkg/providers](https://github.com/helmfile/vals/tree/master/pkg/providers) for the implementations of all the providers. The package names corresponds to the URI schemes.
@@ -823,6 +825,67 @@ Examples:
 - `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/{username,password,uri,notes,item}` gets username, password, uri, notes or the whole item of the given item id
 - `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/notes#/key1` gets the *key1* from the yaml stored as note in the item
 
+### HTTP JSON
+
+This provider retrieves values stored in JSON hosted by a HTTP frontend.
+
+This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages.  
+
+Given the diverse array of JSON structures that can be encountered, utilizing jsonquery with XPath presents a more effective approach for handling this variability in data structures.
+
+This provider requires an xpath to be provided.
+
+Do not include the protocol scheme i.e. http/https. Provider defaults to scheme https (http is available, see below)
+
+Examples:
+
+#### Fetch string value
+
+`ref+httpjson://<domain>/<path>?[insecure=false&floatAsInt=false]#/<xpath>`
+
+Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos:
+```json
+[
+    {
+        "name": "chartify"
+    },
+    {
+        "name": "go-yaml"
+    }
+]
+``` 
+```
+# To get name="chartify" using https protocol you would use:
+ref+httpjson://api.github.com/users/helmfile/repos#///*[1]/name
+
+# To get name="go-yaml" using https protocol you would use:
+ref+httpjson://api.github.com/users/helmfile/repos#///*[2]/name
+
+# To get name="go-yaml" using http protocol you would use:
+ref+httpjson://api.github.com/users/helmfile/repos?insecure=true#///*[2]/
+```
+
+#### Fetch integer value
+
+`ref+httpjson://<domain>/<path>?[insecure=false&floatAsInt=false]#/<xpath>`
+
+Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos:
+```json
+[
+    {
+        "id": 251296379
+    }
+]
+``` 
+```
+# Running the following will return: 2.51296379e+08
+ref+httpjson://api.github.com/users/helmfile/repos#///*[1]/id
+
+# Running the following will return: 251296379
+ref+httpjson://api.github.com/users/helmfile/repos?floatAsInt=true#///*[1]/id
+```
+
+
 ## Advanced Usages
 
 ### Discriminating config and secrets

go.mod

@@ -30,6 +30,8 @@ require (
 )
 
 require (
+	github.com/antchfx/jsonquery v1.3.3 // indirect
+	github.com/antchfx/xpath v1.2.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/go-openapi/analysis v0.23.0 // indirect
 	github.com/go-openapi/errors v0.22.0 // indirect

go.sum

@@ -76,6 +76,10 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
 github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
 github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/antchfx/jsonquery v1.3.3 h1:zjZpbnZhYng3uOAbIfdNq81A9mMEeuDJeYIpeKpZ4es=
+github.com/antchfx/jsonquery v1.3.3/go.mod h1:1JG4DqRlRCHgVYDPY1ioYFAGSXGfWHzNgrbiGQHsWck=
+github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes=
+github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=

pkg/providers/httpjson/httpjson.go

@@ -0,0 +1,160 @@
+package httpjson
+
+import (
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/antchfx/jsonquery"
+	"github.com/antchfx/xpath"
+
+	"github.com/helmfile/vals/pkg/api"
+	"github.com/helmfile/vals/pkg/log"
+)
+
+type provider struct {
+	// Keeping track of httpjson services since we need a service per url
+	protocol   string
+	log        *log.Logger
+	docs       map[string]*jsonquery.Node
+	floatAsInt bool
+}
+
+func New(l *log.Logger, cfg api.StaticConfig) *provider {
+	p := &provider{
+		log: l,
+	}
+
+	// Should the protocol be insecure i.e. http
+	insecureArg := cfg.String("insecure")
+	p.protocol = "https"
+	if insecureArg == "true" {
+		p.protocol = "http"
+	}
+
+	// By default JSON will return large integers as float64
+	floatAsIntArg := cfg.String("floatAsInt")
+	p.floatAsInt = false
+	if floatAsIntArg == "true" {
+		p.floatAsInt = true
+	}
+
+	// Initialize docs map to store the json object for use multiple times
+	if len(p.docs) == 0 {
+		p.docs = make(map[string]*jsonquery.Node)
+	}
+
+	return p
+}
+
+func GetXpathFromUri(uri string) (xpathExpression string, err error) {
+	paths := strings.Split(uri, "#/")
+	if len(paths) == 1 {
+		return "", fmt.Errorf("no xpath expression found in uri: %s", uri)
+	}
+	_, err = xpath.Compile(paths[1])
+	if err != nil {
+		return "", fmt.Errorf("unable to compile xpath expression '%s' from uri: %s", xpathExpression, uri)
+	}
+	xpathExpression = paths[1]
+
+	return xpathExpression, nil
+}
+
+func GetUrlFromUri(uri string, protocol string) (string, error) {
+	// Remove httpjson:// prefix
+	trimmedStr := strings.TrimPrefix(uri, "httpjson://")
+	// Attempt to split uri on argument
+	uriParts := strings.Split(trimmedStr, "?")
+	urlDomain := ""
+	if len(uriParts) == 1 {
+		// Attempt to split uri on parameter
+		urlDomain = strings.Split(trimmedStr, "#")[0]
+	} else {
+		urlDomain = uriParts[0]
+	}
+	if urlDomain == "" {
+		return "", fmt.Errorf("no domain found in uri: %s", uri)
+	}
+	fullURL := fmt.Sprintf("%s://%s", protocol, urlDomain)
+	_, err := url.Parse(fullURL)
+	if err != nil {
+		return "", fmt.Errorf("invalid domain: %s", err.Error())
+	}
+
+	return fullURL, nil
+}
+
+func (p *provider) GetJsonDoc(url string) error {
+	if _, ok := p.docs[url]; !ok {
+		doc, err := jsonquery.LoadURL(url)
+		if err != nil {
+			return fmt.Errorf("error fetching json document at %v: %v", url, err)
+		}
+		p.log.Debugf("httpjson: successfully retrieved JSON data from: %s", url)
+		p.docs[url] = doc
+	}
+
+	return nil
+}
+
+func (p *provider) GetString(uri string) (string, error) {
+	url, err := GetUrlFromUri(uri, p.protocol)
+	if err != nil {
+		return "", err
+	}
+	err = p.GetJsonDoc(url)
+	if err != nil {
+		return "", err
+	}
+	xpathQuery, err := GetXpathFromUri(uri)
+	if err != nil {
+		return "", err
+	}
+
+	returnValue := ""
+	var values []string
+	node, err := jsonquery.Query(p.docs[url], xpathQuery)
+	if err != nil || node == nil {
+		return "", fmt.Errorf("unable to query doc for value with xpath query using %v", uri)
+	}
+
+	if node.FirstChild.Data != node.LastChild.Data {
+		return "", fmt.Errorf("location %v has child nodes at %v, please use a more granular query", xpathQuery, url)
+	}
+
+	childNodesLength := countChildNodes(node)
+
+	if childNodesLength > 1 {
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			values = append(values, child.Value().(string))
+		}
+		returnValue = strings.Join(values, ",")
+	} else {
+		returnValue = node.FirstChild.Value().(string)
+	}
+
+	if p.floatAsInt {
+		intValue, err := strconv.ParseFloat(returnValue, 64)
+		if err != nil {
+			return "", fmt.Errorf("unable to convert possible float to int for value: %v", returnValue)
+		}
+		returnValue = fmt.Sprintf("%.0f", intValue)
+	}
+
+	return returnValue, nil
+}
+
+func countChildNodes(node *jsonquery.Node) int {
+	// Check if there are more child nodes i.e. keys under this json key
+	count := 0
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		count++
+	}
+	return count
+}
+
+func (p *provider) GetStringMap(key string) (map[string]interface{}, error) {
+	return nil, fmt.Errorf("we should not be in the GetStringMap method")
+}

pkg/stringmapprovider/stringmapprovider.go

@@ -11,6 +11,7 @@ import (
 	"github.com/helmfile/vals/pkg/providers/doppler"
 	"github.com/helmfile/vals/pkg/providers/gcpsecrets"
 	"github.com/helmfile/vals/pkg/providers/gkms"
+	"github.com/helmfile/vals/pkg/providers/httpjson"
 	"github.com/helmfile/vals/pkg/providers/k8s"
 	"github.com/helmfile/vals/pkg/providers/onepasswordconnect"
 	"github.com/helmfile/vals/pkg/providers/sops"
@@ -46,6 +47,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringMapProvi
 		return gkms.New(l, provider), nil
 	case "k8s":
 		return k8s.New(l, provider)
+	case "httpjson":
+		return httpjson.New(l, provider), nil
 	}
 
 	return nil, fmt.Errorf("failed initializing string-map provider from config: %v", provider)

pkg/stringprovider/stringprovider.go

@@ -15,6 +15,7 @@ import (
 	"github.com/helmfile/vals/pkg/providers/gitlab"
 	"github.com/helmfile/vals/pkg/providers/gkms"
 	"github.com/helmfile/vals/pkg/providers/hcpvaultsecrets"
+	"github.com/helmfile/vals/pkg/providers/httpjson"
 	"github.com/helmfile/vals/pkg/providers/k8s"
 	"github.com/helmfile/vals/pkg/providers/onepasswordconnect"
 	"github.com/helmfile/vals/pkg/providers/pulumi"
@@ -73,6 +74,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringProvider
 		return conjur.New(l, provider), nil
 	case "hcpvaultsecrets":
 		return hcpvaultsecrets.New(l, provider), nil
+	case "httpjson":
+		return httpjson.New(l, provider), nil
 	}
 
 	return nil, fmt.Errorf("failed initializing string provider from config: %v", provider)

vals.go

@@ -35,6 +35,7 @@ import (
 	"github.com/helmfile/vals/pkg/providers/gkms"
 	"github.com/helmfile/vals/pkg/providers/googlesheets"
 	"github.com/helmfile/vals/pkg/providers/hcpvaultsecrets"
+	"github.com/helmfile/vals/pkg/providers/httpjson"
 	"github.com/helmfile/vals/pkg/providers/k8s"
 	"github.com/helmfile/vals/pkg/providers/onepasswordconnect"
 	"github.com/helmfile/vals/pkg/providers/pulumi"
@@ -96,6 +97,7 @@ const (
 	ProviderK8s                = "k8s"
 	ProviderConjur             = "conjur"
 	ProviderHCPVaultSecrets    = "hcpvaultsecrets"
+	ProviderHttpJsonManager    = "httpjson"
 	ProviderBitwarden          = "bw"
 )
 
@@ -264,6 +266,9 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) {
 		case ProviderHCPVaultSecrets:
 			p := hcpvaultsecrets.New(r.logger, conf)
 			return p, nil
+		case ProviderHttpJsonManager:
+			p := httpjson.New(r.logger, conf)
+			return p, nil
 		case ProviderBitwarden:
 			p := bitwarden.New(r.logger, conf)
 			return p, nil
@@ -375,6 +380,19 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) {
 					if !ok {
 						return "", fmt.Errorf("error reading map from cache: unsupported value type %T", cachedMap)
 					}
+				} else if uri.Scheme == "httpjson" {
+					// Due to the unpredictability in the structure of the JSON object,
+					// an alternative parsing method is used here.
+					// The standard approach couldn't be applied because the JSON object
+					// may vary in its key-value pairs and nesting depth, making it difficult
+					// to reliably parse using conventional methods.
+					// This alternative approach allows for flexible handling of the JSON
+					// object, accommodating different configurations and variations.
+					value, err := p.GetString(key)
+					if err != nil {
+						return "", err
+					}
+					return value, nil
 				} else {
 					obj, err = p.GetStringMap(path)
 					if err != nil {