Skip to content

Commit

Permalink
267: Add http json functionality
Browse files Browse the repository at this point in the history
Signed-off-by: Paul Grant <paulfgrant01@gmail.com>
  • Loading branch information
Paul Grant authored and paulfgrant01 committed Mar 25, 2024
1 parent 994ec61 commit 469266f
Show file tree
Hide file tree
Showing 8 changed files with 344 additions and 0 deletions.
61 changes: 61 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ It supports various backends including:
- Conjur
- HCP Vault Secrets
- Bitwarden
- HTTP JSON

- Use `vals eval -f refs.yaml` to replace all the `ref`s in the file to actual values and secrets.
- Use `vals exec -f env.yaml -- <COMMAND>` to populate envvars and execute the command.
Expand Down Expand Up @@ -225,6 +226,7 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/
- [Kubernetes](#kubernetes)
- [Conjur](#conjur)
- [HCP Vault Secrets](#hcp-vault-secrets)
- [HTTP JSON](#http-json)
- [Bitwarden](#bitwarden)

Please see [pkg/providers](https://github.com/helmfile/vals/tree/master/pkg/providers) for the implementations of all the providers. The package names corresponds to the URI schemes.
Expand Down Expand Up @@ -823,6 +825,65 @@ Examples:
- `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/{username,password,uri,notes,item}` gets username, password, uri, notes or the whole item of the given item id
- `ref+bw://4d084b01-87e7-4411-8de9-2476ab9f3f48/notes#/key1` gets the *key1* from the yaml stored as note in the item
### HTTP JSON
This provider retrieves values stored in JSON hosted by an HTTP frontend.
This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages.
Given the diverse array of JSON structures that can be encountered, utilizing jsonquery with XPath presents a more effective approach for handling this variability in data structures.
This provider requires an xpath to be provided in singleparam mode (as last argument).
Do not include the protocol scheme i.e. http/https. Provider defaults to scheme https
Examples:
#### Fetch string value
`ref+httpjson://<domain>/<path>?[insecure=false&floatAsInt=false]mode=singleparam#/<xpath>`
Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos:
```json
[
{
"name": "chartify"
},
{
"name": "go-yaml"
}
]
```
```
# To get name="chartify" using https protocol you would use:
ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/name
# To get name="go-yaml" using https protocol you would use:
ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[2]/name
# To get name="go-yaml" using http protocol you would use:
ref+httpjson://api.github.com/users/helmfile/repos?insecure=true&mode=singleparam#///*[2]/name
```

#### Fetch integer value

`ref+httpjson://<domain>/<path>?[insecure=false&floatAsInt=false]mode=singleparam#/<xpath>`

Let's say you want to fetch the below JSON object from https://api.github.com/users/helmfile/repos:
```json
[
{
"id": 251296379
}
]
```
```
# Running the following will return: 2.51296379e+08
ref+httpjson://api.github.com/users/helmfile/repos?mode=singleparam#///*[1]/id
# Running the following will return: 251296379
ref+httpjson://api.github.com/users/helmfile/repos?floatAsInt=true&mode=singleparam#///*[1]/id
```



## Advanced Usages

### Discriminating config and secrets
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ require (
)

require (
github.com/antchfx/jsonquery v1.3.3 // indirect
github.com/antchfx/xpath v1.2.3 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/go-openapi/analysis v0.23.0 // indirect
github.com/go-openapi/errors v0.22.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/antchfx/jsonquery v1.3.3 h1:zjZpbnZhYng3uOAbIfdNq81A9mMEeuDJeYIpeKpZ4es=
github.com/antchfx/jsonquery v1.3.3/go.mod h1:1JG4DqRlRCHgVYDPY1ioYFAGSXGfWHzNgrbiGQHsWck=
github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes=
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
Expand Down
136 changes: 136 additions & 0 deletions pkg/providers/httpjson/httpjson.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package httpjson

import (
"fmt"
"strconv"
"strings"

"github.com/antchfx/jsonquery"
"github.com/helmfile/vals/pkg/api"
"github.com/helmfile/vals/pkg/log"
)

type provider struct {
// Keeping track of httpjson services since we need a service per url
protocol string
log *log.Logger
docs map[string]*jsonquery.Node
floatAsInt bool
}

func New(l *log.Logger, cfg api.StaticConfig) *provider {
p := &provider{
log: l,
}

// Should the protocol be insecure i.e. http
insecureArg := cfg.String("insecure")
p.protocol = "https"
if insecureArg == "true" {
p.protocol = "http"
}

// By default JSON will return large integers as float64
floatAsIntArg := cfg.String("floatAsInt")
p.floatAsInt = false
if floatAsIntArg == "true" {
p.floatAsInt = true
}

// Initialize docs map to store the json object for use multiple times
if len(p.docs) == 0 {
p.docs = make(map[string]*jsonquery.Node)
}

return p
}

func GetXpathFromUri(uri string) (xpath string, err error) {
found := strings.Split(uri, "mode=singleparam#")[1]
found = strings.Split(found, "&")[0]
xpath = strings.TrimPrefix(found, "/")

return xpath, nil
}

func getUrlFromUri(uri string, protocol string) (string, error) {
uriParts := strings.Split(uri, "?")
if len(uriParts) < 2 {
return "", fmt.Errorf("error getting url from uri: %v, ensure xpath singleparam is set as a query parameter", uri)
}
url := strings.Replace(uriParts[0], "httpjson", protocol, 1)

return url, nil
}

func (p *provider) GetJsonDoc(url string) error {
if _, ok := p.docs[url]; !ok {
doc, err := jsonquery.LoadURL(url)
if err != nil {
return fmt.Errorf("error fetching json document at %v: %v", url, err)
}
p.docs[url] = doc
}

return nil
}

func (p *provider) GetString(uri string) (string, error) {
url, err := getUrlFromUri(uri, p.protocol)
if err != nil {
return "", err
}
err = p.GetJsonDoc(url)
if err != nil {
return "", err
}
xpathQuery, err := GetXpathFromUri(uri)
if err != nil {
return "", err
}

returnValue := ""
var values []string
node, err := jsonquery.Query(p.docs[url], xpathQuery)
if err != nil || node == nil {
return "", fmt.Errorf("unable to query doc for value with xpath query using %v", uri)
}

if node.FirstChild.Data != node.LastChild.Data {
return "", fmt.Errorf("location %v has child nodes at %v, please use a more granular query", xpathQuery, url)
}

childNodesLength := countChildNodes(node)

if childNodesLength > 1 {
for child := node.FirstChild; child != nil; child = child.NextSibling {
values = append(values, child.Value().(string))
}
returnValue = strings.Join(values, ",")

} else {
returnValue = node.FirstChild.Value().(string)
}

if p.floatAsInt {
intValue, err := strconv.ParseFloat(returnValue, 64)
if err != nil {
return "", fmt.Errorf("unable to convert possible float to int for value: %v", returnValue)
}
returnValue = fmt.Sprintf("%.0f", intValue)
}

return returnValue, nil
}

func countChildNodes(node *jsonquery.Node) int {
count := 0
for child := node.FirstChild; child != nil; child = child.NextSibling {
count++
}
return count
}

func (p *provider) GetStringMap(key string) (map[string]interface{}, error) {
return nil, fmt.Errorf("we should not be in the GetStringMap method")
}
3 changes: 3 additions & 0 deletions pkg/stringmapprovider/stringmapprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/helmfile/vals/pkg/providers/doppler"
"github.com/helmfile/vals/pkg/providers/gcpsecrets"
"github.com/helmfile/vals/pkg/providers/gkms"
"github.com/helmfile/vals/pkg/providers/httpjson"
"github.com/helmfile/vals/pkg/providers/k8s"
"github.com/helmfile/vals/pkg/providers/onepasswordconnect"
"github.com/helmfile/vals/pkg/providers/sops"
Expand Down Expand Up @@ -46,6 +47,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringMapProvi
return gkms.New(l, provider), nil
case "k8s":
return k8s.New(l, provider)
case "httpjson":
return httpjson.New(l, provider), nil
}

return nil, fmt.Errorf("failed initializing string-map provider from config: %v", provider)
Expand Down
3 changes: 3 additions & 0 deletions pkg/stringprovider/stringprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/helmfile/vals/pkg/providers/gitlab"
"github.com/helmfile/vals/pkg/providers/gkms"
"github.com/helmfile/vals/pkg/providers/hcpvaultsecrets"
"github.com/helmfile/vals/pkg/providers/httpjson"
"github.com/helmfile/vals/pkg/providers/k8s"
"github.com/helmfile/vals/pkg/providers/onepasswordconnect"
"github.com/helmfile/vals/pkg/providers/pulumi"
Expand Down Expand Up @@ -73,6 +74,8 @@ func New(l *log.Logger, provider api.StaticConfig) (api.LazyLoadedStringProvider
return conjur.New(l, provider), nil
case "hcpvaultsecrets":
return hcpvaultsecrets.New(l, provider), nil
case "httpjson":
return httpjson.New(l, provider), nil
}

return nil, fmt.Errorf("failed initializing string provider from config: %v", provider)
Expand Down
19 changes: 19 additions & 0 deletions vals.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/helmfile/vals/pkg/providers/gkms"
"github.com/helmfile/vals/pkg/providers/googlesheets"
"github.com/helmfile/vals/pkg/providers/hcpvaultsecrets"
"github.com/helmfile/vals/pkg/providers/httpjson"
"github.com/helmfile/vals/pkg/providers/k8s"
"github.com/helmfile/vals/pkg/providers/onepasswordconnect"
"github.com/helmfile/vals/pkg/providers/pulumi"
Expand Down Expand Up @@ -96,6 +97,7 @@ const (
ProviderK8s = "k8s"
ProviderConjur = "conjur"
ProviderHCPVaultSecrets = "hcpvaultsecrets"
ProviderHttpJsonManager = "httpjson"
ProviderBitwarden = "bw"
)

Expand Down Expand Up @@ -264,6 +266,9 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) {
case ProviderHCPVaultSecrets:
p := hcpvaultsecrets.New(r.logger, conf)
return p, nil
case ProviderHttpJsonManager:
p := httpjson.New(r.logger, conf)
return p, nil
case ProviderBitwarden:
p := bitwarden.New(r.logger, conf)
return p, nil
Expand Down Expand Up @@ -367,6 +372,7 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) {
}

return str, nil

} else {
mapRequestURI := key[:strings.LastIndex(key, uri.Fragment)-1]
var obj map[string]interface{}
Expand All @@ -375,6 +381,19 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) {
if !ok {
return "", fmt.Errorf("error reading map from cache: unsupported value type %T", cachedMap)
}
} else if uri.Scheme == "httpjson" {
// Due to the unpredictability in the structure of the JSON object,
// an alternative parsing method is used here.
// The standard approach couldn't be applied because the JSON object
// may vary in its key-value pairs and nesting depth, making it difficult
// to reliably parse using conventional methods.
// This alternative approach allows for flexible handling of the JSON
// object, accommodating different configurations and variations.
value, err := p.GetString(uri.String())
if err != nil {
return "", err
}
return value, nil
} else {
obj, err = p.GetStringMap(path)
if err != nil {
Expand Down
Loading

0 comments on commit 469266f

Please sign in to comment.