tpl/data: Misc header improvements, tests, allow multiple headers of …

…same key

Closes #5617

common/types/convert.go

@@ -15,21 +15,52 @@ package types
 
 import (
 	"encoding/json"
+	"fmt"
 	"html/template"
+	"reflect"
 
 	"github.com/spf13/cast"
 )
 
-// ToStringSlicePreserveString converts v to a string slice.
-// If v is a string, it will be wrapped in a string slice.
+// ToStringSlicePreserveString is the same as ToStringSlicePreserveStringE,
+// but it never fails.
 func ToStringSlicePreserveString(v interface{}) []string {
+	vv, _ := ToStringSlicePreserveStringE(v)
+	return vv
+}
+
+// ToStringSlicePreserveStringE converts v to a string slice.
+// If v is a string, it will be wrapped in a string slice.
+func ToStringSlicePreserveStringE(v interface{}) ([]string, error) {
 	if v == nil {
-		return nil
+		return nil, nil
 	}
 	if sds, ok := v.(string); ok {
-		return []string{sds}
+		return []string{sds}, nil
+	}
+	result, err := cast.ToStringSliceE(v)
+	if err == nil {
+		return result, nil
 	}
-	return cast.ToStringSlice(v)
+
+	// Probably []int or similar. Fall back to reflect.
+	vv := reflect.ValueOf(v)
+
+	switch vv.Kind() {
+	case reflect.Slice, reflect.Array:
+		result = make([]string, vv.Len())
+		for i := 0; i < vv.Len(); i++ {
+			s, err := cast.ToStringE(vv.Index(i).Interface())
+			if err != nil {
+				return nil, err
+			}
+			result[i] = s
+		}
+		return result, nil
+	default:
+		return nil, fmt.Errorf("failed to convert %T to a string slice", v)
+	}
+
 }
 
 // TypeToString converts v to a string if it's a valid string type.

common/types/convert_test.go

@@ -24,7 +24,9 @@ func TestToStringSlicePreserveString(t *testing.T) {
 	c := qt.New(t)
 
 	c.Assert(ToStringSlicePreserveString("Hugo"), qt.DeepEquals, []string{"Hugo"})
+	c.Assert(ToStringSlicePreserveString(qt.Commentf("Hugo")), qt.DeepEquals, []string{"Hugo"})
 	c.Assert(ToStringSlicePreserveString([]interface{}{"A", "B"}), qt.DeepEquals, []string{"A", "B"})
+	c.Assert(ToStringSlicePreserveString([]int{1, 3}), qt.DeepEquals, []string{"1", "3"})
 	c.Assert(ToStringSlicePreserveString(nil), qt.IsNil)
 }
 

docs/content/en/templates/data-templates.md

@@ -114,19 +114,10 @@ You can use the following code to render the `Short Description` in your layout:
 
 Note the use of the [`markdownify` template function][markdownify]. This will send the description through the Blackfriday Markdown rendering engine.
 
-<!-- begin "Data-drive Content" page -->
 
-## Data-Driven Content
+## Get Remote Data
 
-In addition to the [data files](/extras/datafiles/) feature, Hugo also has a "data-driven content" feature, which lets you load any [JSON](https://www.json.org/) or [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) file from nearly any resource.
-
-Data-driven content currently consists of two functions, `getJSON` and `getCSV`, which are available in all template files.
-
-## Implementation details
-
-### Call the Functions with a URL
-
-In your template, call the functions like this:
+Use `getJSON` or `getCSV` to get remote data:
 
 ```
 {{ $dataJ := getJSON "url" }}
@@ -155,19 +146,18 @@ This will resolve internally to the following:
 {{ $gistJ := getJSON "https://api.github.com/users/GITHUB_USERNAME/gists" }}
 ```
 
-Finally, you can range over an array. This example will output the
-first 5 gists for a GitHub user:
+### Add HTTP headers
+
+{{< new-in "0.84.0" >}} Both `getJSON` and `getCSV` takes an optional map as the last argument, e.g.:
 
 ```
-<ul>
-  {{ $urlPre := "https://api.github.com" }}
-  {{ $gistJ := getJSON $urlPre "/users/GITHUB_USERNAME/gists" }}
-  {{ range first 5 $gistJ }}
-    {{ if .public }}
-      <li><a href="{{ .html_url }}" target="_blank">{{ .description }}</a></li>
-    {{ end }}
-  {{ end }}
-</ul>
+{{ $data := getJSON "https://example.org/api" (dict "Authorization" "Bearer abcd")  }}
+```
+
+If you need multiple values for the same header key, use a slice:
+
+```
+{{ $data := getJSON "https://example.org/api" (dict "X-List" (slice "a" "b" "c"))  }}
 ```
 
 ### Example for CSV files

tpl/data/data.go

@@ -23,6 +23,10 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/maps"
+
+	"github.com/gohugoio/hugo/common/types"
+
 	"github.com/gohugoio/hugo/common/constants"
 	"github.com/gohugoio/hugo/common/loggers"
 
@@ -59,14 +63,10 @@ type Namespace struct {
 // If you provide multiple parts for the URL they will be joined together to the final URL.
 // GetCSV returns nil or a slice slice to use in a short code.
 func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err error) {
-	url := joinURL(args)
+	url, headers := toURLAndHeaders(args)
 	cache := ns.cacheGetCSV
 
 	unmarshal := func(b []byte) (bool, error) {
-		if !bytes.Contains(b, []byte(sep)) {
-			return false, _errors.Errorf("cannot find separator %s in CSV for %s", sep, url)
-		}
-
 		if d, err = parseCSV(b, sep); err != nil {
 			err = _errors.Wrapf(err, "failed to parse CSV file %s", url)
 
@@ -82,17 +82,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
 		return nil, _errors.Wrapf(err, "failed to create request for getCSV for resource %s", url)
 	}
 
-	req.Header.Add("Accept", "text/csv")
-	req.Header.Add("Accept", "text/plain")
-
-	// Add custom user headers to the get request
-	finalArg := args[len(args)-1]
-
-	if userHeaders, ok := finalArg.(map[string]interface{}); ok {
-		for key, val := range userHeaders {
-			req.Header.Add(key, val.(string))
-		}
-	}
+	// Add custom user headers.
+	addUserProvidedHeaders(headers, req)
+	addDefaultHeaders(req, "text/csv", "text/plain")
 
 	err = ns.getResource(cache, unmarshal, req)
 	if err != nil {
@@ -108,7 +100,7 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
 // GetJSON returns nil or parsed JSON to use in a short code.
 func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
 	var v interface{}
-	url := joinURL(args)
+	url, headers := toURLAndHeaders(args)
 	cache := ns.cacheGetJSON
 
 	req, err := http.NewRequest("GET", url, nil)
@@ -124,17 +116,8 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
 		return false, nil
 	}
 
-	req.Header.Add("Accept", "application/json")
-	req.Header.Add("User-Agent", "Hugo Static Site Generator")
-
-	// Add custom user headers to the get request
-	finalArg := args[len(args)-1]
-
-	if userHeaders, ok := finalArg.(map[string]interface{}); ok {
-		for key, val := range userHeaders {
-			req.Header.Add(key, val.(string))
-		}
-	}
+	addUserProvidedHeaders(headers, req)
+	addDefaultHeaders(req, "application/json")
 
 	err = ns.getResource(cache, unmarshal, req)
 	if err != nil {
@@ -145,8 +128,64 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
 	return v, nil
 }
 
-func joinURL(urlParts []interface{}) string {
-	return strings.Join(cast.ToStringSlice(urlParts), "")
+func addDefaultHeaders(req *http.Request, accepts ...string) {
+	for _, accept := range accepts {
+		if !hasHeaderValue(req.Header, "Accept", accept) {
+			req.Header.Add("Accept", accept)
+		}
+	}
+	if !hasHeaderKey(req.Header, "User-Agent") {
+		req.Header.Add("User-Agent", "Hugo Static Site Generator")
+	}
+}
+
+func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
+	if headers == nil {
+		return
+	}
+	for key, val := range headers {
+		vals := types.ToStringSlicePreserveString(val)
+		for _, s := range vals {
+			req.Header.Add(key, s)
+		}
+	}
+}
+
+func hasHeaderValue(m http.Header, key, value string) bool {
+	var s []string
+	var ok bool
+
+	if s, ok = m[key]; !ok {
+		return false
+	}
+
+	for _, v := range s {
+		if v == value {
+			return true
+		}
+	}
+	return false
+}
+
+func hasHeaderKey(m http.Header, key string) bool {
+	_, ok := m[key]
+	return ok
+}
+
+func toURLAndHeaders(urlParts []interface{}) (string, map[string]interface{}) {
+	if len(urlParts) == 0 {
+		return "", nil
+	}
+
+	// The last argument may be a map.
+	headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1])
+	if err == nil {
+		urlParts = urlParts[:len(urlParts)-1]
+	} else {
+		headers = nil
+	}
+
+	return strings.Join(cast.ToStringSlice(urlParts), ""), headers
 }
 
 // parseCSV parses bytes of CSV data into a slice slice string or an error