Allow hosts to be used to configure http monitors (elastic#13703)

Currently only the http monitor allows configuring endpoints to hit via urls. Everyone else relies on hosts similar to metricbeat. This PR ensures that we also allow configuration with hosts. Eventually, the urls field can be deprecated.

(cherry picked from commit 717771f)

CHANGELOG.next.asciidoc

@@ -310,6 +310,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Enable `add_observer_metadata` processor in default config. {pull}11394[11394]
 - Record HTTP body metadata and optionally contents in `http.response.body.*` fields. {pull}13022[13022]
+- Allow `hosts` to be used to configure http monitors {pull}13703[13703]
 
 *Journalbeat*
 

heartbeat/docs/autodiscover-kubernetes-config.asciidoc

@@ -12,7 +12,7 @@ heartbeat.autodiscover:
               kubernetes.annotations.prometheus.io.scrape: "true"
           config:
             - type: http
-              urls: ["${data.host}:${data.port}"]
+              hosts: ["${data.host}:${data.port}"]
               schedule: "@every 1s"
               timeout: 1s
 -------------------------------------------------------------------------------------

heartbeat/docs/heartbeat-options.asciidoc

@@ -33,7 +33,7 @@ heartbeat.monitors:
   check.receive: "Check"
 - type: http
   schedule: '@every 5s'
-  urls: ["http://localhost:80/service/status"]
+  hosts: ["http://localhost:80/service/status"]
   check.response.status: 200
 heartbeat.scheduler:
   limit: 10
@@ -68,7 +68,7 @@ monitor definitions only, e.g. what is normally under the `heartbeat.monitors` s
 # /path/to/my/monitors.d/localhost_service_check.yml
 - type: http
   schedule: '@every 5s'
-  urls: ["http://localhost:80/service/status"]
+  hosts: ["http://localhost:80/service/status"]
   check.response.status: 200
 ----------------------------------------------------------------------
 
@@ -368,7 +368,7 @@ the host returns the expected response. These options are valid when the
 
 [float]
 [[monitor-http-urls]]
-==== `urls`
+==== `hosts`
 
 A list of URLs to ping.
 
@@ -378,7 +378,7 @@ Example configuration:
 -------------------------------------------------------------------------------
 - type: http
   schedule: '@every 5s'
-  urls: ["http://myhost:80"]
+  hosts: ["http://myhost:80"]
 -------------------------------------------------------------------------------
 
 
@@ -419,7 +419,7 @@ Example configuration:
 -------------------------------------------------------------------------------
 - type: http
   schedule: '@every 5s'
-  urls: ["https://myhost:443"]
+  hosts: ["https://myhost:443"]
   ssl:
     certificate_authorities: ['/etc/ca.crt']
     supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"]
@@ -454,7 +454,7 @@ Example configuration:
 -------------------------------------------------------------------------------
 - type: http
   schedule: '@every 5s'
-  urls: ["http://myhost:80"]
+  hosts: ["http://myhost:80"]
   check.request.method: HEAD
   check.response.status: 200
 -------------------------------------------------------------------------------
@@ -481,7 +481,7 @@ contains JSON:
 -------------------------------------------------------------------------------
 - type: http
   schedule: '@every 5s'
-  urls: ["https://myhost:80"]
+  hosts: ["https://myhost:80"]
   check.request:
     method: GET
     headers:
@@ -502,7 +502,7 @@ patterns:
 -------------------------------------------------------------------------------
 - type: http
   schedule: '@every 5s'
-  urls: ["https://myhost:80"]
+  hosts: ["https://myhost:80"]
   check.request:
     method: GET
     headers:
@@ -521,7 +521,7 @@ regex:
 -------------------------------------------------------------------------------
 - type: http
   schedule: '@every 5s'
-  urls: ["https://myhost:80"]
+  hosts: ["https://myhost:80"]
   check.request:
     method: GET
     headers:

heartbeat/monitors.d/sample.http.yml.disabled

@@ -15,7 +15,7 @@
   schedule: '@every 5s' # every 5 seconds from start of beat
 
   # Configure URLs to ping
-  urls: ["http://localhost:9200"]
+  hosts: ["http://localhost:9200"]
 
   # Configure IP protocol types to ping on if hostnames are configured.
   # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`.

heartbeat/monitors/active/http/config.go

@@ -19,19 +19,19 @@ package http
 
 import (
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
-	"github.com/elastic/beats/libbeat/conditions"
-
+	"github.com/elastic/beats/heartbeat/monitors"
 	"github.com/elastic/beats/libbeat/common/match"
 	"github.com/elastic/beats/libbeat/common/transport/tlscommon"
-
-	"github.com/elastic/beats/heartbeat/monitors"
+	"github.com/elastic/beats/libbeat/conditions"
 )
 
 type Config struct {
-	URLs         []string       `config:"urls" validate:"required"`
+	URLs         []string       `config:"urls"`
+	Hosts        []string       `config:"hosts"`
 	ProxyURL     string         `config:"proxy_url"`
 	Timeout      time.Duration  `config:"timeout"`
 	MaxRedirects int            `config:"max_redirects"`
@@ -113,6 +113,7 @@ var defaultConfig = Config{
 	},
 }
 
+// Validate validates of the responseConfig object is valid or not
 func (r *responseConfig) Validate() error {
 	switch strings.ToLower(r.IncludeBody) {
 	case "always", "on_error", "never":
@@ -127,6 +128,7 @@ func (r *responseConfig) Validate() error {
 	return nil
 }
 
+// Validate validates of the requestParameters object is valid or not
 func (r *requestParameters) Validate() error {
 	switch strings.ToUpper(r.Method) {
 	case "HEAD", "GET", "POST":
@@ -137,6 +139,7 @@ func (r *requestParameters) Validate() error {
 	return nil
 }
 
+// Validate validates of the compressionConfig object is valid or not
 func (c *compressionConfig) Validate() error {
 	t := strings.ToLower(c.Type)
 	if t != "" && t != "gzip" {
@@ -153,3 +156,37 @@ func (c *compressionConfig) Validate() error {
 
 	return nil
 }
+
+// Validate validates of the Config object is valid or not
+func (c *Config) Validate() error {
+	if len(c.Hosts) == 0 && len(c.URLs) == 0 {
+		return fmt.Errorf("hosts is a mandatory parameter")
+	}
+
+	if len(c.URLs) != 0 {
+		c.Hosts = append(c.Hosts, c.URLs...)
+	}
+
+	// updateScheme looks at TLS config to decide if http or https should be used to update the host
+	updateScheme := func(host string) string {
+		if c.TLS != nil && *c.TLS.Enabled == true {
+			return fmt.Sprint("https://", host)
+		}
+		return fmt.Sprint("http://", host)
+	}
+
+	// Check if the URL is not parseable. If yes, then append scheme.
+	// If the url is valid but host or scheme is empty which can occur when someone configures host:port
+	// then update the scheme there as well.
+	for i := 0; i < len(c.Hosts); i++ {
+		host := c.Hosts[i]
+		u, err := url.ParseRequestURI(host)
+		if err != nil {
+			c.Hosts[i] = updateScheme(host)
+		} else if u.Scheme == "" || u.Host == "" {
+			c.Hosts[i] = updateScheme(host)
+		}
+	}
+
+	return nil
+}

heartbeat/monitors/active/http/config_test.go

@@ -0,0 +1,99 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package http
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var tests = []struct {
+	description   string
+	host          string
+	url           string
+	convertedHost string
+	result        bool
+}{
+	{
+		"Validate if neither urls nor host specified returns error",
+		"",
+		"",
+		"",
+		false,
+	},
+	{
+		"Validate if only urls are present then the config is moved to hosts",
+		"",
+		"http://localhost:8080",
+		"http://localhost:8080",
+		true,
+	},
+	{
+		"Validate if only hosts are present then the config is valid",
+		"http://localhost:8080",
+		"",
+		"http://localhost:8080",
+		true,
+	},
+	{
+		"Validate if no scheme is present then it is added correctly",
+		"localhost",
+		"",
+		"http://localhost",
+		true,
+	},
+	{
+		"Validate if no scheme is present but has a port then it is added correctly",
+		"localhost:8080",
+		"",
+		"http://localhost:8080",
+		true,
+	},
+	{
+		"Validate if schemes like unix are honored",
+		"unix://localhost:8080",
+		"",
+		"unix://localhost:8080",
+		true,
+	},
+}
+
+func TestConfigValidate(t *testing.T) {
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			config := Config{}
+			if test.host != "" {
+				config.Hosts = []string{test.host}
+			}
+
+			if test.url != "" {
+				config.URLs = []string{test.url}
+			}
+
+			err := config.Validate()
+			if test.result {
+				assert.Nil(t, err)
+				assert.Equal(t, test.convertedHost, config.Hosts[0])
+			} else {
+				assert.NotNil(t, err)
+			}
+
+		})
+	}
+}

heartbeat/monitors/active/http/http.go

@@ -95,8 +95,8 @@ func create(
 		}
 	}
 
-	js = make([]jobs.Job, len(config.URLs))
-	for i, urlStr := range config.URLs {
+	js = make([]jobs.Job, len(config.Hosts))
+	for i, urlStr := range config.Hosts {
 		u, _ := url.Parse(urlStr)
 		if err != nil {
 			return nil, 0, err
@@ -112,7 +112,7 @@ func create(
 		js[i] = wrappers.WithURLField(u, job)
 	}
 
-	return js, len(config.URLs), nil
+	return js, len(config.Hosts), nil
 }
 
 func newRoundTripper(config *Config, tls *transport.TLSConfig) (*http.Transport, error) {