Add haproxy http stats (elastic#5819)

* Haproxy module: Initial refactor and tests for http stats
* Haproxy http stats with basic authentication
* Haproxy show info is not supported in http stats endpoint
* Added documentation for HAproxy http stats frontend
* Use errors library for errors in haproxy module

CHANGELOG.asciidoc

@@ -36,6 +36,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Rename `heap_init` field to `heap.init`. {pull}5320[5320]
 - Rename `http.response.status_code` field to `http.response.code`. {pull}5521[5521]
 - Rename `golang.heap.system.optained` field to `golang.heap.system.obtained`. {issue}5703[5703]
+- Support haproxy stats gathering using http (additionaly to tcp socket). {pull}5819[5819]
 
 *Packetbeat*
 

metricbeat/docs/modules/haproxy.asciidoc

@@ -5,16 +5,39 @@ This file is generated! See scripts/docs_collector.py
 [[metricbeat-module-haproxy]]
 == HAProxy module
 
-This module collects stats from http://www.haproxy.org/[HAProxy]. To configure
-HAProxy to collect stats, you must enable the stats socket via TCP. For example,
-to enable stats reporting via any local IP on port 14567, place this statement
-under the `global` or `default` section of the haproxy config:
+This module collects stats from http://www.haproxy.org/[HAProxy]. It supports
+collection from using TCP sockets or HTTP with or without basic authentication.
 
-`stats socket 127.0.0.1:14567`
+To configure HAProxy to collect stats, you must enable the stats module, it can
+be done by enabling a TCP socket, or by adding an HTTP stats frontend.
+
+Metricbeat can collect two metric sets from HAproxy, `info` and `stats`. `info`
+is not available when using HTTP stats frontend.
+
+For example, to enable stats reporting via any local IP on port 14567, place
+this statement under the `global` or `default` section of the haproxy config:
+
+[source,haproxy]
+----
+ stats socket 127.0.0.1:14567
+----
 
 NOTE: You should use an internal private IP, or secure this with a firewall
 rule, so that only designated hosts can access this data.
 
+To configure the HTTP stats frontend, a frontend with stats enabled has to
+be added. For example, to open this frontend to any IP on port 14567 with
+required authentication add this to the haproxy config:
+
+[source,haproxy]
+----
+ listen stats
+   bind 0.0.0.0:14569
+   stats enable
+   stats uri /stats
+   stats auth admin:admin
+----
+
 [float]
 === Compatibility
 

metricbeat/module/haproxy/_meta/docs.asciidoc

@@ -1,13 +1,36 @@
-This module collects stats from http://www.haproxy.org/[HAProxy]. To configure
-HAProxy to collect stats, you must enable the stats socket via TCP. For example,
-to enable stats reporting via any local IP on port 14567, place this statement
-under the `global` or `default` section of the haproxy config:
+This module collects stats from http://www.haproxy.org/[HAProxy]. It supports
+collection from using TCP sockets or HTTP with or without basic authentication.
 
-`stats socket 127.0.0.1:14567`
+To configure HAProxy to collect stats, you must enable the stats module, it can
+be done by enabling a TCP socket, or by adding an HTTP stats frontend.
+
+Metricbeat can collect two metric sets from HAproxy, `info` and `stats`. `info`
+is not available when using HTTP stats frontend.
+
+For example, to enable stats reporting via any local IP on port 14567, place
+this statement under the `global` or `default` section of the haproxy config:
+
+[source,haproxy]
+----
+ stats socket 127.0.0.1:14567
+----
 
 NOTE: You should use an internal private IP, or secure this with a firewall
 rule, so that only designated hosts can access this data.
 
+To configure the HTTP stats frontend, a frontend with stats enabled has to
+be added. For example, to open this frontend to any IP on port 14567 with
+required authentication add this to the haproxy config:
+
+[source,haproxy]
+----
+ listen stats
+   bind 0.0.0.0:14569
+   stats enable
+   stats uri /stats
+   stats auth admin:admin
+----
+
 [float]
 === Compatibility
 

metricbeat/module/haproxy/_meta/haproxy.conf

@@ -33,6 +33,21 @@ defaults
   option httpchk HEAD /haproxy?monitor HTTP/1.0
   timeout check 5s
 
+listen stat
+
+  bind 0.0.0.0:14568
+
+  stats enable
+  stats uri /stats
+
+listen stat-auth
+
+  bind 0.0.0.0:14569
+
+  stats enable
+  stats uri /stats
+  stats auth admin:admin
+
 listen http-webservices 
 
   bind 0.0.0.0:8888

metricbeat/module/haproxy/haproxy.go

@@ -3,17 +3,18 @@ package haproxy
 import (
 	"bytes"
 	"encoding/csv"
-	"errors"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
+	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/elastic/beats/metricbeat/mb/parse"
 
 	"github.com/gocarina/gocsv"
 	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
 )
 
 // HostParser is used for parsing the configured HAProxy hosts.
@@ -137,60 +138,37 @@ type Info struct {
 }
 
 // Client is an instance of the HAProxy client
+type clientProto interface {
+	Stat() (*bytes.Buffer, error)
+	Info() (*bytes.Buffer, error)
+}
+
 type Client struct {
-	Address     string
-	ProtoScheme string
+	proto clientProto
 }
 
 // NewHaproxyClient returns a new instance of HaproxyClient
 func NewHaproxyClient(address string) (*Client, error) {
-	parts := strings.Split(address, "://")
-	if len(parts) != 2 {
-		return nil, errors.New("must have protocol scheme and address")
-	}
-
-	if parts[0] != "tcp" && parts[0] != "unix" {
-		return nil, errors.New("invalid protocol scheme")
-	}
-
-	return &Client{
-		Address:     parts[1],
-		ProtoScheme: parts[0],
-	}, nil
-}
-
-// Run sends a designated command to the haproxy stats socket
-func (c *Client) run(cmd string) (*bytes.Buffer, error) {
-	var conn net.Conn
-	response := bytes.NewBuffer(nil)
-
-	conn, err := net.Dial(c.ProtoScheme, c.Address)
-	if err != nil {
-		return response, err
-	}
-
-	defer conn.Close()
-
-	_, err = conn.Write([]byte(cmd + "\n"))
-	if err != nil {
-		return response, err
-	}
-
-	_, err = io.Copy(response, conn)
+	u, err := url.Parse(address)
 	if err != nil {
-		return response, err
+		return nil, errors.Wrap(err, "invalid url")
 	}
 
-	if strings.HasPrefix(response.String(), "Unknown command") {
-		return response, fmt.Errorf("unknown command: %s", cmd)
+	switch u.Scheme {
+	case "tcp":
+		return &Client{&unixProto{Network: u.Scheme, Address: u.Host}}, nil
+	case "unix":
+		return &Client{&unixProto{Network: u.Scheme, Address: u.Path}}, nil
+	case "http", "https":
+		return &Client{&httpProto{URL: u}}, nil
+	default:
+		return nil, errors.Errorf("invalid protocol scheme: %s", u.Scheme)
 	}
-
-	return response, nil
 }
 
 // GetStat returns the result from the 'show stat' command
 func (c *Client) GetStat() ([]*Stat, error) {
-	runResult, err := c.run("show stat")
+	runResult, err := c.proto.Stat()
 	if err != nil {
 		return nil, err
 	}
@@ -201,15 +179,15 @@ func (c *Client) GetStat() ([]*Stat, error) {
 
 	err = gocsv.UnmarshalCSV(csvReader, &statRes)
 	if err != nil {
-		return nil, fmt.Errorf("error parsing CSV: %s", err)
+		return nil, errors.Errorf("error parsing CSV: %s", err)
 	}
 
 	return statRes, nil
 }
 
 // GetInfo returns the result from the 'show stat' command
 func (c *Client) GetInfo() (*Info, error) {
-	res, err := c.run("show info")
+	res, err := c.proto.Info()
 	if err != nil {
 		return nil, err
 	}
@@ -243,3 +221,86 @@ func (c *Client) GetInfo() (*Info, error) {
 
 	return nil, err
 }
+
+type unixProto struct {
+	Network string
+	Address string
+}
+
+// Run sends a designated command to the haproxy stats socket
+func (p *unixProto) run(cmd string) (*bytes.Buffer, error) {
+	var conn net.Conn
+	response := bytes.NewBuffer(nil)
+
+	conn, err := net.Dial(p.Network, p.Address)
+	if err != nil {
+		return response, err
+	}
+	defer conn.Close()
+
+	_, err = conn.Write([]byte(cmd + "\n"))
+	if err != nil {
+		return response, err
+	}
+
+	_, err = io.Copy(response, conn)
+	if err != nil {
+		return response, err
+	}
+
+	if strings.HasPrefix(response.String(), "Unknown command") {
+		return response, errors.Errorf("unknown command: %s", cmd)
+	}
+
+	return response, nil
+}
+
+func (p *unixProto) Stat() (*bytes.Buffer, error) {
+	return p.run("show stat")
+}
+
+func (p *unixProto) Info() (*bytes.Buffer, error) {
+	return p.run("show info")
+}
+
+type httpProto struct {
+	URL *url.URL
+}
+
+func (p *httpProto) Stat() (*bytes.Buffer, error) {
+	url := p.URL.String()
+	// Force csv format
+	if !strings.HasSuffix(url, ";csv") {
+		url += ";csv"
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if p.URL.User != nil {
+		password, _ := p.URL.User.Password()
+		req.SetBasicAuth(p.URL.User.Username(), password)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, errors.Errorf("couldn't connect: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("invalid response: %s", resp.Status)
+	}
+
+	d, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Errorf("couldn't read response body: %v", err)
+	}
+	return bytes.NewBuffer(d), nil
+}
+
+func (p *httpProto) Info() (*bytes.Buffer, error) {
+	return nil, errors.New("not supported")
+}

metricbeat/module/haproxy/info/info.go

@@ -36,9 +36,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // Fetch fetches info stats from the haproxy service.
 func (m *MetricSet) Fetch() (common.MapStr, error) {
-	// haproxy doesn't accept a username or password so ignore them if they
-	// are in the URL.
-	hapc, err := haproxy.NewHaproxyClient(m.HostData().SanitizedURI)
+	hapc, err := haproxy.NewHaproxyClient(m.HostData().URI)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed creating haproxy client")
 	}

metricbeat/module/haproxy/stat/stat.go

@@ -36,9 +36,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 // Fetch methods returns a list of stats metrics.
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
-	// haproxy doesn't accept a username or password so ignore them if they
-	// are in the URL.
-	hapc, err := haproxy.NewHaproxyClient(m.HostData().SanitizedURI)
+	hapc, err := haproxy.NewHaproxyClient(m.HostData().URI)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed creating haproxy client")
 	}