Consul catalog provider

README.md

@@ -2,14 +2,15 @@
   <img class="logo" src="https://raw.githubusercontent.com/umputun/reproxy/master/site/src/logo-bg.svg" width="355px" height="142px" alt="Reproxy | Simple Reverse Proxy"/>
 </div>
 
-Reproxy is a simple edge HTTP(s) server / reverse proxy supporting various providers (docker, static, file). One or more providers supply information about the requested server, requested URL, destination URL, and health check URL. It is distributed as a single binary or as a docker container.
+Reproxy is a simple edge HTTP(s) server / reverse proxy supporting various providers (docker, static, file, consul catalog). One or more providers supply information about the requested server, requested URL, destination URL, and health check URL. It is distributed as a single binary or as a docker container.
 
 - Automatic SSL termination with <a href="https://letsencrypt.org/" rel="nofollow noopener noreferrer" target="_blank">Let's Encrypt</a>
 - Support of user-provided SSL certificates
 - Simple but flexible proxy rules
 - Static, command-line proxy rules provider
 - Dynamic, file-based proxy rules provider
 - Docker provider with an automatic discovery
+- Consul Catalog provider with discovery by service tags
 - Support of multiple (virtual) hosts
 - Optional traffic compression
 - User-defined limits and timeouts
@@ -99,6 +100,37 @@ If no `reproxy.route` defined, the default is `http://<container_name>:<containe
 
 This is a dynamic provider and any change in container's status will be applied automatically.
 
+### Consul Catalog
+
+Use: `reproxy --consul-catalog.enabled`
+
+Consul Catalog provider periodically (every second by default) calls Consul API for obtaining services, which has any tag with `reproxy.` prefix.
+
+You can redefine check interval with `--consul-catalog.interval` command line flag.
+
+Also, you can redefine consul address with `--consul-catalog.address` command line option. Default address is `http://127.0.0.1:8500`.
+
+For example:
+```
+reproxy --consul-catalog.enabled --consul-catalog.address=http://192.168.1.100:8500 --consul-catalog.interval=10s  
+```
+
+By default, provider sets values for every service:
+- enabled `false`
+- server `*`
+- route `^/(.*)`
+- dest `http://<SERVICE_ADDRESS_FROM_CONSUL>/$1`
+- ping `http://<SERVICE_ADDRESS_FROM_CONSUL>/ping`
+
+This default can be changed with tags:
+
+- `reproxy.server` - server (hostname) to match. Also, can be a list of comma-separated servers.
+- `reproxy.route` - source route (location)
+- `reproxy.dest` - destination path. Note: this is not full url, but just the path which will be appended to service's ip:port
+- `reproxy.port` - destination port for the discovered service
+- `reproxy.ping` - ping path for the destination service.
+- `reproxy.enabled` - enable (`yes`, `true`, `1`) or disable (`any different value`) service from reproxy destinations.
+
 ## SSL support
 
 SSL mode (by default none) can be set to `auto` (ACME/LE certificates), `static` (existing certificate) or `none`. If `auto` turned on SSL certificate will be issued automatically for all discovered server names. User can override it by setting `--ssl.fqdn` value(s)

app/discovery/discovery.go

@@ -49,9 +49,10 @@ type ProviderID string
 
 // enum of all provider ids
 const (
-	PIDocker ProviderID = "docker"
-	PIStatic ProviderID = "static"
-	PIFile   ProviderID = "file"
+	PIDocker        ProviderID = "docker"
+	PIStatic        ProviderID = "static"
+	PIFile          ProviderID = "file"
+	PIConsulCatalog ProviderID = "consul-catalog"
 )
 
 // MatchType defines the type of mapper (rule)

app/discovery/provider/consulcatalog/client.go

@@ -0,0 +1,159 @@
+package consulcatalog
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+)
+
+// HTTPClient represents interface for http client
+type HTTPClient interface {
+	Do(r *http.Request) (*http.Response, error)
+}
+
+// consulClient allows to get consul services with 'reproxy' tags
+// 1. Client calls https://www.consul.io/api-docs/catalog#list-services API for get services list.
+// It returns services list with names and tags (without addresses)
+// Next, Client filters this list for exclude services without 'reproxy' tags
+// 2. Client calls https://www.consul.io/api-docs/catalog#list-nodes-for-service API for every service
+// This API returns data about every service instance. Include address, port and more
+// Client stores services addresses and ports to internal storage
+type consulClient struct {
+	address    string
+	httpClient HTTPClient
+}
+
+// NewClient creates new Consul consulClient
+func NewClient(address string, httpClient HTTPClient) ConsulClient {
+	cl := &consulClient{
+		address:    strings.TrimSuffix(address, "/"),
+		httpClient: httpClient,
+	}
+
+	return cl
+}
+
+// Get implements ConsulClient interface and returns consul services list,
+// which have any tag with 'reproxy.' prefix
+func (cl *consulClient) Get() ([]consulService, error) {
+	var result []consulService //nolint:prealloc // We cannot calc slice size
+
+	serviceNames, err := cl.getServiceNames()
+	if err != nil {
+		return nil, fmt.Errorf("error get service names, %w", err)
+	}
+
+	for _, serviceName := range serviceNames {
+		services, err := cl.getServices(serviceName)
+		if err != nil {
+			return nil, fmt.Errorf("error get nodes for service name %s, %w", serviceName, err)
+		}
+		result = append(result, services...)
+	}
+
+	return result, nil
+}
+
+func (cl *consulClient) getServiceNames() ([]string, error) {
+	req, err := http.NewRequest(http.MethodGet, cl.address+"/v1/catalog/services", nil)
+	if err != nil {
+		return nil, fmt.Errorf("error create a http request, %w", err)
+	}
+
+	resp, err := cl.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error send request to consul, %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response status code %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error read response body, %w", err)
+	}
+	err = resp.Body.Close()
+	if err != nil {
+		log.Printf("[ERROR] error close body, %v", err)
+	}
+
+	result := map[string][]string{}
+
+	err = json.Unmarshal(body, &result)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshal consul response, %w", err)
+	}
+
+	return cl.filterServices(result), nil
+}
+
+func (cl *consulClient) filterServices(src map[string][]string) []string {
+	var result []string
+
+	for serviceName, tags := range src {
+		for _, tag := range tags {
+			if strings.HasPrefix(tag, "reproxy.") {
+				result = append(result, serviceName)
+			}
+		}
+	}
+
+	sort.Strings(result)
+
+	return result
+}
+
+func (cl *consulClient) getServices(serviceName string) ([]consulService, error) {
+	req, err := http.NewRequest(http.MethodGet, cl.address+"/v1/catalog/service/"+serviceName, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error create a http request, %w", err)
+	}
+
+	resp, err := cl.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error send request to consul, %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response status code %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error read response body, %w", err)
+	}
+	err = resp.Body.Close()
+	if err != nil {
+		log.Printf("[ERROR] error close body, %v", err)
+	}
+
+	var services []consulService
+
+	err = json.Unmarshal(body, &services)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshal consul response, %w", err)
+	}
+
+	for idx, s := range services {
+		s.Labels = make(map[string]string)
+		for _, t := range s.ServiceTags {
+			if strings.HasPrefix(t, "reproxy.") {
+				delimiterIdx := strings.IndexByte(t, '=')
+				if delimiterIdx == -1 || delimiterIdx <= len("reproxy.") {
+					s.Labels[t] = ""
+					continue
+				}
+
+				s.Labels[t[:delimiterIdx]] = t[delimiterIdx+1:]
+			}
+		}
+		services[idx] = s
+	}
+
+	return services, nil
+}