Support configurable certificate stores

* Issue #27: change certificates via API
* Issue #28: refactor listener config
* Issue #70: support Vault
* Issue #85: SNI support

.travis.yml

@@ -2,3 +2,11 @@ language: go
 
 go:
     - 1.6.2
+
+before_script:
+  - wget https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip
+  - unzip consul_0.6.4_linux_amd64.zip
+  - ./consul --version
+
+script:
+  - ../consul agent -server -bootstrap-expect 1 -data-dir /tmp/consul &

cert/consul_source.go

@@ -0,0 +1,132 @@
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"log"
+	"net/url"
+	"path"
+	"reflect"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+)
+
+type ConsulSource struct {
+	CertURL     string
+	ClientCAURL string
+}
+
+const kvURLPrefix = "/v1/kv/"
+
+func parseConsulURL(consulURL, stripPrefix string) (client *api.Client, key string, err error) {
+	u, err := url.Parse(consulURL)
+	if err != nil {
+		return nil, "", err
+	}
+	var token string
+	if len(u.Query()["token"]) > 0 {
+		token = u.Query()["token"][0]
+	}
+	client, err = api.NewClient(&api.Config{Address: u.Host, Scheme: u.Scheme, Token: token})
+	if err != nil {
+		return nil, "", err
+	}
+	key = u.RequestURI()[len(stripPrefix):]
+	return client, key, nil
+}
+
+func (s ConsulSource) LoadClientCAs() (*x509.CertPool, error) {
+	if s.ClientCAURL == "" {
+		return nil, nil
+	}
+
+	client, key, err := parseConsulURL(s.ClientCAURL, kvURLPrefix)
+	if err != nil {
+		return nil, err
+	}
+
+	pemBlocks, _, err := getCerts(client, key, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(pemBlocks) == 0 {
+		return nil, nil
+	}
+
+	x := x509.NewCertPool()
+	for name, pemBlock := range pemBlocks {
+		if !x.AppendCertsFromPEM(pemBlock) {
+			log.Printf("[WARN] cert: Failed to add client CA certificate from %s", name)
+			continue
+		}
+	}
+
+	log.Printf("[INFO] cert: Load client CA certs from %s", s.ClientCAURL)
+	return x, nil
+}
+
+func (s ConsulSource) Certificates() chan []tls.Certificate {
+	if s.CertURL == "" {
+		return nil
+	}
+
+	client, key, err := parseConsulURL(s.CertURL, kvURLPrefix)
+	if err != nil {
+		log.Printf("[ERROR] cert: Failed to create consul client. %s", err)
+	}
+
+	pemBlocksCh := make(chan map[string][]byte, 1)
+	go watchKV(client, key, pemBlocksCh)
+
+	ch := make(chan []tls.Certificate, 1)
+	go func() {
+		for pemBlocks := range pemBlocksCh {
+			certs, err := loadCertificates(pemBlocks)
+			if err != nil {
+				log.Printf("[ERROR] cert: Failed to load certificates. %s", err)
+				continue
+			}
+			ch <- certs
+		}
+	}()
+	return ch
+}
+
+// watchKV monitors a key in the KV store for changes.
+func watchKV(client *api.Client, key string, pemBlocks chan map[string][]byte) {
+	var lastIndex uint64
+	var lastValue map[string][]byte
+
+	for {
+		value, index, err := getCerts(client, key, lastIndex)
+		if err != nil {
+			log.Printf("[WARN] cert: Error fetching certificates from %s. %v", key, err)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		if !reflect.DeepEqual(value, lastValue) || index != lastIndex {
+			log.Printf("[INFO] cert: Certificate index changed to #%d", index)
+			pemBlocks <- value
+			lastValue, lastIndex = value, index
+		}
+	}
+}
+
+func getCerts(client *api.Client, key string, waitIndex uint64) (pemBlocks map[string][]byte, lastIndex uint64, err error) {
+	q := &api.QueryOptions{RequireConsistent: true, WaitIndex: waitIndex}
+	kvpairs, meta, err := client.KV().List(key, q)
+	if err != nil {
+		return nil, 0, err
+	}
+	if len(kvpairs) == 0 {
+		return nil, meta.LastIndex, nil
+	}
+	pemBlocks = map[string][]byte{}
+	for _, kvpair := range kvpairs {
+		pemBlocks[path.Base(kvpair.Key)] = kvpair.Value
+	}
+	return pemBlocks, meta.LastIndex, nil
+}

cert/file_source.go

@@ -0,0 +1,62 @@
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"log"
+)
+
+// FileSource implements a certificate source for one
+// TLS and one client authentication certificate.
+// The certificates are loaded during startup and are cached
+// in memory until the program exits.
+// It exists to support the legacy configuration only. The
+// PathSource should be used instead.
+type FileSource struct {
+	CertFile       string
+	KeyFile        string
+	ClientAuthFile string
+}
+
+func (s FileSource) LoadClientCAs() (*x509.CertPool, error) {
+	if s.ClientAuthFile == "" {
+		return nil, nil
+	}
+
+	pemBlock, err := ioutil.ReadFile(s.ClientAuthFile)
+	if err != nil {
+		return nil, fmt.Errorf("cert: cannot load client CAs. %s", err)
+	}
+
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(pemBlock) {
+		return nil, fmt.Errorf("cert: failed to add client auth certs from %s", s.ClientAuthFile)
+	}
+
+	return pool, nil
+}
+
+func (s FileSource) Certificates() chan []tls.Certificate {
+	ch := make(chan []tls.Certificate, 1)
+	ch <- []tls.Certificate{loadX509KeyPair(s.CertFile, s.KeyFile)}
+	close(ch)
+	return ch
+}
+
+func loadX509KeyPair(certFile, keyFile string) tls.Certificate {
+	if certFile == "" {
+		log.Fatalf("[FATAL] cert: CertFile is required")
+	}
+
+	if keyFile == "" {
+		keyFile = certFile
+	}
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Fatalf("[FATAL] cert: Error loading certificate. %s", err)
+	}
+	return cert
+}

cert/http_source.go

@@ -0,0 +1,68 @@
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"log"
+	"time"
+)
+
+// HTTPSource implements a certificate source which loads
+// TLS and client authentication certificates from an HTTP/HTTPS server.
+// The CertURL/ClientCAURL must point to a text file in the directory
+// of the certificates. The text file contains all files that should
+// be loaded from this directory - one filename per line.
+// The TLS certificates are updated automatically when Refresh
+// is not zero. Refresh cannot be less than one second to prevent
+// busy loops.
+type HTTPSource struct {
+	CertURL     string
+	ClientCAURL string
+	Refresh     time.Duration
+}
+
+func (s HTTPSource) LoadClientCAs() (*x509.CertPool, error) {
+	pemBlocks, err := loadURL(s.ClientCAURL)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(pemBlocks) == 0 {
+		return nil, nil
+	}
+
+	x := x509.NewCertPool()
+	for name, pemBlock := range pemBlocks {
+		if !x.AppendCertsFromPEM(pemBlock) {
+			log.Printf("[WARN] cert: Could not add client CA certificate from %s", name)
+			continue
+		}
+	}
+
+	log.Printf("[INFO] cert: Load client CA certs from %s", s.ClientCAURL)
+	return x, nil
+}
+
+func (s HTTPSource) Certificates() chan []tls.Certificate {
+	loadCerts := func() ([]tls.Certificate, error) {
+		pemBlocks, err := loadURL(s.CertURL)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(pemBlocks) == 0 {
+			return nil, nil
+		}
+
+		certs, err := loadCertificates(pemBlocks)
+		if err != nil {
+			return nil, err
+		}
+
+		return certs, nil
+	}
+
+	ch := make(chan []tls.Certificate, 1)
+	go watch(ch, s.Refresh, s.CertURL, loadCerts)
+	return ch
+}