Add openstack ssl provider in add_cloud_metadata (elastic#21590)

Add a new provider to query metadata from OpenStack deployments using HTTPS.
Add the usual SSL settings available in other features that support TLS.

CHANGELOG.next.asciidoc

@@ -453,6 +453,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Cloud Foundry metadata is cached to disk. {pull}20775[20775]
 - Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
 - Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525]
+- Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590]
 
 *Auditbeat*
 

libbeat/processors/add_cloud_metadata/add_cloud_metadata.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
 	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
@@ -53,6 +54,7 @@ type addCloudMetadata struct {
 type initData struct {
 	fetchers  []metadataFetcher
 	timeout   time.Duration
+	tlsConfig *tlscommon.TLSConfig
 	overwrite bool
 }
 
@@ -63,14 +65,24 @@ func New(c *common.Config) (processors.Processor, error) {
 		return nil, errors.Wrap(err, "failed to unpack add_cloud_metadata config")
 	}
 
+	tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS)
+	if err != nil {
+		return nil, errors.Wrap(err, "TLS configuration load")
+	}
+
 	initProviders := selectProviders(config.Providers, cloudMetaProviders)
 	fetchers, err := setupFetchers(initProviders, c)
 	if err != nil {
 		return nil, err
 	}
 	p := &addCloudMetadata{
-		initData: &initData{fetchers, config.Timeout, config.Overwrite},
-		logger:   logp.NewLogger("add_cloud_metadata"),
+		initData: &initData{
+			fetchers:  fetchers,
+			timeout:   config.Timeout,
+			tlsConfig: tlsConfig,
+			overwrite: config.Overwrite,
+		},
+		logger: logp.NewLogger("add_cloud_metadata"),
 	}
 
 	go p.init()

libbeat/processors/add_cloud_metadata/config.go

@@ -20,12 +20,15 @@ package add_cloud_metadata
 import (
 	"fmt"
 	"time"
+
+	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 )
 
 type config struct {
-	Timeout   time.Duration `config:"timeout"`   // Amount of time to wait for responses from the metadata services.
-	Overwrite bool          `config:"overwrite"` // Overwrite if cloud.* fields already exist.
-	Providers providerList  `config:"providers"` // List of providers to probe
+	Timeout   time.Duration     `config:"timeout"`   // Amount of time to wait for responses from the metadata services.
+	TLS       *tlscommon.Config `config:"ssl"`       // TLS configuration
+	Overwrite bool              `config:"overwrite"` // Overwrite if cloud.* fields already exist.
+	Providers providerList      `config:"providers"` // List of providers to probe
 }
 
 type providerList []string

libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc

@@ -52,12 +52,16 @@ List of names the `providers` setting supports:
 - "aws", or "ec2" for Amazon Web Services (enabled by default).
 - "gcp" for Google Copmute Enging (enabled by default).
 - "openstack", or "nova" for Openstack Nova (enabled by default).
+- "openstack-ssl", or "nova-ssl" for Openstack Nova when SSL metadata APIs are enabled (enabled by default).
 - "tencent", or "qcloud" for Tencent Cloud (disabled by default).
 
 The third optional configuration setting is `overwrite`. When `overwrite` is
 `true`, `add_cloud_metadata` overwrites existing `cloud.*` fields (`false` by
 default).
 
+The `add_cloud_metadata` processor supports SSL options to configure the http
+client used to query cloud metadata. See <<configuration-ssl>> for more information.
+
 The metadata that is added to events varies by hosting provider. Below are
 examples for each of the supported providers.
 

libbeat/processors/add_cloud_metadata/http_fetcher.go

@@ -27,6 +27,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 )
 
 type httpMetadataFetcher struct {
@@ -127,9 +128,15 @@ func (f *httpMetadataFetcher) fetchRaw(
 
 // getMetadataURLs loads config and generates the metadata URLs.
 func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string) ([]string, error) {
+	return getMetadataURLsWithScheme(c, "http", defaultHost, metadataURIs)
+}
+
+// getMetadataURLsWithScheme loads config and generates the metadata URLs.
+func getMetadataURLsWithScheme(c *common.Config, scheme string, defaultHost string, metadataURIs []string) ([]string, error) {
 	var urls []string
 	config := struct {
-		MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only).
+		MetadataHostAndPort string            `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only).
+		TLSConfig           *tlscommon.Config `config:"ssl"`
 	}{
 		MetadataHostAndPort: defaultHost,
 	}
@@ -138,7 +145,7 @@ func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string
 		return urls, errors.Wrap(err, "failed to unpack add_cloud_metadata config")
 	}
 	for _, uri := range metadataURIs {
-		urls = append(urls, "http://"+config.MetadataHostAndPort+uri)
+		urls = append(urls, scheme+"://"+config.MetadataHostAndPort+uri)
 	}
 	return urls, nil
 }

libbeat/processors/add_cloud_metadata/provider_openstack_nova.go

@@ -32,16 +32,24 @@ const (
 // OpenStack Nova Metadata Service
 // Document https://docs.openstack.org/nova/latest/user/metadata-service.html
 var openstackNovaMetadataFetcher = provider{
-	Name: "openstack-nova",
+	Name:   "openstack-nova",
+	Local:  true,
+	Create: buildOpenstackNovaCreate("http"),
+}
 
-	Local: true,
+var openstackNovaSSLMetadataFetcher = provider{
+	Name:   "openstack-nova-ssl",
+	Local:  true,
+	Create: buildOpenstackNovaCreate("https"),
+}
 
-	Create: func(provider string, c *common.Config) (metadataFetcher, error) {
+func buildOpenstackNovaCreate(scheme string) func(provider string, c *common.Config) (metadataFetcher, error) {
+	return func(provider string, c *common.Config) (metadataFetcher, error) {
 		osSchema := func(m map[string]interface{}) common.MapStr {
 			return common.MapStr(m)
 		}
 
-		urls, err := getMetadataURLs(c, metadataHost, []string{
+		urls, err := getMetadataURLsWithScheme(c, scheme, metadataHost, []string{
 			osMetadataInstanceIDURI,
 			osMetadataInstanceTypeURI,
 			osMetadataHostnameURI,
@@ -71,5 +79,5 @@ var openstackNovaMetadataFetcher = provider{
 		}
 		fetcher := &httpMetadataFetcher{"openstack", nil, responseHandlers, osSchema}
 		return fetcher, nil
-	},
+	}
 }

libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go

@@ -29,8 +29,8 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
-func initOpenstackNovaTestServer() *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+func openstackNovaMetadataHandler() http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.RequestURI == osMetadataInstanceIDURI {
 			w.Write([]byte("i-0000ffac"))
 			return
@@ -49,13 +49,13 @@ func initOpenstackNovaTestServer() *httptest.Server {
 		}
 
 		http.Error(w, "not found", http.StatusNotFound)
-	}))
+	})
 }
 
 func TestRetrieveOpenstackNovaMetadata(t *testing.T) {
 	logp.TestingSetup()
 
-	server := initOpenstackNovaTestServer()
+	server := httptest.NewServer(openstackNovaMetadataHandler())
 	defer server.Close()
 
 	config, err := common.NewConfigFrom(map[string]interface{}{
@@ -66,6 +66,28 @@ func TestRetrieveOpenstackNovaMetadata(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	assertOpenstackNova(t, config)
+}
+
+func TestRetrieveOpenstackNovaMetadataWithHTTPS(t *testing.T) {
+	logp.TestingSetup()
+
+	server := httptest.NewTLSServer(openstackNovaMetadataHandler())
+	defer server.Close()
+
+	config, err := common.NewConfigFrom(map[string]interface{}{
+		"host":                  server.Listener.Addr().String(),
+		"ssl.verification_mode": "none",
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertOpenstackNova(t, config)
+}
+
+func assertOpenstackNova(t *testing.T, config *common.Config) {
 	p, err := New(config)
 	if err != nil {
 		t.Fatal(err)

libbeat/processors/add_cloud_metadata/providers.go

@@ -51,17 +51,19 @@ type result struct {
 }
 
 var cloudMetaProviders = map[string]provider{
-	"alibaba":      alibabaCloudMetadataFetcher,
-	"ecs":          alibabaCloudMetadataFetcher,
-	"azure":        azureVMMetadataFetcher,
-	"digitalocean": doMetadataFetcher,
-	"aws":          ec2MetadataFetcher,
-	"ec2":          ec2MetadataFetcher,
-	"gcp":          gceMetadataFetcher,
-	"openstack":    openstackNovaMetadataFetcher,
-	"nova":         openstackNovaMetadataFetcher,
-	"qcloud":       qcloudMetadataFetcher,
-	"tencent":      qcloudMetadataFetcher,
+	"alibaba":       alibabaCloudMetadataFetcher,
+	"ecs":           alibabaCloudMetadataFetcher,
+	"azure":         azureVMMetadataFetcher,
+	"digitalocean":  doMetadataFetcher,
+	"aws":           ec2MetadataFetcher,
+	"ec2":           ec2MetadataFetcher,
+	"gcp":           gceMetadataFetcher,
+	"openstack":     openstackNovaMetadataFetcher,
+	"nova":          openstackNovaMetadataFetcher,
+	"openstack-ssl": openstackNovaSSLMetadataFetcher,
+	"nova-ssl":      openstackNovaSSLMetadataFetcher,
+	"qcloud":        qcloudMetadataFetcher,
+	"tencent":       qcloudMetadataFetcher,
 }
 
 func selectProviders(configList providerList, providers map[string]provider) map[string]provider {
@@ -138,6 +140,7 @@ func (p *addCloudMetadata) fetchMetadata() *result {
 				Timeout:   p.initData.timeout,
 				KeepAlive: 0,
 			}).DialContext,
+			TLSClientConfig: p.initData.tlsConfig.ToConfig(),
 		},
 	}