add_cloud_metadata: env var override for providers (#38669)

* add_cloud_metadata: env var override for providers

Add support for configuring the add_cloud_metadata providers
with an environment variable, $BEATS_ADD_CLOUD_METADATA_PROVIDERS.

This may be useful when deploying Elastic Agent standalone in a
cloud provider managed Kubernetes cluster, where the cloud provider
is known at deployment time.

* Fix changelog

* appease golangci-lint

* More appeasement

(cherry picked from commit d0399e0)

CHANGELOG.next.asciidoc

@@ -106,6 +106,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments
 - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843]
 - Update to Go 1.21.9. {pulk}38727[38727]
+- Enable early event encoding in the Elasticsearch output, improving cpu and memory use {pull}38572[38572]
+- The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669]
 
 *Auditbeat*
 

libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc

@@ -50,6 +50,9 @@ cloud or on-premise).
 The second optional setting is `providers`. The `providers` settings accepts a
 list of cloud provider names to be used. If `providers` is not configured, then
 all providers that do not access a remote endpoint are enabled by default.
+The list of providers may alternatively be configured with the environment
+variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS`, by setting it to a comma-separated
+list of provider names.
 
 List of names the `providers` setting supports:
 

libbeat/processors/add_cloud_metadata/providers.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
+	"strings"
 	"time"
 
 	conf "github.com/elastic/elastic-agent-libs/config"
@@ -73,6 +75,21 @@ func selectProviders(configList providerList, providers map[string]provider) map
 }
 
 func providersFilter(configList providerList, allProviders map[string]provider) func(string) bool {
+	if v, ok := os.LookupEnv("BEATS_ADD_CLOUD_METADATA_PROVIDERS"); ok {
+		// We allow users to override the config and defaults with
+		// this environment variable as a workaround in case the
+		// configured/default providers misbehave.
+		configList = nil
+		for _, name := range strings.Split(v, ",") {
+			configList = append(configList, strings.TrimSpace(name))
+		}
+		if len(configList) == 0 {
+			// User explicitly disabled all providers.
+			return func(string) bool {
+				return false
+			}
+		}
+	}
 	if len(configList) == 0 {
 		return func(name string) bool {
 			ff, ok := allProviders[name]

libbeat/processors/add_cloud_metadata/providers_test.go

@@ -18,6 +18,7 @@
 package add_cloud_metadata
 
 import (
+	"os"
 	"sort"
 	"testing"
 
@@ -26,25 +27,38 @@ import (
 	conf "github.com/elastic/elastic-agent-libs/config"
 )
 
+func init() {
+	os.Unsetenv("BEATS_ADD_CLOUD_METADATA_PROVIDERS")
+}
+
 func TestProvidersFilter(t *testing.T) {
-	var all []string
 	var allLocal []string
 	for name, ff := range cloudMetaProviders {
-		all = append(all, name)
 		if ff.Local {
 			allLocal = append(allLocal, name)
 		}
 	}
 
 	cases := map[string]struct {
 		config   map[string]interface{}
+		env      string
 		fail     bool
 		expected []string
 	}{
 		"all with local access only if not configured": {
 			config:   map[string]interface{}{},
 			expected: allLocal,
 		},
+		"BEATS_ADD_CLOUD_METADATA_PROVIDERS overrides default": {
+			config:   map[string]interface{}{},
+			env:      "alibaba, digitalocean",
+			expected: []string{"alibaba", "digitalocean"},
+		},
+		"none if BEATS_ADD_CLOUD_METADATA_PROVIDERS is explicitly set to an empty list": {
+			config:   map[string]interface{}{},
+			env:      " ",
+			expected: nil,
+		},
 		"fail to load if unknown name is used": {
 			config: map[string]interface{}{
 				"providers": []string{"unknown"},
@@ -56,18 +70,25 @@ func TestProvidersFilter(t *testing.T) {
 				"providers": []string{"aws", "gcp", "digitalocean"},
 			},
 		},
+		"BEATS_ADD_CLOUD_METADATA_PROVIDERS overrides selected": {
+			config: map[string]interface{}{
+				"providers": []string{"aws", "gcp", "digitalocean"},
+			},
+			env:      "alibaba, digitalocean",
+			expected: []string{"alibaba", "digitalocean"},
+		},
 	}
 
 	copyStrings := func(in []string) (out []string) {
-		for _, str := range in {
-			out = append(out, str)
-		}
-		return out
+		return append(out, in...)
 	}
 
 	for name, test := range cases {
 		t.Run(name, func(t *testing.T) {
 			rawConfig := conf.MustNewConfigFrom(test.config)
+			if test.env != "" {
+				t.Setenv("BEATS_ADD_CLOUD_METADATA_PROVIDERS", test.env)
+			}
 
 			config := defaultConfig()
 			err := rawConfig.Unpack(&config)