[confmap] Add featuregate to use stable expansion rules (#10391)

#### Description
This PR adds a feature gate that will handle transitioning users away
from expandconverter, specifically expanding `$VAR` syntax.

The wholistic strategy is:
1. create a new feature gate, `confmap.unifyEnvVarExpansion`, that will
be the single feature gate to manage unifying collector configuraiton
resolution.
2. Update expandconverter to return an error if the feature gate is
enabled and it is about to expand `$VAR` syntax.
3. Update `otelcol.NewCommand` to set a `DefaultScheme="env"` when the
feature gate is enabled and no `DefaultScheme` is set, this handles
`${VAR}` syntax.
  4. Separately, deprecate `expandconverter`.
  5. Follow a normal feature gate cycle.
6. Removed the `confmap.unifyEnvVarExpansion` feature gates and
`expandconverter` at the same time

Supersedes
#10259

#### Link to tracking issue
Related to
#10161
Related to
#8215
Related to
#7111

#### Testing
Unit tests

.chloggen/confmap-add-expand-featuregate-3.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: otelcol/expandconverter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `confmap.unifyEnvVarExpansion` feature gate to allow enabling Collector/Configuration SIG environment variable expansion rules.
+
+# One or more tracking issues or pull requests related to the change
+issues: [10391]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  When enabled, this feature gate will:
+  - Disable expansion of BASH-style env vars (`$FOO`)
+  - `${FOO}` will be expanded as if it was `${env:FOO}
+  See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/rfcs/env-vars.md for more details.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/confmap-add-expand-featuregate.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: confmap
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `confmap.unifyEnvVarExpansion` feature gate to allow enabling Collector/Configuration SIG environment variable expansion rules.
+
+# One or more tracking issues or pull requests related to the change
+issues: [10259]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  When enabled, this feature gate will:
+    - Disable expansion of BASH-style env vars (`$FOO`)
+    - `${FOO}` will be expanded as if it was `${env:FOO}
+  See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/rfcs/env-vars.md for more details.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

confmap/converter/expandconverter/expand.go

@@ -5,6 +5,7 @@ package expandconverter // import "go.opentelemetry.io/collector/confmap/convert
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"regexp"
@@ -13,6 +14,7 @@ import (
 
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/confmap/internal/envvar"
+	"go.opentelemetry.io/collector/internal/featuregates"
 )
 
 type converter struct {
@@ -92,6 +94,10 @@ func (c converter) expandEnv(s string) (string, error) {
 		// in order to make sure we don't log a warning for ${VAR}
 		var regex = regexp.MustCompile(fmt.Sprintf(`\$%s`, regexp.QuoteMeta(str)))
 		if _, exists := c.loggedDeprecations[str]; !exists && regex.MatchString(s) {
+			if featuregates.UseUnifiedEnvVarExpansionRules.IsEnabled() {
+				err = errors.New("$VAR expansion is not supported when feature gate confmap.unifyEnvVarExpansion is enabled")
+				return ""
+			}
 			msg := fmt.Sprintf("Variable substitution using $VAR will be deprecated in favor of ${VAR} and ${env:VAR}, please update $%s", str)
 			c.logger.Warn(msg, zap.String("variable", str))
 			c.loggedDeprecations[str] = struct{}{}

confmap/converter/expandconverter/expand_test.go

@@ -18,6 +18,8 @@ import (
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
 	"go.opentelemetry.io/collector/confmap/internal/envvar"
+	"go.opentelemetry.io/collector/featuregate"
+	"go.opentelemetry.io/collector/internal/featuregates"
 )
 
 func TestNewExpandConverter(t *testing.T) {
@@ -56,6 +58,31 @@ func TestNewExpandConverter(t *testing.T) {
 	}
 }
 
+func TestNewExpandConverter_UseUnifiedEnvVarExpansionRules(t *testing.T) {
+	require.NoError(t, featuregate.GlobalRegistry().Set(featuregates.UseUnifiedEnvVarExpansionRules.ID(), true))
+	t.Cleanup(func() {
+		require.NoError(t, featuregate.GlobalRegistry().Set(featuregates.UseUnifiedEnvVarExpansionRules.ID(), false))
+	})
+
+	const valueExtra = "some string"
+	const valueExtraMapValue = "some map value"
+	const valueExtraListMapValue = "some list map value"
+	const valueExtraListElement = "some list value"
+	t.Setenv("EXTRA", valueExtra)
+	t.Setenv("EXTRA_MAP_VALUE_1", valueExtraMapValue+"_1")
+	t.Setenv("EXTRA_MAP_VALUE_2", valueExtraMapValue+"_2")
+	t.Setenv("EXTRA_LIST_MAP_VALUE_1", valueExtraListMapValue+"_1")
+	t.Setenv("EXTRA_LIST_MAP_VALUE_2", valueExtraListMapValue+"_2")
+	t.Setenv("EXTRA_LIST_VALUE_1", valueExtraListElement+"_1")
+	t.Setenv("EXTRA_LIST_VALUE_2", valueExtraListElement+"_2")
+
+	conf, err := confmaptest.LoadConf(filepath.Join("testdata", "expand-with-all-env.yaml"))
+	require.NoError(t, err, "Unable to get config")
+
+	// Test that expanded configs are the same with the simple config with no env vars.
+	require.ErrorContains(t, createConverter().Convert(context.Background(), conf), "$VAR expansion is not supported when feature gate confmap.unifyEnvVarExpansion is enabled")
+}
+
 func TestNewExpandConverter_EscapedMaps(t *testing.T) {
 	const receiverExtraMapValue = "some map value"
 	t.Setenv("MAP_VALUE", receiverExtraMapValue)

confmap/converter/expandconverter/go.mod

@@ -4,14 +4,17 @@ go 1.21.0
 
 require (
 	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/collector v0.102.1
 	go.opentelemetry.io/collector/confmap v0.102.1
+	go.opentelemetry.io/collector/featuregate v1.9.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.27.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.1.1 // indirect
@@ -22,4 +25,18 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace go.opentelemetry.io/collector/confmap => ../../
+replace go.opentelemetry.io/collector/component => ../../../component
+
+replace go.opentelemetry.io/collector/confmap => ../..
+
+replace go.opentelemetry.io/collector => ../../..
+
+replace go.opentelemetry.io/collector/config/configtelemetry => ../../../config/configtelemetry
+
+replace go.opentelemetry.io/collector/pdata/testdata => ../../../pdata/testdata
+
+replace go.opentelemetry.io/collector/pdata => ../../../pdata
+
+replace go.opentelemetry.io/collector/featuregate => ../../../featuregate
+
+replace go.opentelemetry.io/collector/consumer => ../../../consumer

confmap/go.mod

@@ -16,9 +16,12 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )
 
 retract (