[builder] Support for --skip-new-go-module

.chloggen/builder-skip-module.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: builder
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add a --skip-new-go-module flag to skip creating a module in the output directory.
+
+# One or more tracking issues or pull requests related to the change
+issues: [9252]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/builder/README.md

@@ -145,3 +145,23 @@ then commit the code in a git repo. A CI can sync the code and execute
 ocb --skip-generate --skip-get-modules --config=config.yaml
 ```
 to only execute the compilation step.
+
+### Avoiding the use of a new go.mod file
+
+There is an additional option that controls one aspect of the build
+process, which specifically allows skipping the use of a new `go.mod`
+file.  When the `--skip-new-go-module` command-line flag is supplied,
+the build process issues a `go get` command for each component,
+relying on the Go toolchain to update the enclosing Go module
+appropriately.
+
+This command will avoid downgrading a dependency in the enclosing
+module, according to
+[`semver.Compare()`](https://pkg.go.dev/golang.org/x/mod/semver#Compare),
+and will instead issue a log indicating that the component was not
+upgraded.
+
+This mode cannot be used in conjunction with several features that
+control the generated `go.mod` file, including `replaces`, `excludes`,
+and the component `path` override.  For each of these features, users
+are expected to modify the enclosing `go.mod` directly.

cmd/builder/go.mod

@@ -18,6 +18,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
+	golang.org/x/mod v0.14.0
 )
 
 require (

cmd/builder/internal/builder/config.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-version"
 	"go.uber.org/multierr"
@@ -18,15 +19,20 @@ import (
 
 const defaultOtelColVersion = "0.97.0"
 
-// ErrInvalidGoMod indicates an invalid gomod
-var ErrInvalidGoMod = errors.New("invalid gomod specification for module")
+var (
+	// ErrInvalidGoMod indicates an invalid gomod
+	ErrInvalidGoMod = errors.New("invalid gomod specification for module")
+	// ErrIncompatibleConfigurationValues indicates that there is configuration that cannot be combined
+	ErrIncompatibleConfigurationValues = errors.New("cannot combine configuration values")
+)
 
 // Config holds the builder's configuration
 type Config struct {
 	Logger          *zap.Logger
 	SkipGenerate    bool   `mapstructure:"-"`
 	SkipCompilation bool   `mapstructure:"-"`
 	SkipGetModules  bool   `mapstructure:"-"`
+	SkipNewGoModule bool   `mapstructure:"-"`
 	LDFlags         string `mapstructure:"-"`
 	Verbose         bool   `mapstructure:"-"`
 
@@ -38,6 +44,8 @@ type Config struct {
 	Connectors   []Module     `mapstructure:"connectors"`
 	Replaces     []string     `mapstructure:"replaces"`
 	Excludes     []string     `mapstructure:"excludes"`
+
+	downloadModules retry `mapstructure:"-"`
 }
 
 // Distribution holds the parameters for the final binary
@@ -62,6 +70,11 @@ type Module struct {
 	Path   string `mapstructure:"path"`   // an optional path to the local version of this module
 }
 
+type retry struct {
+	numRetries int
+	wait       time.Duration
+}
+
 // NewDefaultConfig creates a new config, with default values
 func NewDefaultConfig() Config {
 	log, err := zap.NewDevelopment()
@@ -81,17 +94,25 @@ func NewDefaultConfig() Config {
 			OtelColVersion: defaultOtelColVersion,
 			Module:         "go.opentelemetry.io/collector/cmd/builder",
 		},
+
+		// basic retry if error from go mod command (in case of transient network error). This could be improved
+		// retry 3 times with 5 second spacing interval
+		downloadModules: retry{
+			numRetries: 3,
+			wait:       5 * time.Second,
+		},
 	}
 }
 
 // Validate checks whether the current configuration is valid
 func (c *Config) Validate() error {
 	return multierr.Combine(
-		validateModules(c.Extensions),
-		validateModules(c.Receivers),
-		validateModules(c.Exporters),
-		validateModules(c.Processors),
-		validateModules(c.Connectors),
+		c.validateModules(c.Extensions),
+		c.validateModules(c.Receivers),
+		c.validateModules(c.Exporters),
+		c.validateModules(c.Processors),
+		c.validateModules(c.Connectors),
+		c.validateFlags(),
 	)
 }
 
@@ -158,11 +179,21 @@ func (c *Config) ParseModules() error {
 	return nil
 }
 
-func validateModules(mods []Module) error {
+func (c *Config) validateFlags() error {
+	if c.SkipNewGoModule && (len(c.Replaces) != 0 || len(c.Excludes) != 0) {
+		return fmt.Errorf("%w excludes or replaces with --skip-new-go-module; please modify the enclosing go.mod file directly", ErrIncompatibleConfigurationValues)
+	}
+	return nil
+}
+
+func (c *Config) validateModules(mods []Module) error {
 	for _, mod := range mods {
 		if mod.GoMod == "" {
 			return fmt.Errorf("module %q: %w", mod.GoMod, ErrInvalidGoMod)
 		}
+		if mod.Path != "" && c.SkipNewGoModule {
+			return fmt.Errorf("%w cannot modify mod.path \"%v\" combined with --skip-new-go-module; please modify the enclosing go.mod file directly", ErrIncompatibleConfigurationValues, mod.Path)
+		}
 	}
 	return nil
 }
@@ -171,7 +202,7 @@ func parseModules(mods []Module) ([]Module, error) {
 	var parsedModules []Module
 	for _, mod := range mods {
 		if mod.Import == "" {
-			mod.Import = strings.Split(mod.GoMod, " ")[0]
+			mod.Import, _, _ = strings.Cut(mod.GoMod, " ")
 		}
 
 		if mod.Name == "" {

cmd/builder/internal/builder/config_test.go

@@ -4,7 +4,6 @@
 package builder
 
 import (
-	"errors"
 	"os"
 	"strings"
 	"testing"
@@ -122,10 +121,37 @@ func TestInvalidModule(t *testing.T) {
 			},
 			err: ErrInvalidGoMod,
 		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Extensions: []Module{{
+					GoMod: "some-module",
+					Path:  "invalid",
+				}},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Replaces:        []string{"", ""},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Excludes:        []string{"", ""},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
 	}
 
 	for _, test := range configurations {
-		assert.True(t, errors.Is(test.cfg.Validate(), test.err))
+		assert.ErrorIs(t, test.cfg.Validate(), test.err)
 	}
 }