Fix libbeat argument parsing to allow beats to be imported in other code bases.

[leehinman]

• Revert "refactor fleet mode detection and storage" #41151
• Revert "Don't register cfgfile flags on flag.CommandLine in func init()" #41152

had to be reverted because of APM issues:

• add flag.Parse so HandleFlags works correctly apm-server#14214
• flag provided but not defined: -environment apm-server#14280

Also when trying to add beats to elastic-agent we discovered:

• Move backwards compatible flag manipulation #41136

We need to rework Beats argument parsing and init calls to make libbeat both safe to import in other code bases but also provide guidance for APM to use the modified code correctly.

Locations that need updating:

libbeat

• beats/libbeat/plugin/cli.go

  Line 55 in 6d4fbfc

  flag.Var(plugins, "plugin", "Load additional plugins")

• beats/libbeat/cfgfile/cfgfile.go

  Lines 54 to 64 in 2ff38df

  	func init() {
  	// add '-path.x' options overwriting paths in 'overwrites' config
  	makePathFlag := func(name, usage string) *string {
  	return config.ConfigOverwriteFlag(nil, overwrites, name, name, "", usage)
  	}
  	homePath = makePathFlag("path.home", "Home path")
  	configPath = makePathFlag("path.config", "Configuration path")
  	makePathFlag("path.data", "Data path")
  	makePathFlag("path.logs", "Logs path")
  	}

• beats/libbeat/common/fleetmode/fleet_mode.go

  Lines 29 to 49 in 2ff38df

  	func Enabled() bool {
  	type management struct {
  	Enabled bool `config:"management.enabled"`
  	}
  	var managementSettings management
  	cfgFlag := flag.Lookup("E")
  	if cfgFlag == nil {
  	return false
  	}
  	cfgObject, _ := cfgFlag.Value.(*config.SettingsFlag)
  	cliCfg := cfgObject.Config()
  	err := cliCfg.Unpack(&managementSettings)
  	if err != nil {
  	return false
  	}
  	return managementSettings.Enabled
  	}

• beats/libbeat/statestore/internal/storecompliance/storecompliance.go

  Lines 48 to 51 in 2ff38df

  	func init() {
  	flag.StringVar(&defaultTempDir, "dir", "", "Temporary directory for use by the test")
  	flag.BoolVar(&keepTmpDir, "keep", false, "Keep temporary test directories")
  	}

• beats/libbeat/cmd/root.go

  Lines 36 to 43 in 2ff38df

  	func init() {
  	// backwards compatibility workaround, convert -flags to --flags:
  	for i, arg := range os.Args[1:] {
  	if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "--") && len(arg) > 2 {
  	os.Args[1+i] = "-" + arg
  	}
  	}
  	}

• beats/libbeat/publisher/pipeline/module.go

  Lines 53 to 55 in 2ff38df

  	func init() {
  	flag.BoolVar(&publishDisabled, "N", false, "Disable actual publishing for testing")
  	}

• beats/libbeat/cmd/root.go

  Lines 98 to 115 in 2ff38df

  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("E"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("c"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("e"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("environment"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.home"))
  	rootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("strict.perms"))
  	if f := flag.CommandLine.Lookup("plugin"); f != nil {
  	rootCmd.PersistentFlags().AddGoFlag(f)
  	}
  	// Inherit root flags from run command
  	// TODO deprecate when root command no longer executes run (7.0)
  	rootCmd.Flags().AddFlagSet(rootCmd.RunCmd.Flags())

• beats/libbeat/cmd/instance/beat.go

  Lines 979 to 982 in 2ff38df

  	func (b *Beat) handleFlags() error {
  	flag.Parse()
  	return cfgfile.HandleFlags()
  	}

• beats/libbeat/cmd/run.go

  Lines 44 to 47 in 2ff38df

  	runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("N"))
  	runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("httpprof"))
  	runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("cpuprofile"))
  	runCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("memprofile"))

• beats/libbeat/common/config.go

  Line 35 in 2ff38df

  var flagStrictPerms = flag.Bool("strict.perms", true, "Strict permission checking on config files")

filebeat

• beats/filebeat/fileset/flags.go

  Lines 28 to 32 in 6339a42

  	// Modules related command line flags.
  	var (
  	modulesFlag = flag.String("modules", "", "List of enabled modules (comma separated)")
  	moduleOverrides = config.SettingFlag(nil, "M", "Module configuration overwrite")
  	)

• beats/filebeat/cmd/root.go

  Lines 50 to 52 in 6339a42

  	runFlags := pflag.NewFlagSet(Name, pflag.ExitOnError)
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("once"))
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("modules"))

• beats/filebeat/cmd/root.go

  Lines 66 to 74 in 6339a42

  	func Filebeat(inputs beater.PluginFactory, settings instance.Settings) *cmd.BeatsRootCmd {
  	command := cmd.GenRootCmdWithSettings(beater.New(inputs), settings)
  	command.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
  	command.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
  	command.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
  	command.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
  	command.AddCommand(genGenerateCmd())
  	return command
  	}

• beats/filebeat/beater/filebeat.go

  Line 67 in 6339a42

  var once = flag.Bool("once", false, "Run filebeat only once until all harvesters reach EOF")

metricbeat

• beats/metricbeat/internal/sysinit/init.go

  Line 31 in 6339a42

  var hostfsCLI = flag.String("system.hostfs", "", "Mount point of the host's filesystem for use in monitoring a host from within a container")

• beats/metricbeat/cmd/root.go

  Lines 62 to 63 in 6339a42

  	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs"))

• beats/metricbeat/helper/windows/run.go

  Lines 35 to 42 in 6339a42

  	var (
  	goarch = flag.String("goarch", "", "GOARCH value")
  	output = flag.String("output", "", "output file")
  	cmd = flag.String("cmd", "", "command to run")
  	)
  	func main() {
  	flag.Parse()

• beats/metricbeat/helper/windows/pdh/mkpdh_defs.go

  Lines 33 to 35 in 6339a42

  	var (
  	output = flag.String("output", "defs_pdh_windows.go", "output file")
  	)

packetbeat

• beats/packetbeat/cmd/root.go

  Lines 51 to 57 in 6339a42

  	func PacketbeatSettings(globals processors.PluginConfig) instance.Settings {
  	runFlags := pflag.NewFlagSet(Name, pflag.ExitOnError)
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("I"))
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("t"))
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("O"))
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("l"))
  	runFlags.AddGoFlag(flag.CommandLine.Lookup("dump"))

• beats/packetbeat/beater/packetbeat.go

  Lines 61 to 67 in 6339a42

  	var cmdLineArgs = flags{
  	file: flag.String("I", "", "Read packet data from specified file"),
  	loop: flag.Int("l", 1, "Loop file. 0 - loop forever"),
  	oneAtAtime: flag.Bool("O", false, "Read packets one at a time (press Enter)"),
  	topSpeed: flag.Bool("t", false, "Read packets as fast as possible, without sleeping"),
  	dumpfile: flag.String("dump", "", "Write all captured packets to libpcap files with this prefix - a timestamp and pcap extension are added"),
  	}

x-pack

• beats/x-pack/filebeat/module/cisco/shared/gen-ftd-ecs-mapping.go

  Lines 22 to 23 in 6339a42

  	var (
  	outputFile = flag.String("output", "ftd-processor.yml", "Output file")

• beats/x-pack/metricbeat/module/panw/client.go

  Lines 36 to 40 in 6339a42

  	func GetPanwClient(config *Config) (PanwClient, error) {
  	// If running tests, return a test client
  	if flag.Lookup("test.v") != nil {
  	return &PanwTestClient{}, nil
  	}

• beats/x-pack/filebeat/module/cisco/shared/gen-ftd-ecs-mapping.go

  Lines 22 to 23 in 6339a42

  	var (
  	outputFile = flag.String("output", "ftd-processor.yml", "Output file")

• beats/x-pack/filebeat/input/netflow/fields_gen.go

  Lines 26 to 29 in 6339a42

  	var (
  	outputFile = flag.String("output", "fields.yml", "Output file")
  	header = flag.String("header", "fields.header.yml", "File with header fields to prepend")
  	)

• beats/x-pack/filebeat/input/netflow/decoder/fields/gen.go

  Lines 20 to 27 in 6339a42

  	var (
  	outputFile = flag.String("output", "zfields.go", "Output file")
  	export = flag.String("export", "fields", "Name used to export this fields")
  	nameCol = flag.Int("column-name", 0, "Index of column with field name")
  	penCol = flag.Int("column-pen", 0, "Index of column with PEN ID")
  	idCol = flag.Int("column-id", 0, "Index of column with field ID")
  	typeCol = flag.Int("column-type", 0, "Index of column with field type")
  	)

[elasticmachine]

Pinging @elastic/elastic-agent-data-plane (Team:Elastic-Agent-Data-Plane)