[Ingest Manager] New Agent structure

dev-tools/mage/settings.go

@@ -85,6 +85,7 @@ var (
 		"beat_doc_branch":   BeatDocBranch,
 		"beat_version":      BeatQualifiedVersion,
 		"commit":            CommitHash,
+		"commit_short":      CommitHashShort,
 		"date":              BuildDate,
 		"elastic_beats_dir": ElasticBeatsDir,
 		"go_version":        GoVersion,
@@ -238,6 +239,15 @@ func CommitHash() (string, error) {
 	return commitHash, err
 }
 
+// CommitHashShort returns the short length git commit hash.
+func CommitHashShort() (string, error) {
+	shortHash, err := CommitHash()
+	if len(shortHash) > 6 {
+		shortHash = shortHash[:6]
+	}
+	return shortHash, err
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -30,18 +30,19 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint
 
 RUN groupadd --gid 1000 {{ .BeatName }}
 
-RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
+RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/logs && \
     chown -R root:{{ .BeatName }} {{ $beatHome }} && \
     find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
     find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
     chmod 0750 {{ $beatBinary }} && \
+    chmod 0750 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
 {{- if .linux_capabilities }}
     setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
 {{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
     chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
-    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/data/logs
 
 {{- if ne .user "root" }}
 RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }}

x-pack/elastic-agent/pkg/agent/application/reexec/manager.go

@@ -19,7 +19,7 @@ var (
 type ExecManager interface {
 	// ReExec asynchronously re-executes command in the same PID and memory address
 	// as the currently running application.
-	ReExec()
+	ReExec(argOverrides ...string)
 
 	// ShutdownChan returns the shutdown channel the main function should use to
 	// handle shutdown of the current running application.
@@ -56,12 +56,12 @@ func newManager(log *logger.Logger, exec string) *manager {
 	}
 }
 
-func (m *manager) ReExec() {
+func (m *manager) ReExec(argOverrides ...string) {
 	go func() {
 		close(m.trigger)
 		<-m.shutdown
 
-		if err := reexec(m.logger, m.exec); err != nil {
+		if err := reexec(m.logger, m.exec, argOverrides...); err != nil {
 			// panic; because there is no going back, everything is shutdown
 			panic(err)
 		}

x-pack/elastic-agent/pkg/agent/application/reexec/reexec.go

@@ -15,11 +15,12 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
-func reexec(log *logger.Logger, executable string) error {
+func reexec(log *logger.Logger, executable string, argOverrides ...string) error {
 	// force log sync, before re-exec
 	_ = log.Sync()
 
 	args := []string{filepath.Base(executable)}
 	args = append(args, os.Args[1:]...)
+	args = append(args, argOverrides...)
 	return unix.Exec(executable, args, os.Environ())
 }

x-pack/elastic-agent/pkg/agent/application/reexec/reexec_windows.go

@@ -28,12 +28,13 @@ import (
 //   current process just exits.
 //
 // * Sub-process - As a sub-process a new child is spawned and the current process just exits.
-func reexec(log *logger.Logger, executable string) error {
+func reexec(log *logger.Logger, executable string, argOverrides ...string) error {
 	svc, status, err := getService()
 	if err == nil {
 		// running as a service; spawn re-exec windows sub-process
 		log.Infof("Running as Windows service %s; triggering service restart", svc.Name)
 		args := []string{filepath.Base(executable), "reexec_windows", svc.Name, strconv.Itoa(int(status.ProcessId))}
+		args = append(args, argOverrides...)
 		cmd := exec.Cmd{
 			Path:   executable,
 			Args:   args,
@@ -51,6 +52,7 @@ func reexec(log *logger.Logger, executable string) error {
 		log.Infof("Running as Windows process; spawning new child process")
 		args := []string{filepath.Base(executable)}
 		args = append(args, os.Args[1:]...)
+		args = append(args, argOverrides...)
 		cmd := exec.Cmd{
 			Path:   executable,
 			Args:   args,

x-pack/elastic-agent/pkg/agent/cmd/common.go

@@ -6,20 +6,32 @@ package cmd
 
 import (
 	"flag"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/spf13/cobra"
 
 	// import logp flags
 	_ "github.com/elastic/beats/v7/libbeat/logp/configure"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/basecmd"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
-const defaultConfig = "elastic-agent.yml"
+const (
+	defaultConfig = "elastic-agent.yml"
+	hashLen       = 6
+	commitFile    = ".build_hash.txt"
+)
 
 type globalFlags struct {
 	PathConfigFile string
@@ -71,7 +83,75 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 	if reexec != nil {
 		cmd.AddCommand(reexec)
 	}
+	cmd.PersistentPreRunE = preRunCheck(flags)
 	cmd.Run = run.Run
 
 	return cmd
 }
+
+func preRunCheck(flags *globalFlags) func(cmd *cobra.Command, args []string) error {
+	return func(cmd *cobra.Command, args []string) error {
+		commitFilepath := filepath.Join(paths.Home(), commitFile)
+		content, err := ioutil.ReadFile(commitFilepath)
+
+		// file is not present we are at child
+		if os.IsNotExist(err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		pathConfigFile := flags.Config()
+		rawConfig, err := config.LoadYAML(pathConfigFile)
+		if err != nil {
+			return errors.New(err,
+				fmt.Sprintf("could not read configuration file %s", pathConfigFile),
+				errors.TypeFilesystem,
+				errors.M(errors.MetaKeyPath, pathConfigFile))
+		}
+
+		cfg, err := configuration.NewFromConfig(rawConfig)
+		if err != nil {
+			return errors.New(err,
+				fmt.Sprintf("could not parse configuration file %s", pathConfigFile),
+				errors.TypeFilesystem,
+				errors.M(errors.MetaKeyPath, pathConfigFile))
+		}
+
+		logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig)
+		if err != nil {
+			return err
+		}
+
+		reexecPath := filepath.Join(paths.Data(), hashedDirName(content), os.Args[0])
+		rexLogger := logger.Named("reexec")
+		argsOverrides := []string{
+			"--path.data", paths.Data(),
+			"--path.home", filepath.Dir(reexecPath),
+			"--path.config", paths.Config(),
+		}
+		rm := reexec.Manager(rexLogger, reexecPath)
+		rm.ReExec(argsOverrides...)
+
+		// trigger reexec
+		rm.ShutdownComplete()
+
+		// prevent running Run function
+		os.Exit(0)
+
+		return nil
+	}
+}
+
+func hashedDirName(filecontent []byte) string {
+	s := strings.TrimSpace(string(filecontent))
+	if len(s) == 0 {
+		return "elastic-agent"
+	}
+
+	if len(s) > hashLen {
+		s = s[:hashLen]
+	}
+
+	return fmt.Sprintf("elastic-agent-%s", s)
+}

x-pack/elastic-agent/pkg/artifact/config.go

@@ -47,13 +47,14 @@ type Config struct {
 
 // DefaultConfig creates a config with pre-set default values.
 func DefaultConfig() *Config {
+	homePath := paths.Home()
 	dataPath := paths.Data()
 	return &Config{
 		SourceURI:       "https://artifacts.elastic.co/downloads/",
-		TargetDirectory: filepath.Join(dataPath, "downloads"),
+		TargetDirectory: filepath.Join(homePath, "downloads"),
 		Timeout:         30 * time.Second,
 		PgpFile:         filepath.Join(dataPath, "elastic.pgp"),
-		InstallPath:     filepath.Join(dataPath, "install"),
+		InstallPath:     filepath.Join(homePath, "install"),
 	}
 }
 

x-pack/elastic-agent/pkg/core/plugin/process/start.go

@@ -152,6 +152,6 @@ func injectLogLevel(logLevel string, args []string) []string {
 }
 
 func injectDataPath(args []string, pipelineID, id string) []string {
-	dataPath := filepath.Join(paths.Data(), "run", pipelineID, id)
+	dataPath := filepath.Join(paths.Home(), "run", pipelineID, id)
 	return append(args, "-E", "path.data="+dataPath)
 }