[Elastic Agent] Send Agent logs to elasticsearch

libbeat/logp/configure/logging.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"strings"
 
+	"go.uber.org/zap/zapcore"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
@@ -45,7 +47,7 @@ func init() {
 
 // Logging builds a logp.Config based on the given common.Config and the specified
 // CLI flags.
-func Logging(beatName string, cfg *common.Config) error {
+func Logging(beatName string, cfg *common.Config, wrapSink ...func(zapcore.Core) zapcore.Core) error {
 	config := logp.DefaultConfig(environment)
 	config.Beat = beatName
 	if cfg != nil {
@@ -55,7 +57,7 @@ func Logging(beatName string, cfg *common.Config) error {
 	}
 
 	applyFlags(&config)
-	return logp.Configure(config)
+	return logp.Configure(config, wrapSink...)
 }
 
 func applyFlags(cfg *logp.Config) {

libbeat/logp/core.go

@@ -61,7 +61,7 @@ type coreLogger struct {
 }
 
 // Configure configures the logp package.
-func Configure(cfg Config) error {
+func Configure(cfg Config, wrapSink ...func(zapcore.Core) zapcore.Core) error {
 	var (
 		sink         zapcore.Core
 		observedLogs *observer.ObservedLogs
@@ -70,7 +70,7 @@ func Configure(cfg Config) error {
 
 	// Build a single output (stderr has priority if more than one are enabled).
 	if cfg.toObserver {
-		sink, observedLogs = observer.New(cfg.Level.zapLevel())
+		sink, observedLogs = observer.New(cfg.Level.ZapLevel())
 	} else {
 		sink, err = createLogOutput(cfg)
 	}
@@ -105,6 +105,9 @@ func Configure(cfg Config) error {
 		sink = selectiveWrapper(sink, selectors)
 	}
 
+	for _, wrapper := range wrapSink {
+		sink = wrapper(sink)
+	}
 	root := zap.New(sink, makeOptions(cfg)...)
 	storeLogger(&coreLogger{
 		selectors:    selectors,
@@ -191,24 +194,24 @@ func makeOptions(cfg Config) []zap.Option {
 
 func makeStderrOutput(cfg Config) (zapcore.Core, error) {
 	stderr := zapcore.Lock(os.Stderr)
-	return newCore(cfg, buildEncoder(cfg), stderr, cfg.Level.zapLevel()), nil
+	return newCore(cfg, buildEncoder(cfg), stderr, cfg.Level.ZapLevel()), nil
 }
 
 func makeDiscardOutput(cfg Config) (zapcore.Core, error) {
 	discard := zapcore.AddSync(ioutil.Discard)
-	return newCore(cfg, buildEncoder(cfg), discard, cfg.Level.zapLevel()), nil
+	return newCore(cfg, buildEncoder(cfg), discard, cfg.Level.ZapLevel()), nil
 }
 
 func makeSyslogOutput(cfg Config) (zapcore.Core, error) {
-	core, err := newSyslog(buildEncoder(cfg), cfg.Level.zapLevel())
+	core, err := newSyslog(buildEncoder(cfg), cfg.Level.ZapLevel())
 	if err != nil {
 		return nil, err
 	}
 	return wrappedCore(cfg, core), nil
 }
 
 func makeEventLogOutput(cfg Config) (zapcore.Core, error) {
-	core, err := newEventLog(cfg.Beat, buildEncoder(cfg), cfg.Level.zapLevel())
+	core, err := newEventLog(cfg.Beat, buildEncoder(cfg), cfg.Level.ZapLevel())
 	if err != nil {
 		return nil, err
 	}
@@ -234,7 +237,7 @@ func makeFileOutput(cfg Config) (zapcore.Core, error) {
 		return nil, errors.Wrap(err, "failed to create file rotator")
 	}
 
-	return newCore(cfg, buildEncoder(cfg), rotator, cfg.Level.zapLevel()), nil
+	return newCore(cfg, buildEncoder(cfg), rotator, cfg.Level.ZapLevel()), nil
 }
 
 func newCore(cfg Config, enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapcore.LevelEnabler) zapcore.Core {

libbeat/logp/encoding.go

@@ -44,13 +44,13 @@ func buildEncoder(cfg Config) zapcore.Encoder {
 	var encCfg zapcore.EncoderConfig
 	var encCreator encoderCreator
 	if cfg.JSON {
-		encCfg = jsonEncoderConfig()
+		encCfg = JSONEncoderConfig()
 		encCreator = zapcore.NewJSONEncoder
 	} else if cfg.ToSyslog {
-		encCfg = syslogEncoderConfig()
+		encCfg = SyslogEncoderConfig()
 		encCreator = zapcore.NewConsoleEncoder
 	} else {
-		encCfg = consoleEncoderConfig()
+		encCfg = ConsoleEncoderConfig()
 		encCreator = zapcore.NewConsoleEncoder
 	}
 
@@ -60,19 +60,19 @@ func buildEncoder(cfg Config) zapcore.Encoder {
 	return encCreator(encCfg)
 }
 
-func jsonEncoderConfig() zapcore.EncoderConfig {
+func JSONEncoderConfig() zapcore.EncoderConfig {
 	return baseEncodingConfig
 }
 
-func consoleEncoderConfig() zapcore.EncoderConfig {
+func ConsoleEncoderConfig() zapcore.EncoderConfig {
 	c := baseEncodingConfig
 	c.EncodeLevel = zapcore.CapitalLevelEncoder
 	c.EncodeName = bracketedNameEncoder
 	return c
 }
 
-func syslogEncoderConfig() zapcore.EncoderConfig {
-	c := consoleEncoderConfig()
+func SyslogEncoderConfig() zapcore.EncoderConfig {
+	c := ConsoleEncoderConfig()
 	// Time is generally added by syslog.
 	// But when logging with ECS the empty TimeKey will be
 	// ignored and @timestamp is still added to log line

libbeat/logp/level.go

@@ -101,7 +101,7 @@ func (l Level) MarshalJSON() ([]byte, error) {
 	return nil, errors.Errorf("invalid level '%d'", l)
 }
 
-func (l Level) zapLevel() zapcore.Level {
+func (l Level) ZapLevel() zapcore.Level {
 	z, found := zapLevels[l]
 	if found {
 		return z

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -79,3 +79,4 @@
 - Agent now installs/uninstalls Elastic Endpoint {pull}19248[19248]
 - Agent now downloads Elastic Endpoint {pull}19503[19503]
 - Agent now load balances across multiple Kibana instances {pull}19628[19628]
+- Agent now sends its own logs to elasticsearch {pull}19811[19811]

x-pack/elastic-agent/pkg/agent/application/paths/paths.go

@@ -13,6 +13,7 @@ import (
 var (
 	homePath string
 	dataPath string
+	logsPath string
 )
 
 func init() {
@@ -21,6 +22,7 @@ func init() {
 	fs := flag.CommandLine
 	fs.StringVar(&homePath, "path.home", exePath, "Agent root path")
 	fs.StringVar(&dataPath, "path.data", filepath.Join(exePath, "data"), "Data path contains Agent managed binaries")
+	fs.StringVar(&logsPath, "path.logs", exePath, "Logs path contains Agent log output")
 }
 
 // Home returns a directory where binary lives
@@ -29,13 +31,17 @@ func Home() string {
 	return homePath
 }
 
-// Data returns a home directory of current user
+// Data returns the data directory for Agent
 func Data() string {
 	return dataPath
 }
 
-func retrieveExecutablePath() string {
+// Logs returns a the log directory for Agent
+func Logs() string {
+	return logsPath
+}
 
+func retrieveExecutablePath() string {
 	execPath, err := os.Executable()
 	if err != nil {
 		panic(err)

x-pack/elastic-agent/pkg/agent/cmd/common.go

@@ -52,6 +52,7 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.home"))
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data"))
+	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs"))
 
 	cmd.PersistentFlags().StringVarP(&flags.PathConfigFile, "", "c", defaultConfig, fmt.Sprintf(`Configuration file, relative to path.config (default "%s")`, defaultConfig))
 	cmd.PersistentFlags().StringVarP(&flags.PathConfig, "path.config", "", "${path.home}", "Configuration path")