From 223b64fac6bd0dc87d723c5d1e56da61604622b9 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 11:08:36 +0200 Subject: [PATCH 1/9] send ecs metadata --- .../application/filters/constraints_filter.go | 45 ++--------- .../agent/application/info/agent_metadata.go | 74 +++++++++++++++++++ .../pkg/agent/application/local_meta.go | 19 +++-- 3 files changed, 90 insertions(+), 48 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go index 9241f3dd3e6f..2cb92cffd975 100644 --- a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go +++ b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go @@ -6,7 +6,6 @@ package filters import ( "fmt" - "runtime" "github.com/Masterminds/semver" @@ -15,8 +14,6 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/boolexp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" - "github.com/elastic/go-sysinfo" ) const ( @@ -25,24 +22,6 @@ const ( validateVersionFuncName = "validate_version" ) -// List of variables available to be used in constraint definitions. -const ( - // `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier. - agentIDKey = "agent.id" - // `agent.version` specifies current version of an agent. - agentVersionKey = "agent.version" - // `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips). - hostArchKey = "host.architecture" - // `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows). - osFamilyKey = "os.family" - // `os.kernel` specifies current version of a kernel in a semver format. - osKernelKey = "os.kernel" - // `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows). - osPlatformKey = "os.platform" - // `os.version` specifies version of underlying operating system (e.g. 10.12.6). - osVersionKey = "os.version" -) - var ( boolexpVarStore *constraintVarStore boolexpMethodsRegs *boolexp.MethodsReg @@ -245,30 +224,20 @@ func newVarStore() (*constraintVarStore, error) { } func initVarStore(store *constraintVarStore) error { - sysInfo, err := sysinfo.Host() + agentInfo, err := info.NewAgentInfo() if err != nil { return err } - agentInfo, err := info.NewAgentInfo() + meta, err := agentInfo.ECSMetadata() if err != nil { - return err + return errors.New(err, "failed to gather host metadata") } - info := sysInfo.Info() - - // Agent - store.vars[agentIDKey] = agentInfo.AgentID() - store.vars[agentVersionKey] = release.Version() - - // Host - store.vars[hostArchKey] = info.Architecture - - // Operating system - store.vars[osFamilyKey] = runtime.GOOS - store.vars[osKernelKey] = info.KernelVersion - store.vars[osPlatformKey] = info.OS.Family - store.vars[osVersionKey] = info.OS.Version + // keep existing, overwrite gathered + for k, v := range meta { + store.vars[k] = v + } return nil } diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go new file mode 100644 index 000000000000..4f4514378b95 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package info + +import ( + "os" + "runtime" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/go-sysinfo" +) + +// List of variables available to be used in constraint definitions. +const ( + // `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier. + agentIDKey = "agent.id" + // `agent.version` specifies current version of an agent. + agentVersionKey = "agent.version" + // `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips). + hostArchKey = "host.architecture" + // `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows). + osFamilyKey = "os.family" + // `os.kernel` specifies current version of a kernel in a semver format. + osKernelKey = "os.kernel" + // `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows). + osPlatformKey = "os.platform" + // `os.version` specifies version of underlying operating system (e.g. 10.12.6). + osVersionKey = "os.version" + // `host.hostname` specifies hostname of the host. + hostHostnameKey = "host.hostname" + // `host.name` specifies hostname of the host. + hostNameKey = "host.name" +) + +// AgentID returns an agent identifier. +func (i *AgentInfo) ECSMetadata() (map[string]interface{}, error) { + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + + // TODO: remove these values when kibana migrates to ECS + meta := map[string]interface{}{ + "platform": runtime.GOOS, + "version": release.Version(), + "host": hostname, + } + + sysInfo, err := sysinfo.Host() + if err != nil { + return nil, err + } + + info := sysInfo.Info() + + // Agent + meta[agentIDKey] = i.agentID + meta[agentVersionKey] = release.Version() + + // Host + meta[hostArchKey] = info.Architecture + meta[hostHostnameKey] = hostname + meta[hostNameKey] = hostname + + // Operating system + meta[osFamilyKey] = runtime.GOOS + meta[osKernelKey] = info.KernelVersion + meta[osPlatformKey] = info.OS.Family + meta[osVersionKey] = info.OS.Version + + return meta, nil +} diff --git a/x-pack/elastic-agent/pkg/agent/application/local_meta.go b/x-pack/elastic-agent/pkg/agent/application/local_meta.go index 47e358b62627..3456075baa9c 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_meta.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_meta.go @@ -5,21 +5,20 @@ package application import ( - "os" - "runtime" - - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" ) func metadata() (map[string]interface{}, error) { - hostname, err := os.Hostname() + agentInfo, err := info.NewAgentInfo() if err != nil { return nil, err } - return map[string]interface{}{ - "platform": runtime.GOOS, - "version": release.Version(), - "host": hostname, - }, nil + meta, err := agentInfo.ECSMetadata() + if err != nil { + return nil, errors.New(err, "failed to gather host metadata") + } + + return meta, nil } From ea4acc28b14fea1a07f931195c89bf570a7fa5da Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 11:31:48 +0200 Subject: [PATCH 2/9] nil ref fix --- .../pkg/core/plugin/app/monitoring/beats/beats_monitor.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go index 14d191c54779..f1fb92d3a716 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/beats_monitor.go @@ -46,7 +46,12 @@ func (b *Monitor) Reload(rawConfig *config.Config) error { return err } - b.config = cfg.MonitoringConfig + if cfg == nil || cfg.MonitoringConfig == nil { + b.config = &monitoringConfig.MonitoringConfig{} + } else { + b.config = cfg.MonitoringConfig + } + return nil } From 0cb9ed7168ea49d5126021d8b03ca66f08a3f3ab Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 11:38:11 +0200 Subject: [PATCH 3/9] changelog --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index edb9a65b1412..b81acebc9e89 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -36,3 +36,4 @@ - Display the stability of the agent at enroll and start. {pull}17336[17336] - Expose stream.* variables in events {pull}17468[17468] - Monitoring configuration reloadable {pull}17855[17855] +- Pack ECS metadata to request payload send to fleet {pull}17894[17894] From 719e121487c07ca49c2799b92da0e3e68683e834 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 12:06:18 +0200 Subject: [PATCH 4/9] lint --- .../elastic-agent/pkg/agent/application/info/agent_metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go index 4f4514378b95..09c74fa7ed57 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -34,7 +34,7 @@ const ( hostNameKey = "host.name" ) -// AgentID returns an agent identifier. +// ECSMetadata returns an agent ECS compliant metadata. func (i *AgentInfo) ECSMetadata() (map[string]interface{}, error) { hostname, err := os.Hostname() if err != nil { From 13ea056eb89fda22052c5ab3ca1f0bf132782a22 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 13:32:23 +0200 Subject: [PATCH 5/9] pack metadata needed by endpoint --- .../agent/application/info/agent_metadata.go | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go index 09c74fa7ed57..c4644595c67b 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -5,11 +5,14 @@ package info import ( + "fmt" "os" "runtime" + "strings" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" ) // List of variables available to be used in constraint definitions. @@ -18,8 +21,7 @@ const ( agentIDKey = "agent.id" // `agent.version` specifies current version of an agent. agentVersionKey = "agent.version" - // `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips). - hostArchKey = "host.architecture" + // `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows). osFamilyKey = "os.family" // `os.kernel` specifies current version of a kernel in a semver format. @@ -28,10 +30,27 @@ const ( osPlatformKey = "os.platform" // `os.version` specifies version of underlying operating system (e.g. 10.12.6). osVersionKey = "os.version" + // `os.name` is a operating system name. + // Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-os.html + osNameKey = "os.name" + // `os.full` is an operating system name, including the version or code name. + osFullKey = "os.full" + + // `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips). + hostArchKey = "host.architecture" // `host.hostname` specifies hostname of the host. hostHostnameKey = "host.hostname" // `host.name` specifies hostname of the host. hostNameKey = "host.name" + // `host.id` is a Unique host id. + // As hostname is not always unique, use values that are meaningful in your environment. + hostIDKey = "host.id" + // `host.ip` is Host ip addresses. + // Note: this field should contain an array of values. + hostIPKey = "host.ip" + // `host.mac` is Host mac addresses. + // Note: this field should contain an array of values. + hostMACKey = "host.mac" ) // ECSMetadata returns an agent ECS compliant metadata. @@ -63,12 +82,34 @@ func (i *AgentInfo) ECSMetadata() (map[string]interface{}, error) { meta[hostArchKey] = info.Architecture meta[hostHostnameKey] = hostname meta[hostNameKey] = hostname + meta[hostIDKey] = info.UniqueID + meta[hostIPKey] = fmt.Sprintf("[%s]", strings.Join(info.IPs, ",")) + meta[hostMACKey] = fmt.Sprintf("[%s]", strings.Join(info.MACs, ",")) // Operating system meta[osFamilyKey] = runtime.GOOS meta[osKernelKey] = info.KernelVersion meta[osPlatformKey] = info.OS.Family meta[osVersionKey] = info.OS.Version + meta[osNameKey] = info.OS.Name + meta[osFullKey] = getFullOSName(info) return meta, nil } + +func getFullOSName(info types.HostInfo) string { + var sb strings.Builder + sb.WriteString(info.OS.Name) + if codeName := info.OS.Codename; codeName != "" { + sb.WriteString(" ") + sb.WriteString(codeName) + } + + if version := info.OS.Version; version != "" { + sb.WriteString("(") + sb.WriteString(version) + sb.WriteString(")") + } + + return sb.String() +} From bf847141ad8074fb4bea9b0e6cae6ec53cf4cf31 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 14:32:25 +0200 Subject: [PATCH 6/9] comments --- x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go | 2 +- .../pkg/agent/application/info/agent_metadata.go | 2 +- x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go index b1743e924179..b203e0c11eb0 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go @@ -141,7 +141,7 @@ func (c *EnrollCmd) Execute() error { metadata, err := metadata() if err != nil { - return errors.New(err, "acquiring hostname") + c.log.Error(errors.New(err, "acquiring hostname")) } r := &fleetapi.EnrollRequest{ diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go index c4644595c67b..79371e766008 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -74,7 +74,7 @@ func (i *AgentInfo) ECSMetadata() (map[string]interface{}, error) { info := sysInfo.Info() - // Agent + // Agent meta[agentIDKey] = i.agentID meta[agentVersionKey] = release.Version() diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go index c7410baa0e7b..17426246490f 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go @@ -79,13 +79,13 @@ type EnrollRequest struct { EnrollAPIKey string `json:"-"` Type EnrollType `json:"type"` SharedID string `json:"sharedId,omitempty"` - Metadata Metadata `json:"metadata"` + Metadata Metadata `json:"metadata,omitempty"` } // Metadata is a all the metadata send or received from the elastic-agent. type Metadata struct { - Local map[string]interface{} `json:"local"` - UserProvided map[string]interface{} `json:"user_provided"` + Local map[string]interface{} `json:"local,omitempty"` + UserProvided map[string]interface{} `json:"user_provided,omitempty"` } // Validate validates the enrollment request before sending it to the API. From bb531f1b82b4bb8fb0c17cc2bbd3b47cdfe67f85 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 14:33:06 +0200 Subject: [PATCH 7/9] comments --- x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go index 97c1964f8ac5..e4f021fdaad1 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go @@ -182,6 +182,8 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse, var metaData map[string]interface{} if m, err := metadata(); err == nil { metaData = m + } else { + f.log.Error(errors.New("failed to load metadata", err)) } // checkin From 1aa4885dee994b6c661dcdab5f81e3421c97e5bb Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 14:53:16 +0200 Subject: [PATCH 8/9] omitempty? --- x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go index 17426246490f..3192a12ec18c 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go @@ -79,13 +79,13 @@ type EnrollRequest struct { EnrollAPIKey string `json:"-"` Type EnrollType `json:"type"` SharedID string `json:"sharedId,omitempty"` - Metadata Metadata `json:"metadata,omitempty"` + Metadata Metadata `json:"metadata"` } // Metadata is a all the metadata send or received from the elastic-agent. type Metadata struct { Local map[string]interface{} `json:"local,omitempty"` - UserProvided map[string]interface{} `json:"user_provided,omitempty"` + UserProvided map[string]interface{} `json:"user_provided"` } // Validate validates the enrollment request before sending it to the API. From ba0c6b5dded56559634193ff62184ad5dd75d0a2 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 22 Apr 2020 15:03:46 +0200 Subject: [PATCH 9/9] fail if metadata missing on enroll --- x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go | 2 +- x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go index b203e0c11eb0..323937b080c8 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go @@ -141,7 +141,7 @@ func (c *EnrollCmd) Execute() error { metadata, err := metadata() if err != nil { - c.log.Error(errors.New(err, "acquiring hostname")) + return errors.New(err, "acquiring metadata failed") } r := &fleetapi.EnrollRequest{ diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go index 3192a12ec18c..c7410baa0e7b 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go @@ -84,7 +84,7 @@ type EnrollRequest struct { // Metadata is a all the metadata send or received from the elastic-agent. type Metadata struct { - Local map[string]interface{} `json:"local,omitempty"` + Local map[string]interface{} `json:"local"` UserProvided map[string]interface{} `json:"user_provided"` }