diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 221823cc3fe..7ca7ebe502d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Added logging of system info at Beat startup. {issue}5946[5946] - Do not log errors if X-Pack Monitoring is enabled but Elastisearch X-Pack is not. {pull}6627[6627] - Add rename processor. {pull}6292[6292] +- Add IP-addresses and MAC-addresses to add_host_metadata. {pull}6878[6878] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index cbce75f67af..c7deedb152a 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -288,7 +288,13 @@ auditbeat.modules: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index b68b0318285..29f229e38f2 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -3258,6 +3258,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-kubernetes-processor]] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 323759c6c05..ca69a19e7af 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -851,6 +851,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-icinga]] diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index bbca33f8e09..37058c34980 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -806,7 +806,13 @@ filebeat.inputs: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 679dc605129..c353af7c42e 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -421,6 +421,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-http]] diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 1acb33bbda8..4eae7eb17f8 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -397,7 +397,13 @@ heartbeat.scheduler: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 986aab97f6c..ee10b06e185 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -183,7 +183,13 @@ # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 49d72f5051c..71a0874709a 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -727,6 +727,17 @@ forget metadata for a container, 60s by default. beta[] +[source,yaml] +------------------------------------------------------------------------------- +processors: +- add_host_metadata: + netinfo.enabled: false +------------------------------------------------------------------------------- + +It has the following settings: + +`netinfo.enabled`:: (Optional) Default false. Include IP adresses and MAC addresses as fields host.ip and host.mac + The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. The fields added to the event are looking as following: @@ -742,7 +753,9 @@ The fields added to the event are looking as following: "build":"16G1212", "platform":"darwin", "version":"10.12.6" - } + }, + ip: ["192.168.0.1", "10.0.0.1"], + mac: ["00:25:96:12:34:56", "72:00:06:ff:79:f1"] } } ------------------------------------------------------------------------------- diff --git a/libbeat/processors/add_host_metadata/_meta/fields.yml b/libbeat/processors/add_host_metadata/_meta/fields.yml index 9897d770762..b579df3b272 100644 --- a/libbeat/processors/add_host_metadata/_meta/fields.yml +++ b/libbeat/processors/add_host_metadata/_meta/fields.yml @@ -31,3 +31,12 @@ type: keyword description: > OS family (e.g. redhat, debian, freebsd, windows). + - name: ip + type: ip + description: > + List of IP-addresses. + - name: mac + type: keyword + description: > + List of hardware-addresses, usually MAC-addresses. + diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 65d8dbb6544..f27557e7d13 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -1,10 +1,16 @@ package add_host_metadata import ( + "fmt" + "net" "time" + "github.com/joeshaw/multierror" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/go-sysinfo" "github.com/elastic/go-sysinfo/types" @@ -18,19 +24,27 @@ type addHostMetadata struct { info types.HostInfo lastUpdate time.Time data common.MapStr + config Config } const ( + processorName = "add_host_metadata" cacheExpiration = time.Minute * 5 ) -func newHostMetadataProcessor(_ *common.Config) (processors.Processor, error) { +func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName) + } + h, err := sysinfo.Host() if err != nil { return nil, err } p := &addHostMetadata{ - info: h.Info(), + info: h.Info(), + config: config, } return p, nil } @@ -71,10 +85,72 @@ func (p *addHostMetadata) loadData() { if p.info.OS.Build != "" { p.data.Put("host.os.build", p.info.OS.Build) } + + if p.config.NetInfoEnabled { + // IP-address and MAC-address + var ipList, hwList, err = p.getNetInfo() + if err != nil { + logp.Info("Error when getting network information %v", err) + } + + if len(ipList) > 0 { + p.data.Put("host.ip", ipList) + } + if len(hwList) > 0 { + p.data.Put("host.mac", hwList) + } + } + p.lastUpdate = time.Now() } } +func (p addHostMetadata) getNetInfo() ([]string, []string, error) { + var ipList []string + var hwList []string + + // Get all interfaces and loop through them + ifaces, err := net.Interfaces() + if err != nil { + return nil, nil, err + } + + // Keep track of all errors + var errs multierror.Errors + + for _, i := range ifaces { + // Skip loopback interfaces + if i.Flags&net.FlagLoopback == net.FlagLoopback { + continue + } + + hw := i.HardwareAddr.String() + // Skip empty hardware addresses + if hw != "" { + hwList = append(hwList, hw) + } + + addrs, err := i.Addrs() + if err != nil { + // If we get an error, keep track of it and continue with the next interface + errs = append(errs, err) + continue + } + + for _, addr := range addrs { + switch v := addr.(type) { + case *net.IPNet: + ipList = append(ipList, v.IP.String()) + case *net.IPAddr: + ipList = append(ipList, v.IP.String()) + } + } + } + + return ipList, hwList, errs.Err() +} + func (p addHostMetadata) String() string { - return "add_host_metadata=[]" + return fmt.Sprintf("%v=[netinfo.enabled=[%v]]", + processorName, p.config.NetInfoEnabled) } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 0192f57b325..d8e42594c82 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -13,12 +13,48 @@ import ( "github.com/elastic/go-sysinfo/types" ) -func TestRun(t *testing.T) { +func TestConfigDefault(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{}, Timestamp: time.Now(), } - p, err := newHostMetadataProcessor(nil) + testConfig, err := common.NewConfigFrom(map[string]interface{}{}) + assert.NoError(t, err) + + p, err := newHostMetadataProcessor(testConfig) + if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + assert.IsType(t, types.ErrNotImplemented, err) + return + } + assert.NoError(t, err) + + newEvent, err := p.Run(event) + assert.NoError(t, err) + + v, err := newEvent.GetValue("host.os.family") + assert.NoError(t, err) + assert.NotNil(t, v) + + v, err = newEvent.GetValue("host.ip") + assert.Error(t, err) + assert.Nil(t, v) + + v, err = newEvent.GetValue("host.mac") + assert.Error(t, err) + assert.Nil(t, v) +} + +func TestConfigNetInfoEnabled(t *testing.T) { + event := &beat.Event{ + Fields: common.MapStr{}, + Timestamp: time.Now(), + } + testConfig, err := common.NewConfigFrom(map[string]interface{}{ + "netinfo.enabled": true, + }) + assert.NoError(t, err) + + p, err := newHostMetadataProcessor(testConfig) if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" { assert.IsType(t, types.ErrNotImplemented, err) return @@ -31,4 +67,12 @@ func TestRun(t *testing.T) { v, err := newEvent.GetValue("host.os.family") assert.NoError(t, err) assert.NotNil(t, v) + + v, err = newEvent.GetValue("host.ip") + assert.NoError(t, err) + assert.NotNil(t, v) + + v, err = newEvent.GetValue("host.mac") + assert.NoError(t, err) + assert.NotNil(t, v) } diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go new file mode 100644 index 00000000000..5ead35b4c0b --- /dev/null +++ b/libbeat/processors/add_host_metadata/config.go @@ -0,0 +1,12 @@ +package add_host_metadata + +// Config for add_host_metadata processor. +type Config struct { + NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event +} + +func defaultConfig() Config { + return Config{ + NetInfoEnabled: false, + } +} diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index f12ca0cf74c..d9e61cf0420 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5653,6 +5653,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-http]] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 049ce10527e..9f9a865af1d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -763,7 +763,13 @@ metricbeat.modules: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index f3d6ae692c6..9c405c0906d 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -2316,6 +2316,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-http]] diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 88c5013f614..f8773f3f62a 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -660,7 +660,13 @@ packetbeat.protocols: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index 89fc00914d3..48b30ad9c3a 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -634,6 +634,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-kubernetes-processor]] diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index cbd999a7a86..c2e7fed1b8d 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -212,7 +212,13 @@ winlogbeat.event_logs: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# #============================= Elastic Cloud ==================================