Add cache.ttl to add_host_metadata (#9359)

Fix: Refresh host info when cache expires.

Add config parameter `cache.ttl` to control cache expiration (default 5m).

CHANGELOG.asciidoc

@@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 
 - Fix autodiscover configurations stopping when metadata is missing. {pull}8851[8851]
 - Log events at the debug level when dropped by encoding problems. {pull}9251[9251]
+- Refresh host metadata in add_host_metadata. {pull}9359[9359]
 
 *Auditbeat*
 
@@ -70,6 +71,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 
 - Unify dashboard exporter tools. {pull}9097[9097]
 - Use _doc as document type of the Elasticsearch major version is 7. {pull}9056[9056]
+- Add cache.ttl to add_host_metadata. {pull}9359[9359]
 
 *Auditbeat*
 

libbeat/docs/processors-using.asciidoc

@@ -894,12 +894,15 @@ beta[]
 processors:
 - add_host_metadata:
     netinfo.enabled: false
+    cache.ttl: 5m
 -------------------------------------------------------------------------------
 
 It has the following settings:
 
 `netinfo.enabled`:: (Optional) Default false. Include IP addresses and MAC addresses as fields host.ip and host.mac
 
+`cache.ttl`:: (Optional) The processor uses an internal cache for the host metadata. This sets the cache expiration time. The default is 5m, negative values disable caching altogether.
+
 The `add_host_metadata` processor annotates each event with relevant metadata from the host machine.
 The fields added to the event are looking as following:
 
@@ -923,8 +926,6 @@ The fields added to the event are looking as following:
 }
 -------------------------------------------------------------------------------
 
-NOTE: The host information is refreshed every 5 minutes.
-
 [[dissect]]
 === Dissect strings
 

libbeat/processors/add_host_metadata/add_host_metadata.go

@@ -32,15 +32,13 @@ import (
 	"github.com/elastic/beats/libbeat/metric/system/host"
 	"github.com/elastic/beats/libbeat/processors"
 	"github.com/elastic/go-sysinfo"
-	"github.com/elastic/go-sysinfo/types"
 )
 
 func init() {
 	processors.RegisterPlugin("add_host_metadata", newHostMetadataProcessor)
 }
 
 type addHostMetadata struct {
-	info       types.HostInfo
 	lastUpdate struct {
 		time.Time
 		sync.Mutex
@@ -50,8 +48,7 @@ type addHostMetadata struct {
 }
 
 const (
-	processorName   = "add_host_metadata"
-	cacheExpiration = time.Minute * 5
+	processorName = "add_host_metadata"
 )
 
 func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) {
@@ -60,12 +57,7 @@ func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error)
 		return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName)
 	}
 
-	h, err := sysinfo.Host()
-	if err != nil {
-		return nil, err
-	}
 	p := &addHostMetadata{
-		info:   h.Info(),
 		config: config,
 		data:   common.NewMapStrPointer(nil),
 	}
@@ -75,28 +67,41 @@ func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error)
 
 // Run enriches the given event with the host meta data
 func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) {
-	p.loadData()
+	err := p.loadData()
+	if err != nil {
+		return nil, err
+	}
+
 	event.Fields.DeepUpdate(p.data.Get().Clone())
 	return event, nil
 }
 
 func (p *addHostMetadata) expired() bool {
+	if p.config.CacheTTL <= 0 {
+		return true
+	}
+
 	p.lastUpdate.Lock()
 	defer p.lastUpdate.Unlock()
 
-	if p.lastUpdate.Add(cacheExpiration).After(time.Now()) {
+	if p.lastUpdate.Add(p.config.CacheTTL).After(time.Now()) {
 		return false
 	}
 	p.lastUpdate.Time = time.Now()
 	return true
 }
 
-func (p *addHostMetadata) loadData() {
+func (p *addHostMetadata) loadData() error {
 	if !p.expired() {
-		return
+		return nil
+	}
+
+	h, err := sysinfo.Host()
+	if err != nil {
+		return err
 	}
 
-	data := host.MapHostInfo(p.info)
+	data := host.MapHostInfo(h.Info())
 	if p.config.NetInfoEnabled {
 		// IP-address and MAC-address
 		var ipList, hwList, err = p.getNetInfo()
@@ -113,6 +118,7 @@ func (p *addHostMetadata) loadData() {
 	}
 
 	p.data.Set(data)
+	return nil
 }
 
 func (p *addHostMetadata) getNetInfo() ([]string, []string, error) {
@@ -161,6 +167,6 @@ func (p *addHostMetadata) getNetInfo() ([]string, []string, error) {
 }
 
 func (p *addHostMetadata) String() string {
-	return fmt.Sprintf("%v=[netinfo.enabled=[%v]]",
-		processorName, p.config.NetInfoEnabled)
+	return fmt.Sprintf("%v=[netinfo.enabled=[%v], cache.ttl=[%v]]",
+		processorName, p.config.NetInfoEnabled, p.config.CacheTTL)
 }

libbeat/processors/add_host_metadata/config.go

@@ -17,13 +17,19 @@
 
 package add_host_metadata
 
+import (
+	"time"
+)
+
 // Config for add_host_metadata processor.
 type Config struct {
-	NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event
+	NetInfoEnabled bool          `config:"netinfo.enabled"` // Add IP and MAC to event
+	CacheTTL       time.Duration `config:"cache.ttl"`
 }
 
 func defaultConfig() Config {
 	return Config{
 		NetInfoEnabled: false,
+		CacheTTL:       5 * time.Minute,
 	}
 }