Convert source field in Filebeat (elastic#8902)

The source field in Filebeat was used to store the file path for logs or the source ip for syslog, udp, tcp input. As source is in ECS an object the fields are now moved to ECS pattern. * For UDP, TCP, syslog input the source field is converted to log.source.ip * For the log input the source field is converted to log.file.path Done: * Test files updated * Changelog updated * Migration file updated * `source` removed from fields.yml, two new fields added
andrewvc · Nov 7, 2018 · 0e42349 · 0e42349
1 parent 4d718fa
commit 0e42349
Show file tree

Hide file tree

Showing 15 changed files with 66 additions and 20 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -21,11 +21,12 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Rename beat.name to agent.type, beat.hostname to agent.hostname, beat.version to agent.version.
 
 *Filebeat*
-- Rename `fileset.name` to `event.name`.
-- Rename `fileset.module` to `event.module`.
 
+- Rename `fileset.name` to `event.name`. {pull}8879[8879]
+- Rename `fileset.module` to `event.module`. {pull}8879[8879]
+- Rename source to log.file.path and log.source.ip {pull}8902[8902]
 - Remove the deprecated `prospector(s)` option in the configuration use `input(s)` instead. {pull}8909[8909]
-- Rename `offset` to `log.offset`.
+- Rename `offset` to `log.offset`. {pull}8923[8923]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -24,6 +24,10 @@
 - from: fileset.module
   to: event.module
   alias: true
+
+- from: source
+  to: ["log.file.path", "log.source.ip"]
+  alias: false
   copy_to: false
 
 - from: beat.name

diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml
@@ -3,13 +3,20 @@
   description: >
     Contains log file lines.
   fields:
-    - name: source
+
+    - name: log.file.path
       type: keyword
-      required: true
+      required: false
       description: >
         The file from which the line was read. This field contains the absolute path to the file.
         For example: `/var/log/system.log`.
 
+    - name: log.source.ip
+      type: ip
+      required: false
+      description: >
+        Source IP from which the log event was read / sent from.
+
     - name: log.offset
       type: long
       required: false

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4609,16 +4609,28 @@ Contains log file lines.
 
 
 
-*`source`*::
+*`log.file.path`*::
 +
 --
 type: keyword
 
-required: True
+required: False
 
 The file from which the line was read. This field contains the absolute path to the file. For example: `/var/log/system.log`.
 
 
+--
+
+*`log.source.ip`*::
++
+--
+type: ip
+
+required: False
+
+Source IP from which the log event was read / sent from.
+
+
 --
 
 *`log.offset`*::

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/input/log/harvester.go b/filebeat/input/log/harvester.go
@@ -306,9 +306,11 @@ func (h *Harvester) Run() error {
 		// Check if data should be added to event. Only export non empty events.
 		if !message.IsEmpty() && h.shouldExportLine(text) {
 			fields := common.MapStr{
-				"source": state.Source,
 				"log": common.MapStr{
 					"offset": startingOffset, // Offset here is the offset before the starting char.
+					"file": common.MapStr{
+						"path": state.Source,
+					},
 				},
 			}
 			fields.DeepUpdate(message.Fields)

diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go
@@ -203,7 +203,11 @@ func (p *Input) Wait() {
 func createEvent(ev *event, metadata inputsource.NetworkMetadata, timezone *time.Location, log *logp.Logger) *beat.Event {
 	f := common.MapStr{
 		"message": strings.TrimRight(ev.Message(), "\n"),
-		"source":  metadata.RemoteAddr.String(),
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"ip": metadata.RemoteAddr.String(),
+			},
+		},
 	}
 
 	syslog := common.MapStr{}

diff --git a/filebeat/input/syslog/input_test.go b/filebeat/input/syslog/input_test.go
@@ -40,7 +40,11 @@ func TestWhenPriorityIsSet(t *testing.T) {
 	event := createEvent(e, m, time.Local, logp.NewLogger("syslog"))
 
 	expected := common.MapStr{
-		"source":   "127.0.0.1",
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"ip": "127.0.0.1",
+			},
+		},
 		"message":  "hello world",
 		"hostname": "wopr",
 		"process": common.MapStr{
@@ -69,7 +73,11 @@ func TestWhenPriorityIsNotSet(t *testing.T) {
 	m := dummyMetadata()
 	event := createEvent(e, m, time.Local, logp.NewLogger("syslog"))
 	expected := common.MapStr{
-		"source":   "127.0.0.1",
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"ip": "127.0.0.1",
+			},
+		},
 		"message":  "hello world",
 		"hostname": "wopr",
 		"process": common.MapStr{

diff --git a/filebeat/input/tcp/input.go b/filebeat/input/tcp/input.go
@@ -131,7 +131,11 @@ func createEvent(raw []byte, metadata inputsource.NetworkMetadata) *util.Data {
 		Timestamp: time.Now(),
 		Fields: common.MapStr{
 			"message": string(raw),
-			"source":  metadata.RemoteAddr.String(),
+			"log": common.MapStr{
+				"source": common.MapStr{
+					"ip": metadata.RemoteAddr.String(),
+				},
+			},
 		},
 	}
 	return data

diff --git a/filebeat/input/tcp/input_test.go b/filebeat/input/tcp/input_test.go
@@ -42,6 +42,6 @@ func TestCreateEvent(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, string(message), m)
 
-	from, _ := event.GetValue("source")
+	from, _ := event.GetValue("log.source.ip")
 	assert.Equal(t, ip, from)
 }
diff --git a/filebeat/input/udp/input.go b/filebeat/input/udp/input.go
@@ -74,7 +74,11 @@ func NewInput(
 			},
 			Fields: common.MapStr{
 				"message": string(data),
-				"source":  metadata.RemoteAddr.String(),
+				"log": common.MapStr{
+					"source": common.MapStr{
+						"ip": metadata.RemoteAddr.String(),
+					},
+				},
 			},
 		}
 		forwarder.Send(e)

diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-11-06T15:26:20.085Z", 
+        "@timestamp": "2018-11-06T18:29:13.579Z", 
         "event.dataset": "startup", 
         "event.module": "icinga", 
         "icinga.startup.facility": "cli", 
@@ -10,7 +10,7 @@
         "log.offset": 0
     }, 
     {
-        "@timestamp": "2018-11-06T15:26:20.085Z", 
+        "@timestamp": "2018-11-06T18:29:13.579Z", 
         "event.dataset": "startup", 
         "event.module": "icinga", 
         "icinga.startup.facility": "cli", 

diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -194,7 +194,7 @@ def clean_keys(obj):
     # The create timestamps area always new
     time_keys = ["read_timestamp", "event.created"]
     # source path and beat.version can be different for each run
-    other_keys = ["source", "agent.version"]
+    other_keys = ["log.file.path", "agent.version"]
 
     for key in host_keys + time_keys + other_keys:
         delete_key(obj, key)

diff --git a/filebeat/tests/system/test_processors.py b/filebeat/tests/system/test_processors.py
@@ -66,7 +66,7 @@ def test_drop_event(self):
             path=os.path.abspath(self.working_dir) + "/test*.log",
             processors=[{
                 "drop_event": {
-                    "when": "contains.source: test1",
+                    "when": "contains.log.file.path: test1",
                 },
             }]
         )

diff --git a/filebeat/tests/system/test_syslog.py b/filebeat/tests/system/test_syslog.py
@@ -100,4 +100,4 @@ def assert_syslog(self, syslog):
         assert syslog["syslog.priority"] == 13
         assert syslog["syslog.severity_label"] == "Notice"
         assert syslog["syslog.facility_label"] == "user-level"
-        assert len(syslog["source"]) > 0
+        assert len(syslog["log.source.ip"]) > 0