Add the ability to set custom fields

CHANGELOG.asciidoc

@@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff]
 
 *Filebeat*
 - Default config for ignore_older is now infinite instead of 24h, means ignore_older is disabled by default. Use close_older to only close file handlers.
+- Scalar values in used in the `fields` configuration setting are no longer automatically converted to strings. {pull}1092[1092]
 
 *Winlogbeat*
 
@@ -72,6 +73,7 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff]
 - Improve logstash and elasticsearch backoff behavior. {pull}927[927]
 - Add experimental Kafka output. {pull}942[942]
 - Add config file option to configure GOMAXPROCS. {pull}969[969]
+- Added a `fields` and `fields_under_root` as options available under the `shipper` configuration {pull}1092[1092]
 
 *Packetbeat*
 - Change the DNS library used throughout the dns package to github.com/miekg/dns. {pull}803[803]
@@ -84,10 +86,12 @@ https://github.com/elastic/beats/compare/v1.1.0...master[Check the HEAD diff]
 - Add multiline support for combining multiple related lines into one event. {issue}461[461]
 - Add close_older configuration option to complete ignore_older https://github.com/elastic/filebeat/issues/181[181]
 - Add experimental option to enable filebeat publisher pipeline to operate asynchonrously {pull}782[782]
+- Added the ability to set a list of tags for each prospector {pull}1092[1092]
 
 *Winlogbeat*
 - Add caching of event metadata handles and the system render context for the wineventlog API {pull}888[888]
 - Improve config validation by checking for unknown top-level YAML keys. {pull}1100[1100]
+- Added the ability to set tags, fields, and fields_under_root as options for each event log {pull}1092[1092]
 
 ==== Deprecated
 

filebeat/config/config.go

@@ -57,11 +57,11 @@ type ProspectorConfig struct {
 }
 
 type HarvesterConfig struct {
+	common.EventMetadata `config:",inline"` // Fields and tags to add to events.
+
 	BufferSize         int    `config:"harvester_buffer_size"`
 	DocumentType       string `config:"document_type"`
 	Encoding           string `config:"encoding"`
-	Fields             common.MapStr
-	FieldsUnderRoot    bool   `config:"fields_under_root"`
 	InputType          string `config:"input_type"`
 	TailFiles          bool   `config:"tail_files"`
 	Backoff            string `config:"backoff"`

filebeat/docs/reference/configuration/filebeat-options.asciidoc

@@ -92,27 +92,51 @@ The following example configures Filebeat to ignore all the files that have a `g
   exclude_files: [".gz$"]
 -------------------------------------------------------------------------------------
 
+===== tags
+
+A list of tags that the Beat includes in the `tags` field of each published
+event. Tags make it easy to select specific events in Kibana or apply
+conditional filtering in Logstash. These tags will be appended to the list of
+tags specified in the `shipper` configuration.
+
+Example:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+filebeat:
+  prospectors:
+    - paths: ["/var/log/app/*.json"]
+      tags: ["json"]
+--------------------------------------------------------------------------------
+
 [[configuration-fields]]
 ===== fields
 
-Optional fields that you can specify to add additional information to the output. For
-example, you might add fields that you can use for filtering log data. Fields can be
-scalar values, arrays, dictionaries, or any nested combination of these. All scalar values will be interpreted as strings. By default,
-the fields that you specify here will be grouped under a `fields` sub-dictionary in the output document. To store the custom fields as top-level fields, set the `fields_under_root` option to true.
+Optional fields that you can specify to add additional information to the
+output. For example, you might add fields that you can use for filtering log
+data. Fields can be scalar values, arrays, dictionaries, or any nested
+combination of these. By default, the fields that you specify here will be
+grouped under a `fields` sub-dictionary in the output document. To store the
+custom fields as top-level fields, set the `fields_under_root` option to true.
+If a duplicate field is declared in the `shipper` configuration, then its value
+will be overwritten by the value declared here.
 
 [source,yaml]
--------------------------------------------------------------------------------------
-fields:
-    level: debug
-    review: 1
+--------------------------------------------------------------------------------
+filebeat:
+  prospectors:
+    - paths: ["/var/log/app/*.log"]
+      fields:
+        app_id: query_engine_12
+--------------------------------------------------------------------------------
 
--------------------------------------------------------------------------------------
 [[fields-under-root]]
 ===== fields_under_root
 
-If this option is set to true, the custom <<configuration-fields>> are stored as top-level fields
-in the output document instead of being grouped under a `fields` sub-dictionary.
-If the custom field names conflict with other field names added by Filebeat, the custom fields overwrite the other fields.
+If this option is set to true, the custom <<configuration-fields>> are stored as
+top-level fields in the output document instead of being grouped under a
+`fields` sub-dictionary. If the custom field names conflict with other field
+names added by Filebeat, then the custom fields overwrite the other fields.
 
 [[ignore-older]]
 ===== ignore_older

filebeat/filebeat.yml

@@ -360,6 +360,17 @@ shipper:
   # logical properties.
   #tags: ["service-X", "web-tier"]
 
+  # Optional fields that you can specify to add additional information to the
+  # output. Fields can be scalar values, arrays, dictionaries, or any nested
+  # combination of these. 
+  #fields:
+  #  env: staging
+
+  # If this option is set to true, the custom fields are stored as top-level
+  # fields in the output document instead of being grouped under a fields
+  # sub-dictionary. Default is false.
+  #fields_under_root: false
+
   # Uncomment the following if you want to ignore transactions created
   # by the server on which the shipper is installed. This option is useful
   # to remove duplicates if shippers are installed on multiple servers.

filebeat/harvester/log.go

@@ -127,20 +127,18 @@ func (h *Harvester) Harvest() {
 		}
 
 		if h.shouldExportLine(text) {
-			// Sends text to spooler
 			event := &input.FileEvent{
-				ReadTime:     ts,
-				Source:       &h.Path,
-				InputType:    h.Config.InputType,
-				DocumentType: h.Config.DocumentType,
-				Offset:       h.Offset,
-				Bytes:        bytesRead,
-				Text:         &text,
-				Fields:       h.Config.Fields,
-				Fileinfo:     &info,
+				EventMetadata: h.Config.EventMetadata,
+				ReadTime:      ts,
+				Source:        &h.Path,
+				InputType:     h.Config.InputType,
+				DocumentType:  h.Config.DocumentType,
+				Offset:        h.Offset,
+				Bytes:         bytesRead,
+				Text:          &text,
+				Fileinfo:      &info,
 			}
 
-			event.SetFieldsUnderRoot(h.Config.FieldsUnderRoot)
 			h.SpoolerChan <- event // ship the new event downstream
 		}
 

filebeat/input/file.go

@@ -17,17 +17,15 @@ type File struct {
 
 // FileEvent is sent to the output and must contain all relevant information
 type FileEvent struct {
+	common.EventMetadata
 	ReadTime     time.Time
 	Source       *string
 	InputType    string
 	DocumentType string
 	Offset       int64
 	Bytes        int
 	Text         *string
-	Fields       common.MapStr
 	Fileinfo     *os.FileInfo
-
-	fieldsUnderRoot bool
 }
 
 type FileState struct {
@@ -57,37 +55,16 @@ func (f *FileEvent) GetState() *FileState {
 	return state
 }
 
-// SetFieldsUnderRoot sets whether the fields should be added
-// top level to the output documentation (fieldsUnderRoot = true) or
-// under a fields dictionary.
-func (f *FileEvent) SetFieldsUnderRoot(fieldsUnderRoot bool) {
-	f.fieldsUnderRoot = fieldsUnderRoot
-}
-
 func (f *FileEvent) ToMapStr() common.MapStr {
 	event := common.MapStr{
-		"@timestamp": common.Time(f.ReadTime),
-		"source":     f.Source,
-		"offset":     f.Offset, // Offset here is the offset before the starting char.
-		"message":    f.Text,
-		"type":       f.DocumentType,
-		"input_type": f.InputType,
-		"count":      1,
-	}
-
-	if f.Fields != nil {
-		if f.fieldsUnderRoot {
-			for key, value := range f.Fields {
-				// in case of conflicts, overwrite
-				_, found := event[key]
-				if found {
-					logp.Warn("Overwriting %s key", key)
-				}
-				event[key] = value
-			}
-		} else {
-			event["fields"] = f.Fields
-		}
+		common.EventMetadataKey: f.EventMetadata,
+		"@timestamp":            common.Time(f.ReadTime),
+		"source":                f.Source,
+		"offset":                f.Offset, // Offset here is the offset before the starting char.
+		"message":               f.Text,
+		"type":                  f.DocumentType,
+		"input_type":            f.InputType,
+		"count":                 1,
 	}
 
 	return event

filebeat/input/file_test.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/elastic/beats/libbeat/common"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -112,23 +111,3 @@ func TestFileEventToMapStr(t *testing.T) {
 	_, found := mapStr["fields"]
 	assert.False(t, found)
 }
-
-func TestFieldsUnderRoot(t *testing.T) {
-	event := FileEvent{
-		Fields: common.MapStr{
-			"hello": "world",
-		},
-	}
-	event.SetFieldsUnderRoot(true)
-	mapStr := event.ToMapStr()
-	_, found := mapStr["fields"]
-	assert.False(t, found)
-	assert.Equal(t, "world", mapStr["hello"])
-
-	event.SetFieldsUnderRoot(false)
-	mapStr = event.ToMapStr()
-	_, found = mapStr["hello"]
-	assert.False(t, found)
-	_, found = mapStr["fields"]
-	assert.True(t, found)
-}

filebeat/tests/system/test_fields.py

@@ -15,7 +15,7 @@ def test_custom_fields(self):
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/test.log",
-            fields={"hello": "world"}
+            fields={"hello": "world", "number": 2}
         )
 
         with open(self.working_dir + "/test.log", "w") as f:
@@ -28,6 +28,7 @@ def test_custom_fields(self):
         output = self.read_output()
         doc = output[0]
         assert doc["fields.hello"] == "world"
+        assert doc["fields.number"] == 2
 
     def test_custom_fields_under_root(self):
         """