Add DeDot in add_docker_metadata processor (#9505)

* Never default to a qualifier when none of them are set. (#9148) Remove default version qualifier and rename the environment variable to set it from `BEAT_VERSION_QUALIFIER` to `VERSION_QUALIFIER` this will align with other parts of the stack. **Tested with filebeat.** ``` ❯ ./filebeat version [08:39:01] filebeat version 7.0.0 (amd64), libbeat 7.0.0 [0a0c267 built 2018-11-19 13:38:15 +0000 UTC] ``` **Without the patch** ``` ❯ ./filebeat version [08:40:07] filebeat version 7.0.0-alpha1 (amd64), libbeat 7.0.0-alpha1 [b007837 built 2018-11-19 13:39:59 +0000 UTC] ``` Fixes: #8384 * Add DeDot in add_docker_metadata processor * Add dedot into config and default to be false * Update changelog and documentation * Add documentation into processors-using.asciidoc * Run mage fmt update under x-pack filebeat * Run mage fmt update on x-pack metricbeat * Run update again * Run make update fmt from top level dir * Remove mistakes from rebase * Remove repeated doc
elastic · Dec 17, 2018 · 58573a9 · 58573a9
1 parent f310b52
commit 58573a9
Show file tree

Hide file tree

Showing 19 changed files with 89 additions and 3 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -238,6 +238,7 @@ https://github.com/elastic/beats/compare/v6.5.0...v7.0.0-alpha1[View commits]
 - Dissect will now flag event on parsing error. {pull}8751[8751]
 - add_cloud_metadata initialization is performed asynchronously to avoid delays on startup. {pull}8845[8845]
 - Added the `redirect_stderr` option that allows panics to be logged to log files. {pull}8430[8430]
+- Add DeDot method in add_docker_metadata processor in libbeat. {issue}9350[9350] {pull}9505[9505]
 
 *Filebeat*
 

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -291,6 +291,7 @@ auditbeat.modules:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/auditbeat/docs/getting-started.asciidoc b/auditbeat/docs/getting-started.asciidoc
@@ -97,6 +97,22 @@ endif::[]
 [[docker]]
 *docker:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {stack-version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+["source","sh",subs="attributes"]
+------------------------------------------------
+curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz
+tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz
+------------------------------------------------
+
+endif::[]
+
 See <<running-on-docker, Running on Docker>> for deploying Docker containers.
 
 [[win]]

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -364,7 +364,7 @@
   to: user_agent.original
   alias: true
 
- # Note: `http` is not officially in ECS yet
+# Note: `http` is not officially in ECS yet
 
 - from: nginx.access.response_code
   to: http.response.status_code

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -973,6 +973,7 @@ filebeat.inputs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -424,6 +424,7 @@ heartbeat.scheduler:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -225,6 +225,7 @@ journalbeat.inputs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml
@@ -179,6 +179,7 @@
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc
@@ -848,6 +848,7 @@ processors:
     #match_source_index: 4
     #match_short_id: true
     #cleanup_timeout: 60
+    #labels.dedot: false
     # To connect to Docker over TLS you must specify a client and CA certificate.
     #ssl:
     #  certificate_authority: "/etc/pki/root/ca.pem"
@@ -885,6 +886,9 @@ for container ID. It defaults to 4 to match
 `cleanup_timeout`:: (Optional) Time of inactivity to consider we can clean and
 forget metadata for a container, 60s by default.
 
+`labels.dedot`:: (Optional) Default to be false. If set to true, replace dots in
+ labels with `_`.
+
 [[add-host-metadata]]
 === Add Host metadata
 

diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go
@@ -60,6 +60,7 @@ type addDockerMetadata struct {
 	pidFields []string      // Field names that contain PIDs.
 	cgroups   *common.Cache // Cache of PID (int) to cgropus (map[string]string).
 	hostFS    string        // Directory where /proc is found
+	dedot     bool          // If set to true, replace dots in labels with `_`.
 }
 
 func newDockerMetadataProcessor(cfg *common.Config) (processors.Processor, error) {
@@ -103,6 +104,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker.
 		sourceProcessor: sourceProcessor,
 		pidFields:       config.MatchPIDs,
 		hostFS:          config.HostFS,
+		dedot:           config.DeDot,
 	}, nil
 }
 
@@ -174,7 +176,12 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) {
 		if len(container.Labels) > 0 {
 			labels := common.MapStr{}
 			for k, v := range container.Labels {
-				safemapstr.Put(labels, k, v)
+				if d.dedot {
+					label := common.DeDot(k)
+					labels.Put(label, v)
+				} else {
+					safemapstr.Put(labels, k, v)
+				}
 			}
 			meta.Put("container.labels", labels)
 		}

diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go
@@ -149,6 +149,51 @@ func TestMatchContainer(t *testing.T) {
 	}, result.Fields)
 }
 
+func TestMatchContainerWithDedot(t *testing.T) {
+	testConfig, err := common.NewConfigFrom(map[string]interface{}{
+		"match_fields": []string{"foo"},
+		"labels.dedot": true,
+	})
+	assert.NoError(t, err)
+
+	p, err := buildDockerMetadataProcessor(testConfig, MockWatcherFactory(
+		map[string]*docker.Container{
+			"container_id": &docker.Container{
+				ID:    "container_id",
+				Image: "image",
+				Name:  "name",
+				Labels: map[string]string{
+					"a.x":   "1",
+					"b":     "2",
+					"b.foo": "3",
+				},
+			},
+		}))
+	assert.NoError(t, err, "initializing add_docker_metadata processor")
+
+	input := common.MapStr{
+		"foo": "container_id",
+	}
+	result, err := p.Run(&beat.Event{Fields: input})
+	assert.NoError(t, err, "processing an event")
+
+	assert.EqualValues(t, common.MapStr{
+		"docker": common.MapStr{
+			"container": common.MapStr{
+				"id":    "container_id",
+				"image": "image",
+				"labels": common.MapStr{
+					"a_x":   "1",
+					"b":     "2",
+					"b_foo": "3",
+				},
+				"name": "name",
+			},
+		},
+		"foo": "container_id",
+	}, result.Fields)
+}
+
 func TestMatchSource(t *testing.T) {
 	// Use defaults
 	testConfig, err := common.NewConfigFrom(map[string]interface{}{})

diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go
@@ -33,6 +33,7 @@ type Config struct {
 	SourceIndex  int               `config:"match_source_index"` // Index in the source path split by / to look for container ID.
 	MatchPIDs    []string          `config:"match_pids"`         // A list of fields containing process IDs (PIDs).
 	HostFS       string            `config:"system.hostfs"`      // Specifies the mount point of the host’s filesystem for use in monitoring a host from within a container.
+	DeDot        bool              `config:"labels.dedot"`       // If set to true, replace dots in labels with `_`.
 
 	// Annotations are kept after container is killed, until they haven't been
 	// accessed for a full `cleanup_timeout`:
@@ -45,5 +46,6 @@ func defaultConfig() Config {
 		MatchSource: true,
 		SourceIndex: 4, // Use 4 to match the CID in /var/lib/docker/containers/<container_id>/*.log.
 		MatchPIDs:   []string{"process.pid", "process.ppid"},
+		DeDot:       false,
 	}
 }
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -864,6 +864,7 @@ metricbeat.modules:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/metricbeat/module/elasticsearch/node/_meta/data.json b/metricbeat/module/elasticsearch/node/_meta/data.json
@@ -48,4 +48,4 @@
     "service": {
         "name": "elasticsearch"
     }
-}
+}
diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
@@ -668,6 +668,7 @@ packetbeat.protocols:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
@@ -208,6 +208,7 @@ winlogbeat.event_logs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -1007,6 +1007,7 @@ filebeat.inputs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml
@@ -273,6 +273,7 @@ functionbeat.provider.aws.functions:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -872,6 +872,7 @@ metricbeat.modules:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"