From ee30832697ec2709758af6b032681071caa75e44 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jsorianopastor@gmail.com>
Date: Tue, 21 Apr 2020 09:44:23 +0200
Subject: [PATCH] Cherry-pick #17759 to 7.7: Add fields validation for
 histogram subfields (#17786)

Add validation to histogram subfields in a way that they don't need to
be added to the mappings, but they can still be checked. This is helpful
to avoid having to add exceptions to all the modules based on Prometheus
when Elasticsearch types are used.

(cherry picked from commit 4912b78a4a046aa2baa2c5459fedac4b23de6467)
---
 CHANGELOG-developer.next.asciidoc                         | 1 +
 libbeat/mapping/field.go                                  | 4 ++++
 libbeat/tests/system/beat/beat.py                         | 4 ++++
 .../module/prometheus/collector/_meta/testdata/config.yml | 8 +-------
 .../module/prometheus/collector/collector_test.go         | 3 +++
 5 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
index df3b40ef7ec..ed051294799 100644
--- a/CHANGELOG-developer.next.asciidoc
+++ b/CHANGELOG-developer.next.asciidoc
@@ -60,3 +60,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Compare event by event in `testadata` framework to avoid sorting problems {pull}13747[13747]
 - Added a `default_field` option to fields in fields.yml to offer a way to exclude fields from the default_field list. {issue}14262[14262] {pull}14341[14341]
 - `supported-versions.yml` can be used in metricbeat python system tests to obtain the build args for docker compose builds. {pull}14520[14520]
+- Add fields validation for histogram subfields. {pull}17759[17759]
diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go
index 66a580dac5e..7b2ba52e618 100644
--- a/libbeat/mapping/field.go
+++ b/libbeat/mapping/field.go
@@ -354,6 +354,10 @@ func (f Fields) getKeys(namespace string) []string {
 		} else {
 			keys = append(keys, field.Fields.getKeys(fieldName)...)
 		}
+		if field.ObjectType == "histogram" {
+			keys = append(keys, fieldName+".values")
+			keys = append(keys, fieldName+".counts")
+		}
 	}
 
 	return keys
diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
index 160df982f56..fa8e0ad3cb4 100644
--- a/libbeat/tests/system/beat/beat.py
+++ b/libbeat/tests/system/beat/beat.py
@@ -573,6 +573,10 @@ def extract_fields(doc_list, name):
                     if field.get("type") in ["object", "geo_point"]:
                         dictfields.append(newName)
 
+                if field.get("type") == "object" and field.get("object_type") == "histogram":
+                    fields.append(newName + ".values")
+                    fields.append(newName + ".counts")
+
                 if field.get("type") == "alias":
                     aliases.append(newName)
 
diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
index e7b2a5e67b4..380627ba09f 100644
--- a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
+++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
@@ -1,13 +1,7 @@
 type: http
 url: "/metrics"
 suffix: plain
-omit_documented_fields_check:
-   # these are not mapped by this module but the oss one
-  - prometheus.labels.*
-  # histogram values & counts are not mapped (it's part of the type data)
-  - '*.histogram.values'
-  - '*.histogram.counts'
 remove_fields_from_comparison: ["prometheus.labels.instance"]
 module:
   use_types: true
-  rate_counters: true
\ No newline at end of file
+  rate_counters: true
diff --git a/x-pack/metricbeat/module/prometheus/collector/collector_test.go b/x-pack/metricbeat/module/prometheus/collector/collector_test.go
index 0a3950bdc75..5f42875dd76 100644
--- a/x-pack/metricbeat/module/prometheus/collector/collector_test.go
+++ b/x-pack/metricbeat/module/prometheus/collector/collector_test.go
@@ -12,6 +12,9 @@ import (
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 
 	_ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus"
+
+	// Import common fields for validation
+	_ "github.com/elastic/beats/v7/metricbeat/module/prometheus"
 )
 
 func TestData(t *testing.T) {