Update beats framework to a4b06ae

NOTICE.txt

@@ -264,7 +264,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 1c1b7d7427a422ec9da0707b1830e9cb03837266
+Revision: a4b06ae0f035889416ceb0df0b552ec3c573ca1a
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/dev-tools/ecs-migration.yml

@@ -45,6 +45,10 @@
   alias: true
   copy_to: false
 
+# Filebeat modules
+
+## Suricata module
+
 - from: source_ecs.ip
   to: source.ip
   alias: true
@@ -155,6 +159,59 @@
   alias: true
   copy_to: false
 
+## Apache
+
+- from: apache2.access.user_name
+  to: user.name
+  alias: true
+  copy_to: false
+
+- from: apache2.access.method
+  to: http.request.method
+  alias: true
+  copy_to: false
+
+- from: apache2.access.url
+  to: url.original
+  alias: true
+  copy_to: false
+
+- from: apache2.access.http_version
+  to: http.version
+  alias: true
+  copy_to: false
+
+- from: apache2.access.response_code
+  to: http.response.status_code
+  alias: true
+  copy_to: false
+
+- from: apache2.access.referrer
+  to: http.request.referrer
+  alias: true
+  copy_to: false
+
+- from: apache2.access.agent
+  to: user_agent.original
+  alias: true
+  copy_to: false
+
+- from: read_timestamp
+  to: event.created
+  alias: false
+  copy_to: false
+
+# These expand all fields under geoip and user_agent
+- from: apache2.access.geoip.*
+  to: source.geo.*
+  alias: false
+  copy_to: false
+
+- from: apache2.access.user_agent.*
+  to: user_agent.*
+  alias: false
+  copy_to: false
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain

_beats/libbeat/_meta/fields.ecs.yml

@@ -1042,28 +1042,6 @@
 
       type: group
       fields:
-        - name: href
-          type: text
-          description: >
-            Full url. The field is stored as keyword.
-
-            `url.href` is a [multi field](https://www.elastic.co/guide/en/
-            elasticsearch/reference/6.2/
-            multi-fields.html#_multi_fields_with_multiple_analyzers). The data is
-            stored as keyword `url.href` and test
-            `url.href.analyzed`. These fields enable you to run a query against part
-            of the url still works splitting up the URL at ingest time.
-
-            `href` is an analyzed field so the parsed information can be accessed
-            through `href.analyzed` in queries.
-
-          multi_fields:
-            - name: raw
-              type: keyword
-              description: >
-                The full URL. This is a non-analyzed field that is useful
-                for aggregations.
-          example: https://elastic.co:443/search?q=elasticsearch#top
         - name: scheme
           type: keyword
           description: >

_beats/libbeat/scripts/generate_fields_docs.py

@@ -41,12 +41,12 @@ def document_fields(output, section, sections, path):
             document_field(output, field, newpath)
 
 
-def document_field(output, field, path):
+def document_field(output, field, field_path):
 
-    if "path" not in field:
-        field["path"] = path
+    if "field_path" not in field:
+        field["field_path"] = field_path
 
-    output.write("*`{}`*::\n+\n--\n".format(field["path"]))
+    output.write("*`{}`*::\n+\n--\n".format(field["field_path"]))
 
     if "deprecated" in field:
         output.write("\ndeprecated[{}]\n\n".format(field["deprecated"]))
@@ -59,7 +59,8 @@ def document_field(output, field, path):
         output.write("format: {}\n\n".format(field["format"]))
     if "required" in field:
         output.write("required: {}\n\n".format(field["required"]))
-
+    if "path" in field:
+        output.write("alias to: {}\n\n".format(field["path"]))
     if "description" in field:
         output.write("{}\n\n".format(field["description"]))
 
@@ -73,7 +74,7 @@ def document_field(output, field, path):
 
     if "multi_fields" in field:
         for subfield in field["multi_fields"]:
-            document_field(output, subfield, path + "." + subfield["name"])
+            document_field(output, subfield, field_path + "." + subfield["name"])
     output.write("--\n\n")
 
 

changelogs/head.asciidoc

@@ -10,6 +10,8 @@ https://github.com/elastic/apm-server/compare/6.5\...master[View commits]
 - Add `span.sync` property to intake json spec and index field in ES. {pull}1548[1548].
 - Make `service.framework` properties optional and nullable {pull}1546[1546].
 - Update Go to 1.11.2 {pull}1605[1605].
+- Use _doc as document type for Elasticsearch >= 7.0.0 {pull}XXX[XXX].
+- Automatically cap signed integers to 63bits {pull}XXX[XXX].
 
 [float]
 ==== Bug fixes

docs/fields.asciidoc

@@ -2762,30 +2762,6 @@ URL fields provide a complete URL, with scheme, host, and path. The URL object c
 
 
 
-*`url.href`*::
-+
---
-type: text
-
-example: https://elastic.co:443/search?q=elasticsearch#top
-
-Full url. The field is stored as keyword.
-`url.href` is a [multi field](https://www.elastic.co/guide/en/ elasticsearch/reference/6.2/ multi-fields.html#_multi_fields_with_multiple_analyzers). The data is stored as keyword `url.href` and test `url.href.analyzed`. These fields enable you to run a query against part of the url still works splitting up the URL at ingest time.
-`href` is an analyzed field so the parsed information can be accessed through `href.analyzed` in queries.
-
-
-*`url.href.raw`*::
-+
---
-type: keyword
-
-The full URL. This is a non-analyzed field that is useful for aggregations.
-
-
---
-
---
-
 *`url.scheme`*::
 +
 --
@@ -3064,18 +3040,22 @@ Minor version of the operating system.
 
 --
 
-*`agent.type`*::
+*`beat.name`*::
 +
 --
 type: alias
 
+alias to: agent.type
+
 --
 
-*`agent.hostname`*::
+*`beat.hostname`*::
 +
 --
 type: alias
 
+alias to: agent.hostname
+
 --
 
 [[exported-fields-host-processor]]

tests/system/error.approved.json

@@ -1,7 +1,7 @@
 [
     {
         "_score": 0.2876821,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "YEUQC2IBWUGW2FxIuwuv",
         "_source": {
             "beat": {
@@ -64,7 +64,7 @@
     },
     {
         "_score": 0.13353139,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "XkUQC2IBWUGW2FxIuwuv",
         "_source": {
             "transaction": {
@@ -330,7 +330,7 @@
     },
     {
         "_score": 0.13353139,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "X0UQC2IBWUGW2FxIuwuv",
         "_source": {
             "beat": {
@@ -394,7 +394,7 @@
     },
     {
         "_score": 0.13353139,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "YUUQC2IBWUGW2FxIuwuv",
         "_source": {
             "beat": {

tests/system/spans.approved.json

@@ -1,7 +1,7 @@
 [
     {
         "_score": 0.9808292,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "IkU4C2IBWUGW2FxIExjw",
         "_source": {
             "transaction": {
@@ -43,7 +43,7 @@
     },
     {
         "_score": 0.6931472,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "J0U4C2IBWUGW2FxIExjw",
         "_source": {
             "transaction": {
@@ -90,7 +90,7 @@
     },
     {
         "_score": 0.2876821,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "I0U4C2IBWUGW2FxIExjw",
         "_source": {
             "transaction": {
@@ -132,7 +132,7 @@
     },
     {
         "_score": 0.18232156,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "IEU4C2IBWUGW2FxIExjw",
         "_source": {
             "transaction": {
@@ -224,7 +224,7 @@
     },
     {
         "_score": 0.18232156,
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "IUU4C2IBWUGW2FxIExjw",
         "_source": {
             "transaction": {

tests/system/test_integration.py

@@ -74,7 +74,7 @@ def test_mark_navigation_timing(self):
         self.assert_no_logged_warnings()
         mappings = self.es.indices.get_field_mapping(index=self.index_name, fields="transaction.marks.*")
         found_other = False
-        for name, metric in mappings[self.index_name]["mappings"]["doc"].items():
+        for name, metric in mappings[self.index_name]["mappings"]["_doc"].items():
             for mapping in metric["mapping"].values():
                 mtype = mapping["type"]
                 if name.startswith("transaction.marks.navigationTiming."):
@@ -531,7 +531,7 @@ def test_metric_doc(self):
         self.load_docs_with_template(self.get_metricset_payload_path(), self.metrics_url, 'metric', 1)
         mappings = self.es.indices.get_field_mapping(index=self.index_name, fields="system.process.cpu.total.norm.pct")
         expected_type = "scaled_float"
-        actual_type = mappings[self.index_name]["mappings"]["doc"]["system.process.cpu.total.norm.pct"]["mapping"]["pct"]["type"]
+        actual_type = mappings[self.index_name]["mappings"]["_doc"]["system.process.cpu.total.norm.pct"]["mapping"]["pct"]["type"]
         assert expected_type == actual_type, "want: {}, got: {}".format(expected_type, actual_type)
 
 

tests/system/transaction.approved.json

@@ -2,7 +2,7 @@
     {
         "_id": "XkTYCmIBWUGW2FxI4_kt",
         "_score": 0.6931472,
-        "_type": "doc",
+        "_type": "_doc",
         "_source": {
             "beat": {
                 "name": "ed7e2cf02cd9",
@@ -154,7 +154,7 @@
     {
         "_id": "ZUTYCmIBWUGW2FxI4_kt",
         "_score": 0.6931472,
-        "_type": "doc",
+        "_type": "_doc",
         "_source": {
             "beat": {
                 "name": "ed7e2cf02cd9",
@@ -225,7 +225,7 @@
     {
         "_id": "ZETYCmIBWUGW2FxI4_kt",
         "_score": 0.6931472,
-        "_type": "doc",
+        "_type": "_doc",
         "_source": {
             "processor": {
                 "name": "transaction",
@@ -291,7 +291,7 @@
     {
         "_id": "Y0TYCmIBWUGW2FxI4_kt",
         "_score": 0.2876821,
-        "_type": "doc",
+        "_type": "_doc",
         "_source": {
             "beat": {
                 "name": "ed7e2cf02cd9",