Add the content of the body under http.request.body and http.response…

….body if include_body_for is set. (#2167)

CHANGELOG.asciidoc

@@ -20,6 +20,8 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha5...master[Check the HEAD d
 - Change field type system.process.cpu.start_time from keyword to date. {issue}1565[1565]
 
 *Packetbeat*
+- Group HTTP fields under `http.request` and `http.response` {pull}2167[2167]
+- Export `http.request.body` and `http.response.body` when configured under `include_body_for` {pull}2167[2167]
 
 *Topbeat*
 

packetbeat/docs/fields.asciidoc

@@ -1114,44 +1114,71 @@ optional TCP connection id
 HTTP-specific event fields.
 
 
+[float]
+== http Fields
+
+Information about the HTTP request and response.
+
 
 [float]
-=== http.code
+== request Fields
 
-example: 404
+HTTP request
 
-The HTTP status code.
 
 [float]
-=== http.phrase
+=== http.request.params
 
-example: Not found.
+The query parameters or form values. The query parameters are available in the Request-URI and the form values are set in the HTTP body when the content-type is set to `x-www-form-urlencoded`.
 
-The HTTP status phrase.
 
 [float]
-=== http.request_headers
+=== http.request.headers
 
 type: dict
 
 A map containing the captured header fields from the request. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.
 
 
 [float]
-=== http.response_headers
+=== http.request.body
 
-type: dict
+type: text
 
-A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.
+The body of the HTTP request.
+
+[float]
+== response Fields
+
+HTTP response
 
 
 [float]
-=== http.content_length
+=== http.response.code
 
-type: long
+example: 404
 
-The value of the Content-Length header if present.
+The HTTP status code.
+
+[float]
+=== http.response.phrase
+
+example: Not found.
+
+The HTTP status phrase.
+
+[float]
+=== http.response.headers
+
+type: dict
+
+A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.
+
+
+[float]
+=== http.response.body
 
+The body of the HTTP response.
 
 [[exported-fields-icmp]]
 == ICMP Fields

packetbeat/docs/reference/configuration/packetbeat-options.asciidoc

@@ -404,10 +404,15 @@ send all headers by setting this option to true. The default is false.
 
 ===== include_body_for
 
-The list of content types for which Packetbeat includes the full HTTP payload in
-the `response` field. This option should be used together with the <<send-response-option>> option.
+The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under
+`http.request.body` and `http.response.body` for these Content-Types.
 
-Example configuration:
+In addition, if <<send-response-option>> option is enabled, then the HTTP body is exported together with the HTTP
+headers under `response` and if
+<<send-request-option>> enabled, then `request` contains the entire HTTP message including the body.
+
+In the following example, the HTML attachments of the HTTP responses are exported under the `response` field and under
+`http.request.body` or `http.response.body`:
 
 [source,yml]
 ------------------------------------------------------------------------------
@@ -418,6 +423,7 @@ packetbeat.protocols.http:
 ------------------------------------------------------------------------------
 
 
+
 ===== split_cookie
 
 If the `Cookie` or `Set-Cookie` headers are sent, this option controls whether

packetbeat/etc/fields.yml

@@ -837,37 +837,50 @@
   fields:
     - name: http
       type: group
+      description: Information about the HTTP request and response.
       fields:
-        - name: code
-          description: The HTTP status code.
-          example: 404
-
-        - name: phrase
-          description: The HTTP status phrase.
-          example: Not found.
-
-        - name: request_headers
-          type: dict
-          dict-type: keyword
-          description: >
-            A map containing the captured header fields from the request.
-            Which headers to capture is configurable. If headers with the same
-            header name are present in the message, they will be separated by
-            commas.
+        - name: request
+          description: HTTP request
+          type: group
+          fields:
+            - name: params
+              description: >
+                The query parameters or form values. The query parameters are available in the Request-URI
+                and the form values are set in the HTTP body when the content-type is set to `x-www-form-urlencoded`.
+            - name: headers
+              type: dict
+              dict-type: keyword
+              description: >
+                A map containing the captured header fields from the request.
+                Which headers to capture is configurable. If headers with the same
+                header name are present in the message, they will be separated by
+                commas.
+            - name: body
+              type: text
+              description: The body of the HTTP request.
+
+        - name: response
+          description: HTTP response
+          type: group
+          fields:
+            - name: code
+              description: The HTTP status code.
+              example: 404
 
-        - name: response_headers
-          type: dict
-          dict-type: keyword
-          description: >
-            A map containing the captured header fields from the response.
-            Which headers to capture is configurable. If headers with the
-            same header name are present in the message, they will be separated
-            by commas.
+            - name: phrase
+              description: The HTTP status phrase.
+              example: Not found.
 
-        - name: content_length
-          type: long
-          description: >
-            The value of the Content-Length header if present.
+            - name: headers
+              type: dict
+              dict-type: keyword
+              description: >
+                A map containing the captured header fields from the response.
+                Which headers to capture is configurable. If headers with the
+                same header name are present in the message, they will be separated
+                by commas.
+            - name: body
+              description: The body of the HTTP response.
 
 - key: memcache
   title: "Memcache"

packetbeat/packetbeat.template-es2x.json

@@ -30,25 +30,25 @@
           }
         },
         {
-          "http.request_headers": {
+          "http.request.headers": {
             "mapping": {
               "ignore_above": 1024,
               "index": "not_analyzed",
               "type": "string"
             },
             "match_mapping_type": "string",
-            "path_match": "http.request_headers.*"
+            "path_match": "http.request.headers.*"
           }
         },
         {
-          "http.response_headers": {
+          "http.response.headers": {
             "mapping": {
               "ignore_above": 1024,
               "index": "not_analyzed",
               "type": "string"
             },
             "match_mapping_type": "string",
-            "path_match": "http.response_headers.*"
+            "path_match": "http.response.headers.*"
           }
         }
       ],
@@ -508,18 +508,40 @@
         },
         "http": {
           "properties": {
-            "code": {
-              "ignore_above": 1024,
-              "index": "not_analyzed",
-              "type": "string"
-            },
-            "content_length": {
-              "type": "long"
+            "request": {
+              "properties": {
+                "body": {
+                  "index": "analyzed",
+                  "norms": {
+                    "enabled": false
+                  },
+                  "type": "string"
+                },
+                "params": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                }
+              }
             },
-            "phrase": {
-              "ignore_above": 1024,
-              "index": "not_analyzed",
-              "type": "string"
+            "response": {
+              "properties": {
+                "body": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "code": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "phrase": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                }
+              }
             }
           }
         },

packetbeat/packetbeat.template.json

@@ -26,23 +26,23 @@
           }
         },
         {
-          "http.request_headers": {
+          "http.request.headers": {
             "mapping": {
               "ignore_above": 1024,
               "type": "keyword"
             },
             "match_mapping_type": "string",
-            "path_match": "http.request_headers.*"
+            "path_match": "http.request.headers.*"
           }
         },
         {
-          "http.response_headers": {
+          "http.response.headers": {
             "mapping": {
               "ignore_above": 1024,
               "type": "keyword"
             },
             "match_mapping_type": "string",
-            "path_match": "http.response_headers.*"
+            "path_match": "http.response.headers.*"
           }
         }
       ],
@@ -449,16 +449,33 @@
         },
         "http": {
           "properties": {
-            "code": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "content_length": {
-              "type": "long"
+            "request": {
+              "properties": {
+                "body": {
+                  "norms": false,
+                  "type": "text"
+                },
+                "params": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
             },
-            "phrase": {
-              "ignore_above": 1024,
-              "type": "keyword"
+            "response": {
+              "properties": {
+                "body": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "code": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "phrase": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
             }
           }
         },