diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 95fe482d9ad..f8dec156db1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -17,6 +17,8 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha5...master[Check the HEAD d *Metricbeat* *Packetbeat* +- Group HTTP fields under `http.request` and `http.response` {pull}2167[2167] +- Export `http.request.body` and `http.response.body` when configured under `include_body_for` {pull}2167[2167] *Topbeat* diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 5e096de860d..778f83d977d 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -1106,23 +1106,26 @@ optional TCP connection id HTTP-specific event fields. +[float] +== http Fields + +Information about the HTTP request and response. + [float] -=== http.code +== request Fields -example: 404 +HTTP request -The HTTP status code. [float] -=== http.phrase +=== http.request.params -example: Not found. +The query parameters or form values. The query parameters are available in the Request-URI and the form values are set in the HTTP body when the content-type is set to `x-www-form-urlencoded`. -The HTTP status phrase. [float] -=== http.request_headers +=== http.request.headers type: dict @@ -1130,20 +1133,44 @@ A map containing the captured header fields from the request. Which headers to c [float] -=== http.response_headers +=== http.request.body -type: dict +type: text -A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. +The body of the HTTP request. + +[float] +== response Fields + +HTTP response [float] -=== http.content_length +=== http.response.code -type: long +example: 404 -The value of the Content-Length header if present. +The HTTP status code. + +[float] +=== http.response.phrase + +example: Not found. + +The HTTP status phrase. + +[float] +=== http.response.headers + +type: dict + +A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas. + + +[float] +=== http.response.body +The body of the HTTP response. [[exported-fields-icmp]] == ICMP Fields diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index 210bef7f46a..9bc9f66fbf4 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -404,10 +404,15 @@ send all headers by setting this option to true. The default is false. ===== include_body_for -The list of content types for which Packetbeat includes the full HTTP payload in -the `response` field. This option should be used together with the <> option. +The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under +`http.request.body` and `http.response.body` for these Content-Types. -Example configuration: +In addition, if <> option is enabled, then the HTTP body is exported together with the HTTP +headers under `response` and if +<> enabled, then `request` contains the entire HTTP message including the body. + +In the following example, the HTML attachments of the HTTP responses are exported under the `response` field and under +`http.request.body` or `http.response.body`: [source,yml] ------------------------------------------------------------------------------ @@ -418,6 +423,7 @@ packetbeat.protocols.http: ------------------------------------------------------------------------------ + ===== split_cookie If the `Cookie` or `Set-Cookie` headers are sent, this option controls whether diff --git a/packetbeat/etc/fields.yml b/packetbeat/etc/fields.yml index 916d4cd6ee5..59d46e42e6a 100644 --- a/packetbeat/etc/fields.yml +++ b/packetbeat/etc/fields.yml @@ -833,37 +833,50 @@ fields: - name: http type: group + description: Information about the HTTP request and response. fields: - - name: code - description: The HTTP status code. - example: 404 - - - name: phrase - description: The HTTP status phrase. - example: Not found. - - - name: request_headers - type: dict - dict-type: keyword - description: > - A map containing the captured header fields from the request. - Which headers to capture is configurable. If headers with the same - header name are present in the message, they will be separated by - commas. + - name: request + description: HTTP request + type: group + fields: + - name: params + description: > + The query parameters or form values. The query parameters are available in the Request-URI + and the form values are set in the HTTP body when the content-type is set to `x-www-form-urlencoded`. + - name: headers + type: dict + dict-type: keyword + description: > + A map containing the captured header fields from the request. + Which headers to capture is configurable. If headers with the same + header name are present in the message, they will be separated by + commas. + - name: body + type: text + description: The body of the HTTP request. + + - name: response + description: HTTP response + type: group + fields: + - name: code + description: The HTTP status code. + example: 404 - - name: response_headers - type: dict - dict-type: keyword - description: > - A map containing the captured header fields from the response. - Which headers to capture is configurable. If headers with the - same header name are present in the message, they will be separated - by commas. + - name: phrase + description: The HTTP status phrase. + example: Not found. - - name: content_length - type: long - description: > - The value of the Content-Length header if present. + - name: headers + type: dict + dict-type: keyword + description: > + A map containing the captured header fields from the response. + Which headers to capture is configurable. If headers with the + same header name are present in the message, they will be separated + by commas. + - name: body + description: The body of the HTTP response. - key: memcache title: "Memcache" diff --git a/packetbeat/packetbeat.template-es2x.json b/packetbeat/packetbeat.template-es2x.json index bf97a8210c5..828244602aa 100644 --- a/packetbeat/packetbeat.template-es2x.json +++ b/packetbeat/packetbeat.template-es2x.json @@ -30,25 +30,25 @@ } }, { - "http.request_headers": { + "http.request.headers": { "mapping": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, "match_mapping_type": "string", - "path_match": "http.request_headers.*" + "path_match": "http.request.headers.*" } }, { - "http.response_headers": { + "http.response.headers": { "mapping": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" }, "match_mapping_type": "string", - "path_match": "http.response_headers.*" + "path_match": "http.response.headers.*" } } ], @@ -512,18 +512,40 @@ }, "http": { "properties": { - "code": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" - }, - "content_length": { - "type": "long" + "request": { + "properties": { + "body": { + "index": "analyzed", + "norms": { + "enabled": false + }, + "type": "string" + }, + "params": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } }, - "phrase": { - "ignore_above": 1024, - "index": "not_analyzed", - "type": "string" + "response": { + "properties": { + "body": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "code": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "phrase": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } } } }, diff --git a/packetbeat/packetbeat.template.json b/packetbeat/packetbeat.template.json index 4ba8af8375d..421e00a4be7 100644 --- a/packetbeat/packetbeat.template.json +++ b/packetbeat/packetbeat.template.json @@ -26,23 +26,23 @@ } }, { - "http.request_headers": { + "http.request.headers": { "mapping": { "ignore_above": 1024, "type": "keyword" }, "match_mapping_type": "string", - "path_match": "http.request_headers.*" + "path_match": "http.request.headers.*" } }, { - "http.response_headers": { + "http.response.headers": { "mapping": { "ignore_above": 1024, "type": "keyword" }, "match_mapping_type": "string", - "path_match": "http.response_headers.*" + "path_match": "http.response.headers.*" } } ], @@ -451,16 +451,33 @@ }, "http": { "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "content_length": { - "type": "long" + "request": { + "properties": { + "body": { + "norms": false, + "type": "text" + }, + "params": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "phrase": { - "ignore_above": 1024, - "type": "keyword" + "response": { + "properties": { + "body": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "phrase": { + "ignore_above": 1024, + "type": "keyword" + } + } } } }, diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index 207da210ce0..cfab0d9e7b1 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -460,16 +460,21 @@ func (http *HTTP) newTransaction(requ, resp *message) common.MapStr { src, dst = dst, src } - details := common.MapStr{ - "phrase": resp.StatusPhrase, - "code": resp.StatusCode, - "content_length": resp.ContentLength, - } - if http.parserConfig.SendHeaders { - details["request_headers"] = http.collectHeaders(requ) - details["response_headers"] = http.collectHeaders(resp) + http_details := common.MapStr{ + "request": common.MapStr{ + "params": params, + "headers": http.collectHeaders(requ), + }, + "response": common.MapStr{ + "code": resp.StatusCode, + "phrase": resp.StatusPhrase, + "headers": http.collectHeaders(resp), + }, } + http.setBody(http_details["request"].(common.MapStr), requ) + http.setBody(http_details["response"].(common.MapStr), resp) + event := common.MapStr{ "@timestamp": common.Time(requ.Ts), "type": "http", @@ -477,9 +482,8 @@ func (http *HTTP) newTransaction(requ, resp *message) common.MapStr { "responsetime": responseTime, "method": requ.Method, "path": path, - "params": params, "query": fmt.Sprintf("%s %s", requ.Method, path), - "http": details, + "http": http_details, "bytes_out": resp.Size, "bytes_in": requ.Size, "src": &src, @@ -492,6 +496,7 @@ func (http *HTTP) newTransaction(requ, resp *message) common.MapStr { if http.SendResponse { event["response"] = string(http.cutMessageBody(resp)) } + if len(requ.Notes)+len(resp.Notes) > 0 { event["notes"] = append(requ.Notes, resp.Notes...) } @@ -510,26 +515,48 @@ func (http *HTTP) publishTransaction(event common.MapStr) { } func (http *HTTP) collectHeaders(m *message) interface{} { - if !http.SplitCookie { - return m.Headers - } - cookie := "cookie" - if !m.IsRequest { - cookie = "set-cookie" + hdrs := map[string]interface{}{} + + hdrs["content-length"] = m.ContentLength + if len(m.ContentType) > 0 { + hdrs["content-type"] = m.ContentType } - hdrs := map[string]interface{}{} - for name, value := range m.Headers { - if name == cookie { - hdrs[name] = splitCookiesHeader(string(value)) - } else { - hdrs[name] = value + if http.parserConfig.SendHeaders { + + cookie := "cookie" + if !m.IsRequest { + cookie = "set-cookie" + } + + for name, value := range m.Headers { + if strings.ToLower(name) == "content-type" { + continue + } + if strings.ToLower(name) == "content-length" { + continue + } + if http.SplitCookie { + if name == cookie { + hdrs[name] = splitCookiesHeader(string(value)) + } + } else { + hdrs[name] = value + } } } + fmt.Println("Headers: ", hdrs) return hdrs } +func (http *HTTP) setBody(result common.MapStr, m *message) { + body := string(http.extractBody(m)) + if len(body) > 0 { + result["body"] = body + } +} + func splitCookiesHeader(headerVal string) map[string]string { cookies := map[string]string{} @@ -553,25 +580,32 @@ func parseCookieValue(raw string) string { return raw } -func (http *HTTP) cutMessageBody(m *message) []byte { - cutMsg := []byte{} - - // add headers always - cutMsg = m.Raw[:m.bodyOffset] +func (http *HTTP) extractBody(m *message) []byte { + body := []byte{} - // add body if len(m.ContentType) == 0 || http.shouldIncludeInBody(m.ContentType) { if len(m.chunkedBody) > 0 { - cutMsg = append(cutMsg, m.chunkedBody...) + body = append(body, m.chunkedBody...) } else { if isDebug { debugf("Body to include: [%s]", m.Raw[m.bodyOffset:]) } - cutMsg = append(cutMsg, m.Raw[m.bodyOffset:]...) + body = append(body, m.Raw[m.bodyOffset:]...) } } - return cutMsg + return body +} + +func (http *HTTP) cutMessageBody(m *message) []byte { + cutMsg := []byte{} + + // add headers always + cutMsg = m.Raw[:m.bodyOffset] + + // add body + return append(cutMsg, http.extractBody(m)...) + } func (http *HTTP) shouldIncludeInBody(contenttype []byte) bool { @@ -663,7 +697,7 @@ func (http *HTTP) hideSecrets(values url.Values) url.Values { // extractParameters parses the URL and the form parameters and replaces the secrets // with the string xxxxx. The parameters containing secrets are defined in http.Hide_secrets. -// Returns the Request URI path and the (ajdusted) parameters. +// Returns the Request URI path and the (adjusted) parameters. func (http *HTTP) extractParameters(m *message, msg []byte) (path string, params string, err error) { var values url.Values @@ -677,6 +711,7 @@ func (http *HTTP) extractParameters(m *message, msg []byte) (path string, params paramsMap := http.hideSecrets(values) if m.ContentLength > 0 && bytes.Contains(m.ContentType, []byte("urlencoded")) { + values, err = url.ParseQuery(string(msg[m.bodyOffset:])) if err != nil { return @@ -686,12 +721,11 @@ func (http *HTTP) extractParameters(m *message, msg []byte) (path string, params paramsMap[key] = value } } - params = paramsMap.Encode() + params = paramsMap.Encode() if isDetailed { - detailedf("Parameters: %s", params) + detailedf("Form parameters: %s", params) } - return } diff --git a/packetbeat/tests/system/pcaps/http_post_json.pcap b/packetbeat/tests/system/pcaps/http_post_json.pcap new file mode 100644 index 00000000000..0f66d0d58bd Binary files /dev/null and b/packetbeat/tests/system/pcaps/http_post_json.pcap differ diff --git a/packetbeat/tests/system/test_0006_wsgi.py b/packetbeat/tests/system/test_0006_wsgi.py index 09777ccce10..0e844596527 100644 --- a/packetbeat/tests/system/test_0006_wsgi.py +++ b/packetbeat/tests/system/test_0006_wsgi.py @@ -23,8 +23,8 @@ def test_long_answer(self): assert o["status"] == "OK" assert o["method"] == "GET" assert o["path"] == "/" - assert o["http.code"] == 200 - assert o["http.phrase"] == "OK" + assert o["http.response.code"] == 200 + assert o["http.response.phrase"] == "OK" assert "request" not in objs[0] assert "response" not in objs[0] @@ -46,7 +46,7 @@ def test_drum_interraction(self): assert objs[13]["status"] == "Error" assert objs[13]["path"] == "/comment/" - assert objs[13]["http.code"] == 500 + assert objs[13]["http.response.code"] == 500 def test_send_options(self): """ @@ -89,8 +89,8 @@ def test_send_headers_options(self): assert len(objs) == 1 o = objs[0] - assert "http.request_headers" not in o - assert "http.response_headers" not in o + assert "http.request.headers" in o + assert "http.response.headers" in o self.render_config_template( http_ports=[8888], @@ -102,12 +102,12 @@ def test_send_headers_options(self): assert len(objs) == 1 o = objs[0] - assert "http.request_headers" in o - assert "http.response_headers" in o - assert o["http.request_headers"]["cache-control"] == "max-age=0" - assert len(o["http.request_headers"]) == 9 - assert len(o["http.response_headers"]) == 7 - assert isinstance(o["http.response_headers"]["set-cookie"], + assert "http.request.headers" in o + assert "http.response.headers" in o + assert o["http.request.headers"]["cache-control"] == "max-age=0" + assert len(o["http.request.headers"]) > 0 + assert len(o["http.response.headers"]) > 0 + assert isinstance(o["http.response.headers"]["set-cookie"], basestring) self.render_config_template( @@ -121,11 +121,11 @@ def test_send_headers_options(self): assert len(objs) == 1 o = objs[0] - assert "http.request_headers" in o - assert "http.response_headers" in o - assert len(o["http.request_headers"]) == 1 - assert len(o["http.response_headers"]) == 1 - assert "user-agent" in o["http.request_headers"] + assert "http.request.headers" in o + assert "http.response.headers" in o + assert len(o["http.request.headers"]) > 0 + assert len(o["http.response.headers"]) > 0 + assert "user-agent" in o["http.request.headers"] def test_split_cookie(self): self.render_config_template( @@ -138,12 +138,13 @@ def test_split_cookie(self): objs = self.read_output() assert len(objs) == 1 o = objs[0] + print o - assert len(o["http.request_headers"]) == 9 - assert len(o["http.response_headers"]) == 7 + assert len(o["http.request.headers"]) > 0 + assert len(o["http.response.headers"]) > 0 - assert isinstance(o["http.request_headers"]["cookie"], dict) - assert len(o["http.request_headers"]["cookie"]) == 6 + assert isinstance(o["http.request.headers"]["cookie"], dict) + assert len(o["http.request.headers"]["cookie"]) == 6 - assert isinstance(o["http.response_headers"]["set-cookie"], dict) - assert len(o["http.response_headers"]["set-cookie"]) == 4 + assert isinstance(o["http.response.headers"]["set-cookie"], dict) + assert len(o["http.response.headers"]["set-cookie"]) == 4 diff --git a/packetbeat/tests/system/test_0010_http_10_connection_close.py b/packetbeat/tests/system/test_0010_http_10_connection_close.py index 0f05cb5cc18..47408d03486 100644 --- a/packetbeat/tests/system/test_0010_http_10_connection_close.py +++ b/packetbeat/tests/system/test_0010_http_10_connection_close.py @@ -11,9 +11,10 @@ def test_http_sample(self): assert len(objs) == 1 obj = objs[0] + assert obj["status"] == "OK" - assert obj["http.content_length"] == 11422 - assert obj["http.code"] == 200 + assert obj["http.response.headers"]["content-length"] == 11422 + assert obj["http.response.code"] == 200 assert obj["type"] == "http" assert obj["client_ip"] == "127.0.0.1" assert obj["client_port"] == 37885 diff --git a/packetbeat/tests/system/test_0012_http_basicauth.py b/packetbeat/tests/system/test_0012_http_basicauth.py index 13c34539cd3..9cae8aa6ab8 100644 --- a/packetbeat/tests/system/test_0012_http_basicauth.py +++ b/packetbeat/tests/system/test_0012_http_basicauth.py @@ -21,7 +21,7 @@ def test_http_auth_headers(self): assert len(objs) >= 1 assert all([o["type"] == "http" for o in objs]) - assert all([o["http.request_headers"]["authorization"] == "*" + assert all([o["http.request.headers"]["authorization"] == "*" is not None for o in objs]) diff --git a/packetbeat/tests/system/test_0019_hide_params.py b/packetbeat/tests/system/test_0019_hide_params.py index 65640a4ed7b..94537e58735 100644 --- a/packetbeat/tests/system/test_0019_hide_params.py +++ b/packetbeat/tests/system/test_0019_hide_params.py @@ -22,7 +22,7 @@ def test_http_hide_post(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert o["params"] == "pass=xxxxx&user=monica" + assert o["http.request.params"] == "pass=xxxxx&user=monica" assert o["path"] == "/login" for _, val in o.items(): if isinstance(val, basestring): @@ -43,7 +43,7 @@ def test_http_hide_get(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert o["params"] == "pass=xxxxx&user=monica" + assert o["http.request.params"] == "pass=xxxxx&user=monica" assert o["path"] == "/login" for _, val in o.items(): if isinstance(val, basestring): @@ -61,5 +61,5 @@ def test_http_hide_post_default(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert o["params"] == "pass=secret&user=monica" + assert o["http.request.params"] == "pass=secret&user=monica" assert o["path"] == "/login" diff --git a/packetbeat/tests/system/test_0023_http_params.py b/packetbeat/tests/system/test_0023_http_params.py index f94faee839e..22a7821472d 100644 --- a/packetbeat/tests/system/test_0023_http_params.py +++ b/packetbeat/tests/system/test_0023_http_params.py @@ -19,8 +19,8 @@ def test_http_post(self): assert len(objs) == 1 o = objs[0] assert o["type"] == "http" - assert len(o["params"]) > 0 - assert o["params"] == "address=anklamerstr.14b&telephon=8932784368&" +\ + assert len(o["http.request.params"]) > 0 + assert o["http.request.params"] == "address=anklamerstr.14b&telephon=8932784368&" +\ "user=monica" def test_http_get(self): @@ -29,11 +29,12 @@ def test_http_get(self): """ self.render_config_template() self.run_packetbeat(pcap="http_url_params.pcap", - debug_selectors=["http", "httpdetailed"]) + debug_selectors=["*"]) objs = self.read_output() assert len(objs) == 1 o = objs[0] + print o assert o["type"] == "http" - assert len(o["params"]) > 0 - assert o["params"] == "input=packetbeat&src_ip=192.35.243.1" + assert len(o["http.request.params"]) > 0 + assert o["http.request.params"] == "input=packetbeat&src_ip=192.35.243.1" diff --git a/packetbeat/tests/system/test_0024_http_query.py b/packetbeat/tests/system/test_0024_http_query.py index d579cd557a6..c340fc09ef7 100644 --- a/packetbeat/tests/system/test_0024_http_query.py +++ b/packetbeat/tests/system/test_0024_http_query.py @@ -34,4 +34,4 @@ def test_http_get(self): o = objs[0] assert o["type"] == "http" assert o["query"] == "GET /dashboard/transactions" - assert o["params"] == "input=packetbeat&src_ip=192.35.243.1" + assert o["http.request.params"] == "input=packetbeat&src_ip=192.35.243.1" diff --git a/packetbeat/tests/system/test_0060_processors.py b/packetbeat/tests/system/test_0060_processors.py index c95800557f2..fc7984013f5 100644 --- a/packetbeat/tests/system/test_0060_processors.py +++ b/packetbeat/tests/system/test_0060_processors.py @@ -9,7 +9,7 @@ def test_drop_map_fields(self): http_send_all_headers=True, processors=[{ "drop_fields": { - "fields": ["http.request_headers"] + "fields": ["http.request.headers"] }, }] ) @@ -25,8 +25,8 @@ def test_drop_map_fields(self): assert objs[1]["status"] == "OK" assert objs[2]["status"] == "Error" - assert "http.request_headers" not in objs[0] - assert "http.response_headers" in objs[0] + assert "http.request.headers" not in objs[0] + assert "http.response.headers" in objs[0] def test_drop_fields_with_cond(self): @@ -34,8 +34,8 @@ def test_drop_fields_with_cond(self): http_send_all_headers=True, processors=[{ "drop_fields": { - "fields": ["http.request_headers", "http.response_headers"], - "when": "equals.http.code: 200", + "fields": ["http.request.headers", "http.response.headers"], + "when": "equals.http.response.code: 200", }, }] ) @@ -47,17 +47,17 @@ def test_drop_fields_with_cond(self): assert len(objs) == 3 assert all([o["type"] == "http" for o in objs]) - assert "http.request_headers" not in objs[0] - assert "http.response_headers" not in objs[0] + assert "http.request.headers" not in objs[0] + assert "http.response.headers" not in objs[0] assert "status" in objs[0] - assert "http.code" in objs[0] + assert "http.response.code" in objs[0] - assert "http.request_headers" in objs[1] - assert "http.response_headers" in objs[1] + assert "http.request.headers" in objs[1] + assert "http.response.headers" in objs[1] - assert "http.request_headers" in objs[2] - assert "http.response_headers" in objs[2] + assert "http.request.headers" in objs[2] + assert "http.response.headers" in objs[2] def test_include_fields_with_cond(self): @@ -79,13 +79,13 @@ def test_include_fields_with_cond(self): assert len(objs) == 3 assert all([o["type"] == "http" for o in objs]) - assert "http.request_headers" not in objs[0] - assert "http.response_headers" not in objs[0] + assert "http.request.headers" in objs[0] + assert "http.response.headers" in objs[0] - assert "response" not in objs[0] - assert "request" not in objs[0] + assert "response" in objs[0] + assert "request" in objs[0] - assert "http.code" in objs[0] + assert "http.response.code" in objs[0] assert "request" in objs[1] assert "response" in objs[1] @@ -101,7 +101,7 @@ def test_drop_fields_with_cond_range(self): processors=[{ "drop_fields": { "fields": ["request", "response"], - "when": "range.http.code.lt: 300", + "when": "range.http.response.code.lt: 300", }, }] ) @@ -113,11 +113,12 @@ def test_drop_fields_with_cond_range(self): assert len(objs) == 3 assert all([o["type"] == "http" for o in objs]) + print(objs[0]) assert "response" not in objs[0] assert "request" not in objs[0] assert "status" in objs[0] - assert "http.code" in objs[0] + assert "http.response.code" in objs[0] assert "request" in objs[1] assert "response" in objs[1] @@ -130,7 +131,7 @@ def test_drop_event_with_cond(self): self.render_config_template( processors=[{ "drop_event": { - "when": "range.http.code.lt: 300", + "when": "range.http.response.code.lt: 300", }, }] ) @@ -139,10 +140,11 @@ def test_drop_event_with_cond(self): debug_selectors=["http", "httpdetailed"]) objs = self.read_output(required_fields=["@timestamp", "type"]) + print(objs) assert len(objs) == 2 assert all([o["type"] == "http" for o in objs]) - assert all([o["http.code"] > 300 for o in objs]) + assert all([o["http.response.code"] > 300 for o in objs]) def test_drop_end_fields(self): @@ -150,7 +152,7 @@ def test_drop_end_fields(self): http_send_all_headers=True, processors=[{ "drop_fields": { - "fields": ["http.response_headers.transfer-encoding"] + "fields": ["http.response.headers.transfer-encoding"] }, }] ) @@ -166,12 +168,12 @@ def test_drop_end_fields(self): assert objs[1]["status"] == "OK" assert objs[2]["status"] == "Error" - assert "http.request_headers" in objs[0] - assert "http.response_headers" in objs[0] + assert "http.request.headers" in objs[0] + assert "http.response.headers" in objs[0] # check if filtering deleted the # htp.response_headers.transfer-encoding - assert "transfer-encoding" not in objs[0]["http.response_headers"] + assert "transfer-encoding" not in objs[0]["http.response.headers"] def test_drop_unknown_field(self): @@ -179,7 +181,7 @@ def test_drop_unknown_field(self): http_send_all_headers=True, processors=[{ "drop_fields": { - "fields": ["http.response_headers.transfer-encoding-test"] + "fields": ["http.response.headers.transfer-encoding-test"] }, }] ) @@ -195,12 +197,12 @@ def test_drop_unknown_field(self): assert objs[1]["status"] == "OK" assert objs[2]["status"] == "Error" - assert "http.request_headers" in objs[0] - assert "http.response_headers" in objs[0] + assert "http.request.headers" in objs[0] + assert "http.response.headers" in objs[0] # check that htp.response_headers.transfer-encoding # still exists - assert "transfer-encoding" in objs[0]["http.response_headers"] + assert "transfer-encoding" in objs[0]["http.response.headers"] def test_drop_event(self): @@ -238,8 +240,8 @@ def test_include_empty_list(self): ) assert len(objs) == 3 - assert "http.request_headers" not in objs[0] - assert "http.response_headers" not in objs[0] + assert "http.request.headers" not in objs[0] + assert "http.response.headers" not in objs[0] def test_drop_no_fields(self): self.render_config_template( @@ -272,7 +274,7 @@ def test_drop_and_include_fields_failed_cond(self): }, }, { "drop_fields": { - "fields": ["http.request_headers", "http.response_headers"], + "fields": ["http.request.headers", "http.response.headers"], "when": "equals.status: OK", }, }] @@ -287,11 +289,11 @@ def test_drop_and_include_fields_failed_cond(self): assert len(objs) == 3 assert all([o["type"] == "http" for o in objs]) - assert "http.request_headers" in objs[0] - assert "http.response_headers" in objs[0] + assert "http.request.headers" in objs[0] + assert "http.response.headers" in objs[0] - assert "http.request_headers" in objs[1] - assert "http.response_headers" in objs[1] + assert "http.request.headers" in objs[1] + assert "http.response.headers" in objs[1] def test_drop_and_include_fields(self): @@ -303,8 +305,8 @@ def test_drop_and_include_fields(self): }, }, { "drop_fields": { - "fields": ["http.request_headers", "http.response_headers"], - "when": "equals.http.code: 200", + "fields": ["http.request.headers", "http.response.headers"], + "when": "equals.http.response.code: 200", }, }] ) @@ -318,11 +320,11 @@ def test_drop_and_include_fields(self): assert len(objs) == 3 assert all([o["type"] == "http" for o in objs]) - assert "http.request_headers" not in objs[0] - assert "http.response_headers" not in objs[0] + assert "http.request.headers" not in objs[0] + assert "http.response.headers" not in objs[0] - assert "http.request_headers" in objs[1] - assert "http.response_headers" in objs[1] + assert "http.request.headers" in objs[1] + assert "http.response.headers" in objs[1] def test_condition_and(self): @@ -334,7 +336,7 @@ def test_condition_and(self): "when": """ and: - equals.type: http - - equals.http.code: 200 + - equals.http.response.code: 200 """ }, }] @@ -359,8 +361,8 @@ def test_condition_or(self): "drop_event": { "when": """ or: - - equals.http.code: 404 - - equals.http.code: 200 + - equals.http.response.code: 404 + - equals.http.response.code: 200 """ }, }] @@ -372,6 +374,7 @@ def test_condition_or(self): required_fields=["@timestamp", "type"], ) + print(objs) assert len(objs) == 1 assert all([o["type"] == "http" for o in objs]) @@ -381,7 +384,7 @@ def test_condition_not(self): http_send_all_headers=True, processors=[{ "drop_event": { - "when.not": "equals.http.code: 200", + "when.not": "equals.http.response.code: 200", }, }] ) diff --git a/packetbeat/tests/system/test_0062_http_headers.py b/packetbeat/tests/system/test_0062_http_headers.py new file mode 100644 index 00000000000..596efd433e8 --- /dev/null +++ b/packetbeat/tests/system/test_0062_http_headers.py @@ -0,0 +1,60 @@ +from packetbeat import BaseTest + +""" +Tests for checking if the headers from HTTP request/response are exported correctly. +""" + + +class Test(BaseTest): + + def test_http_send_headers(self): + """ + Check that content-length and content-type are sent even if they are not set under send_headers option. + """ + self.render_config_template( + http_send_headers=["host"], + ) + self.run_packetbeat(pcap="http_post.pcap", + debug_selectors=["http", "httpdetailed"]) + objs = self.read_output() + + assert len(objs) == 1 + o = objs[0] + print(o) + assert o["type"] == "http" + + assert len(o["http.request.headers"]) > 0 + assert "content-length" in o["http.request.headers"] + assert "content-type" in o["http.request.headers"] + + assert len(o["http.response.headers"]) > 0 + assert "content-length" in o["http.response.headers"] + assert "content-type" in o["http.response.headers"] + + def test_http_send_all_headers(self): + """ + Check that all headers are sent. + """ + self.render_config_template( + http_send_all_headers=True, + ) + self.run_packetbeat(pcap="http_post.pcap", + debug_selectors=["http", "httpdetailed"]) + objs = self.read_output() + + assert len(objs) == 1 + o = objs[0] + assert o["type"] == "http" + + assert len(o["http.request.headers"]) > 0 + assert "content-length" in o["http.request.headers"] + assert "content-type" in o["http.request.headers"] + assert "host" in o["http.request.headers"] + assert "user-agent" in o["http.request.headers"] + + assert len(o["http.response.headers"]) > 0 + assert "content-length" in o["http.response.headers"] + assert "content-type" in o["http.response.headers"] + assert "date" in o["http.response.headers"] + assert "connection" in o["http.response.headers"] + assert "server" in o["http.response.headers"] diff --git a/packetbeat/tests/system/test_0063_http_body.py b/packetbeat/tests/system/test_0063_http_body.py new file mode 100644 index 00000000000..2bcf84e54ab --- /dev/null +++ b/packetbeat/tests/system/test_0063_http_body.py @@ -0,0 +1,92 @@ +from packetbeat import BaseTest + +""" +Tests for checking if the body is exported correctly when send_body_for is set. +""" + + +class Test(BaseTest): + + def test_include_body(self): + """ + Check that the http body is exported only for some http messages that have the + content type in the list defined by include_body_for. + """ + self.render_config_template( + http_include_body_for=["x-www-form-urlencoded"], + http_send_response=True, + ) + self.run_packetbeat(pcap="http_post.pcap", + debug_selectors=["http", "httpdetailed"]) + objs = self.read_output() + + assert len(objs) == 1 + o = objs[0] + + assert o["type"] == "http" + + assert o["http.request.headers"]["content-type"] == "application/x-www-form-urlencoded" + assert o["http.response.headers"]["content-type"] == "text/html; charset=utf-8" + + assert len(o["http.request.body"]) > 0 + assert "http.response.body" not in o + + # without body + assert len(o["response"]) == 172 + + assert "request" not in o + + def test_include_body_for_both_request_response(self): + """ + Check that the http body is exported only for some http messages that have the + content type in the list defined by include_body_for. + """ + self.render_config_template( + http_include_body_for=["x-www-form-urlencoded", "text/html"], + ) + self.run_packetbeat(pcap="http_post.pcap", + debug_selectors=["http", "httpdetailed"]) + objs = self.read_output() + + assert len(objs) == 1 + o = objs[0] + + assert o["type"] == "http" + + assert o["http.request.headers"]["content-type"] == "application/x-www-form-urlencoded" + assert o["http.response.headers"]["content-type"] == "text/html; charset=utf-8" + + assert len(o["http.request.body"]) > 0 + assert len(o["http.response.body"]) > 0 + + assert "request" not in o + assert "response" not in o + + def test_wrong_content_type(self): + """ + Check if the body is exported for both request and response. + Also checks that http.request.params is exported. + """ + self.render_config_template( + http_include_body_for=["x-www-form-urlencoded", "json"], + http_ports=[80, 8080, 8000, 5000, 8002, 9200], + ) + self.run_packetbeat(pcap="http_post_json.pcap", + debug_selectors=["*"]) + objs = self.read_output() + + assert len(objs) == 1 + o = objs[0] + print o + + assert o["type"] == "http" + + assert o["http.request.headers"]["content-type"] == "application/x-www-form-urlencoded; charset=UTF-8" + assert o["http.response.headers"]["content-type"] == "application/json; charset=UTF-8" + + assert o["http.request.params"] == "%7B+%22query%22%3A+%7B+%22match_all%22%3A+%7B%7D%7D%7D%0A=" + assert len(o["http.request.body"]) > 0 + assert len(o["http.response.body"]) > 0 + + assert "request" not in o + assert "response" not in o