From bbc1e8fa7ab18bcc65d547f1f3a448c2e154ee86 Mon Sep 17 00:00:00 2001 From: OhBonsai Date: Tue, 23 Jun 2020 23:52:32 +0800 Subject: [PATCH 1/6] refactor(packet beat): Improve support for 100-continue --- packetbeat/protos/http/http.go | 6 ++++ packetbeat/protos/http/http_parser.go | 7 ++-- .../tests/system/pcaps/http_100_continue.pcap | Bin 0 -> 6075 bytes .../system/test_0070_http_100_continue.py | 34 ++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 packetbeat/tests/system/pcaps/http_100_continue.pcap create mode 100644 packetbeat/tests/system/test_0070_http_100_continue.py diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index efa344ab163b..2a09982a713f 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -457,6 +457,12 @@ func (http *httpPlugin) flushResponses(conn *httpConnectionData) { unmatchedResponses.Add(1) resp := conn.responses.pop() debugf("Response from unknown transaction: %s. Reporting error.", resp.tcpTuple) + + if bytes.Equal(resp.statusPhrase, continueStatePhrase) && resp.statusCode == 100 { + debugf("Drop first 100-continue response") + return + } + event := http.newTransaction(nil, resp) http.publishTransaction(event) } diff --git a/packetbeat/protos/http/http_parser.go b/packetbeat/protos/http/http_parser.go index 748ea9dc712b..4f0342e9fc19 100644 --- a/packetbeat/protos/http/http_parser.go +++ b/packetbeat/protos/http/http_parser.go @@ -110,9 +110,10 @@ var ( constCRLF = []byte("\r\n") - constClose = []byte("close") - constKeepAlive = []byte("keep-alive") - constHTTPVersion = []byte("HTTP/") + constClose = []byte("close") + constKeepAlive = []byte("keep-alive") + constHTTPVersion = []byte("HTTP/") + continueStatePhrase = []byte("Continue") nameContentLength = []byte("content-length") nameContentType = []byte("content-type") diff --git a/packetbeat/tests/system/pcaps/http_100_continue.pcap b/packetbeat/tests/system/pcaps/http_100_continue.pcap new file mode 100644 index 0000000000000000000000000000000000000000..be1438e3080030cc2559e2a2e24c2e35fdcfd679 GIT binary patch literal 6075 zcmb`Ldr(wW9LMkOA{KZ85d}eQyL@0TT1~hv7o?V5*jNZ{xlyoJ#?C0D$SHNnOb4f1BdZ3O*1O$r0HPa-(Bvq_wMQ*HO8N_XV0GR z_jf+$ckVs+o;>`)+h!p~82Sr*_U86KvphRNSb+|T35Zn>PVW=)Vr6^`{WFC&@Y&SO zB=%eiTTb8aH{ZG?6bphmX7&-2#S&{WJrI}GHFX~K#h3+wd^+GGqfHU?NsjI#v2KPT z0=!mKCF8@mfL`pR1fc>=3Rno0sV`Df9s<~R_@vFCOz%mprvNfMjCZrwF4$MAZz=To1?QIHJvNIrZVlSl_%*F!lvp%=9tM@ekk&CNA5 zJ~BsrjvJxL!EqGnL3fqL(T{bFZ#UFq{LfEwaZHxjp1xmqYdMBd}1g z1{cRM5_`wK`x7n4aE`OoI?_1)0v{ParcgPO4M{QJ549Y_Im#C5BaY;A13og^6oGn# z4Nj)L<5-Dp@~z%W<0Es_=lCi#IXI3bJ?QQW<+wM^P>+_S9bEO8 zCb9Ktr&ee=hI1^tOnn;1@8Kh(O`&omA3lz07$U%HWhlprbdGvVmzewJmxpP5WRChA zmqU|-qk6r9?h%dSTCD3It1bHb^|0CBaB&=$anZV@DY$3CIhL!}CE`dvWln?72yr%# zPsYXWak*P;b?|E6pA_kQRo;`;RowF zvRZNYZ8fzXr>(IK|6Au(S6A5zCl|7~c^;qNZgqK_4%Zs`SXxw2z~Ywq6mNc6z2Zin zv&HMOO`ANWcrrT5oKB?)U*_0ySX^amlj209u%IB{>2doT+${==n}bgnm|xx2q}Z*F zrY2W|)8TLMxNXn+Jno_Xg^Ii0zs7DYVsR{v5Ffs8#LbyNd^*zdUEXLB6yT*$5!3RO zMn^ofe9%sXc3BW{FO{AKk{DQN<^cwQ38inkpL%T0L_KaW)C`&S609B2UWCa|1Zzf& zKA~>4)(5qzW^!~B>h0(x^>>Y#-(9tDur6uNHY`Oj^Yuq9!d&b@3T-J&>CrPET#A4; zmco<~%)Fnd1eXF8Y+by7#ZgI8A^Ia0;&iKXjl=6x{4T^3F)rpoLP zYZ~0ON~>rWwVfp9i8UmOuV_zEv!NCZb#GB?t>X2e;6;03L1~`oReUWjKl)pmY8`&1 zmVV)bcdbvfcZhzE-{BJNMR}sH1)Iy~t7}2CFi)&=VBN2yOccr?OC zMw>$AXn_wOM=OR1@LHf96p3Tf!yE_2i4r@pHRS+}k9<(n=U4$v4vrH^54s06j_Z(5 z_EtkZCLC9|IEoT0I(B-SmSZ@_xt~)XaU`EP@R8A`P&twhA4d^G1bA)NIFjBM(2H7+ z8F=~BdEpK(cJw)3fhGq>+??>Dsq;ec#SY~&J=0K+?AxcfI7$+`dj8-Y9$X&ZLw*`Z zAADrADO8T+!^cs=5CLAn2N%k_2YOLCW@68bJN6@uk6e%X9M3?LgJUM?LHD>;kFChZ zWHSp-;Cl))0xv#s4w7{*DX;A6y}IA>+s3ypz5E(ZZs!Newd2D4@~kfL-&y4=t1IU$ zUbd*(URpdA|MH3-ol-gt2LW;@f`>n&k;X?}aeaisl4JlL literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/test_0070_http_100_continue.py b/packetbeat/tests/system/test_0070_http_100_continue.py new file mode 100644 index 000000000000..4e8d48db26d9 --- /dev/null +++ b/packetbeat/tests/system/test_0070_http_100_continue.py @@ -0,0 +1,34 @@ +from packetbeat import BaseTest + +""" +Tests for checking expect 100-continue only generate 1 event +""" + + +class Test(BaseTest): + + def test_http_100_continue(self): + """ + Should only generate one event + """ + self.render_config_template( + iface_device="lo0", + http_ports=["9200"], + http_send_all_headers=True + ) + self.run_packetbeat(pcap="http_100_continue.pcap") + objs = self.read_output_json() + + assert len(objs) == 1 + o = objs[0] + + assert o["type"] == "http" + assert "request" in o["http"] + assert "headers" in o["http"]["request"] + assert o["http"]["request"]["headers"]["expect"] == "100-continue" + + assert "response" in o["http"] + + assert not "error" in o + + From e2482e0bca4bbfb32979b97262cd0a5392b1d4de Mon Sep 17 00:00:00 2001 From: OhBonsai Date: Fri, 17 Jul 2020 16:49:32 +0800 Subject: [PATCH 2/6] test(packetbeat): 100-continue only generate one event without error --- packetbeat/tests/system/test_0070_http_100_continue.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packetbeat/tests/system/test_0070_http_100_continue.py b/packetbeat/tests/system/test_0070_http_100_continue.py index 4e8d48db26d9..877bb90a2806 100644 --- a/packetbeat/tests/system/test_0070_http_100_continue.py +++ b/packetbeat/tests/system/test_0070_http_100_continue.py @@ -30,5 +30,3 @@ def test_http_100_continue(self): assert "response" in o["http"] assert not "error" in o - - From 178538d239d547a28cf9be7a974fe7feefb14823 Mon Sep 17 00:00:00 2001 From: OhBonsai Date: Fri, 17 Jul 2020 17:00:03 +0800 Subject: [PATCH 3/6] test(packetbeat): 100-continue only generate one event without error --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 224aada05d00..261c762f9bea 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -509,6 +509,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Packetbeat* - Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167] +- Add 100-continue support {issue}15830[15830] {pull}19349[19349] + *Functionbeat* - Add basic ECS categorization and `cloud` fields. {pull}19174[19174] From 7252e4c82ec5d839e127e696eccaf8f9083adcb7 Mon Sep 17 00:00:00 2001 From: Bonsai Date: Thu, 23 Jul 2020 11:33:51 +0800 Subject: [PATCH 4/6] Update packetbeat/protos/http/http.go Co-authored-by: Adrian Serrano --- packetbeat/protos/http/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index 2a09982a713f..4b2367c02397 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -458,7 +458,7 @@ func (http *httpPlugin) flushResponses(conn *httpConnectionData) { resp := conn.responses.pop() debugf("Response from unknown transaction: %s. Reporting error.", resp.tcpTuple) - if bytes.Equal(resp.statusPhrase, continueStatePhrase) && resp.statusCode == 100 { + if resp.statusCode == 100 { debugf("Drop first 100-continue response") return } From 9a9b921694a0ae00af38ff09b4e9606a70bac8d2 Mon Sep 17 00:00:00 2001 From: Bonsai Date: Thu, 23 Jul 2020 11:35:10 +0800 Subject: [PATCH 5/6] delete unused string --- packetbeat/protos/http/http_parser.go | 1 - 1 file changed, 1 deletion(-) diff --git a/packetbeat/protos/http/http_parser.go b/packetbeat/protos/http/http_parser.go index 4f0342e9fc19..9209b12df18f 100644 --- a/packetbeat/protos/http/http_parser.go +++ b/packetbeat/protos/http/http_parser.go @@ -113,7 +113,6 @@ var ( constClose = []byte("close") constKeepAlive = []byte("keep-alive") constHTTPVersion = []byte("HTTP/") - continueStatePhrase = []byte("Continue") nameContentLength = []byte("content-length") nameContentType = []byte("content-type") From 651f9f71df998f2d02dd2a79b0dd3e732d541a3a Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 23 Jul 2020 09:41:10 +0200 Subject: [PATCH 6/6] Fix format issue --- packetbeat/protos/http/http_parser.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packetbeat/protos/http/http_parser.go b/packetbeat/protos/http/http_parser.go index 9209b12df18f..748ea9dc712b 100644 --- a/packetbeat/protos/http/http_parser.go +++ b/packetbeat/protos/http/http_parser.go @@ -110,9 +110,9 @@ var ( constCRLF = []byte("\r\n") - constClose = []byte("close") - constKeepAlive = []byte("keep-alive") - constHTTPVersion = []byte("HTTP/") + constClose = []byte("close") + constKeepAlive = []byte("keep-alive") + constHTTPVersion = []byte("HTTP/") nameContentLength = []byte("content-length") nameContentType = []byte("content-type")