From 3887484016d0074a833e0d2b5079af1a7c8e7474 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 3 Dec 2020 08:21:24 +0100 Subject: [PATCH] [Heartbeat] Add tls fields when connecting through proxy (#22190) This updates Heartbeat to enrich an event with TLS information when the connection has been established via an HTTP proxy. Closes #15797 (cherry picked from commit cd16ca0fc539b859909c594d86d5b151b8c29605) --- CHANGELOG.next.asciidoc | 1 + .../active/dialchain/tlsmeta/tlsmeta.go | 7 +- heartbeat/monitors/active/http/http_test.go | 74 +++++++++++++++++++ heartbeat/monitors/active/http/task.go | 8 ++ 4 files changed, 89 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f6ed8fc48ddb..ab509a4df815 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -244,6 +244,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639] - Fixed scheduler shutdown issues which would in rare situations cause a panic due to semaphore misuse. {pull}16397[16397] - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549] +- Fixed missing `tls` fields when connecting to https via proxy. {issue}15797[15797] {pull}22190[22190] *Heartbeat* diff --git a/heartbeat/monitors/active/dialchain/tlsmeta/tlsmeta.go b/heartbeat/monitors/active/dialchain/tlsmeta/tlsmeta.go index 686da2a3241d..f780e903317a 100644 --- a/heartbeat/monitors/active/dialchain/tlsmeta/tlsmeta.go +++ b/heartbeat/monitors/active/dialchain/tlsmeta/tlsmeta.go @@ -33,9 +33,14 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) +// UnknownTLSHandshakeDuration to be used in AddTLSMetadata when the duration of the TLS handshake can't be determined. +const UnknownTLSHandshakeDuration = time.Duration(-1) + func AddTLSMetadata(fields common.MapStr, connState cryptoTLS.ConnectionState, duration time.Duration) { fields.Put("tls.established", true) - fields.Put("tls.rtt.handshake", look.RTT(duration)) + if duration != UnknownTLSHandshakeDuration { + fields.Put("tls.rtt.handshake", look.RTT(duration)) + } versionDetails := tlscommon.TLSVersion(connState.Version).Details() // The only situation in which versionDetails would be nil is if an unknown TLS version were to be // encountered. Not filling the fields here makes sense, since there's no standard 'unknown' value. diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index a0bf14e73c9e..e0e09b761525 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -29,6 +30,7 @@ import ( "os" "path" "reflect" + "sync" "testing" "time" @@ -367,6 +369,24 @@ func runHTTPSServerCheck( time.Sleep(time.Millisecond * 500) } + // When connecting through a proxy, the following fields are missing. + if _, isProxy := reqExtraConfig["proxy_url"]; isProxy { + missing := map[string]interface{}{ + "http.rtt.response_header.us": time.Duration(0), + "http.rtt.content.us": time.Duration(0), + "monitor.ip": "127.0.0.1", + "tcp.rtt.connect.us": time.Duration(0), + "http.rtt.validate.us": time.Duration(0), + "http.rtt.write_request.us": time.Duration(0), + "tls.rtt.handshake.us": time.Duration(0), + } + for k, v := range missing { + if found, err := event.Fields.HasKey(k); !found || err != nil { + event.Fields.Put(k, v) + } + } + } + testslike.Test( t, lookslike.Strict(lookslike.Compose( @@ -611,3 +631,57 @@ func TestNewRoundTripper(t *testing.T) { } } + +func TestProxy(t *testing.T) { + server := httptest.NewTLSServer(hbtest.HelloWorldHandler(http.StatusOK)) + proxy := httptest.NewServer(http.HandlerFunc(httpConnectTunnel)) + runHTTPSServerCheck(t, server, map[string]interface{}{ + "proxy_url": proxy.URL, + }) +} + +func TestTLSProxy(t *testing.T) { + server := httptest.NewTLSServer(hbtest.HelloWorldHandler(http.StatusOK)) + proxy := httptest.NewTLSServer(http.HandlerFunc(httpConnectTunnel)) + runHTTPSServerCheck(t, server, map[string]interface{}{ + "proxy_url": proxy.URL, + }) +} + +func httpConnectTunnel(writer http.ResponseWriter, request *http.Request) { + // This method is adapted from code by Michał Łowicki @mlowicki (CC BY 4.0) + // See https://medium.com/@mlowicki/http-s-proxy-in-golang-in-less-than-100-lines-of-code-6a51c2f2c38c + if request.Method != http.MethodConnect { + http.Error(writer, "Only CONNECT method is supported", http.StatusMethodNotAllowed) + return + } + destConn, err := net.DialTimeout("tcp", request.Host, 10*time.Second) + if err != nil { + http.Error(writer, err.Error(), http.StatusServiceUnavailable) + return + } + writer.WriteHeader(http.StatusOK) + hijacker, ok := writer.(http.Hijacker) + if !ok { + http.Error(writer, "Hijacking not supported", http.StatusInternalServerError) + return + } + clientConn, clientReadWriter, err := hijacker.Hijack() + if err != nil { + http.Error(writer, err.Error(), http.StatusServiceUnavailable) + } + defer destConn.Close() + defer clientConn.Close() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + io.Copy(destConn, clientReadWriter) + wg.Done() + }() + go func() { + io.Copy(clientConn, destConn) + wg.Done() + }() + wg.Wait() +} diff --git a/heartbeat/monitors/active/http/task.go b/heartbeat/monitors/active/http/task.go index 305c7f044af8..39687172931c 100644 --- a/heartbeat/monitors/active/http/task.go +++ b/heartbeat/monitors/active/http/task.go @@ -275,6 +275,14 @@ func execPing( // Mark the end time as now, since we've finished downloading end = time.Now() + // Enrich event with TLS information when available. This is useful when connecting to an HTTPS server through + // a proxy. + if resp.TLS != nil { + tlsFields := common.MapStr{} + tlsmeta.AddTLSMetadata(tlsFields, *resp.TLS, tlsmeta.UnknownTLSHandshakeDuration) + eventext.MergeEventFields(event, tlsFields) + } + // Add total HTTP RTT eventext.MergeEventFields(event, common.MapStr{"http": common.MapStr{ "rtt": common.MapStr{