Cherry-pick #22190 to 7.x: [Heartbeat] Add tls fields when connecting through proxy

CHANGELOG.next.asciidoc

@@ -244,6 +244,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 - Fixed scheduler shutdown issues which would in rare situations cause a panic due to semaphore misuse. {pull}16397[16397]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
+- Fixed missing `tls` fields when connecting to https via proxy. {issue}15797[15797] {pull}22190[22190]
 
 *Heartbeat*
 

heartbeat/monitors/active/dialchain/tlsmeta/tlsmeta.go

@@ -33,9 +33,14 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 )
 
+// UnknownTLSHandshakeDuration to be used in AddTLSMetadata when the duration of the TLS handshake can't be determined.
+const UnknownTLSHandshakeDuration = time.Duration(-1)
+
 func AddTLSMetadata(fields common.MapStr, connState cryptoTLS.ConnectionState, duration time.Duration) {
 	fields.Put("tls.established", true)
-	fields.Put("tls.rtt.handshake", look.RTT(duration))
+	if duration != UnknownTLSHandshakeDuration {
+		fields.Put("tls.rtt.handshake", look.RTT(duration))
+	}
 	versionDetails := tlscommon.TLSVersion(connState.Version).Details()
 	// The only situation in which versionDetails would be nil is if an unknown TLS version were to be
 	// encountered. Not filling the fields here makes sense, since there's no standard 'unknown' value.

heartbeat/monitors/active/http/http_test.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -29,6 +30,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"sync"
 	"testing"
 	"time"
 
@@ -367,6 +369,24 @@ func runHTTPSServerCheck(
 		time.Sleep(time.Millisecond * 500)
 	}
 
+	// When connecting through a proxy, the following fields are missing.
+	if _, isProxy := reqExtraConfig["proxy_url"]; isProxy {
+		missing := map[string]interface{}{
+			"http.rtt.response_header.us": time.Duration(0),
+			"http.rtt.content.us":         time.Duration(0),
+			"monitor.ip":                  "127.0.0.1",
+			"tcp.rtt.connect.us":          time.Duration(0),
+			"http.rtt.validate.us":        time.Duration(0),
+			"http.rtt.write_request.us":   time.Duration(0),
+			"tls.rtt.handshake.us":        time.Duration(0),
+		}
+		for k, v := range missing {
+			if found, err := event.Fields.HasKey(k); !found || err != nil {
+				event.Fields.Put(k, v)
+			}
+		}
+	}
+
 	testslike.Test(
 		t,
 		lookslike.Strict(lookslike.Compose(
@@ -611,3 +631,57 @@ func TestNewRoundTripper(t *testing.T) {
 	}
 
 }
+
+func TestProxy(t *testing.T) {
+	server := httptest.NewTLSServer(hbtest.HelloWorldHandler(http.StatusOK))
+	proxy := httptest.NewServer(http.HandlerFunc(httpConnectTunnel))
+	runHTTPSServerCheck(t, server, map[string]interface{}{
+		"proxy_url": proxy.URL,
+	})
+}
+
+func TestTLSProxy(t *testing.T) {
+	server := httptest.NewTLSServer(hbtest.HelloWorldHandler(http.StatusOK))
+	proxy := httptest.NewTLSServer(http.HandlerFunc(httpConnectTunnel))
+	runHTTPSServerCheck(t, server, map[string]interface{}{
+		"proxy_url": proxy.URL,
+	})
+}
+
+func httpConnectTunnel(writer http.ResponseWriter, request *http.Request) {
+	// This method is adapted from code by Michał Łowicki @mlowicki (CC BY 4.0)
+	// See https://medium.com/@mlowicki/http-s-proxy-in-golang-in-less-than-100-lines-of-code-6a51c2f2c38c
+	if request.Method != http.MethodConnect {
+		http.Error(writer, "Only CONNECT method is supported", http.StatusMethodNotAllowed)
+		return
+	}
+	destConn, err := net.DialTimeout("tcp", request.Host, 10*time.Second)
+	if err != nil {
+		http.Error(writer, err.Error(), http.StatusServiceUnavailable)
+		return
+	}
+	writer.WriteHeader(http.StatusOK)
+	hijacker, ok := writer.(http.Hijacker)
+	if !ok {
+		http.Error(writer, "Hijacking not supported", http.StatusInternalServerError)
+		return
+	}
+	clientConn, clientReadWriter, err := hijacker.Hijack()
+	if err != nil {
+		http.Error(writer, err.Error(), http.StatusServiceUnavailable)
+	}
+	defer destConn.Close()
+	defer clientConn.Close()
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		io.Copy(destConn, clientReadWriter)
+		wg.Done()
+	}()
+	go func() {
+		io.Copy(clientConn, destConn)
+		wg.Done()
+	}()
+	wg.Wait()
+}

heartbeat/monitors/active/http/task.go

@@ -275,6 +275,14 @@ func execPing(
 	// Mark the end time as now, since we've finished downloading
 	end = time.Now()
 
+	// Enrich event with TLS information when available. This is useful when connecting to an HTTPS server through
+	// a proxy.
+	if resp.TLS != nil {
+		tlsFields := common.MapStr{}
+		tlsmeta.AddTLSMetadata(tlsFields, *resp.TLS, tlsmeta.UnknownTLSHandshakeDuration)
+		eventext.MergeEventFields(event, tlsFields)
+	}
+
 	// Add total HTTP RTT
 	eventext.MergeEventFields(event, common.MapStr{"http": common.MapStr{
 		"rtt": common.MapStr{