Support parsing TLS 1.3 supported_versions extension (#8647) (#8772)

This adds support for the new TLS version negotiation mechanism
introduced in TLS 1.3.

It relies on a new extension: `supported_versions`. When this
extension is used in the CLIENT_HELLO message, it features
a list of versions the client is willing to use:

```
"supported_versions": [
  "TLS 1.3",
  "TLS 1.2",
  "TLS 1.1",
  "TLS 1.0"
],
```

If the server supports the extension, it will pick one of the
offered versions and include it in the SERVER_HELLO message:

```
"supported_versions": "TLS 1.3",
```

The TLS parser will report a new field, `tls.version`, that is the
TLS version that has been selected after negotiation, either using
the new negotiation introduced in TLS 1.3 or the legacy negotiation
mechanism that used the version field in HELLO messages.

Updated the TLS dashboard to use the new version field:

- Server version visualization changed to TLS Version
- Client version is not useful anymore, replaced by
  tls.server_certificate.public_key_size

Fixes #8647

CHANGELOG.asciidoc

@@ -214,6 +214,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 
 *Packetbeat*
 
+- Support new TLS version negotiation introduced in TLS 1.3. {issue}8647[8647].
+
 *Winlogbeat*
 
 *Functionbeat*

packetbeat/_meta/kibana/6/dashboard/Packetbeat-tls.json

@@ -447,7 +447,7 @@
                             "id": "2", 
                             "params": {
                                 "customLabel": "TLS version", 
-                                "field": "tls.server_hello.version", 
+                                "field": "tls.version", 
                                 "order": "desc", 
                                 "orderBy": "1", 
                                 "size": 5
@@ -485,7 +485,7 @@
                     }
                 }, 
                 "savedSearchId": "8f0ff590-d37d-11e7-9914-4982455b3063", 
-                "title": "TLS Client Version", 
+                "title": "TLS Server Public Key Size", 
                 "uiStateJSON": {}, 
                 "version": 1, 
                 "visState": {
@@ -501,8 +501,8 @@
                             "enabled": true, 
                             "id": "2", 
                             "params": {
-                                "customLabel": "Client version", 
-                                "field": "tls.client_hello.version", 
+                                "customLabel": "Public Key Size", 
+                                "field": "tls.server_certificate.public_key_size", 
                                 "order": "desc", 
                                 "orderBy": "1", 
                                 "size": 5
@@ -518,7 +518,7 @@
                         "legendPosition": "right", 
                         "type": "pie"
                     }, 
-                    "title": "TLS Client Version", 
+                    "title": "Server Public Key Size", 
                     "type": "pie"
                 }
             }, 
@@ -1153,13 +1153,13 @@
                                     "store": "appState"
                                 }, 
                                 "exists": {
-                                    "field": "tls.server_hello.version"
+                                    "field": "tls.version"
                                 }, 
                                 "meta": {
                                     "alias": null, 
                                     "disabled": false, 
                                     "index": "packetbeat-*", 
-                                    "key": "tls.server_hello.version", 
+                                    "key": "tls.version", 
                                     "negate": false, 
                                     "type": "exists", 
                                     "value": "exists"
@@ -1207,13 +1207,13 @@
                                     "store": "appState"
                                 }, 
                                 "exists": {
-                                    "field": "tls.client_hello.version"
+                                    "field": "tls.server_certificate.public_key_size"
                                 }, 
                                 "meta": {
                                     "alias": null, 
                                     "disabled": false, 
                                     "index": "packetbeat-*", 
-                                    "key": "tls.client_hello.version", 
+                                    "key": "tls.server_certificate.public_key_size", 
                                     "negate": false, 
                                     "type": "exists", 
                                     "value": "exists"
@@ -1238,7 +1238,7 @@
                     "@timestamp", 
                     "desc"
                 ], 
-                "title": "TLS Client Version", 
+                "title": "Server Public Key Size", 
                 "version": 1
             }, 
             "id": "8f0ff590-d37d-11e7-9914-4982455b3063", 
@@ -1626,4 +1626,4 @@
         }
     ], 
     "version": "6.2.4"
-}
+}

packetbeat/docs/fields.asciidoc

@@ -5595,6 +5595,18 @@ TLS-specific event fields.
 
 
 
+*`tls.version`*::
++
+--
+type: keyword
+
+example: TLS 1.3
+
+The version of the TLS protocol used.
+
+
+--
+
 *`tls.handshake_completed`*::
 +
 --
@@ -5699,6 +5711,16 @@ type: keyword
 Length of the session ticket, if provided, or an empty string to advertise support for tickets.
 
 
+--
+
+*`tls.client_hello.extensions.supported_versions`*::
++
+--
+type: keyword
+
+List of TLS versions that the client is willing to use.
+
+
 --
 
 
@@ -5755,6 +5777,16 @@ type: keyword
 Used to announce that a session ticket will be provided by the server. Always an empty string.
 
 
+--
+
+*`tls.server_hello.extensions.supported_versions`*::
++
+--
+type: keyword
+
+Negotiated TLS version to be used.
+
+
 --
 
 [float]

packetbeat/protos/tls/_meta/fields.yml

@@ -6,6 +6,12 @@
     - name: tls
       type: group
       fields:
+        - name: version
+          type: keyword
+          description: >
+            The version of the TLS protocol used.
+          example: "TLS 1.3"
+
         - name: handshake_completed
           type: boolean
           description: >
@@ -69,6 +75,11 @@
                     Length of the session ticket, if provided, or an empty string
                     to advertise support for tickets.
 
+                - name: supported_versions
+                  type: keyword
+                  description: >
+                    List of TLS versions that the client is willing to use.
+
         - name: server_hello
           type: group
           fields:
@@ -105,6 +116,11 @@
                     Used to announce that a session ticket will be provided
                     by the server. Always an empty string.
 
+                - name: supported_versions
+                  type: keyword
+                  description: >
+                    Negotiated TLS version to be used.
+
         - name: client_certificate
           type: group
           description: Certificate provided by the client for authentication.

packetbeat/protos/tls/extensions.go

@@ -66,6 +66,7 @@ var extensionMap = map[uint16]extension{
 	13:     {"signature_algorithms", parseSignatureSchemes, false},
 	16:     {"application_layer_protocol_negotiation", parseALPN, false},
 	35:     {"session_ticket", parseTicket, false},
+	43:     {"supported_versions", parseSupportedVersions, false},
 	0xff01: {"renegotiation_info", ignoreContent, false},
 }
 
@@ -272,3 +273,47 @@ func parseALPN(buffer bufferView) interface{} {
 	}
 	return protos
 }
+
+func parseSupportedVersions(buffer bufferView) interface{} {
+	// Parsing the supported_versions extensions requires knowing whether the
+	// extension is included in a client_hello or server_hello, but a workaround
+	// can be done by looking at the extension length.
+
+	// Server-side extension has length 2: Selected version (2 bytes)
+	if buffer.length() == 2 {
+		var ver tlsVersion
+		if !buffer.read8(0, &ver.major) || !buffer.read8(1, &ver.minor) {
+			return nil
+		}
+		return ver.String()
+	}
+
+	// Client-side extension has at least 3 bytes: 1 byte length + 2 byte entry
+	if buffer.length() >= 3 {
+		var listBytes uint8
+		if !buffer.read8(0, &listBytes) {
+			return nil
+		}
+		if 1+int(listBytes) > buffer.length() || listBytes&1 != 0 {
+			return nil
+		}
+
+		numEntries := int(listBytes) / 2
+		if numEntries == 0 {
+			return nil
+		}
+		list := make([]string, 0, numEntries)
+		for i := 0; i < numEntries; i++ {
+			var val uint16
+			if !buffer.read16Net(1+2*i, &val) {
+				return nil
+			}
+			if !isGreaseValue(val) {
+				list = append(list, tlsVersion{major: uint8(val >> 8), minor: uint8(val & 0xff)}.String())
+			}
+		}
+		return list
+	}
+
+	return nil
+}

packetbeat/protos/tls/extensions_test.go

@@ -157,3 +157,74 @@ func TestParseSrp(t *testing.T) {
 	r = parseSrp(*mkBuf(t, "FF726f6f74", 5))
 	assert.Nil(t, r)
 }
+
+func TestParseSupportedVersions(t *testing.T) {
+	for _, testCase := range []struct {
+		title    string
+		data     string
+		expected interface{}
+	}{
+		{
+			title:    "negotiation",
+			data:     "080304030303020301",
+			expected: []string{"TLS 1.3", "TLS 1.2", "TLS 1.1", "TLS 1.0"},
+		},
+		{
+			title:    "negotiation with GREASE",
+			data:     "0c7a7a0304030303020301fafa",
+			expected: []string{"TLS 1.3", "TLS 1.2", "TLS 1.1", "TLS 1.0"},
+		},
+		{
+			title:    "selected TLS 1.3",
+			data:     "0304",
+			expected: "TLS 1.3",
+		},
+		{
+			title:    "selected future version",
+			data:     "0305",
+			expected: "TLS 1.4",
+		},
+		{
+			title: "empty error",
+			data:  "00",
+		},
+		{
+			title: "odd length error",
+			data:  "0b7a7a0304030303020301FF",
+		},
+		{
+			title: "out of bounds",
+			data:  "FF",
+		},
+		{
+			title: "out of bounds (2)",
+			data:  "805a5a03040302",
+		},
+		{
+			title:    "valid excess data",
+			data:     "0403030304FFFFFFFFFFFFFF",
+			expected: []string{"TLS 1.2", "TLS 1.3"},
+		},
+	} {
+		t.Run(testCase.title, func(t *testing.T) {
+			r := parseSupportedVersions(*mkBuf(t, testCase.data, len(testCase.data)/2))
+			if testCase.expected == nil {
+				assert.Nil(t, r, testCase.data)
+				return
+			}
+			switch v := testCase.expected.(type) {
+			case string:
+				version, ok := r.(string)
+				assert.True(t, ok)
+				assert.Equal(t, v, version)
+			case []string:
+				list, ok := r.([]string)
+				assert.True(t, ok)
+				assert.Len(t, list, len(v))
+				assert.Equal(t, v, list)
+			default:
+				assert.Fail(t, "wrong expected type", v)
+			}
+		})
+	}
+}

packetbeat/protos/tls/tls.go

@@ -358,6 +358,24 @@ func (plugin *tlsPlugin) createEvent(conn *tlsConnectionData) beat.Event {
 	if len(fingerprints) > 0 {
 		tls["fingerprints"] = fingerprints
 	}
+
+	// TLS version in use
+	if conn.handshakeCompleted > 1 {
+		var version string
+		if serverHello != nil {
+			var ok bool
+			if value, exists := serverHello.extensions.Parsed["supported_versions"]; exists {
+				version, ok = value.(string)
+			}
+			if !ok {
+				version = serverHello.version.String()
+			}
+		} else if clientHello != nil {
+			version = clientHello.version.String()
+		}
+		tls["version"] = version
+	}
+
 	fields := common.MapStr{
 		"type":   "tls",
 		"status": status,