Added dns.question.top_level_domain and dns.question.subdomain ECS fi…

…elds (#14578) (#17127) Added the following fields to packetbeat dns.question.subdomain dns.question.top_level_domain Co-authored-by: Adrian Serrano <adrisr83@gmail.com> Co-authored-by: mbudge <mbudge1@gmail.com> (cherry picked from commit bd1d277)
elastic · Mar 20, 2020 · a2c7d21 · a2c7d21
1 parent 82e5090
commit a2c7d21
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 19 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - TLS: Fields have been changed to adapt to ECS. {pull}15497[15497]
 - TLS: The behavior of send_certificates and include_raw_certificates options has changed. {pull}15497[15497]
 - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
+- Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
 
 *Winlogbeat*
 

diff --git a/packetbeat/protos/dns/dns.go b/packetbeat/protos/dns/dns.go
@@ -497,10 +497,28 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional
 		m["question"] = qMapStr
 
 		eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(q.Name)
-		if err == nil {
+		if err == nil && eTLDPlusOne != "" {
+			eTLDPlusOne = strings.TrimRight(eTLDPlusOne, ".")
+
 			// etld_plus_one should be removed for 8.0.0.
 			qMapStr["etld_plus_one"] = eTLDPlusOne
 			qMapStr["registered_domain"] = eTLDPlusOne
+
+			if idx := strings.IndexByte(eTLDPlusOne, '.'); idx != -1 {
+				qMapStr["top_level_domain"] = eTLDPlusOne[idx+1:]
+			}
+
+			subdomain := strings.TrimRight(strings.TrimSuffix(q.Name, eTLDPlusOne), ".")
+			if subdomain != "" {
+				qMapStr["subdomain"] = subdomain
+			}
+		} else if strings.Count(q.Name, ".") == 1 {
+			// Handle publicsuffix.EffectiveTLDPlusOne eTLD+1 error with 1 dot in the domain.
+			s := strings.Split(q.Name, ".")
+			if len(s) == 2 && s[1] != "" {
+				qMapStr["top_level_domain"] = s[1]
+			}
+			qMapStr["registered_domain"] = q.Name
 		}
 	}
 

diff --git a/packetbeat/protos/dns/dns_tcp_test.go b/packetbeat/protos/dns/dns_tcp_test.go
@@ -58,6 +58,7 @@ var (
 		qType:       "A",
 		qName:       "elastic.co",
 		qEtld:       "elastic.co",
+		qTLD:        "co",
 		answers:     []string{"54.201.204.244", "54.200.185.88"},
 		authorities: []string{"NS-835.AWSDNS-40.NET", "NS-1183.AWSDNS-19.ORG", "NS-2007.AWSDNS-58.CO.UK", "NS-66.AWSDNS-08.COM"},
 		request: []byte{
@@ -89,6 +90,7 @@ var (
 		qType:   "AXFR",
 		qName:   "etas.com",
 		qEtld:   "etas.com",
+		qTLD:    "com",
 		answers: []string{"training2003p", "training2003p", "1.1.1.1", "training2003p"},
 		request: []byte{
 			0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x65,
@@ -120,6 +122,8 @@ var (
 		qType:       "PTR",
 		qName:       "131.252.30.192.in-addr.arpa",
 		qEtld:       "192.in-addr.arpa",
+		qSubdomain:  "131.252.30",
+		qTLD:        "in-addr.arpa",
 		answers:     []string{"github.com"},
 		authorities: []string{"ns1.p16.dynect.net", "ns3.p16.dynect.net", "ns4.p16.dynect.net", "ns2.p16.dynect.net"},
 		request: []byte{
@@ -152,6 +156,10 @@ var (
 			"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
 			"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net",
 		qEtld: "sophosxl.net",
+		qSubdomain: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" +
+			"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
+			"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac",
+		qTLD: "net",
 		request: []byte{
 			0x00, 0xed, 0x88, 0xc1, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33,
 			0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34,

diff --git a/packetbeat/protos/dns/dns_test.go b/packetbeat/protos/dns/dns_test.go
@@ -57,6 +57,8 @@ type dnsTestMessage struct {
 	qType       string
 	qName       string
 	qEtld       string
+	qSubdomain  interface{}
+	qTLD        interface{}
 	answers     []string
 	authorities []string
 	additionals []string
@@ -263,6 +265,8 @@ func assertRequest(t testing.TB, m common.MapStr, q dnsTestMessage) {
 	assert.Equal(t, q.qClass, mapValue(t, m, "dns.question.class"))
 	assert.Equal(t, q.qType, mapValue(t, m, "dns.question.type"))
 	assert.Equal(t, q.qName, mapValue(t, m, "dns.question.name"))
+	assert.Equal(t, q.qTLD, mapValue(t, m, "dns.question.top_level_domain"))
+	assert.Equal(t, q.qSubdomain, mapValue(t, m, "dns.question.subdomain"))
 	assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.etld_plus_one"))
 	assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.registered_domain"))
 }

diff --git a/packetbeat/protos/dns/dns_udp_test.go b/packetbeat/protos/dns/dns_udp_test.go
@@ -68,6 +68,7 @@ var (
 		qType:   "A",
 		qName:   "elastic.co",
 		qEtld:   "elastic.co",
+		qTLD:    "co",
 		answers: []string{"54.148.130.30", "54.69.104.66"},
 		request: []byte{
 			0x21, 0x51, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x65, 0x6c, 0x61,
@@ -90,6 +91,7 @@ var (
 		qType:   "IXFR",
 		qName:   "etas.com",
 		qEtld:   "etas.com",
+		qTLD:    "com",
 		answers: []string{"training2003p", "training2003p", "training2003p", "training2003p", "1.1.1.100"},
 		request: []byte{
 			0x40, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x04, 0x65, 0x74, 0x61,
@@ -118,15 +120,17 @@ var (
 	}
 
 	githubPtr = dnsTestMessage{
-		id:      344,
-		opcode:  "QUERY",
-		flags:   []string{"rd", "ra"},
-		rcode:   "NOERROR",
-		qClass:  "IN",
-		qType:   "PTR",
-		qName:   "131.252.30.192.in-addr.arpa",
-		qEtld:   "192.in-addr.arpa",
-		answers: []string{"github.com"},
+		id:         344,
+		opcode:     "QUERY",
+		flags:      []string{"rd", "ra"},
+		rcode:      "NOERROR",
+		qClass:     "IN",
+		qType:      "PTR",
+		qName:      "131.252.30.192.in-addr.arpa",
+		qEtld:      "192.in-addr.arpa",
+		qSubdomain: "131.252.30",
+		qTLD:       "in-addr.arpa",
+		answers:    []string{"github.com"},
 		authorities: []string{"a.root-servers.net", "b.root-servers.net", "c.root-servers.net",
 			"d.root-servers.net", "e.root-servers.net", "f.root-servers.net", "g.root-servers.net",
 			"h.root-servers.net", "i.root-servers.net", "j.root-servers.net", "k.root-servers.net",
@@ -169,6 +173,10 @@ var (
 			"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
 			"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net",
 		qEtld: "sophosxl.net",
+		qSubdomain: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" +
+			"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
+			"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac",
+		qTLD: "net",
 		request: []byte{
 			0x20, 0x2e, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, 0x3f, 0x31,
 			0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, 0x71, 0x70,
@@ -206,15 +214,17 @@ var (
 	}
 
 	ednsSecA = dnsTestMessage{
-		id:      20498,
-		opcode:  "QUERY",
-		flags:   []string{"rd", "ad", "ra"},
-		rcode:   "NOERROR",
-		qClass:  "IN",
-		qType:   "A",
-		qName:   "www.ietf.org",
-		qEtld:   "ietf.org",
-		answers: []string{"64.170.98.30", "iDA8bJnrAEz3jgYnyFRm567a76qlv1V0CqxOSd/o9nvnN0GlZLaVoDmuXpaIaoypbGxwzwgK/LY6CV2k6SWKwicBmpENL26hwyjkFzPDW8kX3ibFhtfsOb8pYe7nBj326actp/7iG+DRuDmPnkYBja+wDYk61doTtkqZg57fn3iS97tjNPCC9C9knRAuDYUG+dVxalazSwYrpvY97dUC1H2spD0g4UdDyCbGA46mouZ4GPzNMewgf948qxrnU8pWPk3nQW5TgLVkGoWgco2owfLElBqf6rJ4LOswuhaw8IpTtmw3FsixxTLQvKOE5nftd1nMhQQd9CaHjoKNAUEz5Q=="},
+		id:         20498,
+		opcode:     "QUERY",
+		flags:      []string{"rd", "ad", "ra"},
+		rcode:      "NOERROR",
+		qClass:     "IN",
+		qType:      "A",
+		qName:      "www.ietf.org",
+		qEtld:      "ietf.org",
+		qTLD:       "org",
+		qSubdomain: "www",
+		answers:    []string{"64.170.98.30", "iDA8bJnrAEz3jgYnyFRm567a76qlv1V0CqxOSd/o9nvnN0GlZLaVoDmuXpaIaoypbGxwzwgK/LY6CV2k6SWKwicBmpENL26hwyjkFzPDW8kX3ibFhtfsOb8pYe7nBj326actp/7iG+DRuDmPnkYBja+wDYk61doTtkqZg57fn3iS97tjNPCC9C9knRAuDYUG+dVxalazSwYrpvY97dUC1H2spD0g4UdDyCbGA46mouZ4GPzNMewgf948qxrnU8pWPk3nQW5TgLVkGoWgco2owfLElBqf6rJ4LOswuhaw8IpTtmw3FsixxTLQvKOE5nftd1nMhQQd9CaHjoKNAUEz5Q=="},
 		request: []byte{
 			0x50, 0x12, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x77, 0x77, 0x77,
 			0x04, 0x69, 0x65, 0x74, 0x66, 0x03, 0x6f, 0x72, 0x67, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00,