From a2c7d213e569be140e6d3c6c9de5003f6b2f86ae Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 20 Mar 2020 10:12:28 -0400 Subject: [PATCH] Added dns.question.top_level_domain and dns.question.subdomain ECS fields (#14578) (#17127) Added the following fields to packetbeat dns.question.subdomain dns.question.top_level_domain Co-authored-by: Adrian Serrano Co-authored-by: mbudge (cherry picked from commit bd1d2772d81d0b10923bd00e541a708471dafea5) --- CHANGELOG.next.asciidoc | 1 + packetbeat/protos/dns/dns.go | 20 +++++++++++- packetbeat/protos/dns/dns_tcp_test.go | 8 +++++ packetbeat/protos/dns/dns_test.go | 4 +++ packetbeat/protos/dns/dns_udp_test.go | 46 ++++++++++++++++----------- 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e7961e4e454..5cf98b41f5b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - TLS: Fields have been changed to adapt to ECS. {pull}15497[15497] - TLS: The behavior of send_certificates and include_raw_certificates options has changed. {pull}15497[15497] - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353] +- Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578] *Winlogbeat* diff --git a/packetbeat/protos/dns/dns.go b/packetbeat/protos/dns/dns.go index dc0227815b8..29efcc9bcdc 100644 --- a/packetbeat/protos/dns/dns.go +++ b/packetbeat/protos/dns/dns.go @@ -497,10 +497,28 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional m["question"] = qMapStr eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(q.Name) - if err == nil { + if err == nil && eTLDPlusOne != "" { + eTLDPlusOne = strings.TrimRight(eTLDPlusOne, ".") + // etld_plus_one should be removed for 8.0.0. qMapStr["etld_plus_one"] = eTLDPlusOne qMapStr["registered_domain"] = eTLDPlusOne + + if idx := strings.IndexByte(eTLDPlusOne, '.'); idx != -1 { + qMapStr["top_level_domain"] = eTLDPlusOne[idx+1:] + } + + subdomain := strings.TrimRight(strings.TrimSuffix(q.Name, eTLDPlusOne), ".") + if subdomain != "" { + qMapStr["subdomain"] = subdomain + } + } else if strings.Count(q.Name, ".") == 1 { + // Handle publicsuffix.EffectiveTLDPlusOne eTLD+1 error with 1 dot in the domain. + s := strings.Split(q.Name, ".") + if len(s) == 2 && s[1] != "" { + qMapStr["top_level_domain"] = s[1] + } + qMapStr["registered_domain"] = q.Name } } diff --git a/packetbeat/protos/dns/dns_tcp_test.go b/packetbeat/protos/dns/dns_tcp_test.go index 18e499afd38..826ba451c32 100644 --- a/packetbeat/protos/dns/dns_tcp_test.go +++ b/packetbeat/protos/dns/dns_tcp_test.go @@ -58,6 +58,7 @@ var ( qType: "A", qName: "elastic.co", qEtld: "elastic.co", + qTLD: "co", answers: []string{"54.201.204.244", "54.200.185.88"}, authorities: []string{"NS-835.AWSDNS-40.NET", "NS-1183.AWSDNS-19.ORG", "NS-2007.AWSDNS-58.CO.UK", "NS-66.AWSDNS-08.COM"}, request: []byte{ @@ -89,6 +90,7 @@ var ( qType: "AXFR", qName: "etas.com", qEtld: "etas.com", + qTLD: "com", answers: []string{"training2003p", "training2003p", "1.1.1.1", "training2003p"}, request: []byte{ 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x65, @@ -120,6 +122,8 @@ var ( qType: "PTR", qName: "131.252.30.192.in-addr.arpa", qEtld: "192.in-addr.arpa", + qSubdomain: "131.252.30", + qTLD: "in-addr.arpa", answers: []string{"github.com"}, authorities: []string{"ns1.p16.dynect.net", "ns3.p16.dynect.net", "ns4.p16.dynect.net", "ns2.p16.dynect.net"}, request: []byte{ @@ -152,6 +156,10 @@ var ( "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net", qEtld: "sophosxl.net", + qSubdomain: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" + + "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac", + qTLD: "net", request: []byte{ 0x00, 0xed, 0x88, 0xc1, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, 0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, diff --git a/packetbeat/protos/dns/dns_test.go b/packetbeat/protos/dns/dns_test.go index db20a19126f..c5ee52eb5eb 100644 --- a/packetbeat/protos/dns/dns_test.go +++ b/packetbeat/protos/dns/dns_test.go @@ -57,6 +57,8 @@ type dnsTestMessage struct { qType string qName string qEtld string + qSubdomain interface{} + qTLD interface{} answers []string authorities []string additionals []string @@ -263,6 +265,8 @@ func assertRequest(t testing.TB, m common.MapStr, q dnsTestMessage) { assert.Equal(t, q.qClass, mapValue(t, m, "dns.question.class")) assert.Equal(t, q.qType, mapValue(t, m, "dns.question.type")) assert.Equal(t, q.qName, mapValue(t, m, "dns.question.name")) + assert.Equal(t, q.qTLD, mapValue(t, m, "dns.question.top_level_domain")) + assert.Equal(t, q.qSubdomain, mapValue(t, m, "dns.question.subdomain")) assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.etld_plus_one")) assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.registered_domain")) } diff --git a/packetbeat/protos/dns/dns_udp_test.go b/packetbeat/protos/dns/dns_udp_test.go index 3a0ca655505..fb482b55676 100644 --- a/packetbeat/protos/dns/dns_udp_test.go +++ b/packetbeat/protos/dns/dns_udp_test.go @@ -68,6 +68,7 @@ var ( qType: "A", qName: "elastic.co", qEtld: "elastic.co", + qTLD: "co", answers: []string{"54.148.130.30", "54.69.104.66"}, request: []byte{ 0x21, 0x51, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x65, 0x6c, 0x61, @@ -90,6 +91,7 @@ var ( qType: "IXFR", qName: "etas.com", qEtld: "etas.com", + qTLD: "com", answers: []string{"training2003p", "training2003p", "training2003p", "training2003p", "1.1.1.100"}, request: []byte{ 0x40, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x04, 0x65, 0x74, 0x61, @@ -118,15 +120,17 @@ var ( } githubPtr = dnsTestMessage{ - id: 344, - opcode: "QUERY", - flags: []string{"rd", "ra"}, - rcode: "NOERROR", - qClass: "IN", - qType: "PTR", - qName: "131.252.30.192.in-addr.arpa", - qEtld: "192.in-addr.arpa", - answers: []string{"github.com"}, + id: 344, + opcode: "QUERY", + flags: []string{"rd", "ra"}, + rcode: "NOERROR", + qClass: "IN", + qType: "PTR", + qName: "131.252.30.192.in-addr.arpa", + qEtld: "192.in-addr.arpa", + qSubdomain: "131.252.30", + qTLD: "in-addr.arpa", + answers: []string{"github.com"}, authorities: []string{"a.root-servers.net", "b.root-servers.net", "c.root-servers.net", "d.root-servers.net", "e.root-servers.net", "f.root-servers.net", "g.root-servers.net", "h.root-servers.net", "i.root-servers.net", "j.root-servers.net", "k.root-servers.net", @@ -169,6 +173,10 @@ var ( "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net", qEtld: "sophosxl.net", + qSubdomain: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" + + "648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" + + "1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac", + qTLD: "net", request: []byte{ 0x20, 0x2e, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, 0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, 0x71, 0x70, @@ -206,15 +214,17 @@ var ( } ednsSecA = dnsTestMessage{ - id: 20498, - opcode: "QUERY", - flags: []string{"rd", "ad", "ra"}, - rcode: "NOERROR", - qClass: "IN", - qType: "A", - qName: "www.ietf.org", - qEtld: "ietf.org", - answers: []string{"64.170.98.30", "iDA8bJnrAEz3jgYnyFRm567a76qlv1V0CqxOSd/o9nvnN0GlZLaVoDmuXpaIaoypbGxwzwgK/LY6CV2k6SWKwicBmpENL26hwyjkFzPDW8kX3ibFhtfsOb8pYe7nBj326actp/7iG+DRuDmPnkYBja+wDYk61doTtkqZg57fn3iS97tjNPCC9C9knRAuDYUG+dVxalazSwYrpvY97dUC1H2spD0g4UdDyCbGA46mouZ4GPzNMewgf948qxrnU8pWPk3nQW5TgLVkGoWgco2owfLElBqf6rJ4LOswuhaw8IpTtmw3FsixxTLQvKOE5nftd1nMhQQd9CaHjoKNAUEz5Q=="}, + id: 20498, + opcode: "QUERY", + flags: []string{"rd", "ad", "ra"}, + rcode: "NOERROR", + qClass: "IN", + qType: "A", + qName: "www.ietf.org", + qEtld: "ietf.org", + qTLD: "org", + qSubdomain: "www", + answers: []string{"64.170.98.30", "iDA8bJnrAEz3jgYnyFRm567a76qlv1V0CqxOSd/o9nvnN0GlZLaVoDmuXpaIaoypbGxwzwgK/LY6CV2k6SWKwicBmpENL26hwyjkFzPDW8kX3ibFhtfsOb8pYe7nBj326actp/7iG+DRuDmPnkYBja+wDYk61doTtkqZg57fn3iS97tjNPCC9C9knRAuDYUG+dVxalazSwYrpvY97dUC1H2spD0g4UdDyCbGA46mouZ4GPzNMewgf948qxrnU8pWPk3nQW5TgLVkGoWgco2owfLElBqf6rJ4LOswuhaw8IpTtmw3FsixxTLQvKOE5nftd1nMhQQd9CaHjoKNAUEz5Q=="}, request: []byte{ 0x50, 0x12, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x77, 0x77, 0x77, 0x04, 0x69, 0x65, 0x74, 0x66, 0x03, 0x6f, 0x72, 0x67, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00,