From d9a56993282a0e6204d39287f278873c8599c8ce Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 23 Jan 2020 12:57:31 +0100 Subject: [PATCH 1/4] Socket dataset: Workaround for bogus dereference in kernel 5.x This is a tentative workaround for the problems in Auditbeat's system/socket dataset when run under 5.x kernels. On older kernels, we could rely on dereferencing a NULL or invalid pointer returning zeroed memory. However, seems that in the tested 5.x kernels is not the case. Dereferencing a NULL pointer returns bogus memory, which causes some wrong codepaths to be taken in a couple of kprobes defined by the dataset. This so far seems only to affect udp_sendmsg and udpv6_sendmsg, which caused it to attribute traffic to bogus IP addresses. In turn this caused the test-connected-udp system tests to fail. --- .../auditbeat/module/system/socket/events.go | 16 ++- .../module/system/socket/guess/deref.go | 116 ++++++++++++++++++ .../auditbeat/module/system/socket/kprobes.go | 4 +- 3 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 x-pack/auditbeat/module/system/socket/guess/deref.go diff --git a/x-pack/auditbeat/module/system/socket/events.go b/x-pack/auditbeat/module/system/socket/events.go index 043a0565b1b..a5cb5c0ebee 100644 --- a/x-pack/auditbeat/module/system/socket/events.go +++ b/x-pack/auditbeat/module/system/socket/events.go @@ -534,14 +534,16 @@ type udpSendMsgCall struct { LPort uint16 `kprobe:"lport"` RPort uint16 `kprobe:"rport"` AltRPort uint16 `kprobe:"altrport"` + // SIPtr is the struct sockaddr_in pointer. + SIPtr uintptr `kprobe:"siptr"` + // SIAF is the address family in (struct sockaddr_in*)->sin_family. + SIAF uint16 `kprobe:"siaf"` } func (e *udpSendMsgCall) asFlow() flow { raddr, rport := e.RAddr, e.RPort - if raddr == 0 { + if e.SIPtr == 0 || e.SIAF != unix.AF_INET { raddr = e.AltRAddr - } - if rport == 0 { rport = e.AltRPort } return flow{ @@ -586,14 +588,16 @@ type udpv6SendMsgCall struct { LPort uint16 `kprobe:"lport"` RPort uint16 `kprobe:"rport"` AltRPort uint16 `kprobe:"altrport"` + // SI6Ptr is the struct sockaddr_in6 pointer. + SI6Ptr uintptr `kprobe:"si6ptr"` + // Si6AF is the address family field ((struct sockaddr_in6*)->sin6_family) + SI6AF uint16 `kprobe:"si6af"` } func (e *udpv6SendMsgCall) asFlow() flow { raddra, raddrb, rport := e.RAddrA, e.RAddrB, e.RPort - if raddra == 0 && raddrb == 0 { + if e.SI6Ptr == 0 || e.SI6AF != unix.AF_INET6 { raddra, raddrb = e.AltRAddrA, e.AltRAddrB - } - if rport == 0 { rport = e.AltRPort } return flow{ diff --git a/x-pack/auditbeat/module/system/socket/guess/deref.go b/x-pack/auditbeat/module/system/socket/guess/deref.go new file mode 100644 index 00000000000..416a58aaa26 --- /dev/null +++ b/x-pack/auditbeat/module/system/socket/guess/deref.go @@ -0,0 +1,116 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build linux,386 linux,amd64 + +package guess + +import ( + "encoding/hex" + "os" + "syscall" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/x-pack/auditbeat/module/system/socket/helper" + "github.com/elastic/beats/x-pack/auditbeat/tracing" +) + +/* + + */ + +func init() { + if err := Registry.AddGuess(&guessDeref{}); err != nil { + panic(err) + } +} + +const ( + flagName = "NULL_PTR_DEREF_IS_GARBAGE" + envVar = "AUDITBEAT_SYSTEM_SOCKET_CHECK_DEREF" +) + +type guessDeref struct { + ctx Context +} + +func (g *guessDeref) Condition(ctx Context) (bool, error) { + v := os.Getenv(envVar) + return v == "1", nil +} + +// Name of this guess. +func (g *guessDeref) Name() string { + return "guess_deref" +} + +// Provides returns the names of discovered variables. +func (g *guessDeref) Provides() []string { + return []string{ + flagName, + } +} + +// Requires declares the variables required to run this guess. +func (g *guessDeref) Requires() []string { + return []string{ + "SYS_UNAME", + } +} + +// Probes returns a kretprobe on prepare_creds that dumps the first bytes +// pointed to by the return value, which is a struct cred. +func (g *guessDeref) Probes() ([]helper.ProbeDef, error) { + return []helper.ProbeDef{ + { + Probe: tracing.Probe{ + Type: tracing.TypeKProbe, + Name: "guess_null_ptr_deref", + Address: "{{.SYS_UNAME}}", + Fetchargs: helper.MakeMemoryDump("{{.P1}}", 0, credDumpBytes), + }, + Decoder: tracing.NewDumpDecoder, + }, + }, nil +} + +// Prepare is a no-op. +func (g *guessDeref) Prepare(ctx Context) error { + g.ctx = ctx + return nil +} + +// Terminate is a no-op. +func (g *guessDeref) Terminate() error { + return nil +} + +func (g *guessDeref) MaxRepeats() int { + return 1000 +} + +// Extract receives the struct cred dump and discovers the offsets. +func (g *guessDeref) Extract(ev interface{}) (common.MapStr, bool) { + raw := ev.([]byte) + if len(raw) != credDumpBytes { + return nil, false + } + for _, val := range raw { + if val != 0 { + g.ctx.Log.Errorf("Found non-empty memory:\n%s", hex.Dump(raw)) + //return nil, false + } + } + return nil, true +} + +// Trigger invokes the SYS_ACCESS syscall: +// int access(const char *pathname, int mode); +// The function call will return an error due to path being NULL, but it will +// have invoked prepare_creds before argument validation. +func (g *guessDeref) Trigger() error { + var ptr *syscall.Utsname + syscall.Uname(ptr) + return nil +} diff --git a/x-pack/auditbeat/module/system/socket/kprobes.go b/x-pack/auditbeat/module/system/socket/kprobes.go index 71887f0108b..7e29d8c5c9d 100644 --- a/x-pack/auditbeat/module/system/socket/kprobes.go +++ b/x-pack/auditbeat/module/system/socket/kprobes.go @@ -293,7 +293,7 @@ var sharedKProbes = []helper.ProbeDef{ Probe: tracing.Probe{ Name: "udp_sendmsg_in", Address: "udp_sendmsg", - Fetchargs: "sock={{.UDP_SENDMSG_SOCK}} size={{.UDP_SENDMSG_LEN}} laddr=+{{.INET_SOCK_LADDR}}({{.UDP_SENDMSG_SOCK}}):u32 lport=+{{.INET_SOCK_LPORT}}({{.UDP_SENDMSG_SOCK}}):u16 raddr=+{{.SOCKADDR_IN_ADDR}}(+0({{.UDP_SENDMSG_MSG}})):u32 rport=+{{.SOCKADDR_IN_PORT}}(+0({{.UDP_SENDMSG_MSG}})):u16 altraddr=+{{.INET_SOCK_RADDR}}({{.UDP_SENDMSG_SOCK}}):u32 altrport=+{{.INET_SOCK_RPORT}}({{.UDP_SENDMSG_SOCK}}):u16", + Fetchargs: "sock={{.UDP_SENDMSG_SOCK}} size={{.UDP_SENDMSG_LEN}} laddr=+{{.INET_SOCK_LADDR}}({{.UDP_SENDMSG_SOCK}}):u32 lport=+{{.INET_SOCK_LPORT}}({{.UDP_SENDMSG_SOCK}}):u16 raddr=+{{.SOCKADDR_IN_ADDR}}(+0({{.UDP_SENDMSG_MSG}})):x32 siptr=+0({{.UDP_SENDMSG_MSG}}) siaf=+{{.SOCKADDR_IN_AF}}(+0({{.UDP_SENDMSG_MSG}})):u16 rport=+{{.SOCKADDR_IN_PORT}}(+0({{.UDP_SENDMSG_MSG}})):u16 altraddr=+{{.INET_SOCK_RADDR}}({{.UDP_SENDMSG_SOCK}}):u32 altrport=+{{.INET_SOCK_RPORT}}({{.UDP_SENDMSG_SOCK}}):u16", }, Decoder: helper.NewStructDecoder(func() interface{} { return new(udpSendMsgCall) }), }, @@ -455,7 +455,7 @@ var ipv6KProbes = []helper.ProbeDef{ Probe: tracing.Probe{ Name: "udpv6_sendmsg_in", Address: "udpv6_sendmsg", - Fetchargs: "sock={{.UDP_SENDMSG_SOCK}} size={{.UDP_SENDMSG_LEN}} laddra={{.INET_SOCK_V6_LADDR_A}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} laddrb={{.INET_SOCK_V6_LADDR_B}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} lport=+{{.INET_SOCK_LPORT}}({{.UDP_SENDMSG_SOCK}}):u16 raddra=+{{.SOCKADDR_IN6_ADDRA}}(+0({{.UDP_SENDMSG_MSG}})):u64 raddrb=+{{.SOCKADDR_IN6_ADDRB}}(+0({{.UDP_SENDMSG_MSG}})):u64 rport=+{{.SOCKADDR_IN6_PORT}}(+0({{.UDP_SENDMSG_MSG}})):u16 altraddra={{.INET_SOCK_V6_RADDR_A}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} altraddrb={{.INET_SOCK_V6_RADDR_B}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} altrport=+{{.INET_SOCK_RPORT}}({{.UDP_SENDMSG_SOCK}}):u16", + Fetchargs: "sock={{.UDP_SENDMSG_SOCK}} size={{.UDP_SENDMSG_LEN}} laddra={{.INET_SOCK_V6_LADDR_A}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} laddrb={{.INET_SOCK_V6_LADDR_B}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} lport=+{{.INET_SOCK_LPORT}}({{.UDP_SENDMSG_SOCK}}):u16 raddra=+{{.SOCKADDR_IN6_ADDRA}}(+0({{.UDP_SENDMSG_MSG}})):u64 raddrb=+{{.SOCKADDR_IN6_ADDRB}}(+0({{.UDP_SENDMSG_MSG}})):u64 rport=+{{.SOCKADDR_IN6_PORT}}(+0({{.UDP_SENDMSG_MSG}})):u16 altraddra={{.INET_SOCK_V6_RADDR_A}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} altraddrb={{.INET_SOCK_V6_RADDR_B}}({{.UDP_SENDMSG_SOCK}}){{.INET_SOCK_V6_TERM}} altrport=+{{.INET_SOCK_RPORT}}({{.UDP_SENDMSG_SOCK}}):u16 si6ptr=+0({{.UDP_SENDMSG_MSG}}) si6af=+{{.SOCKADDR_IN6_AF}}(+0({{.UDP_SENDMSG_MSG}})):u16", }, Decoder: helper.NewStructDecoder(func() interface{} { return new(udpv6SendMsgCall) }), }, From 284acaf98b3aede2d4889c2954654cef0a269521 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 23 Jan 2020 15:19:16 +0100 Subject: [PATCH 2/4] Fix unit tests --- .../auditbeat/module/system/socket/kprobes.go | 2 +- .../module/system/socket/state_test.go | 74 ++++++++++++++++++- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/x-pack/auditbeat/module/system/socket/kprobes.go b/x-pack/auditbeat/module/system/socket/kprobes.go index 7e29d8c5c9d..340e2e520cb 100644 --- a/x-pack/auditbeat/module/system/socket/kprobes.go +++ b/x-pack/auditbeat/module/system/socket/kprobes.go @@ -293,7 +293,7 @@ var sharedKProbes = []helper.ProbeDef{ Probe: tracing.Probe{ Name: "udp_sendmsg_in", Address: "udp_sendmsg", - Fetchargs: "sock={{.UDP_SENDMSG_SOCK}} size={{.UDP_SENDMSG_LEN}} laddr=+{{.INET_SOCK_LADDR}}({{.UDP_SENDMSG_SOCK}}):u32 lport=+{{.INET_SOCK_LPORT}}({{.UDP_SENDMSG_SOCK}}):u16 raddr=+{{.SOCKADDR_IN_ADDR}}(+0({{.UDP_SENDMSG_MSG}})):x32 siptr=+0({{.UDP_SENDMSG_MSG}}) siaf=+{{.SOCKADDR_IN_AF}}(+0({{.UDP_SENDMSG_MSG}})):u16 rport=+{{.SOCKADDR_IN_PORT}}(+0({{.UDP_SENDMSG_MSG}})):u16 altraddr=+{{.INET_SOCK_RADDR}}({{.UDP_SENDMSG_SOCK}}):u32 altrport=+{{.INET_SOCK_RPORT}}({{.UDP_SENDMSG_SOCK}}):u16", + Fetchargs: "sock={{.UDP_SENDMSG_SOCK}} size={{.UDP_SENDMSG_LEN}} laddr=+{{.INET_SOCK_LADDR}}({{.UDP_SENDMSG_SOCK}}):u32 lport=+{{.INET_SOCK_LPORT}}({{.UDP_SENDMSG_SOCK}}):u16 raddr=+{{.SOCKADDR_IN_ADDR}}(+0({{.UDP_SENDMSG_MSG}})):u32 siptr=+0({{.UDP_SENDMSG_MSG}}) siaf=+{{.SOCKADDR_IN_AF}}(+0({{.UDP_SENDMSG_MSG}})):u16 rport=+{{.SOCKADDR_IN_PORT}}(+0({{.UDP_SENDMSG_MSG}})):u16 altraddr=+{{.INET_SOCK_RADDR}}({{.UDP_SENDMSG_SOCK}}):u32 altrport=+{{.INET_SOCK_RPORT}}({{.UDP_SENDMSG_SOCK}}):u16", }, Decoder: helper.NewStructDecoder(func() interface{} { return new(udpSendMsgCall) }), }, diff --git a/x-pack/auditbeat/module/system/socket/state_test.go b/x-pack/auditbeat/module/system/socket/state_test.go index 8133280b4c8..efff2613a1c 100644 --- a/x-pack/auditbeat/module/system/socket/state_test.go +++ b/x-pack/auditbeat/module/system/socket/state_test.go @@ -16,6 +16,7 @@ import ( "github.com/joeshaw/multierror" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "golang.org/x/sys/unix" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/x-pack/auditbeat/module/system/socket/dns" @@ -148,11 +149,9 @@ func TestUDPOutgoingSinglePacketWithProcess(t *testing.T) { Sock: sock, Size: 123, LAddr: lAddr, - RAddr: rAddr, - AltRAddr: 0, + AltRAddr: rAddr, LPort: lPort, - RPort: rPort, - AltRPort: 0, + AltRPort: rPort, }, &inetReleaseCall{Meta: meta(1234, 1235, 17), Sock: sock}, &doExit{Meta: meta(1234, 1234, 18)}, @@ -293,6 +292,14 @@ func ipv4(ip string) uint32 { return tracing.MachineEndian.Uint32(netIP) } +func ipv6(ip string) (hi uint64, lo uint64) { + netIP := net.ParseIP(ip).To16() + if netIP == nil { + panic("bad ip") + } + return tracing.MachineEndian.Uint64(netIP[:]), tracing.MachineEndian.Uint64(netIP[8:]) +} + func feedEvents(evs []event, st *state, t *testing.T) error { for idx, ev := range evs { t.Logf("Delivering event %d: %s", idx, ev.String()) @@ -515,3 +522,62 @@ func TestDNSTracker(t *testing.T) { }.Run(t) }) } + +func TestUDPSendMsgAltLogic(t *testing.T) { + const expectedIPv4 = "6 probe=0 pid=1234 tid=1235 udp_sendmsg(sock=0x0, size=0, 10.11.12.13:1010 -> 10.20.30.40:1234)" + const expectedIPv6 = "6 probe=0 pid=1234 tid=1235 udpv6_sendmsg(sock=0x0, size=0, [fddd::bebe]:1010 -> [fddd::cafe]:1234)" + t.Run("ipv4 non-connected", func(t *testing.T) { + ev := udpSendMsgCall{ + Meta: meta(1234, 1235, 6), + LAddr: ipv4("10.11.12.13"), + LPort: be16(1010), + RAddr: ipv4("10.20.30.40"), + RPort: be16(1234), + AltRAddr: ipv4("192.168.255.255"), + AltRPort: be16(555), + SIPtr: 0x7fffffff, + SIAF: unix.AF_INET, + } + assert.Equal(t, expectedIPv4, ev.String()) + }) + t.Run("ipv4 connected", func(t *testing.T) { + ev := udpSendMsgCall{ + Meta: meta(1234, 1235, 6), + LAddr: ipv4("10.11.12.13"), + LPort: be16(1010), + RAddr: ipv4("192.168.255.255"), + RPort: be16(555), + AltRAddr: ipv4("10.20.30.40"), + AltRPort: be16(1234), + } + assert.Equal(t, expectedIPv4, ev.String()) + }) + t.Run("ipv6 non-connected", func(t *testing.T) { + ev := udpv6SendMsgCall{ + Meta: meta(1234, 1235, 6), + LPort: be16(1010), + RPort: be16(1234), + AltRPort: be16(555), + SI6Ptr: 0x7fffffff, + SI6AF: unix.AF_INET6, + } + ev.LAddrA, ev.LAddrB = ipv6("fddd::bebe") + ev.RAddrA, ev.RAddrB = ipv6("fddd::cafe") + ev.AltRAddrA, ev.AltRAddrB = ipv6("fddd::bad:bad") + assert.Equal(t, expectedIPv6, ev.String()) + }) + + t.Run("ipv6 connected", func(t *testing.T) { + ev := udpv6SendMsgCall{ + Meta: meta(1234, 1235, 6), + LPort: be16(1010), + RPort: be16(555), + AltRPort: be16(1234), + } + ev.LAddrA, ev.LAddrB = ipv6("fddd::bebe") + ev.RAddrA, ev.RAddrB = ipv6("fddd::bad:bad") + ev.AltRAddrA, ev.AltRAddrB = ipv6("fddd::cafe") + assert.Equal(t, expectedIPv6, ev.String()) + }) + +} From 9ae4d676925963d71c14456f99aed8488216377c Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 23 Jan 2020 16:12:58 +0100 Subject: [PATCH 3/4] Cleanup deref check --- .../module/system/socket/guess/deref.go | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/x-pack/auditbeat/module/system/socket/guess/deref.go b/x-pack/auditbeat/module/system/socket/guess/deref.go index 416a58aaa26..0701a33ebf3 100644 --- a/x-pack/auditbeat/module/system/socket/guess/deref.go +++ b/x-pack/auditbeat/module/system/socket/guess/deref.go @@ -9,6 +9,7 @@ package guess import ( "encoding/hex" "os" + "strconv" "syscall" "github.com/elastic/beats/libbeat/common" @@ -17,8 +18,14 @@ import ( ) /* + This is not an actual guess but a helper to check if the kernel kprobe + subsystem returns garbage after dereferencing a null pointer. - */ + This code is run when the AUDITBEAT_SYSTEM_SOCKET_CHECK_DEREF environment + variable is set to a value greater than 0. When set, it will run the given + number of times, print the hexdump to the debug logs if non-zero memory is + found and set the NULL_PTR_DEREF_IS_OK (bool) variable. +*/ func init() { if err := Registry.AddGuess(&guessDeref{}); err != nil { @@ -27,17 +34,27 @@ func init() { } const ( - flagName = "NULL_PTR_DEREF_IS_GARBAGE" + flagName = "NULL_PTR_DEREF_IS_OK" envVar = "AUDITBEAT_SYSTEM_SOCKET_CHECK_DEREF" ) type guessDeref struct { - ctx Context + ctx Context + tries int + garbage bool } -func (g *guessDeref) Condition(ctx Context) (bool, error) { +// Condition allows the guess to run if the environment variable is set to a +// decimal value greater than zero. +func (g *guessDeref) Condition(ctx Context) (run bool, err error) { v := os.Getenv(envVar) - return v == "1", nil + if v == "" { + return false, nil + } + if g.tries, err = strconv.Atoi(v); err != nil || g.tries <= 0 { + return false, nil + } + return true, nil } // Name of this guess. @@ -59,8 +76,8 @@ func (g *guessDeref) Requires() []string { } } -// Probes returns a kretprobe on prepare_creds that dumps the first bytes -// pointed to by the return value, which is a struct cred. +// Probes returns a kprobe on uname() that dumps the first bytes +// pointed to by its first parameter. func (g *guessDeref) Probes() ([]helper.ProbeDef, error) { return []helper.ProbeDef{ { @@ -86,11 +103,13 @@ func (g *guessDeref) Terminate() error { return nil } +// MaxRepeats returns the configured number of repeats. func (g *guessDeref) MaxRepeats() int { - return 1000 + return g.tries } -// Extract receives the struct cred dump and discovers the offsets. +// Extract receives the memory read through a null pointer and checks if it's +// zero or garbage. func (g *guessDeref) Extract(ev interface{}) (common.MapStr, bool) { raw := ev.([]byte) if len(raw) != credDumpBytes { @@ -98,17 +117,21 @@ func (g *guessDeref) Extract(ev interface{}) (common.MapStr, bool) { } for _, val := range raw { if val != 0 { - g.ctx.Log.Errorf("Found non-empty memory:\n%s", hex.Dump(raw)) - //return nil, false + g.ctx.Log.Errorf("Found non-zero memory:\n%s", hex.Dump(raw)) + g.garbage = true + break } } - return nil, true + // Repeat until completed all tries + if g.tries--; g.tries > 0 { + return nil, true + } + return common.MapStr{ + flagName: !g.garbage, + }, true } -// Trigger invokes the SYS_ACCESS syscall: -// int access(const char *pathname, int mode); -// The function call will return an error due to path being NULL, but it will -// have invoked prepare_creds before argument validation. +// Trigger invokes the uname syscall with a null parameter. func (g *guessDeref) Trigger() error { var ptr *syscall.Utsname syscall.Uname(ptr) From 830f1780d236f3382cb4826d7b9552598f498ecf Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 23 Jan 2020 17:01:32 +0100 Subject: [PATCH 4/4] Changelog --- CHANGELOG.next.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8bd298f3098..fe942b000f8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -38,11 +38,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* -TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568] *Auditbeat* +- system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771] *Filebeat*