diff --git a/NOTICE.txt b/NOTICE.txt index dd599cb5cb18..144a3b0f53e1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14216,11 +14216,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-quark -Version: v0.2.0 +Version: v0.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.2.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-quark@v0.3.0/LICENSE.txt: Apache License @@ -22667,11 +22667,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/stretchr/testify -Version: v1.9.0 +Version: v1.10.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.10.0/LICENSE: MIT License diff --git a/go.mod b/go.mod index 2e25c7c9de0d..2544e5da9015 100644 --- a/go.mod +++ b/go.mod @@ -118,7 +118,7 @@ require ( github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/ugorji/go/codec v1.1.8 github.com/vmware/govmomi v0.39.0 @@ -180,7 +180,7 @@ require ( github.com/elastic/elastic-agent-libs v0.17.4 github.com/elastic/elastic-agent-system-metrics v0.11.4 github.com/elastic/go-elasticsearch/v8 v8.14.0 - github.com/elastic/go-quark v0.2.0 + github.com/elastic/go-quark v0.3.0 github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 github.com/elastic/mito v1.16.0 github.com/elastic/mock-es v0.0.0-20240712014503-e5b47ece0015 diff --git a/go.sum b/go.sum index cbf4b3eab9ea..3eda0a87465e 100644 --- a/go.sum +++ b/go.sum @@ -346,8 +346,8 @@ github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko= github.com/elastic/go-perf v0.0.0-20241029065020-30bec95324b8 h1:FD01NjsTes0RxZVQ22ebNYJA4KDdInVnR9cn1hmaMwA= github.com/elastic/go-perf v0.0.0-20241029065020-30bec95324b8/go.mod h1:Nt+pnRYvf0POC+7pXsrv8ubsEOSsaipJP0zlz1Ms1RM= -github.com/elastic/go-quark v0.2.0 h1:r2BL4NzvhESrrL/yA3AcHt8mwF7fvQDssBAUiOL1sdg= -github.com/elastic/go-quark v0.2.0/go.mod h1:/ngqgumD/Z5vnFZ4XPN2kCbxnEfG5/Uc+bRvOBabVVA= +github.com/elastic/go-quark v0.3.0 h1:d4vokx0psEJo+93fnhvWpTJMggPd9rfMJSleoLva4xA= +github.com/elastic/go-quark v0.3.0/go.mod h1:bO/XIGZBUJGxyiJ9FTsSYn9YlfOTRJnmOP+iBE2FyjA= github.com/elastic/go-seccomp-bpf v1.5.0 h1:gJV+U1iP+YC70ySyGUUNk2YLJW5/IkEw4FZBJfW8ZZY= github.com/elastic/go-seccomp-bpf v1.5.0/go.mod h1:umdhQ/3aybliBF2jjiZwS492I/TOKz+ZRvsLT3hVe1o= github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727 h1:yuiN60oaQUz2PtNpNhDI2H6zrCdfiiptmNdwV5WUaKA= @@ -848,8 +848,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= diff --git a/x-pack/auditbeat/module/system/process/config.go b/x-pack/auditbeat/module/system/process/config.go index 52cb6dd98933..3e985fc0ac0c 100644 --- a/x-pack/auditbeat/module/system/process/config.go +++ b/x-pack/auditbeat/module/system/process/config.go @@ -5,6 +5,7 @@ package process import ( + "fmt" "time" "github.com/elastic/beats/v7/auditbeat/helper/hasher" @@ -16,11 +17,19 @@ type Config struct { ProcessStatePeriod time.Duration `config:"process.state.period"` HasherConfig hasher.Config `config:"process.hash"` + Backend string `config:"process.backend"` } // Validate validates the config. func (c *Config) Validate() error { - return c.HasherConfig.Validate() + if err := c.HasherConfig.Validate(); err != nil { + return err + } + if c.Backend != "quark" && c.Backend != "proc" { + return fmt.Errorf("invalid process.backend '%s'", c.Backend) + } + + return nil } func (c *Config) effectiveStatePeriod() time.Duration { @@ -40,4 +49,5 @@ var defaultConfig = Config{ ScanRatePerSec: "50 MiB", ScanRateBytesPerSec: 50 * 1024 * 1024, }, + Backend: "proc", } diff --git a/x-pack/auditbeat/module/system/process/gosysinfo_provider.go b/x-pack/auditbeat/module/system/process/gosysinfo_provider.go index da82a2e18106..a9f45f498eb4 100644 --- a/x-pack/auditbeat/module/system/process/gosysinfo_provider.go +++ b/x-pack/auditbeat/module/system/process/gosysinfo_provider.go @@ -351,27 +351,12 @@ func putIfNotEmpty(mapstr *mapstr.M, key string, value string) { } func processMessage(process *Process, action eventAction) string { - if process.Error != nil { - return fmt.Sprintf("ERROR for PID %d: %v", process.Info.PID, process.Error) - } - - var actionString string - switch action { - case eventActionProcessStarted: - actionString = "STARTED" - case eventActionProcessStopped: - actionString = "STOPPED" - case eventActionExistingProcess: - actionString = "is RUNNING" - } - - var userString string + var username string if process.User != nil { - userString = fmt.Sprintf(" by user %v", process.User.Username) + username = process.User.Username } - return fmt.Sprintf("Process %v (PID: %d)%v %v", - process.Info.Name, process.Info.PID, userString, actionString) + return makeMessage(process.Info.PID, action, process.Info.Name, username, process.Error) } func convertToCacheable(processes []*Process) []cache.Cacheable { diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index c79e87ce0fad..ad310026fa6c 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -7,6 +7,7 @@ package process import ( "encoding/binary" "fmt" + "runtime" "time" "github.com/elastic/beats/v7/auditbeat/ab" @@ -36,6 +37,8 @@ const ( eventActionExistingProcess eventAction = iota eventActionProcessStarted eventActionProcessStopped + eventActionProcessRan + eventActionProcessChangedImage eventActionProcessError ) @@ -47,6 +50,10 @@ func (action eventAction) String() string { return "process_started" case eventActionProcessStopped: return "process_stopped" + case eventActionProcessRan: + return "process_ran" + case eventActionProcessChangedImage: + return "process_changed_image" case eventActionProcessError: return "process_error" default: @@ -62,6 +69,8 @@ func (action eventAction) Type() string { return "start" case eventActionProcessStopped: return "end" + case eventActionProcessChangedImage: + return "change" case eventActionProcessError: return "info" default: @@ -89,6 +98,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", system.ModuleName, metricsetName, err) } + if runtime.GOOS == "linux" && ms.config.Backend == "quark" { + if qm, err := NewFromQuark(base, ms); err == nil { + return qm, nil + } else { + ms.log.Errorf("can't use quark, falling back to sysinfo: %w", err) + } + } + return NewFromSysInfo(base, ms) } @@ -102,3 +119,31 @@ func entityID(hostID string, pid int, startTime time.Time) string { binary.Write(h, binary.LittleEndian, int64(startTime.Nanosecond())) return h.Sum() } + +func makeMessage(pid int, action eventAction, name string, username string, err error) string { + if err != nil { + return fmt.Sprintf("ERROR for PID %d: %v", pid, err) + } + + var actionString string + switch action { + case eventActionProcessStarted: + actionString = "STARTED" + case eventActionProcessStopped: + actionString = "STOPPED" + case eventActionExistingProcess: + actionString = "is RUNNING" + case eventActionProcessRan: + actionString = "RAN" + case eventActionProcessChangedImage: + actionString = "CHANGED IMAGE" + } + + var userString string + if len(username) > 0 { + userString = fmt.Sprintf(" by user %v", username) + } + + return fmt.Sprintf("Process %v (PID: %d)%v %v", + name, pid, userString, actionString) +} diff --git a/x-pack/auditbeat/module/system/process/quark_provider_linux.go b/x-pack/auditbeat/module/system/process/quark_provider_linux.go new file mode 100644 index 000000000000..8b943865a827 --- /dev/null +++ b/x-pack/auditbeat/module/system/process/quark_provider_linux.go @@ -0,0 +1,285 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux && (amd64 || arm64) && cgo + +package process + +import ( + "fmt" + "os/user" + "strconv" + "time" + + "github.com/elastic/beats/v7/auditbeat/helper/hasher" + "github.com/elastic/beats/v7/auditbeat/helper/tty" + "github.com/elastic/beats/v7/libbeat/common/capabilities" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" + "github.com/elastic/elastic-agent-libs/mapstr" + + quark "github.com/elastic/go-quark" +) + +// QuarkMetricSet is a MetricSet with added members used only in by +// quark QuarkMetricSet uses mb.PushReporterV2 instead of +// mb.ReporterV2. More notably we don't do periodic state reports and +// we don't need a cache as it is provided by quark. +type QuarkMetricSet struct { + system.SystemMetricSet + MetricSet + queue *quark.Queue // Quark runtime state + selfMntNsIno uint32 // Mnt inode from current process + cachedHasher *hasher.CachedHasher +} + +// NewFromQuark instantiates the module with quark's backend. +func NewFromQuark(base mb.BaseMetricSet, ms MetricSet) (mb.MetricSet, error) { + var qm QuarkMetricSet + + qm.MetricSet = ms + + ino64, err := selfNsIno("mnt") + if err != nil { + return nil, fmt.Errorf("failed to fetch self mount inode: %w", err) + } + qm.selfMntNsIno = uint32(ino64) + qm.cachedHasher, err = hasher.NewFileHasherWithCache(qm.config.HasherConfig, 4096) + if err != nil { + return nil, fmt.Errorf("can't create hash cache: %w", err) + } + + attr := quark.DefaultQueueAttr() + qm.queue, err = quark.OpenQueue(attr, 1) + if err != nil { + return nil, fmt.Errorf("can't open quark queue: %w", err) + } + stats := qm.queue.Stats() + if stats.Backend == quark.QQ_EBPF { + qm.log.Info("quark using EBPF") + } else if stats.Backend == quark.QQ_KPROBE { + qm.log.Info("quark using KPROBES") + } else { + qm.log.Info("quark using VOODOO") + } + qm.SystemMetricSet = system.NewSystemMetricSet(base) + + return &qm, nil +} + +// Run reads events from quark's queue and pushes them into output. +// The queue is owned by this go-routine and should not be touched +// from outside as there is no synchronization. +func (ms *QuarkMetricSet) Run(r mb.PushReporterV2) { + ms.log.Info("Quark running") + +MainLoop: + for { + // Poll for done + select { + case <-r.Done(): + break MainLoop + default: + } + + x := time.Now() + quarkEvents, err := ms.queue.GetEvents() + if len(quarkEvents) == 1 { + ms.log.Debugf("getevents took %v", time.Since(x)) + } + if err != nil { + ms.log.Error("quark GetEvents, unrecoverable error", err) + break MainLoop + } + if len(quarkEvents) == 0 { + err = ms.queue.Block() + if err != nil { + ms.log.Error("quark Block, unrecoverable error", err) + break MainLoop + } + continue + } + for _, quarkEvent := range quarkEvents { + if !wantedEvent(quarkEvent) { + continue + } + if event, ok := ms.toEvent(quarkEvent); ok { + r.Event(event) + } + } + } + + // Queue is owned by this goroutine, if we ever access it from + // outside, we need to consider synchronization. + ms.cachedHasher.Close() + ms.queue.Close() + ms.queue = nil +} + +// toEvent converts a quark.Event to a mb.Event, returns true if we +// were able to make an event. +func (ms *QuarkMetricSet) toEvent(quarkEvent quark.Event) (mb.Event, bool) { + action, evtype := actionAndTypeOfEvent(quarkEvent) + process := quarkEvent.Process + event := mb.Event{RootFields: mapstr.M{}} + + var username string + var processErr error + defer func() { + // Fill out root message and error.message + event.RootFields.Put("message", + makeMessage(int(process.Pid), action, process.Comm, username, processErr)) + if processErr != nil { + event.RootFields.Put("error.message", processErr.Error()) + } + }() + + // Values that are independent of Proc.Valid + // Fill out event.* + event.RootFields.Put("event.type", evtype) + event.RootFields.Put("event.action", action.String()) + event.RootFields.Put("event.category", []string{"process"}) + event.RootFields.Put("event.kind", "event") + // Fill out process.* + event.RootFields.Put("process.name", process.Comm) + event.RootFields.Put("process.args", process.Cmdline) + event.RootFields.Put("process.args_count", len(process.Cmdline)) + event.RootFields.Put("process.pid", process.Pid) + event.RootFields.Put("process.working_directory", process.Cwd) + event.RootFields.Put("process.executable", process.Filename) + if process.Exit.Valid { + event.RootFields.Put("process.exit_code", process.Exit.ExitCode) + } + if !process.Proc.Valid { + return event, true + } + + // + // Code below can rely on Proc + // + + // Ids + event.RootFields.Put("process.parent.pid", process.Proc.Ppid) + startTime := time.Unix(0, int64(process.Proc.TimeBoot)) + if ms.HostID() != "" { + // TODO unify with sessionview and guarantee loss of precision + event.RootFields.Put("process.entity_id", + entityID(ms.HostID(), int(process.Pid), startTime)) + } + event.RootFields.Put("process.start", startTime) + event.RootFields.Put("user.id", process.Proc.Uid) + event.RootFields.Put("user.group.id", process.Proc.Gid) + event.RootFields.Put("user.effective.id", process.Proc.Euid) + event.RootFields.Put("user.effective.group.id", process.Proc.Egid) + event.RootFields.Put("user.saved.id", process.Proc.Suid) + event.RootFields.Put("user.saved.group.id", process.Proc.Sgid) + if us, err := user.LookupId(strconv.FormatUint(uint64(process.Proc.Uid), 10)); err == nil { + event.RootFields.Put("user.name", us.Username) + username = us.Username + } + if group, err := user.LookupGroupId(strconv.FormatUint(uint64(process.Proc.Gid), 10)); err == nil { + event.RootFields.Put("user.group.name", group.Name) + } + // Tty things + event.RootFields.Put("process.interactive", + tty.InteractiveFromTTY(tty.TTYDev{ + Major: process.Proc.TtyMajor, + Minor: process.Proc.TtyMinor, + })) + if process.Proc.TtyMajor != 0 { + event.RootFields.Put("process.tty.char_device.major", process.Proc.TtyMajor) + event.RootFields.Put("process.tty.char_device.minor", process.Proc.TtyMinor) + } + // Capabilities + capEffective, _ := capabilities.FromUint64(process.Proc.CapEffective) + if len(capEffective) > 0 { + event.RootFields.Put("process.thread.capabilities.effective", capEffective) + } + capPermitted, _ := capabilities.FromUint64(process.Proc.CapPermitted) + if len(capPermitted) > 0 { + event.RootFields.Put("process.thread.capabilities.permitted", capPermitted) + } + // If we are in the same mount namespace of the process, hash + // the file. When quark is running on kprobes, there are + // limitations concerning the full path of the filename, in + // those cases, the path won't start with a slash. + if process.Proc.MntInonum == ms.selfMntNsIno && len(process.Filename) > 0 && process.Filename[0] == '/' { + hashes, err := ms.cachedHasher.HashFile(process.Filename) + if err != nil { + processErr = fmt.Errorf("failed to hash executable %v for PID %v: %w", + process.Filename, process.Pid, err) + ms.log.Error(processErr.Error()) + } else { + for hashType, digest := range hashes { + fieldName := "process.hash." + string(hashType) + event.RootFields.Put(fieldName, digest) + } + } + } else { + ms.log.Debugf("skipping hash %s (inonum %d vs %d)\n", process.Filename, process.Proc.MntInonum, ms.selfMntNsIno) + } + event.RootFields.Put("quark", true) // XXX REMOVE ME + + return event, true +} + +// wantedEvent filters in only the wanted events from quark. +func wantedEvent(quarkEvent quark.Event) bool { + const wanted uint64 = quark.QUARK_EV_FORK | + quark.QUARK_EV_EXEC | + quark.QUARK_EV_EXIT | + quark.QUARK_EV_SNAPSHOT + if quarkEvent.Events&wanted == 0 || + quarkEvent.Process.Pid == 2 || + quarkEvent.Process.Proc.Ppid == 2 { // skip kthreads + + return false + } + + return true +} + +// actionAndTypeOfEvent computes eventAction and event.type out of a quark.Event. +func actionAndTypeOfEvent(quarkEvent quark.Event) (eventAction, []string) { + snap := quarkEvent.Events&quark.QUARK_EV_SNAPSHOT != 0 + fork := quarkEvent.Events&quark.QUARK_EV_FORK != 0 + exec := quarkEvent.Events&quark.QUARK_EV_EXEC != 0 + exit := quarkEvent.Events&quark.QUARK_EV_EXIT != 0 + + // Calculate event.action + // If it's a snap, it's existing + // If it forked + exited and executed or not, we consider ran + // If it execed + exited we consider stopped + // If it execed but didn't fork or exit, we consider changed image + var action eventAction + if snap { + action = eventActionExistingProcess + } else if fork && exit { + action = eventActionProcessRan + } else if fork { + action = eventActionProcessStarted + } else if exit { + action = eventActionProcessStopped + } else if exec { + action = eventActionProcessChangedImage + } else { + action = eventActionProcessError + } + // Calculate event.type + evtype := make([]string, 0, 4) + if snap { + evtype = append(evtype, eventActionExistingProcess.Type()) + } + if fork { + evtype = append(evtype, eventActionProcessStarted.Type()) + } + if exec { + evtype = append(evtype, eventActionProcessChangedImage.Type()) + } + if exit { + evtype = append(evtype, eventActionProcessStopped.Type()) + } + + return action, evtype +} diff --git a/x-pack/auditbeat/module/system/process/quark_provider_other.go b/x-pack/auditbeat/module/system/process/quark_provider_other.go new file mode 100644 index 000000000000..f061dadb0a04 --- /dev/null +++ b/x-pack/auditbeat/module/system/process/quark_provider_other.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !linux || !(amd64 || arm64) || !cgo + +package process + +import ( + "errors" + + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// NewFromQuark instantiates the module with quark's backend. +func NewFromQuark(base mb.BaseMetricSet, ms MetricSet) (mb.MetricSet, error) { + return nil, errors.New("quark is only available on linux on amd64/arm64 and needs cgo") +} diff --git a/x-pack/auditbeat/seccomp_linux.go b/x-pack/auditbeat/seccomp_linux.go index 5dd05618d31c..dc8735f9b94b 100644 --- a/x-pack/auditbeat/seccomp_linux.go +++ b/x-pack/auditbeat/seccomp_linux.go @@ -43,5 +43,12 @@ func init() { ); err != nil { panic(err) } + + // The system/process dataset uses additional syscalls + if err := seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, + "statx", + ); err != nil { + panic(err) + } } }