From cf561e24c1b30330266baddaf39ddd8cd480eee8 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Wed, 30 Jan 2019 17:55:58 +0000 Subject: [PATCH 1/5] [Auditbeat] Login metricset (#9327) Adds the login metricset to the Auditbeat system module as the last of the six initial metricsets. It only works on Linux, and detects not just user logins and logouts, but also system boots and shutdowns. It works by reading the /var/log/wtmp and /var/log/btmp file (and rotated files) present on Linux systems. In reading a file, it is similar to Filebeat, except that UTMP is a binary format, so reading happens using a binary Go reader. (cherry picked from commit 1566e660d752c62f28d3d4fbbea39ed73c11543f) --- CHANGELOG.next.asciidoc | 1 + auditbeat/docs/fields.asciidoc | 22 + x-pack/auditbeat/auditbeat.reference.yml | 7 + x-pack/auditbeat/auditbeat.yml | 5 + x-pack/auditbeat/docs/modules/system.asciidoc | 11 + .../docs/modules/system/login.asciidoc | 21 + x-pack/auditbeat/include/list.go | 1 + .../module/system/_meta/config.yml.tmpl | 13 + .../module/system/_meta/docs.asciidoc | 2 + .../auditbeat/module/system/_meta/fields.yml | 24 +- x-pack/auditbeat/module/system/fields.go | 2 +- .../module/system/login/_meta/data.json | 30 + .../module/system/login/_meta/docs.asciidoc | 7 + .../auditbeat/module/system/login/config.go | 20 + x-pack/auditbeat/module/system/login/login.go | 232 ++++++++ .../module/system/login/login_other.go | 29 + .../module/system/login/login_test.go | 63 +++ x-pack/auditbeat/module/system/login/utmp.go | 535 ++++++++++++++++++ .../auditbeat/module/system/login/utmp_c.go | 120 ++++ x-pack/auditbeat/tests/files/wtmp | Bin 0 -> 384 bytes .../auditbeat/tests/system/auditbeat_xpack.py | 7 +- .../auditbeat/tests/system/test_metricsets.py | 18 + 22 files changed, 1164 insertions(+), 6 deletions(-) create mode 100644 x-pack/auditbeat/docs/modules/system/login.asciidoc create mode 100644 x-pack/auditbeat/module/system/login/_meta/data.json create mode 100644 x-pack/auditbeat/module/system/login/_meta/docs.asciidoc create mode 100644 x-pack/auditbeat/module/system/login/config.go create mode 100644 x-pack/auditbeat/module/system/login/login.go create mode 100644 x-pack/auditbeat/module/system/login/login_other.go create mode 100644 x-pack/auditbeat/module/system/login/login_test.go create mode 100644 x-pack/auditbeat/module/system/login/utmp.go create mode 100644 x-pack/auditbeat/module/system/login/utmp_c.go create mode 100644 x-pack/auditbeat/tests/files/wtmp diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1df0396344d..293a2a63fa7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -144,6 +144,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Add system module. {pull}9546[9546] - System module `process` dataset: Add user information to processes. {pull}9963[9963] - Add system `package` dataset. {pull}10225[10225] +- Add system module `login` dataset. {pull}9327[9327] *Filebeat* diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index 5fde2c9eb58..d04632616e2 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -3724,6 +3724,28 @@ These are the fields generated by the system module. + +*`event.origin`*:: ++ +-- +type: keyword + +Origin of the event. This can be a file path (e.g. `/var/log/log.1`), or the name of the system component that supplied the data (e.g. `netlink`). + + +-- + + +*`user.terminal`*:: ++ +-- +type: keyword + +Terminal of the user. + + +-- + [float] == system.audit fields diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index b9d33bb7197..d5689f79adf 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -112,6 +112,7 @@ auditbeat.modules: - module: system datasets: - host # General host information, e.g. uptime, IPs + - login # User logins, logouts, and system boots. - package # Installed, updated, and removed packages - process # Started and stopped processes - socket # Opened and closed sockets @@ -134,6 +135,12 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true + # File patterns of the login record files. + # wtmp: History of successful logins, logouts, and system shutdowns and boots. + # btmp: Failed login attempts. + login.wtmp_file_pattern: /var/log/wtmp* + login.btmp_file_pattern: /var/log/btmp* + #================================ General ====================================== # The name of the shipper that publishes the network data. It can be used to group diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index 939d037b424..8417f295697 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -50,6 +50,7 @@ auditbeat.modules: - module: system datasets: - host # General host information, e.g. uptime, IPs + - login # User logins, logouts, and system boots. - package # Installed, updated, and removed packages - process # Started and stopped processes - socket # Opened and closed sockets @@ -65,6 +66,10 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true + # File patterns of the login record files. + login.wtmp_file_pattern: /var/log/wtmp* + login.btmp_file_pattern: /var/log/btmp* + #==================== Elasticsearch template setting ========================== setup.template.settings: index.number_of_shards: 3 diff --git a/x-pack/auditbeat/docs/modules/system.asciidoc b/x-pack/auditbeat/docs/modules/system.asciidoc index 6e3c52b3985..d6f902d07df 100644 --- a/x-pack/auditbeat/docs/modules/system.asciidoc +++ b/x-pack/auditbeat/docs/modules/system.asciidoc @@ -51,6 +51,7 @@ sample suggested configuration. - module: system datasets: - host + - login - package - process - socket @@ -87,6 +88,7 @@ so a longer polling interval can be used. - module: system datasets: - host + - login - package - user period: 1m @@ -113,6 +115,7 @@ auditbeat.modules: - module: system datasets: - host # General host information, e.g. uptime, IPs + - login # User logins, logouts, and system boots. - package # Installed, updated, and removed packages - process # Started and stopped processes - socket # Opened and closed sockets @@ -127,6 +130,10 @@ auditbeat.modules: # /etc/passwd and /etc/shadow and store a hash locally to # detect any changes. user.detect_password_changes: true + + # File patterns of the login record files. + login.wtmp_file_pattern: /var/log/wtmp* + login.btmp_file_pattern: /var/log/btmp* ---- [float] @@ -136,6 +143,8 @@ The following datasets are available: * <<{beatname_lc}-dataset-system-host,host>> +* <<{beatname_lc}-dataset-system-login,login>> + * <<{beatname_lc}-dataset-system-package,package>> * <<{beatname_lc}-dataset-system-process,process>> @@ -146,6 +155,8 @@ The following datasets are available: include::system/host.asciidoc[] +include::system/login.asciidoc[] + include::system/package.asciidoc[] include::system/process.asciidoc[] diff --git a/x-pack/auditbeat/docs/modules/system/login.asciidoc b/x-pack/auditbeat/docs/modules/system/login.asciidoc new file mode 100644 index 00000000000..d042abb7add --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/login.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-login"] +=== System login dataset + +include::../../../module/system/login/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/login/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/include/list.go b/x-pack/auditbeat/include/list.go index 4c2e3507ade..1bfa48fc08f 100644 --- a/x-pack/auditbeat/include/list.go +++ b/x-pack/auditbeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/auditbeat/module/system" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/host" + _ "github.com/elastic/beats/x-pack/auditbeat/module/system/login" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/package" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/process" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/socket" diff --git a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl index 5ac79221791..6e7bf4a44b7 100644 --- a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl +++ b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl @@ -7,6 +7,9 @@ - module: system datasets: - host # General host information, e.g. uptime, IPs + {{ if eq .GOOS "linux" -}} + - login # User logins, logouts, and system boots. + {{- end }} {{ if ne .GOOS "windows" -}} - package # Installed, updated, and removed packages {{- end }} @@ -38,3 +41,13 @@ # detect any changes. user.detect_password_changes: true {{- end }} + + {{ if eq .GOOS "linux" -}} + # File patterns of the login record files. +{{- if .Reference }} + # wtmp: History of successful logins, logouts, and system shutdowns and boots. + # btmp: Failed login attempts. +{{- end }} + login.wtmp_file_pattern: /var/log/wtmp* + login.btmp_file_pattern: /var/log/btmp* + {{- end }} diff --git a/x-pack/auditbeat/module/system/_meta/docs.asciidoc b/x-pack/auditbeat/module/system/_meta/docs.asciidoc index e9da6bc8f55..55614c4d69f 100644 --- a/x-pack/auditbeat/module/system/_meta/docs.asciidoc +++ b/x-pack/auditbeat/module/system/_meta/docs.asciidoc @@ -46,6 +46,7 @@ sample suggested configuration. - module: system datasets: - host + - login - package - process - socket @@ -82,6 +83,7 @@ so a longer polling interval can be used. - module: system datasets: - host + - login - package - user period: 1m diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index f27178a0931..fd47f2be771 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -4,7 +4,25 @@ These are the fields generated by the system module. release: experimental fields: - - name: system.audit - type: group + + - name: event + type: group + fields: + - name: origin + type: keyword description: > - fields: + Origin of the event. This can be a file path (e.g. `/var/log/log.1`), + or the name of the system component that supplied the data (e.g. `netlink`). + + - name: user + type: group + fields: + - name: terminal + type: keyword + description: > + Terminal of the user. + + - name: system.audit + type: group + description: > + fields: diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index 5bb495e45ae..a9234035777 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -18,5 +18,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy0WE1v20YQvetXDHKxDSjKpQgKHQokLdAGSBqjtoHerBF3RG693CF2lpbpX18sPyRRWkqRxfDmsfje2/levocnquYglXjKJwBee0NzeHdXG95NABRJ4nThNds5/DYBALjPSAjQEfiMYKXJKIGULDn0pGBZ1fYGE3JWpaHZBMCRIRSaA70U5HRO1qOZQAswr6Hfg8WcOkEzLJX29T8AfFXQHFLHZdFaItL6aLuIGYvfGGNog4jNswgAC0jYetS2O6+pcUHbFbscw3uznbeGTtw9+1q3asvC65x6AhrFhm3aMx+RHJ6HGgi0BYuWhRK2SmYRxiWzH+BU6Okczs/MHgJWjKd1IDn9SipCtmQ2hPYcvjvyoFdtJARwyxETEIS9sqVZ+DMi4ImqNTt1joC/MSfgVZ32HXz3d1A1BZqlM/h8d39UEK9WQn4mlIwQ+PutjoAaMuBI9IPK8fzxV4sWY9KxoL+RA778EaNAl2TaU+JLN+KBerBwXUf05dePjx9/uYmJyDEWxTdwf/v0O6BSjkQoGjtdRIj2jCc4vtwep2CJUOw3zxMsC5ad9rnTMQGXXPq6WLgIM0TbtBsAPYzDdrlVWBj0AXGPdNjrJ33y/W4D2kY7IetZplAuS+vLKay1VbyWm1lU0UE5XaomALZKvmESLP8OUK8w16YalbyBbOkdqQz9FBQtNdoprBzRUtQpjzyTE812VF0tZpzwiZwlMx7ffSRFr6SlOZSyyU1MnjCli7aPFuNoBaEFbcWjMaSAHTjK+ZlUxz/OZjLWlLhtRMHQnIgly4VUkVzp2Fp/jMnWQg4NqDGpdidTjM/ohOy4p2sho/OoycFRlsmOrsUc3CpFv16+LXdkASxKUuY5uuoNgM2LMczSmTHD8vDP18P+Uwq5i5pPADg5u8OPBIIpMrzP7zc/a5ACPAi5g7az4y29j3g5W39P3XKl43L9GWI6SKa0G/tgVwIZ5xSgKfHcT/GdwsnIjDiGAW4dpw5z8AyutIAeDKd6YAsIifm4k7NRIZ5e/Nnubj8BBILeJwD4buGrtuXLFHymJVxKQ4WklLA0qT6QDgcbdSePl/9RcqbARQ13YmOoGlLZlHAQW6Dz4f56vaSKrdr870qgcDq0suatvT0vXsZwvJThRC78UCRgk/yHdQ1H621Lr62nlPZL5Ez6odorUCRyuKGL1OnYdoDHw7uJWvtruLbcdOuNRXshszo7kkH5z4rkpwPZAXYGtyyil4bgGU1JUn+BXEiGitePG38MYF73Dp2hZCHRtW0+VdYYsNKGbqZb3z4qLbg0pBbTAdSF5S1z4GiKXaFNyXEpgBJqjC1BgjY0KdD2ZgpoY86pERNXFX4XdJ2R7Yesjk3Q/oF88qE2KxCiXAZAPXdZEu4IZGuO+mLQIB5Ef2d1RPGPSRYONFw6Bztd8/xQsMOdSmHV6zHdQdcotQBoBcwm/wcAAP//IWXfjA==" + return "eJy0WF+L27gXfc+nuPSlE0g9/OBHWfKw0O7CbqHdDjsp7NtEtm5s7ci6RleeTPrpF8l2YidyMum4hkKjkc45uv90pXfwiLsl8I4dljMAp5zGJby5DwNvZgASObOqcorMEn6dAQCsCmQEYRFcgbBRqCVDjgatcCgh3YXxBhNKkrXGZAZgUaNgXAI+V2hVicYJPYMWYDmbAbwDI0o/4wmNC1xuV+ESckt1FX53k/3/u9lkVa5MGOoWPOJuS1a2Y5E9+O9rWAe0CXoDZwKrQjFkwkCKIGCjNEIlXAE3mOQJrG+fhL3VlPt/yf/W88UejWyA8ZI6yNYEGZUVGTQOXCEccF1VWqEMU6RwosM26LQyj+t50rdFzWhfbAqHtlQmmPVKY6zalZ12TzvQ0WwmEbVUcddEgPsS+zILYrcfPEU6K9R/aw+whoyME8p0wacDLiizIVsKvy7prRoLv+471trzQOVUiQMBjWJNJh8Mn5Hsv28BCJQBIwwxZmQkJxHGlMiNcErh8BrOj0QOPFaMpzUgWvUdZYQsJdIozDV89+hAbVpPMIgDR0yAF/adDCb+Z0TAMHRfJOCvXgJ28N1vr2oBIds+3q/OCqLNhtEljNkEjl8ddHhUHwFnvO9VTmePP1u0GJOKOf0HOeDT7zEKYbNCOcxcbSfc0AC2rZ/Pv7x/eP//eUxEKWJe/AHuLx9+AyGlRWaM+k5VEaKjwQscn+7OUxBHKI6L5wWWNXGvfPYqJoiUaheShSp/oCuTd6V/gHFaLg8KKy2cRzwiHbf6RZt8vd+Dtt7O0DjiBdRpbVy9gK0ykrY8T6KKTtLptWrCMd8o+SIyP/LPCPVGlErvJiVvIFt6i7IQbgESUyXMAjYWMWV5ySJPaFmRmVRXixknfERrUE/Ht4qE6FtuaU6l7GNTZI8ix1d1Hy3G2QwSBpRhJ7RG6XtDiyU9oez4p+lMpjol7hpRMHZOxILllVSRWOnYWntMydZCjh1QU1L1T6YYn1YZmml310JGz6MmBidpJju6FnO0q2T1/fXdckfmwaIkdVkKu/sBwGZhDLO2ekq3fPv782n92V/n+hTXFB8PcPHs9pMY/FDk8L6+3vysgxTgG6M9KTs9a6ljxNezDfvUA1c+Ldcf3qejZFLZqTf2lqGgEj00Zo6GId5LnAL1hMcwwJ2l3IoSHIGtDQgHmnI10gX4wHzoxWxUiMNnd7W52ycATzB4AoCvBj4rUz8vwBWK/aXUZ0iOGXET6iPhcNJRd/Io/RezKwWuA9yFjmHXkPI+hb3YSljn7683Ke7IyP3f3jJUVvlS1qw66vPiaQznUxkuxMKLPAH74D/Nazibbwd6ZRzmeJwiV9KP5V4lmCObG7tIXfZtB3jevXuvtbPhxlBTrfcjyjHqzdWe9Mp/lic/nMj2sAncEbNKNcKT0DVyeA5ecyEkbR/29hjBvBlsuhBc+EBXpnk0DRjh5XW+ONj2QSoWqUa5Xoygrg0dmD1Hk+xSmBwt1QyCfY6RwfC+qykHZeYLECZmnICY2V3l+qDbAs3QZcE3Xvstuuw2DEtgxJJHQB11UeLvCGgCR7gYNIgn3u+1joLdQ1b4DY2nzklP13wvcvYqvEjvBjWm2+hWcBAArYBk9l8AAAD//y0/V6s=" } diff --git a/x-pack/auditbeat/module/system/login/_meta/data.json b/x-pack/auditbeat/module/system/login/_meta/data.json new file mode 100644 index 00000000000..311893d5926 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/_meta/data.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "action": "user_login", + "dataset": "login", + "module": "system", + "origin": "/var/log/wtmp.1", + "outcome": "success", + "type": "event" + }, + "message": "Login by user vagrant (UID: 1000) on pts/1 (PID: 17559) from 10.0.2.2 (IP: 10.0.2.2)", + "process": { + "pid": 17559 + }, + "service": { + "type": "system" + }, + "source": { + "ip": "10.0.2.2" + }, + "user": { + "id": 1000, + "name": "vagrant", + "terminal": "pts/1" + } +} diff --git a/x-pack/auditbeat/module/system/login/_meta/docs.asciidoc b/x-pack/auditbeat/module/system/login/_meta/docs.asciidoc new file mode 100644 index 00000000000..ea1eff21763 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/_meta/docs.asciidoc @@ -0,0 +1,7 @@ +[role="xpack"] + +experimental[] + +This is the `login` dataset of the system module. + +It is implemented for Linux only. diff --git a/x-pack/auditbeat/module/system/login/config.go b/x-pack/auditbeat/module/system/login/config.go new file mode 100644 index 00000000000..2cdfca54904 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/config.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build linux + +package login + +// config defines the metricset's configuration options. +type config struct { + WtmpFilePattern string `config:"login.wtmp_file_pattern"` + BtmpFilePattern string `config:"login.btmp_file_pattern"` +} + +func defaultConfig() config { + return config{ + WtmpFilePattern: "/var/log/wtmp*", + BtmpFilePattern: "/var/log/btmp*", + } +} diff --git a/x-pack/auditbeat/module/system/login/login.go b/x-pack/auditbeat/module/system/login/login.go new file mode 100644 index 00000000000..7af40d1e6a1 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/login.go @@ -0,0 +1,232 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build linux + +package login + +import ( + "fmt" + "net" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/auditbeat/datastore" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" +) + +const ( + moduleName = "system" + metricsetName = "login" + namespace = "system.audit.login" + + bucketName = "login.v1" + + eventTypeEvent = "event" +) + +// loginRecordType represents the type of a login record. +type loginRecordType uint8 + +const ( + bootRecord loginRecordType = iota + 1 + shutdownRecord + userLoginRecord + userLogoutRecord + userLoginFailedRecord +) + +// String returns the string representation of a LoginRecordType. +func (t loginRecordType) string() string { + switch t { + case bootRecord: + return "boot" + case shutdownRecord: + return "shutdown" + + case userLoginFailedRecord: + fallthrough + case userLoginRecord: + return "user_login" + + case userLogoutRecord: + return "user_logout" + default: + return "" + } +} + +// LoginRecord represents a login record. +type LoginRecord struct { + Utmp *Utmp + Type loginRecordType + PID int + TTY string + UID int + Username string + Hostname string + IP *net.IP + Timestamp time.Time + Origin string +} + +func init() { + mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.DefaultMetricSet(), + mb.WithNamespace(namespace), + ) +} + +// MetricSet collects login records from /var/log/wtmp. +type MetricSet struct { + mb.BaseMetricSet + config config + log *logp.Logger + utmpReader *UtmpFileReader +} + +// New constructs a new MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The %v/%v dataset is experimental", moduleName, metricsetName) + + config := defaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, errors.Wrapf(err, "failed to unpack the %v/%v config", moduleName, metricsetName) + } + + bucket, err := datastore.OpenBucket(bucketName) + if err != nil { + return nil, errors.Wrap(err, "failed to open persistent datastore") + } + + ms := &MetricSet{ + BaseMetricSet: base, + config: config, + log: logp.NewLogger(metricsetName), + } + + ms.utmpReader, err = NewUtmpFileReader(ms.log, bucket, config) + if err != nil { + return nil, err + } + + return ms, nil +} + +// Close cleans up the MetricSet when it finishes. +func (ms *MetricSet) Close() error { + return ms.utmpReader.Close() +} + +// Fetch collects any new login records from /var/log/wtmp. It is invoked periodically. +func (ms *MetricSet) Fetch(report mb.ReporterV2) { + count := ms.readAndEmit(report) + + ms.log.Debugf("%d new login records.", count) + + // Save new state to disk + if count > 0 { + err := ms.utmpReader.saveStateToDisk() + if err != nil { + ms.log.Error(err) + report.Error(err) + } + } +} + +// readAndEmit reads and emits login events and returns the number of events. +func (ms *MetricSet) readAndEmit(report mb.ReporterV2) int { + loginRecordC, errorC := ms.utmpReader.ReadNew() + + var count int + for { + select { + case loginRecord, ok := <-loginRecordC: + if !ok { + return count + } + report.Event(ms.loginEvent(&loginRecord)) + count++ + case err, ok := <-errorC: + if !ok { + return count + } + ms.log.Error(err) + } + } +} + +func (ms *MetricSet) loginEvent(loginRecord *LoginRecord) mb.Event { + event := mb.Event{ + Timestamp: loginRecord.Timestamp, + RootFields: common.MapStr{ + "event": common.MapStr{ + "type": eventTypeEvent, + "action": loginRecord.Type.string(), + "origin": loginRecord.Origin, + }, + "message": loginMessage(loginRecord), + // Very useful for development + // "debug": fmt.Sprintf("%v", login.Utmp), + }, + } + + if loginRecord.Username != "" { + event.RootFields.Put("user.name", loginRecord.Username) + + if loginRecord.UID != -1 { + event.RootFields.Put("user.id", loginRecord.UID) + } + } + + if loginRecord.TTY != "" { + event.RootFields.Put("user.terminal", loginRecord.TTY) + } + + if loginRecord.PID != -1 { + event.RootFields.Put("process.pid", loginRecord.PID) + } + + if loginRecord.IP != nil { + event.RootFields.Put("source.ip", loginRecord.IP) + } + + if loginRecord.Hostname != "" && loginRecord.Hostname != loginRecord.IP.String() { + event.RootFields.Put("source.domain", loginRecord.Hostname) + } + + switch loginRecord.Type { + case userLoginRecord: + event.RootFields.Put("event.outcome", "success") + case userLoginFailedRecord: + event.RootFields.Put("event.outcome", "failure") + } + + return event +} + +func loginMessage(loginRecord *LoginRecord) string { + var actionString string + + switch loginRecord.Type { + case bootRecord: + return "System boot" + case shutdownRecord: + return "System shutdown" + case userLoginRecord: + actionString = "Login" + case userLoginFailedRecord: + actionString = "Failed login" + case userLogoutRecord: + actionString = "Logout" + } + + return fmt.Sprintf("%v by user %v (UID: %d) on %v (PID: %d) from %v (IP: %v)", + actionString, loginRecord.Username, loginRecord.UID, loginRecord.TTY, loginRecord.PID, + loginRecord.Hostname, loginRecord.IP) +} diff --git a/x-pack/auditbeat/module/system/login/login_other.go b/x-pack/auditbeat/module/system/login/login_other.go new file mode 100644 index 00000000000..73462e1c2f1 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/login_other.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !linux + +package login + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/mb" +) + +const ( + moduleName = "system" + metricsetName = "login" +) + +func init() { + mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// New returns an error. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", moduleName, metricsetName) +} diff --git a/x-pack/auditbeat/module/system/login/login_test.go b/x-pack/auditbeat/module/system/login/login_test.go new file mode 100644 index 00000000000..cbab49369f5 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/login_test.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build linux + +package login + +import ( + "encoding/binary" + "io/ioutil" + "os" + "testing" + + "github.com/elastic/beats/auditbeat/core" + "github.com/elastic/beats/libbeat/paths" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + if byteOrder != binary.LittleEndian { + t.Skip("Test only works on little-endian systems - skipping.") + } + + defer setup(t)() + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("received error: %+v", errs[0]) + } + + if len(events) == 0 { + t.Fatal("no events were generated") + } else if len(events) != 1 { + t.Fatal("only one event expected") + } + + fullEvent := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent) + mbtest.WriteEventToDataJSON(t, fullEvent, "") +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "datasets": []string{"login"}, + "login.wtmp_file_pattern": "../../../tests/files/wtmp", + "login.btmp_file_pattern": "", + } +} + +// setup is copied from file_integrity/metricset_test.go. +// TODO: Move to shared location and use in all unit tests. +func setup(t testing.TB) func() { + // path.data should be set so that the DB is written to a predictable location. + var err error + paths.Paths.Data, err = ioutil.TempDir("", "beat-data-dir") + if err != nil { + t.Fatal() + } + return func() { os.RemoveAll(paths.Paths.Data) } +} diff --git a/x-pack/auditbeat/module/system/login/utmp.go b/x-pack/auditbeat/module/system/login/utmp.go new file mode 100644 index 00000000000..318ca94927e --- /dev/null +++ b/x-pack/auditbeat/module/system/login/utmp.go @@ -0,0 +1,535 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build linux + +package login + +import ( + "bytes" + "encoding/gob" + "io" + "net" + "os" + "os/user" + "path/filepath" + "sort" + "strconv" + "syscall" + + "github.com/pkg/errors" + + "github.com/elastic/beats/auditbeat/datastore" + "github.com/elastic/beats/libbeat/logp" +) + +const ( + bucketKeyFileRecords = "file_records" + bucketKeyLoginSessions = "login_sessions" +) + +// Inode represents a file's inode on Linux. +type Inode uint64 + +// UtmpType represents the type of a UTMP file and records. +// Two types are possible: wtmp (records from the "good" file, i.e. /var/log/wtmp) +// and btmp (failed logins from /var/log/btmp). +type UtmpType uint8 + +const ( + // Wtmp is the "normal" wtmp file that includes successful logins, logouts, + // and system boots. + Wtmp UtmpType = iota + // Btmp contains bad logins only. + Btmp +) + +// UtmpFile represents a UTMP file at a point in time. +type UtmpFile struct { + Inode Inode + Path string + Size int64 + Offset int64 + Type UtmpType +} + +// UtmpFileReader can read a UTMP formatted file (usually /var/log/wtmp). +type UtmpFileReader struct { + log *logp.Logger + bucket datastore.Bucket + config config + savedUtmpFiles map[Inode]UtmpFile + loginSessions map[string]LoginRecord +} + +// NewUtmpFileReader creates and initializes a new UTMP file reader. +func NewUtmpFileReader(log *logp.Logger, bucket datastore.Bucket, config config) (*UtmpFileReader, error) { + r := &UtmpFileReader{ + log: log, + bucket: bucket, + config: config, + savedUtmpFiles: make(map[Inode]UtmpFile), + loginSessions: make(map[string]LoginRecord), + } + + // Load state (file records, tty mapping) from disk + err := r.restoreStateFromDisk() + if err != nil { + return nil, errors.Wrap(err, "failed to restore state from disk") + } + + return r, nil +} + +// Close performs any cleanup tasks when the UTMP reader is done. +func (r *UtmpFileReader) Close() error { + err := r.bucket.Close() + return errors.Wrap(err, "error closing bucket") +} + +// ReadNew returns any new UTMP entries in any files matching the configured pattern. +func (r *UtmpFileReader) ReadNew() (<-chan LoginRecord, <-chan error) { + loginRecordC := make(chan LoginRecord) + errorC := make(chan error) + + go func() { + defer close(loginRecordC) + defer close(errorC) + + wtmpFiles, err := r.findFiles(r.config.WtmpFilePattern, Wtmp) + if err != nil { + errorC <- errors.Wrap(err, "failed to expand file pattern") + return + } + + btmpFiles, err := r.findFiles(r.config.BtmpFilePattern, Btmp) + if err != nil { + errorC <- errors.Wrap(err, "failed to expand file pattern") + return + } + + utmpFiles := append(wtmpFiles, btmpFiles...) + defer r.deleteOldUtmpFiles(&utmpFiles) + + for _, utmpFile := range utmpFiles { + r.readNewInFile(loginRecordC, errorC, utmpFile) + } + }() + + return loginRecordC, errorC +} + +func (r *UtmpFileReader) findFiles(filePattern string, utmpType UtmpType) ([]UtmpFile, error) { + paths, err := filepath.Glob(filePattern) + if err != nil { + return nil, errors.Wrapf(err, "failed to expand file pattern %v", filePattern) + } + + // Sort paths in reverse order (oldest/most-rotated file first) + sort.Sort(sort.Reverse(sort.StringSlice(paths))) + + var utmpFiles []UtmpFile + for _, path := range paths { + fileInfo, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + // Skip - file might have been rotated out + r.log.Debugf("File %v does not exist anymore.", path) + continue + } else { + return nil, errors.Wrapf(err, "unexpected error when looking up file %v", path) + } + } + + utmpFiles = append(utmpFiles, UtmpFile{ + Inode: Inode(fileInfo.Sys().(*syscall.Stat_t).Ino), + Path: path, + Size: fileInfo.Size(), + Offset: 0, + Type: utmpType, + }) + } + + return utmpFiles, nil +} + +// deleteOldUtmpFiles cleans up old UTMP file records where the inode no longer exists. +func (r *UtmpFileReader) deleteOldUtmpFiles(existingFiles *[]UtmpFile) { + existingInodes := make(map[Inode]struct{}) + for _, utmpFile := range *existingFiles { + existingInodes[utmpFile.Inode] = struct{}{} + } + + for savedInode := range r.savedUtmpFiles { + if _, exists := existingInodes[savedInode]; !exists { + r.log.Debugf("Deleting file record for old inode %d.", savedInode) + delete(r.savedUtmpFiles, savedInode) + } + } +} + +// readNewInFile reads a UTMP formatted file and emits the records after the last known record. +func (r *UtmpFileReader) readNewInFile(loginRecordC chan<- LoginRecord, errorC chan<- error, utmpFile UtmpFile) { + savedUtmpFile, isKnownFile := r.savedUtmpFiles[utmpFile.Inode] + if !isKnownFile { + r.log.Debugf("Found new file: %v (utmpFile=%+v)", utmpFile.Path, utmpFile) + } + + size := utmpFile.Size + oldSize := savedUtmpFile.Size + if size < oldSize { + // UTMP files are append-only and so this is weird. It might be a sign of + // a highly unlikely inode reuse - or of something more nefarious. + // Setting isKnownFile to false so we read the whole file from the beginning. + isKnownFile = false + + r.log.Warnf("Unexpectedly, the file %v is smaller than before (new: %v, old: %v) - reading whole file.", + utmpFile.Path, size, oldSize) + } + + if !isKnownFile && size == 0 { + // Empty new file - save but don't read. + err := r.updateSavedUtmpFile(utmpFile, nil) + if err != nil { + errorC <- errors.Wrapf(err, "error updating file record for file %v", utmpFile.Path) + } + return + } + + if !isKnownFile || size != oldSize { + r.log.Debugf("Reading file %v (utmpFile=%+v)", utmpFile.Path, utmpFile) + + f, err := os.Open(utmpFile.Path) + if err != nil { + errorC <- errors.Wrapf(err, "error opening file %v", utmpFile.Path) + return + } + defer func() { + // Once we start reading a file, we update the file record even if something fails - + // otherwise we will just keep trying to re-read very frequently forever. + r.updateSavedUtmpFile(utmpFile, f) + if err != nil { + errorC <- errors.Wrapf(err, "error updating file record for file %v", utmpFile.Path) + } + + f.Close() + }() + + _, err = f.Seek(utmpFile.Offset, 0) + if err != nil { + errorC <- errors.Wrapf(err, "error setting offset for file %v", utmpFile.Path) + + // Try one more time, this time resetting to the beginning of the file. + _, err = f.Seek(0, 0) + if err != nil { + errorC <- errors.Wrapf(err, "error setting offset 0 for file %v", utmpFile.Path) + + // Even that did not work, so return. + return + } + } + + for { + utmp, err := ReadNextUtmp(f) + if err != nil && err != io.EOF { + errorC <- errors.Wrapf(err, "error reading entry in UTMP file %v", utmpFile.Path) + return + } + + if utmp != nil { + r.log.Debugf("utmp: (ut_type=%d, ut_pid=%d, ut_line=%v, ut_user=%v, ut_host=%v, ut_tv.tv_sec=%v, ut_addr_v6=%v)", + utmp.UtType, utmp.UtPid, utmp.UtLine, utmp.UtUser, utmp.UtHost, utmp.UtTv, utmp.UtAddrV6) + + loginRecord := r.processLoginRecord(utmp) + if loginRecord != nil { + loginRecord.Origin = utmpFile.Path + if utmpFile.Type == Btmp && loginRecord.Type == userLoginRecord { + loginRecord.Type = userLoginFailedRecord + } + + loginRecordC <- *loginRecord + } + } else { + // Eventually, we have read all UTMP records in the file. + break + } + } + } +} + +func (r *UtmpFileReader) updateSavedUtmpFile(utmpFile UtmpFile, f *os.File) error { + if f != nil { + offset, err := f.Seek(0, 1) + if err != nil { + return errors.Wrap(err, "error calling Seek") + } + utmpFile.Offset = offset + } + + r.log.Debugf("Saving UTMP file record (%+v)", utmpFile) + + r.savedUtmpFiles[utmpFile.Inode] = utmpFile + + return nil +} + +// processLoginRecord receives UTMP login records in order and returns +// a corresponding LoginRecord. Some UTMP records do not translate +// into a LoginRecord, in this case the return value is nil. +func (r *UtmpFileReader) processLoginRecord(utmp *Utmp) *LoginRecord { + record := LoginRecord{ + Utmp: utmp, + Timestamp: utmp.UtTv, + UID: -1, + PID: -1, + } + + if utmp.UtLine != "~" { + record.TTY = utmp.UtLine + } + + switch utmp.UtType { + // See utmp(5) for C constants. + case RUN_LVL: + // The runlevel - though a number - is stored as + // the ASCII character of that number. + runlevel := string(rune(utmp.UtPid)) + + // 0 - halt; 6 - reboot + if utmp.UtUser == "shutdown" || runlevel == "0" || runlevel == "6" { + record.Type = shutdownRecord + + // Clear any old logins + // TODO: Issue logout events for login events that are still around + // at this point. + r.loginSessions = make(map[string]LoginRecord) + } else { + // Ignore runlevel changes that are not halt or reboot. + return nil + } + case BOOT_TIME: + if utmp.UtLine == "~" && utmp.UtUser == "reboot" { + record.Type = bootRecord + + // Clear any old logins + // TODO: Issue logout events for login events that are still around + // at this point. + r.loginSessions = make(map[string]LoginRecord) + } else { + // Ignore unknown record + return nil + } + case USER_PROCESS: + record.Type = userLoginRecord + + record.Username = utmp.UtUser + record.UID = lookupUsername(record.Username) + record.PID = utmp.UtPid + record.IP = newIP(utmp.UtAddrV6) + record.Hostname = utmp.UtHost + + // Store TTY from user login record for enrichment when user logout + // record comes along (which, alas, does not contain the username). + r.loginSessions[record.TTY] = record + case DEAD_PROCESS: + savedRecord, found := r.loginSessions[record.TTY] + if found { + record.Type = userLogoutRecord + record.Username = savedRecord.Username + record.UID = savedRecord.UID + record.PID = savedRecord.PID + record.IP = savedRecord.IP + record.Hostname = savedRecord.Hostname + } else { + // Skip - this is usually the DEAD_PROCESS event for + // a previous INIT_PROCESS or LOGIN_PROCESS event - + // those are ignored - (see default case below). + return nil + } + default: + /* + Every other record type is ignored: + - EMPTY - empty record + - NEW_TIME and OLD_TIME - could be useful, but not written when time changes, + at least not using `date` + - INIT_PROCESS and LOGIN_PROCESS - written on boot but do not contain any + interesting information + - ACCOUNTING - not implemented according to manpage + */ + return nil + } + + return &record +} + +// lookupUsername looks up a username and returns its UID. +// It does not pass through errors (e.g. when the user is not found) +// but will return -1 instead. +func lookupUsername(username string) int { + if username != "" { + user, err := user.Lookup(username) + if err == nil { + uid, err := strconv.Atoi(user.Uid) + if err == nil { + return uid + } + } + } + + return -1 +} + +func newIP(utAddrV6 [4]uint32) *net.IP { + var ip net.IP + + // See utmp(5) for the utmp struct fields. + if utAddrV6[1] != 0 || utAddrV6[2] != 0 || utAddrV6[3] != 0 { + // IPv6 + b := make([]byte, 16) + byteOrder.PutUint32(b[:4], utAddrV6[0]) + byteOrder.PutUint32(b[4:8], utAddrV6[1]) + byteOrder.PutUint32(b[8:12], utAddrV6[2]) + byteOrder.PutUint32(b[12:], utAddrV6[3]) + ip = net.IP(b) + } else { + // IPv4 + b := make([]byte, 4) + byteOrder.PutUint32(b, utAddrV6[0]) + ip = net.IP(b) + } + + return &ip +} + +func (r *UtmpFileReader) saveStateToDisk() error { + err := r.saveFileRecordsToDisk() + if err != nil { + return err + } + + err = r.saveLoginSessionsToDisk() + if err != nil { + return err + } + + return nil +} + +func (r *UtmpFileReader) saveFileRecordsToDisk() error { + var buf bytes.Buffer + encoder := gob.NewEncoder(&buf) + + for _, utmpFile := range r.savedUtmpFiles { + err := encoder.Encode(utmpFile) + if err != nil { + return errors.Wrap(err, "error encoding UTMP file record") + } + } + + err := r.bucket.Store(bucketKeyFileRecords, buf.Bytes()) + if err != nil { + return errors.Wrap(err, "error writing UTMP file records to disk") + } + + r.log.Debugf("Wrote %d UTMP file records to disk", len(r.savedUtmpFiles)) + return nil +} + +func (r *UtmpFileReader) saveLoginSessionsToDisk() error { + var buf bytes.Buffer + encoder := gob.NewEncoder(&buf) + + for _, loginRecord := range r.loginSessions { + err := encoder.Encode(loginRecord) + if err != nil { + return errors.Wrap(err, "error encoding login record") + } + } + + err := r.bucket.Store(bucketKeyLoginSessions, buf.Bytes()) + if err != nil { + return errors.Wrap(err, "error writing login records to disk") + } + + r.log.Debugf("Wrote %d open login sessions to disk", len(r.loginSessions)) + return nil +} + +func (r *UtmpFileReader) restoreStateFromDisk() error { + err := r.restoreFileRecordsFromDisk() + if err != nil { + return err + } + + err = r.restoreLoginSessionsFromDisk() + if err != nil { + return err + } + + return nil +} + +func (r *UtmpFileReader) restoreFileRecordsFromDisk() error { + var decoder *gob.Decoder + err := r.bucket.Load(bucketKeyFileRecords, func(blob []byte) error { + if len(blob) > 0 { + buf := bytes.NewBuffer(blob) + decoder = gob.NewDecoder(buf) + } + return nil + }) + if err != nil { + return err + } + + if decoder != nil { + for { + var utmpFile UtmpFile + err = decoder.Decode(&utmpFile) + if err == nil { + r.savedUtmpFiles[utmpFile.Inode] = utmpFile + } else if err == io.EOF { + // Read all + break + } else { + return errors.Wrap(err, "error decoding file record") + } + } + } + r.log.Debugf("Restored %d UTMP file records from disk", len(r.savedUtmpFiles)) + + return nil +} + +func (r *UtmpFileReader) restoreLoginSessionsFromDisk() error { + var decoder *gob.Decoder + err := r.bucket.Load(bucketKeyLoginSessions, func(blob []byte) error { + if len(blob) > 0 { + buf := bytes.NewBuffer(blob) + decoder = gob.NewDecoder(buf) + } + return nil + }) + if err != nil { + return err + } + + if decoder != nil { + for { + loginRecord := new(LoginRecord) + err = decoder.Decode(loginRecord) + if err == nil { + r.loginSessions[loginRecord.TTY] = *loginRecord + } else if err == io.EOF { + // Read all + break + } else { + return errors.Wrap(err, "error decoding login record") + } + } + } + r.log.Debugf("Restored %d open login sessions from disk", len(r.loginSessions)) + + return nil +} diff --git a/x-pack/auditbeat/module/system/login/utmp_c.go b/x-pack/auditbeat/module/system/login/utmp_c.go new file mode 100644 index 00000000000..804683acd53 --- /dev/null +++ b/x-pack/auditbeat/module/system/login/utmp_c.go @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build linux + +// Pure Go reader for UTMP formatted files. +// See utmp(5) and getutent(3) for the C structs and functions this is +// replacing. + +package login + +import ( + "bytes" + "encoding/binary" + "io" + "time" + "unsafe" +) + +var byteOrder = getByteOrder() + +func getByteOrder() binary.ByteOrder { + var b [2]byte + *((*uint16)(unsafe.Pointer(&b[0]))) = 1 + if b[0] == 1 { + return binary.LittleEndian + } + return binary.BigEndian +} + +// UtType represents the ut_type field. See utmp(5). +type UtType int16 + +// Possible values for UtType. +const ( + EMPTY UtType = 0 + RUN_LVL UtType = 1 + BOOT_TIME UtType = 2 + NEW_TIME UtType = 3 + OLD_TIME UtType = 4 + INIT_PROCESS UtType = 5 + LOGIN_PROCESS UtType = 6 + USER_PROCESS UtType = 7 + DEAD_PROCESS UtType = 8 + ACCOUNTING UtType = 9 + + UT_LINESIZE = 32 + UT_NAMESIZE = 32 + UT_HOSTSIZE = 256 +) + +// utmpC is a Go representation of the C utmp struct that the UTMP files consist of. +type utmpC struct { + Type UtType + + // Alignment + _ [2]byte + + Pid int32 + Device [UT_LINESIZE]byte + Terminal [4]byte + Username [UT_NAMESIZE]byte + Hostname [UT_HOSTSIZE]byte + + ExitStatusTermination int16 + ExitStatusExit int16 + + SessionID int32 + + TimeSeconds int32 + TimeMicroseconds int32 + + IP [4]int32 + + Unused [20]byte +} + +// Utmp contains a Go version of UtmpC. +type Utmp struct { + UtType UtType + UtPid int + UtLine string + UtUser string + UtHost string + UtTv time.Time + UtAddrV6 [4]uint32 +} + +// newUtmp creates a Utmp out of a utmpC. +func newUtmp(utmp *utmpC) *Utmp { + // See utmp(5) for the utmp struct fields. + return &Utmp{ + UtType: utmp.Type, + UtPid: int(utmp.Pid), + UtLine: byteToString(utmp.Device[:]), + UtUser: byteToString(utmp.Username[:]), + UtHost: byteToString(utmp.Hostname[:]), + UtTv: time.Unix(int64(utmp.TimeSeconds), int64(utmp.TimeMicroseconds)*1000), + UtAddrV6: [4]uint32{uint32(utmp.IP[0]), uint32(utmp.IP[1]), uint32(utmp.IP[2]), uint32(utmp.IP[3])}, + } +} + +// byteToString converts a NULL terminated char array to a Go string. +func byteToString(b []byte) string { + n := bytes.IndexByte(b, 0) + return string(b[:n]) +} + +// ReadNextUtmp reads the next UTMP entry in a reader pointing to UTMP formatted data. +func ReadNextUtmp(r io.Reader) (*Utmp, error) { + utmpC := new(utmpC) + + err := binary.Read(r, byteOrder, utmpC) + if err != nil { + return nil, err + } + + return newUtmp(utmpC), nil +} diff --git a/x-pack/auditbeat/tests/files/wtmp b/x-pack/auditbeat/tests/files/wtmp new file mode 100644 index 0000000000000000000000000000000000000000..4b49a1c0b74971467e29b7589631033d46aa06a2 GIT binary patch literal 384 zcmZQ)U|=Y+VqhpJDb_avvT*^BbXj70QDR;RE+q_x26_g1Mtb;+!KrsRNo?=(jER}c M%D~0I#6-wn0BThW7ytkO literal 0 HcmV?d00001 diff --git a/x-pack/auditbeat/tests/system/auditbeat_xpack.py b/x-pack/auditbeat/tests/system/auditbeat_xpack.py index bf3b3edf65c..af8e5ae68fa 100644 --- a/x-pack/auditbeat/tests/system/auditbeat_xpack.py +++ b/x-pack/auditbeat/tests/system/auditbeat_xpack.py @@ -29,14 +29,17 @@ def setUp(self): ) # Adapted from metricbeat.py - def check_metricset(self, module, metricset, fields=[], errors_allowed=False, warnings_allowed=False): + def check_metricset(self, module, metricset, fields=[], extras={}, errors_allowed=False, warnings_allowed=False): """ Method to test a metricset for its fields """ + # Set to 1 hour so we only test one Fetch + extras["period"] = "1h" + self.render_config_template(modules=[{ "name": module, "datasets": [metricset], - "period": "10s", + "extras": extras, }]) proc = self.start_beat() self.wait_until(lambda: self.output_lines() > 0) diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index e8007792c2a..ce45b0bbf9b 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -22,6 +22,24 @@ def test_metricset_host(self): # Metricset is experimental and that generates a warning, TODO: remove later self.check_metricset("system", "host", COMMON_FIELDS + fields, warnings_allowed=True) + @unittest.skipUnless(sys.platform == "linux2", "Only implemented for Linux") + @unittest.skipIf(sys.byteorder != "little", "Test only implemented for little-endian systems") + def test_metricset_login(self): + """ + login metricset collects information about logins (successful and failed) and system restarts. + """ + + fields = ["event.origin", "event.outcome", "message", "process.pid", "source.ip", + "user.name", "user.terminal"] + + config = { + "login.wtmp_file_pattern": os.path.abspath(os.path.join(self.beat_path, "tests/files/wtmp")), + "login.btmp_file_pattern": "-1" + } + + # Metricset is experimental and that generates a warning, TODO: remove later + self.check_metricset("system", "login", COMMON_FIELDS + fields, config, warnings_allowed=True) + @unittest.skipIf(sys.platform == "win32", "Not implemented for Windows") @unittest.skipIf(sys.platform == "linux2" and platform.linux_distribution()[0] != "debian", "Only implemented for Debian") From 3ad3a9417843af89ebbbb1102dc54f2f9af6d78f Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Sun, 3 Feb 2019 23:18:31 +0000 Subject: [PATCH 2/5] Add event.outcome. --- x-pack/auditbeat/module/system/_meta/fields.yml | 11 +++++++++++ x-pack/auditbeat/module/system/fields.go | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index fd47f2be771..b360cf75e82 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -13,6 +13,17 @@ description: > Origin of the event. This can be a file path (e.g. `/var/log/log.1`), or the name of the system component that supplied the data (e.g. `netlink`). + - name: outcome + type: keyword + description: > + The outcome of the event. + + If the event describes an action, this fields contains the outcome of + that action. Examples outcomes are `success` and `failure`. Warning: In + future versions of ECS, we plan to provide a list of acceptable values + for this field, please use with caution. + + example: success - name: user type: group diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index a9234035777..7ade7792246 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -18,5 +18,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy0WF+L27gXfc+nuPSlE0g9/OBHWfKw0O7CbqHdDjsp7NtEtm5s7ci6RleeTPrpF8l2YidyMum4hkKjkc45uv90pXfwiLsl8I4dljMAp5zGJby5DwNvZgASObOqcorMEn6dAQCsCmQEYRFcgbBRqCVDjgatcCgh3YXxBhNKkrXGZAZgUaNgXAI+V2hVicYJPYMWYDmbAbwDI0o/4wmNC1xuV+ESckt1FX53k/3/u9lkVa5MGOoWPOJuS1a2Y5E9+O9rWAe0CXoDZwKrQjFkwkCKIGCjNEIlXAE3mOQJrG+fhL3VlPt/yf/W88UejWyA8ZI6yNYEGZUVGTQOXCEccF1VWqEMU6RwosM26LQyj+t50rdFzWhfbAqHtlQmmPVKY6zalZ12TzvQ0WwmEbVUcddEgPsS+zILYrcfPEU6K9R/aw+whoyME8p0wacDLiizIVsKvy7prRoLv+471trzQOVUiQMBjWJNJh8Mn5Hsv28BCJQBIwwxZmQkJxHGlMiNcErh8BrOj0QOPFaMpzUgWvUdZYQsJdIozDV89+hAbVpPMIgDR0yAF/adDCb+Z0TAMHRfJOCvXgJ28N1vr2oBIds+3q/OCqLNhtEljNkEjl8ddHhUHwFnvO9VTmePP1u0GJOKOf0HOeDT7zEKYbNCOcxcbSfc0AC2rZ/Pv7x/eP//eUxEKWJe/AHuLx9+AyGlRWaM+k5VEaKjwQscn+7OUxBHKI6L5wWWNXGvfPYqJoiUaheShSp/oCuTd6V/gHFaLg8KKy2cRzwiHbf6RZt8vd+Dtt7O0DjiBdRpbVy9gK0ykrY8T6KKTtLptWrCMd8o+SIyP/LPCPVGlErvJiVvIFt6i7IQbgESUyXMAjYWMWV5ySJPaFmRmVRXixknfERrUE/Ht4qE6FtuaU6l7GNTZI8ix1d1Hy3G2QwSBpRhJ7RG6XtDiyU9oez4p+lMpjol7hpRMHZOxILllVSRWOnYWntMydZCjh1QU1L1T6YYn1YZmml310JGz6MmBidpJju6FnO0q2T1/fXdckfmwaIkdVkKu/sBwGZhDLO2ekq3fPv782n92V/n+hTXFB8PcPHs9pMY/FDk8L6+3vysgxTgG6M9KTs9a6ljxNezDfvUA1c+Ldcf3qejZFLZqTf2lqGgEj00Zo6GId5LnAL1hMcwwJ2l3IoSHIGtDQgHmnI10gX4wHzoxWxUiMNnd7W52ycATzB4AoCvBj4rUz8vwBWK/aXUZ0iOGXET6iPhcNJRd/Io/RezKwWuA9yFjmHXkPI+hb3YSljn7683Ke7IyP3f3jJUVvlS1qw66vPiaQznUxkuxMKLPAH74D/Nazibbwd6ZRzmeJwiV9KP5V4lmCObG7tIXfZtB3jevXuvtbPhxlBTrfcjyjHqzdWe9Mp/lic/nMj2sAncEbNKNcKT0DVyeA5ecyEkbR/29hjBvBlsuhBc+EBXpnk0DRjh5XW+ONj2QSoWqUa5Xoygrg0dmD1Hk+xSmBwt1QyCfY6RwfC+qykHZeYLECZmnICY2V3l+qDbAs3QZcE3Xvstuuw2DEtgxJJHQB11UeLvCGgCR7gYNIgn3u+1joLdQ1b4DY2nzklP13wvcvYqvEjvBjWm2+hWcBAArYBk9l8AAAD//y0/V6s=" + return "eJy0Wd9v2zgSfvdfMehLE8BVcMChOPjhgLZX3AZot8Emxe5bPBbHEjcUKXCoOO5fvyBFyXJM2XGjCggQMeT3fZxfHDHv4IG2C+AtO6pmAE46RQt4cxsG3swABHFuZe2k0Qv47wwA4K4kJkBL4EqCtSQlGArSZNGRgNU2jLeYUBnRKMpmAJYUIdMC6KkmKyvSDtUMIsBiNgN4BxorP+ORtAtcblvTAgprmjq8d5P9791sY2UhdRjqFjzQdmOsiGOJPfjnW1gHZh30Bs4M7krJkKOGFQHCWiqCGl0JF5QVGSyvHtFeKVP4n+xfy8t5j2ZsgPGSOshogtxUtdGkHbgSHXBT10qSCFMEOuywNTkl9cPyMtvfXuNyU9H5+7srqVu8v8lZP+V6MBxhVsSAGjD3cHNw3h7Rx7nRDqXmsGaH3KOF7bULM/j8hFWtiLuJHCJmyU2eE/MSUAtYrlGqxtIygz/RaqmLBVzrHm/duMYSPJJlaTT7TXz+dDuHDUGtUIMzUFvzKIV3lZLs/AzMc6odrhTBI6qGeAcXXNRtZw51iEdomGAjXQk5NkF6Zx5qd7CAqHkYoQ2TfXGAOrKV1CHYz3VhXNn5z9NmQx1tiGXYCJlOmATwUOJQZmnY9YOHSEeF+mfpAZa7KGlLggq4IPXa2ApbA+/WjBWF3mfPtA48UDvZZ8VQsTK62Bs+Itk/3wMQSA0atWHKjRacJRhXxrgRToGOzuH8aIwDj5XiiQYkK3+QSJCtjFGE+hy+W3Ig19ETDLjjSAnwwn4YTZl/TQjYD90XCfh9UBY7+O7dq5pDqIEfb++OCjLrNZPLmPIJHH+30+FRfQQc8b5XOZ09fotoKSaZcvpPcsD1/1IUaPNSOsp9fZ2QbAgbT7Wn/7y/f//vy5SIClNe/Anurx8+AQphiZmSvpN1gujZ4AmO65vjFIYTFM+L5wmWpeFB+RxUTMCVaVx77Na+zZK66Er/HsZhudwprBU6j/iMdNzqJ23y7bYHjd7OSTvDc2hWjXbNHDZSC7Phyyyp6CCdXqsmNF+tkq+Y+5G/RqjXWEm1nZS8hYz0lkSJbg6CVhL1HNaWaMXilEViqzOproiZJnwgq0lNx3eXCNG3HGkOpfSxifkDFvSq7iNiHM0g1CA1O1SKhO/YLVXmkUTHP01nMtUpcdOKgrFzIhUsr6RKxErHFu0xJVuEHDugpqQankwpPiVz0tPuLkImz6M2BidpJju6iDnaVbL88fpuuSPzYEmSpqrQbn8CsF2YwmysmtIt3//4clh/+s+5IcU5xccDnDy7/SQGP5Q4vM+vN7/qIAX4zmQPys7AWvI54uvZ9vvUHVcxLdf/vU9HyYS0U2/sLUNpKvLQlDuzH+KDxClJTXgMA9xYU1iswBmwjQZ0oEwhR7oAH5j3g5hNCnH05M42d7wC8AR7VwDwTcMXqZuneMMk20ulgnIT72dGwuGgo+7kmdXflJ8pcBngTnQM25aU+xT2Ymu04bbpYkVbo0X/t7cMtZW+lLWrnvV56TSG46kMJ2LhRZ6APvgP8xqO5tuOXmpHBT1PkTPpx3KvRubE5sY+pE77tgM87t7ea3E2XGjTVut+RDomtT7bk175r/LkhwPZHjaDG8Msd1ef8cq1RGE29709RjAv9jZdIpc+0KVur7IDRrgPv5zvbHsvJONKkVjOR1CX2uyYPUeb7AJ1QdY0DMg+x4ymcOuuTAFSX84Bdco4ATG329oNQTcl6X2XBd947Vfk8qswLICJKh4BdaaLEv+NQDpwhA+DFvHA+4PWEdnd56Xf0HjqHPR07fMiZ9+F/xNs92pMt9ENchAAUUA2+ycAAP//fzXZ0w==" } From 3347e264724e9ef9735b6f1bc72279a300b0337c Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Sun, 3 Feb 2019 23:20:35 +0000 Subject: [PATCH 3/5] Replace event.type with event.kind --- x-pack/auditbeat/module/system/login/_meta/data.json | 2 +- x-pack/auditbeat/module/system/login/login.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/auditbeat/module/system/login/_meta/data.json b/x-pack/auditbeat/module/system/login/_meta/data.json index 311893d5926..c4fab5e888a 100644 --- a/x-pack/auditbeat/module/system/login/_meta/data.json +++ b/x-pack/auditbeat/module/system/login/_meta/data.json @@ -7,10 +7,10 @@ "event": { "action": "user_login", "dataset": "login", + "kind": "event" "module": "system", "origin": "/var/log/wtmp.1", "outcome": "success", - "type": "event" }, "message": "Login by user vagrant (UID: 1000) on pts/1 (PID: 17559) from 10.0.2.2 (IP: 10.0.2.2)", "process": { diff --git a/x-pack/auditbeat/module/system/login/login.go b/x-pack/auditbeat/module/system/login/login.go index 7af40d1e6a1..23509d67123 100644 --- a/x-pack/auditbeat/module/system/login/login.go +++ b/x-pack/auditbeat/module/system/login/login.go @@ -166,7 +166,7 @@ func (ms *MetricSet) loginEvent(loginRecord *LoginRecord) mb.Event { Timestamp: loginRecord.Timestamp, RootFields: common.MapStr{ "event": common.MapStr{ - "type": eventTypeEvent, + "kind": eventTypeEvent, "action": loginRecord.Type.string(), "origin": loginRecord.Origin, }, From 735d3e5cc80768f9278086f13320dda0f7557b33 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Sun, 3 Feb 2019 23:45:36 +0000 Subject: [PATCH 4/5] Field docs. --- auditbeat/docs/fields.asciidoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index d04632616e2..0a5ae9aa781 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -3733,6 +3733,19 @@ type: keyword Origin of the event. This can be a file path (e.g. `/var/log/log.1`), or the name of the system component that supplied the data (e.g. `netlink`). +-- + +*`event.outcome`*:: ++ +-- +type: keyword + +example: success + +The outcome of the event. +If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. + + -- From 8129a35e0f204a4f48a27e5762e080f3bb3bca68 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Mon, 4 Feb 2019 21:55:27 +0000 Subject: [PATCH 5/5] Fix data.json --- x-pack/auditbeat/module/system/login/_meta/data.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/auditbeat/module/system/login/_meta/data.json b/x-pack/auditbeat/module/system/login/_meta/data.json index c4fab5e888a..3518c24535a 100644 --- a/x-pack/auditbeat/module/system/login/_meta/data.json +++ b/x-pack/auditbeat/module/system/login/_meta/data.json @@ -7,10 +7,10 @@ "event": { "action": "user_login", "dataset": "login", - "kind": "event" + "kind": "event", "module": "system", "origin": "/var/log/wtmp.1", - "outcome": "success", + "outcome": "success" }, "message": "Login by user vagrant (UID: 1000) on pts/1 (PID: 17559) from 10.0.2.2 (IP: 10.0.2.2)", "process": {