WIP POC Event categorization on Linux auth logs

filebeat/docs/fields.asciidoc

@@ -12004,6 +12004,14 @@ alias to: user.name
 --
 
 
+*`system.auth.ssh.action`*::
++
+--
+The SSH action. Can be one of "Accepted", "Failed", "Invalid".
+
+
+--
+
 *`system.auth.ssh.method`*::
 +
 --

filebeat/module/system/auth/_meta/fields.yml

@@ -31,6 +31,9 @@
     - name: ssh
       type: group
       fields:
+      - name: action
+        description: >
+          The SSH action. Can be one of "Accepted", "Failed", "Invalid".
       - name: method
         description: >
           The SSH authentication method. Can be one of "password" or "publickey".

filebeat/module/system/auth/ingest/pipeline.json

@@ -1,6 +1,8 @@
 {
     "description": "Pipeline for parsing system authorisation/secure logs",
     "processors": [
+        { "set": { "field": "event.kind", "value": "event" } },
+
         {
             "grok": {
                 "field": "message",
@@ -9,33 +11,72 @@
                     "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} user %{DATA:user.name} from %{IPORHOST:source.ip}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sudo(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:long}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:long}, GID=%{NUMBER:group.id:long}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.auth.message}"
+                    "^%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: +%{GREEDYMULTILINE:message}",
+                    "^%{SYSLOGTIMESTAMP:system.auth.timestamp} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: +%{GREEDYMULTILINE:message}"
                 ]
             }
         },
+
         {
-            "remove": {
-                "field": "message"
+            "grok": {
+                "field": "message",
+                "ignore_missing": true,
+                "ignore_failure": true,
+                "patterns": [
+                    "^%{DATA:system.auth.ssh.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
+                    "^%{DATA:system.auth.ssh.action} user %{DATA:user.name} from %{IPORHOST:source.ip}",
+                    "^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}"
+                ]
+            }
+        },
+
+        {
+            "grok": {
+                "field": "message",
+                "ignore_missing": true,
+                "ignore_failure": true,
+                "patterns": [
+                    "^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
+                    "^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:long}",
+                    "^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:long}, GID=%{NUMBER:group.id:long}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$"
+                ]
             }
         },
+
         {
-            "rename": {
-                "field": "system.auth.message",
-                "target_field": "message",
-                "ignore_missing": true
+            "set": {
+                "field": "event.category",
+                "value": "authentication",
+                "if": "ctx.containsKey('process') && ctx.process.containsKey('name') && ctx.process.name == 'sshd'"
+            }
+        },
+        {
+            "set": {
+                "field": "event.action",
+                "value": "ssh_login",
+                "if": "ctx.event.containsKey('category') && ctx.event.category == 'authentication'"
+            }
+        },
+        {
+            "set": {
+                "field": "event.outcome",
+                "value": "success",
+                "if": "ctx.event.containsKey('action') && ctx.event.action == 'ssh_login' && ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('action') && ctx.system.auth.ssh.action == 'Accepted'"
+            }
+        },
+        {
+            "set": {
+                "field": "event.outcome",
+                "value": "failure",
+                "if": "ctx.event.containsKey('action') && ctx.event.action == 'ssh_login' && ctx.system.auth.containsKey('ssh') && ((ctx.system.auth.ssh.containsKey('action') && ctx.system.auth.ssh.action == 'Failed') || (ctx.system.auth.ssh.containsKey('dropped_ip')))"
             }
         },
+
         {
             "set": {
                 "field": "source.ip",
                 "value": "{{system.auth.ssh.dropped_ip}}",
-                "if": "ctx.containsKey('system') && ctx.system.containsKey('auth') && ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('dropped_ip')"
+                "if": "ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('dropped_ip')"
             }
         },
         {

filebeat/module/system/auth/test/test.log

@@ -1,10 +1,31 @@
 Feb 21 21:54:44 localhost sshd[3402]: Accepted publickey for vagrant from 10.0.2.2 port 63673 ssh2: RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84
 Feb 23 00:13:35 localhost sshd[7483]: Accepted password for vagrant from 192.168.33.1 port 58803 ssh2
-Feb 21 21:56:12 localhost sshd[3430]: Invalid user test from 10.0.2.2
 Feb 20 08:35:22 slave22 sshd[5774]: Failed password for root from 116.31.116.24 port 29160 ssh2
-Feb 21 23:35:33 localhost sudo: vagrant : TTY=pts/0 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
+Jan  4 19:02:26 ubuntu-xenial sudo:  vagrant : TTY=pts/0 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
+Jan  4 19:02:26 ubuntu-xenial sudo: pam_unix(sudo:session): session opened for user root by vagrant(uid=0)
+Jan  4 19:02:26 ubuntu-xenial sudo: pam_unix(sudo:session): session closed for user root
 Feb 19 15:30:04 slave22 sshd[18406]: Did not receive identification string from 123.57.245.163
 Feb 23 00:08:48 localhost sudo: vagrant : TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/cat /var/log/secure
 Feb 24 00:13:02 precise32 sudo:      tsg : user NOT in sudoers ; TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
 Feb 22 11:47:05 localhost groupadd[6991]: new group: name=apache, GID=48
 Feb 22 11:47:05 localhost useradd[6995]: new user: name=apache, UID=48, GID=48, home=/usr/share/httpd, shell=/sbin/nologin
+Jan  4 15:43:53 ubuntu-xenial sshd[2124]: Accepted publickey for vagrant from 10.0.2.2 port 64484 ssh2: RSA SHA256:sKpJLT3xGfw7fCcOslGbKVMyIm+2MbYVCqVkV5a1zeQ
+Jan  4 15:43:53 ubuntu-xenial sshd[2124]: pam_unix(sshd:session): session opened for user vagrant by (uid=0)
+Jan  4 15:43:53 ubuntu-xenial systemd: pam_unix(systemd-user:session): session opened for user vagrant by (uid=0)
+Jan  4 15:43:53 ubuntu-xenial systemd-logind[1055]: New session 2 of user vagrant.
+Jan  4 16:20:27 ubuntu-xenial sshd[9448]: Received disconnect from 10.0.2.2 port 50729:11: disconnected by user
+Jan  4 16:20:27 ubuntu-xenial sshd[9448]: Disconnected from 10.0.2.2 port 50729
+Jan  4 16:20:27 ubuntu-xenial sshd[9415]: pam_unix(sshd:session): session closed for user vagrant
+Jan  4 16:20:27 ubuntu-xenial systemd-logind[1055]: Removed session 14.
+Jan  4 15:44:27 ubuntu-xenial sudo: pam_unix(sudo:session): session opened for user root by vagrant(uid=0)
+Jan  4 15:44:27 ubuntu-xenial su[8849]: Successful su for root by root
+Jan  4 15:44:27 ubuntu-xenial su[8849]: + /dev/pts/1 root:root
+Jan  4 15:44:27 ubuntu-xenial su[8849]: pam_unix(su:session): session opened for user root by vagrant(uid=0)
+Jan  4 15:44:27 ubuntu-xenial su[8849]: pam_systemd(su:session): Cannot create session: Already running in a session
+Jan  4 16:04:14 ubuntu-xenial sshd[1061]: Received signal 15; terminating.
+Jan  4 16:04:14 ubuntu-xenial sshd[9230]: Server listening on 0.0.0.0 port 22.
+Jan  4 16:04:14 ubuntu-xenial sshd[9230]: Server listening on :: port 22.
+Jan  4 16:04:14 ubuntu-xenial sudo: pam_unix(sudo:session): session closed for user root
+Jan  4 16:09:22 ubuntu-xenial sshd[9356]: Invalid user mat from 10.0.2.2
+Jan  4 16:09:22 ubuntu-xenial sshd[9356]: input_userauth_request: invalid user mat [preauth]
+Jan  4 16:09:22 ubuntu-xenial sshd[9356]: Connection closed by 10.0.2.2 port 49945 [preauth]