Merge pull request #50 from itkovian/jsonify

Jsonify

Jenkinsfile

@@ -1,6 +1,6 @@
 #!/usr/bin/env groovy
 
-def LOGSTASH_VERSION = "7.10.2"
+def VECTOR_VERSION = "0.37.1"
 
 node {
     stage('checkout git') {
@@ -9,13 +9,14 @@ node {
         sh 'git clean -fxd'
     }
 
-    stage('install logstash') {
-        sh "wget -nv https://artifacts.elastic.co/downloads/logstash/logstash-${LOGSTASH_VERSION}-linux-x86_64.tar.gz"
-        sh "tar -xzf logstash-${LOGSTASH_VERSION}-linux-x86_64.tar.gz"
-        env.PATH = "${pwd()}/logstash-${LOGSTASH_VERSION}/bin:${env.PATH}"
+    stage('install vector') {
+        sh "wget -nv https://packages.timber.io/vector/${VECTOR_VERSION}/vector-${VECTOR_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
+        sh "tar -xzf vector-${VECTOR_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
+        env.PATH = "${pwd()}/vector-x86_64-unknown-linux-gnu/bin/:${env.PATH}"
     }
 
     stage('test') {
+        sh 'python3.6 -V'
         sh 'pip3 install --ignore-installed --prefix $PWD/.vsc-tox tox'
         sh 'export PATH=$PWD/.vsc-tox/bin:$PATH && export PYTHONPATH=$PWD/.vsc-tox/lib/python$(python3 -c "import sys; print(\\"%s.%s\\" % sys.version_info[:2])")/site-packages:$PYTHONPATH && tox -v -c tox.ini'
         sh 'rm -r $PWD/.vsc-tox'

README.md

@@ -1,25 +1,22 @@
-# Logstash patterns
+# Grok patterns
 
 Patterns for parsing and structuring log messages for different
-services with [Logstash](http://logstash.net).
+services with [Vector](http://vector.dev).
 
 Use [this app](https://grokdebug.herokuapp.com/) for debugging your
 patterns! Be careful, there might be subtle differences!
 
 When issuing pull requests to this repository, don't forget to include an example of the messages your commits try to parse!!
 
+## Adding pattern
 
-# Adding pattern
-
-Develop a new pattern typically in its own file, and call the main new pattern `<something>_MSG`.
-
-Than you can do:
- * extend the `RSYSLOGMESSAGE` in the `rsyslog` file with a new pattern,
-   by joining it with a `|` and placing the new pattern before the `GREEDYDATA` one.
-   This requires only a new rpm and no configuration changes. This is very convient for
-   testing.
- * add the new pattern to the list in the test configuration to the `grok`
-   filter before the `RSYSLOGMESSAGE` as follows: `%{RSYSLOGPREFIX}%{<something>_MSG}`
-   This requires a new rpm and configuration change in quattor too. Should only be done
-   when a pattern is considered stable.
+Develop a new PATTERN typically in its own file, and call the main new pattern `<PATTERN>_MSG`.
 
+Then you can:
+  * Add the new message pattern to the Vector config file in the section `[transforms.syslog]`. Typically,
+    you will add it to the top-level list of patterns to try (second argument of parse_groks).
+  * Include the definitions for this grok pattern in its own JSON file `<PATTERN>.json`, in the format 
+    given by the existing JSON files. You will need to add this filename to the `vector.toml` config file
+    under the parse_groks argument `alias_sources`.
+  * Add tests for your pattern in `tests/<PATTERN>.toml`. Specify the input, and compose the VRL program that 
+    asserts the expected structured output.

files/ceph.json

@@ -0,0 +1,18 @@
+{
+  "CEPH_BYTES": "%{NONNEGINT} (?:(?:k|M|G|T|P)?B)",
+  "CEPH_MSG": "%{DATA} (?:%{CEPH_PGMAP}|%{CEPH_SLOW_REQUEST})",
+  "CEPH_PGMAP": ".*? pgmap v(?<pgmapv>[0-9]+): %{NONNEGINT:ceph.pgs} pgs: (?:%{CEPH_PGMAP_DETAIL})(?:; %{CEPH_PGMAP_VOLUME})?(?:; %{CEPH_PGMAP_ACT})?(?:; %{CEPH_PGMAP_OBJ_DEGRADED})?(?:; %{CEPH_PGMAP_OBJ_MISPLACED})?(?:; %{CEPH_PGMAP_OBJ_RECOV})?$",
+  "CEPH_PGMAP_ACT": "(%{CEPH_BYTES:ceph.actread}/s rd, )?(%{CEPH_BYTES:ceph.actwrite}/s wr, )?%{NONNEGINT:ceph.actops} op/s",
+  "CEPH_PGMAP_DETAIL": "(?:%{CEPH_PGMAP_DETAIL_PAT}(?:, %{CEPH_PGMAP_DETAIL_PAT})*)",
+  "CEPH_PGMAP_DETAIL_ACTIVE_CLEAN": "%{NONNEGINT:ceph.active_clean} active\\+clean",
+  "CEPH_PGMAP_DETAIL_PAT": "(?:%{CEPH_PGMAP_DETAIL_ACTIVE_CLEAN}|%{CEPH_PGMAP_DETAIL_REST})",
+  "CEPH_PGMAP_DETAIL_REST": "%{NONNEGINT} (?:[^ ,;]+)",
+  "CEPH_PGMAP_OBJ_DEGRADED": "%{NONNEGINT:ceph.objdegraded}/%{NONNEGINT:ceph.objtotal} objects degraded [^;]*",
+  "CEPH_PGMAP_OBJ_MISPLACED": "%{NONNEGINT:ceph.objmisplaced}/%{NONNEGINT:ceph.objtotal} objects misplaced [^;]*",
+  "CEPH_PGMAP_OBJ_RECOV": "%{CEPH_BYTES:ceph.objrecovthr}/s, %{NONNEGINT:ceph.objrecovhz} objects/s recovering",
+  "CEPH_PGMAP_VOLUME": "%{CEPH_BYTES:ceph.volumedata} data, %{CEPH_BYTES:ceph.volumeused} used, %{CEPH_BYTES:ceph.volumeavail} / %{CEPH_BYTES:ceph.volumetotal} avail",
+  "CEPH_SLOW_REQUEST": ".*? (?<osd>osd.%{NONNEGINT}) .*? (?:%{CEPH_SLOW_REQUEST_SINGLE}|%{CEPH_SLOW_REQUEST_OLDEST}) %{CEPH_SLOW_REQUEST_SECONDS} (?:secs|seconds old)",
+  "CEPH_SLOW_REQUEST_OLDEST": "%{NONNEGINT:ceph.slowrequestnumber} slow requests, .*? oldest blocked for .*?",
+  "CEPH_SLOW_REQUEST_SECONDS": "(?<slowrequestseconds>%{NONNEGINT})(?:\\.%{NONNEGINT})?",
+  "CEPH_SLOW_REQUEST_SINGLE": "slow request"
+}

files/gpfs.json

@@ -0,0 +1,5 @@
+{
+  "GPFSDATA": "%{GREEDYDATA}",
+  "GPFSDATE": "%{DAY}%{SPACE}%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{YEAR}",
+  "GPFSLOG": "%{GPFSDATE:gpfs.timestamp}:%{SPACE}%{GPFSDATA:gpfs.realmsg}"
+}

files/grok-patterns.json

@@ -0,0 +1,91 @@
+{
+
+"USERNAME": "[a-zA-Z0-9._-]+",
+"USER": "%{USERNAME}",
+"EMAILLOCALPART": "[a-zA-Z][a-zA-Z0-9_.+-=:]+",
+"EMAILADDRESS": "%{EMAILLOCALPART}@%{HOSTNAME}",
+"HTTPDUSER": "%{EMAILADDRESS}|%{USER}",
+"INT": "(?:[+-]?(?:[0-9]+))",
+"BASE10NUM": "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))",
+"NUMBER": "(?:%{BASE10NUM})",
+"BASE16NUM": "(?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))",
+"BASE16FLOAT": "\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b",
+
+"POSINT": "\b(?:[1-9][0-9]*)\b",
+"NONNEGINT": "\b(?:[0-9]+)\b",
+"WORD": "\b\w+\b",
+"NOTSPACE": "\S+",
+"SPACE": "\s*",
+"DATA": ".*?",
+"GREEDYDATA": ".*",
+"QUOTEDSTRING": "(?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))",
+"UUID": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}",
+
+"MAC": "(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})",
+"CISCOMAC": "(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})",
+"WINDOWSMAC": "(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})",
+"COMMONMAC": "(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})",
+"IPV6": "((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?",
+"IPV4": "(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])",
+"IP": "(?:%{IPV6}|%{IPV4})",
+"HOSTNAME": "\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)",
+"IPORHOST": "(?:%{IP}|%{HOSTNAME})",
+"HOSTPORT": "%{IPORHOST}:%{POSINT}",
+
+"PATH": "(?:%{UNIXPATH}|%{WINPATH})",
+"UNIXPATH": "(/([\w_%!$@:.,~-]+|\\.)*)+",
+"TTY": "(?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))",
+"WINPATH": "(?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+",
+"URIPROTO": "[A-Za-z]+(\+[A-Za-z+]+)?",
+"URIHOST": "%{IPORHOST}(?::%{POSINT:port})?",
+"URIPATH": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+",
+"#URIPARAM": "\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?",
+"URIPARAM": "\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*",
+"URIPATHPARAM": "%{URIPATH}(?:%{URIPARAM})?",
+"URI": "%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?",
+
+"MONTH": "\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b",
+"MONTHNUM": "(?:0?[1-9]|1[0-2])",
+"MONTHNUM2": "(?:0[1-9]|1[0-2])",
+"MONTHDAY": "(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])",
+
+"DAY": "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)",
+
+"YEAR": "(?>\d\d){1,2}",
+"HOUR": "(?:2[0123]|[01]?[0-9])",
+"MINUTE": "(?:[0-5][0-9])",
+"SECOND": "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)",
+"TIME": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])",
+"DATE_US": "%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}",
+"DATE_EU": "%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}",
+"ISO8601_TIMEZONE": "(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))",
+"ISO8601_SECOND": "(?:%{SECOND}|60)",
+"TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?",
+"DATE": "%{DATE_US}|%{DATE_EU}",
+"DATESTAMP": "%{DATE}[- ]%{TIME}",
+"TZ": "(?:[PMCE][SD]T|UTC)",
+"DATESTAMP_RFC822": "%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}",
+"DATESTAMP_RFC2822": "%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}",
+"DATESTAMP_OTHER": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}",
+"DATESTAMP_EVENTLOG": "%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}",
+"HTTPDERROR_DATE": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}",
+
+"SYSLOGTIMESTAMP": "%{MONTH} +%{MONTHDAY} %{TIME}",
+"PROG": "[\x21-\x5a\x5c\x5e-\x7e]+",
+"SYSLOGPROG": "%{PROG:program}(?:\[%{POSINT:pid}\])?",
+"SYSLOGHOST": "%{IPORHOST}",
+"SYSLOGFACILITY": "<%{NONNEGINT:facility}.%{NONNEGINT:priority}>",
+"HTTPDATE": "%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}",
+
+"QS": "%{QUOTEDSTRING}",
+
+"SYSLOGBASE": "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:",
+"COMMONAPACHELOG": "%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)",
+"COMBINEDAPACHELOG": "%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}",
+"HTTPD20_ERRORLOG": "\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}",
+"HTTPD24_ERRORLOG": "\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}",
+"HTTPD_ERRORLOG": "%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}",
+
+
+"LOGLEVEL": "([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)"
+}

files/hpc-ugent-patterns.json

@@ -0,0 +1,3 @@
+{
+  "USERNAME_PAT": "%{USERNAME}"
+}