-
Notifications
You must be signed in to change notification settings - Fork 104
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #50 from itkovian/jsonify
Jsonify
- Loading branch information
Showing
46 changed files
with
927 additions
and
829 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,25 +1,22 @@ | ||
# Logstash patterns | ||
# Grok patterns | ||
|
||
Patterns for parsing and structuring log messages for different | ||
services with [Logstash](http://logstash.net). | ||
services with [Vector](http://vector.dev). | ||
|
||
Use [this app](https://grokdebug.herokuapp.com/) for debugging your | ||
patterns! Be careful, there might be subtle differences! | ||
|
||
When issuing pull requests to this repository, don't forget to include an example of the messages your commits try to parse!! | ||
|
||
## Adding pattern | ||
|
||
# Adding pattern | ||
|
||
Develop a new pattern typically in its own file, and call the main new pattern `<something>_MSG`. | ||
|
||
Than you can do: | ||
* extend the `RSYSLOGMESSAGE` in the `rsyslog` file with a new pattern, | ||
by joining it with a `|` and placing the new pattern before the `GREEDYDATA` one. | ||
This requires only a new rpm and no configuration changes. This is very convient for | ||
testing. | ||
* add the new pattern to the list in the test configuration to the `grok` | ||
filter before the `RSYSLOGMESSAGE` as follows: `%{RSYSLOGPREFIX}%{<something>_MSG}` | ||
This requires a new rpm and configuration change in quattor too. Should only be done | ||
when a pattern is considered stable. | ||
Develop a new PATTERN typically in its own file, and call the main new pattern `<PATTERN>_MSG`. | ||
|
||
Then you can: | ||
* Add the new message pattern to the Vector config file in the section `[transforms.syslog]`. Typically, | ||
you will add it to the top-level list of patterns to try (second argument of parse_groks). | ||
* Include the definitions for this grok pattern in its own JSON file `<PATTERN>.json`, in the format | ||
given by the existing JSON files. You will need to add this filename to the `vector.toml` config file | ||
under the parse_groks argument `alias_sources`. | ||
* Add tests for your pattern in `tests/<PATTERN>.toml`. Specify the input, and compose the VRL program that | ||
asserts the expected structured output. |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
{ | ||
"CEPH_BYTES": "%{NONNEGINT} (?:(?:k|M|G|T|P)?B)", | ||
"CEPH_MSG": "%{DATA} (?:%{CEPH_PGMAP}|%{CEPH_SLOW_REQUEST})", | ||
"CEPH_PGMAP": ".*? pgmap v(?<pgmapv>[0-9]+): %{NONNEGINT:ceph.pgs} pgs: (?:%{CEPH_PGMAP_DETAIL})(?:; %{CEPH_PGMAP_VOLUME})?(?:; %{CEPH_PGMAP_ACT})?(?:; %{CEPH_PGMAP_OBJ_DEGRADED})?(?:; %{CEPH_PGMAP_OBJ_MISPLACED})?(?:; %{CEPH_PGMAP_OBJ_RECOV})?$", | ||
"CEPH_PGMAP_ACT": "(%{CEPH_BYTES:ceph.actread}/s rd, )?(%{CEPH_BYTES:ceph.actwrite}/s wr, )?%{NONNEGINT:ceph.actops} op/s", | ||
"CEPH_PGMAP_DETAIL": "(?:%{CEPH_PGMAP_DETAIL_PAT}(?:, %{CEPH_PGMAP_DETAIL_PAT})*)", | ||
"CEPH_PGMAP_DETAIL_ACTIVE_CLEAN": "%{NONNEGINT:ceph.active_clean} active\\+clean", | ||
"CEPH_PGMAP_DETAIL_PAT": "(?:%{CEPH_PGMAP_DETAIL_ACTIVE_CLEAN}|%{CEPH_PGMAP_DETAIL_REST})", | ||
"CEPH_PGMAP_DETAIL_REST": "%{NONNEGINT} (?:[^ ,;]+)", | ||
"CEPH_PGMAP_OBJ_DEGRADED": "%{NONNEGINT:ceph.objdegraded}/%{NONNEGINT:ceph.objtotal} objects degraded [^;]*", | ||
"CEPH_PGMAP_OBJ_MISPLACED": "%{NONNEGINT:ceph.objmisplaced}/%{NONNEGINT:ceph.objtotal} objects misplaced [^;]*", | ||
"CEPH_PGMAP_OBJ_RECOV": "%{CEPH_BYTES:ceph.objrecovthr}/s, %{NONNEGINT:ceph.objrecovhz} objects/s recovering", | ||
"CEPH_PGMAP_VOLUME": "%{CEPH_BYTES:ceph.volumedata} data, %{CEPH_BYTES:ceph.volumeused} used, %{CEPH_BYTES:ceph.volumeavail} / %{CEPH_BYTES:ceph.volumetotal} avail", | ||
"CEPH_SLOW_REQUEST": ".*? (?<osd>osd.%{NONNEGINT}) .*? (?:%{CEPH_SLOW_REQUEST_SINGLE}|%{CEPH_SLOW_REQUEST_OLDEST}) %{CEPH_SLOW_REQUEST_SECONDS} (?:secs|seconds old)", | ||
"CEPH_SLOW_REQUEST_OLDEST": "%{NONNEGINT:ceph.slowrequestnumber} slow requests, .*? oldest blocked for .*?", | ||
"CEPH_SLOW_REQUEST_SECONDS": "(?<slowrequestseconds>%{NONNEGINT})(?:\\.%{NONNEGINT})?", | ||
"CEPH_SLOW_REQUEST_SINGLE": "slow request" | ||
} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"GPFSDATA": "%{GREEDYDATA}", | ||
"GPFSDATE": "%{DAY}%{SPACE}%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{YEAR}", | ||
"GPFSLOG": "%{GPFSDATE:gpfs.timestamp}:%{SPACE}%{GPFSDATA:gpfs.realmsg}" | ||
} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
{ | ||
|
||
"USERNAME": "[a-zA-Z0-9._-]+", | ||
"USER": "%{USERNAME}", | ||
"EMAILLOCALPART": "[a-zA-Z][a-zA-Z0-9_.+-=:]+", | ||
"EMAILADDRESS": "%{EMAILLOCALPART}@%{HOSTNAME}", | ||
"HTTPDUSER": "%{EMAILADDRESS}|%{USER}", | ||
"INT": "(?:[+-]?(?:[0-9]+))", | ||
"BASE10NUM": "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))", | ||
"NUMBER": "(?:%{BASE10NUM})", | ||
"BASE16NUM": "(?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))", | ||
"BASE16FLOAT": "\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b", | ||
|
||
"POSINT": "\b(?:[1-9][0-9]*)\b", | ||
"NONNEGINT": "\b(?:[0-9]+)\b", | ||
"WORD": "\b\w+\b", | ||
"NOTSPACE": "\S+", | ||
"SPACE": "\s*", | ||
"DATA": ".*?", | ||
"GREEDYDATA": ".*", | ||
"QUOTEDSTRING": "(?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))", | ||
"UUID": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}", | ||
|
||
"MAC": "(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})", | ||
"CISCOMAC": "(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})", | ||
"WINDOWSMAC": "(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})", | ||
"COMMONMAC": "(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})", | ||
"IPV6": "((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?", | ||
"IPV4": "(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])", | ||
"IP": "(?:%{IPV6}|%{IPV4})", | ||
"HOSTNAME": "\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)", | ||
"IPORHOST": "(?:%{IP}|%{HOSTNAME})", | ||
"HOSTPORT": "%{IPORHOST}:%{POSINT}", | ||
|
||
"PATH": "(?:%{UNIXPATH}|%{WINPATH})", | ||
"UNIXPATH": "(/([\w_%!$@:.,~-]+|\\.)*)+", | ||
"TTY": "(?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))", | ||
"WINPATH": "(?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+", | ||
"URIPROTO": "[A-Za-z]+(\+[A-Za-z+]+)?", | ||
"URIHOST": "%{IPORHOST}(?::%{POSINT:port})?", | ||
"URIPATH": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+", | ||
"#URIPARAM": "\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?", | ||
"URIPARAM": "\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*", | ||
"URIPATHPARAM": "%{URIPATH}(?:%{URIPARAM})?", | ||
"URI": "%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?", | ||
|
||
"MONTH": "\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b", | ||
"MONTHNUM": "(?:0?[1-9]|1[0-2])", | ||
"MONTHNUM2": "(?:0[1-9]|1[0-2])", | ||
"MONTHDAY": "(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])", | ||
|
||
"DAY": "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)", | ||
|
||
"YEAR": "(?>\d\d){1,2}", | ||
"HOUR": "(?:2[0123]|[01]?[0-9])", | ||
"MINUTE": "(?:[0-5][0-9])", | ||
"SECOND": "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)", | ||
"TIME": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])", | ||
"DATE_US": "%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}", | ||
"DATE_EU": "%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}", | ||
"ISO8601_TIMEZONE": "(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))", | ||
"ISO8601_SECOND": "(?:%{SECOND}|60)", | ||
"TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?", | ||
"DATE": "%{DATE_US}|%{DATE_EU}", | ||
"DATESTAMP": "%{DATE}[- ]%{TIME}", | ||
"TZ": "(?:[PMCE][SD]T|UTC)", | ||
"DATESTAMP_RFC822": "%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}", | ||
"DATESTAMP_RFC2822": "%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}", | ||
"DATESTAMP_OTHER": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}", | ||
"DATESTAMP_EVENTLOG": "%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}", | ||
"HTTPDERROR_DATE": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}", | ||
|
||
"SYSLOGTIMESTAMP": "%{MONTH} +%{MONTHDAY} %{TIME}", | ||
"PROG": "[\x21-\x5a\x5c\x5e-\x7e]+", | ||
"SYSLOGPROG": "%{PROG:program}(?:\[%{POSINT:pid}\])?", | ||
"SYSLOGHOST": "%{IPORHOST}", | ||
"SYSLOGFACILITY": "<%{NONNEGINT:facility}.%{NONNEGINT:priority}>", | ||
"HTTPDATE": "%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}", | ||
|
||
"QS": "%{QUOTEDSTRING}", | ||
|
||
"SYSLOGBASE": "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:", | ||
"COMMONAPACHELOG": "%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)", | ||
"COMBINEDAPACHELOG": "%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}", | ||
"HTTPD20_ERRORLOG": "\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}", | ||
"HTTPD24_ERRORLOG": "\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}", | ||
"HTTPD_ERRORLOG": "%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}", | ||
|
||
|
||
"LOGLEVEL": "([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)" | ||
} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"USERNAME_PAT": "%{USERNAME}" | ||
} |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.