Skip to content

Commit

Permalink
Comply with PostgreSQL database name format (#7198) (#7333)
Browse files Browse the repository at this point in the history
Support `_` and `$` in PostgreSQL database names as all of those are valid characters in a name.

Reference:
> SQL identifiers and key words must begin with a letter (a-z, but also letters with diacritical marks and non-Latin letters) or an underscore (_). Subsequent characters in an identifier or key word can be letters, underscores, digits (0-9), or dollar signs ($). Note that dollar signs are not allowed in identifiers according to the letter of the SQL standard, so their use might render applications less portable.

(Source: https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS)

Closes #7195

(cherry picked from commit d21b351)
  • Loading branch information
ph authored and ruflin committed Jun 15, 2018
1 parent 6456e7b commit ba10d41
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 11 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ https://github.com/elastic/beats/compare/v6.3.0...6.3[Check the HEAD diff]

*Filebeat*

- Comply with PostgreSQL database name format {pull}7198[7198]

*Heartbeat*

*Metricbeat*
Expand Down
11 changes: 6 additions & 5 deletions filebeat/module/postgresql/log/ingest/pipeline.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,17 @@
"field": "message",
"ignore_missing": true,
"patterns": [
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{HOSTNAME:postgresql.log.database} %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{HOSTNAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{HOSTNAME:postgresql.log.database} %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{HOSTNAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database} %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database} %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}",
"%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}"
],
"pattern_definitions": {
"LOCALDATETIME": "[-0-9]+ %{TIME}",
"GREEDYDATA": ".*",
"MULTILINEQUERY" : "(.|\n|\t)*?;$"
"MULTILINEQUERY" : "(.|\n|\t)*?;$",
"POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*"
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,6 @@
AND pg_catalog.pg_table_is_visible(c.oid)
ORDER BY 1,2;
2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);
2017-07-31 13:46:02.670 CEST [5502] postgres@clients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());
2017-07-31 13:46:23.016 CEST [5502] postgres@clients LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());
2017-07-31 13:46:55.637 CEST [5502] postgres@clients LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);
2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());
2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());
2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);
Original file line number Diff line number Diff line change
Expand Up @@ -732,7 +732,7 @@
"log": {
"duration": "10.540",
"thread_id": "5502",
"database": "clients",
"database": "c$lients",
"level": "LOG",
"timezone": "CEST",
"query": "insert into cats(name, toy, born) values('kate', 'ball', now());",
Expand Down Expand Up @@ -781,7 +781,7 @@
"log": {
"duration": "5.156",
"thread_id": "5502",
"database": "clients",
"database": "_clients$db",
"level": "LOG",
"timezone": "CEST",
"query": "insert into cats(name, toy, born) values('frida', 'horse', now());",
Expand Down Expand Up @@ -830,7 +830,7 @@
"log": {
"duration": "25.871",
"thread_id": "5502",
"database": "clients",
"database": "clients_db",
"level": "LOG",
"timezone": "CEST",
"query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);",
Expand Down

0 comments on commit ba10d41

Please sign in to comment.