From fcaca9be0a070952c3d03df9e0031af8867b8b00 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Tue, 1 Nov 2016 13:06:24 +0100 Subject: [PATCH] Add username/pass options for PostgreSQL (#2890) Similar to #2889 but for PostgreSQL. Also adds docs to the Postgres module, which were missing, and adjusted the integration tests to use the username option instead of the full URL. (cherry picked from commit f0b52e1926ef88dff0fa17f5341c16144a831c56) --- CHANGELOG.asciidoc | 2 + metricbeat/docker-compose.yml | 3 +- metricbeat/docs/modules/postgresql.asciidoc | 64 +++++++++++++++- metricbeat/etc/beat.full.yml | 14 +++- metricbeat/metricbeat.full.yml | 14 +++- metricbeat/module/postgresql/_meta/config.yml | 14 +++- .../module/postgresql/_meta/docs.asciidoc | 50 ++++++++++++- .../module/postgresql/activity/activity.go | 22 +++++- .../activity/activity_integration_test.go | 2 + .../module/postgresql/bgwriter/bgwriter.go | 22 +++++- .../bgwriter/bgwriter_integration_test.go | 2 + .../module/postgresql/database/database.go | 21 +++++- .../database/database_integration_test.go | 2 + metricbeat/module/postgresql/postgresql.go | 68 +++++++++++++++++ .../module/postgresql/postgresql_test.go | 74 +++++++++++++++++++ metricbeat/module/postgresql/testing.go | 8 ++ .../tests/system/config/metricbeat.yml.j2 | 8 ++ metricbeat/tests/system/test_postgresql.py | 18 ++++- 18 files changed, 380 insertions(+), 28 deletions(-) create mode 100644 metricbeat/module/postgresql/postgresql_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f85074ff45c..6e9f13c2ae7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -48,6 +48,8 @@ https://github.com/elastic/beats/compare/v5.0.0...5.0[Check the HEAD diff] *Metricbeat* +- Add username and password config options to the PostgreSQL module. {pull}2889[2890] + *Packetbeat* *Topbeat* diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 359fe389903..3511d853eec 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -21,9 +21,10 @@ beat: - MYSQL_DSN=root:test@tcp(mysql:3306)/ - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - POSTGRESQL_DSN=postgres://postgres@postgresql:5432?sslmode=disable + - POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable - POSTGRESQL_HOST=postgresql - POSTGRESQL_PORT=5432 + - POSTGRESQL_USERNAME=postgres - ZOOKEEPER_HOST=zookeeper - ZOOKEEPER_PORT=2181 - HAPROXY_HOST=haproxy diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc index af10fbf3d86..f4f3c8634ea 100644 --- a/metricbeat/docs/modules/postgresql.asciidoc +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -3,10 +3,56 @@ This file is generated! See scripts/docs_collector.py //// [[metricbeat-module-postgresql]] -== postgresql Module +== PostgreSQL Module -This is the postgresql Module. +This module periodically fetches metrics from +https://www.postgresql.org/[PostgreSQL] servers. +[float] +=== Module-Specific Configuration Notes + +When configuring the `hosts` option, you must use Postgres URLs of the following +format: + +----------------------------------- +[postgres://][user:pass@]host[:port][?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"] +---------------------------------------------------------------------- + +WARNING: In case you use username and password in the hosts array, this +information will be sent with each event as part of the `metricset.host` field. +To prevent sending username and password the config options `username` and +`password` can be used. + +[source,yaml] +---- +- module: postgresql + metricsets: ["status"] + hosts: ["postgres://localhost:5432"] + username: root + password: test +---- + +[float] +=== Compatibility + +This module was tested with PostgreSQL 9.5.3 and is expected to work with all +versions >= 9. [float] @@ -33,10 +79,20 @@ metricbeat.modules: #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass ---- diff --git a/metricbeat/etc/beat.full.yml b/metricbeat/etc/beat.full.yml index 87d30646a98..adb58e0ebcd 100644 --- a/metricbeat/etc/beat.full.yml +++ b/metricbeat/etc/beat.full.yml @@ -129,10 +129,20 @@ metricbeat.modules: #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass #-------------------------------- Redis Module ------------------------------- diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 95f874d32fb..08d3e95a6c4 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -129,10 +129,20 @@ metricbeat.modules: #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass #-------------------------------- Redis Module ------------------------------- diff --git a/metricbeat/module/postgresql/_meta/config.yml b/metricbeat/module/postgresql/_meta/config.yml index d86fe1992db..2998207382b 100644 --- a/metricbeat/module/postgresql/_meta/config.yml +++ b/metricbeat/module/postgresql/_meta/config.yml @@ -13,8 +13,18 @@ #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass diff --git a/metricbeat/module/postgresql/_meta/docs.asciidoc b/metricbeat/module/postgresql/_meta/docs.asciidoc index accedf4b569..c9a83e2cad0 100644 --- a/metricbeat/module/postgresql/_meta/docs.asciidoc +++ b/metricbeat/module/postgresql/_meta/docs.asciidoc @@ -1,4 +1,50 @@ -== postgresql Module +== PostgreSQL Module -This is the postgresql Module. +This module periodically fetches metrics from +https://www.postgresql.org/[PostgreSQL] servers. +[float] +=== Module-Specific Configuration Notes + +When configuring the `hosts` option, you must use Postgres URLs of the following +format: + +----------------------------------- +[postgres://][user:pass@]host[:port][?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"] +---------------------------------------------------------------------- + +WARNING: In case you use username and password in the hosts array, this +information will be sent with each event as part of the `metricset.host` field. +To prevent sending username and password the config options `username` and +`password` can be used. + +[source,yaml] +---- +- module: postgresql + metricsets: ["status"] + hosts: ["postgres://localhost:5432"] + username: root + password: test +---- + +[float] +=== Compatibility + +This module was tested with PostgreSQL 9.5.3 and is expected to work with all +versions >= 9. diff --git a/metricbeat/module/postgresql/activity/activity.go b/metricbeat/module/postgresql/activity/activity.go index fc296e6a9df..5ec7c63e359 100644 --- a/metricbeat/module/postgresql/activity/activity.go +++ b/metricbeat/module/postgresql/activity/activity.go @@ -22,6 +22,7 @@ func init() { // MetricSet type defines all fields of the Postgresql MetricSet type MetricSet struct { mb.BaseMetricSet + connectionString string } // New create a new instance of the MetricSet @@ -29,22 +30,35 @@ type MetricSet struct { // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + Username string `config:"username"` + Password string `config:"password"` + }{ + Username: "", + Password: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password, + base.Module().Config().Timeout) + if err != nil { + return nil, err + } + return &MetricSet{ - BaseMetricSet: base, + BaseMetricSet: base, + connectionString: url, }, nil } // Fetch implements the data gathering and data conversion to the right format. func (m *MetricSet) Fetch() ([]common.MapStr, error) { - // TODO: Find a way to pass the timeout - db, err := sql.Open("postgres", m.Host()) + db, err := sql.Open("postgres", m.connectionString) if err != nil { return nil, err } diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go index bd0d8fef565..d8faabc200e 100644 --- a/metricbeat/module/postgresql/activity/activity_integration_test.go +++ b/metricbeat/module/postgresql/activity/activity_integration_test.go @@ -52,5 +52,7 @@ func getConfig() map[string]interface{} { "module": "postgresql", "metricsets": []string{"activity"}, "hosts": []string{postgresql.GetEnvDSN()}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), } } diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter.go b/metricbeat/module/postgresql/bgwriter/bgwriter.go index 1af04bcb4a4..49672683de8 100644 --- a/metricbeat/module/postgresql/bgwriter/bgwriter.go +++ b/metricbeat/module/postgresql/bgwriter/bgwriter.go @@ -23,26 +23,40 @@ func init() { // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet + connectionString string } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - - config := struct{}{} + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + Username string `config:"username"` + Password string `config:"password"` + }{ + Username: "", + Password: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password, + base.Module().Config().Timeout) + if err != nil { + return nil, err + } + return &MetricSet{ - BaseMetricSet: base, + BaseMetricSet: base, + connectionString: url, }, nil } // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch() (common.MapStr, error) { - db, err := sql.Open("postgres", m.Host()) + db, err := sql.Open("postgres", m.connectionString) if err != nil { return nil, err } diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go index 20cd6a14943..d72a826af2f 100644 --- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go +++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go @@ -54,5 +54,7 @@ func getConfig() map[string]interface{} { "module": "postgresql", "metricsets": []string{"bgwriter"}, "hosts": []string{postgresql.GetEnvDSN()}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), } } diff --git a/metricbeat/module/postgresql/database/database.go b/metricbeat/module/postgresql/database/database.go index 7afdbb0e515..f72fa1e09c1 100644 --- a/metricbeat/module/postgresql/database/database.go +++ b/metricbeat/module/postgresql/database/database.go @@ -22,26 +22,41 @@ func init() { // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet + connectionString string } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + Username string `config:"username"` + Password string `config:"password"` + }{ + Username: "", + Password: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password, + base.Module().Config().Timeout) + if err != nil { + return nil, err + } + return &MetricSet{ - BaseMetricSet: base, + BaseMetricSet: base, + connectionString: url, }, nil } // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch() ([]common.MapStr, error) { - db, err := sql.Open("postgres", m.Host()) + db, err := sql.Open("postgres", m.connectionString) if err != nil { return nil, err } diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go index 44a5f58c5be..97ec4481b30 100644 --- a/metricbeat/module/postgresql/database/database_integration_test.go +++ b/metricbeat/module/postgresql/database/database_integration_test.go @@ -54,5 +54,7 @@ func getConfig() map[string]interface{} { "module": "postgresql", "metricsets": []string{"database"}, "hosts": []string{postgresql.GetEnvDSN()}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), } } diff --git a/metricbeat/module/postgresql/postgresql.go b/metricbeat/module/postgresql/postgresql.go index 6ebcfcd3595..e300ea22e7e 100644 --- a/metricbeat/module/postgresql/postgresql.go +++ b/metricbeat/module/postgresql/postgresql.go @@ -5,6 +5,13 @@ package postgresql import ( "database/sql" + "fmt" + "net" + nurl "net/url" + "sort" + "strconv" + "strings" + "time" "github.com/elastic/beats/libbeat/logp" "github.com/pkg/errors" @@ -45,3 +52,64 @@ func QueryStats(db *sql.DB, query string) ([]map[string]interface{}, error) { } return results, nil } + +// ParseURL parses the given URL and overrides the values of username, password and timeout +// if given. Returns a connection string in the form of `user=pass` ready to be passed to the +// sql.Open call. +// Code adapted from the pg driver: https://github.com/lib/pq/blob/master/url.go#L32 +func ParseURL(url, username, password string, timeout time.Duration) (string, error) { + u, err := nurl.Parse(url) + if err != nil { + return "", err + } + + if u.Scheme != "postgres" && u.Scheme != "postgresql" { + return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme) + } + + var kvs []string + escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`) + accrue := func(k, v string) { + if v != "" { + kvs = append(kvs, k+"="+escaper.Replace(v)) + } + } + + if len(username) > 0 { + accrue("user", username) + accrue("password", password) + } else { + if u.User != nil { + v := u.User.Username() + accrue("user", v) + + v, _ = u.User.Password() + accrue("password", v) + } + } + + if host, port, err := net.SplitHostPort(u.Host); err != nil { + accrue("host", u.Host) + } else { + accrue("host", host) + accrue("port", port) + } + + if u.Path != "" { + accrue("dbname", u.Path[1:]) + } + + q := u.Query() + for k := range q { + if k == "connect_timeout" && timeout != 0 { + continue + } + accrue(k, q.Get(k)) + } + if timeout != 0 { + accrue("connect_timeout", strconv.Itoa(int(timeout.Seconds()))) + } + + sort.Strings(kvs) // Makes testing easier (not a performance concern) + return strings.Join(kvs, " "), nil +} diff --git a/metricbeat/module/postgresql/postgresql_test.go b/metricbeat/module/postgresql/postgresql_test.go new file mode 100644 index 00000000000..c6a01c6eb8c --- /dev/null +++ b/metricbeat/module/postgresql/postgresql_test.go @@ -0,0 +1,74 @@ +package postgresql + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseUrl(t *testing.T) { + tests := []struct { + Name string + URL string + Username string + Password string + Timeout time.Duration + Expected string + }{ + { + Name: "simple test", + URL: "postgres://host1:5432", + Expected: "host=host1 port=5432", + }, + { + Name: "no port", + URL: "postgres://host1", + Expected: "host=host1", + }, + { + Name: "user/pass in URL", + URL: "postgres://user:pass@host1:5432", + Expected: "host=host1 password=pass port=5432 user=user", + }, + { + Name: "user/pass in params", + URL: "postgres://host1:5432", + Username: "user", + Password: "secret", + Expected: "host=host1 password=secret port=5432 user=user", + }, + { + Name: "user/pass override", + URL: "postgres://user1:pass@host1:5432", + Username: "user", + Password: "secret", + Expected: "host=host1 password=secret port=5432 user=user", + }, + { + Name: "timeout no override", + URL: "postgres://host1:5432?connect_timeout=2", + Expected: "connect_timeout=2 host=host1 port=5432", + }, + { + Name: "timeout from param", + URL: "postgres://host1:5432", + Timeout: 3 * time.Second, + Expected: "connect_timeout=3 host=host1 port=5432", + }, + { + Name: "user/pass override, and timeout override", + URL: "postgres://user1:pass@host1:5432?connect_timeout=2", + Username: "user", + Password: "secret", + Timeout: 3 * time.Second, + Expected: "connect_timeout=3 host=host1 password=secret port=5432 user=user", + }, + } + + for _, test := range tests { + url, err := ParseURL(test.URL, test.Username, test.Password, test.Timeout) + assert.NoError(t, err, test.Name) + assert.Equal(t, test.Expected, url, test.Name) + } +} diff --git a/metricbeat/module/postgresql/testing.go b/metricbeat/module/postgresql/testing.go index eabc35b580e..9afe8798358 100644 --- a/metricbeat/module/postgresql/testing.go +++ b/metricbeat/module/postgresql/testing.go @@ -5,3 +5,11 @@ import "os" func GetEnvDSN() string { return os.Getenv("POSTGRESQL_DSN") } + +func GetEnvUsername() string { + return os.Getenv("POSTGRESQL_USERNAME") +} + +func GetEnvPassword() string { + return os.Getenv("POSTGRESQL_PASSWORD") +} diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index bad37431ef7..c7a0e8f47f6 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -14,6 +14,14 @@ metricbeat.modules: {% endfor %} {% endif -%} + {% if m.username -%} + username: {{ m.username }} + {% endif -%} + + {% if m.password -%} + password: {{ m.password }} + {% endif -%} + {% if m.metricsets -%} metricsets: {% for ms in m.metricsets -%} diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py index bc5e9376be5..ca21bb786e4 100644 --- a/metricbeat/tests/system/test_postgresql.py +++ b/metricbeat/tests/system/test_postgresql.py @@ -18,7 +18,8 @@ def common_checks(self, output): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv("POSTGRESQL_DSN")] + return [os.getenv("POSTGRESQL_DSN")], os.getenv("POSTGRESQL_USERNAME"), \ + os.getenv("POSTGRESQL_PASSWORD") @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @attr('integration') @@ -26,10 +27,13 @@ def test_activity(self): """ PostgreSQL module outputs an event. """ + hosts, username, password = self.get_hosts() self.render_config_template(modules=[{ "name": "postgresql", "metricsets": ["activity"], - "hosts": self.get_hosts(), + "hosts": hosts, + "username": username, + "password": password, "period": "5s" }]) proc = self.start_beat() @@ -50,10 +54,13 @@ def test_database(self): """ PostgreSQL module outputs an event. """ + hosts, username, password = self.get_hosts() self.render_config_template(modules=[{ "name": "postgresql", "metricsets": ["database"], - "hosts": self.get_hosts(), + "hosts": hosts, + "username": username, + "password": password, "period": "5s" }]) proc = self.start_beat() @@ -77,10 +84,13 @@ def test_bgwriter(self): """ PostgreSQL module outputs an event. """ + hosts, username, password = self.get_hosts() self.render_config_template(modules=[{ "name": "postgresql", "metricsets": ["bgwriter"], - "hosts": self.get_hosts(), + "hosts": hosts, + "username": username, + "password": password, "period": "5s" }]) proc = self.start_beat()