Skip to content

Commit

Permalink
Add username/pass options for PostgreSQL (elastic#2890) (elastic#2906)
Browse files Browse the repository at this point in the history
Similar to elastic#2889 but for PostgreSQL. Also adds docs to the Postgres module,
which were missing, and adjusted the integration tests to use the username
option instead of the full URL.
(cherry picked from commit f0b52e1)
  • Loading branch information
tsg authored and monicasarbu committed Nov 1, 2016
1 parent 4ab0a6c commit d867eb1
Showing 18 changed files with 380 additions and 28 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
@@ -48,6 +48,8 @@ https://github.com/elastic/beats/compare/v5.0.0...5.0[Check the HEAD diff]

*Metricbeat*

- Add username and password config options to the PostgreSQL module. {pull}2889[2890]

*Packetbeat*

*Topbeat*
3 changes: 2 additions & 1 deletion metricbeat/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -21,9 +21,10 @@ beat:
- MYSQL_DSN=root:test@tcp(mysql:3306)/
- MYSQL_HOST=mysql
- MYSQL_PORT=3306
- POSTGRESQL_DSN=postgres://postgres@postgresql:5432?sslmode=disable
- POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable
- POSTGRESQL_HOST=postgresql
- POSTGRESQL_PORT=5432
- POSTGRESQL_USERNAME=postgres
- ZOOKEEPER_HOST=zookeeper
- ZOOKEEPER_PORT=2181
- HAPROXY_HOST=haproxy
64 changes: 60 additions & 4 deletions metricbeat/docs/modules/postgresql.asciidoc
Original file line number Diff line number Diff line change
@@ -3,10 +3,56 @@ This file is generated! See scripts/docs_collector.py
////

[[metricbeat-module-postgresql]]
== postgresql Module
== PostgreSQL Module

This is the postgresql Module.
This module periodically fetches metrics from
https://www.postgresql.org/[PostgreSQL] servers.

[float]
=== Module-Specific Configuration Notes

When configuring the `hosts` option, you must use Postgres URLs of the following
format:

-----------------------------------
[postgres://][user:pass@]host[:port][?options]
-----------------------------------

The URL can be as simple as:

[source,yaml]
----------------------------------------------------------------------
- module: postgresql
hosts: ["postgres://localhost"]
----------------------------------------------------------------------

Or more complex like:

[source,yaml]
----------------------------------------------------------------------
- module: postgresql
hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"]
----------------------------------------------------------------------

WARNING: In case you use username and password in the hosts array, this
information will be sent with each event as part of the `metricset.host` field.
To prevent sending username and password the config options `username` and
`password` can be used.

[source,yaml]
----
- module: postgresql
metricsets: ["status"]
hosts: ["postgres://localhost:5432"]
username: root
password: test
----

[float]
=== Compatibility

This module was tested with PostgreSQL 9.5.3 and is expected to work with all
versions >= 9.


[float]
@@ -33,10 +79,20 @@ metricbeat.modules:
#period: 10s
# The host must be passed as PostgreSQL DSN. Example:
# postgres://pqgotest:password@localhost:5432?sslmode=disable
# postgres://localhost:5432?sslmode=disable
# The available parameters are documented here:
# https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
#hosts: ["postgres://postgres@localhost:5432"]
#
# Warning: specifying the user/password in the hosts array is possible
# but it will result in the password being present in the output documents.
# We recommend using the username and password options instead.
#hosts: ["postgres://localhost:5432"]
# Username to use when connecting to PostgreSQL. Empty by default.
#username: user
# Password to use when connecting to PostgreSQL. Empty by default.
#password: pass
----

14 changes: 12 additions & 2 deletions metricbeat/etc/beat.full.yml
Original file line number Diff line number Diff line change
@@ -129,10 +129,20 @@ metricbeat.modules:
#period: 10s

# The host must be passed as PostgreSQL DSN. Example:
# postgres://pqgotest:password@localhost:5432?sslmode=disable
# postgres://localhost:5432?sslmode=disable
# The available parameters are documented here:
# https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
#hosts: ["postgres://postgres@localhost:5432"]
#
# Warning: specifying the user/password in the hosts array is possible
# but it will result in the password being present in the output documents.
# We recommend using the username and password options instead.
#hosts: ["postgres://localhost:5432"]

# Username to use when connecting to PostgreSQL. Empty by default.
#username: user

# Password to use when connecting to PostgreSQL. Empty by default.
#password: pass


#-------------------------------- Redis Module -------------------------------
14 changes: 12 additions & 2 deletions metricbeat/metricbeat.full.yml
Original file line number Diff line number Diff line change
@@ -129,10 +129,20 @@ metricbeat.modules:
#period: 10s

# The host must be passed as PostgreSQL DSN. Example:
# postgres://pqgotest:password@localhost:5432?sslmode=disable
# postgres://localhost:5432?sslmode=disable
# The available parameters are documented here:
# https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
#hosts: ["postgres://postgres@localhost:5432"]
#
# Warning: specifying the user/password in the hosts array is possible
# but it will result in the password being present in the output documents.
# We recommend using the username and password options instead.
#hosts: ["postgres://localhost:5432"]

# Username to use when connecting to PostgreSQL. Empty by default.
#username: user

# Password to use when connecting to PostgreSQL. Empty by default.
#password: pass


#-------------------------------- Redis Module -------------------------------
14 changes: 12 additions & 2 deletions metricbeat/module/postgresql/_meta/config.yml
Original file line number Diff line number Diff line change
@@ -13,8 +13,18 @@
#period: 10s

# The host must be passed as PostgreSQL DSN. Example:
# postgres://pqgotest:password@localhost:5432?sslmode=disable
# postgres://localhost:5432?sslmode=disable
# The available parameters are documented here:
# https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
#hosts: ["postgres://postgres@localhost:5432"]
#
# Warning: specifying the user/password in the hosts array is possible
# but it will result in the password being present in the output documents.
# We recommend using the username and password options instead.
#hosts: ["postgres://localhost:5432"]

# Username to use when connecting to PostgreSQL. Empty by default.
#username: user

# Password to use when connecting to PostgreSQL. Empty by default.
#password: pass

50 changes: 48 additions & 2 deletions metricbeat/module/postgresql/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
@@ -1,4 +1,50 @@
== postgresql Module
== PostgreSQL Module

This is the postgresql Module.
This module periodically fetches metrics from
https://www.postgresql.org/[PostgreSQL] servers.

[float]
=== Module-Specific Configuration Notes

When configuring the `hosts` option, you must use Postgres URLs of the following
format:

-----------------------------------
[postgres://][user:pass@]host[:port][?options]
-----------------------------------

The URL can be as simple as:

[source,yaml]
----------------------------------------------------------------------
- module: postgresql
hosts: ["postgres://localhost"]
----------------------------------------------------------------------

Or more complex like:

[source,yaml]
----------------------------------------------------------------------
- module: postgresql
hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"]
----------------------------------------------------------------------

WARNING: In case you use username and password in the hosts array, this
information will be sent with each event as part of the `metricset.host` field.
To prevent sending username and password the config options `username` and
`password` can be used.

[source,yaml]
----
- module: postgresql
metricsets: ["status"]
hosts: ["postgres://localhost:5432"]
username: root
password: test
----

[float]
=== Compatibility

This module was tested with PostgreSQL 9.5.3 and is expected to work with all
versions >= 9.
22 changes: 18 additions & 4 deletions metricbeat/module/postgresql/activity/activity.go
Original file line number Diff line number Diff line change
@@ -22,29 +22,43 @@ func init() {
// MetricSet type defines all fields of the Postgresql MetricSet
type MetricSet struct {
mb.BaseMetricSet
connectionString string
}

// New create a new instance of the MetricSet
// Part of new is also setting up the configuration by processing additional
// configuration entries if needed.
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {

config := struct{}{}
config := struct {
Hosts []string `config:"hosts" validate:"nonzero,required"`
Username string `config:"username"`
Password string `config:"password"`
}{
Username: "",
Password: "",
}

if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}

url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
base.Module().Config().Timeout)
if err != nil {
return nil, err
}

return &MetricSet{
BaseMetricSet: base,
BaseMetricSet: base,
connectionString: url,
}, nil
}

// Fetch implements the data gathering and data conversion to the right format.
func (m *MetricSet) Fetch() ([]common.MapStr, error) {

// TODO: Find a way to pass the timeout
db, err := sql.Open("postgres", m.Host())
db, err := sql.Open("postgres", m.connectionString)
if err != nil {
return nil, err
}
Original file line number Diff line number Diff line change
@@ -52,5 +52,7 @@ func getConfig() map[string]interface{} {
"module": "postgresql",
"metricsets": []string{"activity"},
"hosts": []string{postgresql.GetEnvDSN()},
"username": postgresql.GetEnvUsername(),
"password": postgresql.GetEnvPassword(),
}
}
22 changes: 18 additions & 4 deletions metricbeat/module/postgresql/bgwriter/bgwriter.go
Original file line number Diff line number Diff line change
@@ -23,26 +23,40 @@ func init() {
// MetricSet type defines all fields of the MetricSet
type MetricSet struct {
mb.BaseMetricSet
connectionString string
}

// New create a new instance of the MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {

config := struct{}{}
config := struct {
Hosts []string `config:"hosts" validate:"nonzero,required"`
Username string `config:"username"`
Password string `config:"password"`
}{
Username: "",
Password: "",
}

if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}

url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
base.Module().Config().Timeout)
if err != nil {
return nil, err
}

return &MetricSet{
BaseMetricSet: base,
BaseMetricSet: base,
connectionString: url,
}, nil
}

// Fetch methods implements the data gathering and data conversion to the right format
func (m *MetricSet) Fetch() (common.MapStr, error) {

db, err := sql.Open("postgres", m.Host())
db, err := sql.Open("postgres", m.connectionString)
if err != nil {
return nil, err
}
Original file line number Diff line number Diff line change
@@ -54,5 +54,7 @@ func getConfig() map[string]interface{} {
"module": "postgresql",
"metricsets": []string{"bgwriter"},
"hosts": []string{postgresql.GetEnvDSN()},
"username": postgresql.GetEnvUsername(),
"password": postgresql.GetEnvPassword(),
}
}
21 changes: 18 additions & 3 deletions metricbeat/module/postgresql/database/database.go
Original file line number Diff line number Diff line change
@@ -22,26 +22,41 @@ func init() {
// MetricSet type defines all fields of the MetricSet
type MetricSet struct {
mb.BaseMetricSet
connectionString string
}

// New create a new instance of the MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {

config := struct{}{}
config := struct {
Hosts []string `config:"hosts" validate:"nonzero,required"`
Username string `config:"username"`
Password string `config:"password"`
}{
Username: "",
Password: "",
}

if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}

url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
base.Module().Config().Timeout)
if err != nil {
return nil, err
}

return &MetricSet{
BaseMetricSet: base,
BaseMetricSet: base,
connectionString: url,
}, nil
}

// Fetch methods implements the data gathering and data conversion to the right format
func (m *MetricSet) Fetch() ([]common.MapStr, error) {

db, err := sql.Open("postgres", m.Host())
db, err := sql.Open("postgres", m.connectionString)
if err != nil {
return nil, err
}
Original file line number Diff line number Diff line change
@@ -54,5 +54,7 @@ func getConfig() map[string]interface{} {
"module": "postgresql",
"metricsets": []string{"database"},
"hosts": []string{postgresql.GetEnvDSN()},
"username": postgresql.GetEnvUsername(),
"password": postgresql.GetEnvPassword(),
}
}
Loading

0 comments on commit d867eb1

Please sign in to comment.