From 41941aaaef0a5b4d7e3ee3e69697cb9a116d4db1 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 7 Jul 2020 16:11:30 +0200 Subject: [PATCH 01/21] Atomic commit --- metricbeat/docs/fields.asciidoc | 510 ++++++++++++++++++ metricbeat/docs/modules/mysql.asciidoc | 4 + .../docs/modules/mysql/performance.asciidoc | 22 + metricbeat/docs/modules/sql.asciidoc | 354 +++++++++++- metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/helper/sql/sql.go | 205 +++++++ metricbeat/helper/sql/sql_test.go | 188 +++++++ metricbeat/include/list_common.go | 1 + metricbeat/module/mysql/fields.go | 2 +- metricbeat/module/mysql/module.yml | 3 + .../module/mysql/performance/_meta/data.json | 62 +++ .../mysql/performance/_meta/docs.asciidoc | 0 .../module/mysql/performance/_meta/fields.yml | 6 + .../module/mysql/performance/manifest.yml | 19 + .../module/mysql/query/_meta/fields.yml | 274 ++++++++++ metricbeat/module/mysql/query/query.go | 132 +++++ .../metricbeat/module/sql/_meta/docs.asciidoc | 354 +++++++++++- x-pack/metricbeat/module/sql/_meta/fields.yml | 5 + x-pack/metricbeat/module/sql/fields.go | 2 +- x-pack/metricbeat/module/sql/query/query.go | 197 ++----- 20 files changed, 2191 insertions(+), 152 deletions(-) create mode 100644 metricbeat/docs/modules/mysql/performance.asciidoc create mode 100644 metricbeat/helper/sql/sql.go create mode 100644 metricbeat/helper/sql/sql_test.go create mode 100644 metricbeat/module/mysql/performance/_meta/data.json create mode 100644 metricbeat/module/mysql/performance/_meta/docs.asciidoc create mode 100644 metricbeat/module/mysql/performance/_meta/fields.yml create mode 100644 metricbeat/module/mysql/performance/manifest.yml create mode 100644 metricbeat/module/mysql/query/_meta/fields.yml create mode 100644 metricbeat/module/mysql/query/query.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2e6f35dfece9..655a64e9c075 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -31956,6 +31956,506 @@ Replication status fields. +*`mysql.galera_status.repl.data_bytes`*:: ++ +-- +Total size of data replicated. + + +type: long + +-- + +*`mysql.galera_status.repl.keys`*:: ++ +-- +Total number of keys replicated. + + +type: long + +-- + +*`mysql.galera_status.repl.keys_bytes`*:: ++ +-- +Total size of keys replicated. + + +type: long + +-- + +*`mysql.galera_status.repl.other_bytes`*:: ++ +-- +Total size of other bits replicated. + + +type: long + +-- + +*`mysql.galera_status.repl.count`*:: ++ +-- +Total number of write-sets replicated (sent to other nodes). + + +type: long + +-- + +*`mysql.galera_status.repl.bytes`*:: ++ +-- +Total size of write-sets replicated. + + +type: long + +-- + +[float] +=== overview + +`performance` contains the metrics that were obtained by the status SQL query. + + +[float] +=== galera_status + +`galera_status` contains the metrics that were obtained by the status SQL query on Galera. + + + +[float] +=== apply + +Apply status fields. + + + +*`mysql.galera_status.apply.oooe`*:: ++ +-- +How often applier started write-set applying out-of-order (parallelization efficiency). + + +type: double + +-- + +*`mysql.galera_status.apply.oool`*:: ++ +-- +How often write-set was so slow to apply that write-set with higher seqno's were applied earlier. Values closer to 0 refer to a greater gap between slow and fast write-sets. + + +type: double + +-- + +*`mysql.galera_status.apply.window`*:: ++ +-- +Average distance between highest and lowest concurrently applied seqno. + + +type: double + +-- + +[float] +=== cert + +Certification status fields. + + + +*`mysql.galera_status.cert.deps_distance`*:: ++ +-- +Average distance between highest and lowest seqno value that can be possibly applied in parallel (potential degree of parallelization). + + +type: double + +-- + +*`mysql.galera_status.cert.index_size`*:: ++ +-- +The number of entries in the certification index. + + +type: long + +-- + +*`mysql.galera_status.cert.interval`*:: ++ +-- +Average number of transactions received while a transaction replicates. + + +type: double + +-- + +[float] +=== cluster + +Cluster status fields. + + + +*`mysql.galera_status.cluster.conf_id`*:: ++ +-- +Total number of cluster membership changes happened. + + +type: long + +-- + +*`mysql.galera_status.cluster.size`*:: ++ +-- +Current number of members in the cluster. + + +type: long + +-- + +*`mysql.galera_status.cluster.status`*:: ++ +-- +Status of this cluster component. That is, whether the node is part of a PRIMARY or NON_PRIMARY component. + + +type: keyword + +-- + +[float] +=== commit + +Commit status fields. + + + +*`mysql.galera_status.commit.oooe`*:: ++ +-- +How often a transaction was committed out of order. + + +type: double + +-- + +*`mysql.galera_status.commit.window`*:: ++ +-- +Average distance between highest and lowest concurrently committed seqno. + + +type: long + +-- + +*`mysql.galera_status.connected`*:: ++ +-- +If the value is OFF, the node has not yet connected to any of the cluster components. This may be due to misconfiguration. Check the error log for proper diagnostics. + + +type: keyword + +-- + +[float] +=== evs + +Evs Fields. + + + +*`mysql.galera_status.evs.evict`*:: ++ +-- +Lists the UUID's of all nodes evicted from the cluster. Evicted nodes cannot rejoin the cluster until you restart their mysqld processes. + + +type: keyword + +-- + +*`mysql.galera_status.evs.state`*:: ++ +-- +Shows the internal state of the EVS Protocol. + + +type: keyword + +-- + +[float] +=== flow_ctl + +Flow Control fields. + + + +*`mysql.galera_status.flow_ctl.paused`*:: ++ +-- +The fraction of time since the last FLUSH STATUS command that replication was paused due to flow control. In other words, how much the slave lag is slowing down the cluster. + + +type: double + +-- + +*`mysql.galera_status.flow_ctl.paused_ns`*:: ++ +-- +The total time spent in a paused state measured in nanoseconds. + + +type: long + +-- + +*`mysql.galera_status.flow_ctl.recv`*:: ++ +-- +Returns the number of FC_PAUSE events the node has received, including those the node has sent. Unlike most status variables, the counter for this one does not reset every time you run the query. + + +type: long + +-- + +*`mysql.galera_status.flow_ctl.sent`*:: ++ +-- +Returns the number of FC_PAUSE events the node has sent. Unlike most status variables, the counter for this one does not reset every time you run the query. + + +type: long + +-- + +*`mysql.galera_status.last_committed`*:: ++ +-- +The sequence number, or seqno, of the last committed transaction. + + +type: long + +-- + +[float] +=== local + +Node specific Cluster status fields. + + + +*`mysql.galera_status.local.bf_aborts`*:: ++ +-- +Total number of local transactions that were aborted by slave transactions while in execution. + + +type: long + +-- + +*`mysql.galera_status.local.cert_failures`*:: ++ +-- +Total number of local transactions that failed certification test. + + +type: long + +-- + +*`mysql.galera_status.local.commits`*:: ++ +-- +Total number of local transactions committed. + + +type: long + +-- + +[float] +=== recv + +Node specific recv fields. + + + +*`mysql.galera_status.local.recv.queue`*:: ++ +-- +Current (instantaneous) length of the recv queue. + + +type: long + +-- + +*`mysql.galera_status.local.recv.queue_avg`*:: ++ +-- +Recv queue length averaged over interval since the last FLUSH STATUS command. Values considerably larger than 0.0 mean that the node cannot apply write-sets as fast as they are received and will generate a lot of replication throttling. + + +type: double + +-- + +*`mysql.galera_status.local.recv.queue_max`*:: ++ +-- +The maximum length of the recv queue since the last FLUSH STATUS command. + + +type: long + +-- + +*`mysql.galera_status.local.recv.queue_min`*:: ++ +-- +The minimum length of the recv queue since the last FLUSH STATUS command. + + +type: long + +-- + +*`mysql.galera_status.local.replays`*:: ++ +-- +Total number of transaction replays due to asymmetric lock granularity. + + +type: long + +-- + +[float] +=== send + +Node specific sent fields. + + + +*`mysql.galera_status.local.send.queue`*:: ++ +-- +Current (instantaneous) length of the send queue. + + +type: long + +-- + +*`mysql.galera_status.local.send.queue_avg`*:: ++ +-- +Send queue length averaged over time since the last FLUSH STATUS command. Values considerably larger than 0.0 indicate replication throttling or network throughput issue. + + +type: double + +-- + +*`mysql.galera_status.local.send.queue_max`*:: ++ +-- +The maximum length of the send queue since the last FLUSH STATUS command. + + +type: long + +-- + +*`mysql.galera_status.local.send.queue_min`*:: ++ +-- +The minimum length of the send queue since the last FLUSH STATUS command. + + +type: long + +-- + +*`mysql.galera_status.local.state`*:: ++ +-- +Internal Galera Cluster FSM state number. + + +type: keyword + +-- + +*`mysql.galera_status.ready`*:: ++ +-- +Whether the server is ready to accept queries. + + +type: keyword + +-- + +[float] +=== received + +Write-Set receive status fields. + + + +*`mysql.galera_status.received.count`*:: ++ +-- +Total number of write-sets received from other nodes. + + +type: long + +-- + +*`mysql.galera_status.received.bytes`*:: ++ +-- +Total size of write-sets received from other nodes. + + +type: long + +-- + +[float] +=== repl + +Replication status fields. + + + *`mysql.galera_status.repl.data_bytes`*:: + -- @@ -37123,6 +37623,16 @@ type: object Non-numeric values collected. +type: object + +-- + +*`sql.metrics.boolean.*`*:: ++ +-- +Boolean values collected. + + type: object -- diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index 549e8e1dafcc..2febcb85a4cf 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -86,9 +86,13 @@ The following metricsets are available: * <> +* <> + * <> include::mysql/galera_status.asciidoc[] +include::mysql/performance.asciidoc[] + include::mysql/status.asciidoc[] diff --git a/metricbeat/docs/modules/mysql/performance.asciidoc b/metricbeat/docs/modules/mysql/performance.asciidoc new file mode 100644 index 000000000000..8e45e7ce6cfe --- /dev/null +++ b/metricbeat/docs/modules/mysql/performance.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mysql-performance]] +=== MySQL performance metricset + +include::../../../module/mysql/performance/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mysql/performance/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index 61daf5b8a18b..7b5730a3b8b9 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -8,9 +8,361 @@ This file is generated! See scripts/mage/docs_collector.go beta[] -This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. +The SQL module allows to execute custom queries against an SQL database and store the results to Elasticsearch. +The currently supported databases are the ones already included in Metricbeat, which are: +- PostgreSQL +- MySQL +- Oracle +- Microsoft SQL +- CockroachDB +== Quickstart + +You can setup the module by activating it first running + + metricbeat module enable sql + +Once it is activated, open `modules.d/sql.yml` and fill the required fields. This is an example that captures Innodb related metrics from the result of the query `SHOW GLOBAL STATUS LIKE 'Innodb_system%'` in a MySQL database: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["root:root@tcp(localhost:3306)/ps"] + + driver: "mysql" + sql_query: "SHOW GLOBAL STATUS LIKE 'Innodb_system%'" + sql_response_format: variables +---- + +.SHOW GLOBAL STATUS LIKE 'Innodb_system%' +|==== +|Variable_name|Value + +|Innodb_system_rows_deleted|0 +|Innodb_system_rows_inserted|0 +|Innodb_system_rows_read|5062 +|Innodb_system_rows_updated|315 +|==== + + +Keys in the YAML are defined as follow: + +- `driver`: The drivers currently supported are those which already have a Metricbeat module like `mssql` or `postgres`. +- `sql_query`: Is the single query you want to run +- `sql_response_format`: You have 2 options here: + - `variables`: Expects a table which looks like a key/value result. With 2 columns, left column will be considered a key and the right column the value. This mode generates a single event on each fetch operation. + - `table`: Table mode can contain any number of columns and a single event will be generated for each row. + +Results will be grouped by type in the result event for convenient mapping in Elasticsearch. So `strings` values will be grouped into `sql.strings`, `numeric` into `sql.numeric` and so on and so forth. + +The event generated with the example above looks like this: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:09:14.407Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "172.18.0.2:3306", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 1272810 + }, + "sql": { + "driver": "mysql", + "query": "SHOW GLOBAL STATUS LIKE 'Innodb_system%'", + "metrics": { + "numeric": { + "innodb_system_rows_updated": 315, + "innodb_system_rows_deleted": 0, + "innodb_system_rows_inserted": 0, + "innodb_system_rows_read": 5062 + } + } + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "488431bd-bd3c-4442-ad51-0c50eb555787", + "id": "670ef211-87f0-4f38-8beb-655c377f1629" + } +} +---- + +In this example, we are querying PostgreSQL and generate a "table" result, hence a single event for each row returned + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table +---- + +.SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database +|==== +|datid|datname|blks_read|blks_hit|tup_returned|tup_fetched|stats_reset + +|69448|stuff|8652|205976|1484625|53218|2020-06-07 22:50:12 +|13408|postgres|0|0|0|0| +|13407|template0|0|0|0|0| +|==== + +With 3 rows on the table, three events will be generated with the contents of each row. As an example, below you can see the event created for the first row: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T14:47:35.481Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "localhost:5432", + "type": "sql" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "1bffe66d-a1ae-4ed6-985a-fd48548a1971", + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic" + }, + "sql": { + "metrics": { + "numeric": { + "tup_fetched": 53350, + "datid": 69448, + "blks_read": 8652, + "blks_hit": 206501, + "tup_returned": 1.491873e+06 + }, + "string": { + "stats_reset": "2020-06-07T20:50:12.632975Z", + "datname": "stuff" + } + }, + "driver": "postgres", + "query": "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 14076705 + }, + "metricset": { + "name": "query", + "period": 10000 + } +} +---- + + +== More examples + +=== Oracle: + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] + + driver: "oracle" + sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS' + sql_response_format: table +---- + + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:41:02.200Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "metrics": { + "numeric": { + "hit ratio": 0.9742963357937117, + "physical_reads": 17161, + "db_block_gets": 122221, + "consistent_gets": 545427 + }, + "string": { + "name": "DEFAULT" + } + }, + "driver": "oracle", + "query": "SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) \"Hit Ratio\" FROM V$BUFFER_POOL_STATISTICS" + }, + "metricset": { + "period": 10000, + "name": "query" + }, + "service": { + "address": "172.17.0.3:1521", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 39233704 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788" + } +} +---- + +=== MSSQL + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["sqlserver://SA:password@localhost"] + + driver: "mssql" + sql_query: 'SELECT * FROM sys.dm_db_log_space_usage' + sql_response_format: table +---- + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:39:14.421Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "driver": "mssql", + "query": "SELECT * FROM sys.dm_db_log_space_usage", + "metrics": { + "numeric": { + "log_space_in_bytes_since_last_backup": 524288, + "database_id": 1, + "total_log_size_in_bytes": 2.08896e+06, + "used_log_space_in_bytes": 954368, + "used_log_space_in_percent": 45.686275482177734 + } + } + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 40750570 + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.17.0.2", + "type": "sql" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "3da88889-036e-47cb-a88b-275037fa2bc9" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + } +} +---- + +=== Two or more queries + +If you want to launch two or more queries, you need to specify them with their full configuration for each query. For example: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table + +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT * FROM pg_catalog.pg_tables pt WHERE schemaname ='pg_catalog'" + sql_response_format: table +--- [float] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 917f9818abdf..067c794e6e6a 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -191,7 +191,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> beta[] +.3+| .3+| |<> beta[] +|<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> diff --git a/metricbeat/helper/sql/sql.go b/metricbeat/helper/sql/sql.go new file mode 100644 index 000000000000..a5887a7b02a4 --- /dev/null +++ b/metricbeat/helper/sql/sql.go @@ -0,0 +1,205 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sql + +import ( + "context" + "database/sql" + "fmt" + + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type DbClient struct { + *sql.DB + logger *logp.Logger +} + +type sqlRow interface { + Scan(dest ...interface{}) error + Next() bool + Columns() ([]string, error) + Err() error +} + +// DB gets a client ready to query the database +func DB(driver, uri string, l *logp.Logger) (*DbClient, error) { + dbx, err := sql.Open(switchDriverName(driver), uri) + if err != nil { + return nil, errors.Wrap(err, "opening connection") + } + err = dbx.Ping() + if err != nil { + return nil, errors.Wrap(err, "testing connection") + } + + return &DbClient{DB: dbx, logger: l}, nil +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (d *DbClient) FetchTableMode(ctx context.Context, q string) ([]common.MapStr, error) { + rows, err := d.QueryContext(ctx, q) + if err != nil { + return nil, err + } + return d.fetchTableMode(rows) +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (d *DbClient) fetchTableMode(rows sqlRow) ([]common.MapStr, error) { + // Extracted from + // https://stackoverflow.com/questions/23507531/is-golangs-sql-package-incapable-of-ad-hoc-exploratory-queries/23507765#23507765 + cols, err := rows.Columns() + if err != nil { + return nil, errors.Wrap(err, "error getting columns") + } + + for k, v := range cols { + cols[k] = strings.ToLower(v) + } + + vals := make([]interface{}, len(cols)) + for i := 0; i < len(cols); i++ { + vals[i] = new(interface{}) + } + + rr := make([]common.MapStr, 0) + for rows.Next() { + err = rows.Scan(vals...) + if err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to scan rows")) + continue + } + + r := common.MapStr{} + + for i, c := range cols { + value := getValue(vals[i].(*interface{})) + r.Put(c, value) + } + + rr = append(rr, r) + } + + if err = rows.Err(); err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to read rows")) + } + + return rr, nil +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (d *DbClient) FetchVariableMode(ctx context.Context, q string) (common.MapStr, error) { + rows, err := d.QueryContext(ctx, q) + if err != nil { + return nil, err + } + return d.fetchVariableMode(rows) +} + +// fetchVariableMode scan the rows and publishes the event for querys that return the response in a key/value format. +func (d *DbClient) fetchVariableMode(rows sqlRow) (common.MapStr, error) { + data := common.MapStr{} + + for rows.Next() { + var key string + var val interface{} + err := rows.Scan(&key, &val) + if err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to scan rows")) + continue + } + + key = strings.ToLower(key) + data[key] = val + } + + if err := rows.Err(); err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to read rows")) + } + + r := common.MapStr{} + + for key, value := range data { + value := getValue(&value) + r.Put(key, value) + } + + return r, nil +} + +func ToDotKeys(ms common.MapStr) common.MapStr { + dotMap := common.MapStr{} + for k, v := range ms { + dotMap.Put(strings.Replace(k, "_", ".", -1), v) + } + + return dotMap +} + +func getValue(pval *interface{}) interface{} { + switch v := (*pval).(type) { + case nil, bool: + return v + case []byte: + s := string(v) + num, err := strconv.ParseFloat(s, 64) + if err == nil { + return num + } + return s + case time.Time: + return v.Format(time.RFC3339Nano) + case []interface{}: + return v + default: + s := fmt.Sprint(v) + num, err := strconv.ParseFloat(s, 64) + if err == nil { + return num + } + return s + } +} + +// switchDriverName switches between driver name and a pretty name for a driver. For example, 'oracle' driver is called +// 'godror' so this detail implementation must be hidden to the user, that should only choose and see 'oracle' as driver +func switchDriverName(d string) string { + switch d { + case "oracle": + return "godror" + case "cockroachdb": + return "postgres" + case "cockroach": + return "postgres" + case "postgresql": + return "postgres" + } + + return d +} diff --git a/metricbeat/helper/sql/sql_test.go b/metricbeat/helper/sql/sql_test.go new file mode 100644 index 000000000000..31a496454f63 --- /dev/null +++ b/metricbeat/helper/sql/sql_test.go @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sql + +import ( + "math" + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/common" +) + +type kv struct { + k string + v interface{} +} + +type mockVariableMode struct { + index int + results []kv +} + +func (m *mockVariableMode) Scan(dest ...interface{}) error { + d1 := dest[0].(*interface{}) + *d1 = m.results[m.index].k + + d2 := dest[1].(*interface{}) + *d2 = m.results[m.index].v + + m.index++ + + return nil +} + +func (m *mockVariableMode) Next() bool { + return m.index < len(m.results) +} + +func (m mockVariableMode) Columns() ([]string, error) { + return []string{"key", "value"}, nil +} + +func (m mockVariableMode) Err() error { + return nil +} + +type mockTableMode struct { + results []kv + totalResults int +} + +func (m *mockTableMode) Scan(dest ...interface{}) error { + for i, d := range dest { + d1 := d.(*interface{}) + *d1 = m.results[i].v + } + + m.totalResults++ + + return nil +} + +func (m *mockTableMode) Next() bool { + return m.totalResults < len(m.results) +} + +func (m *mockTableMode) Columns() ([]string, error) { + return []string{"hello", "integer", "signed_integer", "unsigned_integer", "float64", "float32", "null", "boolean", "array", "byte_array", "time"}, nil +} + +func (m mockTableMode) Err() error { + return nil +} + +var results = []kv{ + {k: "hello", v: "world"}, + {k: "integer", v: int(10)}, + {k: "signed_integer", v: int(-10)}, + {k: "unsigned_integer", v: uint(100)}, + {k: "float64", v: float64(-13.2)}, + {k: "float32", v: float32(13.2)}, + {k: "null", v: nil}, + {k: "boolean", v: true}, + {k: "array", v: []interface{}{0, 1, 2}}, + {k: "byte_array", v: []byte("byte_array")}, + {k: "time", v: time.Now()}, +} + +func TestFetchVariableMode(t *testing.T) { + db := DbClient{} + + ms, err := db.fetchVariableMode(&mockVariableMode{results: results}) + if err != nil { + t.Fatal(err) + } + + for _, res := range results { + checkValue(t, res, ms) + } +} + +func TestFetchTableMode(t *testing.T) { + db := DbClient{} + + mss, err := db.fetchTableMode(&mockTableMode{results: results}) + if err != nil { + t.Fatal(err) + } + + for _, ms := range mss { + for _, res := range results { + checkValue(t, res, ms) + } + } +} + +func checkValue(t *testing.T, res kv, ms common.MapStr) { + switch v := res.v.(type) { + case string, bool: + if ms[res.k] != v { + t.Fail() + } + case nil: + if ms[res.k] != nil { + t.Fail() + } + case int: + if ms[res.k] != float64(v) { + t.Fail() + } + case uint: + if ms[res.k] != float64(v) { + t.Fail() + } + case float32: + if math.Abs(float64(ms[res.k].(float64)-float64(v))) > 1 { + t.Fail() + } + case float64: + if ms[res.k] != v { + t.Fail() + } + case []interface{}: + for i, val := range v { + if ms[res.k].([]interface{})[i] != val { + t.Fail() + } + } + case []byte: + ar := ms[res.k].(string) + if ar != string(v) { + t.Fail() + } + case time.Time: + ar := ms[res.k].(string) + if v.Format(time.RFC3339Nano) != ar { + t.Fail() + } + default: + if ms[res.k] != res.v { + t.Fail() + } + } +} + +func TestToDotKeys(t *testing.T) { + ms := common.MapStr{"key_value": "value"} + ms = ToDotKeys(ms) + + if ms["key"].(common.MapStr)["value"] != "value" { + t.Fail() + } +} diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index f5935a8141c9..39fadeea0e02 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -112,6 +112,7 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/munin/node" _ "github.com/elastic/beats/v7/metricbeat/module/mysql" _ "github.com/elastic/beats/v7/metricbeat/module/mysql/galera_status" + _ "github.com/elastic/beats/v7/metricbeat/module/mysql/query" _ "github.com/elastic/beats/v7/metricbeat/module/mysql/status" _ "github.com/elastic/beats/v7/metricbeat/module/nats" _ "github.com/elastic/beats/v7/metricbeat/module/nats/connections" diff --git a/metricbeat/module/mysql/fields.go b/metricbeat/module/mysql/fields.go index 283f3e4df4bd..100f37839ebf 100644 --- a/metricbeat/module/mysql/fields.go +++ b/metricbeat/module/mysql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetMysql returns asset data. // This is the base64 encoded gzipped contents of module/mysql. func AssetMysql() string { - return "eJzkXF1z2zazvs+v2MlN0xnb0+tcnBnXcVrPxElqOe2cK3ZFrkgcgwADgJLZX38GC1KiJFKiZFHO21c3iSWSeHaxePYDC17CE1XvIa/sd/kGwAkn6T28va8mf3x6+wYgIRsbUTih1Xv4nzcAAPwbWDJzMmAdutJCTs6I2EKspaTYUQIzo/Nw6dUbAJtp46JYq5lI38MMpaU3AIYkoaX3kOIbgJkgmdj3PMYlKMxphct/XFX4S40ui/qbDnD+8zff9TfEWjkUyoLLaInQZehgQYZAT/2va1CXj/hekqmu6j/bwNrgUpRkMAoqWP7aBdR/lsJOyWHr+x4hWJC1EYYLNK34inpu/GyxPKAV/MZPvGoNsyldW0IsClmt/dIn3R5J/OfaP6wBFUa92rioC0sbj9aatn5sICW6nMqun/fg8p/f9QL0zJFikUUwbOPteGGEo0tLLihDqBR06S717FKbhAy8K9CglCTFP+hHAJrNRCxIxdXPK/F2SSRHlmglwQItWA1W6gU4HQSq7Wd1jXAZZCLNvA7ou9I/2WBdQTEJEBqvoM2pC58/UZZkIZbakvFj/AKGZuG/CKkhdGQgxcKvggWRCmBQJTBD28JhB+huIVSiF6No73pOBlOCRFiHKqYlXNaMdYxY6oX/b6xVXBpDyslqqSVWXY8MDf6YjDvV4roh48RMxMEGX7TIEips1Aj+6tplRcLc21Uw1RgVTAkKba2YtjQuFDRLEd4V2pFyAiUklBoi0DPYWKhDVqdQCT1HVvzTrwepVXqcFh4zAlXmUzIeHSlnBFkvhufueG0+GccgvI7MHMdhlGbWVpidQWUx9jdZMBSTmHvGzIQkwPavYKiQXhjqW9fLNSFL68icbFmEx71sQfiwJRLJGCagHcqWQmvpISf/jc1EAXGGKiULGRYFKUoGWMFI9noTSK4Ft4a5tNmAfgjCzZBpHeMTVQttuhQ+AOYkTLU3z0zYpUpjnRdakXJX8OhpRNgLWGTkvJ/z4JVOCIT1LOH8zQhfH+7urx/+F7SBz18+R82fqwftsWSd5+J0/M5P+/Gjp7VV7+ONoAYfSemSFcth08u9+/F2fLRvX4kyyLtrpTgV6jSBbhPfg/5uxqYanKGw8OXjx4uV8WZoQWkHFbnV4Bx4qSosB9peDVtG5P2SsJBj5b1s4r2uhlzYkLuVhh3SFdxkFD/xI8kYbUDqFGbaQGF0QQYSganS1ol4H+HTfJMHjl4jcDu38PGopUFzEW+u1X2TNQQRAHwS1oWM7du3uw8/MTOhlDxnNgzc5KCdJLouYLg63Buj8vNt6P/0OgNDqZyQUOkSDHEi438VJqTTiZ+kmKztdcawQdX9rPEyps70ImiG4xaFMozWGOvtnxP4arTTsZZ7rGgm9SKK3Wbgc7QpffRZyY1Wzmh5JNsWWNqtxQ8n4VsfOc5MTbJeWSInsMJzmdeb9JnUx0/fJr/D5PH68duEmcuzGgfQTSzWMHQA2ix1r0kuNJi20tufOwWa3aafeXsBmV5AXsZZqDlInHsEqecnn9v5hDnRi0MjhAAqUv1BwssCb8eRV1Bc4aMa4Z1XrYpghTmhLU3ILBQqbSnWKhmyZgzF8xFwP5ArTV39WQVhH2+ir9ffJrdAc8/n6/6gCcovQKhYlomfDZdpS+uX2bVwpv35pqR4Isi1XQYfczQCp5Js8D2xLv3qZfbniEsrgkRTcEaGLDkPzVRB20xKZTCHtSrbLg4i1U/OZ9XnD6ioRkl+1UfLEKWTCDtUtUdNfqlY+l6S55agowsfEHMAdNEQNRPOKjpqhYD7MOt4K1k9mrM/+0myBcU+bz5N5jedRTjVxo3CQhu5H+tiPaVelXYZRajsBoZduy5k3EIBPVNc7tA7bFSfohkKWRp6Tfk8BEo2Ch6ObF9+BVs51iuhXxr8Sz1Cl60PhLlu836UHkuHHdbeBvq9pLIrKoF9Oh0IGFqFhHdC+QzMoSJd2p9Bkkpd1pAKC8Nwduh3C3qE8z50e+KuAwR4WEJrMGPIKRPQczLLMtyQmKzbm0Crlq6VFQkZnMoKJJqUCxao4JerX3yMosIyWrqpOisIxf1VPR3Qcom9dzhkV1cBGlrV8nzQuBBSQkqKjI+KEKTmPL4dRrrMaOekUOlBc5Xj88im5v1Xjs8iL/Ne8zpsloaIJdQ5xBJqTLFWzFVIrM5BsZv1Yqxsk5SgrfKw3+mJ+AlSg6qUaIQbGD7252EnI18fGv5ryNer7Acl38kSWjf5Dk2GX0a8QiW8n9FDgz5GVuQW2jzxt2WaFaUDYe2BGn1NilwZwb+KIk8m1hlKZHdNYSz0bSyzmo+T+7pKEfhzT55lCJPuPo6j6tB/tfZN6j4gYcMgzNZxTIXjpFXs3fBrQo1TZYF/ccQzIdc8+aV7f+UotYct79eK1JbRFxeFQ62NK74DbHFauRGzOSv+oReCbccVp5rzhxYHv6z1AR1G51GhH2q1LT6EZp7oLGGYH+ZwYGfS2uHg2CLPhC5Y/1S4AzGORTI7OabBB+84cnW6vXiHNMe8AtVs67S3o6CbTHY1W56oy7LNOesNrs1nZ8dlKPSdihmv67rhy3ygFKTGqbKtVd7rTeu+0ifFWNYbFwERJIISbljUpeOew7C9Qa0n1RvSckieWN81vqB1yROdo7xw1q+9emz/Xy9Au8d6jyOdCiX1JqwjDWavJWCc0VUi7FNU2lP1GO0ZbaSBdrHY0cvtV/8wXmwHL7KeOJhv1SZHNyLhrhknj7IR1qGUDQuMtF23T8YjxWhc234BGvAu83nMySziMTzuOJvgBTBK3+E67/IwjeRDiJL7uM8ALIxzCLKepqsRsK06rAajM6VSonP402Krx9mDbENn3u12Wv3wXePux3cay0gOyuVFcE+ON97H9lB+uJk400DjibTq/JdYnWuquG9wdMUJZcm4qJvVTz4aJyujTtFMljaL6trkKOs1x+eIe69G5gVd0GYNeCRLO8sKtc4Q5uMTwegkoPmYQTeBnsYhBOM9z9wnJGlHcf5UHvfD7afbx9um5F3vK3DnbVkMOrdjt8+CnR7l3efJ7cPj0SgtSdrRJX0qlJPbT7c3x6Msi2TXdsypUH77+uH6wBlvbYH5e164trph8aZQzo2LoRmsVagK1YTQ39/0JtZ9oXXWunnnQrhMKLBOG+K28dRgbi+gDM2O/ql/lGRDyaZ55BXcuVVfI9c24ebLffT17vNvoA3/f/J4/Xg3eby7We6z7YtSvzfjvKraltrSqj7LXN/VJJqtLbFp1aSc3EbjlXG8jtnIVhpen8MeZV9sqLr5+/4x+vpw+/X64bb1zc2nL5Pbi9X83D9GD7eT28eh85OhSuSpDu7t35XrOFu10xoGWMS2VSyPJdx8ub+/e2xN3wAeOpPncSKnulxq9MJChnOCKZGqATRHS9idD4BNz0HmSOr4aST0zZ6xik29GtyaPc+0AcI4gxil9KtK8IJpAXv3M8xKxcHpBSwyEWd1h5qUFeg4Lo2FuiluSqkI6bBfhaQSPoATx2RtOJXuah4Mp8EGqCg3JhLqDPbXzO1SY6UlC8hcgSkBqVQo+smCXii4L6UTlw+oUoIHwgREXkhWb9gY5bZzFjUIP+QQhqECzRg2fL3RDU/NWFBkaMMJoIW+DH+E1e4D03D6bNgRDByh52tIE9dMmN4uy1M2t3SZCY/NR8krPtBSnxznYz5eIwN7dJ5os1lkXAEMsXfn9ehhAnougylaSkArQI9oIHbZ3+N6JvDLdqInWpuFKxgmgaLnH0ACj4JnQSgWZN+xXVjnja5u9zNL4FEIXdpjpTCdXaNnFGJrFczEsw8RtRXhEO5gOaIfz6b839wDMxM+oB3A6FrKKY4Wk3RA965p09l6eQoyM21ynp+AaeWbhqTROKdCi3E6Pg6URGJM/pcG0iHwox9nRjwS4IngiO4Qcc5ULmivhzBkvbz53CnHZUPeiWDEmbGGytRQrKt6ltLJdFOOkyaDy/37cjYjExVd79HaFe3t0Up/pLfM78q82H732+bgvbw6iFUf2Y3p1PhkxactasU0gVljbWpvh/yWGJJJw653SukPv9YKAq+gC3BGpCmZdnXDOe5Xn9WTFrUUGrGM6CKblY4PcWvTe5nSi52OdXXmE5PX1tsCTQ5l0aMmrxvvs7xe2OQtufACqZSPBRhDttAqHKbW/uH1O9GASYf1L3I6WNusGa/tUEXsUTZfNVTZfe0U+zKhQXrelQ2t99T2Bg17g4/B4Uf7WP9mF0jvgmh6/fglBehwX0jVkkoY15ckjSVWvPUCpiBes+gZU22lvSIPMht+xn+J2ay5vj3K27KXH9Nctg3l5aYBm5vQnT0248jTHZowilqm5etzXiKVob6jaWOK5Ic92cxIdD0tY2OLUY98MklyYeNXEGMfAXChHQ3BtLRVuye5alXeUUodeus5YUlyoYR1PvyYEx9SzAiTC7BlnAGG+oTU8ZOFuhaKCRZ8bYY22/vyx3W9sas4s2NqevX7ok5+NWe6c/thb+X2jK4Hsz4IcAbb42qJUPVeYnd0Gk56Y3LJUDnz9XpRTTPiYGvh+6P6JWT/0SK3TgzYchreWeNktXzBWnNMIMO5995hnfJGSAjPd59W7NNcf41wTK29NagSnb9tKcRzlnAC643roMPBwjRu9TX8hk5FjDLYQINjWJDa/ersMxJFX09hG8bYOqv3futFE+tS1q0G6ISdVavIaI2RUSWQYbKszibCUOyXC1+eCPt0gO14+t+V0Y+hDLt8y+t2TYTdUQ+TJJXC3CtPVhv5eZObh/cRbqXd/NACDebkyLSfM1hTCxQuOluU+VmbPIgZulKhn17Du4WbqGdFrVfwV0aquUMRJat6vjZ1Zze/Ni+luscFYklYu3uOk3COQuJU0kXznJA9WLA6p7XEJOxe8itnUNTVXjbutfmd+WAqq1tgml1k/tcuN/Hr1/+SpfCoQXzCaop2UuGpalMbRzItJDoUsvuC5v8PAAD//zEqUrk=" + return "eJzsXE9z2ziyv+dTdM1lMlWOa845vCqvx9l1VZxkLWen3onbIlsknkGAAUApmk//Cg1SoiRSomRR9ttHXhJLFPHrRqP/4Qd+gGdafoR8aX/IdwBOOEkf4ZeH5eSfn395B5CQjY0onNDqI/zXOwAA/g4smTkZsA5daSEnZ0RsIdZSUuwogZnRebj1+h2AzbRxUazVTKQfYYbS0jsAQ5LQ0kdI8R3ATJBM7Ece4wMozGmNy19uWfhbjS6L6pMWcP76N//q3xBr5VAoCy6jFUKXoYMFGQI99d9uQF094kdJZnld/dkE1gSXoiSDUVDB6ts2oP5aCTslh43PO4RgQTZG6C/QdMl3VHPjZ4vlAa3g7/zE68Yw29I1JcSikMuNb7qkOyCJv278w2pQYdTrrZvasDTxaK1p58saUqLLqWz7+gAuf/1DL0DPHCkWWQTDNt6OF0Y4+mDJBWUIlYIu3Qc9+6BNQgbeF2hQSpLiL/QjAM1mIhak4uVva/H2SSQHlmgtwQItWA1W6gU4HQSq7Gd9j3AZZCLNvA7oh9K/2mBdQTEJEBqvoO2pC9e/UJZkIZbakvFj/A6GZuG/CKkhdGQgxcKvggWRCmBQJTBD28Bhe+huIVSiF4No72ZOBlOCRFiHKqYVXNaMdYxY6oX/b6xVXBpDysnlSkusug4ZavwxGXeuxXVLxomZiIMNvmiRJVTYqBb81bXLioS5t6tgqjEqmBIU2loxbWhcKKiXIrwvtCPlBEpIKDVEoGewtVD7rE6hEvoZWfFXtx6kVulpWnjKCFSZT8l4dKScEWS9GN53xxvzyTh64XVk5jiMR6lnbY3ZGVQWY/8jC4ZiEnPvMTMhCbD5LRgqpBeGutb1ak3I0joyZ1sW4XEvWxA+bYlEMoQJaIeyodBKesjJf2IzUUCcoUrJQoZFQYqSHlYwkL3eBifXgFvBXNlsQN8H4XbKtInxmZYLbdoU3gPmJEy1N89M2JVKY50XWpFy1/Dk3YiwV7DIyPk458ErnRAI672E8z9G+PZ4/3Dz+N+gDXz5+iWq/1w/6IAl6zwX5/Pv/LS3nz1trHqfbwQ1+ExKl6xYTpteHt1Pt+OTY/talF7RXSvFpVCrCbSb+AH09zM21RAMhYWvnz5drY03QwtKO1iSWw/OiZdahuVAu6thx4h8XBIWclz6KJv4qKshFzbUbqXhgHQNtxnFz/xIMkYbkDqFmTZQGF2QgURgqrR1Ij7k8Gm+7QdOXiNwN7fw6aSlQXMRb6/VQ5PVBxEAfBbWhYrt+/f7P35lz4RS8pzZMHBdg7Y60U0Bw93htzEqP9+G/kdvemAolRMSlroEQ1zI+G+FCeV04icpJms7gzFsuepur/EyT53pRdAM5y0KZRitNta7f03gm9FOx1oesKKZ1IsodtuJz8mm9MlXJbdaOaPlid62wNLuLH44i7/1mePMVE7WK0vkBFZ4X+b1Jn0l9enz98k/YPJ08/R9wp7LezVOoOtcrPbQAWi91L0mudFgmkpvXvcKNIdNP/P2CjK9gLyMs9BzkDj3CFLvn3xt5wvmRC+OzRACqEh1JwkvS7wdZ15BcYXPaoQPXpUqghXmhLY0obJQqLSlWKukz5oxFM8HwP1IrjRV92edhH26jb7dfJ/cAc29P9+MB3VSfgVCxbJM/Gy4TFvavM1upDPN67uS4pkg13aVfMzRCJxKsiH2xLr0q5e9P2dcWhEkmkIwMmTJeWhmGbTNTqkM5rDRZdvng0h1O+eL6vMNKqpWkl/10SpFaXWELao6oCa/VCz9KMn7lqCjK58QcwJ0VTtqdjjr7KiRAh7CrOOdYvVkn/3FT5ItKPZ183kqv+kswqk2bhAvtFX7sS42S+p1a5dRhM5u8LAb94WKWyignxSXe/QOW92naIZCloZeUz4PgZKthocj21VfwU6N9UroVwb/0ojQZus9YW7avB+lw9Jhj7U3gf4oqWzLSuCQTnsChkYj4b1QvgJzqEiX9jeQpFKX1U6FhWE4e/S7Az3CeRe6A3nXEQI8rqDVmDHUlAnoOZlVG65PTtYeTaDRS9fKioQMTuUSJJqUGxao4Pfr332OosIyWoWpqioIzf11Px3Qcou9czjkULcENLTu5fmkcSGkhJQUGZ8VIUjNdXwzjXSZ0c5JodKj5irHnwObmo9fOf4UeZl3mtdxs9RHLKEuIZZQQ4q19lyFxOUlXOx2vxiXti5K0C7zsN/pHfEzpAZVKdEI1zN97K7DzuZ8fWr4H+N8vcreqPOdrKC1O9++xfDLHK9QCe9ndLhBnyMrcgttnvnTMs2K0oGw9kiNvqaLXBvBf5SLPJtYF2iR3deNscDbWFU1nyYPVZci+M8DdZYhTNp5HCf1of9s7JtUPCBhwyDsreOYCsdFqzi44VenGueqAv/kjGdCrn7yS/f+ykF6DzvRr5GprbIvbgqHXht3fHvY4nTpBqzmrPiLXgi2mVeca84fGz74ZdQHdBhdRoV+qPW2eB8380wXScP8MMcDu5DWjgfHFnkhdMH6p8IdiXEoJ7PXx9T44D1nrk43F28fcswruJpdna6meU5mLqi5X9zuTvbRLQsyM21yVDG9mGzZdD2bPNf6GmmlzWuklY600pFWOtJKR1rpSCsdaaUjrXSklY600pFWuot/pJWOtFIYaaUjrbRdcyOtFEZa6XG4R1rpq+vzDSpqpJVW10grHWmll0Q/0kpPAAwjrXSklb4FztRIKx1ppSOtdKSVtlwjrXSklY600pFWOtJKt+GOtNKRVtoObKSVjrTSs6I+klbak4S5j2x5Jpbl8XzSphxVo+9cnvGm6hu+LAZKQWqYLttG573atO5qfVKMZbVxERBBIihhwqIuHXMOw/YGNZ5UbUjLPnVi9avhBa1anugc5YWzfu1VY/v/egGar+49EEinQkm9DetEgzloCRhndJ0I+xyV9lwcowOjDTTQPi928nL7m38YL7ajF1lHHsw/1SZHN6DD3TBOHmUrrUMpay8w0HbdIRlPFKMObYcFqMG7zNcxZ7OIp/C402yCF8AgvMNNv8vD1JL3cZTM474AsDDOMcg6SFcDYFszrHqjM6VSonX482KrxjmAbEtnPuy2Wn3/XeP2x7cay0AByuVFCE+ON96HjlB+uJm40EDDibRm/ktcXmqqmDc4uOKEsmRc1O7Vzz4aFyuDTtFMljaLqt7kIOs1x58Rc68G9gu6oO0e8ECWdpEVap0hzId3BIM7Ac3HDNod6HkCQjDey8x9QpL2NOfPFXH/uPt893RXt7yrfQVm3pZFr3M7dvcs2PlR3n+Z3D0+nYzSkqQ9LOlzoZzcfb67PR1lWST7tmPOhfL7tz9ujpzxxhaY/80L11Y7LN4Uypm4GMhgjUZV6CYEfn/NTax4oVXVuv3LhXCZUGCdNsS08dRgbq+gDGRH/9R/lmRDy6Z+5DXcuzWvkXubcPv1Ifp2/+XvoA3/f/J083Q/ebq/Xe2zHcpSf9TjvKraVtrSqjrLXP2qLjQbW2LTZV1yMo3GK+N0HbORrTW8OYcdyr7aUnX998NT9O3x7tvN413jk9vPXyd3V+v5eXiKHu8md0995ydDlchzHdw7vCvXcrZqrzX0sIhdq1gdS7j9+vBw/9SYvh5+6EKRx4mcqnap0QsLGc4JpkSqAlAfLeFw3gM2/QwyR1LHzwOhr/eMVWyq1eA27HmmDRDGGcQopV9VghdMA9j732BWKk5Or2CRiTirGGpSLkHHcWksVKS4KaUilMN+FZJK+ABOHJO14VS6q/xgOA3WQ0W5MZFQF7C/em5XGistWUD2FZgSkEqFol8t6IWCh1I68eERVUrwSJiAyAvJ6g0bo0w7Z1GD8H0OYRgq0AxhwzdbbHiqx4IiQxtOAC30h/BHWO0+MQ2nz/odwcABOF99SFwzYTpZluckt7SZCY/NR8mXfKClOjnOx3y8RnpydJ5pmywyrACGOLrzevQwAb0vgylaSkArQI+oJ3bZzXG9EPgVneiZNmbhGvpJoOjnG5DAo+BZEIoFOXRsFzb9Rhvb/cISeBRCl/ZUKUwra/SCQuysgpn46VNEbUU4hNtbjujt2ZT/mzkwM+ET2h4eXUs5xcFykhboPjRtB1svT/X2KZ6fgGkdm/qU0TinQothGB9HSiIxJv9NDekY+NHbmRGPBHgiOKM7RpwLtQua6yEMWS1vPnfKeVmfdyIYcWGsoTPVF+u6n6V0Mt2W46zF4Gr/vpzNyERF23u09mV7B7TSnemt6rsyL3bf/bY9eKdf7eVVnziM6dT4YsWXLWrtaYJnjbWpoh3yW2JIJrV3vVdK//G3SkHgFXQFzog0JdPsbjjHfPVZNWlRQ6ERy4guslnp+BC3Np23Kb3YG1jXZz4xeW29LdDkUBYdavK68THL64VN3pILL5BK+ViAMWQLrcJhau0fXr0TDdjpsP5FTkdrmzXjtR26iB3K5rv6KruLTnGoEuql533V0CantjNpOJh89E4/msf6t1kgnQui5vrxSwrQ4aGUqiGVMK6rSBpKrHjnBUxBvHrRM6bKSjtF7mU2/Iz/J2azEfoOKG/HXt6muewaystNA7Y3oVs5NsPI056aMIpKptXrc14ilaGuo2lDiuSHPdvMSHQdlLGhxahGPpskubDxK4hxyAFwox0NwbS0yyYnednovKOUOnDruWBJcqGEdT79mBMfUswIkyuwZZwBhv6E1PGzhaoXigkWfG+GNjv48sdNvXGouHBgqrn6XVknv5oz3bv9cLBze8HQg1kXBLiA7XG3RKhqL7E9Ow0nvTH5wFC58vV6UTUZsbe18O+j6iVk/6dFbpwYsOU0vLPGyeXqBWv1MYEM5z56h3XKGyEhPd9/WrFLc909wiG19otBlej8l4ZCvM8STmC1cR102FuYOqy+RtzQqYhRBhuocfRLUttfnX1BR9HFKWzCGFpn1d5vtWhiXcqKaoBO2NlynRlteGRUCWSYrLqziTAU++XCtyfCPh9hO97976voh1CGXb3ldbcnwuGow5MkS4W5V55cbtXndW0e3ke4U3bzQws0mJMj03xOb00tULjoYlnmF23yIGZgpUK3ew3vFq6znrVrvYY/M1L1LxRRsu7na1Mxu/m1eSlVHBeIJWEV7jlPwjkKiVNJV/VzQvVgweqcNgqTsHvJr5xBUXV72bg35nfmk6msosDUu8j8r11t4lev/yVL4VG9/AmrKdrrCs/Vm9o6kmkh0aGR3ZU0/28AAAD//8MvSdc=" } diff --git a/metricbeat/module/mysql/module.yml b/metricbeat/module/mysql/module.yml index ea9854af761e..d07603c61792 100644 --- a/metricbeat/module/mysql/module.yml +++ b/metricbeat/module/mysql/module.yml @@ -1,3 +1,6 @@ dashboards: - id: 66881e90-0006-11e7-bf7f-c9acc3d3e306 file: Metricbeat-mysql-overview.json +name: mysql +metricsets: + - performance diff --git a/metricbeat/module/mysql/performance/_meta/data.json b/metricbeat/module/mysql/performance/_meta/data.json new file mode 100644 index 000000000000..d6526f926548 --- /dev/null +++ b/metricbeat/module/mysql/performance/_meta/data.json @@ -0,0 +1,62 @@ +{ + "@timestamp": "2020-06-18T08:38:39.025Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "c1f810aa-6f0e-4550-b29e-96ee1f402d65", + "id": "de510575-3657-4099-812a-aafd2febc68e" + }, + "service": { + "address": "root:root@tcp(172.18.0.2:3306)/", + "type": "mysql" + }, + "mysql": { + "performance": { + "events_statements": { + "max": { + "timer": { + "wait": 8.918075e+09 + } + }, + "last": { + "seen": "2020-06-18 08:38:05.753769" + }, + "quantile": { + "95": 9.120108393e+09 + }, + "digest": { + "text": "SELECT * FROM `performance_schema` . `global_status` WHERE `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ?" + }, + "count": { + "star": 2 + }, + "avg": { + "timer": { + "wait": 4.846196e+09 + } + } + } + } + }, + "event": { + "duration": 3293342, + "dataset": "mysql.performance", + "module": "mysql" + }, + "metricset": { + "period": 10000, + "name": "performance" + }, + "ecs": { + "version": "1.5.0" + } +} diff --git a/metricbeat/module/mysql/performance/_meta/docs.asciidoc b/metricbeat/module/mysql/performance/_meta/docs.asciidoc new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/metricbeat/module/mysql/performance/_meta/fields.yml b/metricbeat/module/mysql/performance/_meta/fields.yml new file mode 100644 index 000000000000..a796a65099c6 --- /dev/null +++ b/metricbeat/module/mysql/performance/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: overview + type: group + description: > + `performance` contains the metrics that were obtained by the status SQL query. + release: ga + fields: diff --git a/metricbeat/module/mysql/performance/manifest.yml b/metricbeat/module/mysql/performance/manifest.yml new file mode 100644 index 000000000000..9694546ddbd5 --- /dev/null +++ b/metricbeat/module/mysql/performance/manifest.yml @@ -0,0 +1,19 @@ +default: true +input: + module: mysql + metricset: query + defaults: + sql_response_format: table + namespace: performance + sql_queries: + - query: > + SELECT digest_text , count_star , avg_timer_wait , max_timer_wait , last_seen , quantile_95 + FROM performance_schema.events_statements_summary_by_digest + ORDER BY avg_timer_wait DESC + LIMIT 10 + query_namespace: events_statements + - query: > + SELECT object_schema, object_name, index_name, count_fetch + FROM performance_schema.table_io_waits_summary_by_index_usage + WHERE count_fetch > 0 + query_namespace: table_io_waits diff --git a/metricbeat/module/mysql/query/_meta/fields.yml b/metricbeat/module/mysql/query/_meta/fields.yml new file mode 100644 index 000000000000..1ffa8aaaed37 --- /dev/null +++ b/metricbeat/module/mysql/query/_meta/fields.yml @@ -0,0 +1,274 @@ +- name: galera_status + type: group + release: beta + description: > + `galera_status` contains the metrics that were obtained by the status SQL query on Galera. + fields: + - name: apply + type: group + description: > + Apply status fields. + fields: + - name: oooe + type: double + description: > + How often applier started write-set applying out-of-order (parallelization efficiency). + + - name: oool + type: double + description: > + How often write-set was so slow to apply that write-set with higher seqno's were applied earlier. + Values closer to 0 refer to a greater gap between slow and fast write-sets. + + - name: window + type: double + description: > + Average distance between highest and lowest concurrently applied seqno. + + - name: cert + type: group + description: > + Certification status fields. + fields: + - name: deps_distance + type: double + description: > + Average distance between highest and lowest seqno value that can be possibly applied in parallel (potential degree of parallelization). + + - name: index_size + type: long + description: > + The number of entries in the certification index. + + - name: interval + type: double + description: > + Average number of transactions received while a transaction replicates. + + - name: cluster + type: group + description: > + Cluster status fields. + fields: + - name: conf_id + type: long + description: > + Total number of cluster membership changes happened. + + - name: size + type: long + description: > + Current number of members in the cluster. + + - name: status + type: keyword + description: > + Status of this cluster component. That is, whether the node is part of a PRIMARY or NON_PRIMARY component. + + - name: commit + type: group + description: > + Commit status fields. + fields: + - name: oooe + type: double + description: > + How often a transaction was committed out of order. + + - name: window + type: long + description: > + Average distance between highest and lowest concurrently committed seqno. + + - name: connected + type: keyword + description: > + If the value is OFF, the node has not yet connected to any of the cluster components. + This may be due to misconfiguration. Check the error log for proper diagnostics. + + - name: evs + type: group + description: > + Evs Fields. + fields: + - name: evict + type: keyword + description: > + Lists the UUID's of all nodes evicted from the cluster. + Evicted nodes cannot rejoin the cluster until you restart their mysqld processes. + + - name: state + type: keyword + description: > + Shows the internal state of the EVS Protocol. + + - name: flow_ctl + type: group + description: > + Flow Control fields. + fields: + - name: paused + type: double + description: > + The fraction of time since the last FLUSH STATUS command that replication was paused due to flow control. + In other words, how much the slave lag is slowing down the cluster. + + - name: paused_ns + type: long + description: > + The total time spent in a paused state measured in nanoseconds. + + - name: recv + type: long + description: > + Returns the number of FC_PAUSE events the node has received, including those the node has sent. + Unlike most status variables, the counter for this one does not reset every time you run the query. + + - name: sent + type: long + description: > + Returns the number of FC_PAUSE events the node has sent. + Unlike most status variables, the counter for this one does not reset every time you run the query. + + - name: last_committed + type: long + description: > + The sequence number, or seqno, of the last committed transaction. + + - name: local + type: group + description: > + Node specific Cluster status fields. + fields: + - name: bf_aborts + type: long + description: > + Total number of local transactions that were aborted by slave transactions while in execution. + + - name: cert_failures + type: long + description: > + Total number of local transactions that failed certification test. + + - name: commits + type: long + description: > + Total number of local transactions committed. + + - name: recv + type: group + description: > + Node specific recv fields. + fields: + - name: queue + type: long + description: > + Current (instantaneous) length of the recv queue. + + - name: queue_avg + type: double + description: > + Recv queue length averaged over interval since the last FLUSH STATUS command. + Values considerably larger than 0.0 mean that the node cannot apply write-sets as fast + as they are received and will generate a lot of replication throttling. + + - name: queue_max + type: long + description: > + The maximum length of the recv queue since the last FLUSH STATUS command. + + - name: queue_min + type: long + description: > + The minimum length of the recv queue since the last FLUSH STATUS command. + + - name: replays + type: long + description: > + Total number of transaction replays due to asymmetric lock granularity. + + - name: send + type: group + description: > + Node specific sent fields. + fields: + - name: queue + type: long + description: > + Current (instantaneous) length of the send queue. + + - name: queue_avg + type: double + description: > + Send queue length averaged over time since the last FLUSH STATUS command. + Values considerably larger than 0.0 indicate replication throttling or network throughput issue. + + - name: queue_max + type: long + description: > + The maximum length of the send queue since the last FLUSH STATUS command. + + - name: queue_min + type: long + description: > + The minimum length of the send queue since the last FLUSH STATUS command. + + - name: state + type: keyword + description: > + Internal Galera Cluster FSM state number. + + - name: ready + type: keyword + description: > + Whether the server is ready to accept queries. + + - name: received + type: group + description: > + Write-Set receive status fields. + fields: + - name: count + type: long + description: > + Total number of write-sets received from other nodes. + + - name: bytes + type: long + description: > + Total size of write-sets received from other nodes. + + - name: repl + type: group + description: > + Replication status fields. + fields: + - name: data_bytes + type: long + description: > + Total size of data replicated. + + - name: keys + type: long + description: > + Total number of keys replicated. + + - name: keys_bytes + type: long + description: > + Total size of keys replicated. + + - name: other_bytes + type: long + description: > + Total size of other bits replicated. + + - name: count + type: long + description: > + Total number of write-sets replicated (sent to other nodes). + + - name: bytes + type: long + description: > + Total size of write-sets replicated. diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go new file mode 100644 index 000000000000..79c936223623 --- /dev/null +++ b/metricbeat/module/mysql/query/query.go @@ -0,0 +1,132 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package status fetches MySQL server status metrics. + +For more information on the query it uses, see: +http://dev.mysql.com/doc/refman/5.7/en/show-status.html +*/ +package query + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/helper/sql" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func init() { + mb.Registry.MustAddMetricSet("mysql", "query", New, + mb.DefaultMetricSet(), + ) +} + +type query struct { + QueryNamespace string `config:"query_namespace"` + Query string `config:"query"` +} + +// MetricSet for fetching MySQL server status. +type MetricSet struct { + mb.BaseMetricSet + db *sql.DbClient + Config struct { + Queries []query `config:"sql_queries" validate:"nonzero,required"` + Namespace string `config:"namespace" validate:"nonzero,required"` + ResponseFormat string `config:"sql_response_format"` + } +} + +// New creates and returns a new MetricSet instance. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The mysql 'query' metricset is beta.") + + b := &MetricSet{BaseMetricSet: base} + + if err := base.Module().UnpackConfig(&b.Config); err != nil { + return nil, err + } + + return b, nil +} + +// Fetch fetches status messages from a mysql host. +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + if m.db == nil { + var err error + m.db, err = sql.DB("mysql", m.HostData().URI, m.Logger()) + if err != nil { + return errors.Wrap(err, "mysql-status fetch failed") + } + } + + for _, q := range m.Config.Queries { + err := m.fetchQuery(ctx, q, reporter) + if err != nil { + m.Logger().Error("error doing query", err) + } + } + + return nil +} + +func (m *MetricSet) fetchQuery(ctx context.Context, q query, reporter mb.ReporterV2) error { + if m.Config.ResponseFormat == "table" { + mss, err := m.db.FetchTableMode(ctx, q.Query) + if err != nil { + return err + } + + for _, ms := range mss { + event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} + if q.QueryNamespace != "" { + event.ModuleFields[m.Config.Namespace] = common.MapStr{q.QueryNamespace: sql.ToDotKeys(ms)} + } else { + event.ModuleFields[m.Config.Namespace] = sql.ToDotKeys(ms) + } + reporter.Event(event) + } + } else { + ms, err := m.db.FetchVariableMode(ctx, q.Query) + if err != nil { + return err + } + + event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} + if q.QueryNamespace != "" { + event.ModuleFields[m.Config.Namespace] = common.MapStr{q.QueryNamespace: sql.ToDotKeys(ms)} + } else { + event.ModuleFields[m.Config.Namespace] = sql.ToDotKeys(ms) + } + reporter.Event(event) + } + + return nil +} + +// Close closes the database connection and prevents future queries. +func (m *MetricSet) Close() error { + if m.db == nil { + return nil + } + return errors.Wrap(m.db.Close(), "failed to close mysql database client") +} diff --git a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc index f22edf1fa200..b6896851920a 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc @@ -1,3 +1,355 @@ -This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. +The SQL module allows to execute custom queries against an SQL database and store the results to Elasticsearch. +The currently supported databases are the ones already included in Metricbeat, which are: +- PostgreSQL +- MySQL +- Oracle +- Microsoft SQL +- CockroachDB +== Quickstart + +You can setup the module by activating it first running + + metricbeat module enable sql + +Once it is activated, open `modules.d/sql.yml` and fill the required fields. This is an example that captures Innodb related metrics from the result of the query `SHOW GLOBAL STATUS LIKE 'Innodb_system%'` in a MySQL database: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["root:root@tcp(localhost:3306)/ps"] + + driver: "mysql" + sql_query: "SHOW GLOBAL STATUS LIKE 'Innodb_system%'" + sql_response_format: variables +---- + +.SHOW GLOBAL STATUS LIKE 'Innodb_system%' +|==== +|Variable_name|Value + +|Innodb_system_rows_deleted|0 +|Innodb_system_rows_inserted|0 +|Innodb_system_rows_read|5062 +|Innodb_system_rows_updated|315 +|==== + + +Keys in the YAML are defined as follow: + +- `driver`: The drivers currently supported are those which already have a Metricbeat module like `mssql` or `postgres`. +- `sql_query`: Is the single query you want to run +- `sql_response_format`: You have 2 options here: + - `variables`: Expects a table which looks like a key/value result. With 2 columns, left column will be considered a key and the right column the value. This mode generates a single event on each fetch operation. + - `table`: Table mode can contain any number of columns and a single event will be generated for each row. + +Results will be grouped by type in the result event for convenient mapping in Elasticsearch. So `strings` values will be grouped into `sql.strings`, `numeric` into `sql.numeric` and so on and so forth. + +The event generated with the example above looks like this: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:09:14.407Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "172.18.0.2:3306", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 1272810 + }, + "sql": { + "driver": "mysql", + "query": "SHOW GLOBAL STATUS LIKE 'Innodb_system%'", + "metrics": { + "numeric": { + "innodb_system_rows_updated": 315, + "innodb_system_rows_deleted": 0, + "innodb_system_rows_inserted": 0, + "innodb_system_rows_read": 5062 + } + } + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "488431bd-bd3c-4442-ad51-0c50eb555787", + "id": "670ef211-87f0-4f38-8beb-655c377f1629" + } +} +---- + +In this example, we are querying PostgreSQL and generate a "table" result, hence a single event for each row returned + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table +---- + +.SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database +|==== +|datid|datname|blks_read|blks_hit|tup_returned|tup_fetched|stats_reset + +|69448|stuff|8652|205976|1484625|53218|2020-06-07 22:50:12 +|13408|postgres|0|0|0|0| +|13407|template0|0|0|0|0| +|==== + +With 3 rows on the table, three events will be generated with the contents of each row. As an example, below you can see the event created for the first row: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T14:47:35.481Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "localhost:5432", + "type": "sql" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "1bffe66d-a1ae-4ed6-985a-fd48548a1971", + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic" + }, + "sql": { + "metrics": { + "numeric": { + "tup_fetched": 53350, + "datid": 69448, + "blks_read": 8652, + "blks_hit": 206501, + "tup_returned": 1.491873e+06 + }, + "string": { + "stats_reset": "2020-06-07T20:50:12.632975Z", + "datname": "stuff" + } + }, + "driver": "postgres", + "query": "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 14076705 + }, + "metricset": { + "name": "query", + "period": 10000 + } +} +---- + + +== More examples + +=== Oracle: + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] + + driver: "oracle" + sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS' + sql_response_format: table +---- + + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:41:02.200Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "metrics": { + "numeric": { + "hit ratio": 0.9742963357937117, + "physical_reads": 17161, + "db_block_gets": 122221, + "consistent_gets": 545427 + }, + "string": { + "name": "DEFAULT" + } + }, + "driver": "oracle", + "query": "SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) \"Hit Ratio\" FROM V$BUFFER_POOL_STATISTICS" + }, + "metricset": { + "period": 10000, + "name": "query" + }, + "service": { + "address": "172.17.0.3:1521", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 39233704 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788" + } +} +---- + +=== MSSQL + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["sqlserver://SA:password@localhost"] + + driver: "mssql" + sql_query: 'SELECT * FROM sys.dm_db_log_space_usage' + sql_response_format: table +---- + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:39:14.421Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "driver": "mssql", + "query": "SELECT * FROM sys.dm_db_log_space_usage", + "metrics": { + "numeric": { + "log_space_in_bytes_since_last_backup": 524288, + "database_id": 1, + "total_log_size_in_bytes": 2.08896e+06, + "used_log_space_in_bytes": 954368, + "used_log_space_in_percent": 45.686275482177734 + } + } + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 40750570 + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.17.0.2", + "type": "sql" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "3da88889-036e-47cb-a88b-275037fa2bc9" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + } +} +---- + +=== Two or more queries + +If you want to launch two or more queries, you need to specify them with their full configuration for each query. For example: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table + +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT * FROM pg_catalog.pg_tables pt WHERE schemaname ='pg_catalog'" + sql_response_format: table +--- diff --git a/x-pack/metricbeat/module/sql/_meta/fields.yml b/x-pack/metricbeat/module/sql/_meta/fields.yml index 844136cedb83..af2667920323 100644 --- a/x-pack/metricbeat/module/sql/_meta/fields.yml +++ b/x-pack/metricbeat/module/sql/_meta/fields.yml @@ -25,3 +25,8 @@ object_type: keyword description: > Non-numeric values collected. + - name: metrics.boolean.* + type: object + object_type: keyword + description: > + Boolean values collected. diff --git a/x-pack/metricbeat/module/sql/fields.go b/x-pack/metricbeat/module/sql/fields.go index 8b2c4da414d1..e346fd7f8b95 100644 --- a/x-pack/metricbeat/module/sql/fields.go +++ b/x-pack/metricbeat/module/sql/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSql returns asset data. // This is the base64 encoded gzipped contents of module/sql. func AssetSql() string { - return "eJykkkFuwjAQRfc5xRfLSnAAL7rqEiEhDlA59gdcnBjsMW1uX5HEKFWKVFQvv2fmvUy8xImdQrr4ChAnngqL3Xa9qIBIT52oUFN0BVgmE91ZXGgVXisA2G3XaILNnthTzJEJDSU6k7CPoYHuK6wWXevECtg7eptU37xEqxsW+O1Id6bCIYZ8HpNp/bTHRndlvMel9cTuM0Q7yX+RLuetn4GcaCEB/KLJQsiRuGTGbjWj9vH/oNvbiMLquSZ4TyNlcXNquWhzw+jM6mVmEOoPGpnEQ/A+3NqQa8+/6W0Gxv0vjnK0j7WSRNcenrZ6amub0C7Hz8dV+8wHZj+f7HcAAAD//yd20AA=" + return "eJy0k0Fu8jAQhfc5xRPLX4IDZPEvqi4REuIAlWM/wMXJgD2mze0rEoJSpa1AVb18npnvy0Se48C2RDqFAlCvgSVmm/VyVgCRgSaxREU1BeCYbPRH9dKU+F8AwGa9RC0uB2JLtXsm1NTobcI2Sg3TVTijpjKJBbD1DC6VXfMcjak5wC9H2yNL7KLk4zUZ1497XPRnxls8tB7Yvkl0o/wL6eE8dzOQEx1UwHfarITuiVNmbBcTahf/Drq+jBhYHddKCLQ6LG5KHS6aXDN6u/g3MZDqlVZHcR+89LdOchV4n96qZ9z+4lWO7nutpNE3u4etHtraSpr59fNxNiHzLrNKJNA0f6v21EN+1Pr8kj4CAAD//07X+ko=" } diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index d6e91f60eaeb..3c85127dd511 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -7,14 +7,13 @@ package query import ( "context" "fmt" - "strconv" - "strings" - "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/helper/sql" "github.com/jmoiron/sqlx" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" ) @@ -81,167 +80,81 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It calls m.fetchTableMode() or m.fetchVariableMode() depending on the response // format of the query. func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { - db, err := m.DB() + db, err := sql.DB(m.Driver, m.HostData().URI, m.Logger()) if err != nil { return errors.Wrap(err, "error opening connection") } - - rows, err := db.QueryxContext(ctx, m.Query) - if err != nil { - return errors.Wrap(err, "error executing query") - } - defer rows.Close() + defer db.Close() if m.ResponseFormat == tableResponseFormat { - return m.fetchTableMode(rows, report) - } - - return m.fetchVariableMode(rows, report) -} - -// DB gets a client ready to query the database -func (m *MetricSet) DB() (*sqlx.DB, error) { - if m.db == nil { - db, err := sqlx.Open(switchDriverName(m.Driver), m.HostData().URI) - if err != nil { - return nil, errors.Wrap(err, "opening connection") - } - err = db.Ping() - if err != nil { - return nil, errors.Wrap(err, "testing connection") - } - - m.db = db - } - return m.db, nil -} - -// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. -func (m *MetricSet) fetchTableMode(rows *sqlx.Rows, report mb.ReporterV2) error { - - // Extracted from - // https://stackoverflow.com/questions/23507531/is-golangs-sql-package-incapable-of-ad-hoc-exploratory-queries/23507765#23507765 - cols, err := rows.Columns() - if err != nil { - return errors.Wrap(err, "error getting columns") - } - - for k, v := range cols { - cols[k] = strings.ToLower(v) - } - - vals := make([]interface{}, len(cols)) - for i := 0; i < len(cols); i++ { - vals[i] = new(interface{}) - } - - for rows.Next() { - err = rows.Scan(vals...) + mss, err := db.FetchTableMode(ctx, m.Query) if err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to scan rows")) - continue + return err } - numericMetrics := common.MapStr{} - stringMetrics := common.MapStr{} - - for i := 0; i < len(vals); i++ { - value := getValue(vals[i].(*interface{})) - num, err := strconv.ParseFloat(value, 64) - if err == nil { - numericMetrics[cols[i]] = num - } else { - stringMetrics[cols[i]] = value - } - + for _, ms := range mss { + report.Event(m.getEvent(ms)) } - report.Event(mb.Event{ - RootFields: common.MapStr{ - "sql": common.MapStr{ - "driver": m.Driver, - "query": m.Query, - "metrics": common.MapStr{ - "numeric": numericMetrics, - "string": stringMetrics, - }, - }, - }, - }) + return nil } - if err = rows.Err(); err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to read rows")) + ms, err := db.FetchVariableMode(ctx, m.Query) + if err != nil { + return err } + report.Event(m.getEvent(ms)) return nil } -// fetchVariableMode scan the rows and publishes the event for querys that return the response in a key/value format. -func (m *MetricSet) fetchVariableMode(rows *sqlx.Rows, report mb.ReporterV2) error { - data := common.MapStr{} - for rows.Next() { - var key string - var val interface{} - err := rows.Scan(&key, &val) - if err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to scan rows")) - continue - } - - key = strings.ToLower(key) - data[key] = val +func (m *MetricSet) getEvent(ms common.MapStr) mb.Event { + return mb.Event{ + RootFields: common.MapStr{ + "sql": common.MapStr{ + "driver": m.Driver, + "query": m.Query, + "metrics": getMetrics(ms), + }, + }, } +} - if err := rows.Err(); err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to read rows")) - } +func getMetrics(ms common.MapStr) (ret common.MapStr) { + ret = common.MapStr{} numericMetrics := common.MapStr{} stringMetrics := common.MapStr{} - - for key, value := range data { - value := getValue(&value) - num, err := strconv.ParseFloat(value, 64) - if err == nil { - numericMetrics[key] = num - } else { - stringMetrics[key] = value + boolMetrics := common.MapStr{} + + for k, v := range ms { + switch v.(type) { + case float64: + numericMetrics.Put(k, v) + case string: + stringMetrics.Put(k, v) + case bool: + boolMetrics.Put(k, v) + case nil: + //Ignore because a nil has no data type and thus cannot be indexed + default: + stringMetrics.Put(k, v) } } - report.Event(mb.Event{ - RootFields: common.MapStr{ - "sql": common.MapStr{ - "driver": m.Driver, - "query": m.Query, - "metrics": common.MapStr{ - "numeric": numericMetrics, - "string": stringMetrics, - }, - }, - }, - }) + if len(numericMetrics) > 0 { + ret.Put("numeric", numericMetrics) + } - return nil -} + if len(stringMetrics) > 0 { + ret.Put("string", stringMetrics) + } -func getValue(pval *interface{}) string { - switch v := (*pval).(type) { - case nil: - return "NULL" - case bool: - if v { - return "true" - } - return "false" - case []byte: - return string(v) - case time.Time: - return v.Format(time.RFC3339Nano) - default: - return fmt.Sprint(v) + if len(boolMetrics) > 0 { + ret.Put("bool", boolMetrics) } + + return } // Close closes the connection pool releasing its resources @@ -251,13 +164,3 @@ func (m *MetricSet) Close() error { } return errors.Wrap(m.db.Close(), "closing connection") } - -// switchDriverName switches between driver name and a pretty name for a driver. For example, 'oracle' driver is called -// 'godror' so this detail implementation must be hidden to the user, that should only choose and see 'oracle' as driver -func switchDriverName(d string) string { - if d == "oracle" { - return "godror" - } - - return d -} From 295b25e26f063c61cc7f3584d7282326f228b4bd Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 7 Jul 2020 16:13:23 +0200 Subject: [PATCH 02/21] Fix import order --- x-pack/metricbeat/module/sql/query/query.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index 3c85127dd511..37f8edd87f9f 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -8,13 +8,12 @@ import ( "context" "fmt" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/metricbeat/helper/sql" - "github.com/jmoiron/sqlx" "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/helper/sql" "github.com/elastic/beats/v7/metricbeat/mb" ) From b0fb6ddc1805d03b1ddfb4d05e0419da9febffbf Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 7 Jul 2020 18:33:57 +0200 Subject: [PATCH 03/21] Fix unit test --- metricbeat/helper/sql/sql_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/helper/sql/sql_test.go b/metricbeat/helper/sql/sql_test.go index 31a496454f63..9fbaeb8643ec 100644 --- a/metricbeat/helper/sql/sql_test.go +++ b/metricbeat/helper/sql/sql_test.go @@ -36,7 +36,7 @@ type mockVariableMode struct { } func (m *mockVariableMode) Scan(dest ...interface{}) error { - d1 := dest[0].(*interface{}) + d1 := dest[0].(*string) *d1 = m.results[m.index].k d2 := dest[1].(*interface{}) From a06c76b2646aa2ff55812362fdb9c0963f34fc55 Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 8 Jul 2020 15:47:21 +0200 Subject: [PATCH 04/21] Raise go.mod version to 1.14 too --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 44231c67d658..a666752561e6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.13 +go 1.14 require ( 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b From 2384aa2e4b0edd02a95e93f87f6ee07617a265d5 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 12:06:45 +0200 Subject: [PATCH 05/21] Updated fields --- metricbeat/docs/fields.asciidoc | 445 ++---------------- .../docs/modules/mysql/performance.asciidoc | 2 + metricbeat/docs/modules_list.asciidoc | 2 +- .../module/mysql/performance/_meta/fields.yml | 49 +- .../module/mysql/query/_meta/fields.yml | 272 +---------- 5 files changed, 82 insertions(+), 688 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 655a64e9c075..93fd19e00e02 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -32017,504 +32017,121 @@ type: long -- [float] -=== overview - -`performance` contains the metrics that were obtained by the status SQL query. - - -[float] -=== galera_status - -`galera_status` contains the metrics that were obtained by the status SQL query on Galera. - - - -[float] -=== apply - -Apply status fields. - - - -*`mysql.galera_status.apply.oooe`*:: -+ --- -How often applier started write-set applying out-of-order (parallelization efficiency). - - -type: double - --- - -*`mysql.galera_status.apply.oool`*:: -+ --- -How often write-set was so slow to apply that write-set with higher seqno's were applied earlier. Values closer to 0 refer to a greater gap between slow and fast write-sets. - - -type: double - --- - -*`mysql.galera_status.apply.window`*:: -+ --- -Average distance between highest and lowest concurrently applied seqno. - - -type: double - --- - -[float] -=== cert - -Certification status fields. - - - -*`mysql.galera_status.cert.deps_distance`*:: -+ --- -Average distance between highest and lowest seqno value that can be possibly applied in parallel (potential degree of parallelization). - - -type: double - --- - -*`mysql.galera_status.cert.index_size`*:: -+ --- -The number of entries in the certification index. - - -type: long - --- - -*`mysql.galera_status.cert.interval`*:: -+ --- -Average number of transactions received while a transaction replicates. - - -type: double - --- - -[float] -=== cluster - -Cluster status fields. - - - -*`mysql.galera_status.cluster.conf_id`*:: -+ --- -Total number of cluster membership changes happened. - - -type: long - --- - -*`mysql.galera_status.cluster.size`*:: -+ --- -Current number of members in the cluster. - - -type: long - --- - -*`mysql.galera_status.cluster.status`*:: -+ --- -Status of this cluster component. That is, whether the node is part of a PRIMARY or NON_PRIMARY component. - - -type: keyword - --- - -[float] -=== commit - -Commit status fields. - - - -*`mysql.galera_status.commit.oooe`*:: -+ --- -How often a transaction was committed out of order. - - -type: double - --- - -*`mysql.galera_status.commit.window`*:: -+ --- -Average distance between highest and lowest concurrently committed seqno. - - -type: long - --- - -*`mysql.galera_status.connected`*:: -+ --- -If the value is OFF, the node has not yet connected to any of the cluster components. This may be due to misconfiguration. Check the error log for proper diagnostics. - - -type: keyword - --- - -[float] -=== evs - -Evs Fields. - - - -*`mysql.galera_status.evs.evict`*:: -+ --- -Lists the UUID's of all nodes evicted from the cluster. Evicted nodes cannot rejoin the cluster until you restart their mysqld processes. - - -type: keyword - --- - -*`mysql.galera_status.evs.state`*:: -+ --- -Shows the internal state of the EVS Protocol. - - -type: keyword - --- - -[float] -=== flow_ctl - -Flow Control fields. - - - -*`mysql.galera_status.flow_ctl.paused`*:: -+ --- -The fraction of time since the last FLUSH STATUS command that replication was paused due to flow control. In other words, how much the slave lag is slowing down the cluster. - - -type: double - --- - -*`mysql.galera_status.flow_ctl.paused_ns`*:: -+ --- -The total time spent in a paused state measured in nanoseconds. - - -type: long - --- - -*`mysql.galera_status.flow_ctl.recv`*:: -+ --- -Returns the number of FC_PAUSE events the node has received, including those the node has sent. Unlike most status variables, the counter for this one does not reset every time you run the query. - - -type: long - --- - -*`mysql.galera_status.flow_ctl.sent`*:: -+ --- -Returns the number of FC_PAUSE events the node has sent. Unlike most status variables, the counter for this one does not reset every time you run the query. - - -type: long - --- - -*`mysql.galera_status.last_committed`*:: -+ --- -The sequence number, or seqno, of the last committed transaction. +=== performance +`performance` contains metrics related to the performance of a MySQL instance -type: long --- [float] -=== local - -Node specific Cluster status fields. - - - -*`mysql.galera_status.local.bf_aborts`*:: -+ --- -Total number of local transactions that were aborted by slave transactions while in execution. - +=== events_statements -type: long +Records statement events summarized by schema and digest --- -*`mysql.galera_status.local.cert_failures`*:: +*`mysql.performance.events_statements.max.timer.wait`*:: + -- -Total number of local transactions that failed certification test. - +Maximum wait time of the summarized events that are timed type: long -- -*`mysql.galera_status.local.commits`*:: +*`mysql.performance.events_statements.last.seen`*:: + -- -Total number of local transactions committed. - +Time at which the digest was most recently seen -type: long +type: date -- -[float] -=== recv - -Node specific recv fields. - - - -*`mysql.galera_status.local.recv.queue`*:: +*`mysql.performance.events_statements.quantile.95`*:: + -- -Current (instantaneous) length of the recv queue. - +TODO type: long -- -*`mysql.galera_status.local.recv.queue_avg`*:: -+ --- -Recv queue length averaged over interval since the last FLUSH STATUS command. Values considerably larger than 0.0 mean that the node cannot apply write-sets as fast as they are received and will generate a lot of replication throttling. - - -type: double - --- - -*`mysql.galera_status.local.recv.queue_max`*:: +*`mysql.performance.events_statements.digest`*:: + -- -The maximum length of the recv queue since the last FLUSH STATUS command. - +Performance schema digest -type: long +type: text -- -*`mysql.galera_status.local.recv.queue_min`*:: +*`mysql.performance.events_statements.count.star`*:: + -- -The minimum length of the recv queue since the last FLUSH STATUS command. - +Number of summarized events type: long -- -*`mysql.galera_status.local.replays`*:: +*`mysql.performance.events_statements.avg.timer.wait`*:: + -- -Total number of transaction replays due to asymmetric lock granularity. - +Average wait time of the summarized events that are timed type: long -- [float] -=== send - -Node specific sent fields. - - - -*`mysql.galera_status.local.send.queue`*:: -+ --- -Current (instantaneous) length of the send queue. - - -type: long - --- - -*`mysql.galera_status.local.send.queue_avg`*:: -+ --- -Send queue length averaged over time since the last FLUSH STATUS command. Values considerably larger than 0.0 indicate replication throttling or network throughput issue. - - -type: double - --- - -*`mysql.galera_status.local.send.queue_max`*:: -+ --- -The maximum length of the send queue since the last FLUSH STATUS command. - - -type: long - --- +=== table_io_waits -*`mysql.galera_status.local.send.queue_min`*:: -+ --- -The minimum length of the send queue since the last FLUSH STATUS command. +Records table I/O waits by index -type: long - --- -*`mysql.galera_status.local.state`*:: +*`mysql.performance.table_io_waits.object.schema`*:: + -- -Internal Galera Cluster FSM state number. - +Schema name type: keyword -- -*`mysql.galera_status.ready`*:: +*`mysql.performance.table_io_waits.object.name`*:: + -- -Whether the server is ready to accept queries. - +Table name type: keyword -- -[float] -=== received - -Write-Set receive status fields. - - - -*`mysql.galera_status.received.count`*:: +*`mysql.performance.table_io_waits.index.name`*:: + -- -Total number of write-sets received from other nodes. +Name of the index that was used when the table I/O wait event was recorded. PRIMARY indicates that table I/O used the primary index. NULL means that table I/O used no index. Inserts are counted against INDEX_NAME = NULL -type: long +type: keyword -- -*`mysql.galera_status.received.bytes`*:: +*`mysql.performance.table_io_waits.count.fetch`*:: + -- -Total size of write-sets received from other nodes. - +Number of all fetch operations > 0 type: long -- [float] -=== repl - -Replication status fields. - - - -*`mysql.galera_status.repl.data_bytes`*:: -+ --- -Total size of data replicated. - - -type: long - --- - -*`mysql.galera_status.repl.keys`*:: -+ --- -Total number of keys replicated. - - -type: long - --- - -*`mysql.galera_status.repl.keys_bytes`*:: -+ --- -Total size of keys replicated. - - -type: long - --- - -*`mysql.galera_status.repl.other_bytes`*:: -+ --- -Total size of other bits replicated. - - -type: long - --- - -*`mysql.galera_status.repl.count`*:: -+ --- -Total number of write-sets replicated (sent to other nodes). - - -type: long - --- - -*`mysql.galera_status.repl.bytes`*:: -+ --- -Total size of write-sets replicated. - +=== query -type: long +`query` metricset fetches custom queries from the user to a MySQL instance. --- [float] === status diff --git a/metricbeat/docs/modules/mysql/performance.asciidoc b/metricbeat/docs/modules/mysql/performance.asciidoc index 8e45e7ce6cfe..e0e47239f21e 100644 --- a/metricbeat/docs/modules/mysql/performance.asciidoc +++ b/metricbeat/docs/modules/mysql/performance.asciidoc @@ -5,6 +5,8 @@ This file is generated! See scripts/mage/docs_collector.go [[metricbeat-metricset-mysql-performance]] === MySQL performance metricset +beta[] + include::../../../module/mysql/performance/_meta/docs.asciidoc[] This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 067c794e6e6a..f4b90748963d 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -192,7 +192,7 @@ This file is generated! See scripts/mage/docs_collector.go .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .3+| .3+| |<> beta[] -|<> +|<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> diff --git a/metricbeat/module/mysql/performance/_meta/fields.yml b/metricbeat/module/mysql/performance/_meta/fields.yml index a796a65099c6..e2321064fe8a 100644 --- a/metricbeat/module/mysql/performance/_meta/fields.yml +++ b/metricbeat/module/mysql/performance/_meta/fields.yml @@ -1,6 +1,49 @@ -- name: overview +- name: performance type: group description: > - `performance` contains the metrics that were obtained by the status SQL query. - release: ga + `performance` contains metrics related to the performance of a MySQL instance + release: beta fields: + - name: events_statements + description: Records statement events summarized by schema and digest + type: group + fields: + - name: 'max.timer.wait' + type: long + description: Maximum wait time of the summarized events that are timed + - name: 'last.seen' + type: date + description: Time at which the digest was most recently seen + - name: 'quantile.95' + type: long + description: TODO + - name: digest + type: text + description: Performance schema digest + - name: 'count.star' + type: long + description: Number of summarized events + - name: 'avg.timer.wait' + type: long + description: Average wait time of the summarized events that are timed + - name: table_io_waits + type: group + description: Records table I/O waits by index + fields: + - name: object + type: group + fields: + - name: schema + type: keyword + description: Schema name + - name: name + type: keyword + description: Table name + - name: index.name + type: keyword + description: > + Name of the index that was used when the table I/O wait event was recorded. PRIMARY indicates that table I/O + used the primary index. NULL means that table I/O used no index. Inserts are counted against INDEX_NAME = NULL + - name: count.fetch + type: long + description: Number of all fetch operations > 0 diff --git a/metricbeat/module/mysql/query/_meta/fields.yml b/metricbeat/module/mysql/query/_meta/fields.yml index 1ffa8aaaed37..31215e369a5f 100644 --- a/metricbeat/module/mysql/query/_meta/fields.yml +++ b/metricbeat/module/mysql/query/_meta/fields.yml @@ -1,274 +1,6 @@ -- name: galera_status +- name: query type: group release: beta description: > - `galera_status` contains the metrics that were obtained by the status SQL query on Galera. + `query` metricset fetches custom queries from the user to a MySQL instance. fields: - - name: apply - type: group - description: > - Apply status fields. - fields: - - name: oooe - type: double - description: > - How often applier started write-set applying out-of-order (parallelization efficiency). - - - name: oool - type: double - description: > - How often write-set was so slow to apply that write-set with higher seqno's were applied earlier. - Values closer to 0 refer to a greater gap between slow and fast write-sets. - - - name: window - type: double - description: > - Average distance between highest and lowest concurrently applied seqno. - - - name: cert - type: group - description: > - Certification status fields. - fields: - - name: deps_distance - type: double - description: > - Average distance between highest and lowest seqno value that can be possibly applied in parallel (potential degree of parallelization). - - - name: index_size - type: long - description: > - The number of entries in the certification index. - - - name: interval - type: double - description: > - Average number of transactions received while a transaction replicates. - - - name: cluster - type: group - description: > - Cluster status fields. - fields: - - name: conf_id - type: long - description: > - Total number of cluster membership changes happened. - - - name: size - type: long - description: > - Current number of members in the cluster. - - - name: status - type: keyword - description: > - Status of this cluster component. That is, whether the node is part of a PRIMARY or NON_PRIMARY component. - - - name: commit - type: group - description: > - Commit status fields. - fields: - - name: oooe - type: double - description: > - How often a transaction was committed out of order. - - - name: window - type: long - description: > - Average distance between highest and lowest concurrently committed seqno. - - - name: connected - type: keyword - description: > - If the value is OFF, the node has not yet connected to any of the cluster components. - This may be due to misconfiguration. Check the error log for proper diagnostics. - - - name: evs - type: group - description: > - Evs Fields. - fields: - - name: evict - type: keyword - description: > - Lists the UUID's of all nodes evicted from the cluster. - Evicted nodes cannot rejoin the cluster until you restart their mysqld processes. - - - name: state - type: keyword - description: > - Shows the internal state of the EVS Protocol. - - - name: flow_ctl - type: group - description: > - Flow Control fields. - fields: - - name: paused - type: double - description: > - The fraction of time since the last FLUSH STATUS command that replication was paused due to flow control. - In other words, how much the slave lag is slowing down the cluster. - - - name: paused_ns - type: long - description: > - The total time spent in a paused state measured in nanoseconds. - - - name: recv - type: long - description: > - Returns the number of FC_PAUSE events the node has received, including those the node has sent. - Unlike most status variables, the counter for this one does not reset every time you run the query. - - - name: sent - type: long - description: > - Returns the number of FC_PAUSE events the node has sent. - Unlike most status variables, the counter for this one does not reset every time you run the query. - - - name: last_committed - type: long - description: > - The sequence number, or seqno, of the last committed transaction. - - - name: local - type: group - description: > - Node specific Cluster status fields. - fields: - - name: bf_aborts - type: long - description: > - Total number of local transactions that were aborted by slave transactions while in execution. - - - name: cert_failures - type: long - description: > - Total number of local transactions that failed certification test. - - - name: commits - type: long - description: > - Total number of local transactions committed. - - - name: recv - type: group - description: > - Node specific recv fields. - fields: - - name: queue - type: long - description: > - Current (instantaneous) length of the recv queue. - - - name: queue_avg - type: double - description: > - Recv queue length averaged over interval since the last FLUSH STATUS command. - Values considerably larger than 0.0 mean that the node cannot apply write-sets as fast - as they are received and will generate a lot of replication throttling. - - - name: queue_max - type: long - description: > - The maximum length of the recv queue since the last FLUSH STATUS command. - - - name: queue_min - type: long - description: > - The minimum length of the recv queue since the last FLUSH STATUS command. - - - name: replays - type: long - description: > - Total number of transaction replays due to asymmetric lock granularity. - - - name: send - type: group - description: > - Node specific sent fields. - fields: - - name: queue - type: long - description: > - Current (instantaneous) length of the send queue. - - - name: queue_avg - type: double - description: > - Send queue length averaged over time since the last FLUSH STATUS command. - Values considerably larger than 0.0 indicate replication throttling or network throughput issue. - - - name: queue_max - type: long - description: > - The maximum length of the send queue since the last FLUSH STATUS command. - - - name: queue_min - type: long - description: > - The minimum length of the send queue since the last FLUSH STATUS command. - - - name: state - type: keyword - description: > - Internal Galera Cluster FSM state number. - - - name: ready - type: keyword - description: > - Whether the server is ready to accept queries. - - - name: received - type: group - description: > - Write-Set receive status fields. - fields: - - name: count - type: long - description: > - Total number of write-sets received from other nodes. - - - name: bytes - type: long - description: > - Total size of write-sets received from other nodes. - - - name: repl - type: group - description: > - Replication status fields. - fields: - - name: data_bytes - type: long - description: > - Total size of data replicated. - - - name: keys - type: long - description: > - Total number of keys replicated. - - - name: keys_bytes - type: long - description: > - Total size of keys replicated. - - - name: other_bytes - type: long - description: > - Total size of other bits replicated. - - - name: count - type: long - description: > - Total number of write-sets replicated (sent to other nodes). - - - name: bytes - type: long - description: > - Total size of write-sets replicated. From 87ea79beb7046597bda61641354702c2e517967e Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 12:47:26 +0200 Subject: [PATCH 06/21] Add Developer changelog --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 20b7c8f8a584..7e7d54c73f69 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -95,3 +95,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Add IP* fields to `fields.yml` generator script in Filebeat. {issue}17998[17998] {pull}18256[18256] - Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606] - Remove vendor folder from repository. {pull}18655[18655] +- Added SQL helper that can be used from any Metricbeat module {pull}18955[18955] From 52112663fc223fa46c5ef93df01ee9a2594df842 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 13:03:03 +0200 Subject: [PATCH 07/21] Added changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 19afa10de673..ba7ace914fa3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -533,6 +533,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for named ports in autodiscover. {pull}19398[19398] - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] +- Added to MySQL `performance` as lightweight metricset and `query` as parent lightweight metricset {pull}18955[18955] *Packetbeat* From eb5ade50dc7b8fbe1324c0fbe0014c7b3dc9c744 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 13:03:26 +0200 Subject: [PATCH 08/21] Document ToDotKeys function --- metricbeat/helper/sql/sql.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/helper/sql/sql.go b/metricbeat/helper/sql/sql.go index a5887a7b02a4..db670b25b99a 100644 --- a/metricbeat/helper/sql/sql.go +++ b/metricbeat/helper/sql/sql.go @@ -153,6 +153,8 @@ func (d *DbClient) fetchVariableMode(rows sqlRow) (common.MapStr, error) { return r, nil } +// ToDotKeys takes the root keys of a common.Mapstr and rewrites them replacing underscores with dots. Check tests +// to see an example. func ToDotKeys(ms common.MapStr) common.MapStr { dotMap := common.MapStr{} for k, v := range ms { From 786642b14a30c156c0584b7c56b5e9784e43de52 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 13:04:20 +0200 Subject: [PATCH 09/21] Added flag to activate dedotting and response format per query --- .../module/mysql/performance/manifest.yml | 9 ++-- metricbeat/module/mysql/query/query.go | 51 +++++++++++-------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/metricbeat/module/mysql/performance/manifest.yml b/metricbeat/module/mysql/performance/manifest.yml index 9694546ddbd5..4792e12fe79b 100644 --- a/metricbeat/module/mysql/performance/manifest.yml +++ b/metricbeat/module/mysql/performance/manifest.yml @@ -3,17 +3,20 @@ input: module: mysql metricset: query defaults: - sql_response_format: table namespace: performance - sql_queries: + queries: - query: > - SELECT digest_text , count_star , avg_timer_wait , max_timer_wait , last_seen , quantile_95 + SELECT digest_text, count_star, avg_timer_wait, max_timer_wait, last_seen, quantile_95 FROM performance_schema.events_statements_summary_by_digest ORDER BY avg_timer_wait DESC LIMIT 10 query_namespace: events_statements + response_format: table + dedot.enabled: false - query: > SELECT object_schema, object_name, index_name, count_fetch FROM performance_schema.table_io_waits_summary_by_index_usage WHERE count_fetch > 0 query_namespace: table_io_waits + response_format: table + dedot.enabled: false diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index 79c936223623..d7639c616aee 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -25,7 +25,6 @@ package query import ( "context" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -41,8 +40,10 @@ func init() { } type query struct { - QueryNamespace string `config:"query_namespace"` - Query string `config:"query"` + Namespace string `config:"query_namespace" validate:"nonzero,required"` + Query string `config:"query" validate:"nonzero,required"` + ResponseFormat string `config:"response_format" validate:"nonzero,required"` + DeDotEnabled bool `config:"dedot.enabled"` } // MetricSet for fetching MySQL server status. @@ -50,9 +51,8 @@ type MetricSet struct { mb.BaseMetricSet db *sql.DbClient Config struct { - Queries []query `config:"sql_queries" validate:"nonzero,required"` - Namespace string `config:"namespace" validate:"nonzero,required"` - ResponseFormat string `config:"sql_response_format"` + Queries []query `config:"queries" validate:"nonzero,required"` + Namespace string `config:"namespace" validate:"nonzero,required"` } } @@ -89,40 +89,47 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { return nil } -func (m *MetricSet) fetchQuery(ctx context.Context, q query, reporter mb.ReporterV2) error { - if m.Config.ResponseFormat == "table" { - mss, err := m.db.FetchTableMode(ctx, q.Query) +func (m *MetricSet) fetchQuery(ctx context.Context, query query, reporter mb.ReporterV2) error { + if query.ResponseFormat == "table" { + mss, err := m.db.FetchTableMode(ctx, query.Query) if err != nil { return err } for _, ms := range mss { - event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} - if q.QueryNamespace != "" { - event.ModuleFields[m.Config.Namespace] = common.MapStr{q.QueryNamespace: sql.ToDotKeys(ms)} - } else { - event.ModuleFields[m.Config.Namespace] = sql.ToDotKeys(ms) - } + event := m.transformMapStrToEvent(query,ms) reporter.Event(event) } } else { - ms, err := m.db.FetchVariableMode(ctx, q.Query) + ms, err := m.db.FetchVariableMode(ctx, query.Query) if err != nil { return err } - event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} - if q.QueryNamespace != "" { - event.ModuleFields[m.Config.Namespace] = common.MapStr{q.QueryNamespace: sql.ToDotKeys(ms)} - } else { - event.ModuleFields[m.Config.Namespace] = sql.ToDotKeys(ms) - } + event := m.transformMapStrToEvent(query,ms) reporter.Event(event) } return nil } +func (m *MetricSet) transformMapStrToEvent(query query, ms common.MapStr) mb.Event { + event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} + + data := ms + if query.DeDotEnabled { + data = sql.ToDotKeys(ms) + } + + if query.Namespace != "" { + event.ModuleFields[m.Config.Namespace] = common.MapStr{query.Namespace: data} + } else { + event.ModuleFields[m.Config.Namespace] = data + } + + return event +} + // Close closes the database connection and prevents future queries. func (m *MetricSet) Close() error { if m.db == nil { From 885bd71394d09b68ebd1c754173d0b74d5d37918 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 13:13:51 +0200 Subject: [PATCH 10/21] Modified fields.go --- metricbeat/module/mysql/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/mysql/fields.go b/metricbeat/module/mysql/fields.go index 100f37839ebf..eaa6eaf08422 100644 --- a/metricbeat/module/mysql/fields.go +++ b/metricbeat/module/mysql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetMysql returns asset data. // This is the base64 encoded gzipped contents of module/mysql. func AssetMysql() string { - return "eJzsXE9z2ziyv+dTdM1lMlWOa845vCqvx9l1VZxkLWen3onbIlsknkGAAUApmk//Cg1SoiRSomRR9ttHXhJLFPHrRqP/4Qd+gGdafoR8aX/IdwBOOEkf4ZeH5eSfn395B5CQjY0onNDqI/zXOwAA/g4smTkZsA5daSEnZ0RsIdZSUuwogZnRebj1+h2AzbRxUazVTKQfYYbS0jsAQ5LQ0kdI8R3ATJBM7Ece4wMozGmNy19uWfhbjS6L6pMWcP76N//q3xBr5VAoCy6jFUKXoYMFGQI99d9uQF094kdJZnld/dkE1gSXoiSDUVDB6ts2oP5aCTslh43PO4RgQTZG6C/QdMl3VHPjZ4vlAa3g7/zE68Yw29I1JcSikMuNb7qkOyCJv278w2pQYdTrrZvasDTxaK1p58saUqLLqWz7+gAuf/1DL0DPHCkWWQTDNt6OF0Y4+mDJBWUIlYIu3Qc9+6BNQgbeF2hQSpLiL/QjAM1mIhak4uVva/H2SSQHlmgtwQItWA1W6gU4HQSq7Gd9j3AZZCLNvA7oh9K/2mBdQTEJEBqvoO2pC9e/UJZkIZbakvFj/A6GZuG/CKkhdGQgxcKvggWRCmBQJTBD28Bhe+huIVSiF4No72ZOBlOCRFiHKqYVXNaMdYxY6oX/b6xVXBpDysnlSkusug4ZavwxGXeuxXVLxomZiIMNvmiRJVTYqBb81bXLioS5t6tgqjEqmBIU2loxbWhcKKiXIrwvtCPlBEpIKDVEoGewtVD7rE6hEvoZWfFXtx6kVulpWnjKCFSZT8l4dKScEWS9GN53xxvzyTh64XVk5jiMR6lnbY3ZGVQWY/8jC4ZiEnPvMTMhCbD5LRgqpBeGutb1ak3I0joyZ1sW4XEvWxA+bYlEMoQJaIeyodBKesjJf2IzUUCcoUrJQoZFQYqSHlYwkL3eBifXgFvBXNlsQN8H4XbKtInxmZYLbdoU3gPmJEy1N89M2JVKY50XWpFy1/Dk3YiwV7DIyPk458ErnRAI672E8z9G+PZ4/3Dz+N+gDXz5+iWq/1w/6IAl6zwX5/Pv/LS3nz1trHqfbwQ1+ExKl6xYTpteHt1Pt+OTY/talF7RXSvFpVCrCbSb+AH09zM21RAMhYWvnz5drY03QwtKO1iSWw/OiZdahuVAu6thx4h8XBIWclz6KJv4qKshFzbUbqXhgHQNtxnFz/xIMkYbkDqFmTZQGF2QgURgqrR1Ij7k8Gm+7QdOXiNwN7fw6aSlQXMRb6/VQ5PVBxEAfBbWhYrt+/f7P35lz4RS8pzZMHBdg7Y60U0Bw93htzEqP9+G/kdvemAolRMSlroEQ1zI+G+FCeV04icpJms7gzFsuepur/EyT53pRdAM5y0KZRitNta7f03gm9FOx1oesKKZ1IsodtuJz8mm9MlXJbdaOaPlid62wNLuLH44i7/1mePMVE7WK0vkBFZ4X+b1Jn0l9enz98k/YPJ08/R9wp7LezVOoOtcrPbQAWi91L0mudFgmkpvXvcKNIdNP/P2CjK9gLyMs9BzkDj3CFLvn3xt5wvmRC+OzRACqEh1JwkvS7wdZ15BcYXPaoQPXpUqghXmhLY0obJQqLSlWKukz5oxFM8HwP1IrjRV92edhH26jb7dfJ/cAc29P9+MB3VSfgVCxbJM/Gy4TFvavM1upDPN67uS4pkg13aVfMzRCJxKsiH2xLr0q5e9P2dcWhEkmkIwMmTJeWhmGbTNTqkM5rDRZdvng0h1O+eL6vMNKqpWkl/10SpFaXWELao6oCa/VCz9KMn7lqCjK58QcwJ0VTtqdjjr7KiRAh7CrOOdYvVkn/3FT5ItKPZ183kqv+kswqk2bhAvtFX7sS42S+p1a5dRhM5u8LAb94WKWyignxSXe/QOW92naIZCloZeUz4PgZKthocj21VfwU6N9UroVwb/0ojQZus9YW7avB+lw9Jhj7U3gf4oqWzLSuCQTnsChkYj4b1QvgJzqEiX9jeQpFKX1U6FhWE4e/S7Az3CeRe6A3nXEQI8rqDVmDHUlAnoOZlVG65PTtYeTaDRS9fKioQMTuUSJJqUGxao4Pfr332OosIyWoWpqioIzf11Px3Qcou9czjkULcENLTu5fmkcSGkhJQUGZ8VIUjNdXwzjXSZ0c5JodKj5irHnwObmo9fOf4UeZl3mtdxs9RHLKEuIZZQQ4q19lyFxOUlXOx2vxiXti5K0C7zsN/pHfEzpAZVKdEI1zN97K7DzuZ8fWr4H+N8vcreqPOdrKC1O9++xfDLHK9QCe9ndLhBnyMrcgttnvnTMs2K0oGw9kiNvqaLXBvBf5SLPJtYF2iR3deNscDbWFU1nyYPVZci+M8DdZYhTNp5HCf1of9s7JtUPCBhwyDsreOYCsdFqzi44VenGueqAv/kjGdCrn7yS/f+ykF6DzvRr5GprbIvbgqHXht3fHvY4nTpBqzmrPiLXgi2mVeca84fGz74ZdQHdBhdRoV+qPW2eB8380wXScP8MMcDu5DWjgfHFnkhdMH6p8IdiXEoJ7PXx9T44D1nrk43F28fcswruJpdna6meU5mLqi5X9zuTvbRLQsyM21yVDG9mGzZdD2bPNf6GmmlzWuklY600pFWOtJKR1rpSCsdaaUjrXSklY600pFWuot/pJWOtFIYaaUjrbRdcyOtFEZa6XG4R1rpq+vzDSpqpJVW10grHWmll0Q/0kpPAAwjrXSklb4FztRIKx1ppSOtdKSVtlwjrXSklY600pFWOtJKt+GOtNKRVtoObKSVjrTSs6I+klbak4S5j2x5Jpbl8XzSphxVo+9cnvGm6hu+LAZKQWqYLttG573atO5qfVKMZbVxERBBIihhwqIuHXMOw/YGNZ5UbUjLPnVi9avhBa1anugc5YWzfu1VY/v/egGar+49EEinQkm9DetEgzloCRhndJ0I+xyV9lwcowOjDTTQPi928nL7m38YL7ajF1lHHsw/1SZHN6DD3TBOHmUrrUMpay8w0HbdIRlPFKMObYcFqMG7zNcxZ7OIp/C402yCF8AgvMNNv8vD1JL3cZTM474AsDDOMcg6SFcDYFszrHqjM6VSonX482KrxjmAbEtnPuy2Wn3/XeP2x7cay0AByuVFCE+ON96HjlB+uJm40EDDibRm/ktcXmqqmDc4uOKEsmRc1O7Vzz4aFyuDTtFMljaLqt7kIOs1x58Rc68G9gu6oO0e8ECWdpEVap0hzId3BIM7Ac3HDNod6HkCQjDey8x9QpL2NOfPFXH/uPt893RXt7yrfQVm3pZFr3M7dvcs2PlR3n+Z3D0+nYzSkqQ9LOlzoZzcfb67PR1lWST7tmPOhfL7tz9ujpzxxhaY/80L11Y7LN4Uypm4GMhgjUZV6CYEfn/NTax4oVXVuv3LhXCZUGCdNsS08dRgbq+gDGRH/9R/lmRDy6Z+5DXcuzWvkXubcPv1Ifp2/+XvoA3/f/J083Q/ebq/Xe2zHcpSf9TjvKraVtrSqjrLXP2qLjQbW2LTZV1yMo3GK+N0HbORrTW8OYcdyr7aUnX998NT9O3x7tvN413jk9vPXyd3V+v5eXiKHu8md0995ydDlchzHdw7vCvXcrZqrzX0sIhdq1gdS7j9+vBw/9SYvh5+6EKRx4mcqnap0QsLGc4JpkSqAlAfLeFw3gM2/QwyR1LHzwOhr/eMVWyq1eA27HmmDRDGGcQopV9VghdMA9j732BWKk5Or2CRiTirGGpSLkHHcWksVKS4KaUilMN+FZJK+ABOHJO14VS6q/xgOA3WQ0W5MZFQF7C/em5XGistWUD2FZgSkEqFol8t6IWCh1I68eERVUrwSJiAyAvJ6g0bo0w7Z1GD8H0OYRgq0AxhwzdbbHiqx4IiQxtOAC30h/BHWO0+MQ2nz/odwcABOF99SFwzYTpZluckt7SZCY/NR8mXfKClOjnOx3y8RnpydJ5pmywyrACGOLrzevQwAb0vgylaSkArQI+oJ3bZzXG9EPgVneiZNmbhGvpJoOjnG5DAo+BZEIoFOXRsFzb9Rhvb/cISeBRCl/ZUKUwra/SCQuysgpn46VNEbUU4hNtbjujt2ZT/mzkwM+ET2h4eXUs5xcFykhboPjRtB1svT/X2KZ6fgGkdm/qU0TinQothGB9HSiIxJv9NDekY+NHbmRGPBHgiOKM7RpwLtQua6yEMWS1vPnfKeVmfdyIYcWGsoTPVF+u6n6V0Mt2W46zF4Gr/vpzNyERF23u09mV7B7TSnemt6rsyL3bf/bY9eKdf7eVVnziM6dT4YsWXLWrtaYJnjbWpoh3yW2JIJrV3vVdK//G3SkHgFXQFzog0JdPsbjjHfPVZNWlRQ6ERy4guslnp+BC3Np23Kb3YG1jXZz4xeW29LdDkUBYdavK68THL64VN3pILL5BK+ViAMWQLrcJhau0fXr0TDdjpsP5FTkdrmzXjtR26iB3K5rv6KruLTnGoEuql533V0CantjNpOJh89E4/msf6t1kgnQui5vrxSwrQ4aGUqiGVMK6rSBpKrHjnBUxBvHrRM6bKSjtF7mU2/Iz/J2azEfoOKG/HXt6muewaystNA7Y3oVs5NsPI056aMIpKptXrc14ilaGuo2lDiuSHPdvMSHQdlLGhxahGPpskubDxK4hxyAFwox0NwbS0yyYnednovKOUOnDruWBJcqGEdT79mBMfUswIkyuwZZwBhv6E1PGzhaoXigkWfG+GNjv48sdNvXGouHBgqrn6XVknv5oz3bv9cLBze8HQg1kXBLiA7XG3RKhqL7E9Ow0nvTH5wFC58vV6UTUZsbe18O+j6iVk/6dFbpwYsOU0vLPGyeXqBWv1MYEM5z56h3XKGyEhPd9/WrFLc909wiG19otBlej8l4ZCvM8STmC1cR102FuYOqy+RtzQqYhRBhuocfRLUttfnX1BR9HFKWzCGFpn1d5vtWhiXcqKaoBO2NlynRlteGRUCWSYrLqziTAU++XCtyfCPh9hO97976voh1CGXb3ldbcnwuGow5MkS4W5V55cbtXndW0e3ke4U3bzQws0mJMj03xOb00tULjoYlnmF23yIGZgpUK3ew3vFq6znrVrvYY/M1L1LxRRsu7na1Mxu/m1eSlVHBeIJWEV7jlPwjkKiVNJV/VzQvVgweqcNgqTsHvJr5xBUXV72bg35nfmk6msosDUu8j8r11t4lev/yVL4VG9/AmrKdrrCs/Vm9o6kmkh0aGR3ZU0/28AAAD//8MvSdc=" + return "eJzkXU9z2zqSv+dTdL1L8qoc7bvsYVO1U+VxnBlXxU7GVubtnvRaZIvEGAQYAJSsfPotNEiKkkiJ+kM5+0aX2BYJ/LrR/9FA3sMzLT9AtrTf5RsAJ5ykD/DL/fLpH59/eQMQk42MyJ3Q6gP85Q0AAH8HlsycDFiHrrCQkTMishBpKSlyFMPM6Cw8OnoDYFNt3CTSaiaSDzBDaekNgCFJaOkDJPgGYCZIxvYDz/EeFGa0wuU/bpn7R40u8vIvLeD85w9+6w+ItHIolAWXUo3QpehgQYZAT/23a1DrIb4XZJaj8tcmsCa4BCUZnAQW1N+2AfWfmtgpOWz8vYMIJmRthv4ETZf8RLk2frWYHtAK/sYjjhrTbFLXpBDzXC7Xvumibg8l/nPtB6tAhVlHGw+1YWni0VrT1pcVpFgXU9n29R5c/vN3vQA9c6SYZBEE23g5Xhjh6L0lF5ghVAK6cO/17L02MRl4l6NBKUmKH+hnAJrNRCRIRctfV+TtokgOTNGKggVasBqs1AtwOhBUys/qGeFSSEWSeh7Qd6Xf2iBdgTExEBrPoM2lC59/oizIQiS1JePn+A0MzcKPCIkhdGQgwdxrwYJIBTCoYpihbeCwPXi3ECrWi0G4dz0ngwlBLKxDFVENlzljHSOWeuF/jLSKCmNIObmsucSs66Chwh+RcedSrhsyTsxEFGTwJCWLKbeTivBX5y4zEuZeroKoRqhgSpBra8W0wXGhoFJFeJdrR8oJlBBTYohAz2BDUftop1AxvUys+NHNB6lVchwXximBKrIpGY+OlDOCrCfD2+5obT0ZRy+8jswch7Eo1aqtMDuDymLkX7JgKCIx9xYzFZIAm9+CoVx6YqhLr2udkIV1ZM6mFmG40xTChy0TEQ8hAtqhbDC0pB4y8n+xqcghSlElZCHFPCdFcQ8pGEheb4KRa8AtYdYyG9D3QbgZMq1jfKblQps2hveA+RSW2otnKmzN0khnuVak3AjG3owIewWLlJz3cx680jGBsN5KOP8ywtfHu/vrx/8FbeDhy8Ok+nU10B5J1lkmzmffebSfP3pa03ofbwQ2+EhKF8xYDptO9+7Hy/HRvn1FSi/vrpXiVKhVBNpFfA/6uxmLanCGwsKXT5+uVsKbogWlHSzJrSbnwEstgzrQtjZsCZH3S8JChkvvZWPvdTVkwobcrTDskEZwk1L0zEOSMdqA1AnMtIHc6JwMxAITpa0T0T6DT/NNO3C0jsDt3MKno1SD5iLa1NV9i9UHEQB8FtaFjO3bt7uPb9kyoZS8ZjZMXOWgrUZ0ncDwdHg3QuXX29C/9LoFhkI5IWGpCzDEiYz/VpiQTsd+kSKyttMZw4ap7rYap1nqVC8CZzhuUSjDbJWw3v7zCb4a7XSk5R4pmkm9mERuM/A5WpQ++azkRitntDzS2uZY2C3lh7PYWx85zkxpZD2zREZghbdlnm/SZ1KfPn97+js8ja/H357YcnmrxgF0FYtVFjoArVTdc5ILDabJ9ObnToFmt+lX3l5BqheQFVEaag4S5x5B4u2Tz+18whzrxaERQgA1Ud1BwmmBt+PIKzAu91GN8M6rZEWQwozQFiZkFgqVthRpFffRGUPRfADcj+QKU1Z/VkHYp5vJ1+tvT7dAc2/P1/1BFZRfgVCRLGK/Gi7VltYfs2vhTPPzTUnxTJBpWwcfczQCp5Js8D2RLrz2svXniEsrglhTcEaGLDkPzSwDt9koFUEc1qpsu2wQqW7jfFF+/oSMqpjktX5ShyithrCFVXvY5FXF0veCvG0JPLryATEHQFeVoWaDs4qOGiHgPsw62kpWj7bZD36RbE6Rz5vPk/lNZxOcauMGsUIbuR/zYj2lXpV2GUWo7AYLu/ZcyLiFAnqhqNjBd9ioPk1mKGRh6DXp8xAo3ih4OLJd+RVs5VivhL4W+FM9Qpus94S5LvN+lg5Jhx3S3gT6vaCiLSqBfTztCRgahYR3QvkMzKEiXdhfQZJKXFoZFSaG4ezg7xb0Cc670O2Juw4g4LGGVmHGkFPGoOdk6jJcn5is3ZtAo5aulRUxGZzKJUg0CRcsUMFvo998jKKCGtVuqswKQnF/VU8HtFxi75wO2dUtAQ2tank+aFwIKSEhRcZHRQhScx7fDCNdarRzUqjkoLXK8GVgUfP+K8MXkRVZp3gdtkp9yBLqEmQJNSRZK8uVS1xewsRu1otxaaukBO0yC/ud3hA/Q2JQFRKNcD3Dx+487GzG14eGfxrj61n2kxrfpxpau/HtmwyfZniFink/o8MM+hhZkVto88x/LZI0LxwIaw/k6GuayJUQ/KlM5NnIukCJ7K4qjIW+jTqr+fR0X1Ypgv3ck2cZwri9j+OoOvTvjX2Tsg9I2DAJW+sootxx0ir2bvhVoca5ssDfOeJ5IleNfOreXzFI7WHL+zUitTr64qJwqLVxxbeHLE6XbsBszoofdCLYZlxxrjV/bNjg01of0OHkMiz0U622xfuYmWe6SBjmpzkc2IW4djg4lsgLoQvSPxXuQIxDGZmdNqbCB+84cnW6qbx9mmNewdRs87TeLyAz0ybbaFlqtyi7Oi4b4zT6LateS0MSy01V7/saD4eWgdAaGyLrNSBdzZ+7mi9D/ZmbPynzP+4yio8UaRNbqJ+uyte2yDI04kdZPIxSypDT+lgktFUO6DbB+0zn2wxfRj72NqMFCvf2JMG4L+NRP1II6KvYbUVNXZ5HxzUL/9h2lFXD8wHeyBKpbmRxexi3hmzswaCDRSrKHa/AR95J4+K/d4jcLuAn68bzvUDlhKTRf/3nabwaf/n4pduhtS3yanxHL21fro3/tSHkpfh0jFrTxvZsZB2a00h7qC3X1rp3T47z5GxyWLWKnCaHFTSHU0kToSd+uCO7Hio957Hg7j++MDbrdZubBA/UWj39F+1ofOgqjvQpZgRZ2ZnldadGW3Q/BcnzQ++ctfOBI+YcM49bR1xrEB11THqG/O8BVzLHc5V7QmiBd6oXKYUdwnV5CELJjxkWGIpHdT9bVbooZbZ+sxUAz8K+zogMTSllI3j49vkzV583RwlvKF09eKcsGWdZNcIOaAyYeK/q4O7h4+3/TB6u72/hv3nE3RHSaEYuSs9kUVBK4PFA5xS6miz8BX7biix473VvTHHM6Q4e+Y8quCAX8JCFqLBOZ1UCveoNKmzVSb8eaOw6ztHZcnl4bHSmYyijNrYlfaOicif0XKnjdbmxelqRQIpWn3TuHvGyq69rb5giLMrOjoAIYuFthHCpLhwfygj9H9QYqezYk30K6eVbwxNa7gmjc5Tl3rnqau4q9G4eQttTaZgKJfUmrCMFZq8kYJTSKBb2eVLYczVh75ltoIl2pXlHq9tf/WCsbAcrWUehkF/18akbMCNdE06eZaPu5b1JaQUG6mfaR+ORZFS5/34C6ig2NYTx2SRiHIY7TiZYAQY5mLFud3maivI+hpIPul0AWJjnEGQdXekDYFu1oPdGZwqlROv058VWzrMH2QbPvNttlfr+bXXtw7cKy0AOymV5cE8csJ/LWO6cbiYuNNFwJK2ORkpcXmqp+GDF4IwTnJ9N2q362Wfjau6gSzSThU0n5ebtIPqa4cuEm9MHtgs636ofDiRpF9FQ6wxhNrwhGNwIaD6H2W5Az+MQgvBeZu1jkrSje+FcHvfj7efb8W1d0giNF3w0qch7HWy224flz4/y7uHp9nF8NEpLclc19Vwon24/394cj7LIOzY6zovy29eP1weueLPkJk7WrXZYq+20slu+UagK1YRwALI6vFEenCHb+uZCuFQosE4b4nN1icHMXkERToP4Uf9RkA0lm2rIEdy51cEPLm3CzZf7yde7h7+BNvzz0/h6fPc0vrupG5H2Ranfq3lelW01t7QqL3sp36oSzUbP0HRZpZy8IemZcTyPWchWHF5fww5mX22wuvr9fjz5+nj79frxtvGXm89fnm6vVutzP5483j7djvuuT4oqlue62WB/21LL4fOd0tBDIraloj63efPl/v5u3Fi+HnboQp7Hiaza6jB6YSHFOcGUSJUAqrO37M57wKaXQPNE6uh5IPRVU52KTKkNbk2eZ9oAYZRChFJ6rRKsMA1g736FWaE4OL0qN6tDC7+US9BRVBgL5amBKSUipMNeC0nFvD0SRWRt2GxwpR3c3GzoZFFmzESoC8hftbY1xwpLFpBtBSYEpBKh6K0FvVBwX0gn3j+iSggeCWMQWS6ZvaFzjM/lMamB+D6nVA3laIaQ4euN44JUzQV5ijbsCS70+/BL0PbGRlavvn4coCm+z8bwTJjOYyjn7P5tExOem+/aWfKJ3/JqnXKnFPs2MT/TZjftsAQYYu/O+uhhAnpbBlO0FINWgB5RT+yy+xDQhcDX/dbPtLYKI+hHgWrvWrkwBR4Fr4JQTMi+e01g3W60HQe8MAUehdCFPZYK03qs5oJEbGnBTLz4EFFbEW4p6U3H5OeTKe4tQ4dcndlFSk2GlnKKg8UkLdC9a9p0tp6esi+S1ydgWvmmPmk0zinXYpiW2AMpkRiR/6aCdAj8yc+zIh4J8EJwRHcIORcqFzT1IUxZqjdfzMFxWZ9Lo4y4MNZQmeqLdVXPUjqebtJx1mSw3r8vZjMyk7ztotFd0d4ernRHenV+V2T59uW4m5N32tVeVnXMbkwnxicrPm1RK0sTLGukTentkK/RIxlX1vVOKf3xryWDwDPoCpwRSUKmWd1wjg/0zcpFmzQYOmEa0U1sWji+5UabzseUXux0rKtLMTB+bb4t0GRQ5B1s8rzxPsvzhUXekgs3bCZ8btIYsrlW4bYZ7QcvL40FNjrMf5HRwdxmznhuhypiB7P5qb7M7mqn2JcJ9eLzrmxo/dBRZ9CwN/joHX407z3a7ALpVIiq149vcUKH+0KqZue5cV1J0lBkRVs3VAbyKqVnTKWUdpLcS2x4jH8TsVlzfXuYtyUvP6e4bAvK6aIBm5vQrT02w9DTHpowipKmuof4FKoMdfXWD0mSn/ZsKyPRdbSMDU1GOfPZKMmEjV6BjH0GoD4HMy3sstmTvGxU3lFKHQ4fcsISZ0IJ63z4MSe+xSEljK/AFlEKGOoTUkfPFspaKMaY87Mp2nTv7djrfGNXcWHHVB1m7Io6+e7yZOf2w97K7QVdD6ZdEOACssfVEqHKvcT26DRchYPxe4bKma/ni6qaEXtLC78/KW9p/X9NcuPEgC2m4VI/J5f1DbTVMYEU5957Bz3ljZAQnu++zqGLc901wiG59otBFevslwZDvM0STmC5cR142JuYyq2+ht/QiYhQBhmocPQLUtv/b5ELGoqunsImjKF5Vu79lkoT6UKWrQbohJ0tV5HRmkVGFUOKcV2djYWhyKsLPx4L+3yA7HjzvyujH4IZtr4Gf7smwu6ow5LES4WZZ55cbuTnVW4eziJupd08aI4GM3JkmuP05tQChZtcLMp80CYLZIauVOg2r+E/X6iinpVpHcHvKanqDUUUr+r52pSd3XyvcEJljwtEkrB09xwn4RyFxKmkq2qckD1YsDqjtcQk7F7ynXx86DdsDaNbX9+ZD6bSsgWm2kXmf229iV/+/whkw9nmfvaE2TTZaQrPVZvauLPCQqxDIbsraP6/AAAA//+E0caG" } From 741e8379ca3624fa7b63516451cdb9df8acb0e6a Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 15:00:40 +0200 Subject: [PATCH 11/21] mage fmt update --- metricbeat/module/mysql/query/query.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index d7639c616aee..8ccd4dcd184e 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -25,6 +25,7 @@ package query import ( "context" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -97,7 +98,7 @@ func (m *MetricSet) fetchQuery(ctx context.Context, query query, reporter mb.Rep } for _, ms := range mss { - event := m.transformMapStrToEvent(query,ms) + event := m.transformMapStrToEvent(query, ms) reporter.Event(event) } } else { @@ -106,7 +107,7 @@ func (m *MetricSet) fetchQuery(ctx context.Context, query query, reporter mb.Rep return err } - event := m.transformMapStrToEvent(query,ms) + event := m.transformMapStrToEvent(query, ms) reporter.Event(event) } From 8eb27ddd502402e35928e541ece36ac10a6c936b Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 16:30:19 +0200 Subject: [PATCH 12/21] Rename of dbclient --- metricbeat/helper/sql/sql.go | 7 ++----- x-pack/metricbeat/module/sql/query/query.go | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/metricbeat/helper/sql/sql.go b/metricbeat/helper/sql/sql.go index db670b25b99a..3aa5d90d3b40 100644 --- a/metricbeat/helper/sql/sql.go +++ b/metricbeat/helper/sql/sql.go @@ -22,9 +22,6 @@ import ( "database/sql" "fmt" - _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" - "strconv" "strings" "time" @@ -47,8 +44,8 @@ type sqlRow interface { Err() error } -// DB gets a client ready to query the database -func DB(driver, uri string, l *logp.Logger) (*DbClient, error) { +// NewDBClient gets a client ready to query the database +func NewDBClient(driver, uri string, l *logp.Logger) (*DbClient, error) { dbx, err := sql.Open(switchDriverName(driver), uri) if err != nil { return nil, errors.Wrap(err, "opening connection") diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index 37f8edd87f9f..9cb5fc7f1118 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -79,7 +79,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It calls m.fetchTableMode() or m.fetchVariableMode() depending on the response // format of the query. func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { - db, err := sql.DB(m.Driver, m.HostData().URI, m.Logger()) + db, err := sql.DBClient(m.Driver, m.HostData().URI, m.Logger()) if err != nil { return errors.Wrap(err, "error opening connection") } From 5ded8fdd594d1f1721d9da88647ce2a294684369 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 16:30:39 +0200 Subject: [PATCH 13/21] rename dedot.enabled to replace_underscores --- metricbeat/module/mysql/query/query.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index 8ccd4dcd184e..8f2056dc8d9b 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -41,10 +41,10 @@ func init() { } type query struct { - Namespace string `config:"query_namespace" validate:"nonzero,required"` - Query string `config:"query" validate:"nonzero,required"` - ResponseFormat string `config:"response_format" validate:"nonzero,required"` - DeDotEnabled bool `config:"dedot.enabled"` + Namespace string `config:"query_namespace"` + Query string `config:"query" validate:"nonzero,required"` + ResponseFormat string `config:"response_format" validate:"nonzero,required"` + ReplaceUnderscores bool `config:"replace_underscores"` } // MetricSet for fetching MySQL server status. @@ -74,7 +74,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { if m.db == nil { var err error - m.db, err = sql.DB("mysql", m.HostData().URI, m.Logger()) + m.db, err = sql.NewDBClient("mysql", m.HostData().URI, m.Logger()) if err != nil { return errors.Wrap(err, "mysql-status fetch failed") } @@ -118,7 +118,7 @@ func (m *MetricSet) transformMapStrToEvent(query query, ms common.MapStr) mb.Eve event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} data := ms - if query.DeDotEnabled { + if query.ReplaceUnderscores { data = sql.ToDotKeys(ms) } From d216a77e64586ff7b65faf4652945b5f6d6a039d Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 9 Jul 2020 19:46:55 +0200 Subject: [PATCH 14/21] rename fields to replace_underscores --- metricbeat/module/mysql/performance/manifest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/mysql/performance/manifest.yml b/metricbeat/module/mysql/performance/manifest.yml index 4792e12fe79b..6aee3f62402f 100644 --- a/metricbeat/module/mysql/performance/manifest.yml +++ b/metricbeat/module/mysql/performance/manifest.yml @@ -12,11 +12,11 @@ input: LIMIT 10 query_namespace: events_statements response_format: table - dedot.enabled: false + replace_underscores: false - query: > SELECT object_schema, object_name, index_name, count_fetch FROM performance_schema.table_io_waits_summary_by_index_usage WHERE count_fetch > 0 query_namespace: table_io_waits response_format: table - dedot.enabled: false + replace_underscores: false From 76564958b8257736e07461e8565491484f3b4777 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 11:46:03 +0200 Subject: [PATCH 15/21] Replace underscores in metrics flag to true --- metricbeat/module/mysql/performance/manifest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/mysql/performance/manifest.yml b/metricbeat/module/mysql/performance/manifest.yml index 6aee3f62402f..b88a2694cf7f 100644 --- a/metricbeat/module/mysql/performance/manifest.yml +++ b/metricbeat/module/mysql/performance/manifest.yml @@ -12,11 +12,11 @@ input: LIMIT 10 query_namespace: events_statements response_format: table - replace_underscores: false + replace_underscores: true - query: > SELECT object_schema, object_name, index_name, count_fetch FROM performance_schema.table_io_waits_summary_by_index_usage WHERE count_fetch > 0 query_namespace: table_io_waits response_format: table - replace_underscores: false + replace_underscores: true From 6ce7715ee17bc4ad8e663b9653e7c32fe359f2b9 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 12:53:36 +0200 Subject: [PATCH 16/21] Add query when logging an error --- metricbeat/module/mysql/query/query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index 8f2056dc8d9b..a88c22c017db 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -83,7 +83,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { for _, q := range m.Config.Queries { err := m.fetchQuery(ctx, q, reporter) if err != nil { - m.Logger().Error("error doing query", err) + m.Logger().Errorf("error doing query %s", q, err) } } From c470edb0a343bfbdc8a303d8d92471037b0d9d72 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 12:54:03 +0200 Subject: [PATCH 17/21] Add mention to SQL module and rephrase Changelog --- CHANGELOG.next.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 41e8b7460770..193ddb33ea53 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -270,6 +270,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix config example in the perfmon configuration files. {pull}19539[19539] - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601] - Fix k8s scheduler compatibility issue. {pull}19699[19699] +- Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 *Packetbeat* @@ -539,7 +540,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] - Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] -- Added to MySQL `performance` as lightweight metricset and `query` as parent lightweight metricset {pull}18955[18955] +- Added `performance` and `query` metricsets to `mysql` module. {pull}18955[18955] - The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706] *Packetbeat* From a5ab37647d852165cfdf9fc4a4b6a1e1a85fa4a8 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 14:02:29 +0200 Subject: [PATCH 18/21] Final changes and mage update --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/mysql/fields.go | 2 +- metricbeat/module/mysql/performance/_meta/fields.yml | 2 +- x-pack/metricbeat/module/sql/query/query.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5a25ba02b4d6..f8d8493a5352 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -32068,7 +32068,7 @@ type: date *`mysql.performance.events_statements.quantile.95`*:: + -- -TODO +The 95th percentile of the statement latency, in picoseconds type: long diff --git a/metricbeat/module/mysql/fields.go b/metricbeat/module/mysql/fields.go index eaa6eaf08422..8b04d180f45f 100644 --- a/metricbeat/module/mysql/fields.go +++ b/metricbeat/module/mysql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetMysql returns asset data. // This is the base64 encoded gzipped contents of module/mysql. func AssetMysql() string { - return "eJzkXU9z2zqSv+dTdL1L8qoc7bvsYVO1U+VxnBlXxU7GVubtnvRaZIvEGAQYAJSsfPotNEiKkkiJ+kM5+0aX2BYJ/LrR/9FA3sMzLT9AtrTf5RsAJ5ykD/DL/fLpH59/eQMQk42MyJ3Q6gP85Q0AAH8HlsycDFiHrrCQkTMishBpKSlyFMPM6Cw8OnoDYFNt3CTSaiaSDzBDaekNgCFJaOkDJPgGYCZIxvYDz/EeFGa0wuU/bpn7R40u8vIvLeD85w9+6w+ItHIolAWXUo3QpehgQYZAT/23a1DrIb4XZJaj8tcmsCa4BCUZnAQW1N+2AfWfmtgpOWz8vYMIJmRthv4ETZf8RLk2frWYHtAK/sYjjhrTbFLXpBDzXC7Xvumibg8l/nPtB6tAhVlHGw+1YWni0VrT1pcVpFgXU9n29R5c/vN3vQA9c6SYZBEE23g5Xhjh6L0lF5ghVAK6cO/17L02MRl4l6NBKUmKH+hnAJrNRCRIRctfV+TtokgOTNGKggVasBqs1AtwOhBUys/qGeFSSEWSeh7Qd6Xf2iBdgTExEBrPoM2lC59/oizIQiS1JePn+A0MzcKPCIkhdGQgwdxrwYJIBTCoYpihbeCwPXi3ECrWi0G4dz0ngwlBLKxDFVENlzljHSOWeuF/jLSKCmNIObmsucSs66Chwh+RcedSrhsyTsxEFGTwJCWLKbeTivBX5y4zEuZeroKoRqhgSpBra8W0wXGhoFJFeJdrR8oJlBBTYohAz2BDUftop1AxvUys+NHNB6lVchwXximBKrIpGY+OlDOCrCfD2+5obT0ZRy+8jswch7Eo1aqtMDuDymLkX7JgKCIx9xYzFZIAm9+CoVx6YqhLr2udkIV1ZM6mFmG40xTChy0TEQ8hAtqhbDC0pB4y8n+xqcghSlElZCHFPCdFcQ8pGEheb4KRa8AtYdYyG9D3QbgZMq1jfKblQps2hveA+RSW2otnKmzN0khnuVak3AjG3owIewWLlJz3cx680jGBsN5KOP8ywtfHu/vrx/8FbeDhy8Ok+nU10B5J1lkmzmffebSfP3pa03ofbwQ2+EhKF8xYDptO9+7Hy/HRvn1FSi/vrpXiVKhVBNpFfA/6uxmLanCGwsKXT5+uVsKbogWlHSzJrSbnwEstgzrQtjZsCZH3S8JChkvvZWPvdTVkwobcrTDskEZwk1L0zEOSMdqA1AnMtIHc6JwMxAITpa0T0T6DT/NNO3C0jsDt3MKno1SD5iLa1NV9i9UHEQB8FtaFjO3bt7uPb9kyoZS8ZjZMXOWgrUZ0ncDwdHg3QuXX29C/9LoFhkI5IWGpCzDEiYz/VpiQTsd+kSKyttMZw4ap7rYap1nqVC8CZzhuUSjDbJWw3v7zCb4a7XSk5R4pmkm9mERuM/A5WpQ++azkRitntDzS2uZY2C3lh7PYWx85zkxpZD2zREZghbdlnm/SZ1KfPn97+js8ja/H357YcnmrxgF0FYtVFjoArVTdc5ILDabJ9ObnToFmt+lX3l5BqheQFVEaag4S5x5B4u2Tz+18whzrxaERQgA1Ud1BwmmBt+PIKzAu91GN8M6rZEWQwozQFiZkFgqVthRpFffRGUPRfADcj+QKU1Z/VkHYp5vJ1+tvT7dAc2/P1/1BFZRfgVCRLGK/Gi7VltYfs2vhTPPzTUnxTJBpWwcfczQCp5Js8D2RLrz2svXniEsrglhTcEaGLDkPzSwDt9koFUEc1qpsu2wQqW7jfFF+/oSMqpjktX5ShyithrCFVXvY5FXF0veCvG0JPLryATEHQFeVoWaDs4qOGiHgPsw62kpWj7bZD36RbE6Rz5vPk/lNZxOcauMGsUIbuR/zYj2lXpV2GUWo7AYLu/ZcyLiFAnqhqNjBd9ioPk1mKGRh6DXp8xAo3ih4OLJd+RVs5VivhL4W+FM9Qpus94S5LvN+lg5Jhx3S3gT6vaCiLSqBfTztCRgahYR3QvkMzKEiXdhfQZJKXFoZFSaG4ezg7xb0Cc670O2Juw4g4LGGVmHGkFPGoOdk6jJcn5is3ZtAo5aulRUxGZzKJUg0CRcsUMFvo998jKKCGtVuqswKQnF/VU8HtFxi75wO2dUtAQ2tank+aFwIKSEhRcZHRQhScx7fDCNdarRzUqjkoLXK8GVgUfP+K8MXkRVZp3gdtkp9yBLqEmQJNSRZK8uVS1xewsRu1otxaaukBO0yC/ud3hA/Q2JQFRKNcD3Dx+487GzG14eGfxrj61n2kxrfpxpau/HtmwyfZniFink/o8MM+hhZkVto88x/LZI0LxwIaw/k6GuayJUQ/KlM5NnIukCJ7K4qjIW+jTqr+fR0X1Ypgv3ck2cZwri9j+OoOvTvjX2Tsg9I2DAJW+sootxx0ir2bvhVoca5ssDfOeJ5IleNfOreXzFI7WHL+zUitTr64qJwqLVxxbeHLE6XbsBszoofdCLYZlxxrjV/bNjg01of0OHkMiz0U622xfuYmWe6SBjmpzkc2IW4djg4lsgLoQvSPxXuQIxDGZmdNqbCB+84cnW6qbx9mmNewdRs87TeLyAz0ybbaFlqtyi7Oi4b4zT6LateS0MSy01V7/saD4eWgdAaGyLrNSBdzZ+7mi9D/ZmbPynzP+4yio8UaRNbqJ+uyte2yDI04kdZPIxSypDT+lgktFUO6DbB+0zn2wxfRj72NqMFCvf2JMG4L+NRP1II6KvYbUVNXZ5HxzUL/9h2lFXD8wHeyBKpbmRxexi3hmzswaCDRSrKHa/AR95J4+K/d4jcLuAn68bzvUDlhKTRf/3nabwaf/n4pduhtS3yanxHL21fro3/tSHkpfh0jFrTxvZsZB2a00h7qC3X1rp3T47z5GxyWLWKnCaHFTSHU0kToSd+uCO7Hio957Hg7j++MDbrdZubBA/UWj39F+1ofOgqjvQpZgRZ2ZnldadGW3Q/BcnzQ++ctfOBI+YcM49bR1xrEB11THqG/O8BVzLHc5V7QmiBd6oXKYUdwnV5CELJjxkWGIpHdT9bVbooZbZ+sxUAz8K+zogMTSllI3j49vkzV583RwlvKF09eKcsGWdZNcIOaAyYeK/q4O7h4+3/TB6u72/hv3nE3RHSaEYuSs9kUVBK4PFA5xS6miz8BX7biix473VvTHHM6Q4e+Y8quCAX8JCFqLBOZ1UCveoNKmzVSb8eaOw6ztHZcnl4bHSmYyijNrYlfaOicif0XKnjdbmxelqRQIpWn3TuHvGyq69rb5giLMrOjoAIYuFthHCpLhwfygj9H9QYqezYk30K6eVbwxNa7gmjc5Tl3rnqau4q9G4eQttTaZgKJfUmrCMFZq8kYJTSKBb2eVLYczVh75ltoIl2pXlHq9tf/WCsbAcrWUehkF/18akbMCNdE06eZaPu5b1JaQUG6mfaR+ORZFS5/34C6ig2NYTx2SRiHIY7TiZYAQY5mLFud3maivI+hpIPul0AWJjnEGQdXekDYFu1oPdGZwqlROv058VWzrMH2QbPvNttlfr+bXXtw7cKy0AOymV5cE8csJ/LWO6cbiYuNNFwJK2ORkpcXmqp+GDF4IwTnJ9N2q362Wfjau6gSzSThU0n5ebtIPqa4cuEm9MHtgs636ofDiRpF9FQ6wxhNrwhGNwIaD6H2W5Az+MQgvBeZu1jkrSje+FcHvfj7efb8W1d0giNF3w0qch7HWy224flz4/y7uHp9nF8NEpLclc19Vwon24/394cj7LIOzY6zovy29eP1weueLPkJk7WrXZYq+20slu+UagK1YRwALI6vFEenCHb+uZCuFQosE4b4nN1icHMXkERToP4Uf9RkA0lm2rIEdy51cEPLm3CzZf7yde7h7+BNvzz0/h6fPc0vrupG5H2Ranfq3lelW01t7QqL3sp36oSzUbP0HRZpZy8IemZcTyPWchWHF5fww5mX22wuvr9fjz5+nj79frxtvGXm89fnm6vVutzP5483j7djvuuT4oqlue62WB/21LL4fOd0tBDIraloj63efPl/v5u3Fi+HnboQp7Hiaza6jB6YSHFOcGUSJUAqrO37M57wKaXQPNE6uh5IPRVU52KTKkNbk2eZ9oAYZRChFJ6rRKsMA1g736FWaE4OL0qN6tDC7+US9BRVBgL5amBKSUipMNeC0nFvD0SRWRt2GxwpR3c3GzoZFFmzESoC8hftbY1xwpLFpBtBSYEpBKh6K0FvVBwX0gn3j+iSggeCWMQWS6ZvaFzjM/lMamB+D6nVA3laIaQ4euN44JUzQV5ijbsCS70+/BL0PbGRlavvn4coCm+z8bwTJjOYyjn7P5tExOem+/aWfKJ3/JqnXKnFPs2MT/TZjftsAQYYu/O+uhhAnpbBlO0FINWgB5RT+yy+xDQhcDX/dbPtLYKI+hHgWrvWrkwBR4Fr4JQTMi+e01g3W60HQe8MAUehdCFPZYK03qs5oJEbGnBTLz4EFFbEW4p6U3H5OeTKe4tQ4dcndlFSk2GlnKKg8UkLdC9a9p0tp6esi+S1ydgWvmmPmk0zinXYpiW2AMpkRiR/6aCdAj8yc+zIh4J8EJwRHcIORcqFzT1IUxZqjdfzMFxWZ9Lo4y4MNZQmeqLdVXPUjqebtJx1mSw3r8vZjMyk7ztotFd0d4ernRHenV+V2T59uW4m5N32tVeVnXMbkwnxicrPm1RK0sTLGukTentkK/RIxlX1vVOKf3xryWDwDPoCpwRSUKmWd1wjg/0zcpFmzQYOmEa0U1sWji+5UabzseUXux0rKtLMTB+bb4t0GRQ5B1s8rzxPsvzhUXekgs3bCZ8btIYsrlW4bYZ7QcvL40FNjrMf5HRwdxmznhuhypiB7P5qb7M7mqn2JcJ9eLzrmxo/dBRZ9CwN/joHX407z3a7ALpVIiq149vcUKH+0KqZue5cV1J0lBkRVs3VAbyKqVnTKWUdpLcS2x4jH8TsVlzfXuYtyUvP6e4bAvK6aIBm5vQrT02w9DTHpowipKmuof4FKoMdfXWD0mSn/ZsKyPRdbSMDU1GOfPZKMmEjV6BjH0GoD4HMy3sstmTvGxU3lFKHQ4fcsISZ0IJ63z4MSe+xSEljK/AFlEKGOoTUkfPFspaKMaY87Mp2nTv7djrfGNXcWHHVB1m7Io6+e7yZOf2w97K7QVdD6ZdEOACssfVEqHKvcT26DRchYPxe4bKma/ni6qaEXtLC78/KW9p/X9NcuPEgC2m4VI/J5f1DbTVMYEU5957Bz3ljZAQnu++zqGLc901wiG59otBFevslwZDvM0STmC5cR142JuYyq2+ht/QiYhQBhmocPQLUtv/b5ELGoqunsImjKF5Vu79lkoT6UKWrQbohJ0tV5HRmkVGFUOKcV2djYWhyKsLPx4L+3yA7HjzvyujH4IZtr4Gf7smwu6ow5LES4WZZ55cbuTnVW4eziJupd08aI4GM3JkmuP05tQChZtcLMp80CYLZIauVOg2r+E/X6iinpVpHcHvKanqDUUUr+r52pSd3XyvcEJljwtEkrB09xwn4RyFxKmkq2qckD1YsDqjtcQk7F7ynXx86DdsDaNbX9+ZD6bSsgWm2kXmf229iV/+/whkw9nmfvaE2TTZaQrPVZvauLPCQqxDIbsraP6/AAAA//+E0caG" + return "eJzkXc1z2ziyv+ev6JrLZKocvbnsYVP1tsrrOLuuip2s7ey8d9K0yBaJNQgwAChF89e/QoNfkkiJ+qCct6tLbIsEft3obzSQd/BCq/eQrew3+QbACSfpPfx0v3r6x6ef3gDEZCMjcie0eg9/eQMAwN+BJbMgA9ahKyxk5IyILERaSoocxTA3OguPTt4A2FQbN420movkPcxRWnoDYEgSWnoPCb4BmAuSsX3Pc7wDhRk1uPzHrXL/qNFFXv6lA5z//M5v/Q6RVg6FsuBSqhG6FB0syRDomf92DWo9xLeCzGpS/toG1gaXoCSD08CC+tsuoP5TEzsjh62/9xDBhKzNMJyg2YqfKNfGrxbTA1rB33jESWuaTeraFGKey9XaN33U7aHEf679YBWoMOtk46EuLG08Wmva+rKCFOtiJru+3oPLf/6ul6DnjhSTLIJgGy/HSyMcvbPkAjOESkAX7p2ev9MmJgNvczQoJUnxB/oZgOZzEQlS0eqXhrxdFMmRKWooWKIFq8FKvQSnA0Gl/DTPCJdCKpLU84C+Kf2zDdIVGBMDofEM2ly68PknyoIsRFJbMn6OX8HQPPyIkBhCRwYSzL0WLIlUAIMqhjnaFg47gHdLoWK9HIV71wsymBDEwjpUEdVwmTPWMWKpl/7HSKuoMIaUk6uaS8y6Hhoq/BEZdy7luiHjxFxEQQZPUrKYcjutCH917jIjYeHlKohqhApmBLm2VsxaHBcKKlWEt7l2pJxACTElhgj0HDYUdYh2ChXT96kVf/TzQWqVHMeF55RAFdmMjEdHyhlB1pPhbXe0tp6MYxBeR2aB41iUatUazM6gshj5lywYikgsvMVMhSTA9rdgKJeeGOrT61onZGEdmbOpRRjuNIXwYctUxGOIgHYoWwwtqYeM/F9sKnKIUlQJWUgxz0lRPEAKRpLXm2DkWnBLmLXMBvRDEG6GTOsYX2i11KaL4QNgPoWl9uKZCluzNNJZrhUpN4Fnb0aEvYJlSs77OQ9e6ZhAWG8lnH8Z4cvj3f314/+CNvDw+WFa/doMtEeSdZaJ89l3Hu3Hj57WtN7HG4ENPpLSBTOWw6bTvfvxcny0b29IGeTdtVKcCnWKQLeI70F/N2dRDc5QWPj88eNVI7wpWlDawYpcMzkHXmoV1IG2tWFLiLxfEhYyXHkvG3uvqyETNuRuhWGHNIGblKIXHpKM0QakTmCuDeRG52QgFpgobZ2I9hl8WmzagaN1BG4XFj4epRq0ENGmru5brCGIAOCTsC5kbF+/3n34mS0TSslrZsPEVQ7aaUTXCQxPh3cjVH69Df1Lr1tgKJQTEla6AEOcyPhvhQnpdOwXKSJre50xbJjqfqtxmqVO9TJwhuMWhTLMVgnr7T+f4IvRTkda7pGiudTLaeQ2A5+jRemjz0putHJGyyOtbY6F3VJ+OIu99ZHj3JRG1jNLZARWeFvm+SZ9JvXx09env8PT8/Xz1ye2XN6qcQBdxWKVhQ5AK1X3nORCg2kzvf25U6DZbfqVt1eQ6iVkRZSGmoPEhUeQePvkczufMMd6eWiEEEBNVX+QcFrg7TjyCozLfVQjvPMqWRGkMCO0hQmZhUKlLUVaxUN0xlC0GAH3I7nClNWfJgj7eDP9cv316RZo4e35uj+ogvIrECqSRexXw6Xa0vpjdi2caX++KileCDJt6+BjgUbgTJINvifShddetv4ccWlFEGsKzsiQJeehmVXgNhulIojDWpVtlw0i1W+cL8rPH5BRFZO81k/rEKXTEHawag+bvKpY+laQty2BR1c+IOYA6Koy1GxwmuioFQLuw6yjrWT1aJv94BfJ5hT5vPk8md9sPsWZNm4UK7SR+zEv1lPqprTLKEJlN1jYtedCxi0U0HeKih18h43q03SOQhaGXpM+D4HijYKHI9uXX8FWjvVK6GuBP9UjdMn6QJjrMu9n6ZF02CHtbaDfCiq6ohLYx9OBgKFVSHgrlM/AHCrShf0FJKnEpZVRYWIYzg7+bkGf4qIP3Z646wACHmtoFWYMOWUMekGmLsMNicm6vQm0aulaWRGTwZlcgUSTcMECFfw6+dXHKCqoUe2myqwgFPebejqg5RJ773TIrm4FaKip5fmgcSmkhIQUGR8VIUjNeXw7jHSp0c5JoZKD1irD7yOLmvdfGX4XWZH1itdhqzSELKEuQZZQY5LVWK5c4uoSJnazXowrWyUlaFdZ2O/0hvgFEoOqkGiEGxg+9udhZzO+PjT8tzG+nmU/qPF9qqF1G9+hyfBphleomPczesygj5EVuaU2L/zXIknzwoGw9kCOvqaJbITg38pEno2sC5TI7qrCWOjbqLOaj0/3ZZUi2M89eZYhjLv7OI6qQ//W2jcp+4CEDZOwtY4iyh0nrWLvhl8VapwrC/yNI54nctXIp+79FaPUHra8XytSq6MvLgqHWhtXfAfI4mzlRszmrPiDTgTbjivOteaPLRt8WusDOpxehoV+qmZbfIiZeaGLhGF+msOBXYhrh4NjibwQuiD9M+EOxDiWkdlpYyp88JYjV6fbyjukOeYVTM02T+v9AjJzbbKNlqVui7Kr47I1Tqvfsuq1NCSx3FT1vq/1cGgZCK2xIbJeA9LX/Lmr+TLUn7n5kzL/4y6j+EiRNrGF+umqfG2LLEMj/iiLh1FKGXJaH4uEtsoB/SZ4n+n8OcPvEx97m8kShfv5JMG4L+NRP1II6KvYraGmLs+j45qFf2w7yqrh+QBvYolUP7K4O4xbQ/bswaCDZSrKHa/AR95J4+K/d4jcLuAn68fzrUDlhKTJn/90Gq98oPvnP7nUS6OfWciGW7UweLFV0eqKu+REVO1l9TvCLuFocDn63vXlGq4vLeUoxa5n1JonbAcn1qE5jSUPtcXbkpf+yXGRnE1+qxaT0+S3guZwJmkq9NQPd2S3RGUfeCy4+6/PjM16m8DNhQdqu579i3Y0TPQVVYYUQYKs7MwO+1OqLbqfguT5oXfO2vvAEXM+M487R1xrLJ30THqGvPEBG5njucq9JLTAO9zLlMLO4ro8BKHkxwwLDMWTug+uKnmUMlu/2QmAZ2EfaUSGppSyCTx8/fSJq9abo4Q3lK4evFOWjLOsGmHnNAZMvDd2cPfw4fZ/pg/X97fw3zzi7shqMicXpWeyKCgl8HigcwrdUBb+Ar9uRSS8Z7s3FjnmVAiP/HsVlJALeMhCVFinsyrxbnqKClt14K8HKLuOgfS2ah4eU53p+Mqki23J0Giq3EE9V8p5XW7InlZckKLTJ527t7zsBuzbU6YIi7IjJCCCWHgbIVyqC8eHOULfCLVGKjv95JACfPnW+ISWe8noHGW5d666mrsK2duH1/ZUKGZCSb0J60iB2SsJGKU0iYV9mRb2XM3be2YbaaJd6eHR6vZXPxgr28FK1lNg5Fd9fOpGzGTXhJNn2aiXeW9SWoGR+qD20XgkGVXNYD8BdRSbGsKtlONoiXgOwx0nE6wAoxzoWLe7PE1F+RBDyQfkLgAszHMIsp5u9hGwNa3rg9GZQinROf15sZXz7EG2wTPvdjulfng7XvfwncIykoNyWR7cEwfs5zKWO6ebiwtNNB5JzZFKiatLLRUfyBidcYLzs2m3VT/7bFwFHnWJ5rKw6bTc9B1FXzP8PuWm9pHtgs636o4jSdpFNNQ6Q5iNbwhGNwKaz292G9DzOIQgvJdZ+5gk7eh6OJfH/XD76fb5ti5phIYNPtJU5IMORNvtQ/bnR3n38HT7+Hw0SktyVzX1XCifbj/d3hyPssh7NkjOi/Lrlw/XB654u+QmTtatbljNNlzZZd8qVIVqQjg4WR36KA/ckO18cylcKhRYpw3xebzEYGavoAinSPyo/yjIhpJNNeQE7lxzYIRLm3Dz+X765e7hb6AN//z0fP189/R8d1M3MO2LUr9V87wq22puaVVeElO+VSWarV6j2apKOXkj0zPjeB6zkDUcXl/DHmZfbbC6+v3+efrl8fbL9eNt6y83nz4/3V4163P/PH28fbp9Hro+KapYnutGhP3tTh2H1ndKwwCJ2JaK+rznzef7+7vn1vINsEMX8jxOZNVWh9FLCykuCGZEqgRQndlldz4ANn0PNE+ljl5GQl8146nIlNrg1uR5rg0QRilEKKXXKsEK0wL29heYF4qD06tykzu0/ku5Ah1FhbFQnjaYUSJCOuy1kFTM2yNRRNaGzQZX2sHNzYZeFmXGTIW6gPxVa1tzrLBkAdlWYEJAKhGKfraglwruC+nEu0dUCcEjYQwiyyWzN3Sc8Xk+JjUQP+R0q6EczRgyfL1xzJCquSBP0YY9waV+F34J2t7ayBp0HgBHaKYfsjE8F6b3+Mo5u4a7xITn5jt6VnxSuLySp9wpxaHNzy+02YU7LgGG2LuzPnqYgN6WwQwtxaAVoEc0ELvsPzx0IfB1n/YLra3CBIZRoLq7Vi5MgUfBqyAUE7LvPhRYtxtdxwgvTIFHIXRhj6XCdB7HuSARW1owF999iKitCLebDKZj+uPJFPekoUOuzuwipSZDSznD0WKSDujeNW06W09P2U/J6xMwNb5pSBqNC8q1GKeV9kBKJEbkv6kgHQJ/+uOsiEcCvBAc0R1CzoXKBW19CFOW6s0XenBcNuSyKSMujDVUpoZibepZSsezTTrOmgzW+/fFfE5mmnddULor2tvDlf5Ir87viizfvlR3c/JeuzrIqj6zG9OJ8cmKT1tUY2mCZY20Kb0d8vV7JOPKut4ppT/8tWQQeAZdgTMiSci0qxvO8UHAeblo0xZDp0wjuqlNC8e342jT+5jSy52OtblMA+PX5tsSTQZF3sMmzxvvszxfWOQtuXAzZ8LnLY0hm2sVbqnRfvDysllgo8P8FxkdzG3mjOd2qCL2MJufGsrsvnaKfZnQID7vyobWDyv1Bg17g4/B4Uf7vqTNLpBehah6/fj2J3S4L6Rqd54b15ckjUVWtHWzZSCvUnrGVEppL8mDxIbH+A8RmzXXt4d5W/LyY4rLtqCcLhqwuQnd2WMzDj3doQmjKGmqe4hPocpQX2/9mCT5ac+2MhJdT8vY2GSUM5+NkkzY6BXI2GcA6nMws8Ku2j3Jq1blHaXU4dAiJyxxJpSwzocfC+LbH1LC+ApsEaWAoT4hdfRioayFYow5P5uiTffeqr3ON3YVF3ZM1SHIvqiT7zxPdm4/7K3cXtD1YNoHAS4ge1wtEarcS+yOTsMVOhi/Y6ic+Xq+qKoZcbC08PvT8nbX/9ckt04M2GIWLgN0clXfXFsdE0hx4b130FPeCAnh+e5rIPo4118jHJNrPxlUsc5+ajHE2yzhBJYb14GHg4mp3Opr+A2diAhlkIEKx7Agtfv/JLmgoejrKWzDGJtn5d5vqTSRLmTZaoBO2PmqiYzWLDKqGFKM6+psLAxFXl348VjYlwNkx5v/XRn9GMyw9fX52zURdkc9liReKcw88+RqIz+vcvNwFnEr7eZBczSYkSPTHmcwp5Yo3PRiUeaDNlkgM3SlQr95Df9pQxX1NKZ1Ar+lpKo3FFHc1PO1KTu7+T7ihMoeF4gkYenuOU7CBQqJM0lX1Tghe7BgdUZriUnYveS7/PjQb9gaRre+vnMfTKVlC0y1i8z/2noTv/x/FciGs83D7AmzabrTFJ6rNrVx14WFWIdCdl/Q/H8BAAD//468214=" } diff --git a/metricbeat/module/mysql/performance/_meta/fields.yml b/metricbeat/module/mysql/performance/_meta/fields.yml index e2321064fe8a..9e2c0940b737 100644 --- a/metricbeat/module/mysql/performance/_meta/fields.yml +++ b/metricbeat/module/mysql/performance/_meta/fields.yml @@ -16,7 +16,7 @@ description: Time at which the digest was most recently seen - name: 'quantile.95' type: long - description: TODO + description: The 95th percentile of the statement latency, in picoseconds - name: digest type: text description: Performance schema digest diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index 9cb5fc7f1118..5d43d7f785fc 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -79,7 +79,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It calls m.fetchTableMode() or m.fetchVariableMode() depending on the response // format of the query. func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { - db, err := sql.DBClient(m.Driver, m.HostData().URI, m.Logger()) + db, err := sql.NewDBClient(m.Driver, m.HostData().URI, m.Logger()) if err != nil { return errors.Wrap(err, "error opening connection") } From 68f1dcfffb9fb7ce384f7507585cbefdd47d82be Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 14:08:30 +0200 Subject: [PATCH 19/21] Refactoring --- metricbeat/helper/sql/sql.go | 4 ++-- metricbeat/helper/sql/sql_test.go | 2 +- metricbeat/module/mysql/query/query.go | 15 ++++++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/metricbeat/helper/sql/sql.go b/metricbeat/helper/sql/sql.go index 3aa5d90d3b40..b417e3dc9237 100644 --- a/metricbeat/helper/sql/sql.go +++ b/metricbeat/helper/sql/sql.go @@ -150,9 +150,9 @@ func (d *DbClient) fetchVariableMode(rows sqlRow) (common.MapStr, error) { return r, nil } -// ToDotKeys takes the root keys of a common.Mapstr and rewrites them replacing underscores with dots. Check tests +// ReplaceUnderscores takes the root keys of a common.Mapstr and rewrites them replacing underscores with dots. Check tests // to see an example. -func ToDotKeys(ms common.MapStr) common.MapStr { +func ReplaceUnderscores(ms common.MapStr) common.MapStr { dotMap := common.MapStr{} for k, v := range ms { dotMap.Put(strings.Replace(k, "_", ".", -1), v) diff --git a/metricbeat/helper/sql/sql_test.go b/metricbeat/helper/sql/sql_test.go index 9fbaeb8643ec..1045c8d5b873 100644 --- a/metricbeat/helper/sql/sql_test.go +++ b/metricbeat/helper/sql/sql_test.go @@ -180,7 +180,7 @@ func checkValue(t *testing.T, res kv, ms common.MapStr) { func TestToDotKeys(t *testing.T) { ms := common.MapStr{"key_value": "value"} - ms = ToDotKeys(ms) + ms = ReplaceUnderscores(ms) if ms["key"].(common.MapStr)["value"] != "value" { t.Fail() diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index a88c22c017db..24f09218d479 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -41,10 +41,15 @@ func init() { } type query struct { - Namespace string `config:"query_namespace"` - Query string `config:"query" validate:"nonzero,required"` - ResponseFormat string `config:"response_format" validate:"nonzero,required"` - ReplaceUnderscores bool `config:"replace_underscores"` + // Namespace for the mysql event. It effectively names the metricset. For example using `performance` will name + // all events `mysql.performance.*` + Namespace string `config:"query_namespace"` + // Query to execute that must return the metrics Metricbeat wants to push to Elasticsearch + Query string `config:"query" validate:"nonzero,required"` + // ResponseFormat has 2 possible values: table and variable. Explained in the SQL helper on Metricbeat + ResponseFormat string `config:"response_format" validate:"nonzero,required"` + // If the query returns keys with underscores like `foo_bar` it will replace that with a `.` to get `foo.bar` JSON key + ReplaceUnderscores bool `config:"replace_underscores"` } // MetricSet for fetching MySQL server status. @@ -119,7 +124,7 @@ func (m *MetricSet) transformMapStrToEvent(query query, ms common.MapStr) mb.Eve data := ms if query.ReplaceUnderscores { - data = sql.ToDotKeys(ms) + data = sql.ReplaceUnderscores(ms) } if query.Namespace != "" { From 995410c08b12a1f05cb806a45f1fa3cb5ad00c25 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 15:45:37 +0200 Subject: [PATCH 20/21] Add missing docs and update data.json --- metricbeat/docs/modules/mysql.asciidoc | 4 ++ metricbeat/docs/modules/mysql/query.asciidoc | 18 ++++++ metricbeat/docs/modules_list.asciidoc | 3 +- .../module/mysql/performance/_meta/data.json | 64 +++++++++---------- .../mysql/performance/_meta/docs.asciidoc | 1 + .../module/mysql/query/_meta/docs.asciidoc | 1 + 6 files changed, 58 insertions(+), 33 deletions(-) create mode 100644 metricbeat/docs/modules/mysql/query.asciidoc create mode 100644 metricbeat/module/mysql/query/_meta/docs.asciidoc diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index 2febcb85a4cf..21762cbeb669 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -88,11 +88,15 @@ The following metricsets are available: * <> +* <> + * <> include::mysql/galera_status.asciidoc[] include::mysql/performance.asciidoc[] +include::mysql/query.asciidoc[] + include::mysql/status.asciidoc[] diff --git a/metricbeat/docs/modules/mysql/query.asciidoc b/metricbeat/docs/modules/mysql/query.asciidoc new file mode 100644 index 000000000000..fd8cdf650f9c --- /dev/null +++ b/metricbeat/docs/modules/mysql/query.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mysql-query]] +=== MySQL query metricset + +beta[] + +include::../../../module/mysql/query/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index f4b90748963d..fb9be771c213 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -191,8 +191,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> beta[] +.4+| .4+| |<> beta[] |<> beta[] +|<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> diff --git a/metricbeat/module/mysql/performance/_meta/data.json b/metricbeat/module/mysql/performance/_meta/data.json index d6526f926548..57d4a044d939 100644 --- a/metricbeat/module/mysql/performance/_meta/data.json +++ b/metricbeat/module/mysql/performance/_meta/data.json @@ -1,62 +1,62 @@ { - "@timestamp": "2020-06-18T08:38:39.025Z", + "@timestamp": "2020-07-13T13:43:28.495Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, - "host": { - "name": "mcastro" + "event": { + "duration": 1379935, + "dataset": "mysql.performance", + "module": "mysql" }, - "agent": { - "name": "mcastro", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "c1f810aa-6f0e-4550-b29e-96ee1f402d65", - "id": "de510575-3657-4099-812a-aafd2febc68e" + "metricset": { + "name": "performance", + "period": 10000 }, "service": { - "address": "root:root@tcp(172.18.0.2:3306)/", + "address": "tcp(172.17.0.2:3306)/", "type": "mysql" }, "mysql": { "performance": { "events_statements": { - "max": { - "timer": { - "wait": 8.918075e+09 - } - }, - "last": { - "seen": "2020-06-18 08:38:05.753769" - }, - "quantile": { - "95": 9.120108393e+09 - }, "digest": { - "text": "SELECT * FROM `performance_schema` . `global_status` WHERE `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ? OR `VARIABLE_NAME` = ?" + "text": "SELECT @@SESSION . `auto_increment_increment` AS `auto_increment_increment` , @@`character_set_client` AS `character_set_client` , @@`character_set_connection` AS `character_set_connection` , @@`character_set_results` AS `character_set_results` , @@`character_set_server` AS `character_set_server` , @@`collation_server` AS `collation_server` , @@`collation_connection` AS `collation_connection` , @@`init_connect` AS `init_connect` , @@`interactive_timeout` AS `interactive_timeout` , @@`license` AS `license` , @@`lower_case_table_names` AS `lower_case_table_names` , @@`max_allowed_packet` AS `max_allowed_packet` , @@`net_write_timeout` AS `net_write_timeout` , @@`performance_schema` AS `performance_schema` , @@`sql_mode` AS `sql_mode` , @@`system_time_zone` AS `system_time_zone` , @@`time_zone` AS `time_zone` , @@`transaction_isolation` AS `transaction_isolation` , @@`wait_timeout` AS `wait_timeout`" }, "count": { "star": 2 }, "avg": { "timer": { - "wait": 4.846196e+09 + "wait": 1.78294e+08 + } + }, + "max": { + "timer": { + "wait": 1.89622e+08 } + }, + "last": { + "seen": "2020-07-13 10:04:47.709230" + }, + "quantile": { + "95": 1.90546071e+08 } } } }, - "event": { - "duration": 3293342, - "dataset": "mysql.performance", - "module": "mysql" - }, - "metricset": { - "period": 10000, - "name": "performance" - }, "ecs": { "version": "1.5.0" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "f87e6edc-2f37-45f2-9644-b67b1834abfd" } } diff --git a/metricbeat/module/mysql/performance/_meta/docs.asciidoc b/metricbeat/module/mysql/performance/_meta/docs.asciidoc index e69de29bb2d1..2507cb0a06bc 100644 --- a/metricbeat/module/mysql/performance/_meta/docs.asciidoc +++ b/metricbeat/module/mysql/performance/_meta/docs.asciidoc @@ -0,0 +1 @@ +`performance` metricset fetches performance related metrics (events statements and table io waits) from MySQL diff --git a/metricbeat/module/mysql/query/_meta/docs.asciidoc b/metricbeat/module/mysql/query/_meta/docs.asciidoc new file mode 100644 index 000000000000..137a026c2e48 --- /dev/null +++ b/metricbeat/module/mysql/query/_meta/docs.asciidoc @@ -0,0 +1 @@ +`query` metricset allows for custom execution of metric related queries in MySQL From 54ae3c9f1990192c36bdc8850324e2aa44b59a91 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 13 Jul 2020 17:16:25 +0200 Subject: [PATCH 21/21] Fix docs --- metricbeat/docs/modules/sql.asciidoc | 2 +- x-pack/metricbeat/module/sql/_meta/docs.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index 7b5730a3b8b9..69726a1fe7df 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -362,7 +362,7 @@ If you want to launch two or more queries, you need to specify them with their f driver: "postgres" sql_query: "SELECT * FROM pg_catalog.pg_tables pt WHERE schemaname ='pg_catalog'" sql_response_format: table ---- +---- [float] diff --git a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc index b6896851920a..31751f264ece 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc @@ -352,4 +352,4 @@ If you want to launch two or more queries, you need to specify them with their f driver: "postgres" sql_query: "SELECT * FROM pg_catalog.pg_tables pt WHERE schemaname ='pg_catalog'" sql_response_format: table ---- +----