From f3762e3c8ed24fa9c0e7f0ef1f66dfd787932413 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Wed, 28 Jun 2023 10:41:37 -0700 Subject: [PATCH 1/5] Database wraparound collector and test Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 112 +++++++++++++++++++++++ collector/pg_database_wraparound_test.go | 105 +++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 collector/pg_database_wraparound.go create mode 100644 collector/pg_database_wraparound_test.go diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go new file mode 100644 index 000000000..19f618f83 --- /dev/null +++ b/collector/pg_database_wraparound.go @@ -0,0 +1,112 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "database/sql" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const databaseWraparoundSubsystem = "database_wraparound" + +func init() { + registerCollector(databaseWraparoundSubsystem, defaultDisabled, NewPGDatabaseWraparoundCollector) +} + +type PGDatabaseWraparoundCollector struct { + log log.Logger +} + +func NewPGDatabaseWraparoundCollector(config collectorConfig) (Collector, error) { + return &PGDatabaseWraparoundCollector{log: config.logger}, nil +} + +var ( + databaseWraparoundAgeDatfrozenxid = prometheus.NewDesc( + prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datfrozenxid_seconds"), + "Age of the oldest transaction ID that has not been frozen.", + []string{"datname"}, + prometheus.Labels{}, + ) + databaseWraparoundAgeDatminmxid = prometheus.NewDesc( + prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datminmxid"), + "Age of the oldest multi-transaction ID that has been replaced with a transaction ID.", + []string{"datname"}, + prometheus.Labels{}, + ) + + databaseWraparoundQuery = ` + SELECT + datname, + age(d.datfrozenxid) as age_datfrozenxid, + mxid_age(d.datminmxid) as age_datminmxid + FROM + pg_catalog.pg_database d + WHERE + d.datallowconn + ` +) + +func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + databaseWraparoundQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var datname sql.NullString + var ageDatfrozenxid, ageDatminmxid sql.NullFloat64 + + if err := rows.Scan(&datname, &ageDatfrozenxid, &ageDatminmxid); err != nil { + return err + } + + datnameLabel := "unknown" + if datname.Valid { + datnameLabel = datname.String + } + + ageDatfrozenxidMetric := 0.0 + if ageDatfrozenxid.Valid { + ageDatfrozenxidMetric = ageDatfrozenxid.Float64 + } + + ch <- prometheus.MustNewConstMetric( + databaseWraparoundAgeDatfrozenxid, + prometheus.GaugeValue, + ageDatfrozenxidMetric, datnameLabel, + ) + + ageDatminmxidMetric := 0.0 + if ageDatminmxid.Valid { + ageDatminmxidMetric = ageDatminmxid.Float64 + } + ch <- prometheus.MustNewConstMetric( + databaseWraparoundAgeDatminmxid, + prometheus.GaugeValue, + ageDatminmxidMetric, datnameLabel, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_database_wraparound_test.go b/collector/pg_database_wraparound_test.go new file mode 100644 index 000000000..e9ca70335 --- /dev/null +++ b/collector/pg_database_wraparound_test.go @@ -0,0 +1,105 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGDatabaseWraparoundCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "datname", + "age_datfrozenxid", + "age_datminmxid", + } + rows := sqlmock.NewRows(columns). + AddRow("newreddit", 87126426, 0) + + mock.ExpectQuery(sanitizeQuery(databaseWraparoundQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGDatabaseWraparoundCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGDatabaseWraparoundCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"datname": "newreddit"}, value: 87126426, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"datname": "newreddit"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} + +func TestPGDatabaseWraparoundCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "datname", + "age_datfrozenxid", + "age_datminmxid", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(databaseWraparoundQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGDatabaseWraparoundCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGDatabaseWraparoundCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"datname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"datname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 88465c058860149691262bc8fd25f7b0353e29ae Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 29 Jun 2023 12:45:46 -0700 Subject: [PATCH 2/5] Update collector/pg_database_wraparound.go Co-authored-by: Joe Adams Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index 19f618f83..44fb37147 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -52,8 +52,8 @@ var ( databaseWraparoundQuery = ` SELECT datname, - age(d.datfrozenxid) as age_datfrozenxid, - mxid_age(d.datminmxid) as age_datminmxid + age(d.datfrozenxid) AS age_datfrozenxid, + mxid_age(d.datminmxid) AS age_datminmxid FROM pg_catalog.pg_database d WHERE From 9d7deb5c606045541aa50ae77ea88ed773a1ec7d Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 29 Jun 2023 12:47:00 -0700 Subject: [PATCH 3/5] Add seconds to label Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index 44fb37147..2cf149285 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -43,7 +43,7 @@ var ( prometheus.Labels{}, ) databaseWraparoundAgeDatminmxid = prometheus.NewDesc( - prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datminmxid"), + prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datminmxid_seconds"), "Age of the oldest multi-transaction ID that has been replaced with a transaction ID.", []string{"datname"}, prometheus.Labels{}, From aa1075dcf39fef29db08df1376b19f2075e260af Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 29 Jun 2023 15:19:29 -0700 Subject: [PATCH 4/5] Skip if database name is null Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 13 ++++++------- collector/pg_database_wraparound_test.go | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index 2cf149285..f7a521f8c 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -52,8 +52,8 @@ var ( databaseWraparoundQuery = ` SELECT datname, - age(d.datfrozenxid) AS age_datfrozenxid, - mxid_age(d.datminmxid) AS age_datminmxid + age(d.datfrozenxid) as age_datfrozenxid, + mxid_age(d.datminmxid) as age_datminmxid FROM pg_catalog.pg_database d WHERE @@ -79,9 +79,8 @@ func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *insta return err } - datnameLabel := "unknown" - if datname.Valid { - datnameLabel = datname.String + if !datname.Valid { + continue } ageDatfrozenxidMetric := 0.0 @@ -92,7 +91,7 @@ func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *insta ch <- prometheus.MustNewConstMetric( databaseWraparoundAgeDatfrozenxid, prometheus.GaugeValue, - ageDatfrozenxidMetric, datnameLabel, + ageDatfrozenxidMetric, datname.String, ) ageDatminmxidMetric := 0.0 @@ -102,7 +101,7 @@ func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *insta ch <- prometheus.MustNewConstMetric( databaseWraparoundAgeDatminmxid, prometheus.GaugeValue, - ageDatminmxidMetric, datnameLabel, + ageDatminmxidMetric, datname.String, ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_database_wraparound_test.go b/collector/pg_database_wraparound_test.go index e9ca70335..0c19ddf54 100644 --- a/collector/pg_database_wraparound_test.go +++ b/collector/pg_database_wraparound_test.go @@ -76,7 +76,7 @@ func TestPGDatabaseWraparoundCollectorNull(t *testing.T) { "age_datminmxid", } rows := sqlmock.NewRows(columns). - AddRow(nil, nil, nil) + AddRow("foo", nil, nil) mock.ExpectQuery(sanitizeQuery(databaseWraparoundQuery)).WillReturnRows(rows) @@ -90,8 +90,8 @@ func TestPGDatabaseWraparoundCollectorNull(t *testing.T) { } }() expected := []MetricResult{ - {labels: labelMap{"datname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, - {labels: labelMap{"datname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"datname": "foo"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"datname": "foo"}, value: 0, metricType: dto.MetricType_GAUGE}, } convey.Convey("Metrics comparison", t, func() { for _, expect := range expected { From 06915237bf5b1ce10d22a9e73445f23291a09808 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 29 Jun 2023 15:54:53 -0700 Subject: [PATCH 5/5] Skip null stats and add debug statements Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 22 +++++++------ collector/pg_database_wraparound_test.go | 41 ------------------------ 2 files changed, 13 insertions(+), 50 deletions(-) diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index f7a521f8c..d46270637 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -18,6 +18,7 @@ import ( "database/sql" "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -61,7 +62,7 @@ var ( ` ) -func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { +func (c *PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { db := instance.getDB() rows, err := db.QueryContext(ctx, databaseWraparoundQuery) @@ -80,24 +81,27 @@ func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *insta } if !datname.Valid { + level.Debug(c.log).Log("msg", "Skipping database with NULL name") continue } - - ageDatfrozenxidMetric := 0.0 - if ageDatfrozenxid.Valid { - ageDatfrozenxidMetric = ageDatfrozenxid.Float64 + if !ageDatfrozenxid.Valid { + level.Debug(c.log).Log("msg", "Skipping stat emission with NULL age_datfrozenxid") + continue + } + if !ageDatminmxid.Valid { + level.Debug(c.log).Log("msg", "Skipping stat emission with NULL age_datminmxid") + continue } + ageDatfrozenxidMetric := ageDatfrozenxid.Float64 + ch <- prometheus.MustNewConstMetric( databaseWraparoundAgeDatfrozenxid, prometheus.GaugeValue, ageDatfrozenxidMetric, datname.String, ) - ageDatminmxidMetric := 0.0 - if ageDatminmxid.Valid { - ageDatminmxidMetric = ageDatminmxid.Float64 - } + ageDatminmxidMetric := ageDatminmxid.Float64 ch <- prometheus.MustNewConstMetric( databaseWraparoundAgeDatminmxid, prometheus.GaugeValue, diff --git a/collector/pg_database_wraparound_test.go b/collector/pg_database_wraparound_test.go index 0c19ddf54..d0a74c362 100644 --- a/collector/pg_database_wraparound_test.go +++ b/collector/pg_database_wraparound_test.go @@ -62,44 +62,3 @@ func TestPGDatabaseWraparoundCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } - -func TestPGDatabaseWraparoundCollectorNull(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("Error opening a stub db connection: %s", err) - } - defer db.Close() - inst := &instance{db: db} - columns := []string{ - "datname", - "age_datfrozenxid", - "age_datminmxid", - } - rows := sqlmock.NewRows(columns). - AddRow("foo", nil, nil) - - mock.ExpectQuery(sanitizeQuery(databaseWraparoundQuery)).WillReturnRows(rows) - - ch := make(chan prometheus.Metric) - go func() { - defer close(ch) - c := PGDatabaseWraparoundCollector{} - - if err := c.Update(context.Background(), inst, ch); err != nil { - t.Errorf("Error calling PGDatabaseWraparoundCollector.Update: %s", err) - } - }() - expected := []MetricResult{ - {labels: labelMap{"datname": "foo"}, value: 0, metricType: dto.MetricType_GAUGE}, - {labels: labelMap{"datname": "foo"}, value: 0, metricType: dto.MetricType_GAUGE}, - } - convey.Convey("Metrics comparison", t, func() { - for _, expect := range expected { - m := readMetric(<-ch) - convey.So(expect, convey.ShouldResemble, m) - } - }) - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled exceptions: %s", err) - } -}