From 7923c6f21b095671183a58bb98f9a85129f1e804 Mon Sep 17 00:00:00 2001 From: Ergin Babani Date: Mon, 8 Jan 2024 00:06:12 -0500 Subject: [PATCH 1/4] Close connection to base DB after cleanup When running multiple quick tests in parallel it's possible to get a connection failure due to too many clients connected. --- docker-compose.yml | 3 +-- testdb.go | 2 ++ testdb_test.go | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b7411e..0f1c5bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,6 @@ services: - "-c" - "log_statement=all" - "-c" - - "max_connections=1000" + - "max_connections=100" ports: - "5433:5432" - diff --git a/testdb.go b/testdb.go index f1819f8..2f8e4b6 100644 --- a/testdb.go +++ b/testdb.go @@ -154,6 +154,8 @@ func create(t testing.TB, conf Config, migrator Migrator) (*Config, *sql.DB) { } t.Cleanup(func() { + defer baseDB.Close() + // Close the testDB if err := db.Close(); err != nil { t.Fatalf("could not close test database: '%s': %s", instance.Database, err) diff --git a/testdb_test.go b/testdb_test.go index 0ea8c80..6081164 100644 --- a/testdb_test.go +++ b/testdb_test.go @@ -104,12 +104,12 @@ func TestExtensionsInstalled(t *testing.T) { } // These two tests should show that creating many different testdbs in parallel -// is quite fast. Each of the tests creates and destroys 10 databases. +// is quite fast. Each of the tests creates and destroys 50 databases. func TestParallel1(t *testing.T) { t.Parallel() ctx := context.Background() - for i := 0; i < 10; i++ { + for i := 0; i < 50; i++ { t.Run(fmt.Sprintf("subtest_%d", i), func(t *testing.T) { t.Parallel() db := New(t) From 51e9fb83ee24836d68c4d4ca730ee789600b2345 Mon Sep 17 00:00:00 2001 From: Peter Downs Date: Wed, 3 Apr 2024 19:50:49 -0400 Subject: [PATCH 2/4] revert unnecessary changes --- docker-compose.yml | 3 ++- testdb_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0f1c5bd..3b7411e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: - "-c" - "log_statement=all" - "-c" - - "max_connections=100" + - "max_connections=1000" ports: - "5433:5432" + diff --git a/testdb_test.go b/testdb_test.go index 6081164..0ea8c80 100644 --- a/testdb_test.go +++ b/testdb_test.go @@ -104,12 +104,12 @@ func TestExtensionsInstalled(t *testing.T) { } // These two tests should show that creating many different testdbs in parallel -// is quite fast. Each of the tests creates and destroys 50 databases. +// is quite fast. Each of the tests creates and destroys 10 databases. func TestParallel1(t *testing.T) { t.Parallel() ctx := context.Background() - for i := 0; i < 50; i++ { + for i := 0; i < 10; i++ { t.Run(fmt.Sprintf("subtest_%d", i), func(t *testing.T) { t.Parallel() db := New(t) From 2e42e2e97e783088a7dd12d79b972a8de162a32e Mon Sep 17 00:00:00 2001 From: Peter Downs Date: Wed, 3 Apr 2024 19:51:50 -0400 Subject: [PATCH 3/4] bug: use one db connection at any point in time --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- testdb.go | 27 ++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e39cb9a..0c1fa85 100644 --- a/README.md +++ b/README.md @@ -471,10 +471,47 @@ For ramdisk/CI in particular, you may get some ideas from [this blog post](https -c 'fsync=off' -c 'synchronous_commit=off' -c 'full_page_writes=off' --c 'max_connections=100' +-c 'max_connections=1000' -c 'client_min_messages=warning' ``` +## Why are my tests failing because they can't connect to Postgres? + +First, make sure the server is running and you can connect to it. But assuming +you're seeing some kind of failure while running a larger test suite, the most likely +problem is that you're exceeding the maximum number of connections your Postgres server is +configured to accept. This may show up as a few different types of error messages: + +``` +--- FAIL: TestParallel2/subtest_34 (0.01s) + testdb_test.go:134: failed to create instance: failed to create instance from template testdb_tpl_ed8ae75db1176559951eadb85d6be0db: failed to connect to `host=localhost user=postgres database=`: server error (FATAL: sorry, too many clients already (SQLSTATE 53300)) +``` + +or + +``` +--- FAIL: TestParallel2/subtest_25 (0.04s) + testdb_test.go:134: could not create pgtestdb user: sessionlock(testdb-user) failed to open conn: failed to connect to `host=localhost user=postgres database=`: dial error (dial tcp 127.0.0.1:5433: connect: connection refused + ``` + +or + +``` +--- FAIL: TestNew (0.07s) + testdb_test.go:42: failed to migrator.Prepare template testdb_tpl_ed8ae75db1176559951eadb85d6be0db: sessionlock(test-sql-migrator) failed to open conn: failed to connect to `host=localhost user=pgtdbuser database=testdb_tpl_ed8ae75db1176559951eadb85d6be0db`: server error (FATAL: remaining connection slots are reserved for non-replication superuser connections (SQLSTATE 53300)) +``` + +The fundamental way to fix this error is to make sure that you do not attempt +more simultaneous connections to the server than it is ready to accept. Here are +the best ways to do that, from easiest to most complicated: + +* Just don't run so many tests at once. If you're developing locally, and have a fast CI, you may not need to run the full test suite at once. +* Set a higher value for your server's [`max_connections` parameter](https://www.postgresql.org/docs/current/runtime-config-connection.html), which defaults to 100. If you look at the [`docker-compose.yaml`](./docker-compose.yml) in this repo you'll see we set it to 1000. Assuming you're using a dockerized and tmpfs-backed test server on a beefy machine, you should be able to safely crank this really high. +* Run fewer tests at the same time. You'll want to read the [go docs for the `-parallel` test flag and the `-p` build flag](https://pkg.go.dev/cmd/go) very carefully. You can also find these docs by running `go help build` and `go help tesflags`. Basically, `-p N` means go will run tests for up to `N` packages at the same time, and `-parallel Y` means it will run up to `Y` tests in parallel within each package. Both `N` and `Y` default to `GOMAXPROCS`, which is the number of logical CPUs in your machine. So by default, `go test ./...` can run up to `GOMAXPROCS * GOMAXPROCS` tests at the same time. You can try tuning `parallel Y` and `-p N` to set a hard limit of `N * Y` simultaneous tests, but that doesn't mean that you're using at most `N * Y` database connections --- depending on your test and application logic, you may actually use multiple connections at the same time. Sorry, it's confusing. +* Consider putting a connection pooler in front of your test database. This is maybe easiest to do in CI but it's possible locally with a containerized instance of pgbouncer, for example. Your tests will still contend for your database's resources, but it may turn "tests failing" into "tests getting slower". + +It's worth noting that when you call `pgtestdb.New()` or `pgtestdb.Custom()`, the library will only use one active connection at any point in time. So if you have 100 tests running in parallel, pgtestdb will at most consume 100 simultaneous connections. Your own code in your tests may run multiple queries in parallel and consume more connections at once, though. + ## How do I connect to the test databases through `pgx` / `pgxpool` instead of using the sql.DB interface? You can use `pgtestdb.Custom` to connect via `pgx`, `pgxpool`, or any other diff --git a/testdb.go b/testdb.go index 2f8e4b6..2becc49 100644 --- a/testdb.go +++ b/testdb.go @@ -117,11 +117,16 @@ func Custom(t testing.TB, conf Config, migrator Migrator) *Config { // choosing without interference from this existing connection. if err := db.Close(); err != nil { t.Fatalf("could not close test database: '%s': %s", config.Database, err) - return nil // uncreachable + return nil // unreachable } return config } +// create contains the implementation of [New] and [Custom], and is responsible +// for actually creating the instance database to be used by a testcase. +// +// create will use at most one connection to the underlying database at any +// given time. func create(t testing.TB, conf Config, migrator Migrator) (*Config, *sql.DB) { t.Helper() ctx := context.Background() @@ -153,23 +158,35 @@ func create(t testing.TB, conf Config, migrator Migrator) (*Config, *sql.DB) { return nil, nil // unreachable } - t.Cleanup(func() { - defer baseDB.Close() + if err := baseDB.Close(); err != nil { + t.Fatalf("could not close base database: '%s': %s", conf.Database, err) + } + t.Cleanup(func() { // Close the testDB if err := db.Close(); err != nil { t.Fatalf("could not close test database: '%s': %s", instance.Database, err) - return // uncreachable + return // unreachable } // If the test failed, leave the instance around for further investigation if t.Failed() { return } - // Otherwise, remove the instance from the server + + // Otherwise, reconnect to the basedb and remove the instance from the server + baseDB, err := conf.Connect() + if err != nil { + t.Fatalf("could not connect to database: '%s': %s", conf.Database, err) + return + } query := fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, instance.Database) if _, err := baseDB.ExecContext(ctx, query); err != nil { t.Fatalf("could not drop test database '%s': %s", instance.Database, err) } + if err := baseDB.Close(); err != nil { + t.Fatalf("could not close base database: '%s': %s", conf.Database, err) + return // unreachable + } }) // Even if the template previously existed, verify that it was created From 1760483a72f8f4c84c9514d7e294f6c61eb6fb7f Mon Sep 17 00:00:00 2001 From: Peter Downs Date: Thu, 4 Apr 2024 12:46:12 -0400 Subject: [PATCH 4/4] version: v0.0.14 --- README.md | 2 +- VERSION | 2 +- migrators/atlasmigrator/go.mod | 2 +- migrators/dbmatemigrator/go.mod | 2 +- migrators/golangmigrator/go.mod | 2 +- migrators/goosemigrator/go.mod | 2 +- migrators/pgmigrator/go.mod | 2 +- migrators/sqlmigrator/go.mod | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 24b0d71..21b4891 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 🧪 pgtestdb -![Latest Version](https://badgers.space/badge/latest%20version/v0.0.13/blueviolet?corner_radius=m) +![Latest Version](https://badgers.space/badge/latest%20version/v0.0.14/blueviolet?corner_radius=m) ![Golang](https://badgers.space/badge/golang/1.18+/blue?corner_radius=m) pgtestdb makes it cheap and easy to create ephemeral Postgres databases for your diff --git a/VERSION b/VERSION index 03ac640..3802e7d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.0.13 +v0.0.14 diff --git a/migrators/atlasmigrator/go.mod b/migrators/atlasmigrator/go.mod index 9cbd99a..a43798e 100644 --- a/migrators/atlasmigrator/go.mod +++ b/migrators/atlasmigrator/go.mod @@ -6,7 +6,7 @@ replace github.com/peterldowns/pgtestdb => ../../ require ( github.com/jackc/pgx/v5 v5.5.1 - github.com/peterldowns/pgtestdb v0.0.13 + github.com/peterldowns/pgtestdb v0.0.14 github.com/peterldowns/testy v0.0.1 ) diff --git a/migrators/dbmatemigrator/go.mod b/migrators/dbmatemigrator/go.mod index 651fc58..84949fc 100644 --- a/migrators/dbmatemigrator/go.mod +++ b/migrators/dbmatemigrator/go.mod @@ -7,7 +7,7 @@ replace github.com/peterldowns/pgtestdb => ../../ require ( github.com/amacneil/dbmate/v2 v2.4.0 github.com/jackc/pgx/v5 v5.5.1 - github.com/peterldowns/pgtestdb v0.0.13 + github.com/peterldowns/pgtestdb v0.0.14 github.com/peterldowns/testy v0.0.1 ) diff --git a/migrators/golangmigrator/go.mod b/migrators/golangmigrator/go.mod index 56691e5..332f68e 100644 --- a/migrators/golangmigrator/go.mod +++ b/migrators/golangmigrator/go.mod @@ -7,7 +7,7 @@ replace github.com/peterldowns/pgtestdb => ../../ require ( github.com/golang-migrate/migrate/v4 v4.16.0 github.com/jackc/pgx/v5 v5.5.1 - github.com/peterldowns/pgtestdb v0.0.13 + github.com/peterldowns/pgtestdb v0.0.14 github.com/peterldowns/testy v0.0.1 ) diff --git a/migrators/goosemigrator/go.mod b/migrators/goosemigrator/go.mod index e5f2ec2..120386a 100644 --- a/migrators/goosemigrator/go.mod +++ b/migrators/goosemigrator/go.mod @@ -6,7 +6,7 @@ replace github.com/peterldowns/pgtestdb => ../../ require ( github.com/jackc/pgx/v5 v5.5.1 - github.com/peterldowns/pgtestdb v0.0.13 + github.com/peterldowns/pgtestdb v0.0.14 github.com/peterldowns/testy v0.0.1 github.com/pressly/goose/v3 v3.11.2 ) diff --git a/migrators/pgmigrator/go.mod b/migrators/pgmigrator/go.mod index 83f97e9..2749b17 100644 --- a/migrators/pgmigrator/go.mod +++ b/migrators/pgmigrator/go.mod @@ -7,7 +7,7 @@ replace github.com/peterldowns/pgtestdb => ../../ require ( github.com/jackc/pgx/v5 v5.5.1 github.com/peterldowns/pgmigrate v0.0.5 - github.com/peterldowns/pgtestdb v0.0.13 + github.com/peterldowns/pgtestdb v0.0.14 github.com/peterldowns/testy v0.0.1 ) diff --git a/migrators/sqlmigrator/go.mod b/migrators/sqlmigrator/go.mod index 8bf6d43..be08453 100644 --- a/migrators/sqlmigrator/go.mod +++ b/migrators/sqlmigrator/go.mod @@ -6,7 +6,7 @@ replace github.com/peterldowns/pgtestdb => ../../ require ( github.com/jackc/pgx/v5 v5.5.1 - github.com/peterldowns/pgtestdb v0.0.13 + github.com/peterldowns/pgtestdb v0.0.14 github.com/peterldowns/testy v0.0.1 github.com/rubenv/sql-migrate v1.4.0 )