Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove sqlalchemy dependency from postgres connection check #340

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,29 @@ Getting Started
>>> import sqlalchemy

>>> with PostgresContainer("postgres:9.5") as postgres:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's stick with 9.5 here. Pinning the version in the docs is to ensure that things don't accidentally break because one of the dependencies changed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to https://www.postgresql.org/support/versioning/, 9.5 is not supported anymore, the oldest supported version is 11. My suggestion would be to go with 15 and then have another 5 years until the docs have to be changed... :-)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense, let's bump to 15.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

... so now 16 :-)

... engine = sqlalchemy.create_engine(postgres.get_connection_url())
... psql_url = postgres.get_connection_url()
... engine = sqlalchemy.create_engine(psql_url)
... result = engine.execute("select version()")
... version, = result.fetchone()
>>> version
'PostgreSQL 9.5...'
'PostgreSQL ......'

The snippet above will spin up a postgres database in a container. The :code:`get_connection_url()` convenience method returns a :code:`sqlalchemy` compatible url we use to connect to the database and retrieve the database version.

.. doctest::

>>> import asyncpg
>>> from testcontainers.postgres import PostgresContainer

>>> with PostgresContainer("postgres:9.5", driver=None) as postgres:
... psql_url = container.get_connection_url()
... with asyncpg.create_pool(dsn=psql_url,server_settings={"jit": "off"}) as pool:
... conn = await pool.acquire()
... ret = await conn.fetchval("SELECT 1")
... assert ret == 1

This snippet does the same, however the driver is set to None, to influence the :code:`get_connection_url()` convenience method. Note, that the :code:`sqlalchemy` package is no longer a dependency to launch the Postgres container, so your project must provide support for the specified driver.

The snippet above will spin up a postgres database in a container. The :code:`get_connection_url()` convenience method returns a :code:`sqlalchemy` compatible url we use to connect to the database and retrieve the database version.

Installation
------------
Expand Down
58 changes: 54 additions & 4 deletions core/testcontainers/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,77 @@
ADDITIONAL_TRANSIENT_ERRORS = []
try:
from sqlalchemy.exc import DBAPIError

ADDITIONAL_TRANSIENT_ERRORS.append(DBAPIError)
except ImportError:
pass


class DependencyFreeDbContainer(DockerContainer):
"""
A generic database without any package dependencies
"""

def start(self) -> "DbContainer":
self._configure()
super().start()
self._verify_status()
return self

def _verify_status(self) -> "DependencyFreeDbContainer":
"""override this method to ensure the database is running and accepting connections"""
raise NotImplementedError

def _configure(self) -> None:
raise NotImplementedError

def _create_connection_url(
self,
dialect: str,
username: str,
password: str,
host: Optional[str] = None,
port: Optional[int] = None,
dbname: Optional[str] = None,
**kwargs,
) -> str:
if raise_for_deprecated_parameter(kwargs, "db_name", "dbname"):
raise ValueError(f"Unexpected arguments: {','.join(kwargs)}")
if self._container is None:
raise ContainerStartException("container has not been started")
host = host or self.get_container_host_ip()
port = self.get_exposed_port(port)
url = f"{dialect}://{username}:{password}@{host}:{port}"
if dbname:
url = f"{url}/{dbname}"
return url


class DbContainer(DockerContainer):
"""
Generic database container.
"""

@wait_container_is_ready(*ADDITIONAL_TRANSIENT_ERRORS)
def _connect(self) -> None:
import sqlalchemy

engine = sqlalchemy.create_engine(self.get_connection_url())
engine.connect()

def get_connection_url(self) -> str:
raise NotImplementedError

def _create_connection_url(self, dialect: str, username: str, password: str,
host: Optional[str] = None, port: Optional[int] = None,
dbname: Optional[str] = None, **kwargs) -> str:
def _create_connection_url(
self,
dialect: str,
username: str,
password: str,
host: Optional[str] = None,
port: Optional[int] = None,
dbname: Optional[str] = None,
**kwargs,
) -> str:
if raise_for_deprecated_parameter(kwargs, "db_name", "dbname"):
raise ValueError(f"Unexpected arguments: {','.join(kwargs)}")
if self._container is None:
Expand All @@ -52,7 +102,7 @@ def _create_connection_url(self, dialect: str, username: str, password: str,
url = f"{url}/{dbname}"
return url

def start(self) -> 'DbContainer':
def start(self) -> "DbContainer":
self._configure()
super().start()
self._connect()
Expand Down
54 changes: 43 additions & 11 deletions postgres/testcontainers/postgres/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,16 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from time import sleep
from typing import Optional
from testcontainers.core.generic import DbContainer

from testcontainers.core.config import MAX_TRIES, SLEEP_TIME
from testcontainers.core.generic import DependencyFreeDbContainer
from testcontainers.core.utils import raise_for_deprecated_parameter
from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs


class PostgresContainer(DbContainer):
class PostgresContainer(DependencyFreeDbContainer):
"""
Postgres database container.

Expand All @@ -39,16 +43,24 @@ class PostgresContainer(DbContainer):
>>> version
'PostgreSQL 9.5...'
"""
def __init__(self, image: str = "postgres:latest", port: int = 5432,
username: Optional[str] = None, password: Optional[str] = None,
dbname: Optional[str] = None, driver: str = "psycopg2", **kwargs) -> None:

def __init__(
self,
image: str = "postgres:latest",
port: int = 5432,
username: Optional[str] = None,
password: Optional[str] = None,
dbname: Optional[str] = None,
driver: str | None = "psycopg2",
**kwargs,
) -> None:
raise_for_deprecated_parameter(kwargs, "user", "username")
super(PostgresContainer, self).__init__(image=image, **kwargs)
self.username = username or os.environ.get("POSTGRES_USER", "test")
self.password = password or os.environ.get("POSTGRES_PASSWORD", "test")
self.dbname = dbname or os.environ.get("POSTGRES_DB", "test")
self.username: str = username or os.environ.get("POSTGRES_USER", "test")
self.password: str = password or os.environ.get("POSTGRES_PASSWORD", "test")
self.dbname: str = dbname or os.environ.get("POSTGRES_DB", "test")
self.port = port
self.driver = driver
self.driver = f"+{driver}" if driver else ""

self.with_exposed_ports(self.port)

Expand All @@ -59,7 +71,27 @@ def _configure(self) -> None:

def get_connection_url(self, host=None) -> str:
return super()._create_connection_url(
dialect=f"postgresql+{self.driver}", username=self.username,
password=self.password, dbname=self.dbname, host=host,
dialect=f"postgresql{self.driver}",
username=self.username,
password=self.password,
dbname=self.dbname,
host=host,
port=self.port,
)

@wait_container_is_ready()
def _verify_status(self) -> None:
wait_for_logs(
self, ".*database system is ready to accept connections.*", MAX_TRIES, SLEEP_TIME
)

count = 0
while count < MAX_TRIES:
status, _ = self.exec(f"pg_isready -hlocalhost -p{self.port} -U{self.username}")
if status == 0:
return

sleep(SLEEP_TIME)
count += 1

raise RuntimeError("Postgres could not get into a ready state")
24 changes: 10 additions & 14 deletions postgres/tests/test_postgres.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
import sqlalchemy
from testcontainers.postgres import PostgresContainer


def test_docker_run_postgres():
postgres_container = PostgresContainer("postgres:9.5")
with postgres_container as postgres:
engine = sqlalchemy.create_engine(postgres.get_connection_url())
with engine.begin() as connection:
result = connection.execute(sqlalchemy.text("select version()"))
for row in result:
assert row[0].lower().startswith("postgresql 9.5")
# https://www.postgresql.org/support/versioning/
supported_versions = ["11", "12", "13", "14", "latest"]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please add 15 and 16 and maybe remove 11 (not supported anymore)


for version in supported_versions:
postgres_container = PostgresContainer(f"postgres:{version}")
with postgres_container as postgres:
status, msg = postgres.exec(
f"pg_isready -hlocalhost -p{postgres.port} -U{postgres.username}"
)

def test_docker_run_postgres_with_driver_pg8000():
postgres_container = PostgresContainer("postgres:9.5", driver="pg8000")
with postgres_container as postgres:
engine = sqlalchemy.create_engine(postgres.get_connection_url())
with engine.begin() as connection:
connection.execute(sqlalchemy.text("select 1=1"))
assert msg.decode("utf-8").endswith("accepting connections\n")
assert status == 0