Provide the "Initial database schema" alembic migration

jenkins/Pipeline.gy

@@ -28,6 +28,12 @@ pipeline {
                 sh 'jenkins/run tox -e agent-py36 -- agent'
             }
         }
+        stage('Server Alembic Migrations Check') {
+            steps {
+                echo 'Verify alembic migrations cover latest database schema'
+                sh 'jenkins/run-alembic-migrations-check'
+            }
+        }
         stage('Linting, Unit Tests, RPM builds') {
             steps {
                 // If we don't have a sequence number file left over from a

jenkins/run-alembic-migrations-check

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+
+podman run --name postgresql-alembic \
+    --detach \
+    --rm \
+    --network host \
+    --workdir /opt/app-root/src \
+    --env 'PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
+	--env 'TERM=xterm' \
+	--env 'container=oci' \
+    --env 'STI_SCRIPTS_URL=image:///usr/libexec/s2i' \
+    --env 'PGUSER=postgres' \
+    --env 'PLATFORM=el8' \
+    --env 'APP_DATA=/opt/app-root' \
+    --env 'CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql' \
+    --env 'ENABLED_COLLECTIONS' \
+    --env 'POSTGRESQL_VERSION=13' \
+    --env 'APP_ROOT=/opt/app-root' \
+    --env 'STI_SCRIPTS_PATH=/usr/libexec/s2i' \
+    --env 'HOME=/var/lib/pgsql' \
+    --env 'POSTGRESQL_USER=pbench' \
+    --env 'POSTGRESQL_PASSWORD=pbench' \
+    --env 'POSTGRESQL_DATABASE=pbench' \
+    images.paas.redhat.com/pbench/postgresql-13:latest container-entrypoint run-postgresql
+
+trap "podman stop postgresql-alembic" INT ABRT QUIT EXIT
+
+until nc -z localhost 5432; do
+    sleep 1
+done
+
+EXTRA_PODMAN_SWITCHES="${EXTRA_PODMAN_SWITCHES} --network host" jenkins/run tox -e alembic-check

lib/pbench/server/database/alembic.check

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+cd lib/pbench/server/database
+
+# First we run all our migrations to bring the blank database up to speed.
+alembic upgrade head
+
+# Then we check to see if we have any model changes not captured in existing
+# migrations.
+alembic check

lib/pbench/server/database/alembic.ini

@@ -4,11 +4,20 @@
 # path to migration scripts
 script_location = alembic
 
-# template used to generate migration files
-# file_template = %%(rev)s_%%(slug)s
-
-# timezone to use when rendering the date
-# within the migration file as well as the filename.
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
 # string value is passed to dateutil.tz.gettz()
 # leave blank for localtime
 # timezone =
@@ -26,16 +35,27 @@ script_location = alembic
 # versions/ directory
 # sourceless = false
 
-# version location specification; this defaults
+# version location specification; This defaults
 # to alembic/versions.  When using multiple version
-# directories, initial revisions must be specified with --version-path
-# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
 
 # the output encoding used when revision files
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = driver://user:pass@localhost/dbname
+sqlalchemy.url = postgresql://pbench:pbench@localhost:5432/pbench
 
 
 [post_write_hooks]
@@ -44,17 +64,17 @@ sqlalchemy.url = driver://user:pass@localhost/dbname
 # detail and examples
 
 # format using "black" - use the console_scripts runner, against the "black" entrypoint
-# hooks=black
-# black.type=console_scripts
-# black.entrypoint=black
-# black.options=-l 79
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
 
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic
 
 [handlers]
-keys = console, fileHandler
+keys = console
 
 [formatters]
 keys = generic

lib/pbench/server/database/alembic/env.py

@@ -1,62 +1,28 @@
-"""Alembic Migration Driver
-
-This file was auto generated by `alembic init alembic` but was manually altered
-to suit the needs of the Pbench server. Re-running `alembic init alembic` will
-overwrite these changes!
-
-This Python script runs whenever the alembic migration tool is invoked in
-/opt/pbench-server/lib/server/database; it contains instructions to configure
-and generate a SQLAlchemy engine, procure a connection from that engine along
-with a transaction, and then invoke the migration engine, using the connection
-as a source of database connectivity.
-
-This requires access to Pbench library modules and the Pbench server config file
-and thus must be run with
-
-    export PYTHONPATH=/opt/pbench-server/lib:${PYTHONPATH}
-    export _PBENCH_SERVER_CONFIG=/opt/pbench-server/lib/config/pbench-server.cfg
-
-Examples:
-
-   alembic upgrade head     # upgrade database to the latest
-   alembic downgrade base   # downgrade to the original tracked state
-"""
-import logging
-import sys
+from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy import create_engine
+from sqlalchemy import engine_from_config, pool
 
-from pbench.server.api import get_server_config
 from pbench.server.database.database import Database
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
+# This is the Alembic Config object, which provides access to the values within
+# the .ini file in use.
 config = context.config
 
-# Add syslog handler to send logs to journald
-log = logging.getLogger("alembic")
-handler = logging.handlers.SysLogHandler("/dev/log")
-log.addHandler(handler)
-
-# add your model's MetaData object here for 'autogenerate' support:
+# Interpret the config file for Python logging and setup the loggers.
+fileConfig(config.config_file_name)
 
+# Add your model's MetaData object here for 'autogenerate' support:
+#     from myapp import mymodel
+#     target_metadata = mymodel.Base.metadata
 target_metadata = Database.Base.metadata
 
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-try:
-    server_config = get_server_config()
-    url = Database.get_engine_uri(server_config)
-except Exception as e:
-    print(e)
-    sys.exit(1)
+# Other values from the config, defined by the needs of env.py, can be acquired:
+#     my_important_option = config.get_main_option("my_important_option")
+#     ... etc.
 
 
-def run_migrations_offline(url: str):
+def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
     This configures the context with just a URL and not an Engine, though an
@@ -65,6 +31,7 @@ def run_migrations_offline(url: str):
 
     Calls to context.execute() here emit the given string to the script output.
     """
+    url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
         target_metadata=target_metadata,
@@ -76,14 +43,17 @@ def run_migrations_offline(url: str):
         context.run_migrations()
 
 
-def run_migrations_online(url: str):
+def run_migrations_online() -> None:
     """Run migrations in 'online' mode.
 
     In this scenario we need to create an Engine and associate a connection with
     the context.
     """
-
-    connectable = create_engine(url)
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
 
     with connectable.connect() as connection:
         context.configure(connection=connection, target_metadata=target_metadata)
@@ -93,8 +63,6 @@ def run_migrations_online(url: str):
 
 
 if context.is_offline_mode():
-    print("running migration offline")
-    run_migrations_offline(url)
+    run_migrations_offline()
 else:
-    print("running migration online")
-    run_migrations_online(url)
+    run_migrations_online()

lib/pbench/server/database/alembic/script.py.mako

@@ -16,9 +16,9 @@ branch_labels = ${repr(branch_labels)}
 depends_on = ${repr(depends_on)}
 
 
-def upgrade():
+def upgrade() -> None:
     ${upgrades if upgrades else "pass"}
 
 
-def downgrade():
+def downgrade() -> None:
     ${downgrades if downgrades else "pass"}