diff --git a/jenkins/Pipeline.gy b/jenkins/Pipeline.gy index d7ff185463..a301ac6d5f 100644 --- a/jenkins/Pipeline.gy +++ b/jenkins/Pipeline.gy @@ -28,6 +28,12 @@ pipeline { sh 'jenkins/run tox -e agent-py36 -- agent' } } + stage('Server Alembic Migrations Check') { + steps { + echo 'Verify alembic migrations cover latest database schema' + sh 'jenkins/run-alembic-migrations-check' + } + } stage('Linting, Unit Tests, RPM builds') { steps { // If we don't have a sequence number file left over from a diff --git a/jenkins/run-alembic-migrations-check b/jenkins/run-alembic-migrations-check new file mode 100755 index 0000000000..52b93c84b3 --- /dev/null +++ b/jenkins/run-alembic-migrations-check @@ -0,0 +1,32 @@ +#!/bin/bash -e + +podman run --name postgresql-alembic \ + --detach \ + --rm \ + --network host \ + --workdir /opt/app-root/src \ + --env 'PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ + --env 'TERM=xterm' \ + --env 'container=oci' \ + --env 'STI_SCRIPTS_URL=image:///usr/libexec/s2i' \ + --env 'PGUSER=postgres' \ + --env 'PLATFORM=el8' \ + --env 'APP_DATA=/opt/app-root' \ + --env 'CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql' \ + --env 'ENABLED_COLLECTIONS' \ + --env 'POSTGRESQL_VERSION=13' \ + --env 'APP_ROOT=/opt/app-root' \ + --env 'STI_SCRIPTS_PATH=/usr/libexec/s2i' \ + --env 'HOME=/var/lib/pgsql' \ + --env 'POSTGRESQL_USER=pbench' \ + --env 'POSTGRESQL_PASSWORD=pbench' \ + --env 'POSTGRESQL_DATABASE=pbench' \ + images.paas.redhat.com/pbench/postgresql-13:latest container-entrypoint run-postgresql + +trap "podman stop postgresql-alembic" INT ABRT QUIT EXIT + +until nc -z localhost 5432; do + sleep 1 +done + +EXTRA_PODMAN_SWITCHES="${EXTRA_PODMAN_SWITCHES} --network host" jenkins/run tox -e alembic-check diff --git a/lib/pbench/server/database/alembic.check b/lib/pbench/server/database/alembic.check new file mode 100755 index 0000000000..f59f6455dd --- /dev/null +++ b/lib/pbench/server/database/alembic.check @@ -0,0 +1,10 @@ +#!/bin/bash -e + +cd lib/pbench/server/database + +# First we run all our migrations to bring the blank database up to speed. +alembic upgrade head + +# Then we check to see if we have any model changes not captured in existing +# migrations. +alembic check diff --git a/lib/pbench/server/database/alembic.ini b/lib/pbench/server/database/alembic.ini index 1c45b50df2..6edd2fd8f3 100644 --- a/lib/pbench/server/database/alembic.ini +++ b/lib/pbench/server/database/alembic.ini @@ -4,11 +4,20 @@ # path to migration scripts script_location = alembic -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# timezone to use when rendering the date -# within the migration file as well as the filename. +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements # string value is passed to dateutil.tz.gettz() # leave blank for localtime # timezone = @@ -26,16 +35,27 @@ script_location = alembic # versions/ directory # sourceless = false -# version location specification; this defaults +# version location specification; This defaults # to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat alembic/versions +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. # the output encoding used when revision files # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = postgresql://pbench:pbench@localhost:5432/pbench [post_write_hooks] @@ -44,17 +64,17 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # detail and examples # format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks=black -# black.type=console_scripts -# black.entrypoint=black -# black.options=-l 79 +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME # Logging configuration [loggers] keys = root,sqlalchemy,alembic [handlers] -keys = console, fileHandler +keys = console [formatters] keys = generic diff --git a/lib/pbench/server/database/alembic/alembic.ini b/lib/pbench/server/database/alembic/alembic.ini deleted file mode 100644 index 9ccf79ea8b..0000000000 --- a/lib/pbench/server/database/alembic/alembic.ini +++ /dev/null @@ -1,89 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. -prepend_sys_path = . - -# timezone to use when rendering the date -# within the migration file as well as the filename. -# string value is passed to dateutil.tz.gettz() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the -# "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat alembic/versions - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -sqlalchemy.url = driver://user:pass@localhost/dbname - - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/lib/pbench/server/database/alembic/env.py b/lib/pbench/server/database/alembic/env.py index bba332da4c..0de65205a2 100644 --- a/lib/pbench/server/database/alembic/env.py +++ b/lib/pbench/server/database/alembic/env.py @@ -1,62 +1,28 @@ -"""Alembic Migration Driver - -This file was auto generated by `alembic init alembic` but was manually altered -to suit the needs of the Pbench server. Re-running `alembic init alembic` will -overwrite these changes! - -This Python script runs whenever the alembic migration tool is invoked in -/opt/pbench-server/lib/server/database; it contains instructions to configure -and generate a SQLAlchemy engine, procure a connection from that engine along -with a transaction, and then invoke the migration engine, using the connection -as a source of database connectivity. - -This requires access to Pbench library modules and the Pbench server config file -and thus must be run with - - export PYTHONPATH=/opt/pbench-server/lib:${PYTHONPATH} - export _PBENCH_SERVER_CONFIG=/opt/pbench-server/lib/config/pbench-server.cfg - -Examples: - - alembic upgrade head # upgrade database to the latest - alembic downgrade base # downgrade to the original tracked state -""" -import logging -import sys +from logging.config import fileConfig from alembic import context -from sqlalchemy import create_engine +from sqlalchemy import engine_from_config, pool -from pbench.server.api import get_server_config from pbench.server.database.database import Database -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. +# This is the Alembic Config object, which provides access to the values within +# the .ini file in use. config = context.config -# Add syslog handler to send logs to journald -log = logging.getLogger("alembic") -handler = logging.handlers.SysLogHandler("/dev/log") -log.addHandler(handler) - -# add your model's MetaData object here for 'autogenerate' support: +# Interpret the config file for Python logging and setup the loggers. +fileConfig(config.config_file_name) +# Add your model's MetaData object here for 'autogenerate' support: +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata target_metadata = Database.Base.metadata -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - -try: - server_config = get_server_config() - url = Database.get_engine_uri(server_config) -except Exception as e: - print(e) - sys.exit(1) +# Other values from the config, defined by the needs of env.py, can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. -def run_migrations_offline(url: str): +def run_migrations_offline() -> None: """Run migrations in 'offline' mode. This configures the context with just a URL and not an Engine, though an @@ -65,6 +31,7 @@ def run_migrations_offline(url: str): Calls to context.execute() here emit the given string to the script output. """ + url = config.get_main_option("sqlalchemy.url") context.configure( url=url, target_metadata=target_metadata, @@ -76,14 +43,17 @@ def run_migrations_offline(url: str): context.run_migrations() -def run_migrations_online(url: str): +def run_migrations_online() -> None: """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context. """ - - connectable = create_engine(url) + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) with connectable.connect() as connection: context.configure(connection=connection, target_metadata=target_metadata) @@ -93,8 +63,6 @@ def run_migrations_online(url: str): if context.is_offline_mode(): - print("running migration offline") - run_migrations_offline(url) + run_migrations_offline() else: - print("running migration online") - run_migrations_online(url) + run_migrations_online() diff --git a/lib/pbench/server/database/alembic/script.py.mako b/lib/pbench/server/database/alembic/script.py.mako index 2c0156303a..55df2863d2 100644 --- a/lib/pbench/server/database/alembic/script.py.mako +++ b/lib/pbench/server/database/alembic/script.py.mako @@ -16,9 +16,9 @@ branch_labels = ${repr(branch_labels)} depends_on = ${repr(depends_on)} -def upgrade(): +def upgrade() -> None: ${upgrades if upgrades else "pass"} -def downgrade(): +def downgrade() -> None: ${downgrades if downgrades else "pass"} diff --git a/lib/pbench/server/database/alembic/versions/62eddcec4817_add_dataset_access.py b/lib/pbench/server/database/alembic/versions/62eddcec4817_add_dataset_access.py deleted file mode 100644 index 776ce30bd1..0000000000 --- a/lib/pbench/server/database/alembic/versions/62eddcec4817_add_dataset_access.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Add Dataset.access - -Revision ID: 62eddcec4817 -Revises: base -Create Date: 2021-05-27 16:22:13.714761 - -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = "62eddcec4817" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - """ - Upgrade the "base" revision with changes necessary to support publishing - datasets. - - 1. Add the "access" column, and set all existing rows to "private". This - can't be done in a single step, apparently. Instead, we add the column, - set the value in all existing rows, and then mark it non-nullable. - 2. The document map can be extremely large, so change the dataset metadata - "value" column from 2048 character String to unbounded Text. - """ - op.add_column("datasets", sa.Column("access", sa.String(255), default="private")) - op.execute("UPDATE datasets SET access = 'private'") - op.alter_column("datasets", "access", nullable=False) - op.alter_column( - "dataset_metadata", "value", type_=sa.JSON, existing_type=sa.String(2048) - ) - - -def downgrade(): - """ - Reverse the upgrade if we're downgrading to "base" revision - """ - op.drop_column("datasets", "access") - op.alter_column( - "dataset_metadata", "value", type_=sa.String(2048), existing_type=sa.JSON - ) diff --git a/lib/pbench/server/database/alembic/versions/fa12f45a2a5a_initial_database_schema.py b/lib/pbench/server/database/alembic/versions/fa12f45a2a5a_initial_database_schema.py new file mode 100644 index 0000000000..b0b66926d2 --- /dev/null +++ b/lib/pbench/server/database/alembic/versions/fa12f45a2a5a_initial_database_schema.py @@ -0,0 +1,169 @@ +"""Initial database schema + +Revision ID: fa12f45a2a5a +Revises: +Create Date: 2023-01-16 18:33:29.144835 + +Since we are adding Alembic migrations after we have already been using our +database in various contexts, this "Initial database schema" migration describes +how to bring an empty database up to the state of the database as of commit +6a764f154. That commit was the latest working version of the Pbench Server +deployed in Red Hat's staging environment. +""" +from alembic import op +import sqlalchemy as sa + +from pbench.server.database.models import TZDateTime + +# revision identifiers, used by Alembic. +revision = "fa12f45a2a5a" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "audit", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("root_id", sa.Integer(), nullable=True), + sa.Column("name", sa.String(length=128), nullable=True), + sa.Column( + "operation", + sa.Enum("CREATE", "READ", "UPDATE", "DELETE", name="operationcode"), + nullable=False, + ), + sa.Column( + "object_type", + sa.Enum("DATASET", "CONFIG", "NONE", "TEMPLATE", "TOKEN", name="audittype"), + nullable=True, + ), + sa.Column("object_id", sa.String(length=128), nullable=True), + sa.Column("object_name", sa.String(length=256), nullable=True), + sa.Column("user_id", sa.String(length=128), nullable=True), + sa.Column("user_name", sa.String(length=256), nullable=True), + sa.Column( + "status", + sa.Enum("BEGIN", "SUCCESS", "FAILURE", "WARNING", name="auditstatus"), + nullable=False, + ), + sa.Column( + "reason", + sa.Enum("PERMISSION", "INTERNAL", "CONSISTENCY", name="auditreason"), + nullable=True, + ), + sa.Column("attributes", sa.JSON(), nullable=True), + sa.Column("timestamp", TZDateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "datasets", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("name", sa.String(length=1024), nullable=False), + sa.Column("owner_id", sa.String(length=255), nullable=False), + sa.Column("access", sa.String(length=255), nullable=False), + sa.Column("resource_id", sa.String(length=255), nullable=False), + sa.Column("uploaded", TZDateTime(), nullable=False), + sa.Column("created", TZDateTime(), nullable=True), + sa.Column( + "state", + sa.Enum( + "UPLOADING", + "UPLOADED", + "INDEXING", + "INDEXED", + "DELETING", + "DELETED", + name="states", + ), + nullable=False, + ), + sa.Column("transition", TZDateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("resource_id"), + ) + op.create_table( + "serverconfig", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("key", sa.String(length=255), nullable=False), + sa.Column("value", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_serverconfig_key"), "serverconfig", ["key"], unique=True) + op.create_table( + "templates", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("idxname", sa.String(length=255), nullable=False), + sa.Column("template_name", sa.String(length=255), nullable=False), + sa.Column("file", sa.String(length=255), nullable=False), + sa.Column("mtime", sa.DateTime(), nullable=False), + sa.Column("template_pattern", sa.String(length=255), nullable=False), + sa.Column("index_template", sa.String(length=225), nullable=False), + sa.Column("settings", sa.JSON(), nullable=False), + sa.Column("mappings", sa.JSON(), nullable=False), + sa.Column("version", sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("idxname"), + sa.UniqueConstraint("name"), + sa.UniqueConstraint("template_name"), + ) + op.create_table( + "users", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("username", sa.String(length=255), nullable=False), + sa.Column("first_name", sa.String(length=255), nullable=False), + sa.Column("last_name", sa.String(length=255), nullable=False), + sa.Column("password", sa.LargeBinary(length=128), nullable=False), + sa.Column("registered_on", sa.DateTime(), nullable=False), + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("role", sa.Enum("ADMIN", name="roles"), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + sa.UniqueConstraint("username"), + ) + op.create_table( + "active_tokens", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("token", sa.String(length=500), nullable=False), + sa.Column("created", sa.DateTime(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_active_tokens_token"), "active_tokens", ["token"], unique=True + ) + op.create_table( + "dataset_metadata", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("key", sa.String(length=255), nullable=False), + sa.Column("value", sa.JSON(), nullable=True), + sa.Column("dataset_ref", sa.Integer(), nullable=False), + sa.Column("user_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["dataset_ref"], + ["datasets.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_dataset_metadata_key"), "dataset_metadata", ["key"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_dataset_metadata_key"), table_name="dataset_metadata") + op.drop_table("dataset_metadata") + op.drop_index(op.f("ix_active_tokens_token"), table_name="active_tokens") + op.drop_table("active_tokens") + op.drop_table("users") + op.drop_table("templates") + op.drop_index(op.f("ix_serverconfig_key"), table_name="serverconfig") + op.drop_table("serverconfig") + op.drop_table("datasets") + op.drop_table("audit") + # ### end Alembic commands ### diff --git a/server/requirements.txt b/server/requirements.txt index dedd1f875d..2638a2eee3 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,4 +1,4 @@ -alembic>=1.6.4 +alembic>=1.9.0 aniso8601>=9.0.1 Bcrypt-Flask boto3 diff --git a/tox.ini b/tox.ini index f29b1d6977..3dd35b91cb 100644 --- a/tox.ini +++ b/tox.ini @@ -40,3 +40,10 @@ basepython = python3.6 deps = -r{toxinidir}/agent/requirements.txt -r{toxinidir}/agent/test-requirements.txt + +[testenv:alembic-check] +description = Verify alembic migrations cover latest database schema +deps = + -r{toxinidir}/server/requirements.txt +commands = + bash -c "{toxinidir}/lib/pbench/server/database/alembic.check"