Add Alembic support

Add support for Alembic to manage the UWS schema that vo-cutouts
obtains from Safir.

Dockerfile

@@ -12,7 +12,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.12.5-slim-bookworm as base-image
+FROM python:3.12.5-slim-bookworm AS base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .
@@ -52,6 +52,13 @@ RUN useradd --create-home appuser
 # Copy the virtualenv
 COPY --from=install-image /opt/venv /opt/venv
 
+# Copy the Alembic configuration and migrations, and set that path as the
+# working directory so that Alembic can be run with a simple entry command
+# and no extra configuration.
+COPY --from=install-image /workdir/alembic.ini /app/alembic.ini
+COPY --from=install-image /workdir/alembic /app/alembic
+WORKDIR /app
+
 # Copy the startup script
 COPY scripts/start-frontend.sh /start-frontend.sh
 

alembic.ini

@@ -0,0 +1,22 @@
+# Alembic configuration for vo-cutouts.
+#
+# This file does not retain the comments that are generated as part of the
+# default template, since they will get out of date with newer versions of
+# Alembic. See the Alembic documentation for details about each setting and
+# for settings that are not used here.
+
+[alembic]
+script_location = %(here)s/alembic
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+prepend_sys_path = .
+timezone = UTC
+version_path_separator = os
+
+[post_write_hooks]
+hooks = ruff ruff_format
+ruff.type = exec
+ruff.executable = ruff
+ruff.options = check --fix REVISION_SCRIPT_FILENAME
+ruff_format.type = exec
+ruff_format.executable = ruff
+ruff_format.options = format REVISION_SCRIPT_FILENAME

alembic/README.md

@@ -0,0 +1,14 @@
+# vo-cutouts Alembic configuration
+
+This directory contains the Alembic configuration for managing the vo-cutouts UWS database.
+It is installed into the vo-cutouts Docker image and is used to check whether the schema is up-to-date at startup of any vo-cutouts component.
+It is also used by the Helm hook that updates the vo-cutouts UWS schema if `config.updateSchema` is enabled.
+
+## Generating new migrations
+
+For detailed instructions on how to generate a new Alembic migration, see [the Safir documentation](https://safir.lsst.io/user-guide/database/schema#create-migration).
+
+One of the files in this directory is here only to support creating migrations.
+`docker-compose.yaml` is a [docker-compose](https://docs.docker.com/compose/) configuration file that starts a PostgreSQL instance suitable for generating schema migrations.
+This file is not used at runtime.
+It is used by the tox environment described in the above documentation.

alembic/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3"
+services:
+  postgresql:
+    image: "postgres:latest"
+    hostname: "postgresql"
+    container_name: "postgresql"
+    environment:
+      POSTGRES_PASSWORD: "INSECURE"
+      POSTGRES_USER: "vo-cutouts"
+      POSTGRES_DB: "vo-cutouts"
+    ports:
+      - "5432:5432"

alembic/env.py

@@ -0,0 +1,22 @@
+"""Alembic migration environment."""
+
+from safir.database import run_migrations_offline, run_migrations_online
+from safir.logging import configure_alembic_logging, configure_logging
+from safir.uws import UWSSchemaBase
+
+from alembic import context
+from vocutouts.config import config
+
+# Configure structlog.
+configure_logging(name="vo-cutouts", log_level=config.log_level)
+configure_alembic_logging()
+
+# Run the migrations.
+if context.is_offline_mode():
+    run_migrations_offline(UWSSchemaBase.metadata, config.database_url)
+else:
+    run_migrations_online(
+        UWSSchemaBase.metadata,
+        config.database_url,
+        config.database_password,
+    )

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/20240912_1902_5ab72a20365b_initial_uws_schema.py

@@ -0,0 +1,27 @@
+"""Initial UWS schema
+
+Revision ID: 5ab72a20365b
+Revises:
+Create Date: 2024-09-12 19:02:23.855138+00:00
+
+"""
+
+from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "5ab72a20365b"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

changelog.d/20240913_143738_rra_DM_46034.md

@@ -0,0 +1,3 @@
+### New features
+
+- Use Alembic to manage the schema of the UWS database. When upgrading to this version, set `config.updateSchema` to true in the Helm configuration for the first deployment. This release contains no schema changes, but needs to perform a migration to add the Alembic version information. The vo-cutouts components will now refuse to start if the database schema has changed and the database has not yet been migrated.

pyproject.toml

@@ -26,11 +26,15 @@ dependencies = [
     "python-multipart",
     "uvicorn[standard]",
     # Other dependencies.
+    "alembic[tz]",
     "astropy",
     "click",
     "pydantic>2",
     "pydantic-settings",
-    "safir[uws]>=6.2.0",
+    #"safir[uws]>=6.2.0",
+    "safir-arq @ git+https://github.com/lsst-sqre/safir@tickets/DM-46034#subdirectory=safir-arq",
+    "safir-logging @ git+https://github.com/lsst-sqre/safir@tickets/DM-46034#subdirectory=safir-logging",
+    "safir[uws] @ git+https://github.com/lsst-sqre/safir@tickets/DM-46034#subdirectory=safir",
     "structlog",
 ]
 dynamic = ["version"]

requirements/main.txt

@@ -1,5 +1,9 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile --universal --generate-hashes --output-file requirements/main.txt pyproject.toml
+alembic==1.13.2 \
+    --hash=sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef \
+    --hash=sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953
+    # via vo-cutouts (pyproject.toml)
 annotated-types==0.7.0 \
     --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
     --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
@@ -748,6 +752,10 @@ lxml==5.3.0 \
     --hash=sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945 \
     --hash=sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8
     # via pydantic-xml
+mako==1.3.5 \
+    --hash=sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a \
+    --hash=sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc
+    # via alembic
 markupsafe==2.1.5 \
     --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \
     --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \
@@ -809,7 +817,9 @@ markupsafe==2.1.5 \
     --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \
     --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \
     --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68
-    # via jinja2
+    # via
+    #   jinja2
+    #   mako
 numpy==2.1.1 \
     --hash=sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5 \
     --hash=sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0 \
@@ -1120,18 +1130,16 @@ rsa==4.9 \
     --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \
     --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21
     # via google-auth
-safir==6.3.0 \
-    --hash=sha256:2fcd64bf37dd42eacedd6378341b2487cd06dbaf1f28403301b8d80f60a4fb56 \
-    --hash=sha256:6ad7dad520d87d853628849ef95a348c55dbd0180ad3f15c1cf2f7f8fe32f915
+safir @ git+https://github.com/lsst-sqre/safir@9d480f46f588ec94e2d126d4d2285a170042a30b#subdirectory=safir
     # via vo-cutouts (pyproject.toml)
-safir-arq==6.3.0 \
-    --hash=sha256:c034e34fa79a7ebb4cfe3c1cb527e4c64c2195b63f919e2b623ae9dd72ac7e1b \
-    --hash=sha256:db110ce3fb0419d1d4a8d83b524ae9a09aa680d0e3f9323115cce836945b10e9
-    # via safir
-safir-logging==6.3.0 \
-    --hash=sha256:491dfe85de89a3f2daa29c491a22a0551f0961444490418d91ec50c040ae16eb \
-    --hash=sha256:e14754ab0bba6cfa248c3fc4cb5ca28410d97ff3965e831eab6581ed37485e79
-    # via safir
+safir-arq @ git+https://github.com/lsst-sqre/safir@9d480f46f588ec94e2d126d4d2285a170042a30b#subdirectory=safir-arq
+    # via
+    #   vo-cutouts (pyproject.toml)
+    #   safir
+safir-logging @ git+https://github.com/lsst-sqre/safir@9d480f46f588ec94e2d126d4d2285a170042a30b#subdirectory=safir-logging
+    # via
+    #   vo-cutouts (pyproject.toml)
+    #   safir
 sniffio==1.3.1 \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
@@ -1188,7 +1196,9 @@ sqlalchemy==2.0.34 \
     --hash=sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06 \
     --hash=sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec \
     --hash=sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580
-    # via safir
+    # via
+    #   alembic
+    #   safir
 starlette==0.38.5 \
     --hash=sha256:04a92830a9b6eb1442c766199d62260c3d4dc9c4f9188360626b1e0273cb7077 \
     --hash=sha256:632f420a9d13e3ee2a6f18f437b0a9f1faecb0bc42e1942aa2ea0e379a4c4206
@@ -1207,6 +1217,7 @@ typing-extensions==4.12.2 \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
     # via
+    #   alembic
     #   fastapi
     #   pydantic
     #   pydantic-core

ruff-shared.toml

@@ -27,42 +27,45 @@ docstring-code-format = true
 
 [lint]
 ignore = [
-    "ANN401",  # sometimes Any is the right type
-    "ARG001",  # unused function arguments are often legitimate
-    "ARG002",  # unused method arguments are often legitimate
-    "ARG005",  # unused lambda arguments are often legitimate
-    "BLE001",  # we want to catch and report Exception in background tasks
-    "C414",    # nested sorted is how you sort by multiple keys with reverse
-    "D102",    # sometimes we use docstring inheritence
-    "D104",    # don't see the point of documenting every package
-    "D105",    # our style doesn't require docstrings for magic methods
-    "D106",    # Pydantic uses a nested Config class that doesn't warrant docs
-    "D205",    # our documentation style allows a folded first line
-    "EM101",   # justification (duplicate string in traceback) is silly
-    "EM102",   # justification (duplicate string in traceback) is silly
-    "FBT003",  # positional booleans are normal for Pydantic field defaults
-    "FIX002",  # point of a TODO comment is that we're not ready to fix it
-    "G004",    # forbidding logging f-strings is appealing, but not our style
-    "RET505",  # disagree that omitting else always makes code more readable
-    "PLR0911", # often many returns is clearer and simpler style
-    "PLR0913", # factory pattern uses constructors with many arguments
-    "PLR2004", # too aggressive about magic values
-    "PLW0603", # yes global is discouraged but if needed, it's needed
-    "S105",    # good idea but too many false positives on non-passwords
-    "S106",    # good idea but too many false positives on non-passwords
-    "S107",    # good idea but too many false positives on non-passwords
-    "S603",    # not going to manually mark every subprocess call as reviewed
-    "S607",    # using PATH is not a security vulnerability
-    "SIM102",  # sometimes the formatting of nested if statements is clearer
-    "SIM117",  # sometimes nested with contexts are clearer
-    "TCH001",  # we decided to not maintain separate TYPE_CHECKING blocks
-    "TCH002",  # we decided to not maintain separate TYPE_CHECKING blocks
-    "TCH003",  # we decided to not maintain separate TYPE_CHECKING blocks
-    "TD003",   # we don't require issues be created for TODOs
-    "TID252",  # if we're going to use relative imports, use them always
-    "TRY003",  # good general advice but lint is way too aggressive
-    "TRY301",  # sometimes raising exceptions inside try is the best flow
-    "UP040",   # PEP 695 type aliases not yet supported by mypy
+    "ANN401",   # sometimes Any is the right type
+    "ARG001",   # unused function arguments are often legitimate
+    "ARG002",   # unused method arguments are often legitimate
+    "ARG003",   # unused class method arguments are often legitimate
+    "ARG005",   # unused lambda arguments are often legitimate
+    "ASYNC109", # many async functions use asyncio.timeout internally
+    "BLE001",   # we want to catch and report Exception in background tasks
+    "C414",     # nested sorted is how you sort by multiple keys with reverse
+    "D102",     # sometimes we use docstring inheritence
+    "D104",     # don't see the point of documenting every package
+    "D105",     # our style doesn't require docstrings for magic methods
+    "D106",     # Pydantic uses a nested Config class that doesn't warrant docs
+    "D205",     # our documentation style allows a folded first line
+    "EM101",    # justification (duplicate string in traceback) is silly
+    "EM102",    # justification (duplicate string in traceback) is silly
+    "FBT003",   # positional booleans are normal for Pydantic field defaults
+    "FIX002",   # point of a TODO comment is that we're not ready to fix it
+    "PD011",    # attempts to enforce pandas conventions for all data types
+    "G004",     # forbidding logging f-strings is appealing, but not our style
+    "RET505",   # disagree that omitting else always makes code more readable
+    "PLR0911",  # often many returns is clearer and simpler style
+    "PLR0913",  # factory pattern uses constructors with many arguments
+    "PLR2004",  # too aggressive about magic values
+    "PLW0603",  # yes global is discouraged but if needed, it's needed
+    "S105",     # good idea but too many false positives on non-passwords
+    "S106",     # good idea but too many false positives on non-passwords
+    "S107",     # good idea but too many false positives on non-passwords
+    "S603",     # not going to manually mark every subprocess call as reviewed
+    "S607",     # using PATH is not a security vulnerability
+    "SIM102",   # sometimes the formatting of nested if statements is clearer
+    "SIM117",   # sometimes nested with contexts are clearer
+    "TCH001",   # we decided to not maintain separate TYPE_CHECKING blocks
+    "TCH002",   # we decided to not maintain separate TYPE_CHECKING blocks
+    "TCH003",   # we decided to not maintain separate TYPE_CHECKING blocks
+    "TD003",    # we don't require issues be created for TODOs
+    "TID252",   # if we're going to use relative imports, use them always
+    "TRY003",   # good general advice but lint is way too aggressive
+    "TRY301",   # sometimes raising exceptions inside try is the best flow
+    "UP040",    # PEP 695 type aliases not yet supported by mypy
 
     # The following settings should be disabled when using ruff format
     # per https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
@@ -80,16 +83,24 @@ ignore = [
     "COM819",
     "ISC001",
     "ISC002",
-
-    # Temporary bug workarounds.
-    "S113",    # https://github.com/astral-sh/ruff/issues/12210
 ]
 select = ["ALL"]
 
 [lint.per-file-ignores]
+"alembic/**" = [
+    "INP001",  # Alembic files are magical
+    "D103",    # Alembic methods do not need docstrings
+    "D400",    # Alembic migrations have their own docstring format
+]
+"noxfile.py" = [
+    "T201",    # print makes sense as output from nox rules
+]
 "src/*/handlers/**" = [
     "D103",    # FastAPI handlers should not have docstrings
 ]
+"*/src/*/handlers/**" = [
+    "D103",    # FastAPI handlers should not have docstrings
+]
 "tests/**" = [
     "C901",    # tests are allowed to be complex, sometimes that's convenient
     "D101",    # tests don't need docstrings
@@ -101,6 +112,17 @@ select = ["ALL"]
     "S301",    # allow tests for whether code can be pickled
     "SLF001",  # tests are allowed to access private members
 ]
+"*/tests/**" = [
+    "C901",    # tests are allowed to be complex, sometimes that's convenient
+    "D101",    # tests don't need docstrings
+    "D103",    # tests don't need docstrings
+    "PLR0915", # tests are allowed to be long, sometimes that's convenient
+    "PT012",   # way too aggressive about limiting pytest.raises blocks
+    "S101",    # tests should use assert
+    "S106",    # tests are allowed to hard-code dummy passwords
+    "S301",    # allow tests for whether code can be pickled
+    "SLF001",  # tests are allowed to access private members
+]
 
 # These are too useful as attributes or methods to allow the conflict with the
 # built-in to rule out their use.
@@ -118,8 +140,5 @@ builtins-ignorelist = [
 fixture-parentheses = false
 mark-parentheses = false
 
-[lint.mccabe]
-max-complexity = 11
-
 [lint.pydocstyle]
 convention = "numpy"