Merge pull request #860 from CDCgov/devex-753/liquibase

753: Database Migrations with Liquibase

.github/workflows/terraform-deploy_reusable.yml

@@ -76,10 +76,22 @@ jobs:
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
-      - name: Run Db migration
+      - name: Extract database hostname and password into GitHub Env
         run: |
-          export PGPASSWORD=$(az account get-access-token --resource-type oss-rdbms --query "[accessToken]" -o tsv)
-          psql "host=$(terraform output -raw database_hostname) port=5432 dbname=postgres user=cdcti-github sslmode=require" -c "DO \$\$ BEGIN CREATE TYPE message_status AS ENUM ('PENDING', 'DELIVERED', 'FAILED'); EXCEPTION WHEN duplicate_object THEN null; END \$\$; CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz, time_delivered timestamptz, delivery_status message_status, failure_reason varchar(1000)); GRANT ALL ON metadata TO azure_pg_admin; ALTER TABLE metadata OWNER TO azure_pg_admin; ALTER TYPE message_status OWNER TO azure_pg_admin"
+          DATABASE_HOSTNAME=$(terraform output -raw database_hostname)
+          DATABASE_PASSWORD=$(az account get-access-token --resource-type oss-rdbms --query "[accessToken]" -o tsv)
+          echo "::add-mask::$DATABASE_HOSTNAME"
+          echo "::add-mask::$DATABASE_PASSWORD"
+          echo "DATABASE_HOSTNAME=$DATABASE_HOSTNAME" >> "$GITHUB_ENV"
+          echo "DATABASE_PASSWORD=$DATABASE_PASSWORD" >> "$GITHUB_ENV"
+
+      - name: Run Db migration
+        uses: liquibase-github-actions/update@v4.25.1
+        with:
+          changelogFile: ./etor/databaseMigrations/root.yml
+          url: "jdbc:postgresql://${{ env.DATABASE_HOSTNAME }}:5432/postgres"
+          username: cdcti-github
+          password: ${{ env.DATABASE_PASSWORD }}
 
       - id: export-terraform-output
         name: Export Terraform Output

.secrets.baseline

@@ -138,23 +138,23 @@
         "filename": "README.md",
         "hashed_secret": "367e3228ed16bf72b36de9a4134ee8b825cafacb",
         "is_verified": false,
-        "line_number": 278,
+        "line_number": 279,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "README.md",
         "hashed_secret": "40bd7d88eae0468b048e62e1056ac390970b2b51",
         "is_verified": false,
-        "line_number": 283,
+        "line_number": 284,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "README.md",
         "hashed_secret": "0d46754ae17642645ca041edaac9a1c1569f5edc",
         "is_verified": false,
-        "line_number": 288,
+        "line_number": 289,
         "is_secret": false
       }
     ],
@@ -164,7 +164,7 @@
         "filename": "docker-compose.postgres.yml",
         "hashed_secret": "0f7866a6cab6f2793ea9f68e92935e4d726d58b5",
         "is_verified": false,
-        "line_number": 11,
+        "line_number": 12,
         "is_secret": false
       }
     ],
@@ -269,5 +269,5 @@
       }
     ]
   },
-  "generated_at": "2024-02-08T18:23:50Z"
+  "generated_at": "2024-02-09T15:36:39Z"
 }

README.md

@@ -31,6 +31,7 @@ The additional requirements needed to contribute towards development are...
 - [Locust.io](https://docs.locust.io/en/stable/installation.html)
 - [Python](https://docs.python-guide.org/starting/installation/)
 - [Terraform](https://www.terraform.io)
+- [Liquibase](https://www.liquibase.com/download)
 
 ### Generating .env File
 

docker-compose.postgres.yml

@@ -1,4 +1,5 @@
-# Find the SQL schema in `/.github/workflows/terraform-deploy_reusable.yml`.
+# Run the following to migrate...
+# liquibase update --changelog-file ./etor/databaseMigrations/root.yml --url jdbc:postgresql://localhost:5433/intermediary --username intermediary --password 'changeIT!' --label-filter '!azure'
 
 version: "3.7"
 

docs/database.md

@@ -1,13 +1,13 @@
 # Database
 
 ## Requirements
-Choose a Postgres client, [pgAdmin](https://www.pgadmin.org/) is the most full featured, intelliJ also has a built in plugin.
+Choose a Postgres client, [pgAdmin](https://www.pgadmin.org/) is the most full featured, IntelliJ also has a built-in plugin.
 
 ## Connecting to Local Database
 1. Run the docker file ./docker-compose.postgres.yml in the root of the project
 2. Run the `generate_env.sh` script:
 
-    ```bash
+    ```shell
     ./generate_env.sh
     ```
 
@@ -22,7 +22,7 @@ Choose a Postgres client, [pgAdmin](https://www.pgadmin.org/) is the most full f
 3. Inside of the Azure database page select the Networking option from the left hand nav
 4. Click the link that says `Add current client IP address` and then save the page
     1. NOTE: You should only add your local IP address on a temporary basis, you should remove it after the verification is complete
-5. On the left hand navigation select Authentication and select the `Add Microsoft Entra Admins` link to add your user to the list
+5. On the left hand navigation select Authentication and select the `Add Microsoft Entra Admins` link to add your user to the list. Select Okay and then save the underlying page
     1. NOTE: This permission should only be added temporarily and removed after you are finished with verification
 6. Enter new connection settings from Azure into your db client of choice
    1. Password will come from step 8 of these instructions and can be left blank
@@ -31,6 +31,44 @@ Choose a Postgres client, [pgAdmin](https://www.pgadmin.org/) is the most full f
 
 ## Modifying the database schema
 To modify the schema there are a few locations in the code we need to update.
-1. [Line 77 of this file contains the table creation statements](.github/workflows/terraform-deploy_reusable.yml) To modify column data types or add or remove columns, go there.
+1. Add the update inside the databaseMigrations folder in the repo.  See the [section on migrations](#database-migrations) for details
 2. The `PostgresDao.java` contains our queries for the database. Remember to update both the save and the fetch methods
 3. `DatabasePartnerMetadataStorage` is what calls our dao to perform the db operation, you will need to update the inputs here
+
+## Database Migrations
+
+We use [Liquibase](https://www.liquibase.com/download) to handle the migrations.  Our migration files are located in
+[`/etor/databaseMigrations`](/etor/databaseMigrations).
+
+### Adding Migrations
+
+Documentation on Liquibase can be found [here](https://docs.liquibase.com/).
+
+Our change log files are organized by business domain.  For example, `metadata.yml` includes migrations about our
+metadata.  You may need to create a new file if the migrations are for a business domain not previously covered.  Add
+the new file to [`/etor/databaseMigrations`](/etor/databaseMigrations) with the name of the business domain.  For
+example, `dogcow.yml` for migrations associated with DogCows.  Liquibase supports multiple change log file types, but we
+prefer YAML.  YAML supports specialized change types but also supports arbitrary SQL if the flexibility is needed.
+
+A couple of concepts we adhere to...
+1. For each new file, start the `id` at 1 and increment from there.
+2. Use your GitHub username as the `author`.
+3. Provide a `comment` that better describes what or why you are trying to accomplish.
+4. If a migration only applies to Azure, include a `label` of `azure`.
+5. Include only one change per `changeSet`.
+
+Reference this new file in [`root.yml`](/etor/databaseMigrations/root.yml) by adding it at the bottom of all the other
+includes except for any includes that are required to be last (e.g. the `etor/databaseMigrations/azure.yml` include).
+
+### Running
+
+Our deployed databases have migrations automatically ran by our CD processes.
+
+To run migrations when running our database locally, run the following...
+
+```shell
+liquibase update --changelog-file ./etor/databaseMigrations/root.yml --url jdbc:postgresql://localhost:5433/intermediary --username intermediary --password 'changeIT!' --label-filter '!azure'
+```
+
+Notice the `--label-filter '!azure'`.  This will prevent the Azure-specific migrations from running and failing in the
+local environment.

etor/databaseMigrations/azure.yml

@@ -0,0 +1,11 @@
+databaseChangeLog:
+  - changeSet:
+      id: 1
+      author: halprin
+      labels: azure
+      context: azure
+      runAlways: true
+      comment: Fix ownership of anything created in every migration run
+      changes:
+        - sql:
+            sql: REASSIGN OWNED BY CURRENT_USER TO azure_pg_admin

etor/databaseMigrations/metadata.yml

@@ -0,0 +1,51 @@
+databaseChangeLog:
+  - changeSet:
+      id: 1
+      author: jeff.crichlake
+      labels: create-type
+      context: metadata
+      comment: create status type
+      changes:
+       - sql:
+           sql: CREATE TYPE message_status AS ENUM ('PENDING', 'DELIVERED', 'FAILED')
+
+  - changeSet:
+      id: 2
+      author:  jeff.crichlake
+      labels: create-metadata-table
+      context: metadata
+      comment: create partner metadata table
+      changes:
+       - createTable:
+            tableName: metadata
+            columns:
+              - column:
+                  name: received_message_id
+                  type: varchar(40)
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: sent_message_id
+                  type: varchar(40)
+              - column:
+                  name: sender
+                  type: varchar(30)
+              - column:
+                  name: receiver
+                  type: varchar(30)
+              - column:
+                  name: hash_of_order
+                  type: varchar(1000)
+              - column:
+                  name: time_received
+                  type: timestamptz
+              - column:
+                  name: time_delivered
+                  type: timestamptz
+              - column:
+                  name: delivery_status
+                  type: message_status
+              - column:
+                  name: failure_reason
+                  type: varchar(1000)

etor/databaseMigrations/root.yml

@@ -0,0 +1,7 @@
+databaseChangeLog:
+  - include:
+      file: etor/databaseMigrations/metadata.yml
+  - include:
+      file: etor/databaseMigrations/azure.yml
+      comment: This include should always be last in this list
+      labels: azure

load-execute.sh

@@ -23,8 +23,7 @@ start_database() {
 
 migrate_database() {
     echo 'Migrating database'
-    export 'PGPASSWORD=changeIT!'
-    psql --set=sslmode=require -h localhost -p 5433 -d intermediary -U intermediary -c "CREATE TYPE message_status AS ENUM ('PENDING', 'DELIVERED', 'FAILED'); CREATE TABLE IF NOT EXISTS metadata (received_message_id varchar(40) PRIMARY KEY, sent_message_id varchar(40), sender varchar(30), receiver varchar(30), hash_of_order varchar(1000), time_received timestamptz, time_delivered timestamptz, delivery_status message_status, failure_reason varchar(1000));"
+    liquibase update --changelog-file ./etor/databaseMigrations/root.yml --url jdbc:postgresql://localhost:5433/intermediary --username intermediary --password 'changeIT!' --label-filter '!azure'
     echo "Database migrated"
 }