From 028c953a04f9563522cc5444aef7926bd536a165 Mon Sep 17 00:00:00 2001 From: Nicolas <25302138+NicolasCwy@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:52:46 +0800 Subject: [PATCH] [#12048] Update liquibase configuration (#12930) * Update gradle config * Update liquibase config for v9 * Turn off table generate for prod * Update of changelog file * Add configuration for generating changelog * Add schema migration docs --------- Co-authored-by: FergusMok --- build.gradle | 22 + docs/schema-migration.md | 34 ++ .../teammates/common/util/HibernateUtil.java | 6 +- .../db/changelog/db.changelog-root.xml | 2 +- .../db/changelog/db.changelog-v9.0.0.xml | 536 ++++++++++++++++++ .../db/changelog/db.changelog-v9.xml | 82 --- 6 files changed, 598 insertions(+), 84 deletions(-) create mode 100644 docs/schema-migration.md create mode 100644 src/main/resources/db/changelog/db.changelog-v9.0.0.xml delete mode 100644 src/main/resources/db/changelog/db.changelog-v9.xml diff --git a/build.gradle b/build.gradle index 90abb909473..f448d003a8f 100644 --- a/build.gradle +++ b/build.gradle @@ -101,7 +101,9 @@ dependencies { exclude group: "org.apache.jmeter", module: "bom" } + liquibaseRuntime("org.liquibase:liquibase-core:4.19.0") liquibaseRuntime("info.picocli:picocli:4.7.1") + liquibaseRuntime("org.postgresql:postgresql:42.7.2") liquibaseRuntime(sourceSets.main.output) } @@ -136,6 +138,10 @@ sourceSets { } } +if (!project.hasProperty("runList")) { + project.ext.set("runList", "main") +} + liquibase { activities { main { @@ -145,7 +151,23 @@ liquibase { username project.properties['liquibaseUsername'] password project.properties['liquibasePassword'] } + snapshot { + url project.properties['liquibaseDbUrl'] + username project.properties['liquibaseUsername'] + password project.properties['liquibasePassword'] + snapshotFormat "json" + outputFile "liquibase-snapshot.json" + } + diffMain { + searchPath "${projectDir}" + changeLogFile "src/main/resources/db/changelog/db.changelog-new.xml" + referenceUrl project.properties['liquibaseDbUrl'] + referenceUsername project.properties['liquibaseUsername'] + referencePassword project.properties['liquibasePassword'] + url "offline:postgresql?snapshot=liquibase-snapshot.json" + } } + runList = project.ext.runList } tasks.withType(cz.habarta.typescript.generator.gradle.GenerateTask) { diff --git a/docs/schema-migration.md b/docs/schema-migration.md new file mode 100644 index 00000000000..a88f49ece0f --- /dev/null +++ b/docs/schema-migration.md @@ -0,0 +1,34 @@ + + title: "Schema Migration" + + +# SQL Schema Migration + +Teammates uses _[Liquibase]_(https://docs.liquibase.com/start/home.html), a database schema change management solution that enables developers to revise and release database changes to production. The maintainers in charge of releases (Release Leader) will be in charge of generating a _Liquibase_ changelog prior to each release to keep the production databases schema in sync with the code. Therefore this section is just for documentation purposes for contributors. + +## Liquibase in Teammates +_Liquibase_ is made available using the [gradle plugin](https://github.com/liquibase/liquibase-gradle-plugin), providing _liquibase_ functions as tasks. Try `gradle tasks | grep "liquibase"` to see all the tasks available. In teammates, change logs (more in the next section) are written in _XML_. + +### Liquibase connection +Amend the `liquibaseDbUrl`, `liquibaseUsername` and `liquibasePassword` in `gradle.properties` to allow the _Liquibase_ plugin to connect your database. + +## Change logs, change sets and change types +A _change log_ is a file that contains a series of _change sets_ (analagous to a transaction) which applies _change types_ (actions). You can refer to this page on liquibase on the types of [change types](https://docs.liquibase.com/change-types/home.html) that can be used. + +## Gradle Activities for Liquibase +Activities in Gradle are a way of specifying different variables provided by gradle to the Liquibase plugin. The argument `runList` provided by `-pRunList=` e.g `./gradlew liquibaseSnapshot -PrunList=snapshot` is used to specify which activity to be used for the Liquibase command. In this case the `liquibaseSnapshot` command is run using the `snapshot` activity. + +Here is a brief description of the activities defined for Liquibase +1. Main: The default activity used by Liquibase commands and is used for running changelogs against a database. This is used by default if a `runList` is not defined +2. Snapshot: Used to specify output format and name for snapshots i.e JSON +3. diffMain: Specify the reference and the target database to generate changelog that contains operations to update reference database to the state of the target database. i.e the reference is the JSON file generated by the snapshot command, this can be replaced with a live database which is used as reference. + +## Generating/ Updating liquibase change logs +1. Ensure `diff-main` activity in `build.gradle` is pointing to the latest release changelog `src/main/resources/db/changelog/db.changelog-.xml` +2. Delete the `postgres-data` folder to clear any old database schemas +3. Run `git checkout ` and +4. Run the server using `./gradlew serverRun` to generate tables found on branch +5. Generate snapshot of database by running `./gradlew liquibaseSnapshot -PrunList=snapshot`, the snapshot will be output to `liquibase-snapshot.json` +6. Checkout your branch and repeat steps 2 and 4 to generate the tables found on your branch +7. Run `./gradlew liquibaseDiffChangeLog -PrunList=diffMain` to generate changeLog to resolve database schema differences + diff --git a/src/main/java/teammates/common/util/HibernateUtil.java b/src/main/java/teammates/common/util/HibernateUtil.java index acad8a4d13a..8edcca6c807 100644 --- a/src/main/java/teammates/common/util/HibernateUtil.java +++ b/src/main/java/teammates/common/util/HibernateUtil.java @@ -116,7 +116,7 @@ public static void buildSessionFactory(String dbUrl, String username, String pas .setProperty("hibernate.connection.username", username) .setProperty("hibernate.connection.password", password) .setProperty("hibernate.connection.url", dbUrl) - .setProperty("hibernate.hbm2ddl.auto", "update") + .setProperty("hibernate.hbm2ddl.auto", "validate") .setProperty("show_sql", "true") .setProperty("hibernate.current_session_context_class", "thread") .setProperty("hibernate.hikari.minimumIdle", "10") @@ -130,6 +130,10 @@ public static void buildSessionFactory(String dbUrl, String username, String pas // .setProperty("hibernate.jdbc.fetch_size", "50") .addPackage("teammates.storage.sqlentity"); + if (Config.IS_DEV_SERVER) { + config.setProperty("hibernate.hbm2ddl.auto", "update"); + } + for (Class cls : ANNOTATED_CLASSES) { config = config.addAnnotatedClass(cls); } diff --git a/src/main/resources/db/changelog/db.changelog-root.xml b/src/main/resources/db/changelog/db.changelog-root.xml index 66d4b7d7c88..af5c5e34d7e 100644 --- a/src/main/resources/db/changelog/db.changelog-root.xml +++ b/src/main/resources/db/changelog/db.changelog-root.xml @@ -4,5 +4,5 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> - + diff --git a/src/main/resources/db/changelog/db.changelog-v9.0.0.xml b/src/main/resources/db/changelog/db.changelog-v9.0.0.xml new file mode 100644 index 00000000000..205645520ff --- /dev/null +++ b/src/main/resources/db/changelog/db.changelog-v9.0.0.xml @@ -0,0 +1,536 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/db/changelog/db.changelog-v9.xml b/src/main/resources/db/changelog/db.changelog-v9.xml deleted file mode 100644 index 57b6e7f9587..00000000000 --- a/src/main/resources/db/changelog/db.changelog-v9.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -