chore: Correct db upgrade and add ci avoid regression (#14620)

add atlasgo to ci https://atlasgo.io/declarative/diff related: #11286 BTW, it seems some ddl change only exist in 3.1.3、3.1.5、3.1.6 but do not exist in dev which will also cause an error when updating from those version
apache · Aug 30, 2023 · 25e841a · 25e841a
1 parent 742f0c3
commit 25e841a
Show file tree

Hide file tree

Showing 19 changed files with 482 additions and 318 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -42,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
+      db-schema: ${{ steps.filter.outputs.db-schema }}
     steps:
       - uses: actions/checkout@v2
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
@@ -50,6 +51,8 @@ jobs:
           filters: |
             not-ignore:
               - '!(docs/**)'
+            db-schema:
+              - 'dolphinscheduler-dao/src/main/resources/sql/**'
   build:
     name: Backend-Build
     needs: paths-filter
@@ -80,7 +83,7 @@ jobs:
       - name: Build and Package on ${{ matrix.java }}
         run: |
           ./mvnw -B clean install \
-                 -Prelease,docker \
+                 -Prelease \
                  -Dmaven.test.skip=true \
                  -Dspotless.skip=true \
                  -Dhttp.keepAlive=false \
@@ -120,20 +123,151 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
+  schema-check:
+    runs-on: ubuntu-latest
+    if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
+    timeout-minutes: 20
+    needs: build
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: mysql
+          MYSQL_DATABASE: dolphinscheduler_dev
+        ports:
+          - 3306:3306
+        options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dolphinscheduler_dev
+        ports:
+          - 5432:5432
+        options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
+    strategy:
+      fail-fast: false
+      matrix:
+        db: ["mysql", "postgresql"]
+        version: ["2.0.9", "3.0.6", "3.1.8"]
+    steps:
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: 8
+          distribution: 'adopt'
+      - name: Install Atlas and Create Dir
+        run: |
+          mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
+          curl -sSf https://atlasgo.sh | sh
+      - name: Download Tarball
+        uses: actions/download-artifact@v2
+        with:
+          name: binary-package-8
+          path: dolphinscheduler/dev
+      - name: Set Env
+        run: |
+          VERSION=${{ matrix.version }}
+          echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV
+      - name: Prepare
+        run: |
+          wget https://dlcdn.apache.org/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
+          tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
+          tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1
+
+          if [[ ${{ matrix.db }} == "mysql" ]]; then
+            MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+            MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
+            wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
+            for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
+              if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
+                cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
+              else
+                for d in alert-server api-server master-server worker-server tools; do
+                  cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
+                done
+              fi
+            done
+            docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}"; 
+          else
+            docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
+          fi
+      - name: Check
+        run: |
+          if [[ $DATABASE_VERSION -lt 300 ]]; then
+            chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
+          else
+            chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
+          fi
+          if [[ ${{ matrix.db }} == "mysql" ]]; then
+            export DATABASE="mysql"
+            export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
+            export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
+            export SPRING_DATASOURCE_USERNAME="root"
+            export SPRING_DATASOURCE_PASSWORD="mysql"
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+
+            export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
+            if [[ $DATABASE_VERSION -lt 300 ]]; then
+              bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
+            else
+              bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
+            fi
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+
+            atlas_result=$(atlas schema diff \
+              --from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
+              --to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
+            if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
+              echo "================================================================================================"
+              echo "                                !!!!! For Contributors !!!!!"
+              echo "================================================================================================"
+              echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
+              echo "${atlas_result}"
+              exit 1
+            fi
+          else
+            export DATABASE="postgresql"
+            export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
+            export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
+            export SPRING_DATASOURCE_USERNAME="postgres"
+            export SPRING_DATASOURCE_PASSWORD="postgres"
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+          
+            export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
+            if [[ $DATABASE_VERSION -lt 300 ]]; then
+              bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
+            else
+              bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
+            fi
+            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
+          
+            atlas_result=$(atlas schema diff \
+              --from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
+              --to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
+            if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
+              echo "================================================================================================"
+              echo "                                !!!!! For Contributors !!!!!"
+              echo "================================================================================================"
+              echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
+              echo "${atlas_result}"
+              exit 1
+            fi
+          fi
   result:
     name: Build
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    needs: [ build, paths-filter, cluster-test ]
+    needs: [ build, paths-filter, cluster-test, schema-check ]
     if: always()
     steps:
       - name: Status
         run: |
-          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
+          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
             echo "Skip Build!"
             exit 0
           fi
-          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
+          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' || ${{ needs.schema-check.result }} != 'success' ]]; then
             echo "Build Failed!"
             exit -1
           fi
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
@@ -1052,7 +1052,7 @@ CREATE TABLE t_ds_version
 -- Records of t_ds_version
 -- ----------------------------
 INSERT INTO t_ds_version
-VALUES ('1', 'dev');
+VALUES ('1', '3.3.0');
 
 
 -- ----------------------------

diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
@@ -1043,7 +1043,7 @@ CREATE TABLE `t_ds_version` (
 -- ----------------------------
 -- Records of t_ds_version
 -- ----------------------------
-INSERT IGNORE INTO `t_ds_version` VALUES ('1', 'dev');
+INSERT IGNORE INTO `t_ds_version` VALUES ('1', '3.3.0');
 
 
 -- ----------------------------

diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
@@ -1041,7 +1041,7 @@ INSERT INTO t_ds_queue(queue_name, queue, create_time, update_time)
 VALUES ('default', 'default', '2018-11-29 10:22:33', '2018-11-29 10:22:33');
 
 -- Records of t_ds_queue,default queue name : default
-INSERT INTO t_ds_version(version) VALUES ('dev');
+INSERT INTO t_ds_version(version) VALUES ('3.3.0');
 
 --
 -- Table structure for table t_ds_plugin_define

diff --git a/dolphinscheduler-dao/src/main/resources/sql/soft_version b/dolphinscheduler-dao/src/main/resources/sql/soft_version
@@ -1 +1 @@
-dev
+3.3.0
diff --git a/...duler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql b/...duler-dao/src/main/resources/sql/upgrade/3.1.1_schema/postgresql/dolphinscheduler_ddl.sql
@@ -15,19 +15,8 @@
  * limitations under the License.
 */
 
---- rename t_ds_fav_task task_name to task_type
-DO $$
-DECLARE
-v_schema varchar;
-BEGIN
-    v_schema =current_schema();
-  IF EXISTS(SELECT *
-    FROM information_schema.columns
-    WHERE table_name='t_ds_fav_task' and column_name='task_name')
-  then
-   EXECUTE 'ALTER TABLE IF EXISTS ' || quote_ident(v_schema) ||'.t_ds_fav_task RENAME COLUMN task_name TO task_type';
-END IF;
-END $$;
+-- rename columns
+-- ALTER TABLE IF EXISTS t_ds_fav_task RENAME COLUMN task_name TO task_type;
 
---- add column
+-- add column
 ALTER TABLE t_ds_worker_group ADD COLUMN IF NOT EXISTS description varchar(255) DEFAULT NULL;
diff --git a/...nscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql b/...nscheduler-dao/src/main/resources/sql/upgrade/3.2.0_schema/mysql/dolphinscheduler_ddl.sql
@@ -16,6 +16,14 @@
 */
 SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));
 
+CREATE TABLE if not exists `t_ds_fav_task`
+(
+    `id`        bigint      NOT NULL AUTO_INCREMENT COMMENT 'id',
+    `task_type` varchar(64) NOT NULL COMMENT 'favorite task type name',
+    `user_id`   int         NOT NULL COMMENT 'user id',
+    PRIMARY KEY (`id`)
+) ENGINE = InnoDB AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8 COLLATE = utf8_bin;
+
 -- uc_dolphin_T_t_ds_command_R_test_flag
 drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_command_R_test_flag;
 delimiter d//
@@ -97,8 +105,7 @@ delimiter ;
 CALL uc_dolphin_T_t_ds_task_instance_R_test_flag;
 DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_R_test_flag;
 
-delimiter d//
-CREATE TABLE `t_ds_trigger_relation` (
+CREATE TABLE if not exists `t_ds_trigger_relation` (
     `id` bigint(20) NOT NULL AUTO_INCREMENT,
     `trigger_type` int(11) NOT NULL DEFAULT '0' COMMENT '0 process 1 task',
     `trigger_code` bigint(20) NOT NULL,
@@ -109,8 +116,6 @@ CREATE TABLE `t_ds_trigger_relation` (
     KEY `t_ds_trigger_relation_trigger_code_IDX` (`trigger_code`),
     UNIQUE KEY `t_ds_trigger_relation_UN` (`trigger_type`,`job_id`,`trigger_code`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-d//
-delimiter ;
 
 -- uc_dolphin_T_t_ds_task_definition_R_is_cache
 drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_definition_R_is_cache;
@@ -182,7 +187,7 @@ BEGIN
            AND TABLE_SCHEMA=(SELECT DATABASE())
            AND COLUMN_NAME ='cache_key')
    THEN
-ALTER TABLE t_ds_task_instance ADD `cache_key` varchar(255) DEFAULT null COMMENT 'cache key';
+ALTER TABLE t_ds_task_instance ADD `cache_key` varchar(200) DEFAULT null COMMENT 'cache key';
 END IF;
 END;
 
@@ -202,7 +207,7 @@ BEGIN
         AND TABLE_SCHEMA=(SELECT DATABASE())
         AND INDEX_NAME='cache_key')
     THEN
-ALTER TABLE `t_ds_task_instance` ADD KEY `cache_key`( `cache_key` );
+ALTER TABLE `t_ds_task_instance` ADD KEY `idx_cache_key`( `cache_key` );
 END IF;
 END;
 d//
@@ -234,7 +239,7 @@ END IF;
            AND TABLE_SCHEMA=(SELECT DATABASE())
            AND COLUMN_NAME ='tenant_code')
    THEN
-ALTER TABLE t_ds_process_instance ADD `tenant_code` varchar(64) DEFAULT NULL COMMENT 'tenant code';
+ALTER TABLE t_ds_process_instance ADD `tenant_code` varchar(64) DEFAULT 'default' COMMENT 'tenant code';
 END IF;
 END;
 d//
@@ -356,6 +361,28 @@ ALTER TABLE `t_ds_task_group` MODIFY `name` varchar(255) DEFAULT NULL COMMENT 't
 ALTER TABLE `t_ds_k8s` MODIFY `k8s_name` varchar(255) DEFAULT NULL;
 ALTER TABLE `t_ds_k8s_namespace` MODIFY `namespace` varchar(255) DEFAULT NULL;
 ALTER TABLE `t_ds_cluster` MODIFY `name`        varchar(255) NOT NULL COMMENT 'cluster name';
+ALTER TABLE `t_ds_user` MODIFY COLUMN `tenant_id` int NULL DEFAULT -1 COMMENT 'tenant id';
+
+-- t_ds_fav_task behavior change
+DROP PROCEDURE if EXISTS add_improvement_t_ds_fav_task;
+delimiter d//
+CREATE PROCEDURE add_improvement_t_ds_fav_task()
+BEGIN
+   IF EXISTS (SELECT 1 FROM information_schema.COLUMNS
+           WHERE TABLE_NAME='t_ds_fav_task'
+           AND TABLE_SCHEMA=(SELECT DATABASE())
+           AND COLUMN_NAME ='task_name')
+   THEN
+       ALTER TABLE `t_ds_fav_task` MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT;
+       ALTER TABLE `t_ds_fav_task` DROP COLUMN `task_name`;
+       ALTER TABLE `t_ds_fav_task` ADD COLUMN `task_type` varchar(64) NOT NULL COMMENT 'favorite task type name';
+   END IF;
+END;
+d//
+delimiter ;
+CALL add_improvement_t_ds_fav_task;
+DROP PROCEDURE add_improvement_t_ds_fav_task;
+
 
 -- tenant improvement
 DROP PROCEDURE if EXISTS add_improvement_workflow_run_tenant;
@@ -389,16 +416,26 @@ delimiter ;
 CALL add_improvement_workflow_run_tenant;
 DROP PROCEDURE add_improvement_workflow_run_tenant;
 
--- uc_dolphin_T_t_ds_relation_sub_workflow
-drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_relation_sub_workflow;
+-- t_ds_fav_task behavior change
+drop PROCEDURE if EXISTS drop_t_ds_task_instance_key_foreign_key_instance_id;
 delimiter d//
-CREATE PROCEDURE uc_dolphin_T_t_ds_relation_sub_workflow()
+CREATE PROCEDURE drop_t_ds_task_instance_key_foreign_key_instance_id()
 BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
-           WHERE TABLE_NAME='t_ds_relation_sub_workflow'
-           AND TABLE_SCHEMA=(SELECT DATABASE()))
-   THEN
-CREATE TABLE `t_ds_relation_sub_workflow` (
+    IF EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS
+               WHERE TABLE_SCHEMA = (SELECT DATABASE())
+                 AND TABLE_NAME = 't_ds_task_instance'
+                 AND CONSTRAINT_NAME = 'foreign_key_instance_id')
+    THEN
+        ALTER TABLE `t_ds_task_instance` DROP FOREIGN KEY foreign_key_instance_id;
+    END IF;
+END;
+d//
+delimiter ;
+CALL drop_t_ds_task_instance_key_foreign_key_instance_id;
+DROP PROCEDURE drop_t_ds_task_instance_key_foreign_key_instance_id;
+
+-- table t_ds_relation_sub_workflow
+CREATE TABLE if not exists `t_ds_relation_sub_workflow` (
     `id` bigint NOT NULL AUTO_INCREMENT,
     `parent_workflow_instance_id` bigint  NOT NULL,
     `parent_task_code` bigint  NOT NULL,
@@ -408,7 +445,33 @@ CREATE TABLE `t_ds_relation_sub_workflow` (
     KEY `idx_parent_task_code` (`parent_task_code`),
     KEY `idx_sub_workflow_instance_id` (`sub_workflow_instance_id`)
 );
-END IF;
-END;
 
-d//
+-- table t_ds_project_preference
+CREATE TABLE if not exists `t_ds_project_preference` (
+   `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
+   `code` bigint(20) NOT NULL COMMENT 'encoding',
+   `project_code` bigint(20) NOT NULL COMMENT 'project code',
+   `preferences` varchar(512) NOT NULL COMMENT 'project preferences',
+   `user_id` int(11) DEFAULT NULL COMMENT 'creator id',
+   `state` int(11) DEFAULT '1' comment '1 means enabled, 0 means disabled',
+   `create_time` datetime NOT NULL COMMENT 'create time',
+   `update_time` datetime DEFAULT NULL COMMENT 'update time',
+   PRIMARY KEY (`id`),
+   UNIQUE KEY `unique_project_preference_project_code`(`project_code`),
+   UNIQUE KEY `unique_project_preference_code`(`code`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin;
+
+-- table t_ds_project_parameter
+CREATE TABLE if not exists `t_ds_project_parameter` (
+    `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
+    `param_name` varchar(255) NOT NULL COMMENT 'project parameter name',
+    `param_value` varchar(255) NOT NULL COMMENT 'project parameter value',
+    `code` bigint(20) NOT NULL COMMENT 'encoding',
+    `project_code` bigint(20) NOT NULL COMMENT 'project code',
+    `user_id` int(11) DEFAULT NULL COMMENT 'creator id',
+    `create_time` datetime NOT NULL COMMENT 'create time',
+    `update_time` datetime DEFAULT NULL COMMENT 'update time',
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `unique_project_parameter_name`(`project_code`, `param_name`),
+    UNIQUE KEY `unique_project_parameter_code`(`code`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin;