From 06f85d97b2308d2e268358875f4acfa85588b7e5 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 19 Jan 2022 14:35:45 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"dm/test:=20swap=20upstream=20and=20do?= =?UTF-8?q?wnstream=20of=20sync=5Fdiff=20in=20some=20ca=E2=80=A6=20(#4382)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref pingcap/tiflow#4159 --- .../conf/diff_config_revert_1.toml | 31 ------------------- .../conf/diff_config_revert_2.toml | 31 ------------------- dm/tests/adjust_gtid/run.sh | 6 ++-- .../full_mode/conf/diff_config_revert_1.toml | 30 ------------------ .../full_mode/conf/diff_config_revert_2.toml | 30 ------------------ dm/tests/full_mode/run.sh | 10 ++---- .../conf/diff_config_revert_1.toml | 30 ------------------ .../conf/diff_config_revert_2.toml | 30 ------------------ dm/tests/incremental_mode/run.sh | 6 ++-- 9 files changed, 7 insertions(+), 197 deletions(-) delete mode 100644 dm/tests/adjust_gtid/conf/diff_config_revert_1.toml delete mode 100644 dm/tests/adjust_gtid/conf/diff_config_revert_2.toml delete mode 100644 dm/tests/full_mode/conf/diff_config_revert_1.toml delete mode 100644 dm/tests/full_mode/conf/diff_config_revert_2.toml delete mode 100644 dm/tests/incremental_mode/conf/diff_config_revert_1.toml delete mode 100644 dm/tests/incremental_mode/conf/diff_config_revert_2.toml diff --git a/dm/tests/adjust_gtid/conf/diff_config_revert_1.toml b/dm/tests/adjust_gtid/conf/diff_config_revert_1.toml deleted file mode 100644 index 455ed32451f..00000000000 --- a/dm/tests/adjust_gtid/conf/diff_config_revert_1.toml +++ /dev/null @@ -1,31 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - - -[task] - output-dir = "/tmp/ticdc_dm_test/output" - - source-instances = ["tidb0"] - - target-instance = "mysql1" - - target-check-tables = ["adjust_gtid.t?*"] - - -[data-sources] -[data-sources.mysql1] -host = "127.0.0.1" -port = 3306 -user = "root" -password = "123456" - -[data-sources.tidb0] -host = "127.0.0.1" -port = 4000 -user = "test" -password = "123456" diff --git a/dm/tests/adjust_gtid/conf/diff_config_revert_2.toml b/dm/tests/adjust_gtid/conf/diff_config_revert_2.toml deleted file mode 100644 index 6d38ad7fb17..00000000000 --- a/dm/tests/adjust_gtid/conf/diff_config_revert_2.toml +++ /dev/null @@ -1,31 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - - -[task] - output-dir = "/tmp/ticdc_dm_test/output" - - source-instances = ["tidb0"] - - target-instance = "mysql2" - - target-check-tables = ["adjust_gtid.t?*"] - - -[data-sources] -[data-sources.mysql2] -host = "127.0.0.1" -port = 3307 -user = "root" -password = "123456" - -[data-sources.tidb0] -host = "127.0.0.1" -port = 4000 -user = "test" -password = "123456" diff --git a/dm/tests/adjust_gtid/run.sh b/dm/tests/adjust_gtid/run.sh index 107485d1e31..01676329be1 100755 --- a/dm/tests/adjust_gtid/run.sh +++ b/dm/tests/adjust_gtid/run.sh @@ -85,8 +85,7 @@ function run() { "start-task $WORK_DIR/dm-task.yaml --remove-meta" # use sync_diff_inspector to check full dump loader - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_1.toml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_2.toml + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml name1=$(grep "Log: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') pos1=$(grep "Pos: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') @@ -128,8 +127,7 @@ function run() { check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT # use sync_diff_inspector to check incremental dump loader - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_1.toml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_2.toml + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" diff --git a/dm/tests/full_mode/conf/diff_config_revert_1.toml b/dm/tests/full_mode/conf/diff_config_revert_1.toml deleted file mode 100644 index f690ce6e804..00000000000 --- a/dm/tests/full_mode/conf/diff_config_revert_1.toml +++ /dev/null @@ -1,30 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - -[task] - output-dir = "/tmp/ticdc_dm_test/output" - - source-instances = ["tidb0"] - - target-instance = "mysql1" - - target-check-tables = ["full_mode.t?*"] - - -[data-sources] -[data-sources.mysql1] -host = "127.0.0.1" -port = 3306 -user = "root" -password = "123456" - -[data-sources.tidb0] -host = "127.0.0.1" -port = 4000 -user = "test" -password = "123456" diff --git a/dm/tests/full_mode/conf/diff_config_revert_2.toml b/dm/tests/full_mode/conf/diff_config_revert_2.toml deleted file mode 100644 index 84a3ecd2d43..00000000000 --- a/dm/tests/full_mode/conf/diff_config_revert_2.toml +++ /dev/null @@ -1,30 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - -[task] - output-dir = "/tmp/ticdc_dm_test/output" - - source-instances = ["tidb0"] - - target-instance = "mysql2" - - target-check-tables = ["full_mode.t?*"] - - -[data-sources] -[data-sources.mysql2] -host = "127.0.0.1" -port = 3307 -user = "root" -password = "123456" - -[data-sources.tidb0] -host = "127.0.0.1" -port = 4000 -user = "test" -password = "123456" diff --git a/dm/tests/full_mode/run.sh b/dm/tests/full_mode/run.sh index e26387ba331..80fd8975c52 100755 --- a/dm/tests/full_mode/run.sh +++ b/dm/tests/full_mode/run.sh @@ -56,7 +56,6 @@ function fail_acquire_global_lock() { "you need (at least one of) the RELOAD privilege(s) for this operation" 2 cleanup_data full_mode - cleanup_data_upstream full_mode cleanup_process $* } @@ -101,6 +100,7 @@ function escape_schema() { # start DM task only dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta" + check_sync_diff $WORK_DIR $WORK_DIR/diff_config.toml check_log_contain_with_retry 'clean dump files' $WORK_DIR/worker1/log/dm-worker.log check_log_contain_with_retry 'clean dump files' $WORK_DIR/worker2/log/dm-worker.log @@ -114,7 +114,6 @@ function escape_schema() { check_metric $WORKER2_PORT 'dumpling_dump_finished_tables' 3 0 3 cleanup_data full/mode - cleanup_data_upstream full/mode cleanup_process $* } @@ -127,8 +126,7 @@ function empty_data() { init_cluster dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta" - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_1.toml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_2.toml + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml sleep 1 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ @@ -141,7 +139,6 @@ function empty_data() { check_log_contains $WORK_DIR/worker2/log/dm-worker.log "progress=\"100.00 %\"" cleanup_data full_mode - cleanup_data_upstream full_mode cleanup_process $* } @@ -190,8 +187,7 @@ function run() { dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta" # use sync_diff_inspector to check full dump loader - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_1.toml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_2.toml + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml echo "check dump files have been cleaned" ls $WORK_DIR/worker1/dumped_data.test && exit 1 || echo "worker1 auto removed dump files" diff --git a/dm/tests/incremental_mode/conf/diff_config_revert_1.toml b/dm/tests/incremental_mode/conf/diff_config_revert_1.toml deleted file mode 100644 index 45328c12217..00000000000 --- a/dm/tests/incremental_mode/conf/diff_config_revert_1.toml +++ /dev/null @@ -1,30 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - -[task] - output-dir = "/tmp/ticdc_dm_test/output" - - source-instances = ["tidb0"] - - target-instance = "mysql1" - - target-check-tables = ["incremental_mode.t?*"] - - -[data-sources] -[data-sources.mysql1] -host = "127.0.0.1" -port = 3306 -user = "root" -password = "123456" - -[data-sources.tidb0] -host = "127.0.0.1" -port = 4000 -user = "test" -password = "123456" diff --git a/dm/tests/incremental_mode/conf/diff_config_revert_2.toml b/dm/tests/incremental_mode/conf/diff_config_revert_2.toml deleted file mode 100644 index 16ed402a93f..00000000000 --- a/dm/tests/incremental_mode/conf/diff_config_revert_2.toml +++ /dev/null @@ -1,30 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - -[task] - output-dir = "/tmp/ticdc_dm_test/output" - - source-instances = ["tidb0"] - - target-instance = "mysql2" - - target-check-tables = ["incremental_mode.t?*"] - - -[data-sources] -[data-sources.mysql2] -host = "127.0.0.1" -port = 3307 -user = "root" -password = "123456" - -[data-sources.tidb0] -host = "127.0.0.1" -port = 4000 -user = "test" -password = "123456" diff --git a/dm/tests/incremental_mode/run.sh b/dm/tests/incremental_mode/run.sh index 011c72cefd7..e86ace8800b 100755 --- a/dm/tests/incremental_mode/run.sh +++ b/dm/tests/incremental_mode/run.sh @@ -76,8 +76,7 @@ function run() { sed -i "s/binlog-pos-placeholder-2/4/g" $WORK_DIR/dm-task.yaml dmctl_start_task $WORK_DIR/dm-task.yaml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_1.toml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_2.toml + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml dmctl_stop_task $TASK_NAME @@ -266,8 +265,7 @@ function run() { "resume-task test" \ "\"result\": true" 3 - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_1.toml - check_sync_diff $WORK_DIR $cur/conf/diff_config_revert_2.toml + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml # test rotate binlog, after rotate and ddl, master binlog should be equal to sync binlog run_sql "flush logs;" $MYSQL_PORT1 $MYSQL_PASSWORD1