dm: load table structure from dump files (pingcap#3295) (pingcap#4163)

ti-chi-bot · Jan 13, 2022 · 76de75d · 76de75d
1 parent fcbf280
commit 76de75d
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 14 deletions.
diff --git a/dm/loader/util.go b/dm/loader/util.go
@@ -158,7 +158,6 @@ func cleanDumpFiles(cfg *config.SubTaskConfig) {
 		var lastErr error
 		for f := range files {
 			if strings.HasSuffix(f, ".sql") {
-				// TODO: table structure files are not used now, but we plan to used them in future so not delete them
 				if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") {
 					continue
 				}

diff --git a/dm/syncer/syncer.go b/dm/syncer/syncer.go
@@ -1351,7 +1351,11 @@ func (s *Syncer) Run(ctx context.Context) (err error) {
 	if s.cfg.Mode == config.ModeAll && fresh {
 		delLoadTask = true
 		flushCheckpoint = true
-		// TODO: loadTableStructureFromDump in future
+		err = s.loadTableStructureFromDump(ctx)
+		if err != nil {
+			tctx.L().Warn("error happened when load table structure from dump files", zap.Error(err))
+			cleanDumpFile = false
+		}
 	} else {
 		cleanDumpFile = false
 	}
@@ -2766,7 +2770,6 @@ func (s *Syncer) genRouter() error {
 	return nil
 }
 
-//nolint:unused
 func (s *Syncer) loadTableStructureFromDump(ctx context.Context) error {
 	logger := s.tctx.L()
 

diff --git a/dm/tests/downstream_more_column/run.sh b/dm/tests/downstream_more_column/run.sh
@@ -32,17 +32,8 @@ function run() {
 	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1<100;" "count(1): 2"
 
 	run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
-	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-		"query-status test" \
-		"sourceTable: \`${db1}\`.\`${tb1}\`" 1 \
-		"targetTable: \`${db}\`.\`${tb}\`" 1 \
-		"Column count doesn't match value count" 1
-	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-		"operate-schema set -s mysql-replica-01 test -d ${db1} -t ${tb1} $cur/data/schema.sql" \
-		"\"result\": true" 2
+	# we load table structure from dump files, so there's no error now
 
-	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-		"resume-task test"
 	# check incremental data
 	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1>100 and c1<1000;" "count(1): 2"
 

diff --git a/dm/tests/shardddl1/run.sh b/dm/tests/shardddl1/run.sh
@@ -13,7 +13,10 @@ function DM_001_CASE() {
 	# schema tracker could track per table without error
 	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"query-status test" \
-		"Duplicate column name 'new_col1'" 1
+		"\"result\": true" 2 \
+		"\"synced\": true" 1
+	# only downstream sees a duplicate error, but currently ignored by DM
+	check_log_contain_with_retry "Duplicate column name 'new_col1'" $WORK_DIR/worker1/log/dm-worker.log $WORK_DIR/worker2/log/dm-worker.log
 }
 
 function DM_001() {