checker/:ignore unnecessary privilege for different task modes (#372) (…

…#374)

* ignore unnecessary privilege for different task modes

* grant limited rights for incremental test

* add full_mode test

checker/cmd.go

@@ -39,8 +39,19 @@ func CheckSyncConfig(ctx context.Context, cfgs []*config.SubTaskConfig) error {
 		return nil
 	}
 
-	// all `IgnoreCheckingItems` of sub-task are same, so we take first one
-	checkingItems := config.FilterCheckingItems(cfgs[0].IgnoreCheckingItems)
+	// all `IgnoreCheckingItems` and `Mode` of sub-task are same, so we take first one
+	// for ModeFull we don't need replication privilege; for ModeIncrement we don't need dump privilege
+	ignoreCheckingItems := cfgs[0].IgnoreCheckingItems
+	// we directly append ignore checking items here which may cause duplicate in ignoreCheckingItems
+	// but in config.FilterCheckingItems we only use this to delete map's keys so it is tolerable to append directly here
+	switch cfgs[0].Mode {
+	case config.ModeFull:
+		ignoreCheckingItems = append(ignoreCheckingItems, config.ReplicationPrivilegeChecking,
+			config.BinlogEnableChecking, config.BinlogFormatChecking, config.BinlogRowImageChecking)
+	case config.ModeIncrement:
+		ignoreCheckingItems = append(ignoreCheckingItems, config.DumpPrivilegeChecking)
+	}
+	checkingItems := config.FilterCheckingItems(ignoreCheckingItems)
 	if len(checkingItems) == 0 {
 		return nil
 	}

cmd/dm-master/main.go

@@ -37,7 +37,7 @@ func main() {
 	case flag.ErrHelp:
 		os.Exit(0)
 	default:
-		fmt.Printf("parse cmd flags err %s", err)
+		fmt.Printf("parse cmd flags err %s", errors.ErrorStack(err))
 		os.Exit(2)
 	}
 

cmd/dm-tracer/main.go

@@ -36,7 +36,7 @@ func main() {
 	case flag.ErrHelp:
 		os.Exit(0)
 	default:
-		fmt.Printf("parse cmd flags err %s", err)
+		fmt.Printf("parse cmd flags err %s", errors.ErrorStack(err))
 		os.Exit(2)
 	}
 

cmd/dm-worker/main.go

@@ -41,7 +41,7 @@ func main() {
 	case flag.ErrHelp:
 		os.Exit(0)
 	default:
-		fmt.Printf("parse cmd flags err: %s", err)
+		fmt.Printf("parse cmd flags err: %s", errors.ErrorStack(err))
 		os.Exit(2)
 	}
 

tests/_utils/check_count

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -eu
+
+count=$(grep -c "$1" "$TEST_DIR/sql_res.$TEST_NAME.txt")
+if [ $count != $2 ]; then
+    echo "TEST FAILED: OUTPUT CONTAINS '$1' for $count times, which supposed to be $2 times"
+    echo "____________________________________"
+    cat "$TEST_DIR/sql_res.$TEST_NAME.txt"
+    echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+    exit 1
+fi

tests/full_mode/conf/diff_config.toml

@@ -0,0 +1,58 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 1000
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-rowid = false
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "full_mode"
+tables = ["~t.*"]
+
+[[table-config]]
+schema = "full_mode"
+table = "t1"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "full_mode"
+table  = "t1"
+
+[[table-config]]
+schema = "full_mode"
+table = "t2"
+
+[[table-config.source-tables]]
+instance-id = "source-2"
+schema = "full_mode"
+table  = "t2"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = ""
+instance-id = "source-1"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = ""
+instance-id = "source-2"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""

tests/full_mode/conf/dm-master.toml

@@ -0,0 +1,9 @@
+# Master Configuration.
+
+[[deploy]]
+source-id = "mysql-replica-01"
+dm-worker = "127.0.0.1:8262"
+
+[[deploy]]
+source-id = "mysql-replica-02"
+dm-worker = "127.0.0.1:8263"

tests/full_mode/conf/dm-task.yaml

@@ -0,0 +1,51 @@
+---
+name: test
+task-mode: full
+is-sharding: false
+meta-schema: "dm_meta"
+remove-meta: false
+enable-heartbeat: true
+heartbeat-update-interval: 1
+heartbeat-report-interval: 1
+timezone: "Asia/Shanghai"
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "root"
+  password: ""
+
+mysql-instances:
+  - source-id: "mysql-replica-01"
+    black-white-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+  - source-id: "mysql-replica-02"
+    black-white-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+black-white-list:
+  instance:
+    do-dbs: ["full_mode"]
+
+mydumpers:
+  global:
+    mydumper-path: "./bin/mydumper"
+    threads: 4
+    chunk-filesize: 64
+    skip-tz-utc: true
+    extra-args: ""
+
+loaders:
+  global:
+    pool-size: 16
+    dir: "./dumped_data"
+
+syncers:
+  global:
+    worker-count: 16
+    batch: 100

tests/full_mode/conf/dm-worker1.toml

@@ -0,0 +1,15 @@
+# Worker Configuration.
+
+server-id = 101 # we don't give dm_full account REPLICATION SLAVE privilege here, so we must specify server-id here
+source-id = "mysql-replica-01"
+flavor = ""
+enable-gtid = false
+relay-binlog-name = ""
+relay-binlog-gtid = ""
+
+[from]
+host = "127.0.0.1"
+user = "dm_full"
+password = ""
+port = 3306
+

tests/full_mode/conf/dm-worker2.toml

@@ -0,0 +1,14 @@
+# Worker Configuration.
+
+server-id = 102 # we don't give dm_full_account REPLICATION SLAVE privilege here, so we must specify server-id here
+source-id = "mysql-replica-02"
+flavor = ""
+enable-gtid = false
+relay-binlog-name = ""
+relay-binlog-gtid = ""
+
+[from]
+host = "127.0.0.1"
+user = "dm_full"
+password = ""
+port = 3307

tests/full_mode/data/db1.prepare.sql

@@ -0,0 +1,29 @@
+drop database if exists `full_mode`;
+create database `full_mode`;
+use `full_mode`;
+create table t1 (id int, name varchar(20));
+insert into t1 (id, name) values (1, 'arya'), (2, 'catelyn');
+insert into t1 (id, name) values (3, 'Eddard Stark');
+update t1 set name = 'Arya Stark' where id = 1;
+update t1 set name = 'Catelyn Stark' where name = 'catelyn';
+
+-- test multi column index with generated column
+alter table t1 add column info json;
+alter table t1 add column gen_id int as (info->"$.id");
+alter table t1 add index multi_col(`id`, `gen_id`);
+insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}');
+insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}');
+update t1 set info = '{"id": 120}' where id = 1;
+update t1 set info = '{"id": 121}' where id = 2;
+update t1 set info = '{"id": 122}' where id = 3;
+
+-- test genColumnCache is reset after ddl
+alter table t1 add column info2 varchar(40);
+insert into t1 (id, name, info) values (6, 'gentest', '{"id": 125, "test cache": false}');
+alter table t1 add unique key gen_idx(`gen_id`);
+update t1 set name = 'gentestxx' where gen_id = 123;
+
+insert into t1 (id, name, info) values (7, 'gentest', '{"id": 126}');
+update t1 set name = 'gentestxxxxxx' where gen_id = 124;
+-- delete with unique key
+delete from t1 where gen_id > 124;

tests/full_mode/data/db1.prepare.user.sql

@@ -0,0 +1,7 @@
+drop user if exists 'dm_full';
+flush privileges;
+create user 'dm_full'@'%' identified by '';
+grant all privileges on *.* to 'dm_full'@'%';
+revoke replication slave, replication client on *.* from 'dm_full'@'%';
+revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_full'@'%'; # privileges not supported by TiDB
+flush privileges;

tests/full_mode/data/db2.prepare.sql

@@ -0,0 +1,5 @@
+drop database if exists `full_mode`;
+create database `full_mode`;
+use `full_mode`;
+create table t2 (id int auto_increment, name varchar(20), primary key (`id`));
+insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa');

tests/full_mode/data/db2.prepare.user.sql

@@ -0,0 +1,7 @@
+drop user if exists 'dm_full';
+flush privileges;
+create user 'dm_full'@'%' identified by '';
+grant all privileges on *.* to 'dm_full'@'%';
+revoke replication slave, replication client on *.* from 'dm_full'@'%';
+revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_full'@'%'; # privileges not supported by TiDB
+flush privileges;

tests/full_mode/run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eu
+
+cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source $cur/../_utils/test_prepare
+WORK_DIR=$TEST_DIR/$TEST_NAME
+
+function run() {
+    run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1
+    check_contains 'Query OK, 2 rows affected'
+    run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2
+    check_contains 'Query OK, 3 rows affected'
+
+    run_sql_file $cur/data/db1.prepare.user.sql $MYSQL_HOST1 $MYSQL_PORT1
+    check_count 'Query OK, 0 rows affected' 7
+    run_sql_file $cur/data/db2.prepare.user.sql $MYSQL_HOST2 $MYSQL_PORT2
+    check_count 'Query OK, 0 rows affected' 7
+
+    run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
+    check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
+    run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
+    check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
+    run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
+    check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
+
+    # start DM task only
+    dmctl_start_task
+
+    # use sync_diff_inspector to check full dump loader
+    check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
+}
+
+cleanup_data full_mode
+# also cleanup dm processes in case of last run failed
+cleanup_process $*
+run $*
+cleanup_process $*
+
+echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"

tests/incremental_mode/data/db1.prepare.user.sql

@@ -0,0 +1,7 @@
+drop user if exists 'dm_incremental';
+flush privileges;
+create user 'dm_incremental'@'%' identified by '';
+grant all privileges on *.* to 'dm_incremental'@'%';
+revoke select, reload on *.* from 'dm_incremental'@'%';
+revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_incremental'@'%'; # privileges not supported by TiDB
+flush privileges;

tests/incremental_mode/data/db2.prepare.user.sql

@@ -0,0 +1,7 @@
+drop user if exists 'dm_incremental';
+flush privileges;
+create user 'dm_incremental'@'%' identified by '';
+grant all privileges on *.* to 'dm_incremental'@'%';
+revoke select, reload on *.* from 'dm_incremental'@'%';
+revoke create temporary tables, lock tables, create routine, alter routine, event, create tablespace, file, shutdown, execute, process, index on *.* from 'dm_incremental'@'%'; # privileges not supported by TiDB
+flush privileges;

tests/incremental_mode/run.sh

@@ -36,6 +36,21 @@ function run() {
     run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2
 
     # start a task in `incremental` mode
+    # using account with limited privileges
+    kill_dm_worker
+    run_sql_file $cur/data/db1.prepare.user.sql $MYSQL_HOST1 $MYSQL_PORT1
+    check_count 'Query OK, 0 rows affected' 7
+    run_sql_file $cur/data/db2.prepare.user.sql $MYSQL_HOST2 $MYSQL_PORT2
+    check_count 'Query OK, 0 rows affected' 7
+    cat $cur/conf/dm-worker1.toml > $WORK_DIR/dm-worker1.toml
+    sed -i "s/root/dm_incremental/g" $WORK_DIR/dm-worker1.toml
+    cat $cur/conf/dm-worker2.toml > $WORK_DIR/dm-worker2.toml
+    sed -i "s/root/dm_incremental/g" $WORK_DIR/dm-worker2.toml
+    run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $WORK_DIR/dm-worker1.toml
+    check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
+    run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $WORK_DIR/dm-worker2.toml
+    check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
+
     cat $cur/conf/dm-task.yaml > $WORK_DIR/dm-task.yaml
     sed -i "s/task-mode-placeholder/incremental/g" $WORK_DIR/dm-task.yaml
     name1=$(grep "Log: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata|awk -F: '{print $2}'|tr -d ' ')
@@ -46,7 +61,8 @@ function run() {
     sed -i "s/binlog-pos-placeholder-1/$pos1/g" $WORK_DIR/dm-task.yaml
     sed -i "s/binlog-name-placeholder-2/$name2/g" $WORK_DIR/dm-task.yaml
     sed -i "s/binlog-pos-placeholder-2/$pos2/g" $WORK_DIR/dm-task.yaml
-    dmctl_start_task $WORK_DIR/dm-task.yaml
+	sleep 2
+	dmctl_start_task $WORK_DIR/dm-task.yaml
 
     check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
 }

tests/others_integration.txt

@@ -1,7 +1,8 @@
 all_mode
+full_mode
 sequence_sharding
 sequence_safe_mode
 relay_interrupt
 start_task
 initial_unit
-http_apis
+http_apis