From ec821eba57e11d4a0ef5039ba0a444209ab305a1 Mon Sep 17 00:00:00 2001 From: ireneontheway Date: Thu, 17 Sep 2020 16:17:37 +0800 Subject: [PATCH 1/3] Update task-configuration-file-full.md --- en/task-configuration-file-full.md | 1 + 1 file changed, 1 insertion(+) diff --git a/en/task-configuration-file-full.md b/en/task-configuration-file-full.md index e916b6a36..85e7e8714 100644 --- a/en/task-configuration-file-full.md +++ b/en/task-configuration-file-full.md @@ -42,6 +42,7 @@ target-database: # Configuration of the downstream database insta port: 4000 user: "root" password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" # It is recommended to use a password encrypted with dmctl + max-allowed-packet: 67108864 # When setting the DM to internally connect to the TiDB server, the "max_allowed_packet" limit of the TiDB client (the limit of the maximum accepted packet). The unit is bytes. (67108864 by default) session: # The session variables of TiDB, supported since v1.0.6. For details, go to `https://pingcap.com/docs/stable/system-variables` sql_mode: "ANSI_QUOTES,NO_ZERO_IN_DATE,NO_ZERO_DATE" tidb_skip_utf8_check: 1 From 04fb2b703ce52e1c8531e40cc2aadfb310f02279 Mon Sep 17 00:00:00 2001 From: ireneontheway Date: Thu, 17 Sep 2020 16:25:21 +0800 Subject: [PATCH 2/3] en: some mirror updates --- en/dm-worker-intro.md | 2 +- en/glossary.md | 2 +- en/key-features.md | 10 +++++----- en/overview.md | 8 ++++++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/en/dm-worker-intro.md b/en/dm-worker-intro.md index 95e83e745..60f44c371 100644 --- a/en/dm-worker-intro.md +++ b/en/dm-worker-intro.md @@ -23,7 +23,7 @@ A DM-worker task contains multiple logic units, including relay log, Dumper, Loa The relay log persistently stores the binlog data from the upstream MySQL/MariaDB and provides the feature of accessing binlog events for the binlog replication. -Its rationale and features are similar to the secondary relay log of MySQL. For details, see [The Secondary Relay Log](https://dev.mysql.com/doc/refman/5.7/en/slave-logs-relaylog.html). +Its rationale and features are similar to the relay log of MySQL. For details, see [MySQL Relay Log](https://dev.mysql.com/doc/refman/5.7/en/replica-logs-relaylog.html). ### Dumper diff --git a/en/glossary.md b/en/glossary.md index 4dd2b57a0..df9e247e9 100644 --- a/en/glossary.md +++ b/en/glossary.md @@ -75,7 +75,7 @@ In the case of clearly mentioning "full", not explicitly mentioning "full or inc ### Relay log -The relay log refers to the binlog files that DM-worker pulls from the upstream MySQL or MariaDB, and stores in the local disk. The format of the relay log is the standard binlog file, which can be parsed by tools such as [mysqlbinlog](https://dev.mysql.com/doc/refman/8.0/en/mysqlbinlog.html) of a compatible version. +The relay log refers to the binlog files that DM-worker pulls from the upstream MySQL or MariaDB, and stores in the local disk. The format of the relay log is the standard binlog file, which can be parsed by tools such as [Mysql Binlog](https://dev.mysql.com/doc/refman/8.0/en/mysqlbinlog.html) of a compatible version. Its role is similar to [MySQL Relay Log](https://dev.mysql.com/doc/refman/5.7/en/replica-logs-relaylog.html) and [MariaDB Relay Log](https://mariadb.com/kb/en/library/relay-log/). For more details such as the relay log's directory structure, initial migration rules, and data purge in TiDB DM, see [TiDB DM relay log](relay-log.md). diff --git a/en/key-features.md b/en/key-features.md index b0628ddb5..107eb8051 100644 --- a/en/key-features.md +++ b/en/key-features.md @@ -131,10 +131,10 @@ block-allow-list: # This configuration applies to DM versions higher ### Parameter explanation -- `do-dbs`: allow lists of the schemas to be migrated, similar to [`replicate-do-db`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-slave.html#option_mysqld_replicate-do-db) in MySQL -- `ignore-dbs`: block lists of the schemas to be migrated, similar to [`replicate-ignore-db`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-slave.html#option_mysqld_replicate-ignore-db) in MySQL -- `do-tables`: allow lists of the tables to be migrated, similar to [`replicate-do-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-slave.html#option_mysqld_replicate-do-table) in MySQL -- `ignore-tables`: block lists of the tables to be migrated, similar to [`replicate-ignore-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-slave.html#option_mysqld_replicate-ignore-table) in MySQL +- `do-dbs`: allow lists of the schemas to be migrated, similar to [`replicate-do-db`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-do-db) in MySQL +- `ignore-dbs`: block lists of the schemas to be migrated, similar to [`replicate-ignore-db`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-ignore-db) in MySQL +- `do-tables`: allow lists of the tables to be migrated, similar to [`replicate-do-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-do-tabl) in MySQL +- `ignore-tables`: block lists of the tables to be migrated, similar to [`replicate-ignore-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-ignore-table) in MySQL If a value of the above parameters starts with the `~` character, the subsequent characters of this value are treated as a [regular expression](https://golang.org/pkg/regexp/syntax/#hdr-syntax). You can use this parameter to match schema or table names. @@ -146,7 +146,7 @@ The filtering rules corresponding to `do-dbs` and `ignore-dbs` are similar to th > > In DM and in MySQL, the allow and block lists filtering rules are different in the following ways: > -> - In MySQL, [`replicate-wild-do-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-slave.html#option_mysqld_replicate-wild-do-table) and [`replicate-wild-ignore-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-slave.html#option_mysqld_replicate-wild-ignore-table) support wildcard characters. In DM, some parameter values directly supports regular expressions that start with the `~` character. +> - In MySQL, [`replicate-wild-do-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-wild-do-table) and [`replicate-wild-ignore-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-wild-ignore-table) support wildcard characters. In DM, some parameter values directly supports regular expressions that start with the `~` character. > - DM currently only supports binlogs in the `ROW` format, and does not support those in the `STATEMENT` or `MIXED` format. Therefore, the filtering rules in DM correspond to those in the `ROW` format in MySQL. > - MySQL determines a DDL statement only by the database name explicitly specified in the `USE` section of the statement. DM determines a statement first based on the database name section in the DDL statement. If the DDL statement does not contain such a section, DM determines the statement by the `USE` section. Suppose that the SQL statement to be determined is `USE test_db_2; CREATE TABLE test_db_1.test_table (c1 INT PRIMARY KEY)`; that `replicate-do-db=test_db_1` is configured in MySQL and `do-dbs: ["test_db_1"]` is configured in DM. Then this rule only applies to DM and not to MySQL. diff --git a/en/overview.md b/en/overview.md index fd1fa1d91..da9d13fbd 100644 --- a/en/overview.md +++ b/en/overview.md @@ -64,6 +64,14 @@ When you deploy multiple DM-master nodes, all DM-master nodes use the embedded e When the number of deployed DM-worker nodes exceeds the number of upstream MySQL/MariaDB nodes, the extra DM-worker nodes are idle by default. If a DM-worker node goes offline or is isolated from the DM-master leader, DM-master automatically schedules data migration tasks of the original DM-worker node to other idle DM-worker nodes. (If a DM-worker node is isolated, it automatically stops the data migration tasks on it); if there are no available idle DM-worker nodes, the data migration tasks of the original DM-worker cannot be performed. +> **Note:** +> +> When the data migration task is in the process of full export or import, the migration task does not support high availability. Here are the main reasons: +> +> - For the full export, MySQL does not support exporting from a specific snapshot point, which means that after the data migration task is rescheduled or restarted, the export cannot continue from the previous moment of interruption. +> +> - For the full import, DM-worker does not support reading full export data cross the nodes, which means that after the data migration task is scheduled to the new DM-worker node, you cannot read all the exported data on the original DM-worker node before the scheduling occurs. + ## Data migration features This section describes the data migration features provided by the Data Migration tool. From b2e02989f4486a021d173db7518367e778576600 Mon Sep 17 00:00:00 2001 From: ireneontheway <48651140+ireneontheway@users.noreply.github.com> Date: Mon, 21 Sep 2020 10:29:30 +0800 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Keke Yi <40977455+yikeke@users.noreply.github.com> --- en/glossary.md | 2 +- en/key-features.md | 2 +- en/overview.md | 4 ++-- en/task-configuration-file-full.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/en/glossary.md b/en/glossary.md index df9e247e9..4746a44b3 100644 --- a/en/glossary.md +++ b/en/glossary.md @@ -75,7 +75,7 @@ In the case of clearly mentioning "full", not explicitly mentioning "full or inc ### Relay log -The relay log refers to the binlog files that DM-worker pulls from the upstream MySQL or MariaDB, and stores in the local disk. The format of the relay log is the standard binlog file, which can be parsed by tools such as [Mysql Binlog](https://dev.mysql.com/doc/refman/8.0/en/mysqlbinlog.html) of a compatible version. Its role is similar to [MySQL Relay Log](https://dev.mysql.com/doc/refman/5.7/en/replica-logs-relaylog.html) and [MariaDB Relay Log](https://mariadb.com/kb/en/library/relay-log/). +The relay log refers to the binlog files that DM-worker pulls from the upstream MySQL or MariaDB, and stores in the local disk. The format of the relay log is the standard binlog file, which can be parsed by tools such as [mysqlbinlog](https://dev.mysql.com/doc/refman/8.0/en/mysqlbinlog.html) of a compatible version. Its role is similar to [MySQL Relay Log](https://dev.mysql.com/doc/refman/5.7/en/replica-logs-relaylog.html) and [MariaDB Relay Log](https://mariadb.com/kb/en/library/relay-log/). For more details such as the relay log's directory structure, initial migration rules, and data purge in TiDB DM, see [TiDB DM relay log](relay-log.md). diff --git a/en/key-features.md b/en/key-features.md index 107eb8051..5d14b167d 100644 --- a/en/key-features.md +++ b/en/key-features.md @@ -133,7 +133,7 @@ block-allow-list: # This configuration applies to DM versions higher - `do-dbs`: allow lists of the schemas to be migrated, similar to [`replicate-do-db`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-do-db) in MySQL - `ignore-dbs`: block lists of the schemas to be migrated, similar to [`replicate-ignore-db`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-ignore-db) in MySQL -- `do-tables`: allow lists of the tables to be migrated, similar to [`replicate-do-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-do-tabl) in MySQL +- `do-tables`: allow lists of the tables to be migrated, similar to [`replicate-do-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-do-table) in MySQL - `ignore-tables`: block lists of the tables to be migrated, similar to [`replicate-ignore-table`](https://dev.mysql.com/doc/refman/5.7/en/replication-options-replica.html#option_mysqld_replicate-ignore-table) in MySQL If a value of the above parameters starts with the `~` character, the subsequent characters of this value are treated as a [regular expression](https://golang.org/pkg/regexp/syntax/#hdr-syntax). You can use this parameter to match schema or table names. diff --git a/en/overview.md b/en/overview.md index da9d13fbd..39b7b6887 100644 --- a/en/overview.md +++ b/en/overview.md @@ -68,9 +68,9 @@ When the number of deployed DM-worker nodes exceeds the number of upstream MySQL > > When the data migration task is in the process of full export or import, the migration task does not support high availability. Here are the main reasons: > -> - For the full export, MySQL does not support exporting from a specific snapshot point, which means that after the data migration task is rescheduled or restarted, the export cannot continue from the previous moment of interruption. +> - For the full export, MySQL does not support exporting from a specific snapshot point yet. This means that after the data migration task is rescheduled or restarted, the export cannot resume from the previous interruption point. > -> - For the full import, DM-worker does not support reading full export data cross the nodes, which means that after the data migration task is scheduled to the new DM-worker node, you cannot read all the exported data on the original DM-worker node before the scheduling occurs. +> - For the full import, DM-worker does not support reading exported full data across the nodes yet. This means that after the data migration task is scheduled to a new DM-worker node, you cannot read the exported full data on the original DM-worker node before the scheduling happens. ## Data migration features diff --git a/en/task-configuration-file-full.md b/en/task-configuration-file-full.md index 85e7e8714..6bce9cd49 100644 --- a/en/task-configuration-file-full.md +++ b/en/task-configuration-file-full.md @@ -42,7 +42,7 @@ target-database: # Configuration of the downstream database insta port: 4000 user: "root" password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" # It is recommended to use a password encrypted with dmctl - max-allowed-packet: 67108864 # When setting the DM to internally connect to the TiDB server, the "max_allowed_packet" limit of the TiDB client (the limit of the maximum accepted packet). The unit is bytes. (67108864 by default) + max-allowed-packet: 67108864 # Sets the "max_allowed_packet" limit of the TiDB client (that is, the limit of the maximum accepted packet) when DM internally connects to the TiDB server. The unit is bytes. (67108864 by default) session: # The session variables of TiDB, supported since v1.0.6. For details, go to `https://pingcap.com/docs/stable/system-variables` sql_mode: "ANSI_QUOTES,NO_ZERO_IN_DATE,NO_ZERO_DATE" tidb_skip_utf8_check: 1