*: use show full tables in ListAllDatabasesTables (pingcap#325)

.github/workflows/go.yml

@@ -29,22 +29,113 @@ jobs:
         run: make test WITH_RACE=1
       - uses: codecov/codecov-action@v1
 
-  integration-test:
+  integration-test-mysql-5735:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     strategy:
       fail-fast: true
+    services:
+      mysql:
+        image: mysql:5.7.35
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
     steps:
       - uses: actions/checkout@v2
+      - name: Shutdown Ubuntu MySQL (SUDO)
+        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
       - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
           go-version: 1.16
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Get dependencies
+        run: go mod download
+      - name: Download dependencies
+        run: sh install.sh
+      - name: Integration test
+        run: make integration_test
+      - name: Set up tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+
+  integration-test-mysql-8026:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: true
+    services:
+      mysql:
+        image: mysql:8.0.26
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+      - uses: actions/checkout@v2
+      - name: Shutdown Ubuntu MySQL (SUDO)
+        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Get dependencies
+        run: go mod download
+      - name: Download dependencies
+        run: sh install.sh
+      - name: Integration test
+        run: make integration_test
+      - name: Set up tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+
+  integration-test-mysql-8022:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: true
+    services:
+      mysql:
+        image: mysql:8.0.22
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+      - uses: actions/checkout@v2
+      - name: Shutdown Ubuntu MySQL (SUDO)
+        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Get dependencies
+        run: go mod download
       - name: Download dependencies
         run: sh install.sh
-      - name: Start MySQL
-        run: |
-          sudo systemctl start mysql.service
-          mysqladmin -uroot -proot password ''
       - name: Integration test
         run: make integration_test
+      - name: Set up tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

dumpling/Makefile

@@ -29,7 +29,7 @@ test: failpoint-enable
 
 integration_test: bins failpoint-enable bin/dumpling
 	@make failpoint-disable
-	./tests/run.sh
+	./tests/run.sh $(CASE)
 
 tools:
 	@echo "install tools..."

dumpling/tests/quote/data/quote-database-schema-create-mysql57.sql

@@ -0,0 +1,2 @@
+/*!40101 SET NAMES binary*/;
+CREATE DATABASE `quo``te/database` /*!40100 DEFAULT CHARACTER SET latin1 */;

dumpling/tests/quote/data/quote-database.quote-table-schema-mysql57.sql

@@ -0,0 +1,7 @@
+/*!40101 SET NAMES binary*/;
+CREATE TABLE `quo``te/table` (
+  `quo``te/col` int(11) NOT NULL,
+  `a` int(11) DEFAULT NULL,
+  `gen``id` int(11) GENERATED ALWAYS AS (`quo``te/col`) VIRTUAL,
+  PRIMARY KEY (`quo``te/col`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

dumpling/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql

@@ -0,0 +1,13 @@
+/*!40101 SET NAMES binary*/;
+INSERT INTO `quo``te/table` (`quo``te/col`,`a`) VALUES
+(0,10),
+(1,9),
+(2,8),
+(3,7),
+(4,6),
+(5,5),
+(6,4),
+(7,3),
+(8,2),
+(9,1),
+(10,0);

dumpling/tests/quote/run.sh

@@ -1,13 +1,23 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
 set -eu
 
 mkdir -p "$DUMPLING_OUTPUT_DIR"/data
-cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql"
-cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql"
-cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql"
+
+mysql_version=$(echo "select version()" | mysql -uroot -h127.0.0.1 -P3306 | awk 'NR==2' | awk '{print $1}')
+echo "current user mysql version is $mysql_version"
+if [[ $mysql_version = 5* ]]; then
+  # there is a bug in mysql 5.x, see https://bugs.mysql.com/bug.php?id=96994, so we use different create db/table sql
+  cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql"
+  cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql"
+  cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql"
+else
+  cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql"
+  cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql"
+  cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql"
+fi
 
 db="quo\`te/database"
 run_sql "drop database if exists \`quo\`\`te/database\`"

dumpling/tests/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
@@ -17,7 +17,6 @@ mkdir -p "$DUMPLING_TEST_DIR"
 PATH="tests/_utils:$PATH"
 . "tests/_utils/run_services"
 
-
 file_should_exist bin/tidb-server
 file_should_exist bin/tidb-lightning
 file_should_exist bin/dumpling
@@ -26,20 +25,34 @@ file_should_exist bin/sync_diff_inspector
 trap stop_services EXIT
 start_services
 
-for script in tests/*/run.sh; do
-    echo "****************** Running test $script..."
+run_case_by_fullpath() {
+    script="$1"
+    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Running test $script..."
     DUMPLING_BASE_NAME="$(dirname "$script")"
     export DUMPLING_BASE_NAME
     TEST_NAME="$(basename "$(dirname "$script")")"
     DUMPLING_OUTPUT_DIR="$DUMPLING_TEST_DIR"/sql_res."$TEST_NAME"
     export DUMPLING_OUTPUT_DIR
 
     PATH="tests/_utils:$PATH" \
-    sh "$script"
-
-    echo "Cleaning up test output dir: $DUMPLING_OUTPUT_DIR"
+        bash "$script"
+    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST: $script Passed Cleaning up test output dir: $DUMPLING_OUTPUT_DIR"
     rm -rf "$DUMPLING_OUTPUT_DIR"
-
-done
-
-echo "Passed integration tests."
+}
+
+if [ "$#" -ge 1 ]; then
+    test_case="$@"
+else
+    test_case="*"
+fi
+
+if [ "$test_case" == "*" ]; then
+    for script in tests/*/run.sh; do
+        run_case_by_fullpath "$script"
+    done
+else
+    script="tests/$test_case/run.sh"
+    run_case_by_fullpath "$script"
+fi
+
+echo "Passed integration tests."

dumpling/tests/views/run.sh

@@ -9,7 +9,8 @@ run_sql_file "$DUMPLING_BASE_NAME/data/views-schema-create.sql"
 export DUMPLING_TEST_DATABASE="views"
 
 run_sql "create table t (a bigint, b varchar(255))"
-run_sql "set session collation_connection='utf8mb4_general_ci'; create definer = 'root'@'localhost' view v as select * from t;"
+run_sql_file "$DUMPLING_BASE_NAME/data/views.v-schema-view.sql"
+
 # insert 20 records to `t`.
 run_sql "insert into t values $(seq -s, 20 | sed 's/,*$//g' | sed 's/[0-9]*/(\0,"\0")/g')"
 

dumpling/v4/export/config.go

@@ -733,3 +733,16 @@ func adjustFileFormat(conf *Config) error {
 	}
 	return nil
 }
+
+func matchMysqlBugversion(info ServerInfo) bool {
+	// if 8.0.3 <= mysql8 version < 8.0.23
+	// FLUSH TABLES WITH READ LOCK could block other sessions from executing SHOW TABLE STATUS.
+	// see more in https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-23.html
+	if info.ServerType != ServerTypeMySQL {
+		return false
+	}
+	currentVersion := info.ServerVersion
+	bugVersionStart := semver.New("8.0.2")
+	bugVersionEnd := semver.New("8.0.23")
+	return bugVersionStart.LessThan(*currentVersion) && currentVersion.LessThan(*bugVersionEnd)
+}

dumpling/v4/export/config_test.go

@@ -3,7 +3,7 @@
 package export
 
 import (
-	"context"
+	tcontext "github.com/pingcap/dumpling/v4/context"
 
 	. "github.com/pingcap/check"
 )
@@ -14,7 +14,24 @@ type testConfigSuite struct{}
 
 func (s *testConfigSuite) TestCreateExternalStorage(c *C) {
 	mockConfig := defaultConfigForTest(c)
-	loc, err := mockConfig.createExternalStorage(context.Background())
+	loc, err := mockConfig.createExternalStorage(tcontext.Background())
 	c.Assert(err, IsNil)
 	c.Assert(loc.URI(), Matches, "file:.*")
 }
+
+func (s *testConfigSuite) TestMatchMysqlBugversion(c *C) {
+	cases := []struct {
+		serverInfo ServerInfo
+		expected   bool
+	}{
+		{ParseServerInfo(tcontext.Background(), "5.7.25-TiDB-3.0.6"), false},
+		{ParseServerInfo(tcontext.Background(), "8.0.2"), false},
+		{ParseServerInfo(tcontext.Background(), "8.0.3"), true},
+		{ParseServerInfo(tcontext.Background(), "8.0.22"), true},
+		{ParseServerInfo(tcontext.Background(), "8.0.23"), false},
+	}
+	for _, x := range cases {
+		cmt := Commentf("server info %s", x.serverInfo)
+		c.Assert(x.expected, Equals, matchMysqlBugversion(x.serverInfo), cmt)
+	}
+}

dumpling/v4/export/dump.go

@@ -851,6 +851,19 @@ func extractTiDBRowIDFromDecodedKey(indexField, key string) (string, error) {
 	return "", errors.Errorf("decoded key %s doesn't have %s field", key, indexField)
 }
 
+func getListTableTypeByConf(conf *Config) listTableType {
+	// use listTableByShowTableStatus by default because it has better performance
+	listType := listTableByShowTableStatus
+	if conf.Consistency == consistencyTypeLock {
+		// for consistency lock, we need to build the tables to dump as soon as possible
+		listType = listTableByInfoSchema
+	} else if conf.Consistency == consistencyTypeFlush && matchMysqlBugversion(conf.ServerInfo) {
+		// For some buggy versions of mysql, we need a workaround to get a list of table names.
+		listType = listTableByShowFullTables
+	}
+	return listType
+}
+
 func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) error {
 	databases, err := prepareDumpingDatabases(conf, db)
 	if err != nil {
@@ -861,9 +874,7 @@ func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn)
 	if !conf.NoViews {
 		tableTypes = append(tableTypes, TableTypeView)
 	}
-	// for consistency lock, we need to build the tables to dump as soon as possible
-	asap := conf.Consistency == consistencyTypeLock
-	conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, asap, tableTypes...)
+	conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, getListTableTypeByConf(conf), tableTypes...)
 	if err != nil {
 		return err
 	}
@@ -1045,16 +1056,16 @@ func detectServerInfo(d *Dumper) error {
 // resolveAutoConsistency is an initialization step of Dumper.
 func resolveAutoConsistency(d *Dumper) error {
 	conf := d.conf
-	if conf.Consistency != "auto" {
+	if conf.Consistency != consistencyTypeAuto {
 		return nil
 	}
 	switch conf.ServerInfo.ServerType {
 	case ServerTypeTiDB:
-		conf.Consistency = "snapshot"
+		conf.Consistency = consistencyTypeSnapshot
 	case ServerTypeMySQL, ServerTypeMariaDB:
-		conf.Consistency = "flush"
+		conf.Consistency = consistencyTypeFlush
 	default:
-		conf.Consistency = "none"
+		conf.Consistency = consistencyTypeNone
 	}
 	return nil
 }

dumpling/v4/export/dump_test.go

@@ -88,3 +88,31 @@ func (s *testSQLSuite) TestDumpTableMeta(c *C) {
 		c.Assert(meta.HasImplicitRowID(), Equals, hasImplicitRowID)
 	}
 }
+
+func (s *testSQLSuite) TestGetListTableTypeByConf(c *C) {
+	conf := defaultConfigForTest(c)
+	tctx := tcontext.Background().WithLogger(appLogger)
+	cases := []struct {
+		serverInfo  ServerInfo
+		consistency string
+		expected    listTableType
+	}{
+		{ParseServerInfo(tctx, "5.7.25-TiDB-3.0.6"), consistencyTypeSnapshot, listTableByShowTableStatus},
+		// no bug version
+		{ParseServerInfo(tctx, "8.0.2"), consistencyTypeLock, listTableByInfoSchema},
+		{ParseServerInfo(tctx, "8.0.2"), consistencyTypeFlush, listTableByShowTableStatus},
+		{ParseServerInfo(tctx, "8.0.23"), consistencyTypeNone, listTableByShowTableStatus},
+
+		// bug version
+		{ParseServerInfo(tctx, "8.0.3"), consistencyTypeLock, listTableByInfoSchema},
+		{ParseServerInfo(tctx, "8.0.3"), consistencyTypeFlush, listTableByShowFullTables},
+		{ParseServerInfo(tctx, "8.0.3"), consistencyTypeNone, listTableByShowTableStatus},
+	}
+
+	for _, x := range cases {
+		conf.Consistency = x.consistency
+		conf.ServerInfo = x.serverInfo
+		cmt := Commentf("server info %s consistency %s", x.serverInfo, x.consistency)
+		c.Assert(getListTableTypeByConf(conf), Equals, x.expected, cmt)
+	}
+}