From 0acc68f6b7b02d33bb686738d915b7a590e26d4c Mon Sep 17 00:00:00 2001 From: sylzd Date: Thu, 25 Nov 2021 20:12:22 +0800 Subject: [PATCH 1/5] modify test case temporarily --- dumpling/tests/basic/run.sh | 177 +++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 85 deletions(-) diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh index 6caccce221433..7c427993e4da9 100644 --- a/dumpling/tests/basic/run.sh +++ b/dumpling/tests/basic/run.sh @@ -17,97 +17,104 @@ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (1), (2);" run_dumpling -f "$DB_NAME.$TABLE_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log -cnt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql|wc -l) -echo "records count is ${cnt}" -[ "$cnt" = 2 ] - -# make sure that dumpling log contains version infomation -cnt=$(grep -w "Welcome to dumpling.*Release Version.*Git Commit Hash.*Go Version" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -echo "version info count is ${cnt}" -[ "$cnt" = 1 ] - -# Test for simple WHERE case. -run_sql "drop database if exists \`$DB_NAME\`;" -run_sql "create database \`$DB_NAME\`;" -run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int);" - -seq 10 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (_);" - -run_dumpling --where "a >= 3 and a <= 9" -f "$DB_NAME.$TABLE_NAME" - -actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) -expected=$(seq 3 9) -echo "expected ${expected}, actual ${actual}" -[ "$actual" = "$expected" ] - -# Test for OR WHERE case. Better dump MySQL here because Dumpling has some special handle for concurrently dump TiDB tables. -export DUMPLING_TEST_PORT=3306 -run_sql "drop database if exists \`$DB_NAME\`;" -run_sql "create database \`$DB_NAME\`;" -run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int primary key, b int);" - -seq 0 99 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` (a,b) values (_, 99-_);" -run_sql "analyze table \`$DB_NAME\`.\`$TABLE_NAME\`;" -run_dumpling --where "b <= 4 or b >= 95" -f "$DB_NAME.$TABLE_NAME" --rows 10 - -actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) -expected=$(seq 0 4) -echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql ${expected}, actual ${actual}" -[ "$actual" = "$expected" ] -actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql | cut -c2-3) -expected=$(seq 95 99) -echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql ${expected}, actual ${actual}" -[ "$actual" = "$expected" ] - -seq 1 8 | xargs -I\? file_not_exist ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.00000000\?.sql - -# Test for specifying --filetype sql with --sql, should report an error -set +e -run_dumpling --sql "select * from \`$DB_NAME\`.\`$TABLE_NAME\`" --filetype sql > ${DUMPLING_OUTPUT_DIR}/dumpling.log -set -e - -actual=$(grep -w "unsupported config.FileType 'sql' when we specify --sql, please unset --filetype or set it to 'csv'" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -echo "expected 1 return error when specifying --filetype sql and --sql, actual ${actual}" -[ "$actual" = 1 ] - +#nt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql|wc -l) +#echo "records count is ${cnt}" +#[ "$cnt" = 2 ] +# +## make sure that dumpling log contains version infomation +#cnt=$(grep -w "Welcome to dumpling.*Release Version.*Git Commit Hash.*Go Version" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +#echo "version info count is ${cnt}" +#[ "$cnt" = 1 ] +# +## Test for simple WHERE case. +#run_sql "drop database if exists \`$DB_NAME\`;" +#run_sql "create database \`$DB_NAME\`;" +#run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int);" +# +#seq 10 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (_);" +# +#run_dumpling --where "a >= 3 and a <= 9" -f "$DB_NAME.$TABLE_NAME" +# +#actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) +#expected=$(seq 3 9) +#echo "expected ${expected}, actual ${actual}" +#[ "$actual" = "$expected" ] +# +## Test for OR WHERE case. Better dump MySQL here because Dumpling has some special handle for concurrently dump TiDB tables. +#export DUMPLING_TEST_PORT=3306 +#run_sql "drop database if exists \`$DB_NAME\`;" +#run_sql "create database \`$DB_NAME\`;" +#run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int primary key, b int);" +# +#seq 0 99 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` (a,b) values (_, 99-_);" +#run_sql "analyze table \`$DB_NAME\`.\`$TABLE_NAME\`;" +#run_dumpling --where "b <= 4 or b >= 95" -f "$DB_NAME.$TABLE_NAME" --rows 10 +# +#actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) +#expected=$(seq 0 4) +#echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql ${expected}, actual ${actual}" +#[ "$actual" = "$expected" ] +#actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql | cut -c2-3) +#expected=$(seq 95 99) +#echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql ${expected}, actual ${actual}" +#[ "$actual" = "$expected" ] +# +#seq 1 8 | xargs -I\? file_not_exist ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.00000000\?.sql +# +## Test for specifying --filetype sql with --sql, should report an error +#set +e +#run_dumpling --sql "select * from \`$DB_NAME\`.\`$TABLE_NAME\`" --filetype sql > ${DUMPLING_OUTPUT_DIR}/dumpling.log +#set -e +# +#actual=$(grep -w "unsupported config.FileType 'sql' when we specify --sql, please unset --filetype or set it to 'csv'" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +#echo "expected 1 return error when specifying --filetype sql and --sql, actual ${actual}" +#[ "$actual" = 1 ] +# export DUMPLING_TEST_PORT=4000 -# Test for --sql option. +## Test for --sql option. run_sql "drop database if exists \`$DB_NAME\`;" run_sql "create database \`$DB_NAME\`;" run_sql "create sequence \`$DB_NAME\`.\`$SEQUENCE_NAME\` increment by 1;" - +# run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)" +# +#actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) +#echo "expected 1, actual ${actual}" +#[ "$actual" = 1 ] +# +#run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)" +# +#actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) +#echo "expected 2, actual ${actual}" +#[ "$actual" = 2 ] -actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) +# Test for dump with sequence +run_dumpling | tee ${DUMPLING_OUTPUT_DIR}/dumpling.log +actual=$(grep -w "dump data successfully" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) echo "expected 1, actual ${actual}" -[ "$actual" = 1 ] - -run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)" - -actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) -echo "expected 2, actual ${actual}" -[ "$actual" = 2 ] +#[ "$actual" = 1 ] # Test for tidb_mem_quota_query configuration -export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/PrintTiDBMemQuotaQuery=1*return" -run_dumpling > ${DUMPLING_OUTPUT_DIR}/dumpling.log -actual=$(grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -echo "expected 1, actual ${actual}" -[ "$actual" = 1 ] - -export GO_FAILPOINTS="" - -# Test for wrong sql causing panic problem: https://github.com/pingcap/dumpling/pull/234#issuecomment-759996695 -set +e -run_dumpling --sql "test" > ${DUMPLING_OUTPUT_DIR}/dumpling.log 2> ${DUMPLING_OUTPUT_DIR}/dumpling.err -set -e - -# check stderr, should not contain panic info -actual=$(grep -w "panic" ${DUMPLING_OUTPUT_DIR}/dumpling.err|wc -l) -echo "expected panic 0, actual ${actual}" -[ "$actual" = 0 ] - -# check stdout, should contain mysql error log -actual=$(grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -echo "expect contain Error 1064, actual ${actual}" -[ "$actual" -ge 1 ] +#export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/PrintTiDBMemQuotaQuery=1*return" +#run_dumpling > ${DUMPLING_OUTPUT_DIR}/dumpling.log +#actual=$(grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +#echo "expected 1, actual ${actual}" +#[ "$actual" = 1 ] +# +#export GO_FAILPOINTS="" +# +## Test for wrong sql causing panic problem: https://github.com/pingcap/dumpling/pull/234#issuecomment-759996695 +#set +e +#run_dumpling --sql "test" > ${DUMPLING_OUTPUT_DIR}/dumpling.log 2> ${DUMPLING_OUTPUT_DIR}/dumpling.err +#set -e +# +## check stderr, should not contain panic info +#actual=$(grep -w "panic" ${DUMPLING_OUTPUT_DIR}/dumpling.err|wc -l) +#echo "expected panic 0, actual ${actual}" +#[ "$actual" = 0 ] +# +## check stdout, should contain mysql error log +#actual=$(grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +#echo "expect contain Error 1064, actual ${actual}" +#[ "$actual" -ge 1 ] +# From 33694f2b346b3b17b94a85050bc21d50574f5c85 Mon Sep 17 00:00:00 2001 From: sylzd Date: Thu, 25 Nov 2021 20:42:55 +0800 Subject: [PATCH 2/5] complete --- dumpling/export/dump.go | 12 ++- dumpling/export/prepare.go | 8 ++ dumpling/tests/basic/run.sh | 177 +++++++++++++++++----------------- executor/infoschema_reader.go | 1 + 4 files changed, 108 insertions(+), 90 deletions(-) diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go index 39637aa4f1b89..16efabd8a7de4 100755 --- a/dumpling/export/dump.go +++ b/dumpling/export/dump.go @@ -37,6 +37,7 @@ import ( var openDBFunc = sql.Open var emptyHandleValsErr = errors.New("empty handleVals for TiDB table") +var UnSupporedSequenceErr = errors.Errorf("unsupported tableType %s for dumping table MetaData", TableTypeSequenceStr) // Dumper is the dump progress structure type Dumper struct { @@ -316,7 +317,12 @@ func (d *Dumper) dumpDatabases(tctx *tcontext.Context, metaConn *sql.Conn, taskC zap.String("table", table.Name)) meta, err := dumpTableMeta(conf, metaConn, dbName, table) if err != nil { - return err + if err == UnSupporedSequenceErr { + tctx.L().Warn("fail to dump table meta: ", log.ShortError(err)) + continue + } else { + return err + } } if !conf.NoSchemas { @@ -890,6 +896,10 @@ func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) } func dumpTableMeta(conf *Config, conn *sql.Conn, db string, table *TableInfo) (TableMeta, error) { + if table.Type == TableTypeSequence { + return nil, UnSupporedSequenceErr + } + tbl := table.Name selectField, selectLen, err := buildSelectField(conn, db, tbl, conf.CompleteInsert) if err != nil { diff --git a/dumpling/export/prepare.go b/dumpling/export/prepare.go index fcc49cc6d7569..3f0e93acd4fca 100644 --- a/dumpling/export/prepare.go +++ b/dumpling/export/prepare.go @@ -116,6 +116,8 @@ const ( TableTypeBase TableType = iota // TableTypeView represents the view table TableTypeView + // TableTypeSequence represents the view table + TableTypeSequence ) const ( @@ -123,6 +125,8 @@ const ( TableTypeBaseStr = "BASE TABLE" // TableTypeViewStr represents the view table string TableTypeViewStr = "VIEW" + // TableTypeSequenceStr represents the view table string + TableTypeSequenceStr = "SEQUENCE" ) func (t TableType) String() string { @@ -131,6 +135,8 @@ func (t TableType) String() string { return TableTypeBaseStr case TableTypeView: return TableTypeViewStr + case TableTypeSequence: + return TableTypeSequenceStr default: return "UNKNOWN" } @@ -143,6 +149,8 @@ func ParseTableType(s string) (TableType, error) { return TableTypeBase, nil case TableTypeViewStr: return TableTypeView, nil + case TableTypeSequenceStr: + return TableTypeSequence, nil default: return TableTypeBase, errors.Errorf("unknown table type %s", s) } diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh index 7c427993e4da9..e162812587a8e 100644 --- a/dumpling/tests/basic/run.sh +++ b/dumpling/tests/basic/run.sh @@ -17,104 +17,103 @@ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (1), (2);" run_dumpling -f "$DB_NAME.$TABLE_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log -#nt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql|wc -l) -#echo "records count is ${cnt}" -#[ "$cnt" = 2 ] -# -## make sure that dumpling log contains version infomation -#cnt=$(grep -w "Welcome to dumpling.*Release Version.*Git Commit Hash.*Go Version" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -#echo "version info count is ${cnt}" -#[ "$cnt" = 1 ] -# -## Test for simple WHERE case. -#run_sql "drop database if exists \`$DB_NAME\`;" -#run_sql "create database \`$DB_NAME\`;" -#run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int);" -# -#seq 10 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (_);" -# -#run_dumpling --where "a >= 3 and a <= 9" -f "$DB_NAME.$TABLE_NAME" -# -#actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) -#expected=$(seq 3 9) -#echo "expected ${expected}, actual ${actual}" -#[ "$actual" = "$expected" ] -# -## Test for OR WHERE case. Better dump MySQL here because Dumpling has some special handle for concurrently dump TiDB tables. -#export DUMPLING_TEST_PORT=3306 -#run_sql "drop database if exists \`$DB_NAME\`;" -#run_sql "create database \`$DB_NAME\`;" -#run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int primary key, b int);" -# -#seq 0 99 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` (a,b) values (_, 99-_);" -#run_sql "analyze table \`$DB_NAME\`.\`$TABLE_NAME\`;" -#run_dumpling --where "b <= 4 or b >= 95" -f "$DB_NAME.$TABLE_NAME" --rows 10 -# -#actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) -#expected=$(seq 0 4) -#echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql ${expected}, actual ${actual}" -#[ "$actual" = "$expected" ] -#actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql | cut -c2-3) -#expected=$(seq 95 99) -#echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql ${expected}, actual ${actual}" -#[ "$actual" = "$expected" ] -# -#seq 1 8 | xargs -I\? file_not_exist ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.00000000\?.sql -# -## Test for specifying --filetype sql with --sql, should report an error -#set +e -#run_dumpling --sql "select * from \`$DB_NAME\`.\`$TABLE_NAME\`" --filetype sql > ${DUMPLING_OUTPUT_DIR}/dumpling.log -#set -e -# -#actual=$(grep -w "unsupported config.FileType 'sql' when we specify --sql, please unset --filetype or set it to 'csv'" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -#echo "expected 1 return error when specifying --filetype sql and --sql, actual ${actual}" -#[ "$actual" = 1 ] -# +cnt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql|wc -l) +echo "records count is ${cnt}" +[ "$cnt" = 2 ] + +# make sure that dumpling log contains version infomation +cnt=$(grep -w "Welcome to dumpling.*Release Version.*Git Commit Hash.*Go Version" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +echo "version info count is ${cnt}" +[ "$cnt" = 1 ] + +# Test for simple WHERE case. +run_sql "drop database if exists \`$DB_NAME\`;" +run_sql "create database \`$DB_NAME\`;" +run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int);" + +seq 10 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (_);" + +run_dumpling --where "a >= 3 and a <= 9" -f "$DB_NAME.$TABLE_NAME" + +actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) +expected=$(seq 3 9) +echo "expected ${expected}, actual ${actual}" +[ "$actual" = "$expected" ] + +# Test for OR WHERE case. Better dump MySQL here because Dumpling has some special handle for concurrently dump TiDB tables. +export DUMPLING_TEST_PORT=3306 +run_sql "drop database if exists \`$DB_NAME\`;" +run_sql "create database \`$DB_NAME\`;" +run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int primary key, b int);" + +seq 0 99 | xargs -I_ run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` (a,b) values (_, 99-_);" +run_sql "analyze table \`$DB_NAME\`.\`$TABLE_NAME\`;" +run_dumpling --where "b <= 4 or b >= 95" -f "$DB_NAME.$TABLE_NAME" --rows 10 + +actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql | cut -c2-2) +expected=$(seq 0 4) +echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql ${expected}, actual ${actual}" +[ "$actual" = "$expected" ] +actual=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql | cut -c2-3) +expected=$(seq 95 99) +echo "expected ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000009.sql ${expected}, actual ${actual}" +[ "$actual" = "$expected" ] + +seq 1 8 | xargs -I\? file_not_exist ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.00000000\?.sql + +# Test for specifying --filetype sql with --sql, should report an error +set +e +run_dumpling --sql "select * from \`$DB_NAME\`.\`$TABLE_NAME\`" --filetype sql > ${DUMPLING_OUTPUT_DIR}/dumpling.log +set -e + +actual=$(grep -w "unsupported config.FileType 'sql' when we specify --sql, please unset --filetype or set it to 'csv'" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +echo "expected 1 return error when specifying --filetype sql and --sql, actual ${actual}" +[ "$actual" = 1 ] + export DUMPLING_TEST_PORT=4000 -## Test for --sql option. +# Test for --sql option. run_sql "drop database if exists \`$DB_NAME\`;" run_sql "create database \`$DB_NAME\`;" run_sql "create sequence \`$DB_NAME\`.\`$SEQUENCE_NAME\` increment by 1;" -# + run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)" -# -#actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) -#echo "expected 1, actual ${actual}" -#[ "$actual" = 1 ] -# -#run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)" -# -#actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) -#echo "expected 2, actual ${actual}" -#[ "$actual" = 2 ] + +actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) +echo "expected 1, actual ${actual}" +[ "$actual" = 1 ] + +run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)" + +actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv) +echo "expected 2, actual ${actual}" +[ "$actual" = 2 ] # Test for dump with sequence run_dumpling | tee ${DUMPLING_OUTPUT_DIR}/dumpling.log actual=$(grep -w "dump data successfully" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) echo "expected 1, actual ${actual}" -#[ "$actual" = 1 ] +[ "$actual" = 1 ] # Test for tidb_mem_quota_query configuration -#export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/PrintTiDBMemQuotaQuery=1*return" -#run_dumpling > ${DUMPLING_OUTPUT_DIR}/dumpling.log -#actual=$(grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -#echo "expected 1, actual ${actual}" -#[ "$actual" = 1 ] -# -#export GO_FAILPOINTS="" -# -## Test for wrong sql causing panic problem: https://github.com/pingcap/dumpling/pull/234#issuecomment-759996695 -#set +e -#run_dumpling --sql "test" > ${DUMPLING_OUTPUT_DIR}/dumpling.log 2> ${DUMPLING_OUTPUT_DIR}/dumpling.err -#set -e -# -## check stderr, should not contain panic info -#actual=$(grep -w "panic" ${DUMPLING_OUTPUT_DIR}/dumpling.err|wc -l) -#echo "expected panic 0, actual ${actual}" -#[ "$actual" = 0 ] -# -## check stdout, should contain mysql error log -#actual=$(grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -#echo "expect contain Error 1064, actual ${actual}" -#[ "$actual" -ge 1 ] -# +export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/PrintTiDBMemQuotaQuery=1*return" +run_dumpling > ${DUMPLING_OUTPUT_DIR}/dumpling.log +actual=$(grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +echo "expected 1, actual ${actual}" +[ "$actual" = 1 ] + +export GO_FAILPOINTS="" + +# Test for wrong sql causing panic problem: https://github.com/pingcap/dumpling/pull/234#issuecomment-759996695 +set +e +run_dumpling --sql "test" > ${DUMPLING_OUTPUT_DIR}/dumpling.log 2> ${DUMPLING_OUTPUT_DIR}/dumpling.err +set -e + +# check stderr, should not contain panic info +actual=$(grep -w "panic" ${DUMPLING_OUTPUT_DIR}/dumpling.err|wc -l) +echo "expected panic 0, actual ${actual}" +[ "$actual" = 0 ] + +# check stdout, should contain mysql error log +actual=$(grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +echo "expect contain Error 1064, actual ${actual}" +[ "$actual" -ge 1 ] diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 483e970d479fc..e305aaf5a039d 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -578,6 +578,7 @@ func (e *memtableRetriever) setDataFromTables(ctx context.Context, sctx sessionc if rowCount == 0 { rowCount = 1 } + table.Comment = "SEQUENCE" } if table.PKIsHandle || table.IsCommonHandle { pkType = "CLUSTERED" From cccec705deef5320f103cdd0f62c8535923ed414 Mon Sep 17 00:00:00 2001 From: sylzd Date: Mon, 29 Nov 2021 11:10:51 +0800 Subject: [PATCH 3/5] change the bugfix method by review --- dumpling/export/dump.go | 27 +++++++++++++++------------ dumpling/export/prepare.go | 1 + dumpling/export/sql.go | 13 +++++++++++++ dumpling/export/sql_test.go | 29 +++++++++++++++++++++++++++++ dumpling/tests/basic/run.sh | 6 +++--- executor/infoschema_reader.go | 1 - 6 files changed, 61 insertions(+), 16 deletions(-) diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go index 16efabd8a7de4..9ea0c10993245 100755 --- a/dumpling/export/dump.go +++ b/dumpling/export/dump.go @@ -37,7 +37,6 @@ import ( var openDBFunc = sql.Open var emptyHandleValsErr = errors.New("empty handleVals for TiDB table") -var UnSupporedSequenceErr = errors.Errorf("unsupported tableType %s for dumping table MetaData", TableTypeSequenceStr) // Dumper is the dump progress structure type Dumper struct { @@ -317,12 +316,7 @@ func (d *Dumper) dumpDatabases(tctx *tcontext.Context, metaConn *sql.Conn, taskC zap.String("table", table.Name)) meta, err := dumpTableMeta(conf, metaConn, dbName, table) if err != nil { - if err == UnSupporedSequenceErr { - tctx.L().Warn("fail to dump table meta: ", log.ShortError(err)) - continue - } else { - return err - } + return err } if !conf.NoSchemas { @@ -886,7 +880,20 @@ func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) if !conf.NoViews { tableTypes = append(tableTypes, TableTypeView) } - conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, getListTableTypeByConf(conf), tableTypes...) + + ifSeqExists, err := CheckIfSeqExists(db) + if err != nil { + return err + } + var listType listTableType + if ifSeqExists { + tctx.L().Warn("dumpling tableType `sequence` is unsupported for now") + listType = listTableByShowFullTables + } else { + listType = getListTableTypeByConf(conf) + } + + conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, listType, tableTypes...) if err != nil { return err } @@ -896,10 +903,6 @@ func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) } func dumpTableMeta(conf *Config, conn *sql.Conn, db string, table *TableInfo) (TableMeta, error) { - if table.Type == TableTypeSequence { - return nil, UnSupporedSequenceErr - } - tbl := table.Name selectField, selectLen, err := buildSelectField(conn, db, tbl, conf.CompleteInsert) if err != nil { diff --git a/dumpling/export/prepare.go b/dumpling/export/prepare.go index 3f0e93acd4fca..777b366c65a78 100644 --- a/dumpling/export/prepare.go +++ b/dumpling/export/prepare.go @@ -117,6 +117,7 @@ const ( // TableTypeView represents the view table TableTypeView // TableTypeSequence represents the view table + // TODO: need to be supported TableTypeSequence ) diff --git a/dumpling/export/sql.go b/dumpling/export/sql.go index 4b0203a1665da..9a984acaaa599 100644 --- a/dumpling/export/sql.go +++ b/dumpling/export/sql.go @@ -691,6 +691,19 @@ func CheckTiDBWithTiKV(db *sql.DB) (bool, error) { return count > 0, nil } +// CheckIfSeqExists use sql to check whether sequence exists +func CheckIfSeqExists(db *sql.Conn) (bool, error) { + var count int + const query = "SELECT COUNT(1) as c FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='SEQUENCE'" + row := db.QueryRowContext(context.Background(), query) + err := row.Scan(&count) + if err != nil { + return false, errors.Annotatef(err, "sql: %s", query) + } + + return count > 0, nil +} + // CheckTiDBEnableTableLock use sql variable to check whether current TiDB has TiKV func CheckTiDBEnableTableLock(db *sql.Conn) (bool, error) { tidbConfig, err := getTiDBConfig(db) diff --git a/dumpling/export/sql_test.go b/dumpling/export/sql_test.go index b6bd835d27e35..c94469d45090c 100644 --- a/dumpling/export/sql_test.go +++ b/dumpling/export/sql_test.go @@ -1692,6 +1692,35 @@ func TestPickupPossibleField(t *testing.T) { } } +func TestCheckIfSeqExists(t *testing.T) { + t.Parallel() + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() + + conn, err := db.Conn(context.Background()) + require.NoError(t, err) + + mock.ExpectQuery("SELECT COUNT"). + WillReturnRows(sqlmock.NewRows([]string{"c"}). + AddRow("1")) + + exists, err := CheckIfSeqExists(conn) + require.NoError(t, err) + require.Equal(t, true, exists) + + mock.ExpectQuery("SELECT COUNT"). + WillReturnRows(sqlmock.NewRows([]string{"c"}). + AddRow("0")) + + exists, err = CheckIfSeqExists(conn) + require.NoError(t, err) + require.Equal(t, false, exists) +} + func makeVersion(major, minor, patch int64, preRelease string) *semver.Version { return &semver.Version{ Major: major, diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh index e162812587a8e..5eccbb77514e5 100644 --- a/dumpling/tests/basic/run.sh +++ b/dumpling/tests/basic/run.sh @@ -90,9 +90,9 @@ echo "expected 2, actual ${actual}" # Test for dump with sequence run_dumpling | tee ${DUMPLING_OUTPUT_DIR}/dumpling.log -actual=$(grep -w "dump data successfully" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) -echo "expected 1, actual ${actual}" -[ "$actual" = 1 ] +actual=$(grep -w "dump failed" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +echo "expected 0, actual ${actual}" +[ "$actual" = 0 ] # Test for tidb_mem_quota_query configuration export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/PrintTiDBMemQuotaQuery=1*return" diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index e305aaf5a039d..483e970d479fc 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -578,7 +578,6 @@ func (e *memtableRetriever) setDataFromTables(ctx context.Context, sctx sessionc if rowCount == 0 { rowCount = 1 } - table.Comment = "SEQUENCE" } if table.PKIsHandle || table.IsCommonHandle { pkType = "CLUSTERED" From 2c3104198bcaeab2189a40bff68ffbc3ec26da8f Mon Sep 17 00:00:00 2001 From: sylzd Date: Sun, 5 Dec 2021 20:37:29 +0800 Subject: [PATCH 4/5] resolve conflicts --- dumpling/export/sql_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/dumpling/export/sql_test.go b/dumpling/export/sql_test.go index 86e626ac5f0fb..49d954f59e5b1 100644 --- a/dumpling/export/sql_test.go +++ b/dumpling/export/sql_test.go @@ -1772,13 +1772,11 @@ func TestCheckIfSeqExists(t *testing.T) { func TestGetCharsetAndDefaultCollation(t *testing.T) { t.Parallel() - db, mock, err := sqlmock.New() require.NoError(t, err) defer func() { require.NoError(t, db.Close()) }() - ctx := context.Background() conn, err := db.Conn(ctx) require.NoError(t, err) From 36b19e5fd9c659ff696b91920b26ed80c193d62f Mon Sep 17 00:00:00 2001 From: sylzd Date: Mon, 6 Dec 2021 11:25:06 +0800 Subject: [PATCH 5/5] Update dumpling/export/sql_test.go Co-authored-by: Chunzhu Li --- dumpling/export/sql_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dumpling/export/sql_test.go b/dumpling/export/sql_test.go index 49d954f59e5b1..44ba39ec01d8c 100644 --- a/dumpling/export/sql_test.go +++ b/dumpling/export/sql_test.go @@ -1750,6 +1750,12 @@ func TestPickupPossibleField(t *testing.T) { func TestCheckIfSeqExists(t *testing.T) { t.Parallel() + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() + conn, err := db.Conn(context.Background()) require.NoError(t, err)