From 4fa0b6a51ab184b8d89dbcc0a8ed1ea62127b589 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 13 Jun 2022 11:41:29 +0800 Subject: [PATCH 01/21] add a new DDL related integration test --- .../ddl/multi_alter_with_write.test | 749 ++++++++++++++++++ 1 file changed, 749 insertions(+) create mode 100644 tests/fullstack-test2/ddl/multi_alter_with_write.test diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test new file mode 100644 index 00000000000..c60ebcb02fb --- /dev/null +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -0,0 +1,749 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this test focus on the case when multi DDL actions happen closely（ and these DDL actions will be fetched in the same regular sync schema duration.) and there are some corresponding insert(write) actions between these DDL actions. Considering that these write actions and these schema change will arrive at tiflash in a different order, we simulate these different order situation to check that our schema module was working correctly. + +# TiDB Timeline ： write cmd 1 ｜ alter cmd 1 ｜ write cmd 2 | alter cmd 2 | write cmd 3 + +# stop regular schema sync +=> DBGInvoke __enable_schema_sync_service('false') + +# Enable the failpoint and make it pause before applying the raft cmd to write a row +>> DBGInvoke __init_fail_point() +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + + +# ----------------------------------------------------------------------------- +# Order 1 : write cmd 1 | alter cmd 1 | write cmd 2 | alter cmd 2 | write cmd 3 +# ----------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +│ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make alter cmd 2 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + + +# --------------------------------------------------------------------------------------------- +# Order 2 : write cmd 1 | alter cmd 1 | write cmd 2 | write cmd 3 --> sync schema(alter cmd 2) +# --------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +│ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +# check what happen after write cmd 3 --> call sync schema and get alter cmd 2 happen +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# ----------------------------------------------------------------------------------------------- +# Order 3 : write cmd 1 | alter cmd 1 | alter cmd 2 | write cmd 2 -->sync schema() | write cmd 3 +# ----------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make alter cmd 2 take effect +>> DBGInvoke __refresh_schemas() + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# ----------------------------------------------------------------------------------------------- +# Order 4 : write cmd 1 | write cmd 2 --> sync schema(alter cmd 1) | alter cmd 2 | write cmd 3 +# ----------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# check no schema change before write cmd 2 take effect +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +│ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make alter cmd 2 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# --------------------------------------------------------------------------------------------------------------------- +# Order 5 : write cmd 1 | write cmd 2 --> sync schema(alter cmd 1) | write cmd 3 --> sync schema(alter cmd 2) +# ---------------------------------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +# check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 happened. +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +│ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ +└─────┴──────┴───────┴─────┴─────┘ + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +# check what happen after write cmd 3 --> should call sync schema, get the alter cmd 2 happened. +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# ----------------------------------------------------------------------------------------------- +# Order 6 : write cmd 1 | write cmd 2 --> sync schema(alter cmd 1 alter cmd 2) | write cmd 3 +# ----------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 1 │ 4.50 │ abc │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +# check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 && alter cmd 2 happened. +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + + +# ------------------------------------------------------------------------------- +# Order 7 : alter cmd 1 | write cmd 1 | write cmd 2 | alter cmd 2 | write cmd 3 +# ------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + + +# add a new pre write to make check the alter cmd 1 more convenient. +mysql> insert into test.t (a, b, c) values (0, 0, ' '); + +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 0 │ 0.00 │ │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +│ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make alter cmd 2 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# -------------------------------------------------------------------------------------------------- +# Order 8 : alter cmd 1 | write cmd 1 | write cmd 2 | write cmd 3 --> sync schema(alter cmd 2) +# -------------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# add a new pre write to make check the alter cmd 1 more convenient. +mysql> insert into test.t (a, b, c) values (0, 0, ' '); + +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 0 │ 0.00 │ │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +│ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +# check what happen after write cmd 3 --> should call sync schema, get the alter cmd 2 happened. +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# -------------------------------------------------------------------------------------------------- +# Order 9 : alter cmd 1 | write cmd 1 | alter cmd 2 | write cmd 2 -->sync schema() | write cmd 3 +# -------------------------------------------------------------------------------------------------- + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + +# add a new pre write to make check the alter cmd 1 more convenient. +mysql> insert into test.t (a, b, c) values (0, 0, ' '); + +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 0 │ 0.00 │ │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# make write cmd 1 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┬─e───┐ +│ 0 │ 0.00 │ │ 0 │ \N │ +│ 1 │ 4.50 │ abc │ 0 │ \N │ +└─────┴──────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make alter cmd 2 take effect +>> DBGInvoke __refresh_schemas() + +# make write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# ------------------------------------------------------------------------------------------------------------------ +# Order 10 : alter cmd 1 | alter cmd 2 | write cmd 1 -->sync schema() | write cmd 2 -->sync schema() | write cmd 3 +# ------------------------------------------------------------------------------------------------------------------ + +mysql> drop table if exists test.t +mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); + +mysql> alter table test.t set tiflash replica 1; +mysql> set session tidb_isolation_read_engines='tiflash'; + + +# add a new pre write to make check the alter cmd 1 more convenient. +mysql> insert into test.t (a, b, c) values (0, 0, ' '); + +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─b────┬─c─────┬─d───┐ +│ 0 │ 0.00 │ │ 0 │ +└─────┴──────┴───────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# write cmd 1 +mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); + +# alter cmd 1 +mysql> alter table test.t add column e decimal(6,1) NULL; + +# make alter cmd 1 take effect +>> DBGInvoke __refresh_schemas() + +# write cmd 2 +mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); + +# alter cmd 2 +mysql> alter table test.t drop column b; + +# make alter cmd 2 take effect +>> DBGInvoke __refresh_schemas() + +# make write cmd 1 and write cmd 2 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +└─────┴───────┴─────┴─────┘ + +# write cmd 3 +mysql> insert into test.t values (4, 'abcd', 10, 0.2); + +# make write cmd 3 take effect +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +┌─a───┬─c─────┬─d───┬─e───┐ +│ 0 │ │ 0 │ \N │ +│ 1 │ abc │ 0 │ \N │ +│ 3 │ ccc │ 3 │ 0.1 │ +│ 4 │ abcd │ 10 │ 0.2 │ +└─────┴───────┴─────┴─────┘ + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) \ No newline at end of file From 3b53512bd37faa74f298a49f69ca92addaecbc50 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 13 Jun 2022 11:42:08 +0800 Subject: [PATCH 02/21] add a new DDL related integration test --- tests/fullstack-test2/ddl/multi_alter_with_write.test | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index c60ebcb02fb..b862d23c91b 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -23,7 +23,6 @@ >> DBGInvoke __init_fail_point() >> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) - # ----------------------------------------------------------------------------- # Order 1 : write cmd 1 | alter cmd 1 | write cmd 2 | alter cmd 2 | write cmd 3 # ----------------------------------------------------------------------------- @@ -197,7 +196,6 @@ mysql> alter table test.t add column e decimal(6,1) NULL; │ 1 │ 4.50 │ abc │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ - # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); From 0a922d9bd1220f1ef1c0df197d9d098f93e33887 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 13 Jun 2022 15:13:26 +0800 Subject: [PATCH 03/21] add write_table in test to make each action take effect --- .../ddl/multi_alter_with_write.test | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index b862d23c91b..eb7e5e20b43 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -39,6 +39,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ @@ -52,6 +54,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -63,6 +67,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -77,6 +83,8 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -89,6 +97,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -115,6 +125,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ @@ -126,6 +138,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -137,6 +151,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -154,6 +170,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + # check what happen after write cmd 3 --> call sync schema and get alter cmd 2 happen => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ @@ -180,6 +198,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ @@ -191,6 +211,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -205,9 +227,13 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -222,6 +248,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -247,6 +275,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ @@ -269,6 +299,9 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + +# check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 happened. => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -283,6 +316,8 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -295,6 +330,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -320,6 +357,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ @@ -336,6 +375,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + # check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 happened. => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ @@ -352,6 +393,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + # check what happen after write cmd 3 --> should call sync schema, get the alter cmd 2 happened. => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ @@ -378,6 +421,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ @@ -397,6 +442,8 @@ mysql> alter table test.t drop column b; # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + # check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 && alter cmd 2 happened. => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ @@ -412,6 +459,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ @@ -438,6 +487,8 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ @@ -452,6 +503,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -463,6 +516,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -477,6 +532,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -492,6 +549,8 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ @@ -505,6 +564,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ @@ -543,6 +604,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -554,6 +617,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -568,6 +633,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -586,6 +653,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + # check what happen after write cmd 3 --> should call sync schema, get the alter cmd 2 happened. => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ @@ -612,6 +681,8 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ @@ -625,6 +696,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -636,6 +709,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -653,9 +728,13 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ @@ -669,6 +748,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ @@ -695,6 +776,8 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ @@ -711,6 +794,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); @@ -720,9 +805,13 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +func> wait_table test t + # make write cmd 1 and write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ @@ -736,6 +825,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +func> wait_table test t + => DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ From 8fc54917b395e11e69ac794b9542f9278bbc507b Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 13 Jun 2022 16:09:48 +0800 Subject: [PATCH 04/21] add some code for test --- .../Storages/Transaction/PartitionStreams.cpp | 6 + .../ddl/multi_alter_with_write.test | 180 +++++------------- 2 files changed, 51 insertions(+), 135 deletions(-) diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 4b2ca6c07a8..78ee4fc6e7a 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -164,10 +164,14 @@ static void writeRegionDataToStorage( /// decoding data. Check the test case for more details. FAIL_POINT_PAUSE(FailPoints::pause_before_apply_raft_cmd); + LOG_FMT_TRACE(log, "[for test only] begin decode and write"); /// Try read then write once. { if (atomic_read_write(false)) + { + LOG_FMT_TRACE(log, "[for test only] first decode sucess"); return; + } } /// If first try failed, sync schema and force read then write. @@ -180,6 +184,8 @@ static void writeRegionDataToStorage( // TODO: Enrich exception message. throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", ErrorCodes::LOGICAL_ERROR); + + LOG_FMT_TRACE(log, "[for test only] second decode sucess"); } } diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index eb7e5e20b43..9bf77994b48 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -39,9 +39,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -54,9 +52,7 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ @@ -67,9 +63,7 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ │ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ @@ -83,9 +77,7 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -97,9 +89,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -108,7 +98,6 @@ func> wait_table test t >> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) - # --------------------------------------------------------------------------------------------- # Order 2 : write cmd 1 | alter cmd 1 | write cmd 2 | write cmd 3 --> sync schema(alter cmd 2) # --------------------------------------------------------------------------------------------- @@ -125,9 +114,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -138,9 +125,7 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ @@ -151,9 +136,7 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ │ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ @@ -170,10 +153,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - # check what happen after write cmd 3 --> call sync schema and get alter cmd 2 happen -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -198,9 +179,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -211,9 +190,7 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ @@ -227,14 +204,10 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -248,9 +221,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -275,9 +246,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -288,7 +257,7 @@ func> wait_table test t mysql> alter table test.t add column e decimal(6,1) NULL; # check no schema change before write cmd 2 take effect -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -299,10 +268,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - # check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 happened. -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ │ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ @@ -316,9 +283,7 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -330,9 +295,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -357,9 +320,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -375,15 +336,15 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - # check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 happened. -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 1 │ 4.50 │ abc │ 0 │ \N │ │ 3 │ 0.20 │ ccc │ 3 │ 0.1 │ └─────┴──────┴───────┴─────┴─────┘ +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + # alter cmd 2 mysql> alter table test.t drop column b; @@ -393,10 +354,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - # check what happen after write cmd 3 --> should call sync schema, get the alter cmd 2 happened. -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -421,9 +380,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ @@ -442,10 +399,8 @@ mysql> alter table test.t drop column b; # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - # check what happen after write cmd 2 --> should call sync schema, get the alter cmd 1 && alter cmd 2 happened. -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -459,9 +414,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 1 │ abc │ 0 │ \N │ │ 3 │ ccc │ 3 │ 0.1 │ @@ -470,7 +423,6 @@ func> wait_table test t >> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) - # ------------------------------------------------------------------------------- # Order 7 : alter cmd 1 | write cmd 1 | write cmd 2 | alter cmd 2 | write cmd 3 # ------------------------------------------------------------------------------- @@ -487,9 +439,7 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ └─────┴──────┴───────┴─────┘ @@ -503,9 +453,7 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ @@ -516,9 +464,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -532,9 +478,7 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -549,9 +493,7 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ @@ -564,9 +506,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ @@ -591,7 +531,7 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ └─────┴──────┴───────┴─────┘ @@ -604,9 +544,7 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ @@ -617,9 +555,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -633,9 +569,7 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -653,10 +587,8 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - # check what happen after write cmd 3 --> should call sync schema, get the alter cmd 2 happened. -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ @@ -681,9 +613,7 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ └─────┴──────┴───────┴─────┘ @@ -696,9 +626,7 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ └─────┴──────┴───────┴─────┴─────┘ @@ -709,9 +637,7 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ │ 1 │ 4.50 │ abc │ 0 │ \N │ @@ -728,14 +654,10 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ @@ -748,9 +670,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ @@ -776,9 +696,7 @@ mysql> insert into test.t (a, b, c) values (0, 0, ' '); >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┐ │ 0 │ 0.00 │ │ 0 │ └─────┴──────┴───────┴─────┘ @@ -794,8 +712,6 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # make alter cmd 1 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); @@ -805,14 +721,10 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -func> wait_table test t - # make write cmd 1 and write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ @@ -825,9 +737,7 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) -func> wait_table test t - -=> DBGInvoke query_mapped('selraw * from \$d.\$t', test, t) +=> DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─c─────┬─d───┬─e───┐ │ 0 │ │ 0 │ \N │ │ 1 │ abc │ 0 │ \N │ From 654c7eb11b5b3116caa9b46a6b0d5f63d5b64b10 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 11:57:49 +0800 Subject: [PATCH 05/21] add wait_table to wait create table in tiflash --- .../ddl/multi_alter_with_write.test | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 9bf77994b48..952346bc99d 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -33,6 +33,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); @@ -108,6 +110,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); @@ -173,6 +177,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); @@ -240,6 +246,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); @@ -314,6 +322,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); @@ -374,6 +384,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); @@ -433,6 +445,7 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); @@ -526,6 +539,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); @@ -608,6 +623,8 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t + # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); @@ -690,6 +707,7 @@ mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar mysql> alter table test.t set tiflash replica 1; mysql> set session tidb_isolation_read_engines='tiflash'; +func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); From 46890f669713ae9ebb7d5840cacff13a4fb9d827 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 18:09:22 +0800 Subject: [PATCH 06/21] add failpoint to make query begin until the write action finished --- dbms/src/Common/FailPoint.cpp | 5 ++- dbms/src/Interpreters/executeQuery.cpp | 11 +++++ .../Storages/Transaction/PartitionStreams.cpp | 45 +++++++++++++++++-- .../ddl/multi_alter_with_write.test | 3 ++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index c6c3caa44ad..0bccdaae286 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -87,6 +87,7 @@ std::unordered_map> FailPointHelper::f M(force_slow_page_storage_snapshot_release) \ M(force_change_all_blobs_to_read_only) + #define APPLY_FOR_FAILPOINTS_ONCE_WITH_CHANNEL(M) \ M(pause_with_alter_locks_acquired) \ M(hang_in_execution) \ @@ -103,7 +104,9 @@ std::unordered_map> FailPointHelper::f M(pause_when_ingesting_to_dt_store) \ M(pause_when_altering_dt_store) \ M(pause_after_copr_streams_acquired) \ - M(pause_before_server_merge_one_delta) + M(pause_before_server_merge_one_delta) \ + M(pause_query_until_write_finish) \ + M(force_pause_query_until_write_finish) namespace FailPoints { diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 96cfc0a58ae..cc38cd23a52 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -54,6 +55,11 @@ extern const int QUERY_IS_TOO_LARGE; extern const int INTO_OUTFILE_NOT_ALLOWED; } // namespace ErrorCodes +namespace FailPoints +{ +extern const char pause_query_until_write_finish[]; +} // namespace FailPoints + namespace { void checkASTSizeLimits(const IAST & ast, const Settings & settings) @@ -415,6 +421,11 @@ BlockIO executeQuery( bool internal, QueryProcessingStage::Enum stage) { + /// Failpoint to make our query begin after the write action finish. + FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); + // auto execute_query_logger = getLogger(context); + // LOG_FMT_INFO(execute_query_logger, "[for test only] Begin ExecuteQuery with query {}", query); + BlockIO streams; SQLQuerySource query_src(query.data(), query.data() + query.size()); std::tie(std::ignore, streams) = executeQueryImpl(query_src, context, internal, stage); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 78ee4fc6e7a..07dc8d2a119 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -32,6 +32,8 @@ #include #include +#include "fiu.h" + namespace DB { namespace FailPoints @@ -39,6 +41,8 @@ namespace FailPoints extern const char pause_before_apply_raft_cmd[]; extern const char pause_before_apply_raft_snapshot[]; extern const char force_set_safepoint_when_decode_block[]; +extern const char force_pause_query_until_write_finish[]; +extern const char pause_query_until_write_finish[]; } // namespace FailPoints namespace ErrorCodes @@ -150,6 +154,7 @@ static void writeRegionDataToStorage( default: throw Exception("Unknown StorageEngine: " + toString(static_cast(storage->engineType())), ErrorCodes::LOGICAL_ERROR); } + write_part_cost = watch.elapsedMilliseconds(); GET_METRIC(tiflash_raft_write_data_to_storage_duration_seconds, type_write).Observe(write_part_cost / 1000.0); if (need_decode) @@ -159,17 +164,37 @@ static void writeRegionDataToStorage( return true; }; + /// In TiFlash, the actions between applying raft log and schema changes are not strictly synchronized. /// There could be a chance that some raft logs come after a table gets tombstoned. Take care of it when /// decoding data. Check the test case for more details. - FAIL_POINT_PAUSE(FailPoints::pause_before_apply_raft_cmd); + fiu_do_on( + FailPoints::pause_before_apply_raft_cmd, + { + FailPointHelper::wait(FailPoints::pause_before_apply_raft_cmd); + + /// we add force_pause_query_until_write_finish failpoint to enable pause_query_until_write_finish here and disable it when write finish to make the query happend until the write totally finished. + fiu_do_on(FailPoints::force_pause_query_until_write_finish, { + FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); + //LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); + }); + }); + + + // LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); - LOG_FMT_TRACE(log, "[for test only] begin decode and write"); + // ::sleep(3); + + // LOG_FMT_INFO(log, "[for test only] begin decode and write"); /// Try read then write once. { if (atomic_read_write(false)) { - LOG_FMT_TRACE(log, "[for test only] first decode sucess"); + //LOG_FMT_INFO(log, "[for test only] first decode sucess"); + + fiu_do_on(FailPoints::force_pause_query_until_write_finish, { + FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); + }); return; } } @@ -180,12 +205,24 @@ static void writeRegionDataToStorage( tmt.getSchemaSyncer()->syncSchemas(context); if (!atomic_read_write(true)) + { // Failure won't be tolerated this time. // TODO: Enrich exception message. + fiu_do_on(FailPoints::force_pause_query_until_write_finish, { + FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); + }); + throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", ErrorCodes::LOGICAL_ERROR); + } + + + //LOG_FMT_INFO(log, "[for test only] second decode sucess"); - LOG_FMT_TRACE(log, "[for test only] second decode sucess"); + fiu_do_on(FailPoints::force_pause_query_until_write_finish, { + FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); + //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + }); } } diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 952346bc99d..69d1ba75724 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -23,6 +23,9 @@ >> DBGInvoke __init_fail_point() >> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) +# Enable the failpoint to make our query only start when the write action finished +>> DBGInvoke __enable_fail_point(force_pause_query_until_write_finish) + # ----------------------------------------------------------------------------- # Order 1 : write cmd 1 | alter cmd 1 | write cmd 2 | alter cmd 2 | write cmd 3 # ----------------------------------------------------------------------------- From 6774ccdf924f70dc41e86b9a762f8aa9c2e9f99c Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 21:11:00 +0800 Subject: [PATCH 07/21] change the failpoint --- dbms/src/Common/FailPoint.cpp | 8 +++++ .../Interpreters/InterpreterSelectQuery.cpp | 16 ++++++++++ dbms/src/Interpreters/executeQuery.cpp | 6 +--- .../Storages/Transaction/PartitionStreams.cpp | 29 ++++++++++--------- .../ddl/multi_alter_with_write.test | 8 +++++ 5 files changed, 49 insertions(+), 18 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index 0bccdaae286..3dd8636321c 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -127,14 +127,20 @@ class FailPointChannel : private boost::noncopyable void wait() { + //std::cout << "[for test only] wait line 130" << std::endl; std::unique_lock lock(m); + //std::cout << "[for test only] wait line 132" << std::endl; cv.wait(lock); + //std::cout << "[for test only] wait line 134" << std::endl; } void notifyAll() { + //std::cout << "[for test only] notifyAll line 139" << std::endl; std::unique_lock lock(m); + //std::cout << "[for test only] notifyAll line 141" << std::endl; cv.notify_all(); + //std::cout << "[for test only] notifyAll line 143" << std::endl; } private: @@ -183,6 +189,7 @@ void FailPointHelper::enableFailPoint(const String & fail_point_name) void FailPointHelper::disableFailPoint(const String & fail_point_name) { + //std::cout << "[for test only] FailPointHelper::disableFailPoint " << fail_point_name << std::endl; if (auto iter = fail_point_wait_channels.find(fail_point_name); iter != fail_point_wait_channels.end()) { /// can not rely on deconstruction to do the notify_all things, because @@ -195,6 +202,7 @@ void FailPointHelper::disableFailPoint(const String & fail_point_name) void FailPointHelper::wait(const String & fail_point_name) { + //std::cout << "[for test only] FailPointHelper::wait " << fail_point_name << std::endl; if (auto iter = fail_point_wait_channels.find(fail_point_name); iter == fail_point_wait_channels.end()) throw Exception("Can not find channel for fail point " + fail_point_name); else diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 51b55f65bd4..a7a07917b53 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -93,6 +94,12 @@ extern const int SCHEMA_VERSION_ERROR; extern const int UNKNOWN_EXCEPTION; } // namespace ErrorCodes + +namespace FailPoints +{ +extern const char pause_query_until_write_finish[]; +} // namespace FailPoints + InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const Context & context_, @@ -131,6 +138,15 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(const Names & required_result_column_names) { + /// Failpoint to make our query begin after the write action finish. + FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); + // fiu_do_on(FailPoints::pause_query_until_write_finish, { + // //LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); + // FailPointHelper::wait(FailPoints::pause_query_until_write_finish); + // }); + + //LOG_FMT_INFO(log, "[for test only] Begin InterpreterSelectQuery"); + ProfileEvents::increment(ProfileEvents::SelectQuery); if (!context.hasQueryContext()) diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index cc38cd23a52..608905e6d47 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -421,11 +422,6 @@ BlockIO executeQuery( bool internal, QueryProcessingStage::Enum stage) { - /// Failpoint to make our query begin after the write action finish. - FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); - // auto execute_query_logger = getLogger(context); - // LOG_FMT_INFO(execute_query_logger, "[for test only] Begin ExecuteQuery with query {}", query); - BlockIO streams; SQLQuerySource query_src(query.data(), query.data() + query.size()); std::tie(std::ignore, streams) = executeQueryImpl(query_src, context, internal, stage); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 07dc8d2a119..490030dbd2e 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -165,27 +165,28 @@ static void writeRegionDataToStorage( }; + /// we add force_pause_query_until_write_finish failpoint to enable pause_query_until_write_finish here and disable it when write finish to make the query happend until the write totally finished. + fiu_do_on(FailPoints::force_pause_query_until_write_finish, { + FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); + //LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); + }); + /// In TiFlash, the actions between applying raft log and schema changes are not strictly synchronized. /// There could be a chance that some raft logs come after a table gets tombstoned. Take care of it when /// decoding data. Check the test case for more details. - fiu_do_on( - FailPoints::pause_before_apply_raft_cmd, - { - FailPointHelper::wait(FailPoints::pause_before_apply_raft_cmd); - - /// we add force_pause_query_until_write_finish failpoint to enable pause_query_until_write_finish here and disable it when write finish to make the query happend until the write totally finished. - fiu_do_on(FailPoints::force_pause_query_until_write_finish, { - FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); - }); - }); + FAIL_POINT_PAUSE(FailPoints::pause_before_apply_raft_cmd); + // fiu_do_on( + // FailPoints::pause_before_apply_raft_cmd, + // { + // FailPointHelper::wait(FailPoints::pause_before_apply_raft_cmd); + // }); - // LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); + //LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); // ::sleep(3); - // LOG_FMT_INFO(log, "[for test only] begin decode and write"); + //LOG_FMT_INFO(log, "[for test only] begin decode and write"); /// Try read then write once. { if (atomic_read_write(false)) @@ -194,6 +195,7 @@ static void writeRegionDataToStorage( fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); + //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); return; } @@ -210,6 +212,7 @@ static void writeRegionDataToStorage( // TODO: Enrich exception message. fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); + //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 69d1ba75724..deeeb168222 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -126,6 +126,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + # alter cmd 1 mysql> alter table test.t add column e decimal(6,1) NULL; @@ -193,6 +195,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); │ 1 │ 4.50 │ abc │ 0 │ └─────┴──────┴───────┴─────┘ +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + # alter cmd 1 mysql> alter table test.t add column e decimal(6,1) NULL; @@ -684,6 +688,8 @@ mysql> alter table test.t drop column b; │ 3 │ ccc │ 3 │ 0.1 │ └─────┴───────┴─────┴─────┘ +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); @@ -752,6 +758,8 @@ mysql> alter table test.t drop column b; │ 3 │ ccc │ 3 │ 0.1 │ └─────┴───────┴─────┴─────┘ +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); From 4c9c99295208489b6dea578aca1cdf1b985b8214 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 21:38:02 +0800 Subject: [PATCH 08/21] add some log for test --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 12 ++++++------ .../Storages/Transaction/PartitionStreams.cpp | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index a7a07917b53..29726ce59a7 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -139,13 +139,13 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(const Names & required_result_column_names) { /// Failpoint to make our query begin after the write action finish. - FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); - // fiu_do_on(FailPoints::pause_query_until_write_finish, { - // //LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); - // FailPointHelper::wait(FailPoints::pause_query_until_write_finish); - // }); + //FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); + fiu_do_on(FailPoints::pause_query_until_write_finish, { + //LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); + FailPointHelper::wait(FailPoints::pause_query_until_write_finish); + }); - //LOG_FMT_INFO(log, "[for test only] Begin InterpreterSelectQuery"); + LOG_FMT_INFO(log, "[for test only] Begin InterpreterSelectQuery"); ProfileEvents::increment(ProfileEvents::SelectQuery); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 490030dbd2e..d185ae62d44 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -168,7 +168,7 @@ static void writeRegionDataToStorage( /// we add force_pause_query_until_write_finish failpoint to enable pause_query_until_write_finish here and disable it when write finish to make the query happend until the write totally finished. fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); + LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); }); /// In TiFlash, the actions between applying raft log and schema changes are not strictly synchronized. @@ -182,20 +182,20 @@ static void writeRegionDataToStorage( // }); - //LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); + LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); // ::sleep(3); - //LOG_FMT_INFO(log, "[for test only] begin decode and write"); + LOG_FMT_INFO(log, "[for test only] begin decode and write"); /// Try read then write once. { if (atomic_read_write(false)) { - //LOG_FMT_INFO(log, "[for test only] first decode sucess"); + LOG_FMT_INFO(log, "[for test only] first decode sucess"); fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); return; } @@ -212,7 +212,7 @@ static void writeRegionDataToStorage( // TODO: Enrich exception message. fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", @@ -220,11 +220,11 @@ static void writeRegionDataToStorage( } - //LOG_FMT_INFO(log, "[for test only] second decode sucess"); + LOG_FMT_INFO(log, "[for test only] second decode sucess"); fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); } } From 3dd140afa4a56b1bbc6ea0195827ec4136ea1756 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 21:53:48 +0800 Subject: [PATCH 09/21] add some log for test --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 2 +- dbms/src/Interpreters/executeQuery.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 29726ce59a7..ebb2c469811 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -141,7 +141,7 @@ void InterpreterSelectQuery::init(const Names & required_result_column_names) /// Failpoint to make our query begin after the write action finish. //FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); fiu_do_on(FailPoints::pause_query_until_write_finish, { - //LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); + LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); FailPointHelper::wait(FailPoints::pause_query_until_write_finish); }); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 608905e6d47..fae474801fc 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -422,6 +422,9 @@ BlockIO executeQuery( bool internal, QueryProcessingStage::Enum stage) { + auto log = getLogger(context); + LOG_FMT_INFO(log, "[for test only] executeQuery query is {}", query); + BlockIO streams; SQLQuerySource query_src(query.data(), query.data() + query.size()); std::tie(std::ignore, streams) = executeQueryImpl(query_src, context, internal, stage); From 4c7b2cc93c4e3377e0723881034fcafe90d7a718 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 22:26:33 +0800 Subject: [PATCH 10/21] change the failpoint --- .../Storages/Transaction/PartitionStreams.cpp | 8 +- .../ddl/multi_alter_with_write.test | 102 +++++++++++++++++- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index d185ae62d44..ccb948f0372 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -166,10 +166,10 @@ static void writeRegionDataToStorage( /// we add force_pause_query_until_write_finish failpoint to enable pause_query_until_write_finish here and disable it when write finish to make the query happend until the write totally finished. - fiu_do_on(FailPoints::force_pause_query_until_write_finish, { - FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); - LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); - }); + // fiu_do_on(FailPoints::force_pause_query_until_write_finish, { + // FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); + // LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); + // }); /// In TiFlash, the actions between applying raft log and schema changes are not strictly synchronized. /// There could be a chance that some raft logs come after a table gets tombstoned. Take care of it when diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index deeeb168222..7f18b172288 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -41,6 +41,9 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -65,6 +68,9 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -91,6 +97,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -118,6 +127,9 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -142,6 +154,9 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -159,6 +174,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -187,6 +205,9 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -217,6 +238,9 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -231,6 +255,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -258,6 +285,9 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -280,6 +310,9 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -307,6 +340,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -334,6 +370,9 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -350,6 +389,9 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -368,6 +410,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -396,6 +441,9 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -415,6 +463,9 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # alter cmd 2 mysql> alter table test.t drop column b; +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -430,6 +481,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -457,6 +511,9 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) => DBGInvoke query_mapped('select * from \$d.\$t', test, t) @@ -481,9 +538,13 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + => DBGInvoke query_mapped('select * from \$d.\$t', test, t) ┌─a───┬─b────┬─c─────┬─d───┬─e───┐ │ 0 │ 0.00 │ │ 0 │ \N │ @@ -495,6 +556,9 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -523,6 +587,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -550,7 +617,10 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); - + +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) => DBGInvoke query_mapped('select * from \$d.\$t', test, t) @@ -574,6 +644,9 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -588,6 +661,9 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -606,6 +682,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -635,6 +714,9 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) => DBGInvoke query_mapped('select * from \$d.\$t', test, t) @@ -658,6 +740,9 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -678,6 +763,9 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -693,6 +781,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -721,6 +812,9 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) => DBGInvoke query_mapped('select * from \$d.\$t', test, t) @@ -748,6 +842,9 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 1 and write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -763,6 +860,9 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); +# enable pause_query_until_write_finish make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_until_write_finish) + # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) From 3dbe45dfb5d0a46a0fe9392bf4d349f059891108 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 22:46:15 +0800 Subject: [PATCH 11/21] comment out these log infos --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 4 ++-- dbms/src/Interpreters/executeQuery.cpp | 4 ++-- dbms/src/Storages/Transaction/PartitionStreams.cpp | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index ebb2c469811..f5ace886066 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -141,11 +141,11 @@ void InterpreterSelectQuery::init(const Names & required_result_column_names) /// Failpoint to make our query begin after the write action finish. //FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); fiu_do_on(FailPoints::pause_query_until_write_finish, { - LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); + //LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); FailPointHelper::wait(FailPoints::pause_query_until_write_finish); }); - LOG_FMT_INFO(log, "[for test only] Begin InterpreterSelectQuery"); + //LOG_FMT_INFO(log, "[for test only] Begin InterpreterSelectQuery"); ProfileEvents::increment(ProfileEvents::SelectQuery); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index fae474801fc..b07b34ee0a5 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -422,8 +422,8 @@ BlockIO executeQuery( bool internal, QueryProcessingStage::Enum stage) { - auto log = getLogger(context); - LOG_FMT_INFO(log, "[for test only] executeQuery query is {}", query); + //auto log = getLogger(context); + //LOG_FMT_INFO(log, "[for test only] executeQuery query is {}", query); BlockIO streams; SQLQuerySource query_src(query.data(), query.data() + query.size()); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index ccb948f0372..06b3cf7abe6 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -182,11 +182,11 @@ static void writeRegionDataToStorage( // }); - LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); + //LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); // ::sleep(3); - LOG_FMT_INFO(log, "[for test only] begin decode and write"); + // LOG_FMT_INFO(log, "[for test only] begin decode and write"); /// Try read then write once. { if (atomic_read_write(false)) @@ -195,7 +195,7 @@ static void writeRegionDataToStorage( fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); return; } @@ -212,7 +212,7 @@ static void writeRegionDataToStorage( // TODO: Enrich exception message. fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", @@ -220,11 +220,11 @@ static void writeRegionDataToStorage( } - LOG_FMT_INFO(log, "[for test only] second decode sucess"); + //LOG_FMT_INFO(log, "[for test only] second decode sucess"); fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); + //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); } } From 0bf25bfd25d8b7bd76f51bdebdcec78fecd38f17 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 22:59:43 +0800 Subject: [PATCH 12/21] remove extra log info --- dbms/src/Common/FailPoint.cpp | 8 ------- .../Interpreters/InterpreterSelectQuery.cpp | 8 +------ dbms/src/Interpreters/executeQuery.cpp | 3 --- .../Storages/Transaction/PartitionStreams.cpp | 24 ------------------- 4 files changed, 1 insertion(+), 42 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index 3dd8636321c..0bccdaae286 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -127,20 +127,14 @@ class FailPointChannel : private boost::noncopyable void wait() { - //std::cout << "[for test only] wait line 130" << std::endl; std::unique_lock lock(m); - //std::cout << "[for test only] wait line 132" << std::endl; cv.wait(lock); - //std::cout << "[for test only] wait line 134" << std::endl; } void notifyAll() { - //std::cout << "[for test only] notifyAll line 139" << std::endl; std::unique_lock lock(m); - //std::cout << "[for test only] notifyAll line 141" << std::endl; cv.notify_all(); - //std::cout << "[for test only] notifyAll line 143" << std::endl; } private: @@ -189,7 +183,6 @@ void FailPointHelper::enableFailPoint(const String & fail_point_name) void FailPointHelper::disableFailPoint(const String & fail_point_name) { - //std::cout << "[for test only] FailPointHelper::disableFailPoint " << fail_point_name << std::endl; if (auto iter = fail_point_wait_channels.find(fail_point_name); iter != fail_point_wait_channels.end()) { /// can not rely on deconstruction to do the notify_all things, because @@ -202,7 +195,6 @@ void FailPointHelper::disableFailPoint(const String & fail_point_name) void FailPointHelper::wait(const String & fail_point_name) { - //std::cout << "[for test only] FailPointHelper::wait " << fail_point_name << std::endl; if (auto iter = fail_point_wait_channels.find(fail_point_name); iter == fail_point_wait_channels.end()) throw Exception("Can not find channel for fail point " + fail_point_name); else diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index f5ace886066..39278ac6ef5 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -139,13 +139,7 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(const Names & required_result_column_names) { /// Failpoint to make our query begin after the write action finish. - //FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); - fiu_do_on(FailPoints::pause_query_until_write_finish, { - //LOG_FMT_INFO(log, "[for test only] FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish)"); - FailPointHelper::wait(FailPoints::pause_query_until_write_finish); - }); - - //LOG_FMT_INFO(log, "[for test only] Begin InterpreterSelectQuery"); + FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); ProfileEvents::increment(ProfileEvents::SelectQuery); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index b07b34ee0a5..608905e6d47 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -422,9 +422,6 @@ BlockIO executeQuery( bool internal, QueryProcessingStage::Enum stage) { - //auto log = getLogger(context); - //LOG_FMT_INFO(log, "[for test only] executeQuery query is {}", query); - BlockIO streams; SQLQuerySource query_src(query.data(), query.data() + query.size()); std::tie(std::ignore, streams) = executeQueryImpl(query_src, context, internal, stage); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 06b3cf7abe6..6eb0c61517b 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -164,29 +164,11 @@ static void writeRegionDataToStorage( return true; }; - - /// we add force_pause_query_until_write_finish failpoint to enable pause_query_until_write_finish here and disable it when write finish to make the query happend until the write totally finished. - // fiu_do_on(FailPoints::force_pause_query_until_write_finish, { - // FailPointHelper::enableFailPoint(FailPoints::pause_query_until_write_finish); - // LOG_FMT_INFO(log, "[for test only] enable FailPoints::pause_query_until_write_finish"); - // }); - /// In TiFlash, the actions between applying raft log and schema changes are not strictly synchronized. /// There could be a chance that some raft logs come after a table gets tombstoned. Take care of it when /// decoding data. Check the test case for more details. FAIL_POINT_PAUSE(FailPoints::pause_before_apply_raft_cmd); - // fiu_do_on( - // FailPoints::pause_before_apply_raft_cmd, - // { - // FailPointHelper::wait(FailPoints::pause_before_apply_raft_cmd); - // }); - - - //LOG_FMT_INFO(log, "[for test only] open the pause failpoint pause_before_apply_raft_cmd"); - // ::sleep(3); - - // LOG_FMT_INFO(log, "[for test only] begin decode and write"); /// Try read then write once. { if (atomic_read_write(false)) @@ -195,7 +177,6 @@ static void writeRegionDataToStorage( fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); return; } @@ -212,19 +193,14 @@ static void writeRegionDataToStorage( // TODO: Enrich exception message. fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", ErrorCodes::LOGICAL_ERROR); } - - //LOG_FMT_INFO(log, "[for test only] second decode sucess"); - fiu_do_on(FailPoints::force_pause_query_until_write_finish, { FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - //LOG_FMT_INFO(log, "[for test only] disable FailPoints::pause_query_until_write_finish"); }); } } From 4e13058a362549ce958187042468e35676a5f267 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 23:01:50 +0800 Subject: [PATCH 13/21] remove extra header --- dbms/src/Storages/Transaction/PartitionStreams.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 6eb0c61517b..393107327a6 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -32,8 +32,6 @@ #include #include -#include "fiu.h" - namespace DB { namespace FailPoints From a53c45297ea644738e376f4b272087b4032654bd Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 14 Jun 2022 23:05:05 +0800 Subject: [PATCH 14/21] remove extra change --- dbms/src/Interpreters/executeQuery.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 608905e6d47..96cfc0a58ae 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -36,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -56,11 +54,6 @@ extern const int QUERY_IS_TOO_LARGE; extern const int INTO_OUTFILE_NOT_ALLOWED; } // namespace ErrorCodes -namespace FailPoints -{ -extern const char pause_query_until_write_finish[]; -} // namespace FailPoints - namespace { void checkASTSizeLimits(const IAST & ast, const Settings & settings) From 45678089a38d2200920b06f0b73ed2451e0005c4 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 15 Jun 2022 00:03:24 +0800 Subject: [PATCH 15/21] add reset code --- tests/fullstack-test2/ddl/multi_alter_with_write.test | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 7f18b172288..8f93b20f411 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -874,4 +874,12 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); │ 4 │ abcd │ 10 │ 0.2 │ └─────┴───────┴─────┴─────┘ ->> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) \ No newline at end of file +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + + +## + +=> DBGInvoke __enable_schema_sync_service('true') +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +>> DBGInvoke __disable_fail_point(force_pause_query_until_write_finish) +>> DBGInvoke __disable_fail_point(pause_query_until_write_finish) \ No newline at end of file From ebcf379f7facb60fe09213913d725dd790114280 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 15 Jun 2022 10:11:32 +0800 Subject: [PATCH 16/21] change failpoint name and add necessary comments --- dbms/src/Common/FailPoint.cpp | 52 ++++--- .../Interpreters/InterpreterSelectQuery.cpp | 4 +- .../Storages/Transaction/PartitionStreams.cpp | 24 ++- .../ddl/multi_alter_with_write.test | 138 +++++++++--------- 4 files changed, 109 insertions(+), 109 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index 0bccdaae286..8d5eaf99cbe 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -85,36 +85,42 @@ std::unordered_map> FailPointHelper::f M(force_remote_read_for_batch_cop) \ M(force_context_path) \ M(force_slow_page_storage_snapshot_release) \ - M(force_change_all_blobs_to_read_only) + M(force_change_all_blobs_to_read_only) \ + M(unblock_query_init_after_write) + + +#define APPLY_FOR_FAILPOINTS_ONCE_WITH_PAUSEABLE(M) \ + M(pause_with_alter_locks_acquired) \ + M(hang_in_execution) \ + M(pause_before_dt_background_delta_merge) \ + M(pause_until_dt_background_delta_merge) \ + M(pause_before_apply_raft_cmd) \ + M(pause_before_apply_raft_snapshot) \ + M(pause_until_apply_raft_snapshot) \ + M(pause_after_copr_streams_acquired_once) +/// the usage of pause_query_init and unblock_query_init_after_write : +/// the failpoint pause_query_init should use with the failpoint unblock_query_init_after_write, to fulfill that the select query action will be blocked before init state to wait the write action finished. +/// In using, we need enable unblock_query_init_after_write in our test code, and before each write statement take effect, we need enable pause_query_init, and when the write action finished, the pause_query_init will be disabled automatically, and then the select query could be continued. +/// you can refer multi_alter_with_write.test for an example -#define APPLY_FOR_FAILPOINTS_ONCE_WITH_CHANNEL(M) \ - M(pause_with_alter_locks_acquired) \ - M(hang_in_execution) \ - M(pause_before_dt_background_delta_merge) \ - M(pause_until_dt_background_delta_merge) \ - M(pause_before_apply_raft_cmd) \ - M(pause_before_apply_raft_snapshot) \ - M(pause_until_apply_raft_snapshot) \ - M(pause_after_copr_streams_acquired_once) +#define APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) \ + M(pause_when_reading_from_dt_stream) \ + M(pause_when_writing_to_dt_store) \ + M(pause_when_ingesting_to_dt_store) \ + M(pause_when_altering_dt_store) \ + M(pause_after_copr_streams_acquired) \ + M(pause_before_server_merge_one_delta) \ + M(pause_query_init) -#define APPLY_FOR_FAILPOINTS_WITH_CHANNEL(M) \ - M(pause_when_reading_from_dt_stream) \ - M(pause_when_writing_to_dt_store) \ - M(pause_when_ingesting_to_dt_store) \ - M(pause_when_altering_dt_store) \ - M(pause_after_copr_streams_acquired) \ - M(pause_before_server_merge_one_delta) \ - M(pause_query_until_write_finish) \ - M(force_pause_query_until_write_finish) namespace FailPoints { #define M(NAME) extern const char(NAME)[] = #NAME ""; APPLY_FOR_FAILPOINTS_ONCE(M) APPLY_FOR_FAILPOINTS(M) -APPLY_FOR_FAILPOINTS_ONCE_WITH_CHANNEL(M) -APPLY_FOR_FAILPOINTS_WITH_CHANNEL(M) +APPLY_FOR_FAILPOINTS_ONCE_WITH_PAUSEABLE(M) +APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) #undef M } // namespace FailPoints @@ -170,11 +176,11 @@ void FailPointHelper::enableFailPoint(const String & fail_point_name) } #define M(NAME) SUB_M(NAME, FIU_ONETIME) - APPLY_FOR_FAILPOINTS_ONCE_WITH_CHANNEL(M) + APPLY_FOR_FAILPOINTS_ONCE_WITH_PAUSEABLE(M) #undef M #define M(NAME) SUB_M(NAME, 0) - APPLY_FOR_FAILPOINTS_WITH_CHANNEL(M) + APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) #undef M #undef SUB_M diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 39278ac6ef5..72e214c9b01 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -97,7 +97,7 @@ extern const int UNKNOWN_EXCEPTION; namespace FailPoints { -extern const char pause_query_until_write_finish[]; +extern const char pause_query_init[]; } // namespace FailPoints InterpreterSelectQuery::InterpreterSelectQuery( @@ -139,7 +139,7 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(const Names & required_result_column_names) { /// Failpoint to make our query begin after the write action finish. - FAIL_POINT_PAUSE(FailPoints::pause_query_until_write_finish); + FAIL_POINT_PAUSE(FailPoints::pause_query_init); ProfileEvents::increment(ProfileEvents::SelectQuery); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 393107327a6..aae39534c4a 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -39,8 +39,8 @@ namespace FailPoints extern const char pause_before_apply_raft_cmd[]; extern const char pause_before_apply_raft_snapshot[]; extern const char force_set_safepoint_when_decode_block[]; -extern const char force_pause_query_until_write_finish[]; -extern const char pause_query_until_write_finish[]; +extern const char unblock_query_init_after_write[]; +extern const char pause_query_init[]; } // namespace FailPoints namespace ErrorCodes @@ -167,15 +167,17 @@ static void writeRegionDataToStorage( /// decoding data. Check the test case for more details. FAIL_POINT_PAUSE(FailPoints::pause_before_apply_raft_cmd); + /// disable pause_query init when the write action finish, to make the query action continue + SCOPE_EXIT({ + fiu_do_on(FailPoints::unblock_query_init_after_write, { + FailPointHelper::disableFailPoint(FailPoints::pause_query_init); + }); + }); + /// Try read then write once. { if (atomic_read_write(false)) { - LOG_FMT_INFO(log, "[for test only] first decode sucess"); - - fiu_do_on(FailPoints::force_pause_query_until_write_finish, { - FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - }); return; } } @@ -189,17 +191,9 @@ static void writeRegionDataToStorage( { // Failure won't be tolerated this time. // TODO: Enrich exception message. - fiu_do_on(FailPoints::force_pause_query_until_write_finish, { - FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - }); - throw Exception("Write region " + std::to_string(region->id()) + " to table " + std::to_string(table_id) + " failed", ErrorCodes::LOGICAL_ERROR); } - - fiu_do_on(FailPoints::force_pause_query_until_write_finish, { - FailPointHelper::disableFailPoint(FailPoints::pause_query_until_write_finish); - }); } } diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 8f93b20f411..1948a5063e9 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -24,7 +24,7 @@ >> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) # Enable the failpoint to make our query only start when the write action finished ->> DBGInvoke __enable_fail_point(force_pause_query_until_write_finish) +>> DBGInvoke __enable_fail_point(unblock_query_init_after_write) # ----------------------------------------------------------------------------- # Order 1 : write cmd 1 | alter cmd 1 | write cmd 2 | alter cmd 2 | write cmd 3 @@ -41,8 +41,8 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -68,8 +68,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -97,8 +97,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -127,8 +127,8 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -154,8 +154,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -174,8 +174,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -205,8 +205,8 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -238,8 +238,8 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -255,8 +255,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -285,8 +285,8 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -310,8 +310,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -340,8 +340,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -370,8 +370,8 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -389,8 +389,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -410,8 +410,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -441,8 +441,8 @@ func> wait_table test t # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -463,8 +463,8 @@ mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); # alter cmd 2 mysql> alter table test.t drop column b; -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -481,8 +481,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -511,8 +511,8 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -538,8 +538,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -556,8 +556,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -587,8 +587,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -618,8 +618,8 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -644,8 +644,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -661,8 +661,8 @@ mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); # write cmd 2 mysql> insert into test.t values (3, 0.2, 'ccc', 3, 0.1); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -682,8 +682,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -714,8 +714,8 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -740,8 +740,8 @@ mysql> alter table test.t add column e decimal(6,1) NULL; # write cmd 1 mysql> insert into test.t (a, b, c) values (1, 4.5, 'abc'); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -763,8 +763,8 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -781,8 +781,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -812,8 +812,8 @@ func> wait_table test t # add a new pre write to make check the alter cmd 1 more convenient. mysql> insert into test.t (a, b, c) values (0, 0, ' '); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -842,8 +842,8 @@ mysql> alter table test.t drop column b; # make alter cmd 2 take effect >> DBGInvoke __refresh_schemas() -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 1 and write cmd 2 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -860,8 +860,8 @@ mysql> alter table test.t drop column b; # write cmd 3 mysql> insert into test.t values (4, 'abcd', 10, 0.2); -# enable pause_query_until_write_finish make query start until write cmd finish ->> DBGInvoke __enable_fail_point(pause_query_until_write_finish) +# enable pause_query_init make query start until write cmd finish +>> DBGInvoke __enable_fail_point(pause_query_init) # make write cmd 3 take effect >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) @@ -881,5 +881,5 @@ mysql> insert into test.t values (4, 'abcd', 10, 0.2); => DBGInvoke __enable_schema_sync_service('true') >> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) ->> DBGInvoke __disable_fail_point(force_pause_query_until_write_finish) ->> DBGInvoke __disable_fail_point(pause_query_until_write_finish) \ No newline at end of file +>> DBGInvoke __disable_fail_point(unblock_query_init_after_write) +>> DBGInvoke __disable_fail_point(pause_query_init) \ No newline at end of file From 7f15573d35680c72bf4e5c583c0d36b1564cc09e Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 15 Jun 2022 10:48:07 +0800 Subject: [PATCH 17/21] change the comments --- dbms/src/Common/FailPoint.cpp | 10 +++++++--- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 1 - 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index 8d5eaf99cbe..67d0a8e6f2f 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -100,9 +100,13 @@ std::unordered_map> FailPointHelper::f M(pause_after_copr_streams_acquired_once) /// the usage of pause_query_init and unblock_query_init_after_write : -/// the failpoint pause_query_init should use with the failpoint unblock_query_init_after_write, to fulfill that the select query action will be blocked before init state to wait the write action finished. -/// In using, we need enable unblock_query_init_after_write in our test code, and before each write statement take effect, we need enable pause_query_init, and when the write action finished, the pause_query_init will be disabled automatically, and then the select query could be continued. -/// you can refer multi_alter_with_write.test for an example +/// the failpoint pause_query_init should use with the failpoint unblock_query_init_after_write, +/// to fulfill that the select query action will be blocked before init state to wait the write action finished. +/// In using, we need enable unblock_query_init_after_write in our test code, +/// and before each write statement take effect, we need enable pause_query_init. +/// When the write action finished, the pause_query_init will be disabled automatically, +/// and then the select query could be continued. +/// you can refer multi_alter_with_write.test for an example. #define APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) \ M(pause_when_reading_from_dt_stream) \ diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 72e214c9b01..23b74125932 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -138,7 +138,6 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(const Names & required_result_column_names) { - /// Failpoint to make our query begin after the write action finish. FAIL_POINT_PAUSE(FailPoints::pause_query_init); ProfileEvents::increment(ProfileEvents::SelectQuery); From 15fb7b866c0065c1a4d526c8a904b221b843f33a Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 15 Jun 2022 13:55:24 +0800 Subject: [PATCH 18/21] change name of define --- dbms/src/Common/FailPoint.cpp | 47 ++++++++----------- .../Interpreters/InterpreterSelectQuery.cpp | 7 +++ .../Storages/Transaction/PartitionStreams.cpp | 3 +- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index 67d0a8e6f2f..10d0a558a50 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -89,32 +89,23 @@ std::unordered_map> FailPointHelper::f M(unblock_query_init_after_write) -#define APPLY_FOR_FAILPOINTS_ONCE_WITH_PAUSEABLE(M) \ - M(pause_with_alter_locks_acquired) \ - M(hang_in_execution) \ - M(pause_before_dt_background_delta_merge) \ - M(pause_until_dt_background_delta_merge) \ - M(pause_before_apply_raft_cmd) \ - M(pause_before_apply_raft_snapshot) \ - M(pause_until_apply_raft_snapshot) \ +#define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) \ + M(pause_with_alter_locks_acquired) \ + M(hang_in_execution) \ + M(pause_before_dt_background_delta_merge) \ + M(pause_until_dt_background_delta_merge) \ + M(pause_before_apply_raft_cmd) \ + M(pause_before_apply_raft_snapshot) \ + M(pause_until_apply_raft_snapshot) \ M(pause_after_copr_streams_acquired_once) -/// the usage of pause_query_init and unblock_query_init_after_write : -/// the failpoint pause_query_init should use with the failpoint unblock_query_init_after_write, -/// to fulfill that the select query action will be blocked before init state to wait the write action finished. -/// In using, we need enable unblock_query_init_after_write in our test code, -/// and before each write statement take effect, we need enable pause_query_init. -/// When the write action finished, the pause_query_init will be disabled automatically, -/// and then the select query could be continued. -/// you can refer multi_alter_with_write.test for an example. - -#define APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) \ - M(pause_when_reading_from_dt_stream) \ - M(pause_when_writing_to_dt_store) \ - M(pause_when_ingesting_to_dt_store) \ - M(pause_when_altering_dt_store) \ - M(pause_after_copr_streams_acquired) \ - M(pause_before_server_merge_one_delta) \ +#define APPLY_FOR_PAUSEABLE_FAILPOINTS(M) \ + M(pause_when_reading_from_dt_stream) \ + M(pause_when_writing_to_dt_store) \ + M(pause_when_ingesting_to_dt_store) \ + M(pause_when_altering_dt_store) \ + M(pause_after_copr_streams_acquired) \ + M(pause_before_server_merge_one_delta) \ M(pause_query_init) @@ -123,8 +114,8 @@ namespace FailPoints #define M(NAME) extern const char(NAME)[] = #NAME ""; APPLY_FOR_FAILPOINTS_ONCE(M) APPLY_FOR_FAILPOINTS(M) -APPLY_FOR_FAILPOINTS_ONCE_WITH_PAUSEABLE(M) -APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) +APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) +APPLY_FOR_PAUSEABLE_FAILPOINTS(M) #undef M } // namespace FailPoints @@ -180,11 +171,11 @@ void FailPointHelper::enableFailPoint(const String & fail_point_name) } #define M(NAME) SUB_M(NAME, FIU_ONETIME) - APPLY_FOR_FAILPOINTS_ONCE_WITH_PAUSEABLE(M) + APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) #undef M #define M(NAME) SUB_M(NAME, 0) - APPLY_FOR_FAILPOINTS_WITH_PAUSEABLE(M) + APPLY_FOR_PAUSEABLE_FAILPOINTS(M) #undef M #undef SUB_M diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 23b74125932..c52792fcab8 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -138,6 +138,13 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(const Names & required_result_column_names) { + /// the failpoint pause_query_init should use with the failpoint unblock_query_init_after_write, + /// to fulfill that the select query action will be blocked before init state to wait the write action finished. + /// In using, we need enable unblock_query_init_after_write in our test code, + /// and before each write statement take effect, we need enable pause_query_init. + /// When the write action finished, the pause_query_init will be disabled automatically, + /// and then the select query could be continued. + /// you can refer multi_alter_with_write.test for an example. FAIL_POINT_PAUSE(FailPoints::pause_query_init); ProfileEvents::increment(ProfileEvents::SelectQuery); diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index aae39534c4a..8c1224908f9 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -167,7 +167,8 @@ static void writeRegionDataToStorage( /// decoding data. Check the test case for more details. FAIL_POINT_PAUSE(FailPoints::pause_before_apply_raft_cmd); - /// disable pause_query init when the write action finish, to make the query action continue + /// disable pause_query_init when the write action finish, to make the query action continue. + /// the usage of unblock_query_init_after_write and pause_query_init can refer to InterpreterSelectQuery::init SCOPE_EXIT({ fiu_do_on(FailPoints::unblock_query_init_after_write, { FailPointHelper::disableFailPoint(FailPoints::pause_query_init); From 18f17cbb362ea735ea1fabb965ab95e9b8cf3b0b Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 20 Jun 2022 15:53:06 +0800 Subject: [PATCH 19/21] remove useless code --- tests/fullstack-test2/ddl/multi_alter_with_write.test | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 1948a5063e9..4de26f766f2 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -34,7 +34,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -120,7 +119,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -198,7 +196,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -278,7 +275,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -363,7 +359,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -434,7 +429,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -504,7 +498,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -611,7 +604,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -707,7 +699,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t @@ -805,7 +796,6 @@ mysql> drop table if exists test.t mysql> create table test.t(a int primary key, b decimal(5,2) not NULL, c varchar(10), d int default 0); mysql> alter table test.t set tiflash replica 1; -mysql> set session tidb_isolation_read_engines='tiflash'; func> wait_table test t From f5c5b0d37326647e3807ad72cdf58fcaa129c241 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 21 Jun 2022 19:38:23 +0800 Subject: [PATCH 20/21] merge master remove extra code --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index c52792fcab8..fe8f04427a0 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -147,8 +147,6 @@ void InterpreterSelectQuery::init(const Names & required_result_column_names) /// you can refer multi_alter_with_write.test for an example. FAIL_POINT_PAUSE(FailPoints::pause_query_init); - ProfileEvents::increment(ProfileEvents::SelectQuery); - if (!context.hasQueryContext()) context.setQueryContext(context); From 366e4edf6aa46a80144a45aa5b4317fbc4a098df Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 21 Jun 2022 20:19:32 +0800 Subject: [PATCH 21/21] change comments --- tests/fullstack-test2/ddl/multi_alter_with_write.test | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/fullstack-test2/ddl/multi_alter_with_write.test b/tests/fullstack-test2/ddl/multi_alter_with_write.test index 4de26f766f2..3284511d775 100644 --- a/tests/fullstack-test2/ddl/multi_alter_with_write.test +++ b/tests/fullstack-test2/ddl/multi_alter_with_write.test @@ -12,7 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this test focus on the case when multi DDL actions happen closely（ and these DDL actions will be fetched in the same regular sync schema duration.) and there are some corresponding insert(write) actions between these DDL actions. Considering that these write actions and these schema change will arrive at tiflash in a different order, we simulate these different order situation to check that our schema module was working correctly. +# this test focus on the case when multi DDL actions happen closely +#（ and these DDL actions will be fetched in the same regular sync schema duration.) +# and there are some corresponding insert(write) actions between these DDL actions. +# Considering that these write actions and these schema change will arrive at +# tiflash in a different order, we simulate these different order situation to check +# that our schema module was working correctly. # TiDB Timeline ： write cmd 1 ｜ alter cmd 1 ｜ write cmd 2 | alter cmd 2 | write cmd 3