From 8f4bdcb96e00482559fabbc3a1107fa1957b4af1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sk=C3=B6ld?= Date: Mon, 2 Mar 2020 16:03:38 +0100 Subject: [PATCH 1/2] Bug#21911930 OPTION FOR MYSQLD TO SHUTDOWN IF FULL BINLOG RECORDING NOT POSSIBLE Added new option --ndb-log-fail-terminate to terminated the mysqld if binlogging of all found ndb tables fail (if binlogging is enabled). Added test case with error injection. Approved-by: Frazer Clement --- .../r/ndb_binlog_startup_fail.result | 12 ++ .../ndb_binlog/t/ndb_binlog_startup_fail.cnf | 11 ++ .../ndb_binlog/t/ndb_binlog_startup_fail.test | 124 ++++++++++++++++++ sql/ha_ndbcluster.cc | 13 +- sql/ha_ndbcluster_binlog.cc | 36 ++++- sql/ha_ndbcluster_binlog.h | 3 +- 6 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/ndb_binlog/r/ndb_binlog_startup_fail.result create mode 100644 mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.cnf create mode 100644 mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test diff --git a/mysql-test/suite/ndb_binlog/r/ndb_binlog_startup_fail.result b/mysql-test/suite/ndb_binlog/r/ndb_binlog_startup_fail.result new file mode 100644 index 000000000000..703aa78ab81a --- /dev/null +++ b/mysql-test/suite/ndb_binlog/r/ndb_binlog_startup_fail.result @@ -0,0 +1,12 @@ +use test; +create table t1 (a int primary key) engine=ndb; +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +Prepare for Server2 to fail +Wait for 'server2' binlog rotate to indicate disconnect +insert into t1 values (11); +Give 'server2' some time to start, and fail, a binlog_setup +Wait for Server2 to fail +Server2 failed as expected +insert into t1 values (10); +drop table test.t1; +Wait for 'server2' to complete setup and get out of read-only mode diff --git a/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.cnf b/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.cnf new file mode 100644 index 000000000000..177539d12032 --- /dev/null +++ b/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.cnf @@ -0,0 +1,11 @@ +!include suite/ndb_binlog/my.cnf + +[mysqld] +ndb-cluster-connection-pool=1 + +[mysqld.1.1] +server_id=1 + +[mysqld.2.1] +server_id=2 +ndb-log-fail-terminate=1 diff --git a/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test b/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test new file mode 100644 index 000000000000..89f1fcf88612 --- /dev/null +++ b/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test @@ -0,0 +1,124 @@ +-- source include/have_multi_ndb.inc +-- source include/have_binlog_format_mixed_or_row.inc +# We are using some debug-only features in this test +--source include/have_debug.inc + +connection server1; +# Find NodeId of the mysqld we are connected to: +--let $node_id= `SHOW STATUS LIKE 'Ndb_cluster_node_id'` +--let $node_id= `SELECT SUBSTRING('$node_id', 20)+0` +--disable_query_log +call mtr.add_suppression("cluster disconnect An incident event"); +--enable_query_log + +connection server2; +# Find NodeId2 of 'server2' we are connected to: +--let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'` +--let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0` + +# Ignore server shutdown/startup failure +--disable_query_log +call mtr.add_suppression("NDB Binlog: FAILED"); +call mtr.add_suppression("NDB Binlog: ndbcluster_handle_incomplete_binlog_setup"); +call mtr.add_suppression("Plugin 'ndbcluster' will be forced to shutdown"); +call mtr.add_suppression("NDB Binlog: Failed setting up binlogging"); +call mtr.add_suppression("TIMESTAMP"); +call mtr.add_suppression("Insecure"); +call mtr.add_suppression("ndbcluster_end"); +call mtr.add_suppression("Attempting backtrace"); +--enable_query_log + +# +# 1 create a table +# 2 Setup error insert on MySQLD2 +# 3 Disconnect MySQLD2 using DUMP code, causing Binlog re-init +# 4 Observe MySQLD2 fail, then restart (and succeed) +# + +connection server1; +use test; + +create table t1 (a int primary key) engine=ndb; +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +connection server2; +disable_query_log; +let $debug_saved = `select @@global.debug`; +set global debug='-d'; # Switch DEBUG/TRACING OFF +set global debug='+d,ndb_binlog_fail_setup'; +enable_query_log; + +--echo Prepare for Server2 to fail +connection server2; +# Find NodeId2 of 'server2' we are connected to: +--let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'` +--let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0` + +# +# Some magic for MTR to allow failure, taken from include/restart_mysqld.inc +# +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.1.expect + +--exec echo "wait" > $_expect_file_name +# +# /end magic part 1 +# + +# +# Cause data nodes to disconnect all API nodes +# Each API will setup binlog again +# MySQLD2 will get stuck and restart +# + +--exec $NDB_MGM --no-defaults -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT +--exec $NDB_MGM --no-defaults -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT + +#connection server1; +#--echo Wait for 'server1' binlog rotate to indicate disconnect +#--let $wait_binlog_event= mysqld-bin.000002 +#--source include/wait_for_binlog_event.inc + +connection server2; +--echo Wait for 'server2' binlog rotate to indicate disconnect +--let $wait_binlog_event= mysqld-bin.000002 +--source include/wait_for_binlog_event.inc + +connection server1; +--source include/ndb_not_readonly.inc +insert into t1 values (11); + +--echo Give 'server2' some time to start, and fail, a binlog_setup +sleep 2; + +--echo Wait for Server2 to fail +connection server2; +--source include/wait_until_disconnected.inc + +--echo Server2 failed as expected + +# +# More magic from include/restart_mysqld.inc +# +--exec echo "restart" > $_expect_file_name + +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect +# +# /end magic part 2 +# + +connection server1; +insert into t1 values (10); +drop table test.t1; + +connection server2; +--echo Wait for 'server2' to complete setup and get out of read-only mode +--source include/ndb_not_readonly.inc +--remove_file $NDB_TOOLS_OUTPUT + +disable_query_log; +set global debug='+d'; # Switch DEBUG/TRACING ON +eval set global debug= '$debug_saved'; +enable_query_log; diff --git a/sql/ha_ndbcluster.cc b/sql/ha_ndbcluster.cc index bc0d7ca11ef9..64c869490acf 100644 --- a/sql/ha_ndbcluster.cc +++ b/sql/ha_ndbcluster.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2004, 2019, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2004, 2020, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -19170,6 +19170,16 @@ static MYSQL_SYSVAR_STR( NULL /* default */ ); +my_bool opt_ndb_log_fail_terminate; +static MYSQL_SYSVAR_BOOL( + log_fail_terminate, /* name */ + opt_ndb_log_fail_terminate, /* var */ + PLUGIN_VAR_OPCMDARG, + "Terminate mysqld if complete logging of all found row events is not possible", + NULL, /* check func. */ + NULL, /* update func. */ + 0 /* default */ +); static MYSQL_SYSVAR_STR( mgmd_host, /* name */ @@ -19400,6 +19410,7 @@ static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(log_empty_epochs), MYSQL_SYSVAR(log_apply_status), MYSQL_SYSVAR(log_transaction_id), + MYSQL_SYSVAR(log_fail_terminate), MYSQL_SYSVAR(clear_apply_status), MYSQL_SYSVAR(connectstring), MYSQL_SYSVAR(mgmd_host), diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 9cf09ff87b11..80ca36693d02 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2006, 2020, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -63,6 +63,7 @@ extern my_bool opt_ndb_log_transaction_id; extern my_bool log_bin_use_v1_row_events; extern my_bool opt_ndb_log_empty_update; extern my_bool opt_ndb_clear_apply_status; +extern my_bool opt_ndb_log_fail_terminate; bool ndb_log_empty_epochs(void); @@ -1468,6 +1469,12 @@ int ndbcluster_find_all_files(THD *thd) } while (unhandled && retries); + if (unhandled) + { + sql_print_error("NDB Binlog: Failed setting up binlogging"); + ndbcluster_handle_incomplete_binlog_setup(); + } + DBUG_RETURN(-(skipped + unhandled)); } @@ -2522,7 +2529,7 @@ class Ndb_schema_event_handler { }; //class Ndb_schema_op static void - print_could_not_discover_error(THD *thd, + handle_could_not_discover_error(THD *thd, const Ndb_schema_op *schema) { sql_print_error("NDB Binlog: Could not discover table '%s.%s' from " @@ -2531,6 +2538,10 @@ class Ndb_schema_event_handler { schema->db, schema->name, schema->query, schema->node_id, my_errno); thd_print_warning_list(thd, "NDB Binlog"); + if (opt_ndb_log_fail_terminate) + { + ndbcluster_handle_incomplete_binlog_setup(); + } } @@ -3214,7 +3225,7 @@ class Ndb_schema_event_handler { // Instantiate a new 'share' for the altered table. if (ndb_create_table_from_engine(m_thd, schema->db, schema->name)) { - print_could_not_discover_error(m_thd, schema); + handle_could_not_discover_error(m_thd, schema); } DBUG_VOID_RETURN; } @@ -3557,7 +3568,7 @@ class Ndb_schema_event_handler { if (ndb_create_table_from_engine(m_thd, schema->db, schema->name)) { - print_could_not_discover_error(m_thd, schema); + handle_could_not_discover_error(m_thd, schema); } DBUG_VOID_RETURN; @@ -3587,7 +3598,7 @@ class Ndb_schema_event_handler { if (ndb_create_table_from_engine(m_thd, schema->db, schema->name)) { - print_could_not_discover_error(m_thd, schema); + handle_could_not_discover_error(m_thd, schema); } DBUG_VOID_RETURN; @@ -4672,6 +4683,13 @@ ndbcluster_check_if_local_table(const char *dbname, const char *tabname) } +void ndbcluster_handle_incomplete_binlog_setup() +{ + sql_print_error("NDB Binlog: ndbcluster_handle_incomplete_binlog_setup"); + if (opt_ndb_log_fail_terminate) + kill_mysql(); +} + /* Common function for setting up everything for logging a table at create/discover. @@ -4775,14 +4793,20 @@ int ndbcluster_create_binlog_setup(THD *thd, Ndb *ndb, if (ndbcluster_create_event_ops(thd, share, ndbtab, event_name.c_ptr())) { - sql_print_error("NDB Binlog:" + sql_print_error("NDB Binlog: " "FAILED CREATE (DISCOVER) EVENT OPERATIONS Event: %s", event_name.c_ptr()); /* a warning has been issued to the client */ break; } + DBUG_EXECUTE_IF("ndb_binlog_fail_setup", { ndbcluster_handle_incomplete_binlog_setup(); }); DBUG_RETURN(0); } + /* + * Handle failure of setting up binlogging of one table during startup + */ + ndbcluster_handle_incomplete_binlog_setup(); + DBUG_RETURN(-1); } diff --git a/sql/ha_ndbcluster_binlog.h b/sql/ha_ndbcluster_binlog.h index c703c36d21c7..18ae6ffd2911 100644 --- a/sql/ha_ndbcluster_binlog.h +++ b/sql/ha_ndbcluster_binlog.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -70,6 +70,7 @@ int ndbcluster_drop_event(THD *thd, Ndb *ndb, NDB_SHARE *share, int ndbcluster_handle_drop_table(THD *thd, Ndb *ndb, NDB_SHARE *share, const char *type_str, const char * db, const char * tabname); +void ndbcluster_handle_incomplete_binlog_setup(); void ndb_rep_event_name(String *event_name, const char *db, const char *tbl, bool full, bool allow_hardcoded_name = true); From 3e5e00d9182e968c4c7617397585a514ada53ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sk=C3=B6ld?= Date: Mon, 2 Mar 2020 16:09:31 +0100 Subject: [PATCH 2/2] Bug#21911930 OPTION FOR MYSQLD TO SHUTDOWN IF FULL BINLOG RECORDING NOT POSSIBLE Added more error log suppression for test case. Approved-by: Frazer Clement --- mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test b/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test index 89f1fcf88612..4d834275025d 100644 --- a/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test +++ b/mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test @@ -26,6 +26,7 @@ call mtr.add_suppression("TIMESTAMP"); call mtr.add_suppression("Insecure"); call mtr.add_suppression("ndbcluster_end"); call mtr.add_suppression("Attempting backtrace"); +call mtr.add_suppression("An incident event has been written"); --enable_query_log #