PS-9237 feature: Include support for utf8mb4_0900_ai_ci in MySQL 5.7

mysql-test/r/percona_default_collation_for_utf8mb4_extensions.result

@@ -0,0 +1,137 @@
+RESET MASTER;
+*** Variables from the default session
+*** (values for all collation variables are expected to be utf8mb4_0900_ai_ci)
+SHOW GLOBAL VARIABLES LIKE '%collation%';
+Variable_name	Value
+collation_connection	utf8mb4_0900_ai_ci
+collation_database	utf8mb4_0900_ai_ci
+collation_server	utf8mb4_0900_ai_ci
+default_collation_for_utf8mb4	utf8mb4_0900_ai_ci
+SHOW SESSION VARIABLES LIKE '%collation%';
+Variable_name	Value
+collation_connection	utf8mb4_0900_ai_ci
+collation_database	utf8mb4_0900_ai_ci
+collation_server	utf8mb4_0900_ai_ci
+default_collation_for_utf8mb4	utf8mb4_0900_ai_ci
+
+*** Variables from the new connection established via MySQL command line client
+*** (values for all collation variables are expected to be utf8mb4_0900_ai_ci)
+Variable_name	Value
+collation_connection	utf8mb4_0900_ai_ci
+collation_database	utf8mb4_0900_ai_ci
+collation_server	utf8mb4_0900_ai_ci
+default_collation_for_utf8mb4	utf8mb4_0900_ai_ci
+Variable_name	Value
+collation_connection	utf8mb4_0900_ai_ci
+collation_database	utf8mb4_0900_ai_ci
+collation_server	utf8mb4_0900_ai_ci
+default_collation_for_utf8mb4	utf8mb4_0900_ai_ci
+
+
+*** Updating collation variables
+SET GLOBAL default_collation_for_utf8mb4 = utf8mb4_general_ci;
+Warnings:
+Warning	1681	Updating 'default_collation_for_utf8mb4' is deprecated. It will be made read-only in a future release.
+SET GLOBAL collation_server = utf8mb4_general_ci;
+SET GLOBAL collation_connection = utf8mb4_general_ci;
+SET GLOBAL collation_database = utf8mb4_general_ci;
+Warnings:
+Warning	1681	Updating 'collation_database' is deprecated. It will be made read-only in a future release.
+
+
+*** Re-connecting
+
+
+*** Variables after re-connecting to the default database
+*** (values for all collation variables except for @@session.collation_database are expected to be utf8mb4_general_ci)
+SHOW GLOBAL VARIABLES LIKE '%collation%';
+Variable_name	Value
+collation_connection	utf8mb4_general_ci
+collation_database	utf8mb4_general_ci
+collation_server	utf8mb4_general_ci
+default_collation_for_utf8mb4	utf8mb4_general_ci
+SHOW SESSION VARIABLES LIKE '%collation%';
+Variable_name	Value
+collation_connection	utf8mb4_general_ci
+collation_database	utf8mb4_0900_ai_ci
+collation_server	utf8mb4_general_ci
+default_collation_for_utf8mb4	utf8mb4_general_ci
+
+
+*** Creating a fresh database
+CREATE DATABASE fresh;
+SHOW CREATE DATABASE fresh;
+Database	Create Database
+fresh	CREATE DATABASE `fresh` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci */ /*!80016 DEFAULT ENCRYPTION='N' */
+
+
+*** Variables after connecting to a fresh database
+*** (values for collation variables expected to be utf8mb4_general_ci)
+SHOW GLOBAL VARIABLES LIKE '%collation%';
+Variable_name	Value
+collation_connection	utf8mb4_general_ci
+collation_database	utf8mb4_general_ci
+collation_server	utf8mb4_general_ci
+default_collation_for_utf8mb4	utf8mb4_general_ci
+SHOW SESSION VARIABLES LIKE '%collation%';
+Variable_name	Value
+collation_connection	utf8mb4_general_ci
+collation_database	utf8mb4_general_ci
+collation_server	utf8mb4_general_ci
+default_collation_for_utf8mb4	utf8mb4_general_ci
+
+*** Variables from the new connection established via MySQL command line client to a fresh database
+*** (values for all collation variables are expected to be utf8mb4_general_ci)
+Variable_name	Value
+collation_connection	utf8mb4_general_ci
+collation_database	utf8mb4_general_ci
+collation_server	utf8mb4_general_ci
+default_collation_for_utf8mb4	utf8mb4_general_ci
+Variable_name	Value
+collation_connection	utf8mb4_general_ci
+collation_database	utf8mb4_general_ci
+collation_server	utf8mb4_general_ci
+default_collation_for_utf8mb4	utf8mb4_general_ci
+
+
+*** Creating tables in the fresh database from the default connection
+CREATE TABLE tbl_internal_before(id BIGINT UNSIGNED);
+SET character_set_client = utf8mb4;
+CREATE TABLE tbl_internal_after(id BIGINT UNSIGNED);
+
+
+*** Creating tables in the fresh database from a connection established via MySQL command line client
+
+
+*** Creating logical dump of the fresh database
+
+
+*** Creating a database for restoring data from the logical dump
+CREATE DATABASE restore;
+SHOW CREATE DATABASE restore;
+Database	Create Database
+restore	CREATE DATABASE `restore` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci */ /*!80016 DEFAULT ENCRYPTION='N' */
+
+
+*** Restoring data from the logical dump
+
+
+*** Checking events in the binary log
+include/assert_grep.inc [Events in the binary log must use utf8mb4_general_ci (45) collation]
+include/assert_grep.inc [Events in the binary log must not use utf8mb4_0900_ai_ci (255) collation]
+
+
+*** Dropping fresh database
+DROP DATABASE restore;
+DROP DATABASE fresh;
+
+
+*** Restoring collation variables
+SET GLOBAL collation_database = utf8mb4_0900_ai_ci;
+Warnings:
+Warning	1681	Updating 'collation_database' is deprecated. It will be made read-only in a future release.
+SET GLOBAL collation_connection = utf8mb4_0900_ai_ci;
+SET GLOBAL collation_server = utf8mb4_0900_ai_ci;
+SET GLOBAL default_collation_for_utf8mb4 = utf8mb4_0900_ai_ci;
+Warnings:
+Warning	1681	Updating 'default_collation_for_utf8mb4' is deprecated. It will be made read-only in a future release.

mysql-test/t/percona_default_collation_for_utf8mb4_extensions.test

@@ -0,0 +1,144 @@
+--source include/count_sessions.inc
+
+# needed for repeatable content of the binary log
+RESET MASTER;
+
+--let $global_col_stmt = SHOW GLOBAL VARIABLES LIKE '%collation%'
+--let $session_col_stmt = SHOW SESSION VARIABLES LIKE '%collation%'
+
+--echo *** Variables from the default session
+--echo *** (values for all collation variables are expected to be utf8mb4_0900_ai_ci)
+eval $global_col_stmt;
+eval $session_col_stmt;
+
+--let $default_database = `SELECT DATABASE()`
+
+--echo
+--echo *** Variables from the new connection established via MySQL command line client
+--echo *** (values for all collation variables are expected to be utf8mb4_0900_ai_ci)
+--exec $MYSQL -e "$global_col_stmt; $session_col_stmt;" $default_database
+
+--echo
+--echo
+--echo *** Updating collation variables
+--let $saved_default_collation_for_utf8mb4 = `SELECT @@global.default_collation_for_utf8mb4`
+--let $saved_collation_server = `SELECT @@global.collation_server`
+--let $saved_collation_connection = `SELECT @@global.collation_connection`
+--let $saved_collation_database = `SELECT @@global.collation_database`
+
+SET GLOBAL default_collation_for_utf8mb4 = utf8mb4_general_ci;
+SET GLOBAL collation_server = utf8mb4_general_ci;
+SET GLOBAL collation_connection = utf8mb4_general_ci;
+SET GLOBAL collation_database = utf8mb4_general_ci;
+
+--echo
+--echo
+--echo *** Re-connecting
+--disconnect default
+--connect (default,localhost,root,,$default_database)
+
+--echo
+--echo
+--echo *** Variables after re-connecting to the default database
+--echo *** (values for all collation variables except for @@session.collation_database are expected to be utf8mb4_general_ci)
+eval $global_col_stmt;
+eval $session_col_stmt;
+
+--echo
+--echo
+--echo *** Creating a fresh database
+--let $fresh_database = fresh
+eval CREATE DATABASE $fresh_database;
+eval SHOW CREATE DATABASE $fresh_database;
+
+--connect (con1,localhost,root,,$fresh_database)
+
+--echo
+--echo
+--echo *** Variables after connecting to a fresh database
+--echo *** (values for collation variables expected to be utf8mb4_general_ci)
+eval $global_col_stmt;
+eval $session_col_stmt;
+
+--echo
+--echo *** Variables from the new connection established via MySQL command line client to a fresh database
+--echo *** (values for all collation variables are expected to be utf8mb4_general_ci)
+--exec $MYSQL -e "$global_col_stmt; $session_col_stmt;" $fresh_database
+
+--let $binlog_file = query_get_value("SHOW MASTER STATUS", File, 1)
+--let $server_port = `SELECT @@port`
+
+--echo
+--echo
+--echo *** Creating tables in the fresh database from the default connection
+CREATE TABLE tbl_internal_before(id BIGINT UNSIGNED);
+SET character_set_client = utf8mb4;
+CREATE TABLE tbl_internal_after(id BIGINT UNSIGNED);
+
+--echo
+--echo
+--echo *** Creating tables in the fresh database from a connection established via MySQL command line client
+--exec $MYSQL -e "CREATE TABLE tbl_external_before(id BIGINT UNSIGNED); SET character_set_client = utf8mb4; CREATE TABLE tbl_external_after(id BIGINT UNSIGNED);" $fresh_database
+
+
+--echo
+--echo
+--echo *** Creating logical dump of the fresh database
+--let $fresh_dump_file = $MYSQL_TMP_DIR/fresh_dump.sql
+--exec $MYSQL_DUMP --column-statistics=0 --no-data $fresh_database > $fresh_dump_file
+
+--echo
+--echo
+--echo *** Creating a database for restoring data from the logical dump
+--let $restore_database = restore
+eval CREATE DATABASE $restore_database;
+eval SHOW CREATE DATABASE $restore_database;
+
+--echo
+--echo
+--echo *** Restoring data from the logical dump
+--exec $MYSQL $restore_database < $fresh_dump_file
+
+--remove_file $fresh_dump_file
+
+--echo
+--echo
+--echo *** Checking events in the binary log
+--let $binlog_dump_file = $MYSQL_TMP_DIR/binlog_dump.sql
+--exec $MYSQL_BINLOG --read-from-remote-server --host=127.0.0.1 --port=$server_port --user=root --to-last-log $binlog_file > $binlog_dump_file
+
+# The binary log is expected to have the following 2 lines with collation 45 (utf8mb4_general_ci)
+# SET @@session.character_set_client=45,@@session.collation_connection=45,@@session.collation_server=45/*!*/;
+# /*!80011 SET @@session.default_collation_for_utf8mb4=45*//*!*/;
+--let $assert_text = Events in the binary log must use utf8mb4_general_ci (45) collation
+--let $assert_file = $binlog_dump_file
+--let $assert_select = =45
+--let $assert_count = 2
+--source include/assert_grep.inc
+
+--let $assert_text = Events in the binary log must not use utf8mb4_0900_ai_ci (255) collation
+--let $assert_file = $binlog_dump_file
+--let $assert_select = =255
+--let $assert_count = 0
+--source include/assert_grep.inc
+
+--remove_file $binlog_dump_file
+
+
+--disconnect con1
+--connection default
+--source include/wait_until_count_sessions.inc
+
+--echo
+--echo
+--echo *** Dropping fresh database
+eval DROP DATABASE $restore_database;
+eval DROP DATABASE $fresh_database;
+
+--echo
+--echo
+--echo *** Restoring collation variables
+eval SET GLOBAL collation_database = $saved_collation_database;
+eval SET GLOBAL collation_connection = $saved_collation_connection;
+eval SET GLOBAL collation_server = $saved_collation_server;
+eval SET GLOBAL default_collation_for_utf8mb4 = $saved_default_collation_for_utf8mb4;

sql/sql_connect.cc

@@ -670,6 +670,24 @@ void reset_mqh(THD *thd, LEX_USER *lu, bool get_them = false) {
 
 bool thd_init_client_charset(THD *thd, uint cs_number) {
   CHARSET_INFO *cs;
+
+  // if the 8.0 client sets 'MYSQL_SET_CHARSET_NAME' option to 'utf8mb4' or
+  // leaves it empty, basically meaning the same, this function will be called
+  // with 'cs_number' equal to 255 (meaning 'utf8mb4_0900_ai_ci')
+
+  // at the same time, if 'default_collation_for_utf8mb4' is set to something
+  // other than default 'utf8mb4' collation ('utf8mb4_0900_ai_ci', number 255),
+  // we need to fix 'cs_number' here by setting it to the corresponding number
+  // of 'default_collation_for_utf8mb4' (currently only 'utf8mb4_general_ci',
+  // number 45, is supported)
+  const auto *primary_utf8mb4_collation =
+      get_charset_by_csname("utf8mb4", MY_CS_PRIMARY, MYF(0));
+  if (thd->variables.default_collation_for_utf8mb4 !=
+      primary_utf8mb4_collation) {
+    if (cs_number == primary_utf8mb4_collation->number) {
+      cs_number = thd->variables.default_collation_for_utf8mb4->number;
+    }
+  }
   /*
    Use server character set and collation if
    - opt_character_set_client_handshake is not set

sql/sys_vars.cc

@@ -1869,6 +1869,21 @@ static bool check_charset(sys_var *, THD *thd, set_var *var) {
         my_error(ER_UNKNOWN_CHARACTER_SET, MYF(0), err.ptr());
         return true;
       }
+      // if 'default_collation_for_utf8mb4' is set to something other than
+      // default 'utf8mb4' collation ('utf8mb4_0900_ai_ci') and if the value
+      // returned by 'get_charset_by_csname()' is also default 'utf8mb4'
+      // collation ('utf8mb4_0900_ai_ci'), meaning that were requesting for
+      // 'utf8mb4', we need to fix the returned value depending on the value of
+      // 'default_collation_for_utf8mb4' (currently, only 'utf8mb4_general_ci'
+      // is possible)
+      const auto *primary_utf8mb4_collation =
+          get_charset_by_csname("utf8mb4", MY_CS_PRIMARY, MYF(0));
+      if (thd->variables.default_collation_for_utf8mb4 !=
+          primary_utf8mb4_collation) {
+        if (var->save_result.ptr == primary_utf8mb4_collation) {
+          var->save_result.ptr = thd->variables.default_collation_for_utf8mb4;
+        }
+      }
       warn_on_deprecated_charset(
           thd, static_cast<const CHARSET_INFO *>(var->save_result.ptr),
           err.ptr());