Merge pull request #4274 from sysown/v2.x-4268

Reworking of SET parser

.gitignore

@@ -116,6 +116,8 @@ test/tap/tap/cpp-dotenv/cpp-dotenv-*
 test/tap/tests/galera_1_timeout_count
 test/tap/tests/galera_2_timeout_no_count
 test/tap/tests/setparser_test
+test/tap/tests/setparser_test2
+test/tap/tests/setparser_test3
 test/tap/tests/reg_test_3504-change_user_libmariadb_helper
 test/tap/tests/reg_test_3504-change_user_libmysql_helper
 test/tap/tests/generate_set_session_csv

include/MySQL_Thread.h

@@ -14,6 +14,8 @@
 
 #include "prometheus_helpers.h"
 
+#include "set_parser.h"
+
 #define MIN_POLL_LEN 8
 #define MIN_POLL_DELETE_RATIO  8
 #define MY_EPOLL_THREAD_MAXEVENTS 128
@@ -214,11 +216,15 @@ class __attribute__((aligned(64))) MySQL_Thread
 		bool query_cache_stores_empty_result;
 	} variables;
 
-  pthread_mutex_t thread_mutex;
-  MySQL_Thread();
-  ~MySQL_Thread();
-  MySQL_Session * create_new_session_and_client_data_stream(int _fd);
-  bool init();
+	pthread_mutex_t thread_mutex;
+
+	// if set_parser_algorithm == 2 , a single thr_SetParser is used
+	SetParser *thr_SetParser;
+
+	MySQL_Thread();
+	~MySQL_Thread();
+	MySQL_Session * create_new_session_and_client_data_stream(int _fd);
+	bool init();
 	void run___get_multiple_idle_connections(int& num_idles);
 	void run___cleanup_mirror_queue();
   	void ProcessAllMyDS_BeforePoll();
@@ -538,6 +544,7 @@ class MySQL_Threads_Handler
 		int query_processor_iterations;
 		int query_processor_regex;
 		int set_query_lock_on_hostgroup;
+		int set_parser_algorithm;
 		int reset_connection_algorithm;
 		int auto_increment_delay_multiplex;
 		int auto_increment_delay_multiplex_timeout_ms;

include/proxysql_structs.h

@@ -815,6 +815,7 @@ __thread int mysql_thread___connect_timeout_server_max;
 __thread int mysql_thread___query_processor_iterations;
 __thread int mysql_thread___query_processor_regex;
 __thread int mysql_thread___set_query_lock_on_hostgroup;
+__thread int mysql_thread___set_parser_algorithm;
 __thread int mysql_thread___reset_connection_algorithm;
 __thread uint32_t mysql_thread___server_capabilities;
 __thread int mysql_thread___auto_increment_delay_multiplex;
@@ -981,6 +982,7 @@ extern __thread int mysql_thread___connect_timeout_server_max;
 extern __thread int mysql_thread___query_processor_iterations;
 extern __thread int mysql_thread___query_processor_regex;
 extern __thread int mysql_thread___set_query_lock_on_hostgroup;
+extern __thread int mysql_thread___set_parser_algorithm;
 extern __thread int mysql_thread___reset_connection_algorithm;
 extern __thread uint32_t mysql_thread___server_capabilities;
 extern __thread int mysql_thread___auto_increment_delay_multiplex;

include/set_parser.h

@@ -4,14 +4,44 @@
 #include <map>
 #include <vector>
 
+#include "re2/re2.h"
+#include "re2/regexp.h"
+
+
 class SetParser {
 	private:
+	// parse1v2 variables used for compile the RE only once
+	bool parse1v2_init;
+	re2::RE2::Options * parse1v2_opt2;
+	re2::RE2 * parse1v2_re;
+	std::string parse1v2_pattern;
 	std::string query;
+#ifdef PARSERDEBUG
+	int verbosity;
+	public:
+	SetParser(std::string q, int verb = 0);
+#else
 	public:
 	SetParser(std::string q);
+#endif
+	// set_query() allows to change the query associated to a SetParser.
+	// This allow to parse multiple queries using just a single SetParser.
+	// At the moment this makes sense only when using parse1v2() because it
+	// allows to compile the regular expression only once
+	void set_query(const std::string& q);
+	// First implementation of the general parser
+	// It uses a single complex RE pattern that is hardcoded
 	std::map<std::string, std::vector<std::string>> parse1();
+	// Second implementation of the general parser .
+	// It uses a RE pattern that is built at runtime .
+	// The final pattern used by parse1v2() is a lot longer than the one used by parse1()
+	// making it very difficult to read, but the code generating it should be clear
+	std::map<std::string, std::vector<std::string>> parse1v2();
+	void generateRE_parse1v2();
+	// First implemenation of the parser for TRANSACTION ISOLATION LEVEL and TRANSACTION READ/WRITE
 	std::map<std::string, std::vector<std::string>> parse2();
 	std::string parse_character_set();
+	~SetParser();
 };
 
 

lib/MySQL_Session.cpp

@@ -6,7 +6,6 @@
 #include "re2/re2.h"
 #include "re2/regexp.h"
 #include "mysqld_error.h"
-#include "set_parser.h"
 
 #include "MySQL_Data_Stream.h"
 #include "query_processor.h"
@@ -6022,7 +6021,15 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
 				proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Parsing SET command %s\n", nq.c_str());
 				proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Parsing SET command = %s\n", nq.c_str());
 				SetParser parser(nq);
-				std::map<std::string, std::vector<std::string>> set = parser.parse1();
+				std::map<std::string, std::vector<std::string>> set = {};
+				if (mysql_thread___set_parser_algorithm == 1) { // legacy behavior
+					set = parser.parse1();
+				} else if (mysql_thread___set_parser_algorithm == 2) { // we use a single SetParser per thread
+					thread->thr_SetParser->set_query(nq); // replace the query
+					set = thread->thr_SetParser->parse1v2(); // use algorithm v2
+				} else {
+					assert(0);
+				}
 				// Flag to be set if any variable within the 'SET' statement fails to be tracked,
 				// due to being unknown or because it's an user defined variable.
 				bool failed_to_parse_var = false;

lib/MySQL_Thread.cpp

@@ -546,6 +546,7 @@ static char * mysql_thread_variables_names[]= {
 	(char *)"query_processor_iterations",
 	(char *)"query_processor_regex",
 	(char *)"set_query_lock_on_hostgroup",
+	(char *)"set_parser_algorithm",
 	(char *)"reset_connection_algorithm",
 	(char *)"auto_increment_delay_multiplex",
 	(char *)"auto_increment_delay_multiplex_timeout_ms",
@@ -1140,6 +1141,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() {
 	variables.query_processor_iterations=0;
 	variables.query_processor_regex=1;
 	variables.set_query_lock_on_hostgroup=1;
+	variables.set_parser_algorithm=1; // in 2.6.0 this must become 2
 	variables.reset_connection_algorithm=2;
 	variables.auto_increment_delay_multiplex=5;
 	variables.auto_increment_delay_multiplex_timeout_ms=10000;
@@ -2234,6 +2236,7 @@ char ** MySQL_Threads_Handler::get_variables_list() {
 		VariablesPointers_int["query_processor_regex"]           = make_tuple(&variables.query_processor_regex,            1,           2, false);
 		VariablesPointers_int["query_retries_on_failure"]        = make_tuple(&variables.query_retries_on_failure,         0,        1000, false);
 		VariablesPointers_int["set_query_lock_on_hostgroup"]     = make_tuple(&variables.set_query_lock_on_hostgroup,      0,           1, false);
+		VariablesPointers_int["set_parser_algorithm"]            = make_tuple(&variables.set_parser_algorithm,             1,           2, false);
 
 		// throttle
 		VariablesPointers_int["throttle_connections_per_sec_to_hostgroup"] = make_tuple(&variables.throttle_connections_per_sec_to_hostgroup, 1, 100*1000*1000, false);
@@ -2856,6 +2859,10 @@ MySQL_Thread::~MySQL_Thread() {
 		free(match_regexes);
 		match_regexes=NULL;
 	}
+	if (thr_SetParser != NULL) {
+		delete thr_SetParser;
+		thr_SetParser = NULL;
+	}
 
 }
 
@@ -2962,6 +2969,7 @@ bool MySQL_Thread::init() {
 	mypolls.add(POLLIN, pipefd[0], NULL, 0);
 	assert(i==0);
 
+	thr_SetParser = new SetParser("");
 	match_regexes=(Session_Regex **)malloc(sizeof(Session_Regex *)*4);
 //	match_regexes[0]=new Session_Regex((char *)"^SET (|SESSION |@@|@@session.)SQL_LOG_BIN( *)(:|)=( *)");
 	match_regexes[0] = NULL; // NOTE: historically we used match_regexes[0] for SET SQL_LOG_BIN . Not anymore
@@ -4016,6 +4024,7 @@ void MySQL_Thread::refresh_variables() {
 	mysql_thread___query_processor_iterations=GloMTH->get_variable_int((char *)"query_processor_iterations");
 	mysql_thread___query_processor_regex=GloMTH->get_variable_int((char *)"query_processor_regex");
 	mysql_thread___set_query_lock_on_hostgroup=GloMTH->get_variable_int((char *)"set_query_lock_on_hostgroup");
+	mysql_thread___set_parser_algorithm=GloMTH->get_variable_int((char *)"set_parser_algorithm");
 	mysql_thread___reset_connection_algorithm=GloMTH->get_variable_int((char *)"reset_connection_algorithm");
 	mysql_thread___auto_increment_delay_multiplex=GloMTH->get_variable_int((char *)"auto_increment_delay_multiplex");
 	mysql_thread___auto_increment_delay_multiplex_timeout_ms=GloMTH->get_variable_int((char *)"auto_increment_delay_multiplex_timeout_ms");
@@ -4251,6 +4260,7 @@ MySQL_Thread::MySQL_Thread() {
 		mysql_thread___default_variables[i] = NULL;
 	}
 	shutdown=0;
+	thr_SetParser = NULL;
 }
 
 void MySQL_Thread::register_session_connection_handler(MySQL_Session *_sess, bool _new) {