Skip to content

Commit

Permalink
Merge pull request #4274 from sysown/v2.x-4268
Browse files Browse the repository at this point in the history
Reworking of SET parser
  • Loading branch information
renecannao authored Jun 30, 2023
2 parents c07d4b0 + a6ff9e3 commit fb53926
Show file tree
Hide file tree
Showing 21 changed files with 847 additions and 153 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ test/tap/tap/cpp-dotenv/cpp-dotenv-*
test/tap/tests/galera_1_timeout_count
test/tap/tests/galera_2_timeout_no_count
test/tap/tests/setparser_test
test/tap/tests/setparser_test2
test/tap/tests/setparser_test3
test/tap/tests/reg_test_3504-change_user_libmariadb_helper
test/tap/tests/reg_test_3504-change_user_libmysql_helper
test/tap/tests/generate_set_session_csv
Expand Down
17 changes: 12 additions & 5 deletions include/MySQL_Thread.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

#include "prometheus_helpers.h"

#include "set_parser.h"

#define MIN_POLL_LEN 8
#define MIN_POLL_DELETE_RATIO 8
#define MY_EPOLL_THREAD_MAXEVENTS 128
Expand Down Expand Up @@ -214,11 +216,15 @@ class __attribute__((aligned(64))) MySQL_Thread
bool query_cache_stores_empty_result;
} variables;

pthread_mutex_t thread_mutex;
MySQL_Thread();
~MySQL_Thread();
MySQL_Session * create_new_session_and_client_data_stream(int _fd);
bool init();
pthread_mutex_t thread_mutex;

// if set_parser_algorithm == 2 , a single thr_SetParser is used
SetParser *thr_SetParser;

MySQL_Thread();
~MySQL_Thread();
MySQL_Session * create_new_session_and_client_data_stream(int _fd);
bool init();
void run___get_multiple_idle_connections(int& num_idles);
void run___cleanup_mirror_queue();
void ProcessAllMyDS_BeforePoll();
Expand Down Expand Up @@ -538,6 +544,7 @@ class MySQL_Threads_Handler
int query_processor_iterations;
int query_processor_regex;
int set_query_lock_on_hostgroup;
int set_parser_algorithm;
int reset_connection_algorithm;
int auto_increment_delay_multiplex;
int auto_increment_delay_multiplex_timeout_ms;
Expand Down
2 changes: 2 additions & 0 deletions include/proxysql_structs.h
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,7 @@ __thread int mysql_thread___connect_timeout_server_max;
__thread int mysql_thread___query_processor_iterations;
__thread int mysql_thread___query_processor_regex;
__thread int mysql_thread___set_query_lock_on_hostgroup;
__thread int mysql_thread___set_parser_algorithm;
__thread int mysql_thread___reset_connection_algorithm;
__thread uint32_t mysql_thread___server_capabilities;
__thread int mysql_thread___auto_increment_delay_multiplex;
Expand Down Expand Up @@ -981,6 +982,7 @@ extern __thread int mysql_thread___connect_timeout_server_max;
extern __thread int mysql_thread___query_processor_iterations;
extern __thread int mysql_thread___query_processor_regex;
extern __thread int mysql_thread___set_query_lock_on_hostgroup;
extern __thread int mysql_thread___set_parser_algorithm;
extern __thread int mysql_thread___reset_connection_algorithm;
extern __thread uint32_t mysql_thread___server_capabilities;
extern __thread int mysql_thread___auto_increment_delay_multiplex;
Expand Down
30 changes: 30 additions & 0 deletions include/set_parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,44 @@
#include <map>
#include <vector>

#include "re2/re2.h"
#include "re2/regexp.h"


class SetParser {
private:
// parse1v2 variables used for compile the RE only once
bool parse1v2_init;
re2::RE2::Options * parse1v2_opt2;
re2::RE2 * parse1v2_re;
std::string parse1v2_pattern;
std::string query;
#ifdef PARSERDEBUG
int verbosity;
public:
SetParser(std::string q, int verb = 0);
#else
public:
SetParser(std::string q);
#endif
// set_query() allows to change the query associated to a SetParser.
// This allow to parse multiple queries using just a single SetParser.
// At the moment this makes sense only when using parse1v2() because it
// allows to compile the regular expression only once
void set_query(const std::string& q);
// First implementation of the general parser
// It uses a single complex RE pattern that is hardcoded
std::map<std::string, std::vector<std::string>> parse1();
// Second implementation of the general parser .
// It uses a RE pattern that is built at runtime .
// The final pattern used by parse1v2() is a lot longer than the one used by parse1()
// making it very difficult to read, but the code generating it should be clear
std::map<std::string, std::vector<std::string>> parse1v2();
void generateRE_parse1v2();
// First implemenation of the parser for TRANSACTION ISOLATION LEVEL and TRANSACTION READ/WRITE
std::map<std::string, std::vector<std::string>> parse2();
std::string parse_character_set();
~SetParser();
};


Expand Down
11 changes: 9 additions & 2 deletions lib/MySQL_Session.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
#include "re2/re2.h"
#include "re2/regexp.h"
#include "mysqld_error.h"
#include "set_parser.h"

#include "MySQL_Data_Stream.h"
#include "query_processor.h"
Expand Down Expand Up @@ -6022,7 +6021,15 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Parsing SET command %s\n", nq.c_str());
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Parsing SET command = %s\n", nq.c_str());
SetParser parser(nq);
std::map<std::string, std::vector<std::string>> set = parser.parse1();
std::map<std::string, std::vector<std::string>> set = {};
if (mysql_thread___set_parser_algorithm == 1) { // legacy behavior
set = parser.parse1();
} else if (mysql_thread___set_parser_algorithm == 2) { // we use a single SetParser per thread
thread->thr_SetParser->set_query(nq); // replace the query
set = thread->thr_SetParser->parse1v2(); // use algorithm v2
} else {
assert(0);
}
// Flag to be set if any variable within the 'SET' statement fails to be tracked,
// due to being unknown or because it's an user defined variable.
bool failed_to_parse_var = false;
Expand Down
10 changes: 10 additions & 0 deletions lib/MySQL_Thread.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,7 @@ static char * mysql_thread_variables_names[]= {
(char *)"query_processor_iterations",
(char *)"query_processor_regex",
(char *)"set_query_lock_on_hostgroup",
(char *)"set_parser_algorithm",
(char *)"reset_connection_algorithm",
(char *)"auto_increment_delay_multiplex",
(char *)"auto_increment_delay_multiplex_timeout_ms",
Expand Down Expand Up @@ -1140,6 +1141,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() {
variables.query_processor_iterations=0;
variables.query_processor_regex=1;
variables.set_query_lock_on_hostgroup=1;
variables.set_parser_algorithm=1; // in 2.6.0 this must become 2
variables.reset_connection_algorithm=2;
variables.auto_increment_delay_multiplex=5;
variables.auto_increment_delay_multiplex_timeout_ms=10000;
Expand Down Expand Up @@ -2234,6 +2236,7 @@ char ** MySQL_Threads_Handler::get_variables_list() {
VariablesPointers_int["query_processor_regex"] = make_tuple(&variables.query_processor_regex, 1, 2, false);
VariablesPointers_int["query_retries_on_failure"] = make_tuple(&variables.query_retries_on_failure, 0, 1000, false);
VariablesPointers_int["set_query_lock_on_hostgroup"] = make_tuple(&variables.set_query_lock_on_hostgroup, 0, 1, false);
VariablesPointers_int["set_parser_algorithm"] = make_tuple(&variables.set_parser_algorithm, 1, 2, false);

// throttle
VariablesPointers_int["throttle_connections_per_sec_to_hostgroup"] = make_tuple(&variables.throttle_connections_per_sec_to_hostgroup, 1, 100*1000*1000, false);
Expand Down Expand Up @@ -2856,6 +2859,10 @@ MySQL_Thread::~MySQL_Thread() {
free(match_regexes);
match_regexes=NULL;
}
if (thr_SetParser != NULL) {
delete thr_SetParser;
thr_SetParser = NULL;
}

}

Expand Down Expand Up @@ -2962,6 +2969,7 @@ bool MySQL_Thread::init() {
mypolls.add(POLLIN, pipefd[0], NULL, 0);
assert(i==0);

thr_SetParser = new SetParser("");
match_regexes=(Session_Regex **)malloc(sizeof(Session_Regex *)*4);
// match_regexes[0]=new Session_Regex((char *)"^SET (|SESSION |@@|@@session.)SQL_LOG_BIN( *)(:|)=( *)");
match_regexes[0] = NULL; // NOTE: historically we used match_regexes[0] for SET SQL_LOG_BIN . Not anymore
Expand Down Expand Up @@ -4016,6 +4024,7 @@ void MySQL_Thread::refresh_variables() {
mysql_thread___query_processor_iterations=GloMTH->get_variable_int((char *)"query_processor_iterations");
mysql_thread___query_processor_regex=GloMTH->get_variable_int((char *)"query_processor_regex");
mysql_thread___set_query_lock_on_hostgroup=GloMTH->get_variable_int((char *)"set_query_lock_on_hostgroup");
mysql_thread___set_parser_algorithm=GloMTH->get_variable_int((char *)"set_parser_algorithm");
mysql_thread___reset_connection_algorithm=GloMTH->get_variable_int((char *)"reset_connection_algorithm");
mysql_thread___auto_increment_delay_multiplex=GloMTH->get_variable_int((char *)"auto_increment_delay_multiplex");
mysql_thread___auto_increment_delay_multiplex_timeout_ms=GloMTH->get_variable_int((char *)"auto_increment_delay_multiplex_timeout_ms");
Expand Down Expand Up @@ -4251,6 +4260,7 @@ MySQL_Thread::MySQL_Thread() {
mysql_thread___default_variables[i] = NULL;
}
shutdown=0;
thr_SetParser = NULL;
}

void MySQL_Thread::register_session_connection_handler(MySQL_Session *_sess, bool _new) {
Expand Down
Loading

0 comments on commit fb53926

Please sign in to comment.