Merge pull request #2030 from sysown/v2.0.4-internal_json

V2.0.4 internal json

deps/Makefile

@@ -118,7 +118,7 @@ sqlite3/sqlite3/sqlite3.o:
 	cd sqlite3 && rm -rf sqlite-amalgamation-3190200
 	cd sqlite3 && tar -zxf sqlite-amalgamation-3190200.tar.gz
 	cd sqlite3/sqlite3 && patch sqlite3.c < ../from_unixtime.patch
-	cd sqlite3/sqlite3 && ${CC} ${MYCFLAGS} -c -o sqlite3.o sqlite3.c -DSQLITE_ENABLE_MEMORY_MANAGEMENT
+	cd sqlite3/sqlite3 && ${CC} ${MYCFLAGS} -c -o sqlite3.o sqlite3.c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_JSON1
 
 sqlite3: sqlite3/sqlite3/sqlite3.o
 

include/MySQL_Data_Stream.h

@@ -84,6 +84,12 @@ class MySQL_Data_Stream
 	unsigned long long wait_until;
 	unsigned long long killed_at;
 	unsigned long long max_connect_time;
+
+	struct {
+		unsigned long long questions;
+		unsigned long long myconnpoll_get;
+		unsigned long long myconnpoll_put;
+	} statuses;
 
 	PtrSizeArray *PSarrayIN;
 	PtrSizeArray *PSarrayOUT;
@@ -180,13 +186,17 @@ class MySQL_Data_Stream
 
 	// safe way to attach a MySQL Connection
 	void attach_connection(MySQL_Connection *mc) {
+		statuses.myconnpoll_get++;
 		myconn=mc;
+		myconn->statuses.myconnpoll_get++;
 		mc->myds=this;
 	}
 
 	// safe way to detach a MySQL Connection
 	void detach_connection() {
 		assert(myconn);
+		myconn->statuses.myconnpoll_put++;
+		statuses.myconnpoll_put++;
 		myconn->myds=NULL;
 		myconn=NULL;
 	}

include/MySQL_HostGroups_Manager.h

@@ -537,6 +537,7 @@ class MySQL_HostGroups_Manager {
 	void drop_all_idle_connections();
 	int get_multiple_idle_connections(int, unsigned long long, MySQL_Connection **, int);
 	SQLite3_result * SQL3_Connection_Pool(bool _reset);
+	SQLite3_result * SQL3_Free_Connections();
 
 	void push_MyConn_to_pool(MySQL_Connection *, bool _lock=true);
 	void push_MyConn_to_pool_array(MySQL_Connection **, unsigned int);

include/MySQL_PreparedStatement.h

@@ -181,6 +181,9 @@ class MySQL_STMTs_meta {
 class MySQL_STMTs_local_v14 {
 	private:
 	bool is_client_;
+	std::stack<uint32_t> free_client_ids;
+	uint32_t local_max_stmt_id;
+	public:
 	// this map associate client_stmt_id to global_stmt_id : this is used only for client connections
 	std::map<uint32_t, uint64_t> client_stmt_to_global_ids;
 	// this multimap associate global_stmt_id to client_stmt_id : this is used only for client connections
@@ -193,9 +196,6 @@ class MySQL_STMTs_local_v14 {
 
 	std::map<uint64_t, MYSQL_STMT *> global_stmt_to_backend_stmt;
 
-	std::stack<uint32_t> free_client_ids;
-	uint32_t local_max_stmt_id;
-	public:
 	MySQL_Session *sess;
 	MySQL_STMTs_local_v14(bool _ic) {
 		local_max_stmt_id = 0;

include/MySQL_Session.h

@@ -3,6 +3,9 @@
 #include "proxysql.h"
 #include "cpp.h"
 
+#include "../deps/json/json.hpp"
+using json = nlohmann::json;
+
 enum proxysql_session_type {
 	PROXYSQL_SESSION_MYSQL,
 	PROXYSQL_SESSION_ADMIN,
@@ -85,6 +88,7 @@ class MySQL_Session
 
 	void handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection();	
 
+	void return_proxysql_internal(PtrSize_t *);
 	bool handler_special_queries(PtrSize_t *);
 	bool handler_CommitRollback(PtrSize_t *);
 	bool handler_SetAutocommit(PtrSize_t *);
@@ -226,6 +230,7 @@ class MySQL_Session
 	void create_new_session_and_reset_connection(MySQL_Data_Stream *_myds);
 	bool handle_command_query_kill(PtrSize_t *);
 	void finishQuery(MySQL_Data_Stream *myds, MySQL_Connection *myconn, bool);
+	void generate_proxysql_internal_session_json(json &);
 };
 
 #define KILL_QUERY       1

include/MySQL_Thread.h

@@ -369,6 +369,7 @@ class MySQL_Threads_Handler
 		int connect_timeout_server;
 		int connect_timeout_server_max;
 		int free_connections_pct;
+		int show_processlist_extended;
 #ifdef IDLE_THREADS
 		int session_idle_ms;
 		bool session_idle_show_processlist;

include/mysql_connection.h

@@ -84,6 +84,14 @@ class MySQL_Connection {
 	MySQL_Connection_userinfo *userinfo;
 	MySQL_Data_Stream *myds;
 	enum MySerStatus server_status; // this to solve a side effect of #774
+
+	bytes_stats_t bytes_info; // bytes statistics
+	struct {
+		unsigned long long questions;
+		unsigned long long myconnpoll_get;
+		unsigned long long myconnpoll_put;
+	} statuses;
+
 	unsigned long largest_query_length;
 	uint32_t status_flags;
 	int async_exit_status; // exit status of MariaDB Client Library Non blocking API

include/proxysql_admin.h

@@ -102,6 +102,7 @@ class ProxySQL_Admin {
 		int stats_mysql_query_cache;
 		int stats_system_cpu;
 		int stats_system_memory;
+		int mysql_show_processlist_extended;
 		bool web_enabled;
 		bool web_enabled_old;
 		int web_port;
@@ -242,6 +243,7 @@ class ProxySQL_Admin {
 	//void stats___mysql_query_digests_reset();
 	void stats___mysql_commands_counters();
 	void stats___mysql_processlist();
+	void stats___mysql_free_connections();
 	void stats___mysql_connection_pool(bool _reset);
 	void stats___mysql_errors(bool reset);
 	void stats___memory_metrics();

include/proxysql_structs.h

@@ -650,6 +650,7 @@ __thread bool mysql_thread___query_digests_normalize_digest_text;
 __thread bool mysql_thread___query_digests_track_hostname;
 __thread int mysql_thread___query_digests_max_digest_length;
 __thread int mysql_thread___query_digests_max_query_length;
+__thread int mysql_thread___show_processlist_extended;
 __thread bool mysql_thread___default_reconnect;
 __thread bool mysql_thread___session_idle_show_processlist;
 __thread bool mysql_thread___sessions_sort;
@@ -773,6 +774,7 @@ extern __thread bool mysql_thread___query_digests_normalize_digest_text;
 extern __thread bool mysql_thread___query_digests_track_hostname;
 extern __thread int mysql_thread___query_digests_max_digest_length;
 extern __thread int mysql_thread___query_digests_max_query_length;
+extern __thread int mysql_thread___show_processlist_extended;
 extern __thread bool mysql_thread___default_reconnect;
 extern __thread bool mysql_thread___session_idle_show_processlist;
 extern __thread bool mysql_thread___sessions_sort;

lib/MySQL_HostGroups_Manager.cpp

@@ -2636,6 +2636,115 @@ void MySQL_HostGroups_Manager::set_incoming_galera_hostgroups(SQLite3_result *s)
 	incoming_galera_hostgroups=s;
 }
 
+SQLite3_result * MySQL_HostGroups_Manager::SQL3_Free_Connections() {
+	const int colnum=13;
+	proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Free Connections in Pool\n");
+	SQLite3_result *result=new SQLite3_result(colnum);
+	result->add_column_definition(SQLITE_TEXT,"fd");
+	result->add_column_definition(SQLITE_TEXT,"hostgroup");
+	result->add_column_definition(SQLITE_TEXT,"srv_host");
+	result->add_column_definition(SQLITE_TEXT,"srv_port");
+	result->add_column_definition(SQLITE_TEXT,"user");
+	result->add_column_definition(SQLITE_TEXT,"schema");
+	result->add_column_definition(SQLITE_TEXT,"init_connect");
+	result->add_column_definition(SQLITE_TEXT,"time_zone");
+	result->add_column_definition(SQLITE_TEXT,"sql_mode");
+	result->add_column_definition(SQLITE_TEXT,"autocommit");
+	result->add_column_definition(SQLITE_TEXT,"idle_ms");
+	result->add_column_definition(SQLITE_TEXT,"statistics");
+	result->add_column_definition(SQLITE_TEXT,"mysql_info");
+	unsigned long long curtime = monotonic_time();
+	wrlock();
+	int i,j, k, l;
+	for (i=0; i<(int)MyHostGroups->len; i++) {
+		MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i);
+		for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) {
+			MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j);
+			if (mysrvc->status!=MYSQL_SERVER_STATUS_ONLINE) {
+				proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port);
+				mysrvc->ConnectionsFree->drop_all_connections();
+			}
+			// drop idle connections if beyond max_connection
+			while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) {
+				//MySQL_Connection *conn=(MySQL_Connection *)mysrvc->ConnectionsFree->conns->remove_index_fast(0);
+				MySQL_Connection *conn=mysrvc->ConnectionsFree->remove(0);
+				delete conn;
+			}
+			char buf[1024];
+			for (l=0; l<mysrvc->ConnectionsFree->conns_length(); l++) {
+				char **pta=(char **)malloc(sizeof(char *)*colnum);
+				MySQL_Connection *conn = mysrvc->ConnectionsFree->index(l);
+				sprintf(buf,"%d", conn->fd);
+				pta[0]=strdup(buf);
+				sprintf(buf,"%d", (int)myhgc->hid);
+				pta[1]=strdup(buf);
+				pta[2]=strdup(mysrvc->address);
+				sprintf(buf,"%d", mysrvc->port);
+				pta[3]=strdup(buf);
+				pta[4] = strdup(conn->userinfo->username);
+				pta[5] = strdup(conn->userinfo->schemaname);
+				pta[6] = NULL;
+				if (conn->options.init_connect) {
+					pta[6] = strdup(conn->options.init_connect);
+				}
+				pta[7] = NULL;
+				if (conn->options.time_zone) {
+					pta[7] = strdup(conn->options.time_zone);
+				}
+				pta[8] = NULL;
+				if (conn->options.sql_mode) {
+					pta[8] = strdup(conn->options.sql_mode);
+				}
+				sprintf(buf,"%d", conn->options.autocommit);
+				pta[9]=strdup(buf);
+				sprintf(buf,"%llu", (curtime-conn->last_time_used)/1000);
+				pta[10]=strdup(buf);
+				{
+					json j;
+					j["bytes_recv"] = conn->bytes_info.bytes_recv;
+					j["bytes_sent"] = conn->bytes_info.bytes_sent;
+					j["myconnpoll_get"] = conn->statuses.myconnpoll_get;
+					j["myconnpoll_put"] = conn->statuses.myconnpoll_put;
+					j["questions"] = conn->statuses.questions;
+					string s = j.dump();
+					pta[11] = strdup(s.c_str());
+				}
+				{
+					MYSQL *_my = conn->mysql;
+					json j;
+					j["host"] = _my->host;
+					j["host_info"] = _my->host_info;
+					j["port"] = _my->port;
+					j["server_version"] = _my->server_version;
+					j["user"] = _my->user;
+					j["unix_socket"] = (_my->unix_socket ? _my->unix_socket : "");
+					j["db"] = (_my->db ? _my->db : "");
+					j["affected_rows"] = _my->affected_rows;
+					j["insert_id"] = _my->insert_id;
+					j["server_status"] = _my->server_status;
+					j["charset"] = _my->charset->nr;
+					j["options"]["charset_name"] = _my->options.charset_name;
+					j["options"]["use_ssl"] = _my->options.use_ssl;
+					j["net"]["last_errno"] = _my->net.last_errno;
+					j["net"]["fd"] = _my->net.fd;
+					j["net"]["max_packet_size"] = _my->net.max_packet_size;
+					j["net"]["sqlstate"] = _my->net.sqlstate;
+					string s = j.dump();
+					pta[12] = strdup(s.c_str());
+				}
+				result->add_row(pta);
+				for (k=0; k<colnum; k++) {
+					if (pta[k])
+						free(pta[k]);
+				}
+				free(pta);
+			}
+		}
+	}
+	wrunlock();
+	return result;
+}
+
 SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset) {
   const int colnum=14;
   proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Connection Pool\n");