Skip to content

Commit

Permalink
TiDB settings
Browse files Browse the repository at this point in the history
  • Loading branch information
@DallasC
DallasC committed Jan 16, 2021
1 parent dc6ecf2 commit 35b27d3
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 76 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
notes.md

89 changes: 13 additions & 76 deletions query-featurized-rl-tuner/environment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,97 +17,38 @@
class Database:
def __init__(self):
self.connection = pymysql.connect(host='localhost',
port=3306,
port=4000,
user='root',
password='db2019',
password='',
db='INFORMATION_SCHEMA',
cursorclass=pycursor.DictCursor)

# self.internal_metric_num = 13 # 13(state) + cumulative()
self.external_metric_num = 2 # [throughput, latency] # num_event / t
self.external_metric_num = 2 # [throughput, latency]
self.internal_metric_num = 57 # 57 for tidb%count metrics, 110 for tidb%count AND tikv%count

# bulk_insert_buffer_size connect_timeout default_week_format delayed_insert_limit delayed_insert_timeout delayed_queue_size div_precision_increment flush_time
# host_cache_size interactive_timeout key_buffer_size key_cache_age_threshold key_cache_block_size key_cache_division_limit
# log_throttle_queries_not_using_indexes log_warnings
# max_allowed_packet max_connect_errors max_error_count max_heap_table_size max_insert_delayed_threads(no range) max_join_size(1, 18446744073709551615)
# max_length_for_sort_data（4， 8388608） max_prepared_stmt_count（0， 104856） max_seeks_for_key（1， 4294967295）
# max_sp_recursion_depth（0,255） max_user_connections（0，4294967295） max_write_lock_count（1,，4294967295） min_examined_row_limit（0，4294967295）
# myisam_max_sort_file_size（？？） myisam_repair_threads（1，4294967295）
self.var_names = ["tidb_distsql_scan_concurrency", "tidb_hash_join_concurrency", "tidb_hashagg_final_concurrency", "tidb_hashagg_partial_concurrency", "tidb_index_join_batch_size", "tidb_index_lookup_concurrency", "tidb_index_lookup_join_concurrency", "tidb_index_lookup_size", "tidb_index_serial_scan_concurrency", "tidb_projection_concurrency", "tidb_window_concurrency", "tidb_union_concurrency", "tidb_init_chunk_size", "tidb_max_chunk_size", "tidb_opt_correlation_exp_factor", "tidb_opt_insubq_to_join_and_agg"] # 16 dynamic variables

self.var_names = ["table_open_cache", "max_connections", "innodb_buffer_pool_size", "binlog_cache_size",
"max_length_for_sort_data", "max_prepared_stmt_count", "max_sp_recursion_depth",
"max_user_connections", "myisam_repair_threads"] # 9 dynamic variables

'''
self.i_names = ["lock_row_lock_time_max","lock_row_lock_time_avg","buffer_pool_size","buffer_pool_pages_total","buffer_pool_pages_misc",
"buffer_pool_pages_data","buffer_pool_bytes_data","buffer_pool_pages_dirty","buffer_pool_bytes_dirty","buffer_pool_pages_free",
"trx_rseg_history_len","file_num_open_files","innodb_page_size",]
["lock_deadlocks","lock_timeouts","lock_row_lock_current_waits","lock_row_lock_time","" ]
'''
self.knob_num = len(self.var_names)
self.internal_metric_num = 65 # default enabled in metric_innodb

# Resource Distribution
# table (col(value_distr, type, index))


def close(self):
self.connection.close()

def fetch_internal_metrics(self):
with self.connection.cursor() as cursor:
######### observation_space
# State_status
# [lock_row_lock_time_max, lock_row_lock_time_avg, buffer_pool_size,
# buffer_pool_pages_total, buffer_pool_pages_misc, buffer_pool_pages_data, buffer_pool_bytes_data,
# buffer_pool_pages_dirty, buffer_pool_bytes_dirty, buffer_pool_pages_free, trx_rseg_history_len,
# file_num_open_files, innodb_page_size]
# Cumulative_status
# [lock_row_lock_current_waits, ]
'''
sql = "select count from INNODB_METRICS where name='lock_row_lock_time_max' or name='lock_row_lock_time_avg'\
or name='buffer_pool_size' or name='buffer_pool_pages_total' or name='buffer_pool_pages_misc' or name='buffer_pool_pages_data'\
or name='buffer_pool_bytes_data' or name='buffer_pool_pages_dirty' or name='buffer_pool_bytes_dirty' or name='buffer_pool_pages_free'\
or name='trx_rseg_history_len' or name='file_num_open_files' or name='innodb_page_size'"
'''
sql = "SELECT count FROM INNODB_METRICS where status='enabled'"
sql = "SELECT sum_value FROM information_schema.metrics_summary WHERE metrics_name LIKE 'tidb%count'" #57 total metrics
# sql = "SELECT * FROM information_schema.metrics_summary WHERE metrics_name LIKE 'tidb%count' OR metrics_name LIKE 'tikv%count" #110 total metrics
cursor.execute(sql)
result = cursor.fetchall()
state_list = np.array([])
for s in result:
state_list = np.append(state_list, [s['count']])
state_list = np.append(state_list, [s['sum_value']])

return state_list

def fetch_knob(self):
with self.connection.cursor() as cursor:
######### action_space
# Common part
# '''
# sql = "select @@table_open_cache, @@max_connections, @@innodb_buffer_pool_size, @@innodb_buffer_pool_instances,\
# @@innodb_log_files_in_group, @@innodb_log_file_size, @@innodb_purge_threads, @@innodb_read_io_threads,\
# @@innodb_write_io_threads, @@binlog_cache_size"
# '''

# Read-only
# innodb_buffer_pool_instances innodb_log_files_in_group innodb_log_file_size innodb_purge_threads innodb_read_io_threads
# innodb_write_io_threads

# Extended part
# innodb_adaptive_max_sleep_delay(0,1000000) innodb_change_buffer_max_size(0,50) innodb_flush_log_at_timeout(1,2700) innodb_flushing_avg_loops(1,1000)
# innodb_max_purge_lag(0,4294967295) innodb_old_blocks_pct(5,95) innodb_read_ahead_threshold(0,64) innodb_replication_delay(0,4294967295)
# innodb_rollback_segments(1,128) innodb_adaptive_flushing_lwm(0,70) innodb_sync_spin_loops (0,4294967295)
# innodb_lock_wait_timeout(1,1073741824) innodb_autoextend_increment(1,1000) innodb_concurrency_tickets(1,4294967295) innodb_max_dirty_pages_pct(0,99)
# innodb_max_dirty_pages_pct_lwm(0,99) innodb_io_capacity(100, 2**32-1) innodb_lru_scan_depth(100, 2**32-1) innodb_old_blocks_time(0, 2**32-1)
# innodb_purge_batch_size(1,5000) innodb_spin_wait_delay(0,2**32-1)

# Non-dynmic
# innodb_sync_array_size metadata_locks_cache_size metadata_locks_hash_instances innodb_log_buffer_size eq_range_index_dive_limit max_length_for_sort_data
# read_rnd_buffer_size table_open_cache_instances transaction_prealloc_size binlog_order_commits query_cache_limit query_cache_size query_cache_type query_prealloc_size
# join_buffer_size tmp_table_size max_seeks_for_key query_alloc_block_size sort_buffer_size thread_cache_size max_write_lock_count

sql = "select @@table_open_cache, @@max_connections, @@innodb_buffer_pool_size, @@binlog_cache_size,\
@@max_length_for_sort_data, @@max_prepared_stmt_count, @@max_sp_recursion_depth, @@max_user_connections, @@myisam_repair_threads"
# @@tidb_opt_agg_push_down, tidb_opt_distinct_agg_push_down # Are both Session only variables not global
sql = "select @@tidb_distsql_scan_concurrency, @@tidb_hash_join_concurrency, @@tidb_hashagg_final_concurrency, @@tidb_hashagg_partial_concurrency, @@tidb_index_join_batch_size, @@tidb_index_lookup_concurrency, @@tidb_index_lookup_join_concurrency, @@tidb_index_lookup_size, @@tidb_index_serial_scan_concurrency, @@tidb_projection_concurrency, @@tidb_window_concurrency, @@tidb_union_concurrency, @@tidb_init_chunk_size, @@tidb_max_chunk_size, @@tidb_opt_correlation_exp_factor, @@tidb_opt_insubq_to_join_and_agg"

cursor.execute(sql)
result = cursor.fetchall()
Expand All @@ -125,8 +66,6 @@ def change_knob_nonrestart(self, actions):
for i in range(self.knob_num):
sql = 'set global %s=%d' % (self.var_names[i], actions[i])
cursor.execute(sql)
# result = cursor.fetchall()
# connection.commit()


o_low = np.array(
Expand Down Expand Up @@ -172,10 +111,8 @@ def change_knob_nonrestart(self, actions):
-10000000000, -10000000000
])

a_low = np.array([400, 151, 134217728, 32768,
1024, 16382, 0, 0, 1])
a_high = np.array([524288, 100000, 2 ** 32 - 1, 4294967295,
8388608, 1048576, 255, 4294967295, 4294967295])
a_low = np.array([1, 1, 1, 1, 500, 1, 1, 500, 1, 1, 1, 1, 1, 32, 0, 0])
a_high = np.array([32, 32, 32, 32, 100000, 32, 32, 100000, 32, 32, 32, 32, 32, 10000, 100, 1])

# Define the environment
class Environment(gym.Env):
Expand All @@ -188,7 +125,7 @@ def __init__(self, db, argus):
p_u_index=argus['p_u_index'], p_i=argus['p_i'], p_d=argus['p_d'])

self.state_num = db.internal_metric_num
self.action_num = db.knob_num # 4 in fact
self.action_num = db.knob_num

# state_space
self.o_low = o_low
Expand Down
10 changes: 10 additions & 0 deletions query-featurized-rl-tuner/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@

numpy==1.14.5
pandas==1.2.0
gym==0.18.0
psutil==5.4.6
pymysql==0.10.0
tensorflow==2.4.0
keras==2.4.3
scikit-learn==0.24.0

0 comments on commit 35b27d3

Please sign in to comment.