From de9662fcca5d029b038c1f91501225230c6e317e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 23 Dec 2022 12:43:55 +0530 Subject: [PATCH 01/25] feat: refactor generate commands of VTOrc to be in a single file Signed-off-by: Manan Gupta --- go/vt/vtorc/db/db.go | 3 +- go/vt/vtorc/db/generate_base.go | 1127 ++++++++++++++-------------- go/vt/vtorc/db/generate_patches.go | 559 -------------- 3 files changed, 551 insertions(+), 1138 deletions(-) delete mode 100644 go/vt/vtorc/db/generate_patches.go diff --git a/go/vt/vtorc/db/db.go b/go/vt/vtorc/db/db.go index b71a80a1dc2..91b46419a20 100644 --- a/go/vt/vtorc/db/db.go +++ b/go/vt/vtorc/db/db.go @@ -160,8 +160,7 @@ func deployStatements(db *sql.DB, queries []string) error { func initVTOrcDB(db *sql.DB) error { log.Info("Initializing vtorc") log.Info("Migrating database schema") - _ = deployStatements(db, generateSQLBase) - _ = deployStatements(db, generateSQLPatches) + _ = deployStatements(db, vtorcBackend) _ = registerVTOrcDeployment(db) if IsSQLite() { diff --git a/go/vt/vtorc/db/generate_base.go b/go/vt/vtorc/db/generate_base.go index 85aed1b2b86..ef1941656da 100644 --- a/go/vt/vtorc/db/generate_base.go +++ b/go/vt/vtorc/db/generate_base.go @@ -16,803 +16,776 @@ package db -// generateSQLBase & generateSQLPatches are lists of SQL statements required to build the vtorc backend -var generateSQLBase = []string{ +// vtorcBackend is a list of SQL statements required to build the vtorc backend +var vtorcBackend = []string{ + ` +DROP TABLE IF EXISTS database_instance +`, + ` +CREATE TABLE database_instance ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + last_checked timestamp not null default (''), + last_seen timestamp NULL DEFAULT NULL, + server_id int NOT NULL, + version varchar(128) NOT NULL, + binlog_format varchar(16) NOT NULL, + log_bin tinyint NOT NULL, + log_replica_updates tinyint NOT NULL, + binary_log_file varchar(128) NOT NULL, + binary_log_pos bigint NOT NULL, + source_host varchar(128) NOT NULL, + source_port smallint NOT NULL, + replica_sql_running tinyint NOT NULL, + replica_io_running tinyint NOT NULL, + source_log_file varchar(128) NOT NULL, + read_source_log_pos bigint NOT NULL, + relay_source_log_file varchar(128) NOT NULL, + exec_source_log_pos bigint NOT NULL, + replication_lag_seconds bigint DEFAULT NULL, + replica_lag_seconds bigint DEFAULT NULL, + cluster_name varchar(128) NOT NULL, read_only TINYint not null default 0, last_sql_error TEXT not null default '', last_io_error TEXT not null default '', oracle_gtid TINYint not null default 0, mariadb_gtid TINYint not null default 0, relay_log_file varchar(128) not null default '', relay_log_pos bigint not null default 0, pseudo_gtid TINYint not null default 0, replication_depth TINYint not null default 0, has_replication_filters TINYint not null default 0, data_center varchar(32) not null default '', physical_environment varchar(32) not null default '', is_co_primary TINYint not null default 0, sql_delay int not null default 0, binlog_server TINYint not null default 0, supports_oracle_gtid TINYint not null default 0, executed_gtid_set text not null default '', server_uuid varchar(64) not null default '', last_attempted_check TIMESTAMP NOT NULL DEFAULT '1971-01-01 00:00:00', gtid_purged text not null default '', has_replication_credentials TINYint not null default 0, allow_tls TINYint not null default 0, semi_sync_enforced TINYint not null default 0, instance_alias varchar(128) not null default '', version_comment varchar(128) NOT NULL DEFAULT '', major_version varchar(16) not null default '', binlog_row_image varchar(16) not null default '', last_discovery_latency bigint not null default 0, semi_sync_primary_enabled TINYint not null default 0, semi_sync_replica_enabled TINYint not null default 0, gtid_mode varchar(32) not null default '', last_check_partial_success tinyint not null default 0, source_uuid varchar(64) not null default '', gtid_errant text not null default '', ancestry_uuid text not null default '', replication_sql_thread_state tinyint signed not null default 0, replication_io_thread_state tinyint signed not null default 0, region varchar(32) not null default '', semi_sync_primary_timeout int NOT NULL DEFAULT 0, semi_sync_primary_wait_for_replica_count int NOT NULL DEFAULT 0, semi_sync_primary_status TINYint NOT NULL DEFAULT 0, semi_sync_replica_status TINYint NOT NULL DEFAULT 0, semi_sync_primary_clients int NOT NULL DEFAULT 0, replication_group_name VARCHAR(64) NOT NULL DEFAULT '', replication_group_is_single_primary_mode TINYint NOT NULL DEFAULT 1, replication_group_member_state VARCHAR(16) NOT NULL DEFAULT '', replication_group_member_role VARCHAR(16) NOT NULL DEFAULT '', replication_group_members text not null default '', replication_group_primary_host varchar(128) NOT NULL DEFAULT '', replication_group_primary_port smallint NOT NULL DEFAULT 0, + PRIMARY KEY (hostname,port) +)`, ` - CREATE TABLE IF NOT EXISTS database_instance ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - last_checked timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_seen timestamp NULL DEFAULT NULL, - server_id int(10) unsigned NOT NULL, - version varchar(128) CHARACTER SET ascii NOT NULL, - binlog_format varchar(16) CHARACTER SET ascii NOT NULL, - log_bin tinyint(3) unsigned NOT NULL, - log_replica_updates tinyint(3) unsigned NOT NULL, - binary_log_file varchar(128) CHARACTER SET ascii NOT NULL, - binary_log_pos bigint(20) unsigned NOT NULL, - source_host varchar(128) CHARACTER SET ascii NOT NULL, - source_port smallint(5) unsigned NOT NULL, - replica_sql_running tinyint(3) unsigned NOT NULL, - replica_io_running tinyint(3) unsigned NOT NULL, - source_log_file varchar(128) CHARACTER SET ascii NOT NULL, - read_source_log_pos bigint(20) unsigned NOT NULL, - relay_source_log_file varchar(128) CHARACTER SET ascii NOT NULL, - exec_source_log_pos bigint(20) unsigned NOT NULL, - replication_lag_seconds bigint(20) unsigned DEFAULT NULL, - replica_lag_seconds bigint(20) unsigned DEFAULT NULL, - cluster_name varchar(128) CHARACTER SET ascii NOT NULL, - PRIMARY KEY (hostname,port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX cluster_name_idx_database_instance ON database_instance(cluster_name) `, ` - DROP INDEX cluster_name_idx ON database_instance +CREATE INDEX last_checked_idx_database_instance ON database_instance(last_checked) `, ` - CREATE INDEX cluster_name_idx_database_instance ON database_instance(cluster_name) +CREATE INDEX last_seen_idx_database_instance ON database_instance(last_seen) `, ` - DROP INDEX last_checked_idx ON database_instance - `, +DROP TABLE IF EXISTS database_instance_maintenance +`, ` - CREATE INDEX last_checked_idx_database_instance ON database_instance(last_checked) - `, +CREATE TABLE database_instance_maintenance ( + database_instance_maintenance_id integer, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + maintenance_active tinyint(4) DEFAULT NULL, + begin_timestamp timestamp NULL DEFAULT NULL, + end_timestamp timestamp NULL DEFAULT NULL, + owner varchar(128) NOT NULL, + reason text NOT NULL, processing_node_hostname varchar(128) not null default '', processing_node_token varchar(128) not null default '', explicitly_bounded TINYint not null default 0, + PRIMARY KEY (database_instance_maintenance_id) +)`, ` - DROP INDEX last_seen_idx ON database_instance +CREATE UNIQUE INDEX maintenance_uidx_database_instance_maintenance ON database_instance_maintenance (maintenance_active, hostname, port) `, ` - CREATE INDEX last_seen_idx_database_instance ON database_instance(last_seen) - `, +DROP TABLE IF EXISTS database_instance_long_running_queries +`, ` - CREATE TABLE IF NOT EXISTS database_instance_maintenance ( - database_instance_maintenance_id int(10) unsigned NOT NULL AUTO_INCREMENT, - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - maintenance_active tinyint(4) DEFAULT NULL, - begin_timestamp timestamp NULL DEFAULT NULL, - end_timestamp timestamp NULL DEFAULT NULL, - owner varchar(128) CHARACTER SET utf8 NOT NULL, - reason text CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (database_instance_maintenance_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE database_instance_long_running_queries ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + process_id bigint(20) NOT NULL, + process_started_at timestamp not null default (''), + process_user varchar(16) NOT NULL, + process_host varchar(128) NOT NULL, + process_db varchar(128) NOT NULL, + process_command varchar(16) NOT NULL, + process_time_seconds int(11) NOT NULL, + process_state varchar(128) NOT NULL, + process_info varchar(1024) NOT NULL, + PRIMARY KEY (hostname,port,process_id) +)`, ` - DROP INDEX maintenance_uidx ON database_instance_maintenance +CREATE INDEX process_started_at_idx_database_instance_long_running_queries ON database_instance_long_running_queries (process_started_at) `, ` - CREATE UNIQUE INDEX maintenance_uidx_database_instance_maintenance ON database_instance_maintenance (maintenance_active, hostname, port) - `, +DROP TABLE IF EXISTS audit +`, ` - CREATE TABLE IF NOT EXISTS database_instance_long_running_queries ( - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - process_id bigint(20) NOT NULL, - process_started_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - process_user varchar(16) CHARACTER SET utf8 NOT NULL, - process_host varchar(128) CHARACTER SET utf8 NOT NULL, - process_db varchar(128) CHARACTER SET utf8 NOT NULL, - process_command varchar(16) CHARACTER SET utf8 NOT NULL, - process_time_seconds int(11) NOT NULL, - process_state varchar(128) CHARACTER SET utf8 NOT NULL, - process_info varchar(1024) CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (hostname,port,process_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE audit ( + audit_id integer, + audit_timestamp timestamp not null default (''), + audit_type varchar(128) NOT NULL, + hostname varchar(128) NOT NULL DEFAULT '', + port smallint NOT NULL, + message text NOT NULL, cluster_name varchar(128) NOT NULL DEFAULT '', + PRIMARY KEY (audit_id) +)`, ` - DROP INDEX process_started_at_idx ON database_instance_long_running_queries +CREATE INDEX audit_timestamp_idx_audit ON audit (audit_timestamp) `, ` - CREATE INDEX process_started_at_idx_database_instance_long_running_queries ON database_instance_long_running_queries (process_started_at) +CREATE INDEX host_port_idx_audit ON audit (hostname, port, audit_timestamp) `, ` - CREATE TABLE IF NOT EXISTS audit ( - audit_id bigint(20) unsigned NOT NULL AUTO_INCREMENT, - audit_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - audit_type varchar(128) CHARACTER SET ascii NOT NULL, - hostname varchar(128) CHARACTER SET ascii NOT NULL DEFAULT '', - port smallint(5) unsigned NOT NULL, - message text CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (audit_id) - ) ENGINE=InnoDB DEFAULT CHARSET=latin1 - `, +DROP TABLE IF EXISTS host_agent +`, ` - DROP INDEX audit_timestamp_idx ON audit - `, +CREATE TABLE host_agent ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + token varchar(128) NOT NULL, + last_submitted timestamp not null default (''), + last_checked timestamp NULL DEFAULT NULL, + last_seen timestamp NULL DEFAULT NULL, + mysql_port smallint DEFAULT NULL, + count_mysql_snapshots smallint NOT NULL, + PRIMARY KEY (hostname) +)`, ` - CREATE INDEX audit_timestamp_idx_audit ON audit (audit_timestamp) +CREATE INDEX token_idx_host_agent ON host_agent (token) `, ` - DROP INDEX host_port_idx ON audit +CREATE INDEX last_submitted_idx_host_agent ON host_agent (last_submitted) `, ` - CREATE INDEX host_port_idx_audit ON audit (hostname, port, audit_timestamp) +CREATE INDEX last_checked_idx_host_agent ON host_agent (last_checked) `, ` - CREATE TABLE IF NOT EXISTS host_agent ( - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - token varchar(128) NOT NULL, - last_submitted timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_checked timestamp NULL DEFAULT NULL, - last_seen timestamp NULL DEFAULT NULL, - mysql_port smallint(5) unsigned DEFAULT NULL, - count_mysql_snapshots smallint(5) unsigned NOT NULL, - PRIMARY KEY (hostname) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX last_seen_idx_host_agent ON host_agent (last_seen) `, ` - DROP INDEX token_idx ON host_agent - `, +DROP TABLE IF EXISTS agent_seed +`, ` - CREATE INDEX token_idx_host_agent ON host_agent (token) - `, +CREATE TABLE agent_seed ( + agent_seed_id integer, + target_hostname varchar(128) NOT NULL, + source_hostname varchar(128) NOT NULL, + start_timestamp timestamp not null default (''), + end_timestamp timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + is_complete tinyint NOT NULL DEFAULT '0', + is_successful tinyint NOT NULL DEFAULT '0', + PRIMARY KEY (agent_seed_id) +)`, ` - DROP INDEX last_submitted_idx ON host_agent +CREATE INDEX target_hostname_idx_agent_seed ON agent_seed (target_hostname,is_complete) `, ` - CREATE INDEX last_submitted_idx_host_agent ON host_agent (last_submitted) +CREATE INDEX source_hostname_idx_agent_seed ON agent_seed (source_hostname,is_complete) `, ` - DROP INDEX last_checked_idx ON host_agent +CREATE INDEX start_timestamp_idx_agent_seed ON agent_seed (start_timestamp) `, ` - CREATE INDEX last_checked_idx_host_agent ON host_agent (last_checked) +CREATE INDEX is_complete_idx_agent_seed ON agent_seed (is_complete,start_timestamp) `, ` - DROP INDEX last_seen_idx ON host_agent +CREATE INDEX is_successful_idx_agent_seed ON agent_seed (is_successful, start_timestamp) `, ` - CREATE INDEX last_seen_idx_host_agent ON host_agent (last_seen) - `, +DROP TABLE IF EXISTS agent_seed_state +`, ` - CREATE TABLE IF NOT EXISTS agent_seed ( - agent_seed_id int(10) unsigned NOT NULL AUTO_INCREMENT, - target_hostname varchar(128) NOT NULL, - source_hostname varchar(128) NOT NULL, - start_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - end_timestamp timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', - is_complete tinyint(3) unsigned NOT NULL DEFAULT '0', - is_successful tinyint(3) unsigned NOT NULL DEFAULT '0', - PRIMARY KEY (agent_seed_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, - ` - DROP INDEX target_hostname_idx ON agent_seed - `, - ` - CREATE INDEX target_hostname_idx_agent_seed ON agent_seed (target_hostname,is_complete) - `, +CREATE TABLE agent_seed_state ( + agent_seed_state_id integer, + agent_seed_id int NOT NULL, + state_timestamp timestamp not null default (''), + state_action varchar(127) NOT NULL, + error_message varchar(255) NOT NULL, + PRIMARY KEY (agent_seed_state_id) +)`, ` - DROP INDEX source_hostname_idx ON agent_seed +CREATE INDEX agent_seed_idx_agent_seed_state ON agent_seed_state (agent_seed_id, state_timestamp) `, ` - CREATE INDEX source_hostname_idx_agent_seed ON agent_seed (source_hostname,is_complete) - `, +DROP TABLE IF EXISTS host_attributes +`, ` - DROP INDEX start_timestamp_idx ON agent_seed - `, +CREATE TABLE host_attributes ( + hostname varchar(128) NOT NULL, + attribute_name varchar(128) NOT NULL, + attribute_value varchar(128) NOT NULL, + submit_timestamp timestamp not null default (''), + expire_timestamp timestamp NULL DEFAULT NULL, + PRIMARY KEY (hostname,attribute_name) +)`, ` - CREATE INDEX start_timestamp_idx_agent_seed ON agent_seed (start_timestamp) +CREATE INDEX attribute_name_idx_host_attributes ON host_attributes (attribute_name) `, ` - DROP INDEX is_complete_idx ON agent_seed +CREATE INDEX attribute_value_idx_host_attributes ON host_attributes (attribute_value) `, ` - CREATE INDEX is_complete_idx_agent_seed ON agent_seed (is_complete,start_timestamp) +CREATE INDEX submit_timestamp_idx_host_attributes ON host_attributes (submit_timestamp) `, ` - DROP INDEX is_successful_idx ON agent_seed +CREATE INDEX expire_timestamp_idx_host_attributes ON host_attributes (expire_timestamp) `, ` - CREATE INDEX is_successful_idx_agent_seed ON agent_seed (is_successful, start_timestamp) - `, +DROP TABLE IF EXISTS hostname_resolve +`, ` - CREATE TABLE IF NOT EXISTS agent_seed_state ( - agent_seed_state_id int(10) unsigned NOT NULL AUTO_INCREMENT, - agent_seed_id int(10) unsigned NOT NULL, - state_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - state_action varchar(127) NOT NULL, - error_message varchar(255) NOT NULL, - PRIMARY KEY (agent_seed_state_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE hostname_resolve ( + hostname varchar(128) NOT NULL, + resolved_hostname varchar(128) NOT NULL, + resolved_timestamp timestamp not null default (''), + PRIMARY KEY (hostname) +)`, ` - DROP INDEX agent_seed_idx ON agent_seed_state +CREATE INDEX resolved_timestamp_idx_hostname_resolve ON hostname_resolve (resolved_timestamp) `, ` - CREATE INDEX agent_seed_idx_agent_seed_state ON agent_seed_state (agent_seed_id, state_timestamp) - `, +DROP TABLE IF EXISTS active_node +`, ` - CREATE TABLE IF NOT EXISTS host_attributes ( - hostname varchar(128) NOT NULL, - attribute_name varchar(128) NOT NULL, - attribute_value varchar(128) NOT NULL, - submit_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - expire_timestamp timestamp NULL DEFAULT NULL, - PRIMARY KEY (hostname,attribute_name) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE active_node ( + anchor tinyint NOT NULL, + hostname varchar(128) NOT NULL, + token varchar(128) NOT NULL, + last_seen_active timestamp not null default (''), first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + PRIMARY KEY (anchor) +)`, ` - DROP INDEX attribute_name_idx ON host_attributes - `, +DROP TABLE IF EXISTS node_health +`, ` - CREATE INDEX attribute_name_idx_host_attributes ON host_attributes (attribute_name) - `, +CREATE TABLE node_health ( + hostname varchar(128) NOT NULL, + token varchar(128) NOT NULL, + last_seen_active timestamp not null default (''), extra_info varchar(128) not null default '', command varchar(128) not null default '', app_version varchar(64) NOT NULL DEFAULT "", first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', db_backend varchar(255) NOT NULL DEFAULT "", incrementing_indicator bigint not null default 0, + PRIMARY KEY (hostname, token) +)`, ` - DROP INDEX attribute_value_idx ON host_attributes - `, +DROP TABLE IF EXISTS topology_recovery +`, ` - CREATE INDEX attribute_value_idx_host_attributes ON host_attributes (attribute_value) - `, +CREATE TABLE topology_recovery ( + recovery_id integer, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + in_active_period tinyint NOT NULL DEFAULT 0, + start_active_period timestamp not null default (''), + end_active_period_unixtime int, + end_recovery timestamp NULL DEFAULT NULL, + processing_node_hostname varchar(128) NOT NULL, + processcing_node_token varchar(128) NOT NULL, + successor_hostname varchar(128) DEFAULT NULL, + successor_port smallint DEFAULT NULL, analysis varchar(128) not null default '', cluster_name varchar(128) not null default '', count_affected_replicas int not null default 0, is_successful TINYint NOT NULL DEFAULT 0, acknowledged TINYint NOT NULL DEFAULT 0, acknowledged_by varchar(128) not null default '', acknowledge_comment text not null default '', participating_instances text not null default '', lost_replicas text not null default '', all_errors text not null default '', acknowledged_at TIMESTAMP NULL, last_detection_id bigint not null default 0, successor_alias varchar(128) DEFAULT NULL, uid varchar(128) not null default '', + PRIMARY KEY (recovery_id) +)`, ` - DROP INDEX submit_timestamp_idx ON host_attributes +CREATE INDEX in_active_start_period_idx_topology_recovery ON topology_recovery (in_active_period, start_active_period) `, ` - CREATE INDEX submit_timestamp_idx_host_attributes ON host_attributes (submit_timestamp) +CREATE INDEX start_active_period_idx_topology_recovery ON topology_recovery (start_active_period) `, ` - DROP INDEX expire_timestamp_idx ON host_attributes +CREATE UNIQUE INDEX hostname_port_active_period_uidx_topology_recovery ON topology_recovery (hostname, port, in_active_period, end_active_period_unixtime) `, ` - CREATE INDEX expire_timestamp_idx_host_attributes ON host_attributes (expire_timestamp) - `, +DROP TABLE IF EXISTS hostname_unresolve +`, ` - CREATE TABLE IF NOT EXISTS hostname_resolve ( - hostname varchar(128) NOT NULL, - resolved_hostname varchar(128) NOT NULL, - resolved_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (hostname) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE hostname_unresolve ( + hostname varchar(128) NOT NULL, + unresolved_hostname varchar(128) NOT NULL, last_registered timestamp not null default (''), + PRIMARY KEY (hostname) +)`, ` - DROP INDEX resolved_timestamp_idx ON hostname_resolve +CREATE INDEX unresolved_hostname_idx_hostname_unresolve ON hostname_unresolve (unresolved_hostname) `, ` - CREATE INDEX resolved_timestamp_idx_hostname_resolve ON hostname_resolve (resolved_timestamp) - `, +DROP TABLE IF EXISTS database_instance_topology_history +`, ` - CREATE TABLE IF NOT EXISTS active_node ( - anchor tinyint unsigned NOT NULL, - hostname varchar(128) CHARACTER SET ascii NOT NULL, - token varchar(128) NOT NULL, - last_seen_active timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (anchor) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE database_instance_topology_history ( + snapshot_unix_timestamp int NOT NULL, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + source_host varchar(128) NOT NULL, + source_port smallint NOT NULL, + cluster_name tinytext NOT NULL, version varchar(128) not null default '', + PRIMARY KEY (snapshot_unix_timestamp, hostname, port) +)`, ` - INSERT IGNORE INTO active_node (anchor, hostname, token, last_seen_active) - VALUES (1, '', '', NOW()) +CREATE INDEX cluster_name_idx_database_instance_topology_history ON database_instance_topology_history (snapshot_unix_timestamp, cluster_name) `, ` - CREATE TABLE IF NOT EXISTS node_health ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - token varchar(128) NOT NULL, - last_seen_active timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (hostname, token) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +DROP TABLE IF EXISTS candidate_database_instance +`, ` - DROP VIEW IF EXISTS _whats_wrong - `, +CREATE TABLE candidate_database_instance ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + last_suggested timestamp not null default (''), priority TINYINT SIGNED NOT NULL DEFAULT 1, promotion_rule text check(promotion_rule in ('must', 'prefer', 'neutral', 'prefer_not', 'must_not')) NOT NULL DEFAULT 'neutral', + PRIMARY KEY (hostname, port) +)`, ` - DROP VIEW IF EXISTS whats_wrong +CREATE INDEX last_suggested_idx_candidate_database_instance ON candidate_database_instance (last_suggested) `, ` - DROP VIEW IF EXISTS whats_wrong_summary - `, +DROP TABLE IF EXISTS database_instance_downtime +`, ` - CREATE TABLE IF NOT EXISTS topology_recovery ( - recovery_id bigint unsigned not null auto_increment, - hostname varchar(128) NOT NULL, - port smallint unsigned NOT NULL, - in_active_period tinyint unsigned NOT NULL DEFAULT 0, - start_active_period timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - end_active_period_unixtime int unsigned, - end_recovery timestamp NULL DEFAULT NULL, - processing_node_hostname varchar(128) CHARACTER SET ascii NOT NULL, - processcing_node_token varchar(128) NOT NULL, - successor_hostname varchar(128) DEFAULT NULL, - successor_port smallint unsigned DEFAULT NULL, - PRIMARY KEY (recovery_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE database_instance_downtime ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + downtime_active tinyint(4) DEFAULT NULL, + begin_timestamp timestamp default (''), + end_timestamp timestamp NULL DEFAULT NULL, + owner varchar(128) NOT NULL, + reason text NOT NULL, + PRIMARY KEY (hostname, port) +)`, ` - DROP INDEX in_active_start_period_idx ON topology_recovery - `, +DROP TABLE IF EXISTS topology_failure_detection +`, ` - CREATE INDEX in_active_start_period_idx_topology_recovery ON topology_recovery (in_active_period, start_active_period) - `, +CREATE TABLE topology_failure_detection ( + detection_id integer, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + in_active_period tinyint NOT NULL DEFAULT '0', + start_active_period timestamp not null default (''), + end_active_period_unixtime int NOT NULL, + processing_node_hostname varchar(128) NOT NULL, + processcing_node_token varchar(128) NOT NULL, + analysis varchar(128) NOT NULL, + cluster_name varchar(128) NOT NULL, + count_affected_replicas int NOT NULL, is_actionable tinyint not null default 0, + PRIMARY KEY (detection_id) +)`, ` - DROP INDEX start_active_period_idx ON topology_recovery +CREATE INDEX in_active_start_period_idx_topology_failure_detection ON topology_failure_detection (in_active_period, start_active_period) `, ` - CREATE INDEX start_active_period_idx_topology_recovery ON topology_recovery (start_active_period) - `, +DROP TABLE IF EXISTS hostname_resolve_history +`, ` - DROP INDEX hostname_port_active_period_uidx ON topology_recovery - `, +CREATE TABLE hostname_resolve_history ( + resolved_hostname varchar(128) NOT NULL, + hostname varchar(128) NOT NULL, + resolved_timestamp timestamp not null default (''), + PRIMARY KEY (resolved_hostname) +)`, ` - CREATE UNIQUE INDEX hostname_port_active_period_uidx_topology_recovery ON topology_recovery (hostname, port, in_active_period, end_active_period_unixtime) +CREATE INDEX hostname_idx_hostname_resolve_history ON hostname_resolve_history (hostname) `, ` - CREATE TABLE IF NOT EXISTS hostname_unresolve ( - hostname varchar(128) NOT NULL, - unresolved_hostname varchar(128) NOT NULL, - PRIMARY KEY (hostname) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX resolved_timestamp_idx_hostname_resolve_history ON hostname_resolve_history (resolved_timestamp) `, ` - DROP INDEX unresolved_hostname_idx ON hostname_unresolve - `, +DROP TABLE IF EXISTS hostname_unresolve_history +`, ` - CREATE INDEX unresolved_hostname_idx_hostname_unresolve ON hostname_unresolve (unresolved_hostname) - `, +CREATE TABLE hostname_unresolve_history ( + unresolved_hostname varchar(128) NOT NULL, + hostname varchar(128) NOT NULL, + last_registered timestamp not null default (''), + PRIMARY KEY (unresolved_hostname) +)`, ` - CREATE TABLE IF NOT EXISTS database_instance_topology_history ( - snapshot_unix_timestamp INT UNSIGNED NOT NULL, - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - source_host varchar(128) CHARACTER SET ascii NOT NULL, - source_port smallint(5) unsigned NOT NULL, - cluster_name tinytext CHARACTER SET ascii NOT NULL, - PRIMARY KEY (snapshot_unix_timestamp, hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX hostname_idx_hostname_unresolve_history ON hostname_unresolve_history (hostname) `, ` - DROP INDEX cluster_name_idx ON database_instance_topology_history +CREATE INDEX last_registered_idx_hostname_unresolve_history ON hostname_unresolve_history (last_registered) `, ` - CREATE INDEX cluster_name_idx_database_instance_topology_history ON database_instance_topology_history (snapshot_unix_timestamp, cluster_name(128)) - `, +DROP TABLE IF EXISTS cluster_domain_name +`, ` - CREATE TABLE IF NOT EXISTS candidate_database_instance ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - last_suggested TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE cluster_domain_name ( + cluster_name varchar(128) NOT NULL, + domain_name varchar(128) NOT NULL, last_registered timestamp not null default (''), + PRIMARY KEY (cluster_name) +)`, ` - DROP INDEX last_suggested_idx ON candidate_database_instance +CREATE INDEX domain_name_idx_cluster_domain_name ON cluster_domain_name (domain_name) `, ` - CREATE INDEX last_suggested_idx_candidate_database_instance ON candidate_database_instance (last_suggested) - `, +DROP TABLE IF EXISTS primary_position_equivalence +`, ` - CREATE TABLE IF NOT EXISTS database_instance_downtime ( - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - downtime_active tinyint(4) DEFAULT NULL, - begin_timestamp timestamp DEFAULT CURRENT_TIMESTAMP, - end_timestamp timestamp NULL DEFAULT NULL, - owner varchar(128) CHARACTER SET utf8 NOT NULL, - reason text CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE primary_position_equivalence ( + equivalence_id integer, + primary1_hostname varchar(128) NOT NULL, + primary1_port smallint NOT NULL, + primary1_binary_log_file varchar(128) NOT NULL, + primary1_binary_log_pos bigint NOT NULL, + primary2_hostname varchar(128) NOT NULL, + primary2_port smallint NOT NULL, + primary2_binary_log_file varchar(128) NOT NULL, + primary2_binary_log_pos bigint NOT NULL, + last_suggested timestamp not null default (''), + PRIMARY KEY (equivalence_id) +)`, ` - CREATE TABLE IF NOT EXISTS topology_failure_detection ( - detection_id bigint(20) unsigned NOT NULL AUTO_INCREMENT, - hostname varchar(128) NOT NULL, - port smallint unsigned NOT NULL, - in_active_period tinyint unsigned NOT NULL DEFAULT '0', - start_active_period timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - end_active_period_unixtime int unsigned NOT NULL, - processing_node_hostname varchar(128) NOT NULL, - processcing_node_token varchar(128) NOT NULL, - analysis varchar(128) NOT NULL, - cluster_name varchar(128) NOT NULL, - count_affected_replicas int unsigned NOT NULL, - PRIMARY KEY (detection_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE UNIQUE INDEX equivalence_uidx_primary_position_equivalence ON primary_position_equivalence (primary1_hostname, primary1_port, primary1_binary_log_file, primary1_binary_log_pos, primary2_hostname, primary2_port) `, ` - DROP INDEX hostname_port_active_period_uidx ON topology_failure_detection +CREATE INDEX primary2_idx_primary_position_equivalence ON primary_position_equivalence (primary2_hostname, primary2_port, primary2_binary_log_file, primary2_binary_log_pos) `, ` - DROP INDEX in_active_start_period_idx ON topology_failure_detection +CREATE INDEX last_suggested_idx_primary_position_equivalence ON primary_position_equivalence (last_suggested) `, ` - CREATE INDEX in_active_start_period_idx_topology_failure_detection ON topology_failure_detection (in_active_period, start_active_period) - `, +DROP TABLE IF EXISTS async_request +`, ` - CREATE TABLE IF NOT EXISTS hostname_resolve_history ( - resolved_hostname varchar(128) NOT NULL, - hostname varchar(128) NOT NULL, - resolved_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (resolved_hostname) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE async_request ( + request_id integer, + command varchar(128) not null, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + destination_hostname varchar(128) NOT NULL, + destination_port smallint NOT NULL, + pattern text NOT NULL, + gtid_hint varchar(32) not null, + begin_timestamp timestamp NULL DEFAULT NULL, + end_timestamp timestamp NULL DEFAULT NULL, + story text NOT NULL, + PRIMARY KEY (request_id) +)`, ` - DROP INDEX hostname ON hostname_resolve_history +CREATE INDEX begin_timestamp_idx_async_request ON async_request (begin_timestamp) `, ` - CREATE INDEX hostname_idx_hostname_resolve_history ON hostname_resolve_history (hostname) +CREATE INDEX end_timestamp_idx_async_request ON async_request (end_timestamp) `, ` - DROP INDEX resolved_timestamp_idx ON hostname_resolve_history - `, +DROP TABLE IF EXISTS blocked_topology_recovery +`, ` - CREATE INDEX resolved_timestamp_idx_hostname_resolve_history ON hostname_resolve_history (resolved_timestamp) - `, +CREATE TABLE blocked_topology_recovery ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + cluster_name varchar(128) NOT NULL, + analysis varchar(128) NOT NULL, + last_blocked_timestamp timestamp not null default (''), + blocking_recovery_id bigint, + PRIMARY KEY (hostname, port) +)`, ` - CREATE TABLE IF NOT EXISTS hostname_unresolve_history ( - unresolved_hostname varchar(128) NOT NULL, - hostname varchar(128) NOT NULL, - last_registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (unresolved_hostname) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX cluster_blocked_idx_blocked_topology_recovery ON blocked_topology_recovery (cluster_name, last_blocked_timestamp) `, ` - DROP INDEX hostname ON hostname_unresolve_history - `, +DROP TABLE IF EXISTS database_instance_last_analysis +`, ` - CREATE INDEX hostname_idx_hostname_unresolve_history ON hostname_unresolve_history (hostname) - `, +CREATE TABLE database_instance_last_analysis ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + analysis_timestamp timestamp not null default (''), + analysis varchar(128) NOT NULL, + PRIMARY KEY (hostname, port) +)`, ` - DROP INDEX last_registered_idx ON hostname_unresolve_history +CREATE INDEX analysis_timestamp_idx_database_instance_last_analysis ON database_instance_last_analysis (analysis_timestamp) `, ` - CREATE INDEX last_registered_idx_hostname_unresolve_history ON hostname_unresolve_history (last_registered) - `, +DROP TABLE IF EXISTS database_instance_analysis_changelog +`, ` - CREATE TABLE IF NOT EXISTS cluster_domain_name ( - cluster_name varchar(128) CHARACTER SET ascii NOT NULL, - domain_name varchar(128) NOT NULL, - PRIMARY KEY (cluster_name) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE database_instance_analysis_changelog ( + changelog_id integer, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + analysis_timestamp timestamp not null default (''), + analysis varchar(128) NOT NULL, + PRIMARY KEY (changelog_id) +)`, ` - DROP INDEX domain_name_idx ON cluster_domain_name +CREATE INDEX analysis_timestamp_idx_database_instance_analysis_changelog ON database_instance_analysis_changelog (analysis_timestamp) `, ` - CREATE INDEX domain_name_idx_cluster_domain_name ON cluster_domain_name (domain_name(32)) - `, +DROP TABLE IF EXISTS node_health_history +`, ` - CREATE TABLE IF NOT EXISTS primary_position_equivalence ( - equivalence_id bigint unsigned not null auto_increment, - primary1_hostname varchar(128) CHARACTER SET ascii NOT NULL, - primary1_port smallint(5) unsigned NOT NULL, - primary1_binary_log_file varchar(128) CHARACTER SET ascii NOT NULL, - primary1_binary_log_pos bigint(20) unsigned NOT NULL, - primary2_hostname varchar(128) CHARACTER SET ascii NOT NULL, - primary2_port smallint(5) unsigned NOT NULL, - primary2_binary_log_file varchar(128) CHARACTER SET ascii NOT NULL, - primary2_binary_log_pos bigint(20) unsigned NOT NULL, - last_suggested TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (equivalence_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE node_health_history ( + history_id integer, + hostname varchar(128) NOT NULL, + token varchar(128) NOT NULL, + first_seen_active timestamp NOT NULL, + extra_info varchar(128) NOT NULL, command varchar(128) not null default '', app_version varchar(64) NOT NULL DEFAULT "", + PRIMARY KEY (history_id) +)`, ` - DROP INDEX equivalence_uidx ON primary_position_equivalence +CREATE INDEX first_seen_active_idx_node_health_history ON node_health_history (first_seen_active) `, ` - CREATE UNIQUE INDEX equivalence_uidx_primary_position_equivalence ON primary_position_equivalence (primary1_hostname, primary1_port, primary1_binary_log_file, primary1_binary_log_pos, primary2_hostname, primary2_port) +CREATE UNIQUE INDEX hostname_token_idx_node_health_history ON node_health_history (hostname, token) `, ` - DROP INDEX primary2_idx ON primary_position_equivalence - `, +DROP TABLE IF EXISTS database_instance_coordinates_history +`, ` - CREATE INDEX primary2_idx_primary_position_equivalence ON primary_position_equivalence (primary2_hostname, primary2_port, primary2_binary_log_file, primary2_binary_log_pos) - `, +CREATE TABLE database_instance_coordinates_history ( + history_id integer, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + recorded_timestamp timestamp not null default (''), + binary_log_file varchar(128) NOT NULL, + binary_log_pos bigint NOT NULL, + relay_log_file varchar(128) NOT NULL, + relay_log_pos bigint NOT NULL, last_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + PRIMARY KEY (history_id) +)`, ` - DROP INDEX last_suggested_idx ON primary_position_equivalence +CREATE INDEX hostname_port_recorded_idx_database_instance_coordinates_history ON database_instance_coordinates_history (hostname, port, recorded_timestamp) `, ` - CREATE INDEX last_suggested_idx_primary_position_equivalence ON primary_position_equivalence (last_suggested) +CREATE INDEX recorded_timestmp_idx_database_instance_coordinates_history ON database_instance_coordinates_history (recorded_timestamp) `, ` - CREATE TABLE IF NOT EXISTS async_request ( - request_id bigint unsigned NOT NULL AUTO_INCREMENT, - command varchar(128) charset ascii not null, - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - destination_hostname varchar(128) NOT NULL, - destination_port smallint(5) unsigned NOT NULL, - pattern text CHARACTER SET utf8 NOT NULL, - gtid_hint varchar(32) charset ascii not null, - begin_timestamp timestamp NULL DEFAULT NULL, - end_timestamp timestamp NULL DEFAULT NULL, - story text CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (request_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +DROP TABLE IF EXISTS database_instance_binlog_files_history +`, ` - DROP INDEX begin_timestamp_idx ON async_request - `, +CREATE TABLE database_instance_binlog_files_history ( + history_id integer, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + binary_log_file varchar(128) NOT NULL, + binary_log_pos bigint NOT NULL, + first_seen timestamp not null default (''), + last_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + PRIMARY KEY (history_id) +)`, ` - CREATE INDEX begin_timestamp_idx_async_request ON async_request (begin_timestamp) +CREATE UNIQUE INDEX hostname_port_file_idx_database_instance_binlog_files_history ON database_instance_binlog_files_history (hostname, port, binary_log_file) `, ` - DROP INDEX end_timestamp_idx ON async_request +CREATE INDEX last_seen_idx_database_instance_binlog_files_history ON database_instance_binlog_files_history (last_seen) `, ` - CREATE INDEX end_timestamp_idx_async_request ON async_request (end_timestamp) - `, +DROP TABLE IF EXISTS database_instance_recent_relaylog_history +`, ` - CREATE TABLE IF NOT EXISTS blocked_topology_recovery ( - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - cluster_name varchar(128) NOT NULL, - analysis varchar(128) NOT NULL, - last_blocked_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - blocking_recovery_id bigint unsigned, - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE database_instance_recent_relaylog_history ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + current_relay_log_file varchar(128) NOT NULL, + current_relay_log_pos bigint NOT NULL, + current_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + prev_relay_log_file varchar(128) NOT NULL, + prev_relay_log_pos bigint NOT NULL, + prev_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + PRIMARY KEY (hostname, port) +)`, ` - DROP INDEX cluster_blocked_idx ON blocked_topology_recovery +CREATE INDEX current_seen_idx_database_instance_recent_relaylog_history ON database_instance_recent_relaylog_history (current_seen) `, ` - CREATE INDEX cluster_blocked_idx_blocked_topology_recovery ON blocked_topology_recovery (cluster_name, last_blocked_timestamp) - `, +DROP TABLE IF EXISTS vtorc_metadata +`, ` - CREATE TABLE IF NOT EXISTS database_instance_last_analysis ( - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - analysis_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - analysis varchar(128) NOT NULL, - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE vtorc_metadata ( + anchor tinyint NOT NULL, + last_deployed_version varchar(128) NOT NULL, + last_deployed_timestamp timestamp NOT NULL, + PRIMARY KEY (anchor) +)`, ` - DROP INDEX analysis_timestamp_idx ON database_instance_last_analysis - `, +DROP TABLE IF EXISTS vtorc_db_deployments +`, ` - CREATE INDEX analysis_timestamp_idx_database_instance_last_analysis ON database_instance_last_analysis (analysis_timestamp) - `, +CREATE TABLE vtorc_db_deployments ( + deployed_version varchar(128) NOT NULL, + deployed_timestamp timestamp NOT NULL, + PRIMARY KEY (deployed_version) +)`, ` - CREATE TABLE IF NOT EXISTS database_instance_analysis_changelog ( - changelog_id bigint unsigned not null auto_increment, - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - analysis_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - analysis varchar(128) NOT NULL, - PRIMARY KEY (changelog_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +DROP TABLE IF EXISTS global_recovery_disable +`, ` - DROP INDEX analysis_timestamp_idx ON database_instance_analysis_changelog - `, +CREATE TABLE global_recovery_disable ( + disable_recovery tinyint NOT NULL , + PRIMARY KEY (disable_recovery) +)`, + ` +DROP TABLE IF EXISTS topology_recovery_steps +`, + ` +CREATE TABLE topology_recovery_steps ( + recovery_step_id integer, + recovery_uid varchar(128) NOT NULL, + audit_at timestamp not null default (''), + message text NOT NULL, + PRIMARY KEY (recovery_step_id) +)`, ` - CREATE INDEX analysis_timestamp_idx_database_instance_analysis_changelog ON database_instance_analysis_changelog (analysis_timestamp) +DROP TABLE IF EXISTS raft_store +`, + ` +CREATE TABLE raft_store ( + store_id integer, + store_key varbinary(512) not null, + store_value blob not null, + PRIMARY KEY (store_id) +)`, + ` +CREATE INDEX store_key_idx_raft_store ON raft_store (store_key) `, ` - CREATE TABLE IF NOT EXISTS node_health_history ( - history_id bigint unsigned not null auto_increment, - hostname varchar(128) CHARACTER SET ascii NOT NULL, - token varchar(128) NOT NULL, - first_seen_active timestamp NOT NULL, - extra_info varchar(128) CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (history_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +DROP TABLE IF EXISTS raft_log +`, ` - DROP INDEX first_seen_active_idx ON node_health_history - `, +CREATE TABLE raft_log ( + log_index integer, + term bigint not null, + log_type int not null, + data blob not null, + PRIMARY KEY (log_index) +)`, + ` +DROP TABLE IF EXISTS raft_snapshot +`, + ` +CREATE TABLE raft_snapshot ( + snapshot_id integer, + snapshot_name varchar(128) NOT NULL, + snapshot_meta varchar(4096) NOT NULL, created_at timestamp not null default (''), + PRIMARY KEY (snapshot_id) +)`, ` - CREATE INDEX first_seen_active_idx_node_health_history ON node_health_history (first_seen_active) +CREATE UNIQUE INDEX snapshot_name_uidx_raft_snapshot ON raft_snapshot (snapshot_name) `, ` - DROP INDEX hostname_token_idx ON node_health_history - `, +DROP TABLE IF EXISTS database_instance_peer_analysis +`, ` - CREATE UNIQUE INDEX hostname_token_idx_node_health_history ON node_health_history (hostname, token) - `, +CREATE TABLE database_instance_peer_analysis ( + peer varchar(128) NOT NULL, + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + analysis_timestamp timestamp not null default (''), + analysis varchar(128) NOT NULL, + PRIMARY KEY (peer, hostname, port) +)`, ` - CREATE TABLE IF NOT EXISTS database_instance_coordinates_history ( - history_id bigint unsigned not null auto_increment, - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - recorded_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - binary_log_file varchar(128) NOT NULL, - binary_log_pos bigint(20) unsigned NOT NULL, - relay_log_file varchar(128) NOT NULL, - relay_log_pos bigint(20) unsigned NOT NULL, - PRIMARY KEY (history_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +DROP TABLE IF EXISTS database_instance_tls +`, ` - DROP INDEX hostname_port_recorded_timestmp_idx ON database_instance_coordinates_history - `, +CREATE TABLE database_instance_tls ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + required tinyint NOT NULL DEFAULT 0, + PRIMARY KEY (hostname,port) +)`, ` - CREATE INDEX hostname_port_recorded_idx_database_instance_coordinates_history ON database_instance_coordinates_history (hostname, port, recorded_timestamp) - `, +DROP TABLE IF EXISTS cluster_injected_pseudo_gtid +`, ` - DROP INDEX recorded_timestmp_idx ON database_instance_coordinates_history - `, +CREATE TABLE cluster_injected_pseudo_gtid ( + cluster_name varchar(128) NOT NULL, + time_injected timestamp not null default (''), + PRIMARY KEY (cluster_name) +)`, ` - CREATE INDEX recorded_timestmp_idx_database_instance_coordinates_history ON database_instance_coordinates_history (recorded_timestamp) - `, +DROP TABLE IF EXISTS hostname_ips +`, ` - CREATE TABLE IF NOT EXISTS database_instance_binlog_files_history ( - history_id bigint unsigned not null auto_increment, - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - binary_log_file varchar(128) NOT NULL, - binary_log_pos bigint(20) unsigned NOT NULL, - first_seen timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', - PRIMARY KEY (history_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE hostname_ips ( + hostname varchar(128) NOT NULL, + ipv4 varchar(128) NOT NULL, + ipv6 varchar(128) NOT NULL, + last_updated timestamp not null default (''), + PRIMARY KEY (hostname) +)`, ` - DROP INDEX hostname_port_file_idx ON database_instance_binlog_files_history - `, +DROP TABLE IF EXISTS database_instance_tags +`, ` - CREATE UNIQUE INDEX hostname_port_file_idx_database_instance_binlog_files_history ON database_instance_binlog_files_history (hostname, port, binary_log_file) - `, +CREATE TABLE database_instance_tags ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + tag_name varchar(128) NOT NULL, + tag_value varchar(128) NOT NULL, + last_updated timestamp not null default (''), + PRIMARY KEY (hostname, port, tag_name) +)`, ` - DROP INDEX last_seen_idx ON database_instance_binlog_files_history +CREATE INDEX tag_name_idx_database_instance_tags ON database_instance_tags (tag_name) `, ` - CREATE INDEX last_seen_idx_database_instance_binlog_files_history ON database_instance_binlog_files_history (last_seen) - `, +DROP TABLE IF EXISTS database_instance_stale_binlog_coordinates +`, ` - CREATE TABLE IF NOT EXISTS database_instance_recent_relaylog_history ( - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - current_relay_log_file varchar(128) NOT NULL, - current_relay_log_pos bigint(20) unsigned NOT NULL, - current_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', - prev_relay_log_file varchar(128) NOT NULL, - prev_relay_log_pos bigint(20) unsigned NOT NULL, - prev_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE database_instance_stale_binlog_coordinates ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + binary_log_file varchar(128) NOT NULL, + binary_log_pos bigint NOT NULL, + first_seen timestamp not null default (''), + PRIMARY KEY (hostname, port) +)`, ` - DROP INDEX current_seen_idx ON database_instance_recent_relaylog_history +CREATE INDEX first_seen_idx_database_instance_stale_binlog_coordinates ON database_instance_stale_binlog_coordinates (first_seen) `, ` - CREATE INDEX current_seen_idx_database_instance_recent_relaylog_history ON database_instance_recent_relaylog_history (current_seen) - `, +DROP TABLE IF EXISTS vitess_tablet +`, ` - CREATE TABLE IF NOT EXISTS vtorc_metadata ( - anchor tinyint unsigned NOT NULL, - last_deployed_version varchar(128) CHARACTER SET ascii NOT NULL, - last_deployed_timestamp timestamp NOT NULL, - PRIMARY KEY (anchor) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE vitess_tablet ( + hostname varchar(128) NOT NULL, + port smallint NOT NULL, + keyspace varchar(128) NOT NULL, + shard varchar(128) NOT NULL, + cell varchar(128) NOT NULL, + tablet_type smallint(5) NOT NULL, + primary_timestamp timestamp NOT NULL, + info varchar(512) NOT NULL, + PRIMARY KEY (hostname, port) +)`, ` - CREATE TABLE IF NOT EXISTS vtorc_db_deployments ( - deployed_version varchar(128) CHARACTER SET ascii NOT NULL, - deployed_timestamp timestamp NOT NULL, - PRIMARY KEY (deployed_version) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX cell_idx_vitess_tablet ON vitess_tablet (cell) `, ` - CREATE TABLE IF NOT EXISTS global_recovery_disable ( - disable_recovery tinyint unsigned NOT NULL COMMENT 'Insert 1 to disable recovery globally', - PRIMARY KEY (disable_recovery) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX ks_idx_vitess_tablet ON vitess_tablet (keyspace, shard) `, ` - CREATE TABLE IF NOT EXISTS topology_recovery_steps ( - recovery_step_id bigint unsigned not null auto_increment, - recovery_uid varchar(128) CHARACTER SET ascii NOT NULL, - audit_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - message text CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (recovery_step_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +DROP TABLE IF EXISTS vitess_keyspace +`, ` - CREATE TABLE IF NOT EXISTS raft_store ( - store_id bigint unsigned not null auto_increment, - store_key varbinary(512) not null, - store_value blob not null, - PRIMARY KEY (store_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii - `, +CREATE TABLE vitess_keyspace ( + keyspace varchar(128) NOT NULL, + keyspace_type smallint(5) NOT NULL, + durability_policy varchar(512) NOT NULL, + PRIMARY KEY (keyspace) +)`, ` - CREATE INDEX store_key_idx_raft_store ON raft_store (store_key) +CREATE INDEX source_host_port_idx_database_instance_database_instance on database_instance (source_host, source_port) `, ` - CREATE TABLE IF NOT EXISTS raft_log ( - log_index bigint unsigned not null auto_increment, - term bigint not null, - log_type int not null, - data blob not null, - PRIMARY KEY (log_index) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX active_timestamp_idx_database_instance_maintenance on database_instance_maintenance (maintenance_active, begin_timestamp) `, ` - CREATE TABLE IF NOT EXISTS raft_snapshot ( - snapshot_id bigint unsigned not null auto_increment, - snapshot_name varchar(128) CHARACTER SET utf8 NOT NULL, - snapshot_meta varchar(4096) CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (snapshot_id) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX active_end_timestamp_idx_database_instance_maintenance on database_instance_maintenance (maintenance_active, end_timestamp) `, ` - CREATE UNIQUE INDEX snapshot_name_uidx_raft_snapshot ON raft_snapshot (snapshot_name) +CREATE INDEX last_registered_idx_hostname_unresolve on hostname_unresolve (last_registered) `, ` - CREATE TABLE IF NOT EXISTS database_instance_peer_analysis ( - peer varchar(128) NOT NULL, - hostname varchar(128) NOT NULL, - port smallint(5) unsigned NOT NULL, - analysis_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - analysis varchar(128) NOT NULL, - PRIMARY KEY (peer, hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX cluster_name_in_active_idx_topology_recovery on topology_recovery (cluster_name, in_active_period) `, ` - CREATE TABLE IF NOT EXISTS database_instance_tls ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - required tinyint unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (hostname,port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX end_recovery_idx_topology_recovery on topology_recovery (end_recovery) `, ` - CREATE TABLE IF NOT EXISTS cluster_injected_pseudo_gtid ( - cluster_name varchar(128) NOT NULL, - time_injected timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (cluster_name) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX last_registered_idx_cluster_domain_name on cluster_domain_name (last_registered) `, ` - CREATE TABLE IF NOT EXISTS hostname_ips ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - ipv4 varchar(128) CHARACTER SET ascii NOT NULL, - ipv6 varchar(128) CHARACTER SET ascii NOT NULL, - last_updated timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (hostname) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX acknowledged_idx_topology_recovery on topology_recovery (acknowledged, acknowledged_at) `, ` - CREATE TABLE IF NOT EXISTS database_instance_tags ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - tag_name varchar(128) CHARACTER SET utf8 NOT NULL, - tag_value varchar(128) CHARACTER SET utf8 NOT NULL, - last_updated timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (hostname, port, tag_name) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX last_blocked_idx_blocked_topology_recovery on blocked_topology_recovery (last_blocked_timestamp) `, ` - CREATE INDEX tag_name_idx_database_instance_tags ON database_instance_tags (tag_name) +CREATE INDEX instance_timestamp_idx_database_instance_analysis_changelog on database_instance_analysis_changelog (hostname, port, analysis_timestamp) `, ` - CREATE TABLE IF NOT EXISTS database_instance_stale_binlog_coordinates ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - binary_log_file varchar(128) NOT NULL, - binary_log_pos bigint(20) unsigned NOT NULL, - first_seen timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX last_detection_idx_topology_recovery on topology_recovery (last_detection_id) `, ` - CREATE INDEX first_seen_idx_database_instance_stale_binlog_coordinates ON database_instance_stale_binlog_coordinates (first_seen) +CREATE INDEX last_seen_active_idx_node_health on node_health (last_seen_active) `, ` - CREATE TABLE IF NOT EXISTS vitess_tablet ( - hostname varchar(128) CHARACTER SET ascii NOT NULL, - port smallint(5) unsigned NOT NULL, - keyspace varchar(128) CHARACTER SET ascii NOT NULL, - shard varchar(128) CHARACTER SET ascii NOT NULL, - cell varchar(128) CHARACTER SET ascii NOT NULL, - tablet_type smallint(5) NOT NULL, - primary_timestamp timestamp NOT NULL, - info varchar(512) CHARACTER SET ascii NOT NULL, - PRIMARY KEY (hostname, port) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE INDEX uid_idx_topology_recovery ON topology_recovery(uid) `, ` - CREATE INDEX cell_idx_vitess_tablet ON vitess_tablet (cell) +CREATE INDEX recovery_uid_idx_topology_recovery_steps ON topology_recovery_steps(recovery_uid) `, ` - CREATE INDEX ks_idx_vitess_tablet ON vitess_tablet (keyspace, shard) +CREATE INDEX end_timestamp_idx_database_instance_downtime ON database_instance_downtime(end_timestamp) `, ` - CREATE TABLE IF NOT EXISTS vitess_keyspace ( - keyspace varchar(128) CHARACTER SET ascii NOT NULL, - keyspace_type smallint(5) NOT NULL, - durability_policy varchar(512) CHARACTER SET ascii NOT NULL, - PRIMARY KEY (keyspace) - ) ENGINE=InnoDB DEFAULT CHARSET=ascii +CREATE UNIQUE INDEX host_port_active_recoverable_uidx_topology_failure_detection ON topology_failure_detection (hostname, port, in_active_period, end_active_period_unixtime, is_actionable) `, } diff --git a/go/vt/vtorc/db/generate_patches.go b/go/vt/vtorc/db/generate_patches.go deleted file mode 100644 index 93099f51a57..00000000000 --- a/go/vt/vtorc/db/generate_patches.go +++ /dev/null @@ -1,559 +0,0 @@ -/* - Copyright 2017 Shlomi Noach, GitHub Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package db - -// generateSQLPatches contains DDLs for patching schema to the latest version. -// Add new statements at the end of the list so they form a changelog. -var generateSQLPatches = []string{ - ` - ALTER TABLE - database_instance - ADD COLUMN read_only TINYINT UNSIGNED NOT NULL AFTER version - `, - ` - ALTER TABLE - database_instance - ADD COLUMN last_sql_error TEXT NOT NULL AFTER exec_source_log_pos - `, - ` - ALTER TABLE - database_instance - ADD COLUMN last_io_error TEXT NOT NULL AFTER last_sql_error - `, - ` - ALTER TABLE - database_instance - ADD COLUMN oracle_gtid TINYINT UNSIGNED NOT NULL AFTER replica_io_running - `, - ` - ALTER TABLE - database_instance - ADD COLUMN mariadb_gtid TINYINT UNSIGNED NOT NULL AFTER oracle_gtid - `, - ` - ALTER TABLE - database_instance - ADD COLUMN relay_log_file varchar(128) CHARACTER SET ascii NOT NULL AFTER exec_source_log_pos - `, - ` - ALTER TABLE - database_instance - ADD COLUMN relay_log_pos bigint unsigned NOT NULL AFTER relay_log_file - `, - ` - DROP INDEX source_host_port_idx ON database_instance - `, - ` - ALTER TABLE - database_instance - ADD INDEX source_host_port_idx_database_instance (source_host, source_port) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN pseudo_gtid TINYINT UNSIGNED NOT NULL AFTER mariadb_gtid - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_depth TINYINT UNSIGNED NOT NULL AFTER cluster_name - `, - ` - ALTER TABLE - database_instance - ADD COLUMN has_replication_filters TINYINT UNSIGNED NOT NULL AFTER replica_io_running - `, - ` - ALTER TABLE - database_instance - ADD COLUMN data_center varchar(32) CHARACTER SET ascii NOT NULL AFTER cluster_name - `, - ` - ALTER TABLE - database_instance - ADD COLUMN physical_environment varchar(32) CHARACTER SET ascii NOT NULL AFTER data_center - `, - ` - ALTER TABLE - database_instance_maintenance - ADD KEY active_timestamp_idx (maintenance_active, begin_timestamp) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN is_co_primary TINYINT UNSIGNED NOT NULL AFTER replication_depth - `, - ` - ALTER TABLE - database_instance_maintenance - ADD KEY active_end_timestamp_idx (maintenance_active, end_timestamp) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN sql_delay INT UNSIGNED NOT NULL AFTER replica_lag_seconds - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN analysis varchar(128) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN cluster_name varchar(128) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN count_affected_replicas int unsigned NOT NULL - `, - ` - ALTER TABLE hostname_unresolve - ADD COLUMN last_registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - ` - ALTER TABLE hostname_unresolve - ADD KEY last_registered_idx (last_registered) - `, - ` - ALTER TABLE topology_recovery - ADD KEY cluster_name_in_active_idx (cluster_name, in_active_period) - `, - ` - ALTER TABLE topology_recovery - ADD KEY end_recovery_idx (end_recovery) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN binlog_server TINYINT UNSIGNED NOT NULL AFTER version - `, - ` - ALTER TABLE cluster_domain_name - ADD COLUMN last_registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - ` - ALTER TABLE cluster_domain_name - ADD KEY last_registered_idx (last_registered) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN supports_oracle_gtid TINYINT UNSIGNED NOT NULL AFTER oracle_gtid - `, - ` - ALTER TABLE - database_instance - ADD COLUMN executed_gtid_set text CHARACTER SET ascii NOT NULL AFTER oracle_gtid - `, - ` - ALTER TABLE - database_instance - ADD COLUMN server_uuid varchar(64) CHARACTER SET ascii NOT NULL AFTER server_id - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN is_successful TINYINT UNSIGNED NOT NULL DEFAULT 0 AFTER processcing_node_token - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN acknowledged TINYINT UNSIGNED NOT NULL DEFAULT 0 - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN acknowledged_by varchar(128) CHARACTER SET utf8 NOT NULL - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN acknowledge_comment text CHARACTER SET utf8 NOT NULL - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN participating_instances text CHARACTER SET ascii NOT NULL after count_affected_replicas - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN lost_replicas text CHARACTER SET ascii NOT NULL after participating_instances - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN all_errors text CHARACTER SET ascii NOT NULL after lost_replicas - `, - ` - ALTER TABLE audit - ADD COLUMN cluster_name varchar(128) CHARACTER SET ascii NOT NULL DEFAULT '' AFTER port - `, - ` - ALTER TABLE candidate_database_instance - ADD COLUMN priority TINYINT SIGNED NOT NULL DEFAULT 1 comment 'positive promote, nagative unpromotes' - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN acknowledged_at TIMESTAMP NULL after acknowledged - `, - ` - ALTER TABLE - topology_recovery - ADD KEY acknowledged_idx (acknowledged, acknowledged_at) - `, - ` - ALTER TABLE - blocked_topology_recovery - ADD KEY last_blocked_idx (last_blocked_timestamp) - `, - ` - ALTER TABLE candidate_database_instance - ADD COLUMN promotion_rule enum('must', 'prefer', 'neutral', 'prefer_not', 'must_not') NOT NULL DEFAULT 'neutral' - `, - ` - ALTER TABLE node_health /* sqlite3-skip */ - DROP PRIMARY KEY, - ADD PRIMARY KEY (hostname, token) - `, - ` - ALTER TABLE node_health - ADD COLUMN extra_info varchar(128) CHARACTER SET utf8 NOT NULL - `, - ` - ALTER TABLE agent_seed /* sqlite3-skip */ - MODIFY end_timestamp timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' - `, - ` - ALTER TABLE active_node /* sqlite3-skip */ - MODIFY last_seen_active timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - - ` - ALTER TABLE node_health /* sqlite3-skip */ - MODIFY last_seen_active timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - ` - ALTER TABLE candidate_database_instance /* sqlite3-skip */ - MODIFY last_suggested timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - ` - ALTER TABLE primary_position_equivalence /* sqlite3-skip */ - MODIFY last_suggested timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - ` - ALTER TABLE - database_instance - ADD COLUMN last_attempted_check TIMESTAMP NOT NULL DEFAULT '1971-01-01 00:00:00' AFTER last_checked - `, - ` - ALTER TABLE - database_instance /* sqlite3-skip */ - MODIFY last_attempted_check TIMESTAMP NOT NULL DEFAULT '1971-01-01 00:00:00' - `, - ` - ALTER TABLE - database_instance_analysis_changelog - ADD KEY instance_timestamp_idx (hostname, port, analysis_timestamp) - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN last_detection_id bigint unsigned NOT NULL - `, - ` - ALTER TABLE - topology_recovery - ADD KEY last_detection_idx (last_detection_id) - `, - ` - ALTER TABLE node_health_history - ADD COLUMN command varchar(128) CHARACTER SET utf8 NOT NULL - `, - ` - ALTER TABLE node_health - ADD COLUMN command varchar(128) CHARACTER SET utf8 NOT NULL - `, - ` - ALTER TABLE database_instance_topology_history - ADD COLUMN version varchar(128) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE - database_instance - ADD COLUMN gtid_purged text CHARACTER SET ascii NOT NULL AFTER executed_gtid_set - `, - ` - ALTER TABLE - database_instance_coordinates_history - ADD COLUMN last_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' AFTER recorded_timestamp - `, - ` - ALTER TABLE - database_instance - ADD COLUMN has_replication_credentials TINYINT UNSIGNED NOT NULL - `, - ` - ALTER TABLE - database_instance - ADD COLUMN allow_tls TINYINT UNSIGNED NOT NULL AFTER sql_delay - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_enforced TINYINT UNSIGNED NOT NULL AFTER physical_environment - `, - ` - ALTER TABLE - database_instance - ADD COLUMN instance_alias varchar(128) CHARACTER SET ascii NOT NULL AFTER physical_environment - `, - ` - ALTER TABLE - topology_recovery - ADD COLUMN successor_alias varchar(128) DEFAULT NULL - `, - ` - ALTER TABLE - database_instance /* sqlite3-skip */ - MODIFY cluster_name varchar(128) NOT NULL - `, - ` - ALTER TABLE - node_health - ADD INDEX last_seen_active_idx (last_seen_active) - `, - ` - ALTER TABLE - database_instance_maintenance - ADD COLUMN processing_node_hostname varchar(128) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE - database_instance_maintenance - ADD COLUMN processing_node_token varchar(128) NOT NULL - `, - ` - ALTER TABLE - database_instance_maintenance - ADD COLUMN explicitly_bounded TINYINT UNSIGNED NOT NULL - `, - ` - ALTER TABLE node_health_history - ADD COLUMN app_version varchar(64) CHARACTER SET ascii NOT NULL DEFAULT "" - `, - ` - ALTER TABLE node_health - ADD COLUMN app_version varchar(64) CHARACTER SET ascii NOT NULL DEFAULT "" - `, - ` - ALTER TABLE node_health_history /* sqlite3-skip */ - MODIFY app_version varchar(64) CHARACTER SET ascii NOT NULL DEFAULT "" - `, - ` - ALTER TABLE node_health /* sqlite3-skip */ - MODIFY app_version varchar(64) CHARACTER SET ascii NOT NULL DEFAULT "" - `, - ` - ALTER TABLE - database_instance - ADD COLUMN version_comment varchar(128) NOT NULL DEFAULT '' - `, - ` - ALTER TABLE active_node - ADD COLUMN first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' - `, - ` - ALTER TABLE node_health - ADD COLUMN first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' - `, - ` - ALTER TABLE database_instance - ADD COLUMN major_version varchar(16) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE - database_instance - ADD COLUMN binlog_row_image varchar(16) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE topology_recovery - ADD COLUMN uid varchar(128) CHARACTER SET ascii NOT NULL - `, - ` - CREATE INDEX uid_idx_topology_recovery ON topology_recovery(uid) - `, - ` - CREATE INDEX recovery_uid_idx_topology_recovery_steps ON topology_recovery_steps(recovery_uid) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN last_discovery_latency bigint not null - `, - ` - CREATE INDEX end_timestamp_idx_database_instance_downtime ON database_instance_downtime(end_timestamp) - `, - ` - ALTER TABLE - topology_failure_detection - ADD COLUMN is_actionable tinyint not null default 0 - `, - ` - DROP INDEX hostname_port_active_period_uidx_topology_failure_detection ON topology_failure_detection - `, - ` - CREATE UNIQUE INDEX host_port_active_recoverable_uidx_topology_failure_detection ON topology_failure_detection (hostname, port, in_active_period, end_active_period_unixtime, is_actionable) - `, - ` - ALTER TABLE raft_snapshot - ADD COLUMN created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - `, - ` - ALTER TABLE node_health - ADD COLUMN db_backend varchar(255) CHARACTER SET ascii NOT NULL DEFAULT "" - `, - ` - ALTER TABLE node_health - ADD COLUMN incrementing_indicator bigint not null default 0 - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_primary_enabled TINYINT UNSIGNED NOT NULL - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_replica_enabled TINYINT UNSIGNED NOT NULL - `, - ` - ALTER TABLE - database_instance - ADD COLUMN gtid_mode varchar(32) CHARACTER SET ascii NOT NULL - `, - ` - ALTER TABLE - database_instance - ADD COLUMN last_check_partial_success tinyint unsigned NOT NULL after last_attempted_check - `, - ` - ALTER TABLE - database_instance - ADD COLUMN source_uuid varchar(64) CHARACTER SET ascii NOT NULL AFTER oracle_gtid - `, - ` - ALTER TABLE - database_instance - ADD COLUMN gtid_errant text CHARACTER SET ascii NOT NULL AFTER gtid_purged - `, - ` - ALTER TABLE - database_instance - ADD COLUMN ancestry_uuid text CHARACTER SET ascii NOT NULL AFTER source_uuid - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_sql_thread_state tinyint signed not null default 0 AFTER replica_io_running - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_io_thread_state tinyint signed not null default 0 AFTER replication_sql_thread_state - `, - ` - ALTER TABLE - database_instance_tags /* sqlite3-skip */ - DROP PRIMARY KEY, - ADD PRIMARY KEY (hostname, port, tag_name) - `, - ` - ALTER TABLE - database_instance - ADD COLUMN region varchar(32) CHARACTER SET ascii NOT NULL AFTER data_center - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_primary_timeout INT UNSIGNED NOT NULL DEFAULT 0 AFTER semi_sync_primary_enabled - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_primary_wait_for_replica_count INT UNSIGNED NOT NULL DEFAULT 0 AFTER semi_sync_primary_timeout - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_primary_status TINYINT UNSIGNED NOT NULL DEFAULT 0 AFTER semi_sync_primary_wait_for_replica_count - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_replica_status TINYINT UNSIGNED NOT NULL DEFAULT 0 AFTER semi_sync_primary_status - `, - ` - ALTER TABLE - database_instance - ADD COLUMN semi_sync_primary_clients INT UNSIGNED NOT NULL DEFAULT 0 AFTER semi_sync_primary_status - `, - ` - ALTER TABLE /* sqlite3-skip */ - database_instance - MODIFY semi_sync_primary_timeout BIGINT UNSIGNED NOT NULL DEFAULT 0 - `, - // Fields related to Replication Group the instance belongs to - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_name VARCHAR(64) CHARACTER SET ascii NOT NULL DEFAULT '' AFTER gtid_mode - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_is_single_primary_mode TINYINT UNSIGNED NOT NULL DEFAULT 1 AFTER replication_group_name - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_member_state VARCHAR(16) CHARACTER SET ascii NOT NULL DEFAULT '' AFTER replication_group_is_single_primary_mode - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_member_role VARCHAR(16) CHARACTER SET ascii NOT NULL DEFAULT '' AFTER replication_group_member_state - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_members text CHARACTER SET ascii NOT NULL AFTER replication_group_member_role - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_primary_host varchar(128) CHARACTER SET ascii NOT NULL DEFAULT '' AFTER replication_group_members - `, - ` - ALTER TABLE - database_instance - ADD COLUMN replication_group_primary_port smallint(5) unsigned NOT NULL DEFAULT 0 AFTER replication_group_primary_host - `, -} From b52cb1b2a0e63a2b0b008b266bdff5de9c8f668d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 23 Dec 2022 12:48:33 +0530 Subject: [PATCH 02/25] refactor: cleanup create table formatting Signed-off-by: Manan Gupta --- go/vt/vtorc/db/generate_base.go | 113 ++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 14 deletions(-) diff --git a/go/vt/vtorc/db/generate_base.go b/go/vt/vtorc/db/generate_base.go index ef1941656da..344f3acb876 100644 --- a/go/vt/vtorc/db/generate_base.go +++ b/go/vt/vtorc/db/generate_base.go @@ -44,7 +44,57 @@ CREATE TABLE database_instance ( exec_source_log_pos bigint NOT NULL, replication_lag_seconds bigint DEFAULT NULL, replica_lag_seconds bigint DEFAULT NULL, - cluster_name varchar(128) NOT NULL, read_only TINYint not null default 0, last_sql_error TEXT not null default '', last_io_error TEXT not null default '', oracle_gtid TINYint not null default 0, mariadb_gtid TINYint not null default 0, relay_log_file varchar(128) not null default '', relay_log_pos bigint not null default 0, pseudo_gtid TINYint not null default 0, replication_depth TINYint not null default 0, has_replication_filters TINYint not null default 0, data_center varchar(32) not null default '', physical_environment varchar(32) not null default '', is_co_primary TINYint not null default 0, sql_delay int not null default 0, binlog_server TINYint not null default 0, supports_oracle_gtid TINYint not null default 0, executed_gtid_set text not null default '', server_uuid varchar(64) not null default '', last_attempted_check TIMESTAMP NOT NULL DEFAULT '1971-01-01 00:00:00', gtid_purged text not null default '', has_replication_credentials TINYint not null default 0, allow_tls TINYint not null default 0, semi_sync_enforced TINYint not null default 0, instance_alias varchar(128) not null default '', version_comment varchar(128) NOT NULL DEFAULT '', major_version varchar(16) not null default '', binlog_row_image varchar(16) not null default '', last_discovery_latency bigint not null default 0, semi_sync_primary_enabled TINYint not null default 0, semi_sync_replica_enabled TINYint not null default 0, gtid_mode varchar(32) not null default '', last_check_partial_success tinyint not null default 0, source_uuid varchar(64) not null default '', gtid_errant text not null default '', ancestry_uuid text not null default '', replication_sql_thread_state tinyint signed not null default 0, replication_io_thread_state tinyint signed not null default 0, region varchar(32) not null default '', semi_sync_primary_timeout int NOT NULL DEFAULT 0, semi_sync_primary_wait_for_replica_count int NOT NULL DEFAULT 0, semi_sync_primary_status TINYint NOT NULL DEFAULT 0, semi_sync_replica_status TINYint NOT NULL DEFAULT 0, semi_sync_primary_clients int NOT NULL DEFAULT 0, replication_group_name VARCHAR(64) NOT NULL DEFAULT '', replication_group_is_single_primary_mode TINYint NOT NULL DEFAULT 1, replication_group_member_state VARCHAR(16) NOT NULL DEFAULT '', replication_group_member_role VARCHAR(16) NOT NULL DEFAULT '', replication_group_members text not null default '', replication_group_primary_host varchar(128) NOT NULL DEFAULT '', replication_group_primary_port smallint NOT NULL DEFAULT 0, + cluster_name varchar(128) NOT NULL, + read_only TINYint not null default 0, + last_sql_error TEXT not null default '', + last_io_error TEXT not null default '', + oracle_gtid TINYint not null default 0, + mariadb_gtid TINYint not null default 0, + relay_log_file varchar(128) not null default '', + relay_log_pos bigint not null default 0, + pseudo_gtid TINYint not null default 0, + replication_depth TINYint not null default 0, + has_replication_filters TINYint not null default 0, + data_center varchar(32) not null default '', + physical_environment varchar(32) not null default '', + is_co_primary TINYint not null default 0, + sql_delay int not null default 0, + binlog_server TINYint not null default 0, + supports_oracle_gtid TINYint not null default 0, + executed_gtid_set text not null default '', + server_uuid varchar(64) not null default '', + last_attempted_check TIMESTAMP NOT NULL DEFAULT '1971-01-01 00:00:00', + gtid_purged text not null default '', + has_replication_credentials TINYint not null default 0, + allow_tls TINYint not null default 0, + semi_sync_enforced TINYint not null default 0, + instance_alias varchar(128) not null default '', + version_comment varchar(128) NOT NULL DEFAULT '', + major_version varchar(16) not null default '', + binlog_row_image varchar(16) not null default '', + last_discovery_latency bigint not null default 0, + semi_sync_primary_enabled TINYint not null default 0, + semi_sync_replica_enabled TINYint not null default 0, + gtid_mode varchar(32) not null default '', + last_check_partial_success tinyint not null default 0, + source_uuid varchar(64) not null default '', + gtid_errant text not null default '', + ancestry_uuid text not null default '', + replication_sql_thread_state tinyint signed not null default 0, + replication_io_thread_state tinyint signed not null default 0, + region varchar(32) not null default '', + semi_sync_primary_timeout int NOT NULL DEFAULT 0, + semi_sync_primary_wait_for_replica_count int NOT NULL DEFAULT 0, + semi_sync_primary_status TINYint NOT NULL DEFAULT 0, + semi_sync_replica_status TINYint NOT NULL DEFAULT 0, + semi_sync_primary_clients int NOT NULL DEFAULT 0, + replication_group_name VARCHAR(64) NOT NULL DEFAULT '', + replication_group_is_single_primary_mode TINYint NOT NULL DEFAULT 1, + replication_group_member_state VARCHAR(16) NOT NULL DEFAULT '', + replication_group_member_role VARCHAR(16) NOT NULL DEFAULT '', + replication_group_members text not null default '', + replication_group_primary_host varchar(128) NOT NULL DEFAULT '', + replication_group_primary_port smallint NOT NULL DEFAULT 0, PRIMARY KEY (hostname,port) )`, ` @@ -68,7 +118,10 @@ CREATE TABLE database_instance_maintenance ( begin_timestamp timestamp NULL DEFAULT NULL, end_timestamp timestamp NULL DEFAULT NULL, owner varchar(128) NOT NULL, - reason text NOT NULL, processing_node_hostname varchar(128) not null default '', processing_node_token varchar(128) not null default '', explicitly_bounded TINYint not null default 0, + reason text NOT NULL, + processing_node_hostname varchar(128) not null default '', + processing_node_token varchar(128) not null default '', + explicitly_bounded TINYint not null default 0, PRIMARY KEY (database_instance_maintenance_id) )`, ` @@ -105,7 +158,8 @@ CREATE TABLE audit ( audit_type varchar(128) NOT NULL, hostname varchar(128) NOT NULL DEFAULT '', port smallint NOT NULL, - message text NOT NULL, cluster_name varchar(128) NOT NULL DEFAULT '', + message text NOT NULL, + cluster_name varchar(128) NOT NULL DEFAULT '', PRIMARY KEY (audit_id) )`, ` @@ -230,7 +284,8 @@ CREATE TABLE active_node ( anchor tinyint NOT NULL, hostname varchar(128) NOT NULL, token varchar(128) NOT NULL, - last_seen_active timestamp not null default (''), first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + last_seen_active timestamp not null default (''), + first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', PRIMARY KEY (anchor) )`, ` @@ -240,7 +295,13 @@ DROP TABLE IF EXISTS node_health CREATE TABLE node_health ( hostname varchar(128) NOT NULL, token varchar(128) NOT NULL, - last_seen_active timestamp not null default (''), extra_info varchar(128) not null default '', command varchar(128) not null default '', app_version varchar(64) NOT NULL DEFAULT "", first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', db_backend varchar(255) NOT NULL DEFAULT "", incrementing_indicator bigint not null default 0, + last_seen_active timestamp not null default (''), + extra_info varchar(128) not null default '', + command varchar(128) not null default '', + app_version varchar(64) NOT NULL DEFAULT "", + first_seen_active timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + db_backend varchar(255) NOT NULL DEFAULT "", + incrementing_indicator bigint not null default 0, PRIMARY KEY (hostname, token) )`, ` @@ -258,7 +319,21 @@ CREATE TABLE topology_recovery ( processing_node_hostname varchar(128) NOT NULL, processcing_node_token varchar(128) NOT NULL, successor_hostname varchar(128) DEFAULT NULL, - successor_port smallint DEFAULT NULL, analysis varchar(128) not null default '', cluster_name varchar(128) not null default '', count_affected_replicas int not null default 0, is_successful TINYint NOT NULL DEFAULT 0, acknowledged TINYint NOT NULL DEFAULT 0, acknowledged_by varchar(128) not null default '', acknowledge_comment text not null default '', participating_instances text not null default '', lost_replicas text not null default '', all_errors text not null default '', acknowledged_at TIMESTAMP NULL, last_detection_id bigint not null default 0, successor_alias varchar(128) DEFAULT NULL, uid varchar(128) not null default '', + successor_port smallint DEFAULT NULL, + analysis varchar(128) not null default '', + cluster_name varchar(128) not null default '', + count_affected_replicas int not null default 0, + is_successful TINYint NOT NULL DEFAULT 0, + acknowledged TINYint NOT NULL DEFAULT 0, + acknowledged_by varchar(128) not null default '', + acknowledge_comment text not null default '', + participating_instances text not null default '', + lost_replicas text not null default '', + all_errors text not null default '', + acknowledged_at TIMESTAMP NULL, + last_detection_id bigint not null default 0, + successor_alias varchar(128) DEFAULT NULL, + uid varchar(128) not null default '', PRIMARY KEY (recovery_id) )`, ` @@ -276,7 +351,8 @@ DROP TABLE IF EXISTS hostname_unresolve ` CREATE TABLE hostname_unresolve ( hostname varchar(128) NOT NULL, - unresolved_hostname varchar(128) NOT NULL, last_registered timestamp not null default (''), + unresolved_hostname varchar(128) NOT NULL, + last_registered timestamp not null default (''), PRIMARY KEY (hostname) )`, ` @@ -292,7 +368,8 @@ CREATE TABLE database_instance_topology_history ( port smallint NOT NULL, source_host varchar(128) NOT NULL, source_port smallint NOT NULL, - cluster_name tinytext NOT NULL, version varchar(128) not null default '', + cluster_name tinytext NOT NULL, + version varchar(128) not null default '', PRIMARY KEY (snapshot_unix_timestamp, hostname, port) )`, ` @@ -305,7 +382,9 @@ DROP TABLE IF EXISTS candidate_database_instance CREATE TABLE candidate_database_instance ( hostname varchar(128) NOT NULL, port smallint NOT NULL, - last_suggested timestamp not null default (''), priority TINYINT SIGNED NOT NULL DEFAULT 1, promotion_rule text check(promotion_rule in ('must', 'prefer', 'neutral', 'prefer_not', 'must_not')) NOT NULL DEFAULT 'neutral', + last_suggested timestamp not null default (''), + priority TINYINT SIGNED NOT NULL DEFAULT 1, + promotion_rule text check(promotion_rule in ('must', 'prefer', 'neutral', 'prefer_not', 'must_not')) NOT NULL DEFAULT 'neutral', PRIMARY KEY (hostname, port) )`, ` @@ -340,7 +419,8 @@ CREATE TABLE topology_failure_detection ( processcing_node_token varchar(128) NOT NULL, analysis varchar(128) NOT NULL, cluster_name varchar(128) NOT NULL, - count_affected_replicas int NOT NULL, is_actionable tinyint not null default 0, + count_affected_replicas int NOT NULL, + is_actionable tinyint not null default 0, PRIMARY KEY (detection_id) )`, ` @@ -384,7 +464,8 @@ DROP TABLE IF EXISTS cluster_domain_name ` CREATE TABLE cluster_domain_name ( cluster_name varchar(128) NOT NULL, - domain_name varchar(128) NOT NULL, last_registered timestamp not null default (''), + domain_name varchar(128) NOT NULL, + last_registered timestamp not null default (''), PRIMARY KEY (cluster_name) )`, ` @@ -494,7 +575,9 @@ CREATE TABLE node_health_history ( hostname varchar(128) NOT NULL, token varchar(128) NOT NULL, first_seen_active timestamp NOT NULL, - extra_info varchar(128) NOT NULL, command varchar(128) not null default '', app_version varchar(64) NOT NULL DEFAULT "", + extra_info varchar(128) NOT NULL, + command varchar(128) not null default '', + app_version varchar(64) NOT NULL DEFAULT "", PRIMARY KEY (history_id) )`, ` @@ -515,7 +598,8 @@ CREATE TABLE database_instance_coordinates_history ( binary_log_file varchar(128) NOT NULL, binary_log_pos bigint NOT NULL, relay_log_file varchar(128) NOT NULL, - relay_log_pos bigint NOT NULL, last_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', + relay_log_pos bigint NOT NULL, + last_seen timestamp NOT NULL DEFAULT '1971-01-01 00:00:00', PRIMARY KEY (history_id) )`, ` @@ -631,7 +715,8 @@ DROP TABLE IF EXISTS raft_snapshot CREATE TABLE raft_snapshot ( snapshot_id integer, snapshot_name varchar(128) NOT NULL, - snapshot_meta varchar(4096) NOT NULL, created_at timestamp not null default (''), + snapshot_meta varchar(4096) NOT NULL, + created_at timestamp not null default (''), PRIMARY KEY (snapshot_id) )`, ` From 1f8cc351f8ca0df481f3ad3494a1896e7845ff71 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 23 Dec 2022 13:11:28 +0530 Subject: [PATCH 03/25] feat: cleanup the usage of IsSQLite and IsMySQL Signed-off-by: Manan Gupta --- go/vt/vtorc/config/config.go | 12 +------ go/vt/vtorc/db/db.go | 68 +++++------------------------------- 2 files changed, 10 insertions(+), 70 deletions(-) diff --git a/go/vt/vtorc/config/config.go b/go/vt/vtorc/config/config.go index fd54e9ed582..3d3dde96034 100644 --- a/go/vt/vtorc/config/config.go +++ b/go/vt/vtorc/config/config.go @@ -162,23 +162,13 @@ func newConfiguration() *Configuration { } func (config *Configuration) postReadAdjustments() error { - if config.IsSQLite() && config.SQLite3DataFile == "" { + if config.SQLite3DataFile == "" { return fmt.Errorf("SQLite3DataFile must be set") } return nil } -// TODO: Simplify the callers and delete this function -func (config *Configuration) IsSQLite() bool { - return true -} - -// TODO: Simplify the callers and delete this function -func (config *Configuration) IsMySQL() bool { - return false -} - // read reads configuration from given file, or silently skips if the file does not exist. // If the file does exist, then it is expected to be in valid JSON format or the function bails out. func read(fileName string) (*Configuration, error) { diff --git a/go/vt/vtorc/db/db.go b/go/vt/vtorc/db/db.go index 91b46419a20..6338813ccec 100644 --- a/go/vt/vtorc/db/db.go +++ b/go/vt/vtorc/db/db.go @@ -54,10 +54,6 @@ func (dummyRes DummySQLResult) RowsAffected() (int64, error) { return 1, nil } -func IsSQLite() bool { - return config.Config.IsSQLite() -} - // OpenTopology returns the DB instance for the vtorc backed database func OpenVTOrc() (db *sql.DB, err error) { var fromCache bool @@ -73,11 +69,8 @@ func OpenVTOrc() (db *sql.DB, err error) { return db, err } -func translateStatement(statement string) (string, error) { - if IsSQLite() { - statement = sqlutils.ToSqlite3Dialect(statement) - } - return statement, nil +func translateStatement(statement string) string { + return sqlutils.ToSqlite3Dialect(statement) } // registerVTOrcDeployment updates the vtorc_metadata table upon successful deployment @@ -102,30 +95,8 @@ func deployStatements(db *sql.DB, queries []string) error { if err != nil { log.Fatal(err.Error()) } - // Ugly workaround ahead. - // Origin of this workaround is the existence of some "timestamp NOT NULL," column definitions, - // where in NO_ZERO_IN_DATE,NO_ZERO_DATE sql_mode are invalid (since default is implicitly "0") - // This means installation of vtorc fails on such configured servers, and in particular on 5.7 - // where this setting is the dfault. - // For purpose of backwards compatability, what we do is force sql_mode to be more relaxed, create the schemas - // along with the "invalid" definition, and then go ahead and fix those definitions via following ALTER statements. - // My bad. - originalSQLMode := "" - if config.Config.IsMySQL() { - _ = tx.QueryRow(`select @@session.sql_mode`).Scan(&originalSQLMode) - if _, err := tx.Exec(`set @@session.sql_mode=REPLACE(@@session.sql_mode, 'NO_ZERO_DATE', '')`); err != nil { - log.Fatal(err.Error()) - } - if _, err := tx.Exec(`set @@session.sql_mode=REPLACE(@@session.sql_mode, 'NO_ZERO_IN_DATE', '')`); err != nil { - log.Fatal(err.Error()) - } - } for _, query := range queries { - query, err := translateStatement(query) - if err != nil { - log.Fatalf("Cannot initiate vtorc: %+v; query=%+v", err, query) - return err - } + query = translateStatement(query) if _, err := tx.Exec(query); err != nil { if strings.Contains(err.Error(), "syntax error") { log.Fatalf("Cannot initiate vtorc: %+v; query=%+v", err, query) @@ -144,11 +115,6 @@ func deployStatements(db *sql.DB, queries []string) error { } } } - if config.Config.IsMySQL() { - if _, err := tx.Exec(`set session sql_mode=?`, originalSQLMode); err != nil { - log.Fatal(err.Error()) - } - } if err := tx.Commit(); err != nil { log.Fatal(err.Error()) } @@ -163,10 +129,8 @@ func initVTOrcDB(db *sql.DB) error { _ = deployStatements(db, vtorcBackend) _ = registerVTOrcDeployment(db) - if IsSQLite() { - _, _ = ExecVTOrc(`PRAGMA journal_mode = WAL`) - _, _ = ExecVTOrc(`PRAGMA synchronous = NORMAL`) - } + _, _ = ExecVTOrc(`PRAGMA journal_mode = WAL`) + _, _ = ExecVTOrc(`PRAGMA synchronous = NORMAL`) return nil } @@ -174,10 +138,7 @@ func initVTOrcDB(db *sql.DB) error { // execInternal func execInternal(db *sql.DB, query string, args ...any) (sql.Result, error) { var err error - query, err = translateStatement(query) - if err != nil { - return nil, err - } + query = translateStatement(query) res, err := sqlutils.ExecNoPrepare(db, query, args...) return res, err } @@ -185,10 +146,7 @@ func execInternal(db *sql.DB, query string, args ...any) (sql.Result, error) { // ExecVTOrc will execute given query on the vtorc backend database. func ExecVTOrc(query string, args ...any) (sql.Result, error) { var err error - query, err = translateStatement(query) - if err != nil { - return nil, err - } + query = translateStatement(query) db, err := OpenVTOrc() if err != nil { return nil, err @@ -199,11 +157,7 @@ func ExecVTOrc(query string, args ...any) (sql.Result, error) { // QueryVTOrcRowsMap func QueryVTOrcRowsMap(query string, onRow func(sqlutils.RowMap) error) error { - query, err := translateStatement(query) - if err != nil { - log.Fatalf("Cannot query vtorc: %+v; query=%+v", err, query) - return err - } + query = translateStatement(query) db, err := OpenVTOrc() if err != nil { return err @@ -214,11 +168,7 @@ func QueryVTOrcRowsMap(query string, onRow func(sqlutils.RowMap) error) error { // QueryVTOrc func QueryVTOrc(query string, argsArray []any, onRow func(sqlutils.RowMap) error) error { - query, err := translateStatement(query) - if err != nil { - log.Fatalf("Cannot query vtorc: %+v; query=%+v", err, query) - return err - } + query = translateStatement(query) db, err := OpenVTOrc() if err != nil { return err From 6a6f76791f85833e06c4f32073bd9bb73f03375b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 23 Dec 2022 13:37:08 +0530 Subject: [PATCH 04/25] feat: remove unused minimal instance Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/minimal_instance.go | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 go/vt/vtorc/inst/minimal_instance.go diff --git a/go/vt/vtorc/inst/minimal_instance.go b/go/vt/vtorc/inst/minimal_instance.go deleted file mode 100644 index 1eeb85663d3..00000000000 --- a/go/vt/vtorc/inst/minimal_instance.go +++ /dev/null @@ -1,15 +0,0 @@ -package inst - -type MinimalInstance struct { - Key InstanceKey - PrimaryKey InstanceKey - ClusterName string -} - -func (minimalInstance *MinimalInstance) ToInstance() *Instance { - return &Instance{ - Key: minimalInstance.Key, - SourceKey: minimalInstance.PrimaryKey, - ClusterName: minimalInstance.ClusterName, - } -} From 19dd8eac743e48411658f6ae0ca874d964d6e59c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 23 Dec 2022 14:03:15 +0530 Subject: [PATCH 05/25] feat: remove unused table cluster_domain_name Signed-off-by: Manan Gupta --- go/vt/vtorc/db/generate_base.go | 16 ------- go/vt/vtorc/inst/analysis_dao.go | 8 +--- go/vt/vtorc/inst/cluster_domain_dao.go | 60 -------------------------- go/vt/vtorc/logic/orchestrator.go | 1 - 4 files changed, 1 insertion(+), 84 deletions(-) delete mode 100644 go/vt/vtorc/inst/cluster_domain_dao.go diff --git a/go/vt/vtorc/db/generate_base.go b/go/vt/vtorc/db/generate_base.go index 344f3acb876..be0ecba4627 100644 --- a/go/vt/vtorc/db/generate_base.go +++ b/go/vt/vtorc/db/generate_base.go @@ -459,19 +459,6 @@ CREATE INDEX hostname_idx_hostname_unresolve_history ON hostname_unresolve_histo CREATE INDEX last_registered_idx_hostname_unresolve_history ON hostname_unresolve_history (last_registered) `, ` -DROP TABLE IF EXISTS cluster_domain_name -`, - ` -CREATE TABLE cluster_domain_name ( - cluster_name varchar(128) NOT NULL, - domain_name varchar(128) NOT NULL, - last_registered timestamp not null default (''), - PRIMARY KEY (cluster_name) -)`, - ` -CREATE INDEX domain_name_idx_cluster_domain_name ON cluster_domain_name (domain_name) - `, - ` DROP TABLE IF EXISTS primary_position_equivalence `, ` @@ -844,9 +831,6 @@ CREATE INDEX cluster_name_in_active_idx_topology_recovery on topology_recovery ( CREATE INDEX end_recovery_idx_topology_recovery on topology_recovery (end_recovery) `, ` -CREATE INDEX last_registered_idx_cluster_domain_name on cluster_domain_name (last_registered) - `, - ` CREATE INDEX acknowledged_idx_topology_recovery on topology_recovery (acknowledged, acknowledged_at) `, ` diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index 657bb6ddea3..eaba5bb215c 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -98,10 +98,7 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) ) ) AS is_stale_binlog_coordinates, MIN( - IFNULL( - cluster_domain_name.domain_name, - primary_instance.cluster_name - ) + primary_instance.cluster_name ) AS cluster_domain, MIN( primary_instance.last_checked <= primary_instance.last_seen @@ -332,9 +329,6 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) AND replica_instance.port = replica_downtime.port AND replica_downtime.downtime_active = 1 ) - LEFT JOIN cluster_domain_name ON ( - cluster_domain_name.cluster_name = primary_instance.cluster_name - ) WHERE database_instance_maintenance.database_instance_maintenance_id IS NULL AND ? IN ('', primary_instance.cluster_name) diff --git a/go/vt/vtorc/inst/cluster_domain_dao.go b/go/vt/vtorc/inst/cluster_domain_dao.go deleted file mode 100644 index 45aed648be1..00000000000 --- a/go/vt/vtorc/inst/cluster_domain_dao.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright 2015 Shlomi Noach, courtesy Booking.com - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package inst - -import ( - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/vtorc/config" - "vitess.io/vitess/go/vt/vtorc/db" -) - -// WriteClusterDomainName will write (and override) the domain name of a cluster -func WriteClusterDomainName(clusterName string, domainName string) error { - writeFunc := func() error { - _, err := db.ExecVTOrc(` - insert into - cluster_domain_name (cluster_name, domain_name, last_registered) - values - (?, ?, NOW()) - on duplicate key update - domain_name=values(domain_name), - last_registered=values(last_registered) - `, - clusterName, domainName) - if err != nil { - log.Error(err) - } - return err - } - return ExecDBWriteFunc(writeFunc) -} - -// ExpireClusterDomainName expires cluster_domain_name entries that haven't been updated recently. -func ExpireClusterDomainName() error { - writeFunc := func() error { - _, err := db.ExecVTOrc(` - delete from cluster_domain_name - where last_registered < NOW() - INTERVAL ? MINUTE - `, config.ExpiryHostnameResolvesMinutes, - ) - if err != nil { - log.Error(err) - } - return err - } - return ExecDBWriteFunc(writeFunc) -} diff --git a/go/vt/vtorc/logic/orchestrator.go b/go/vt/vtorc/logic/orchestrator.go index 77e0fd30993..11b92f6111f 100644 --- a/go/vt/vtorc/logic/orchestrator.go +++ b/go/vt/vtorc/logic/orchestrator.go @@ -395,7 +395,6 @@ func ContinuousDiscovery() { go inst.ExpireMaintenance() go inst.ExpireCandidateInstances() go inst.ExpireHostnameUnresolve() - go inst.ExpireClusterDomainName() go inst.ExpireAudit() go inst.FlushNontrivialResolveCacheToDatabase() go inst.ExpireStaleInstanceBinlogCoordinates() From 1d9d8b08ebc84c37267e00daa9484433c6190537 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 10:22:37 +0530 Subject: [PATCH 06/25] feat: fix vtorc database to store keyspace and shard instead of cluster Signed-off-by: Manan Gupta --- go/vt/vtorc/db/generate_base.go | 34 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/go/vt/vtorc/db/generate_base.go b/go/vt/vtorc/db/generate_base.go index be0ecba4627..465cf11a822 100644 --- a/go/vt/vtorc/db/generate_base.go +++ b/go/vt/vtorc/db/generate_base.go @@ -44,7 +44,6 @@ CREATE TABLE database_instance ( exec_source_log_pos bigint NOT NULL, replication_lag_seconds bigint DEFAULT NULL, replica_lag_seconds bigint DEFAULT NULL, - cluster_name varchar(128) NOT NULL, read_only TINYint not null default 0, last_sql_error TEXT not null default '', last_io_error TEXT not null default '', @@ -98,9 +97,6 @@ CREATE TABLE database_instance ( PRIMARY KEY (hostname,port) )`, ` -CREATE INDEX cluster_name_idx_database_instance ON database_instance(cluster_name) - `, - ` CREATE INDEX last_checked_idx_database_instance ON database_instance(last_checked) `, ` @@ -159,7 +155,8 @@ CREATE TABLE audit ( hostname varchar(128) NOT NULL DEFAULT '', port smallint NOT NULL, message text NOT NULL, - cluster_name varchar(128) NOT NULL DEFAULT '', + keyspace varchar(128) NOT NULL, + shard varchar(128) NOT NULL, PRIMARY KEY (audit_id) )`, ` @@ -321,7 +318,8 @@ CREATE TABLE topology_recovery ( successor_hostname varchar(128) DEFAULT NULL, successor_port smallint DEFAULT NULL, analysis varchar(128) not null default '', - cluster_name varchar(128) not null default '', + keyspace varchar(128) NOT NULL, + shard varchar(128) NOT NULL, count_affected_replicas int not null default 0, is_successful TINYint NOT NULL DEFAULT 0, acknowledged TINYint NOT NULL DEFAULT 0, @@ -368,12 +366,13 @@ CREATE TABLE database_instance_topology_history ( port smallint NOT NULL, source_host varchar(128) NOT NULL, source_port smallint NOT NULL, - cluster_name tinytext NOT NULL, + keyspace varchar(128) NOT NULL, + shard varchar(128) NOT NULL, version varchar(128) not null default '', PRIMARY KEY (snapshot_unix_timestamp, hostname, port) )`, ` -CREATE INDEX cluster_name_idx_database_instance_topology_history ON database_instance_topology_history (snapshot_unix_timestamp, cluster_name) +CREATE INDEX keyspace_shard_idx_database_instance_topology_history ON database_instance_topology_history (snapshot_unix_timestamp, keyspace, shard) `, ` DROP TABLE IF EXISTS candidate_database_instance @@ -418,7 +417,8 @@ CREATE TABLE topology_failure_detection ( processing_node_hostname varchar(128) NOT NULL, processcing_node_token varchar(128) NOT NULL, analysis varchar(128) NOT NULL, - cluster_name varchar(128) NOT NULL, + keyspace varchar(128) NOT NULL, + shard varchar(128) NOT NULL, count_affected_replicas int NOT NULL, is_actionable tinyint not null default 0, PRIMARY KEY (detection_id) @@ -515,14 +515,15 @@ DROP TABLE IF EXISTS blocked_topology_recovery CREATE TABLE blocked_topology_recovery ( hostname varchar(128) NOT NULL, port smallint NOT NULL, - cluster_name varchar(128) NOT NULL, + keyspace varchar(128) NOT NULL, + shard varchar(128) NOT NULL, analysis varchar(128) NOT NULL, last_blocked_timestamp timestamp not null default (''), blocking_recovery_id bigint, PRIMARY KEY (hostname, port) )`, ` -CREATE INDEX cluster_blocked_idx_blocked_topology_recovery ON blocked_topology_recovery (cluster_name, last_blocked_timestamp) +CREATE INDEX keyspace_shard_blocked_idx_blocked_topology_recovery ON blocked_topology_recovery (keyspace, shard, last_blocked_timestamp) `, ` DROP TABLE IF EXISTS database_instance_last_analysis @@ -732,15 +733,6 @@ CREATE TABLE database_instance_tls ( PRIMARY KEY (hostname,port) )`, ` -DROP TABLE IF EXISTS cluster_injected_pseudo_gtid -`, - ` -CREATE TABLE cluster_injected_pseudo_gtid ( - cluster_name varchar(128) NOT NULL, - time_injected timestamp not null default (''), - PRIMARY KEY (cluster_name) -)`, - ` DROP TABLE IF EXISTS hostname_ips `, ` @@ -825,7 +817,7 @@ CREATE INDEX active_end_timestamp_idx_database_instance_maintenance on database_ CREATE INDEX last_registered_idx_hostname_unresolve on hostname_unresolve (last_registered) `, ` -CREATE INDEX cluster_name_in_active_idx_topology_recovery on topology_recovery (cluster_name, in_active_period) +CREATE INDEX keyspace_shard_in_active_idx_topology_recovery on topology_recovery (keyspace, shard, in_active_period) `, ` CREATE INDEX end_recovery_idx_topology_recovery on topology_recovery (end_recovery) From 6997250c7d47d6220262441a563f1b53c34ab44e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 10:36:47 +0530 Subject: [PATCH 07/25] feat: remove unused attributes Signed-off-by: Manan Gupta --- go/vt/vtorc/attributes/attributes.go | 26 ------ go/vt/vtorc/attributes/attributes_dao.go | 111 ----------------------- go/vt/vtorc/db/generate_base.go | 24 ----- go/vt/vtorc/logic/topology_recovery.go | 4 - 4 files changed, 165 deletions(-) delete mode 100644 go/vt/vtorc/attributes/attributes.go delete mode 100644 go/vt/vtorc/attributes/attributes_dao.go diff --git a/go/vt/vtorc/attributes/attributes.go b/go/vt/vtorc/attributes/attributes.go deleted file mode 100644 index 466f57c93d7..00000000000 --- a/go/vt/vtorc/attributes/attributes.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - Copyright 2014 Outbrain Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package attributes - -// HostAttributes presents attributes submitted by a host -type HostAttributes struct { - Hostname string - AttributeName string - AttributeValue string - SubmitTimestamp string - ExpireTimestamp string -} diff --git a/go/vt/vtorc/attributes/attributes_dao.go b/go/vt/vtorc/attributes/attributes_dao.go deleted file mode 100644 index 503d4e4d22c..00000000000 --- a/go/vt/vtorc/attributes/attributes_dao.go +++ /dev/null @@ -1,111 +0,0 @@ -/* - Copyright 2014 Outbrain Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package attributes - -import ( - "fmt" - - "vitess.io/vitess/go/vt/log" - - "github.com/openark/golib/sqlutils" - - "vitess.io/vitess/go/vt/vtorc/db" -) - -// SetHostAttributes is used to set host attributes -func SetHostAttributes(hostname string, attributeName string, attributeValue string) error { - _, err := db.ExecVTOrc(` - replace - into host_attributes ( - hostname, attribute_name, attribute_value, submit_timestamp, expire_timestamp - ) VALUES ( - ?, ?, ?, NOW(), NULL - ) - `, - hostname, - attributeName, - attributeValue, - ) - if err != nil { - log.Error(err) - return err - } - - return err -} - -func getHostAttributesByClause(whereClause string, args []any) ([]HostAttributes, error) { - var res []HostAttributes - query := fmt.Sprintf(` - select - hostname, - attribute_name, - attribute_value, - submit_timestamp , - ifnull(expire_timestamp, '') as expire_timestamp - from - host_attributes - %s - order by - hostname, attribute_name - `, whereClause) - - err := db.QueryVTOrc(query, args, func(m sqlutils.RowMap) error { - hostAttributes := HostAttributes{} - hostAttributes.Hostname = m.GetString("hostname") - hostAttributes.AttributeName = m.GetString("attribute_name") - hostAttributes.AttributeValue = m.GetString("attribute_value") - hostAttributes.SubmitTimestamp = m.GetString("submit_timestamp") - hostAttributes.ExpireTimestamp = m.GetString("expire_timestamp") - - res = append(res, hostAttributes) - return nil - }) - - if err != nil { - log.Error(err) - } - return res, err -} - -// GetHostAttribute expects to return a single attribute for a given hostname/attribute-name combination -// or error on empty result -func GetHostAttribute(hostname string, attributeName string) (string, error) { - whereClause := `where hostname=? and attribute_name=?` - attributes, err := getHostAttributesByClause(whereClause, sqlutils.Args(hostname, attributeName)) - if err != nil { - return "", err - } - if len(attributeName) == 0 { - log.Errorf("No attribute found for %+v, %+v", hostname, attributeName) - return "", fmt.Errorf("No attribute found for %+v, %+v", hostname, attributeName) - } - return attributes[0].AttributeValue, nil -} - -// SetGeneralAttribute sets an attribute not associated with a host. Its a key-value thing -func SetGeneralAttribute(attributeName string, attributeValue string) error { - if attributeName == "" { - return nil - } - return SetHostAttributes("*", attributeName, attributeValue) -} - -// GetGeneralAttribute expects to return a single attribute value (not associated with a specific hostname) -func GetGeneralAttribute(attributeName string) (result string, err error) { - return GetHostAttribute("*", attributeName) -} diff --git a/go/vt/vtorc/db/generate_base.go b/go/vt/vtorc/db/generate_base.go index 465cf11a822..3dd3cc437e4 100644 --- a/go/vt/vtorc/db/generate_base.go +++ b/go/vt/vtorc/db/generate_base.go @@ -237,30 +237,6 @@ CREATE TABLE agent_seed_state ( CREATE INDEX agent_seed_idx_agent_seed_state ON agent_seed_state (agent_seed_id, state_timestamp) `, ` -DROP TABLE IF EXISTS host_attributes -`, - ` -CREATE TABLE host_attributes ( - hostname varchar(128) NOT NULL, - attribute_name varchar(128) NOT NULL, - attribute_value varchar(128) NOT NULL, - submit_timestamp timestamp not null default (''), - expire_timestamp timestamp NULL DEFAULT NULL, - PRIMARY KEY (hostname,attribute_name) -)`, - ` -CREATE INDEX attribute_name_idx_host_attributes ON host_attributes (attribute_name) - `, - ` -CREATE INDEX attribute_value_idx_host_attributes ON host_attributes (attribute_value) - `, - ` -CREATE INDEX submit_timestamp_idx_host_attributes ON host_attributes (submit_timestamp) - `, - ` -CREATE INDEX expire_timestamp_idx_host_attributes ON host_attributes (expire_timestamp) - `, - ` DROP TABLE IF EXISTS hostname_resolve `, ` diff --git a/go/vt/vtorc/logic/topology_recovery.go b/go/vt/vtorc/logic/topology_recovery.go index 808f6d3e21b..451fd028599 100644 --- a/go/vt/vtorc/logic/topology_recovery.go +++ b/go/vt/vtorc/logic/topology_recovery.go @@ -34,7 +34,6 @@ import ( "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/vtorc/attributes" "vitess.io/vitess/go/vt/vtorc/config" "vitess.io/vitess/go/vt/vtorc/inst" "vitess.io/vitess/go/vt/vtorc/util" @@ -327,8 +326,6 @@ func postErsCompletion(topologyRecovery *TopologyRecovery, analysisEntry inst.Re if promotedReplica != nil { // Success! _ = AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: successfully promoted %+v", promotedReplica.Key)) - - _ = attributes.SetGeneralAttribute(analysisEntry.ClusterDetails.ClusterDomain, promotedReplica.Key.StringCode()) } } @@ -820,7 +817,6 @@ func postPrsCompletion(topologyRecovery *TopologyRecovery, analysisEntry inst.Re if promotedReplica != nil { // Success! _ = AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("%+v: successfully promoted %+v", analysisEntry.Analysis, promotedReplica.Key)) - _ = attributes.SetGeneralAttribute(analysisEntry.ClusterDetails.ClusterDomain, promotedReplica.Key.StringCode()) } } From 3bd217d17e27567868ff5e74259ebb50aa189b4f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 10:38:43 +0530 Subject: [PATCH 08/25] feat: remove unused cluster domain Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/analysis_dao.go | 4 ---- go/vt/vtorc/inst/cluster.go | 1 - go/vt/vtorc/test/recovery_analysis.go | 3 --- 3 files changed, 8 deletions(-) diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index eaba5bb215c..0199d574359 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -97,9 +97,6 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) 0 ) ) AS is_stale_binlog_coordinates, - MIN( - primary_instance.cluster_name - ) AS cluster_domain, MIN( primary_instance.last_checked <= primary_instance.last_seen and primary_instance.last_attempted_check <= primary_instance.last_seen + interval ? second @@ -387,7 +384,6 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) } isStaleBinlogCoordinates := m.GetBool("is_stale_binlog_coordinates") a.ClusterDetails.ClusterName = m.GetString("cluster_name") - a.ClusterDetails.ClusterDomain = m.GetString("cluster_domain") a.GTIDMode = m.GetString("gtid_mode") a.LastCheckValid = m.GetBool("is_last_check_valid") a.LastCheckPartialSuccess = m.GetBool("last_check_partial_success") diff --git a/go/vt/vtorc/inst/cluster.go b/go/vt/vtorc/inst/cluster.go index 805b25d7af4..ba4a569e3e1 100644 --- a/go/vt/vtorc/inst/cluster.go +++ b/go/vt/vtorc/inst/cluster.go @@ -19,7 +19,6 @@ package inst // ClusterInfo makes for a cluster status/info summary type ClusterInfo struct { ClusterName string - ClusterDomain string // CNAME/VIP/A-record/whatever of the primary of this cluster CountInstances uint HeuristicLag int64 HasAutomatedPrimaryRecovery bool diff --git a/go/vt/vtorc/test/recovery_analysis.go b/go/vt/vtorc/test/recovery_analysis.go index 4fa748f91cf..b8f2efe731d 100644 --- a/go/vt/vtorc/test/recovery_analysis.go +++ b/go/vt/vtorc/test/recovery_analysis.go @@ -48,7 +48,6 @@ type InfoForRecoveryAnalysis struct { LogPos int64 IsStaleBinlogCoordinates int ClusterName string - ClusterDomain string GTIDMode string LastCheckValid int LastCheckPartialSuccess int @@ -90,7 +89,6 @@ func (info *InfoForRecoveryAnalysis) ConvertToRowMap() sqlutils.RowMap { rowMap := make(sqlutils.RowMap) rowMap["binary_log_file"] = sqlutils.CellData{String: info.LogFile, Valid: true} rowMap["binary_log_pos"] = sqlutils.CellData{String: fmt.Sprintf("%v", info.LogPos), Valid: true} - rowMap["cluster_domain"] = sqlutils.CellData{String: info.ClusterDomain, Valid: true} rowMap["cluster_name"] = sqlutils.CellData{String: info.ClusterName, Valid: true} rowMap["count_binlog_server_replicas"] = sqlutils.CellData{Valid: false} rowMap["count_co_primary_replicas"] = sqlutils.CellData{Valid: false} @@ -164,5 +162,4 @@ func (info *InfoForRecoveryAnalysis) SetValuesFromTabletInfo() { info.Keyspace = info.TabletInfo.Keyspace info.Shard = info.TabletInfo.Shard info.ClusterName = fmt.Sprintf("%v:%v", info.TabletInfo.Keyspace, info.TabletInfo.Shard) - info.ClusterDomain = fmt.Sprintf("%v:%d", info.TabletInfo.MysqlHostname, info.TabletInfo.MysqlPort) } From 36ae49af57589615a11308f5fcb5fdd3be830a06 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 11:20:49 +0530 Subject: [PATCH 09/25] feat: change GetClusterName to GetKeyspaceAndShardName Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/audit_dao.go | 3 +- go/vt/vtorc/inst/instance_dao.go | 20 ++++++-------- go/vt/vtorc/inst/instance_dao_test.go | 40 +++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/go/vt/vtorc/inst/audit_dao.go b/go/vt/vtorc/inst/audit_dao.go index 36c690d2513..44bdfc4a7f9 100644 --- a/go/vt/vtorc/inst/audit_dao.go +++ b/go/vt/vtorc/inst/audit_dao.go @@ -57,7 +57,8 @@ func AuditOperation(auditType string, instanceKey *InstanceKey, message string) } clusterName := "" if instanceKey.Hostname != "" { - clusterName, _ = GetClusterName(instanceKey) + keyspace, shard, _ := GetKeyspaceShardName(instanceKey) + clusterName = GetClusterNameFromKeyspaceAndShard(keyspace, shard) } auditWrittenToFile := false diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index d6610de09b6..e04cf5dde6e 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -69,8 +69,6 @@ const ( GroupReplicationMemberStateError = "ERROR" ) -// instanceKeyInformativeClusterName is a non-authoritative cache; used for auditing or general purpose. -var instanceKeyInformativeClusterName *cache.Cache var forgetInstanceKeys *cache.Cache var accessDeniedCounter = metrics.NewCounter() @@ -95,7 +93,6 @@ func init() { func initializeInstanceDao() { config.WaitForConfigurationToBeLoaded() - instanceKeyInformativeClusterName = cache.New(time.Duration(config.Config.InstancePollSeconds/2)*time.Second, time.Second) forgetInstanceKeys = cache.New(time.Duration(config.Config.InstancePollSeconds*3)*time.Second, time.Second) } @@ -998,28 +995,27 @@ func ResolveUnknownPrimaryHostnameResolves() error { return err } -func GetClusterName(instanceKey *InstanceKey) (clusterName string, err error) { - if clusterName, found := instanceKeyInformativeClusterName.Get(instanceKey.StringCode()); found { - return clusterName.(string), nil - } +// GetKeyspaceShardName gets the keyspace shard name for the given instance key +func GetKeyspaceShardName(instanceKey *InstanceKey) (keyspace string, shard string, err error) { query := ` select - ifnull(max(cluster_name), '') as cluster_name + keyspace, + shard from - database_instance + vitess_tablet where hostname = ? and port = ? ` err = db.QueryVTOrc(query, sqlutils.Args(instanceKey.Hostname, instanceKey.Port), func(m sqlutils.RowMap) error { - clusterName = m.GetString("cluster_name") - instanceKeyInformativeClusterName.Set(instanceKey.StringCode(), clusterName, cache.DefaultExpiration) + keyspace = m.GetString("keyspace") + shard = m.GetString("shard") return nil }) if err != nil { log.Error(err) } - return clusterName, err + return keyspace, shard, err } // ReadOutdatedInstanceKeys reads and returns keys for all instances that are not up to date (i.e. diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index 5d2ba5a73ce..d019b0d8061 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -8,6 +8,9 @@ import ( "testing" "github.com/stretchr/testify/require" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vtorc/db" ) var ( @@ -115,3 +118,40 @@ func fmtArgs(args []any) string { } return b.String() } + +func TestGetKeyspaceShardName(t *testing.T) { + orcDb, err := db.OpenVTOrc() + require.NoError(t, err) + defer func() { + _, err = orcDb.Exec("delete from vitess_tablet") + require.NoError(t, err) + }() + + ks := "ks" + shard := "0" + hostname := "localhost" + var port int32 = 100 + tab100 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone-1", + Uid: 100, + }, + Hostname: hostname, + Keyspace: ks, + Shard: shard, + Type: topodatapb.TabletType_PRIMARY, + MysqlHostname: hostname, + MysqlPort: port, + } + + err = SaveTablet(tab100) + require.NoError(t, err) + + keyspaceRead, shardRead, err := GetKeyspaceShardName(&InstanceKey{ + Hostname: hostname, + Port: int(port), + }) + require.NoError(t, err) + require.Equal(t, ks, keyspaceRead) + require.Equal(t, shard, shardRead) +} From e8eb8b0615120dd9a8151702911f27c8b917d472 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 11:41:27 +0530 Subject: [PATCH 10/25] feat: fix insertion into database_instance Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/instance_dao.go | 2 -- go/vt/vtorc/inst/instance_dao_test.go | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index e04cf5dde6e..428189402a8 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -1172,7 +1172,6 @@ func mkInsertOdkuForInstances(instances []*Instance, instanceWasActuallyFound bo "replication_lag_seconds", "replica_lag_seconds", "sql_delay", - "cluster_name", "data_center", "region", "physical_environment", @@ -1259,7 +1258,6 @@ func mkInsertOdkuForInstances(instances []*Instance, instanceWasActuallyFound bo args = append(args, instance.SecondsBehindPrimary) args = append(args, instance.ReplicationLagSeconds) args = append(args, instance.SQLDelay) - args = append(args, instance.ClusterName) args = append(args, instance.DataCenter) args = append(args, instance.Region) args = append(args, instance.PhysicalEnvironment) diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index d019b0d8061..71d0ed94ff9 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -63,17 +63,17 @@ func TestMkInsertOdkuSingle(t *testing.T) { version, major_version, version_comment, binlog_server, read_only, binlog_format, binlog_row_image, log_bin, log_replica_updates, binary_log_file, binary_log_pos, source_host, source_port, replica_sql_running, replica_io_running, replication_sql_thread_state, replication_io_thread_state, has_replication_filters, supports_oracle_gtid, oracle_gtid, source_uuid, ancestry_uuid, executed_gtid_set, gtid_mode, gtid_purged, gtid_errant, mariadb_gtid, pseudo_gtid, - source_log_file, read_source_log_pos, relay_source_log_file, exec_source_log_pos, relay_log_file, relay_log_pos, last_sql_error, last_io_error, replication_lag_seconds, replica_lag_seconds, sql_delay, cluster_name, data_center, region, physical_environment, replication_depth, is_co_primary, has_replication_credentials, allow_tls, semi_sync_enforced, semi_sync_primary_enabled, semi_sync_primary_timeout, semi_sync_primary_wait_for_replica_count, semi_sync_replica_enabled, semi_sync_primary_status, semi_sync_primary_clients, semi_sync_replica_status, instance_alias, last_discovery_latency, replication_group_name, replication_group_is_single_primary_mode, replication_group_member_state, replication_group_member_role, replication_group_members, replication_group_primary_host, replication_group_primary_port, last_seen) + source_log_file, read_source_log_pos, relay_source_log_file, exec_source_log_pos, relay_log_file, relay_log_pos, last_sql_error, last_io_error, replication_lag_seconds, replica_lag_seconds, sql_delay, data_center, region, physical_environment, replication_depth, is_co_primary, has_replication_credentials, allow_tls, semi_sync_enforced, semi_sync_primary_enabled, semi_sync_primary_timeout, semi_sync_primary_wait_for_replica_count, semi_sync_replica_enabled, semi_sync_primary_status, semi_sync_primary_clients, semi_sync_replica_status, instance_alias, last_discovery_latency, replication_group_name, replication_group_is_single_primary_mode, replication_group_member_state, replication_group_member_role, replication_group_members, replication_group_primary_host, replication_group_primary_port, last_seen) VALUES - (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()) + (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()) ON DUPLICATE KEY UPDATE - hostname=VALUES(hostname), port=VALUES(port), last_checked=VALUES(last_checked), last_attempted_check=VALUES(last_attempted_check), last_check_partial_success=VALUES(last_check_partial_success), server_id=VALUES(server_id), server_uuid=VALUES(server_uuid), version=VALUES(version), major_version=VALUES(major_version), version_comment=VALUES(version_comment), binlog_server=VALUES(binlog_server), read_only=VALUES(read_only), binlog_format=VALUES(binlog_format), binlog_row_image=VALUES(binlog_row_image), log_bin=VALUES(log_bin), log_replica_updates=VALUES(log_replica_updates), binary_log_file=VALUES(binary_log_file), binary_log_pos=VALUES(binary_log_pos), source_host=VALUES(source_host), source_port=VALUES(source_port), replica_sql_running=VALUES(replica_sql_running), replica_io_running=VALUES(replica_io_running), replication_sql_thread_state=VALUES(replication_sql_thread_state), replication_io_thread_state=VALUES(replication_io_thread_state), has_replication_filters=VALUES(has_replication_filters), supports_oracle_gtid=VALUES(supports_oracle_gtid), oracle_gtid=VALUES(oracle_gtid), source_uuid=VALUES(source_uuid), ancestry_uuid=VALUES(ancestry_uuid), executed_gtid_set=VALUES(executed_gtid_set), gtid_mode=VALUES(gtid_mode), gtid_purged=VALUES(gtid_purged), gtid_errant=VALUES(gtid_errant), mariadb_gtid=VALUES(mariadb_gtid), pseudo_gtid=VALUES(pseudo_gtid), source_log_file=VALUES(source_log_file), read_source_log_pos=VALUES(read_source_log_pos), relay_source_log_file=VALUES(relay_source_log_file), exec_source_log_pos=VALUES(exec_source_log_pos), relay_log_file=VALUES(relay_log_file), relay_log_pos=VALUES(relay_log_pos), last_sql_error=VALUES(last_sql_error), last_io_error=VALUES(last_io_error), replication_lag_seconds=VALUES(replication_lag_seconds), replica_lag_seconds=VALUES(replica_lag_seconds), sql_delay=VALUES(sql_delay), cluster_name=VALUES(cluster_name), data_center=VALUES(data_center), region=VALUES(region), physical_environment=VALUES(physical_environment), replication_depth=VALUES(replication_depth), is_co_primary=VALUES(is_co_primary), has_replication_credentials=VALUES(has_replication_credentials), allow_tls=VALUES(allow_tls), + hostname=VALUES(hostname), port=VALUES(port), last_checked=VALUES(last_checked), last_attempted_check=VALUES(last_attempted_check), last_check_partial_success=VALUES(last_check_partial_success), server_id=VALUES(server_id), server_uuid=VALUES(server_uuid), version=VALUES(version), major_version=VALUES(major_version), version_comment=VALUES(version_comment), binlog_server=VALUES(binlog_server), read_only=VALUES(read_only), binlog_format=VALUES(binlog_format), binlog_row_image=VALUES(binlog_row_image), log_bin=VALUES(log_bin), log_replica_updates=VALUES(log_replica_updates), binary_log_file=VALUES(binary_log_file), binary_log_pos=VALUES(binary_log_pos), source_host=VALUES(source_host), source_port=VALUES(source_port), replica_sql_running=VALUES(replica_sql_running), replica_io_running=VALUES(replica_io_running), replication_sql_thread_state=VALUES(replication_sql_thread_state), replication_io_thread_state=VALUES(replication_io_thread_state), has_replication_filters=VALUES(has_replication_filters), supports_oracle_gtid=VALUES(supports_oracle_gtid), oracle_gtid=VALUES(oracle_gtid), source_uuid=VALUES(source_uuid), ancestry_uuid=VALUES(ancestry_uuid), executed_gtid_set=VALUES(executed_gtid_set), gtid_mode=VALUES(gtid_mode), gtid_purged=VALUES(gtid_purged), gtid_errant=VALUES(gtid_errant), mariadb_gtid=VALUES(mariadb_gtid), pseudo_gtid=VALUES(pseudo_gtid), source_log_file=VALUES(source_log_file), read_source_log_pos=VALUES(read_source_log_pos), relay_source_log_file=VALUES(relay_source_log_file), exec_source_log_pos=VALUES(exec_source_log_pos), relay_log_file=VALUES(relay_log_file), relay_log_pos=VALUES(relay_log_pos), last_sql_error=VALUES(last_sql_error), last_io_error=VALUES(last_io_error), replication_lag_seconds=VALUES(replication_lag_seconds), replica_lag_seconds=VALUES(replica_lag_seconds), sql_delay=VALUES(sql_delay), data_center=VALUES(data_center), region=VALUES(region), physical_environment=VALUES(physical_environment), replication_depth=VALUES(replication_depth), is_co_primary=VALUES(is_co_primary), has_replication_credentials=VALUES(has_replication_credentials), allow_tls=VALUES(allow_tls), semi_sync_enforced=VALUES(semi_sync_enforced), semi_sync_primary_enabled=VALUES(semi_sync_primary_enabled), semi_sync_primary_timeout=VALUES(semi_sync_primary_timeout), semi_sync_primary_wait_for_replica_count=VALUES(semi_sync_primary_wait_for_replica_count), semi_sync_replica_enabled=VALUES(semi_sync_replica_enabled), semi_sync_primary_status=VALUES(semi_sync_primary_status), semi_sync_primary_clients=VALUES(semi_sync_primary_clients), semi_sync_replica_status=VALUES(semi_sync_replica_status), instance_alias=VALUES(instance_alias), last_discovery_latency=VALUES(last_discovery_latency), replication_group_name=VALUES(replication_group_name), replication_group_is_single_primary_mode=VALUES(replication_group_is_single_primary_mode), replication_group_member_state=VALUES(replication_group_member_state), replication_group_member_role=VALUES(replication_group_member_role), replication_group_members=VALUES(replication_group_members), replication_group_primary_host=VALUES(replication_group_primary_host), replication_group_primary_port=VALUES(replication_group_primary_port), last_seen=VALUES(last_seen) ` a1 := `i710, 3306, 710, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, - false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 10, , 0, , , {0 false}, {0 false}, 0, , , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, ` + false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 10, , 0, , , {0 false}, {0 false}, 0, , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, ` sql1, args1, err := mkInsertOdkuForInstances(instances[:1], false, true) require.NoError(t, err) @@ -86,22 +86,22 @@ func TestMkInsertOdkuThree(t *testing.T) { // three instances s3 := `INSERT INTO database_instance - (hostname, port, last_checked, last_attempted_check, last_check_partial_success, server_id, server_uuid, version, major_version, version_comment, binlog_server, read_only, binlog_format, binlog_row_image, log_bin, log_replica_updates, binary_log_file, binary_log_pos, source_host, source_port, replica_sql_running, replica_io_running, replication_sql_thread_state, replication_io_thread_state, has_replication_filters, supports_oracle_gtid, oracle_gtid, source_uuid, ancestry_uuid, executed_gtid_set, gtid_mode, gtid_purged, gtid_errant, mariadb_gtid, pseudo_gtid, source_log_file, read_source_log_pos, relay_source_log_file, exec_source_log_pos, relay_log_file, relay_log_pos, last_sql_error, last_io_error, replication_lag_seconds, replica_lag_seconds, sql_delay, cluster_name, data_center, region, physical_environment, replication_depth, is_co_primary, has_replication_credentials, allow_tls, semi_sync_enforced, semi_sync_primary_enabled, semi_sync_primary_timeout, semi_sync_primary_wait_for_replica_count, + (hostname, port, last_checked, last_attempted_check, last_check_partial_success, server_id, server_uuid, version, major_version, version_comment, binlog_server, read_only, binlog_format, binlog_row_image, log_bin, log_replica_updates, binary_log_file, binary_log_pos, source_host, source_port, replica_sql_running, replica_io_running, replication_sql_thread_state, replication_io_thread_state, has_replication_filters, supports_oracle_gtid, oracle_gtid, source_uuid, ancestry_uuid, executed_gtid_set, gtid_mode, gtid_purged, gtid_errant, mariadb_gtid, pseudo_gtid, source_log_file, read_source_log_pos, relay_source_log_file, exec_source_log_pos, relay_log_file, relay_log_pos, last_sql_error, last_io_error, replication_lag_seconds, replica_lag_seconds, sql_delay, data_center, region, physical_environment, replication_depth, is_co_primary, has_replication_credentials, allow_tls, semi_sync_enforced, semi_sync_primary_enabled, semi_sync_primary_timeout, semi_sync_primary_wait_for_replica_count, semi_sync_replica_enabled, semi_sync_primary_status, semi_sync_primary_clients, semi_sync_replica_status, instance_alias, last_discovery_latency, replication_group_name, replication_group_is_single_primary_mode, replication_group_member_state, replication_group_member_role, replication_group_members, replication_group_primary_host, replication_group_primary_port, last_seen) VALUES - (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()), - (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()), - (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()) + (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()), + (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()), + (?, ?, NOW(), NOW(), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NOW()) ON DUPLICATE KEY UPDATE - hostname=VALUES(hostname), port=VALUES(port), last_checked=VALUES(last_checked), last_attempted_check=VALUES(last_attempted_check), last_check_partial_success=VALUES(last_check_partial_success), server_id=VALUES(server_id), server_uuid=VALUES(server_uuid), version=VALUES(version), major_version=VALUES(major_version), version_comment=VALUES(version_comment), binlog_server=VALUES(binlog_server), read_only=VALUES(read_only), binlog_format=VALUES(binlog_format), binlog_row_image=VALUES(binlog_row_image), log_bin=VALUES(log_bin), log_replica_updates=VALUES(log_replica_updates), binary_log_file=VALUES(binary_log_file), binary_log_pos=VALUES(binary_log_pos), source_host=VALUES(source_host), source_port=VALUES(source_port), replica_sql_running=VALUES(replica_sql_running), replica_io_running=VALUES(replica_io_running), replication_sql_thread_state=VALUES(replication_sql_thread_state), replication_io_thread_state=VALUES(replication_io_thread_state), has_replication_filters=VALUES(has_replication_filters), supports_oracle_gtid=VALUES(supports_oracle_gtid), oracle_gtid=VALUES(oracle_gtid), source_uuid=VALUES(source_uuid), ancestry_uuid=VALUES(ancestry_uuid), executed_gtid_set=VALUES(executed_gtid_set), gtid_mode=VALUES(gtid_mode), gtid_purged=VALUES(gtid_purged), gtid_errant=VALUES(gtid_errant), mariadb_gtid=VALUES(mariadb_gtid), pseudo_gtid=VALUES(pseudo_gtid), source_log_file=VALUES(source_log_file), read_source_log_pos=VALUES(read_source_log_pos), relay_source_log_file=VALUES(relay_source_log_file), exec_source_log_pos=VALUES(exec_source_log_pos), relay_log_file=VALUES(relay_log_file), relay_log_pos=VALUES(relay_log_pos), last_sql_error=VALUES(last_sql_error), last_io_error=VALUES(last_io_error), replication_lag_seconds=VALUES(replication_lag_seconds), replica_lag_seconds=VALUES(replica_lag_seconds), sql_delay=VALUES(sql_delay), cluster_name=VALUES(cluster_name), data_center=VALUES(data_center), region=VALUES(region), + hostname=VALUES(hostname), port=VALUES(port), last_checked=VALUES(last_checked), last_attempted_check=VALUES(last_attempted_check), last_check_partial_success=VALUES(last_check_partial_success), server_id=VALUES(server_id), server_uuid=VALUES(server_uuid), version=VALUES(version), major_version=VALUES(major_version), version_comment=VALUES(version_comment), binlog_server=VALUES(binlog_server), read_only=VALUES(read_only), binlog_format=VALUES(binlog_format), binlog_row_image=VALUES(binlog_row_image), log_bin=VALUES(log_bin), log_replica_updates=VALUES(log_replica_updates), binary_log_file=VALUES(binary_log_file), binary_log_pos=VALUES(binary_log_pos), source_host=VALUES(source_host), source_port=VALUES(source_port), replica_sql_running=VALUES(replica_sql_running), replica_io_running=VALUES(replica_io_running), replication_sql_thread_state=VALUES(replication_sql_thread_state), replication_io_thread_state=VALUES(replication_io_thread_state), has_replication_filters=VALUES(has_replication_filters), supports_oracle_gtid=VALUES(supports_oracle_gtid), oracle_gtid=VALUES(oracle_gtid), source_uuid=VALUES(source_uuid), ancestry_uuid=VALUES(ancestry_uuid), executed_gtid_set=VALUES(executed_gtid_set), gtid_mode=VALUES(gtid_mode), gtid_purged=VALUES(gtid_purged), gtid_errant=VALUES(gtid_errant), mariadb_gtid=VALUES(mariadb_gtid), pseudo_gtid=VALUES(pseudo_gtid), source_log_file=VALUES(source_log_file), read_source_log_pos=VALUES(read_source_log_pos), relay_source_log_file=VALUES(relay_source_log_file), exec_source_log_pos=VALUES(exec_source_log_pos), relay_log_file=VALUES(relay_log_file), relay_log_pos=VALUES(relay_log_pos), last_sql_error=VALUES(last_sql_error), last_io_error=VALUES(last_io_error), replication_lag_seconds=VALUES(replication_lag_seconds), replica_lag_seconds=VALUES(replica_lag_seconds), sql_delay=VALUES(sql_delay), data_center=VALUES(data_center), region=VALUES(region), physical_environment=VALUES(physical_environment), replication_depth=VALUES(replication_depth), is_co_primary=VALUES(is_co_primary), has_replication_credentials=VALUES(has_replication_credentials), allow_tls=VALUES(allow_tls), semi_sync_enforced=VALUES(semi_sync_enforced), semi_sync_primary_enabled=VALUES(semi_sync_primary_enabled), semi_sync_primary_timeout=VALUES(semi_sync_primary_timeout), semi_sync_primary_wait_for_replica_count=VALUES(semi_sync_primary_wait_for_replica_count), semi_sync_replica_enabled=VALUES(semi_sync_replica_enabled), semi_sync_primary_status=VALUES(semi_sync_primary_status), semi_sync_primary_clients=VALUES(semi_sync_primary_clients), semi_sync_replica_status=VALUES(semi_sync_replica_status), instance_alias=VALUES(instance_alias), last_discovery_latency=VALUES(last_discovery_latency), replication_group_name=VALUES(replication_group_name), replication_group_is_single_primary_mode=VALUES(replication_group_is_single_primary_mode), replication_group_member_state=VALUES(replication_group_member_state), replication_group_member_role=VALUES(replication_group_member_role), replication_group_members=VALUES(replication_group_members), replication_group_primary_host=VALUES(replication_group_primary_host), replication_group_primary_port=VALUES(replication_group_primary_port), last_seen=VALUES(last_seen) ` a3 := ` - i710, 3306, 710, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 10, , 0, , , {0 false}, {0 false}, 0, , , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, - i720, 3306, 720, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 20, , 0, , , {0 false}, {0 false}, 0, , , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, - i730, 3306, 730, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 30, , 0, , , {0 false}, {0 false}, 0, , , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, + i710, 3306, 710, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 10, , 0, , , {0 false}, {0 false}, 0, , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, + i720, 3306, 720, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 20, , 0, , , {0 false}, {0 false}, 0, , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, + i730, 3306, 730, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 30, , 0, , , {0 false}, {0 false}, 0, , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, , 0, , false, , , [], , 0, ` sql3, args3, err := mkInsertOdkuForInstances(instances[:3], true, true) From 27d18b0345a484a9e020f69a2ff79e0621c9b34b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 12:32:38 +0530 Subject: [PATCH 11/25] feat: fix SnapshotTopologies Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/instance_dao.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 428189402a8..8e762228216 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -1447,10 +1447,10 @@ func SnapshotTopologies() error { _, err := db.ExecVTOrc(` insert ignore into database_instance_topology_history (snapshot_unix_timestamp, - hostname, port, source_host, source_port, cluster_name, version) + hostname, port, source_host, source_port, version) select UNIX_TIMESTAMP(NOW()), - hostname, port, source_host, source_port, cluster_name, version + hostname, port, source_host, source_port, version from database_instance `, From c187f46b7d4c75e0d4a37863bb560050eeaed983 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 12:44:14 +0530 Subject: [PATCH 12/25] feat: remove inject unseen primary and inject seed Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/instance_dao.go | 40 --------------------------- go/vt/vtorc/logic/orchestrator.go | 1 - go/vt/vtorc/logic/tablet_discovery.go | 3 -- 3 files changed, 44 deletions(-) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 8e762228216..335448333e2 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -859,46 +859,6 @@ func readUnseenPrimaryKeys() ([]InstanceKey, error) { return res, nil } -// InjectSeed: intented to be used to inject an instance upon startup, assuming it's not already known to vtorc. -func InjectSeed(instanceKey *InstanceKey) error { - if instanceKey == nil { - return fmt.Errorf("InjectSeed: nil instanceKey") - } - clusterName := instanceKey.StringCode() - // minimal details: - instance := &Instance{Key: *instanceKey, Version: "Unknown", ClusterName: clusterName} - instance.SetSeed() - err := WriteInstance(instance, false, nil) - log.Infof("InjectSeed: %+v, %+v", *instanceKey, err) - _ = AuditOperation("inject-seed", instanceKey, "injected") - return err -} - -// InjectUnseenPrimaries will review primaries of instances that are known to be replicating, yet which are not listed -// in database_instance. Since their replicas are listed as replicating, we can assume that such primaries actually do -// exist: we shall therefore inject them with minimal details into the database_instance table. -func InjectUnseenPrimaries() error { - - unseenPrimaryKeys, err := readUnseenPrimaryKeys() - if err != nil { - return err - } - - operations := 0 - for _, primaryKey := range unseenPrimaryKeys { - primaryKey := primaryKey - clusterName := primaryKey.StringCode() - // minimal details: - instance := Instance{Key: primaryKey, Version: "Unknown", ClusterName: clusterName} - if err := WriteInstance(&instance, false, nil); err == nil { - operations++ - } - } - - _ = AuditOperation("inject-unseen-primaries", nil, fmt.Sprintf("Operations: %d", operations)) - return err -} - // ForgetUnseenInstancesDifferentlyResolved will purge instances which are invalid, and whose hostname // appears on the hostname_resolved table; this means some time in the past their hostname was unresovled, and now // resovled to a different value; the old hostname is never accessed anymore and the old entry should be removed. diff --git a/go/vt/vtorc/logic/orchestrator.go b/go/vt/vtorc/logic/orchestrator.go index 11b92f6111f..dcc30027392 100644 --- a/go/vt/vtorc/logic/orchestrator.go +++ b/go/vt/vtorc/logic/orchestrator.go @@ -385,7 +385,6 @@ func ContinuousDiscovery() { // Various periodic internal maintenance tasks go func() { if IsLeaderOrActive() { - go inst.InjectUnseenPrimaries() go inst.ForgetLongUnseenInstances() go inst.ForgetUnseenInstancesDifferentlyResolved() diff --git a/go/vt/vtorc/logic/tablet_discovery.go b/go/vt/vtorc/logic/tablet_discovery.go index 6716d5a727b..a16394d5a9c 100644 --- a/go/vt/vtorc/logic/tablet_discovery.go +++ b/go/vt/vtorc/logic/tablet_discovery.go @@ -72,9 +72,6 @@ func OpenTabletDiscovery() <-chan time.Time { if _, err := db.ExecVTOrc("delete from vitess_tablet"); err != nil { log.Error(err) } - refreshTabletsUsing(func(instanceKey *inst.InstanceKey) { - _ = inst.InjectSeed(instanceKey) - }, false /* forceRefresh */) return time.Tick(time.Second * time.Duration(config.Config.TopoInformationRefreshSeconds)) //nolint SA1015: using time.Tick leaks the underlying ticker } From 396a2c48913b45bcf14a194b1449152031e027fe Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 12:47:56 +0530 Subject: [PATCH 13/25] feat: remove ClusterName from Instance Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/readtopologyinstance/main_test.go | 2 -- go/vt/vtorc/inst/instance.go | 1 - go/vt/vtorc/inst/instance_dao.go | 2 -- 3 files changed, 5 deletions(-) diff --git a/go/test/endtoend/vtorc/readtopologyinstance/main_test.go b/go/test/endtoend/vtorc/readtopologyinstance/main_test.go index c6426021d6c..5d8c129dbbd 100644 --- a/go/test/endtoend/vtorc/readtopologyinstance/main_test.go +++ b/go/test/endtoend/vtorc/readtopologyinstance/main_test.go @@ -103,7 +103,6 @@ func TestReadTopologyInstanceBufferable(t *testing.T) { assert.False(t, primaryInstance.HasReplicationCredentials) assert.Equal(t, primaryInstance.ReplicationIOThreadState, inst.ReplicationThreadStateNoThread) assert.Equal(t, primaryInstance.ReplicationSQLThreadState, inst.ReplicationThreadStateNoThread) - assert.Equal(t, fmt.Sprintf("%v:%v", keyspace.Name, shard0.Name), primaryInstance.ClusterName) // insert an errant GTID in the replica _, err = utils.RunSQL(t, "insert into vt_insert_test(id, msg) values (10173, 'test 178342')", replica, "vt_ks") @@ -159,5 +158,4 @@ func TestReadTopologyInstanceBufferable(t *testing.T) { assert.False(t, replicaInstance.HasReplicationFilters) assert.LessOrEqual(t, int(replicaInstance.SecondsBehindPrimary.Int64), 1) assert.False(t, replicaInstance.AllowTLS) - assert.Equal(t, fmt.Sprintf("%v:%v", keyspace.Name, shard0.Name), replicaInstance.ClusterName) } diff --git a/go/vt/vtorc/inst/instance.go b/go/vt/vtorc/inst/instance.go index d4b8c7bfe88..dd1526ff090 100644 --- a/go/vt/vtorc/inst/instance.go +++ b/go/vt/vtorc/inst/instance.go @@ -74,7 +74,6 @@ type Instance struct { primaryExecutedGtidSet string // Not exported ReplicationLagSeconds sql.NullInt64 - ClusterName string DataCenter string Region string PhysicalEnvironment string diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 335448333e2..f9b2f7c84d1 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -219,7 +219,6 @@ func ReadTopologyInstanceBufferable(instanceKey *InstanceKey, latency *stopwatch if err != nil { goto Cleanup } - instance.ClusterName = GetClusterNameFromKeyspaceAndShard(tablet.Keyspace, tablet.Shard) fullStatus, err = FullStatus(*instanceKey) if err != nil { @@ -605,7 +604,6 @@ func readInstanceRow(m sqlutils.RowMap) *Instance { instance.SecondsBehindPrimary = m.GetNullInt64("replication_lag_seconds") instance.ReplicationLagSeconds = m.GetNullInt64("replica_lag_seconds") instance.SQLDelay = m.GetUint("sql_delay") - instance.ClusterName = m.GetString("cluster_name") instance.DataCenter = m.GetString("data_center") instance.Region = m.GetString("region") instance.PhysicalEnvironment = m.GetString("physical_environment") From ec8a64f7c7e57c2e02b45b5cacc383f7755454b5 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 14:35:48 +0530 Subject: [PATCH 14/25] feat: fix Audit operations Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/audit_dao.go | 17 +++-- go/vt/vtorc/inst/audit_dao_test.go | 115 +++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 go/vt/vtorc/inst/audit_dao_test.go diff --git a/go/vt/vtorc/inst/audit_dao.go b/go/vt/vtorc/inst/audit_dao.go index 44bdfc4a7f9..0b208c6a3ba 100644 --- a/go/vt/vtorc/inst/audit_dao.go +++ b/go/vt/vtorc/inst/audit_dao.go @@ -55,10 +55,10 @@ func AuditOperation(auditType string, instanceKey *InstanceKey, message string) if instanceKey == nil { instanceKey = &InstanceKey{} } - clusterName := "" + keyspace := "" + shard := "" if instanceKey.Hostname != "" { - keyspace, shard, _ := GetKeyspaceShardName(instanceKey) - clusterName = GetClusterNameFromKeyspaceAndShard(keyspace, shard) + keyspace, shard, _ = GetKeyspaceShardName(instanceKey) } auditWrittenToFile := false @@ -72,7 +72,7 @@ func AuditOperation(auditType string, instanceKey *InstanceKey, message string) } defer f.Close() - text := fmt.Sprintf("%s\t%s\t%s\t%d\t[%s]\t%s\t\n", time.Now().Format("2006-01-02 15:04:05"), auditType, instanceKey.Hostname, instanceKey.Port, clusterName, message) + text := fmt.Sprintf("%s\t%s\t%s\t%d\t[%s:%s]\t%s\t\n", time.Now().Format("2006-01-02 15:04:05"), auditType, instanceKey.Hostname, instanceKey.Port, keyspace, shard, message) if _, err = f.WriteString(text); err != nil { log.Error(err) } @@ -82,15 +82,16 @@ func AuditOperation(auditType string, instanceKey *InstanceKey, message string) _, err := db.ExecVTOrc(` insert into audit ( - audit_timestamp, audit_type, hostname, port, cluster_name, message + audit_timestamp, audit_type, hostname, port, keyspace, shard, message ) VALUES ( - NOW(), ?, ?, ?, ?, ? + NOW(), ?, ?, ?, ?, ?, ? ) `, auditType, instanceKey.Hostname, instanceKey.Port, - clusterName, + keyspace, + shard, message, ) if err != nil { @@ -98,7 +99,7 @@ func AuditOperation(auditType string, instanceKey *InstanceKey, message string) return err } } - logMessage := fmt.Sprintf("auditType:%s instance:%s cluster:%s message:%s", auditType, instanceKey.DisplayString(), clusterName, message) + logMessage := fmt.Sprintf("auditType:%s instance:%s keyspace:%s shard:%s message:%s", auditType, instanceKey.DisplayString(), keyspace, shard, message) if syslogWriter != nil { auditWrittenToFile = true go func() { diff --git a/go/vt/vtorc/inst/audit_dao_test.go b/go/vt/vtorc/inst/audit_dao_test.go new file mode 100644 index 00000000000..c1c28fac2fe --- /dev/null +++ b/go/vt/vtorc/inst/audit_dao_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inst + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vtorc/config" + "vitess.io/vitess/go/vt/vtorc/db" +) + +// TestAuditOperation tests that auditing a operation works as intended based on the configurations. +func TestAuditOperation(t *testing.T) { + // Restore original configurations + originalAuditSysLog := config.Config.AuditToSyslog + originalAuditLogFile := config.Config.AuditLogFile + originalAuditBackend := config.Config.AuditToBackendDB + defer func() { + config.Config.AuditToSyslog = originalAuditSysLog + config.Config.AuditLogFile = originalAuditLogFile + config.Config.AuditToBackendDB = originalAuditBackend + }() + + orcDb, err := db.OpenVTOrc() + require.NoError(t, err) + defer func() { + _, err = orcDb.Exec("delete from audit") + require.NoError(t, err) + _, err = orcDb.Exec("delete from vitess_tablet") + require.NoError(t, err) + }() + + // Store a tablet in the database + ks := "ks" + shard := "0" + hostname := "localhost" + var port int32 = 100 + tab100 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone-1", + Uid: 100, + }, + Hostname: hostname, + Keyspace: ks, + Shard: shard, + Type: topodatapb.TabletType_PRIMARY, + MysqlHostname: hostname, + MysqlPort: port, + } + err = SaveTablet(tab100) + require.NoError(t, err) + + instance100 := &InstanceKey{ + Hostname: hostname, + Port: int(port), + } + auditType := "test-audit-operation" + message := "test-message" + + t.Run("Audit to backend", func(t *testing.T) { + config.Config.AuditLogFile = "" + config.Config.AuditToSyslog = false + config.Config.AuditToBackendDB = true + + // Auditing should succeed as expected + err = AuditOperation(auditType, instance100, message) + require.NoError(t, err) + + audits, err := ReadRecentAudit(instance100, 0) + require.NoError(t, err) + require.Len(t, audits, 1) + require.EqualValues(t, 1, audits[0].AuditID) + require.EqualValues(t, auditType, audits[0].AuditType) + require.EqualValues(t, message, audits[0].Message) + require.EqualValues(t, *instance100, audits[0].AuditInstanceKey) + }) + + t.Run("Audit to File", func(t *testing.T) { + config.Config.AuditToBackendDB = false + config.Config.AuditToSyslog = false + + file, err := os.CreateTemp("", "test-auditing-*") + require.NoError(t, err) + defer os.Remove(file.Name()) + config.Config.AuditLogFile = file.Name() + + err = AuditOperation(auditType, instance100, message) + require.NoError(t, err) + + // Give a little time for the write to succeed and be emptied from the write buffer. + time.Sleep(100 * time.Millisecond) + fileContent, err := os.ReadFile(file.Name()) + require.NoError(t, err) + require.Contains(t, string(fileContent), "\ttest-audit-operation\tlocalhost\t100\t[ks:0]\ttest-message") + }) +} From c2d20bbfe640b352ffdadbb0832d3a9c58e22c0e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 15:01:16 +0530 Subject: [PATCH 15/25] feat: add Keyspace and Shard to cluster information to replace ClusterName Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/analysis_dao.go | 19 +++++++++++-------- go/vt/vtorc/inst/cluster.go | 2 ++ go/vt/vtorc/logic/topology_recovery.go | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index 0199d574359..f0d3427dd6b 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -383,7 +383,9 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) Type: BinaryLog, } isStaleBinlogCoordinates := m.GetBool("is_stale_binlog_coordinates") - a.ClusterDetails.ClusterName = m.GetString("cluster_name") + a.ClusterDetails.Keyspace = m.GetString("keyspace") + a.ClusterDetails.Shard = m.GetString("shard") + a.ClusterDetails.ClusterName = GetClusterNameFromKeyspaceAndShard(a.ClusterDetails.Keyspace, a.ClusterDetails.Shard) a.GTIDMode = m.GetString("gtid_mode") a.LastCheckValid = m.GetBool("is_last_check_valid") a.LastCheckPartialSuccess = m.GetBool("last_check_partial_success") @@ -431,18 +433,19 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) a.IsReadOnly = m.GetUint("read_only") == 1 if !a.LastCheckValid { - analysisMessage := fmt.Sprintf("analysis: ClusterName: %+v, IsPrimary: %+v, LastCheckValid: %+v, LastCheckPartialSuccess: %+v, CountReplicas: %+v, CountValidReplicas: %+v, CountValidReplicatingReplicas: %+v, CountLaggingReplicas: %+v, CountDelayedReplicas: %+v, CountReplicasFailingToConnectToPrimary: %+v", - a.ClusterDetails.ClusterName, a.IsPrimary, a.LastCheckValid, a.LastCheckPartialSuccess, a.CountReplicas, a.CountValidReplicas, a.CountValidReplicatingReplicas, a.CountLaggingReplicas, a.CountDelayedReplicas, a.CountReplicasFailingToConnectToPrimary, + analysisMessage := fmt.Sprintf("analysis: Keyspace: %+v, Shard: %+v, IsPrimary: %+v, LastCheckValid: %+v, LastCheckPartialSuccess: %+v, CountReplicas: %+v, CountValidReplicas: %+v, CountValidReplicatingReplicas: %+v, CountLaggingReplicas: %+v, CountDelayedReplicas: %+v, CountReplicasFailingToConnectToPrimary: %+v", + a.ClusterDetails.Keyspace, a.ClusterDetails.Shard, a.IsPrimary, a.LastCheckValid, a.LastCheckPartialSuccess, a.CountReplicas, a.CountValidReplicas, a.CountValidReplicatingReplicas, a.CountLaggingReplicas, a.CountDelayedReplicas, a.CountReplicasFailingToConnectToPrimary, ) if util.ClearToLog("analysis_dao", analysisMessage) { log.Infof(analysisMessage) } } - if clusters[a.ClusterDetails.ClusterName] == nil { - clusters[a.ClusterDetails.ClusterName] = &clusterAnalysis{} + keyspaceShard := a.ClusterDetails.ClusterName + if clusters[keyspaceShard] == nil { + clusters[keyspaceShard] = &clusterAnalysis{} if a.TabletType == topodatapb.TabletType_PRIMARY { a.IsClusterPrimary = true - clusters[a.ClusterDetails.ClusterName].primaryKey = &a.AnalyzedInstanceKey + clusters[keyspaceShard].primaryKey = &a.AnalyzedInstanceKey } durabilityPolicy := m.GetString("durability_policy") if durabilityPolicy == "" { @@ -454,10 +457,10 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) log.Errorf("can't get the durability policy %v - %v. Skipping keyspace - %v.", durabilityPolicy, err, a.AnalyzedKeyspace) return nil } - clusters[a.ClusterDetails.ClusterName].durability = durability + clusters[keyspaceShard].durability = durability } // ca has clusterwide info - ca := clusters[a.ClusterDetails.ClusterName] + ca := clusters[keyspaceShard] if ca.hasClusterwideAction { // We can only take one cluster level action at a time. return nil diff --git a/go/vt/vtorc/inst/cluster.go b/go/vt/vtorc/inst/cluster.go index ba4a569e3e1..aea7b81ed81 100644 --- a/go/vt/vtorc/inst/cluster.go +++ b/go/vt/vtorc/inst/cluster.go @@ -19,6 +19,8 @@ package inst // ClusterInfo makes for a cluster status/info summary type ClusterInfo struct { ClusterName string + Keyspace string + Shard string CountInstances uint HeuristicLag int64 HasAutomatedPrimaryRecovery bool diff --git a/go/vt/vtorc/logic/topology_recovery.go b/go/vt/vtorc/logic/topology_recovery.go index 451fd028599..7c6969270b2 100644 --- a/go/vt/vtorc/logic/topology_recovery.go +++ b/go/vt/vtorc/logic/topology_recovery.go @@ -827,7 +827,7 @@ func electNewPrimary(ctx context.Context, analysisEntry inst.ReplicationAnalysis _ = AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("found an active or recent recovery on %+v. Will not issue another electNewPrimary.", analysisEntry.AnalyzedInstanceKey)) return false, nil, err } - log.Infof("Analysis: %v, will elect a new primary: %v", analysisEntry.Analysis, analysisEntry.ClusterDetails.ClusterName) + log.Infof("Analysis: %v, will elect a new primary for %v:%v", analysisEntry.Analysis, analysisEntry.ClusterDetails.Keyspace, analysisEntry.ClusterDetails.Shard) var promotedReplica *inst.Instance // This has to be done in the end; whether successful or not, we should mark that the recovery is done. From ec18e0d291b74682bb66857bd6d4855f233be536 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 15:12:42 +0530 Subject: [PATCH 16/25] feat: fix attempt failure detection registeration Signed-off-by: Manan Gupta --- go/vt/vtorc/logic/topology_recovery_dao.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/vt/vtorc/logic/topology_recovery_dao.go b/go/vt/vtorc/logic/topology_recovery_dao.go index f1cfa284de8..1fabe858476 100644 --- a/go/vt/vtorc/logic/topology_recovery_dao.go +++ b/go/vt/vtorc/logic/topology_recovery_dao.go @@ -39,7 +39,8 @@ func AttemptFailureDetectionRegistration(analysisEntry *inst.ReplicationAnalysis process.ThisHostname, util.ProcessToken.Hash, string(analysisEntry.Analysis), - analysisEntry.ClusterDetails.ClusterName, + analysisEntry.ClusterDetails.Keyspace, + analysisEntry.ClusterDetails.Shard, analysisEntry.CountReplicas, analysisEntry.IsActionableRecovery, ) @@ -59,7 +60,8 @@ func AttemptFailureDetectionRegistration(analysisEntry *inst.ReplicationAnalysis processing_node_hostname, processcing_node_token, analysis, - cluster_name, + keyspace, + shard, count_affected_replicas, is_actionable, start_active_period @@ -74,6 +76,7 @@ func AttemptFailureDetectionRegistration(analysisEntry *inst.ReplicationAnalysis ?, ?, ?, + ?, %s ) `, startActivePeriodHint) From e7084eb13b1f3cc745e11b4ec7eba775371e2cfd Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 15:31:06 +0530 Subject: [PATCH 17/25] feat: fix blocked topology recoveries Signed-off-by: Manan Gupta --- go/vt/vtorc/logic/topology_recovery_dao.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go/vt/vtorc/logic/topology_recovery_dao.go b/go/vt/vtorc/logic/topology_recovery_dao.go index 1fabe858476..b8859cbc34b 100644 --- a/go/vt/vtorc/logic/topology_recovery_dao.go +++ b/go/vt/vtorc/logic/topology_recovery_dao.go @@ -251,7 +251,8 @@ func RegisterBlockedRecoveries(analysisEntry *inst.ReplicationAnalysis, blocking into blocked_topology_recovery ( hostname, port, - cluster_name, + keyspace, + shard, analysis, last_blocked_timestamp, blocking_recovery_id @@ -264,13 +265,15 @@ func RegisterBlockedRecoveries(analysisEntry *inst.ReplicationAnalysis, blocking ? ) on duplicate key update - cluster_name=values(cluster_name), + keyspace=values(keyspace), + shard=values(shard), analysis=values(analysis), last_blocked_timestamp=values(last_blocked_timestamp), blocking_recovery_id=values(blocking_recovery_id) `, analysisEntry.AnalyzedInstanceKey.Hostname, analysisEntry.AnalyzedInstanceKey.Port, - analysisEntry.ClusterDetails.ClusterName, + analysisEntry.ClusterDetails.Keyspace, + analysisEntry.ClusterDetails.Shard, string(analysisEntry.Analysis), recovery.ID, ) From c4f3807ec1a970171e9421ee60a34411b4a4bc62 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 15:59:13 +0530 Subject: [PATCH 18/25] feat: fix topology recovery Signed-off-by: Manan Gupta --- go/cmd/vtorc/status.go | 2 +- go/vt/vtorc/logic/topology_recovery_dao.go | 28 ++++---- .../vtorc/logic/topology_recovery_dao_test.go | 70 +++++++++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 go/vt/vtorc/logic/topology_recovery_dao_test.go diff --git a/go/cmd/vtorc/status.go b/go/cmd/vtorc/status.go index bdb54963051..a4d8a59d3fc 100644 --- a/go/cmd/vtorc/status.go +++ b/go/cmd/vtorc/status.go @@ -24,7 +24,7 @@ import ( // addStatusParts adds UI parts to the /debug/status page of VTOrc func addStatusParts() { servenv.AddStatusPart("Recent Recoveries", logic.TopologyRecoveriesTemplate, func() any { - recoveries, _ := logic.ReadRecentRecoveries("", false, 0) + recoveries, _ := logic.ReadRecentRecoveries(false, 0) return recoveries }) } diff --git a/go/vt/vtorc/logic/topology_recovery_dao.go b/go/vt/vtorc/logic/topology_recovery_dao.go index b8859cbc34b..f890a5b4e15 100644 --- a/go/vt/vtorc/logic/topology_recovery_dao.go +++ b/go/vt/vtorc/logic/topology_recovery_dao.go @@ -128,7 +128,8 @@ func writeTopologyRecovery(topologyRecovery *TopologyRecovery) (*TopologyRecover processing_node_hostname, processcing_node_token, analysis, - cluster_name, + keyspace, + shard, count_affected_replicas, last_detection_id ) values ( @@ -144,6 +145,7 @@ func writeTopologyRecovery(topologyRecovery *TopologyRecovery) (*TopologyRecover ?, ?, ?, + ?, (select ifnull(max(detection_id), 0) from topology_failure_detection where hostname=? and port=?) ) `, @@ -152,7 +154,8 @@ func writeTopologyRecovery(topologyRecovery *TopologyRecovery) (*TopologyRecover analysisEntry.AnalyzedInstanceKey.Hostname, analysisEntry.AnalyzedInstanceKey.Port, process.ThisHostname, util.ProcessToken.Hash, string(analysisEntry.Analysis), - analysisEntry.ClusterDetails.ClusterName, + analysisEntry.ClusterDetails.Keyspace, + analysisEntry.ClusterDetails.Shard, analysisEntry.CountReplicas, analysisEntry.AnalyzedInstanceKey.Hostname, analysisEntry.AnalyzedInstanceKey.Port, ) @@ -194,7 +197,7 @@ func AttemptRecoveryRegistration(analysisEntry *inst.ReplicationAnalysis, failIf if failIfClusterInActiveRecovery { // Let's check if this cluster has just experienced a failover and is still in active period. // If so, we reject recovery registration to avoid flapping. - recoveries, err := ReadInActivePeriodClusterRecovery(analysisEntry.ClusterDetails.ClusterName) + recoveries, err := ReadInActivePeriodClusterRecovery(analysisEntry.ClusterDetails.Keyspace, analysisEntry.ClusterDetails.Shard) if err != nil { log.Error(err) return nil, err @@ -452,7 +455,8 @@ func readRecoveries(whereCondition string, limit string, args []any) ([]*Topolog ifnull(successor_port, 0) as successor_port, ifnull(successor_alias, '') as successor_alias, analysis, - cluster_name, + keyspace, + shard, count_affected_replicas, participating_instances, lost_replicas, @@ -484,7 +488,8 @@ func readRecoveries(whereCondition string, limit string, args []any) ([]*Topolog topologyRecovery.AnalysisEntry.AnalyzedInstanceKey.Hostname = m.GetString("hostname") topologyRecovery.AnalysisEntry.AnalyzedInstanceKey.Port = m.GetInt("port") topologyRecovery.AnalysisEntry.Analysis = inst.AnalysisCode(m.GetString("analysis")) - topologyRecovery.AnalysisEntry.ClusterDetails.ClusterName = m.GetString("cluster_name") + topologyRecovery.AnalysisEntry.ClusterDetails.Keyspace = m.GetString("keyspace") + topologyRecovery.AnalysisEntry.ClusterDetails.Shard = m.GetString("shard") topologyRecovery.AnalysisEntry.CountReplicas = m.GetUint("count_affected_replicas") topologyRecovery.SuccessorKey = &inst.InstanceKey{} @@ -517,12 +522,13 @@ func readRecoveries(whereCondition string, limit string, args []any) ([]*Topolog // ReadInActivePeriodClusterRecovery reads recoveries (possibly complete!) that are in active period. // (may be used to block further recoveries on this cluster) -func ReadInActivePeriodClusterRecovery(clusterName string) ([]*TopologyRecovery, error) { +func ReadInActivePeriodClusterRecovery(keyspace string, shard string) ([]*TopologyRecovery, error) { whereClause := ` where in_active_period=1 - and cluster_name=?` - return readRecoveries(whereClause, ``, sqlutils.Args(clusterName)) + and keyspace=? + and shard=?` + return readRecoveries(whereClause, ``, sqlutils.Args(keyspace, shard)) } // ReadInActivePeriodSuccessorInstanceRecovery reads completed recoveries for a given instance, where said instance @@ -537,17 +543,13 @@ func ReadInActivePeriodSuccessorInstanceRecovery(instanceKey *inst.InstanceKey) } // ReadRecentRecoveries reads latest recovery entries from topology_recovery -func ReadRecentRecoveries(clusterName string, unacknowledgedOnly bool, page int) ([]*TopologyRecovery, error) { +func ReadRecentRecoveries(unacknowledgedOnly bool, page int) ([]*TopologyRecovery, error) { whereConditions := []string{} whereClause := "" args := sqlutils.Args() if unacknowledgedOnly { whereConditions = append(whereConditions, `acknowledged=0`) } - if clusterName != "" { - whereConditions = append(whereConditions, `cluster_name=?`) - args = append(args, clusterName) - } if len(whereConditions) > 0 { whereClause = fmt.Sprintf("where %s", strings.Join(whereConditions, " and ")) } diff --git a/go/vt/vtorc/logic/topology_recovery_dao_test.go b/go/vt/vtorc/logic/topology_recovery_dao_test.go new file mode 100644 index 00000000000..b977e9c4371 --- /dev/null +++ b/go/vt/vtorc/logic/topology_recovery_dao_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logic + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/vt/vtorc/db" + "vitess.io/vitess/go/vt/vtorc/inst" +) + +// TestTopologyRecovery tests various operations related to topology recovery like reading from and writing it to the database. +func TestTopologyRecovery(t *testing.T) { + // Open the vtorc + // After the test completes delete everything from the vitess_tablet table + orcDb, err := db.OpenVTOrc() + require.NoError(t, err) + defer func() { + _, err = orcDb.Exec("delete from topology_recovery") + require.NoError(t, err) + }() + + replicationAnalysis := inst.ReplicationAnalysis{ + AnalyzedInstanceKey: inst.InstanceKey{ + Hostname: hostname, + Port: 101, + }, + TabletType: tab101.Type, + ClusterDetails: inst.ClusterInfo{ + Keyspace: keyspace, + Shard: shard, + }, + AnalyzedKeyspace: keyspace, + AnalyzedShard: shard, + Analysis: inst.ReplicaIsWritable, + IsReadOnly: false, + } + topologyRecovery := NewTopologyRecovery(replicationAnalysis) + + t.Run("writing to topology recovery", func(t *testing.T) { + topologyRecovery, err = writeTopologyRecovery(topologyRecovery) + require.NoError(t, err) + // The ID field should be populated after the insert + require.Greater(t, topologyRecovery.ID, int64(0)) + }) + + t.Run("read recoveries", func(t *testing.T) { + recoveries, err := ReadRecentRecoveries(false, 0) + require.NoError(t, err) + require.Len(t, recoveries, 1) + // Assert that the ID field matches the one that we just wrote + require.EqualValues(t, topologyRecovery.ID, recoveries[0].ID) + }) +} From 7b7e46b1becc52c131bcce8ca6599d208eed024b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 16:37:10 +0530 Subject: [PATCH 19/25] feat: reading recovery instances Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/downtime_dao.go | 2 +- go/vt/vtorc/inst/instance_dao.go | 14 ++++++++------ go/vt/vtorc/server/api.go | 11 +---------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/go/vt/vtorc/inst/downtime_dao.go b/go/vt/vtorc/inst/downtime_dao.go index 1d5c33873b5..53b12e325e8 100644 --- a/go/vt/vtorc/inst/downtime_dao.go +++ b/go/vt/vtorc/inst/downtime_dao.go @@ -135,7 +135,7 @@ func renewLostInRecoveryDowntime() error { // expireLostInRecoveryDowntime expires downtime for servers who have been lost in recovery in the last, // but are now replicating. func expireLostInRecoveryDowntime() error { - instances, err := ReadLostInRecoveryInstances("") + instances, err := ReadLostInRecoveryInstances("", "") if err != nil { return err } diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index f9b2f7c84d1..64ad6d48678 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -773,9 +773,10 @@ func ReadReplicaInstancesIncludingBinlogServerSubReplicas(primaryKey *InstanceKe } // ReadProblemInstances reads all instances with problems -func ReadProblemInstances(clusterName string) ([](*Instance), error) { +func ReadProblemInstances(keyspace string, shard string) ([](*Instance), error) { condition := ` - cluster_name LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + keyspace LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) and ( (last_seen < last_checked) or (unix_timestamp() - unix_timestamp(last_checked) > ?) @@ -788,7 +789,7 @@ func ReadProblemInstances(clusterName string) ([](*Instance), error) { ) ` - args := sqlutils.Args(clusterName, clusterName, config.Config.InstancePollSeconds*5, config.Config.ReasonableReplicationLagSeconds, config.Config.ReasonableReplicationLagSeconds) + args := sqlutils.Args(keyspace, keyspace, shard, shard, config.Config.InstancePollSeconds*5, config.Config.ReasonableReplicationLagSeconds, config.Config.ReasonableReplicationLagSeconds) instances, err := readInstancesByCondition(condition, args, "") if err != nil { return instances, err @@ -808,15 +809,16 @@ func ReadProblemInstances(clusterName string) ([](*Instance), error) { // ReadLostInRecoveryInstances returns all instances (potentially filtered by cluster) // which are currently indicated as downtimed due to being lost during a topology recovery. -func ReadLostInRecoveryInstances(clusterName string) ([](*Instance), error) { +func ReadLostInRecoveryInstances(keyspace string, shard string) ([](*Instance), error) { condition := ` ifnull( database_instance_downtime.downtime_active = 1 and database_instance_downtime.end_timestamp > now() and database_instance_downtime.reason = ?, 0) - and ? IN ('', cluster_name) + and ? IN ('', keyspace) + and ? IN ('', shard) ` - return readInstancesByCondition(condition, sqlutils.Args(DowntimeLostInRecoveryMessage, clusterName), "cluster_name asc, replication_depth asc") + return readInstancesByCondition(condition, sqlutils.Args(DowntimeLostInRecoveryMessage, keyspace, shard), "keyspace asc, shard asc, replication_depth asc") } // readUnseenPrimaryKeys will read list of primaries that have never been seen, and yet whose replicas diff --git a/go/vt/vtorc/server/api.go b/go/vt/vtorc/server/api.go index 84b2810ba57..d1ead910d39 100644 --- a/go/vt/vtorc/server/api.go +++ b/go/vt/vtorc/server/api.go @@ -120,18 +120,9 @@ func returnAsJSON(response http.ResponseWriter, code int, stuff any) { // problemsAPIHandler is the handler for the problemsAPI endpoint func problemsAPIHandler(response http.ResponseWriter, request *http.Request) { // This api also supports filtering by shard and keyspace provided. - // Currently, both of them have to be provided in order to filter the instances. - // Once we split the cluster_name field into keyspace and shard, we can support - // filtering just by keyspace as well. shard := request.URL.Query().Get("shard") keyspace := request.URL.Query().Get("keyspace") - clusterName := "" - // Override the cluster name to filter by only when both the parameters - // are specified and not empty - if keyspace != "" && shard != "" { - clusterName = inst.GetClusterNameFromKeyspaceAndShard(keyspace, shard) - } - instances, err := inst.ReadProblemInstances(clusterName) + instances, err := inst.ReadProblemInstances(keyspace, shard) if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError) return From cc22ca5c9456b6f282be0c6dad77b36c13518569 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 16:41:23 +0530 Subject: [PATCH 20/25] feat: fix get replication and analysis Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/analysis_dao.go | 8 ++++---- go/vt/vtorc/inst/analysis_dao_test.go | 2 +- go/vt/vtorc/inst/instance_dao.go | 1 - go/vt/vtorc/logic/topology_recovery.go | 4 ++-- go/vt/vtorc/server/api.go | 11 +---------- go/vt/vtorc/test/recovery_analysis.go | 1 - 6 files changed, 8 insertions(+), 19 deletions(-) diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index f0d3427dd6b..89059d3f25b 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -63,11 +63,11 @@ type clusterAnalysis struct { } // GetReplicationAnalysis will check for replication problems (dead primary; unreachable primary; etc) -func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) ([]ReplicationAnalysis, error) { +func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAnalysisHints) ([]ReplicationAnalysis, error) { result := []ReplicationAnalysis{} // TODO(sougou); deprecate ReduceReplicationAnalysisCount - args := sqlutils.Args(config.Config.ReasonableReplicationLagSeconds, ValidSecondsFromSeenToLastAttemptedCheck(), config.Config.ReasonableReplicationLagSeconds, clusterName) + args := sqlutils.Args(config.Config.ReasonableReplicationLagSeconds, ValidSecondsFromSeenToLastAttemptedCheck(), config.Config.ReasonableReplicationLagSeconds, keyspace, shard) query := ` SELECT vitess_tablet.info AS tablet_info, @@ -85,7 +85,6 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) MIN(primary_instance.physical_environment) AS physical_environment, MIN(primary_instance.source_host) AS source_host, MIN(primary_instance.source_port) AS source_port, - MIN(primary_instance.cluster_name) AS cluster_name, MIN(primary_instance.binary_log_file) AS binary_log_file, MIN(primary_instance.binary_log_pos) AS binary_log_pos, MIN(primary_tablet.info) AS primary_tablet_info, @@ -328,7 +327,8 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) ) WHERE database_instance_maintenance.database_instance_maintenance_id IS NULL - AND ? IN ('', primary_instance.cluster_name) + AND ? IN ('', vitess_keyspace.keyspace) + AND ? IN ('', vitess_tablet.shard) GROUP BY vitess_tablet.hostname, vitess_tablet.port diff --git a/go/vt/vtorc/inst/analysis_dao_test.go b/go/vt/vtorc/inst/analysis_dao_test.go index a705530a585..6c5fc3cade6 100644 --- a/go/vt/vtorc/inst/analysis_dao_test.go +++ b/go/vt/vtorc/inst/analysis_dao_test.go @@ -530,7 +530,7 @@ func TestGetReplicationAnalysis(t *testing.T) { } db.Db = test.NewTestDB([][]sqlutils.RowMap{rowMaps}) - got, err := GetReplicationAnalysis("", &ReplicationAnalysisHints{}) + got, err := GetReplicationAnalysis("", "", &ReplicationAnalysisHints{}) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) return diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 64ad6d48678..8bb831cd796 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -507,7 +507,6 @@ func ReadInstanceClusterAttributes(instance *Instance) (err error) { var primaryOrGroupPrimaryExecutedGtidSet string primaryOrGroupPrimaryDataFound := false - // Read the cluster_name of the _primary_ or _group_primary_ of our instance, derive it from there. query := ` select replication_depth, diff --git a/go/vt/vtorc/logic/topology_recovery.go b/go/vt/vtorc/logic/topology_recovery.go index 7c6969270b2..b8b698f4241 100644 --- a/go/vt/vtorc/logic/topology_recovery.go +++ b/go/vt/vtorc/logic/topology_recovery.go @@ -747,7 +747,7 @@ func executeCheckAndRecoverFunction(analysisEntry inst.ReplicationAnalysis, cand // checkIfAlreadyFixed checks whether the problem that the analysis entry represents has already been fixed by another agent or not func checkIfAlreadyFixed(analysisEntry inst.ReplicationAnalysis) (bool, error) { // Run a replication analysis again. We will check if the problem persisted - analysisEntries, err := inst.GetReplicationAnalysis(analysisEntry.ClusterDetails.ClusterName, &inst.ReplicationAnalysisHints{}) + analysisEntries, err := inst.GetReplicationAnalysis(analysisEntry.ClusterDetails.Keyspace, analysisEntry.ClusterDetails.Shard, &inst.ReplicationAnalysisHints{}) if err != nil { return false, err } @@ -766,7 +766,7 @@ func checkIfAlreadyFixed(analysisEntry inst.ReplicationAnalysis) (bool, error) { // CheckAndRecover is the main entry point for the recovery mechanism func CheckAndRecover(specificInstance *inst.InstanceKey, candidateInstanceKey *inst.InstanceKey, skipProcesses bool) (recoveryAttempted bool, promotedReplicaKey *inst.InstanceKey, err error) { // Allow the analysis to run even if we don't want to recover - replicationAnalysis, err := inst.GetReplicationAnalysis("", &inst.ReplicationAnalysisHints{IncludeDowntimed: true, AuditAnalysis: true}) + replicationAnalysis, err := inst.GetReplicationAnalysis("", "", &inst.ReplicationAnalysisHints{IncludeDowntimed: true, AuditAnalysis: true}) if err != nil { log.Error(err) return false, nil, err diff --git a/go/vt/vtorc/server/api.go b/go/vt/vtorc/server/api.go index d1ead910d39..4743fa0705d 100644 --- a/go/vt/vtorc/server/api.go +++ b/go/vt/vtorc/server/api.go @@ -153,18 +153,9 @@ func enableGlobalRecoveriesAPIHandler(response http.ResponseWriter) { // replicationAnalysisAPIHandler is the handler for the replicationAnalysisAPI endpoint func replicationAnalysisAPIHandler(response http.ResponseWriter, request *http.Request) { // This api also supports filtering by shard and keyspace provided. - // Currently, both of them have to be provided in order to filter the replication analysis. - // Once we split the cluster_name field into keyspace and shard, we can support - // filtering just by keyspace as well. shard := request.URL.Query().Get("shard") keyspace := request.URL.Query().Get("keyspace") - clusterName := "" - // Override the cluster name to filter by only when both the parameters - // are specified and not empty - if keyspace != "" && shard != "" { - clusterName = inst.GetClusterNameFromKeyspaceAndShard(keyspace, shard) - } - analysis, err := inst.GetReplicationAnalysis(clusterName, &inst.ReplicationAnalysisHints{}) + analysis, err := inst.GetReplicationAnalysis(keyspace, shard, &inst.ReplicationAnalysisHints{}) if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError) return diff --git a/go/vt/vtorc/test/recovery_analysis.go b/go/vt/vtorc/test/recovery_analysis.go index b8f2efe731d..0fdd77b01af 100644 --- a/go/vt/vtorc/test/recovery_analysis.go +++ b/go/vt/vtorc/test/recovery_analysis.go @@ -89,7 +89,6 @@ func (info *InfoForRecoveryAnalysis) ConvertToRowMap() sqlutils.RowMap { rowMap := make(sqlutils.RowMap) rowMap["binary_log_file"] = sqlutils.CellData{String: info.LogFile, Valid: true} rowMap["binary_log_pos"] = sqlutils.CellData{String: fmt.Sprintf("%v", info.LogPos), Valid: true} - rowMap["cluster_name"] = sqlutils.CellData{String: info.ClusterName, Valid: true} rowMap["count_binlog_server_replicas"] = sqlutils.CellData{Valid: false} rowMap["count_co_primary_replicas"] = sqlutils.CellData{Valid: false} rowMap["count_delayed_replicas"] = sqlutils.CellData{String: fmt.Sprintf("%v", info.CountDelayedReplicas), Valid: true} From e46e853f37043185dceda931bb432d789fd48cf9 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 19:46:56 +0530 Subject: [PATCH 21/25] feat: fix bug in query Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/instance_dao.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 8bb831cd796..8fae27c884b 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -775,7 +775,7 @@ func ReadReplicaInstancesIncludingBinlogServerSubReplicas(primaryKey *InstanceKe func ReadProblemInstances(keyspace string, shard string) ([](*Instance), error) { condition := ` keyspace LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) - shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + and shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) and ( (last_seen < last_checked) or (unix_timestamp() - unix_timestamp(last_checked) > ?) From 3ffc36f8f518af0399c616bf92bcb03dd076587f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 19:49:25 +0530 Subject: [PATCH 22/25] test: add tests to check that filtering by keyspace works for APIs Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 87312004f7a..9042c1bf522 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -107,6 +107,11 @@ func TestProblemsAPI(t *testing.T) { assert.Equal(t, 200, status, resp) assert.Contains(t, resp, fmt.Sprintf(`"Port": %d`, replica.MySQLPort)) + // Verify that filtering by keyspace also works in the API as intended + status, resp = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?keyspace=ks") + assert.Equal(t, 200, status, resp) + assert.Contains(t, resp, fmt.Sprintf(`"Port": %d`, replica.MySQLPort)) + // Check that filtering using keyspace and shard works status, resp = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?keyspace=ks&shard=80-") assert.Equal(t, 200, status, resp) @@ -150,6 +155,11 @@ func TestProblemsAPI(t *testing.T) { assert.Equal(t, 200, status, resp) assert.Contains(t, resp, fmt.Sprintf(`"InstanceAlias": "%v"`, replica.Alias)) + // Check that filtering using keyspace works + status, resp = utils.MakeAPICall(t, vtorc, "/api/problems?keyspace=ks") + assert.Equal(t, 200, status, resp) + assert.Contains(t, resp, fmt.Sprintf(`"InstanceAlias": "%v"`, replica.Alias)) + // Check that filtering using keyspace and shard works status, resp = utils.MakeAPICall(t, vtorc, "/api/problems?keyspace=ks&shard=80-") assert.Equal(t, 200, status, resp) From 6bce6fdbd55f582814d3ef175fc97411836181c8 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 27 Dec 2022 19:52:42 +0530 Subject: [PATCH 23/25] feat: remove remaining usages of ClusterName Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/analysis_dao.go | 3 +-- go/vt/vtorc/inst/cluster.go | 1 - go/vt/vtorc/inst/instance_dao.go | 4 ++-- go/vt/vtorc/logic/topology_recovery.go | 2 -- go/vt/vtorc/logic/topology_recovery_dao.go | 2 +- go/vt/vtorc/test/recovery_analysis.go | 2 -- 6 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index 89059d3f25b..ac662b36c89 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -385,7 +385,6 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna isStaleBinlogCoordinates := m.GetBool("is_stale_binlog_coordinates") a.ClusterDetails.Keyspace = m.GetString("keyspace") a.ClusterDetails.Shard = m.GetString("shard") - a.ClusterDetails.ClusterName = GetClusterNameFromKeyspaceAndShard(a.ClusterDetails.Keyspace, a.ClusterDetails.Shard) a.GTIDMode = m.GetString("gtid_mode") a.LastCheckValid = m.GetBool("is_last_check_valid") a.LastCheckPartialSuccess = m.GetBool("last_check_partial_success") @@ -440,7 +439,7 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna log.Infof(analysisMessage) } } - keyspaceShard := a.ClusterDetails.ClusterName + keyspaceShard := getKeyspaceShardName(a.ClusterDetails.Keyspace, a.ClusterDetails.Shard) if clusters[keyspaceShard] == nil { clusters[keyspaceShard] = &clusterAnalysis{} if a.TabletType == topodatapb.TabletType_PRIMARY { diff --git a/go/vt/vtorc/inst/cluster.go b/go/vt/vtorc/inst/cluster.go index aea7b81ed81..c3a77485e74 100644 --- a/go/vt/vtorc/inst/cluster.go +++ b/go/vt/vtorc/inst/cluster.go @@ -18,7 +18,6 @@ package inst // ClusterInfo makes for a cluster status/info summary type ClusterInfo struct { - ClusterName string Keyspace string Shard string CountInstances uint diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 8fae27c884b..0db4eb5a181 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -454,8 +454,8 @@ Cleanup: return nil, err } -// GetClusterNameFromKeyspaceAndShard returns the cluster name from keyspace and shard -func GetClusterNameFromKeyspaceAndShard(keyspace, shard string) string { +// getKeyspaceShardName returns a single string having both the keyspace and shard +func getKeyspaceShardName(keyspace, shard string) string { return fmt.Sprintf("%v:%v", keyspace, shard) } diff --git a/go/vt/vtorc/logic/topology_recovery.go b/go/vt/vtorc/logic/topology_recovery.go index b8b698f4241..f4a67f9f564 100644 --- a/go/vt/vtorc/logic/topology_recovery.go +++ b/go/vt/vtorc/logic/topology_recovery.go @@ -99,7 +99,6 @@ type RecoveryAcknowledgement struct { Comment string Key inst.InstanceKey - ClusterName string ID int64 UID string AllRecoveries bool @@ -108,7 +107,6 @@ type RecoveryAcknowledgement struct { // BlockedTopologyRecovery represents an entry in the blocked_topology_recovery table type BlockedTopologyRecovery struct { FailedInstanceKey inst.InstanceKey - ClusterName string Analysis inst.AnalysisCode LastBlockedTimestamp string BlockingRecoveryID int64 diff --git a/go/vt/vtorc/logic/topology_recovery_dao.go b/go/vt/vtorc/logic/topology_recovery_dao.go index f890a5b4e15..d60c0fbfd5b 100644 --- a/go/vt/vtorc/logic/topology_recovery_dao.go +++ b/go/vt/vtorc/logic/topology_recovery_dao.go @@ -204,7 +204,7 @@ func AttemptRecoveryRegistration(analysisEntry *inst.ReplicationAnalysis, failIf } if len(recoveries) > 0 { _ = RegisterBlockedRecoveries(analysisEntry, recoveries) - errMsg := fmt.Sprintf("AttemptRecoveryRegistration: cluster %+v has recently experienced a failover (of %+v) and is in active period. It will not be failed over again. You may acknowledge the failure on this cluster (-c ack-cluster-recoveries) or on %+v (-c ack-instance-recoveries) to remove this blockage", analysisEntry.ClusterDetails.ClusterName, recoveries[0].AnalysisEntry.AnalyzedInstanceKey, recoveries[0].AnalysisEntry.AnalyzedInstanceKey) + errMsg := fmt.Sprintf("AttemptRecoveryRegistration: keyspace %+v shard %+v has recently experienced a failover (of %+v) and is in active period. It will not be failed over again. You may acknowledge the failure on this cluster (-c ack-cluster-recoveries) or on %+v (-c ack-instance-recoveries) to remove this blockage", analysisEntry.ClusterDetails.Keyspace, analysisEntry.ClusterDetails.Shard, recoveries[0].AnalysisEntry.AnalyzedInstanceKey, recoveries[0].AnalysisEntry.AnalyzedInstanceKey) log.Errorf(errMsg) return nil, fmt.Errorf(errMsg) } diff --git a/go/vt/vtorc/test/recovery_analysis.go b/go/vt/vtorc/test/recovery_analysis.go index 0fdd77b01af..5893caafc25 100644 --- a/go/vt/vtorc/test/recovery_analysis.go +++ b/go/vt/vtorc/test/recovery_analysis.go @@ -47,7 +47,6 @@ type InfoForRecoveryAnalysis struct { LogFile string LogPos int64 IsStaleBinlogCoordinates int - ClusterName string GTIDMode string LastCheckValid int LastCheckPartialSuccess int @@ -160,5 +159,4 @@ func (info *InfoForRecoveryAnalysis) SetValuesFromTabletInfo() { info.DataCenter = info.TabletInfo.Alias.Cell info.Keyspace = info.TabletInfo.Keyspace info.Shard = info.TabletInfo.Shard - info.ClusterName = fmt.Sprintf("%v:%v", info.TabletInfo.Keyspace, info.TabletInfo.Shard) } From 6b0e47793d7826e6153fddc97386da981eebe807 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 29 Dec 2022 11:16:42 +0530 Subject: [PATCH 24/25] refactor: fix comment explaining sleep in the test Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/audit_dao_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/vt/vtorc/inst/audit_dao_test.go b/go/vt/vtorc/inst/audit_dao_test.go index c1c28fac2fe..4a6533077c2 100644 --- a/go/vt/vtorc/inst/audit_dao_test.go +++ b/go/vt/vtorc/inst/audit_dao_test.go @@ -106,7 +106,9 @@ func TestAuditOperation(t *testing.T) { err = AuditOperation(auditType, instance100, message) require.NoError(t, err) - // Give a little time for the write to succeed and be emptied from the write buffer. + // Give a little time for the write to succeed since it happens in a separate go-routine + // There is no way to wait for that write to complete. This sleep is required to prevent this test from + // becoming flaky wherein we sometimes read the file before the contents are written. time.Sleep(100 * time.Millisecond) fileContent, err := os.ReadFile(file.Name()) require.NoError(t, err) From 0de78f91db200cea0d6f31007542657cf2c185d8 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 5 Jan 2023 13:26:56 +0530 Subject: [PATCH 25/25] feat: add code to prevent filtering just by shard and add tests for it Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 10 ++++++++++ go/vt/vtorc/server/api.go | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 9042c1bf522..4885a67aa9c 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -116,6 +116,11 @@ func TestProblemsAPI(t *testing.T) { status, resp = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?keyspace=ks&shard=80-") assert.Equal(t, 200, status, resp) assert.Equal(t, "[]", resp) + + // Check that filtering using just the shard fails + status, resp = utils.MakeAPICall(t, vtorc, "/api/replication-analysis?shard=0") + assert.Equal(t, 400, status, resp) + assert.Equal(t, "Filtering by shard without keyspace isn't supported\n", resp) }) t.Run("Enable Recoveries API", func(t *testing.T) { @@ -164,5 +169,10 @@ func TestProblemsAPI(t *testing.T) { status, resp = utils.MakeAPICall(t, vtorc, "/api/problems?keyspace=ks&shard=80-") assert.Equal(t, 200, status, resp) assert.Equal(t, "null", resp) + + // Check that filtering using just the shard fails + status, resp = utils.MakeAPICall(t, vtorc, "/api/problems?shard=0") + assert.Equal(t, 400, status, resp) + assert.Equal(t, "Filtering by shard without keyspace isn't supported\n", resp) }) } diff --git a/go/vt/vtorc/server/api.go b/go/vt/vtorc/server/api.go index 4743fa0705d..1a7f0a1c1da 100644 --- a/go/vt/vtorc/server/api.go +++ b/go/vt/vtorc/server/api.go @@ -41,6 +41,8 @@ const ( enableGlobalRecoveriesAPI = "/api/enable-global-recoveries" replicationAnalysisAPI = "/api/replication-analysis" healthAPI = "/debug/health" + + shardWithoutKeyspaceFilteringErrorStr = "Filtering by shard without keyspace isn't supported" ) var ( @@ -122,6 +124,10 @@ func problemsAPIHandler(response http.ResponseWriter, request *http.Request) { // This api also supports filtering by shard and keyspace provided. shard := request.URL.Query().Get("shard") keyspace := request.URL.Query().Get("keyspace") + if shard != "" && keyspace == "" { + http.Error(response, shardWithoutKeyspaceFilteringErrorStr, http.StatusBadRequest) + return + } instances, err := inst.ReadProblemInstances(keyspace, shard) if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError) @@ -155,6 +161,10 @@ func replicationAnalysisAPIHandler(response http.ResponseWriter, request *http.R // This api also supports filtering by shard and keyspace provided. shard := request.URL.Query().Get("shard") keyspace := request.URL.Query().Get("keyspace") + if shard != "" && keyspace == "" { + http.Error(response, shardWithoutKeyspaceFilteringErrorStr, http.StatusBadRequest) + return + } analysis, err := inst.GetReplicationAnalysis(keyspace, shard, &inst.ReplicationAnalysisHints{}) if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError)