Skip to content

Commit

Permalink
Online DDL: improved row estimation via ANALYE TABLE with --analyze-t…
Browse files Browse the repository at this point in the history 
…able strategy flag (#13352)

* DDL Strategy: support --analyze-table flag

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

* Issue an ANALYZE TABLE if --analyze-table is given. Also, update table_rows if rows_copied exceeds it

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

* use NO_WRITE_TO_BINLOG

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

* update workflows

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

---------

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
  • Loading branch information
@shlomi-noach
shlomi-noach authored Jul 13, 2023
1 parent 9699f11 commit a960b91
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 3 deletions.
7 changes: 7 additions & 0 deletions go/vt/schema/ddl_strategy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const (
cutOverThresholdFlag = "cut-over-threshold"
vreplicationTestSuite = "vreplication-test-suite"
allowForeignKeysFlag = "unsafe-allow-foreign-keys"
analyzeTableFlag = "analyze-table"
)

// DDLStrategy suggests how an ALTER TABLE should run (e.g. "direct", "online", "gh-ost" or "pt-osc")
Expand Down Expand Up @@ -221,6 +222,11 @@ func (setting *DDLStrategySetting) IsAllowForeignKeysFlag() bool {
return setting.hasFlag(allowForeignKeysFlag)
}

// IsAnalyzeTableFlag checks if strategy options include --analyze-table
func (setting *DDLStrategySetting) IsAnalyzeTableFlag() bool {
return setting.hasFlag(analyzeTableFlag)
}

// RuntimeOptions returns the options used as runtime flags for given strategy, removing any internal hint options
func (setting *DDLStrategySetting) RuntimeOptions() []string {
opts, _ := shlex.Split(setting.Options)
Expand All @@ -243,6 +249,7 @@ func (setting *DDLStrategySetting) RuntimeOptions() []string {
case isFlag(opt, fastRangeRotationFlag):
case isFlag(opt, vreplicationTestSuite):
case isFlag(opt, allowForeignKeysFlag):
case isFlag(opt, analyzeTableFlag):
default:
validOpts = append(validOpts, opt)
}
Expand Down
9 changes: 9 additions & 0 deletions go/vt/schema/ddl_strategy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ func TestParseDDLStrategy(t *testing.T) {
fastOverRevertible bool
fastRangeRotation bool
allowForeignKeys bool
analyzeTable bool
cutOverThreshold time.Duration
runtimeOptions string
err error
Expand Down Expand Up @@ -238,6 +239,13 @@ func TestParseDDLStrategy(t *testing.T) {
runtimeOptions: "",
cutOverThreshold: 5 * time.Minute,
},
{
strategyVariable: "vitess --analyze-table",
strategy: DDLStrategyVitess,
options: "--analyze-table",
runtimeOptions: "",
analyzeTable: true,
},
}
for _, ts := range tt {
t.Run(ts.strategyVariable, func(t *testing.T) {
Expand All @@ -253,6 +261,7 @@ func TestParseDDLStrategy(t *testing.T) {
assert.Equal(t, ts.fastOverRevertible, setting.IsPreferInstantDDL())
assert.Equal(t, ts.fastRangeRotation, setting.IsFastRangeRotationFlag())
assert.Equal(t, ts.allowForeignKeys, setting.IsAllowForeignKeysFlag())
assert.Equal(t, ts.analyzeTable, setting.IsAnalyzeTableFlag())
cutOverThreshold, err := setting.CutOverThreshold()
assert.NoError(t, err)
assert.Equal(t, ts.cutOverThreshold, cutOverThreshold)
Expand Down
5 changes: 3 additions & 2 deletions go/vt/vttablet/onlineddl/executor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1316,7 +1316,7 @@ func (e *Executor) initVreplicationOriginalMigration(ctx context.Context, online
}
}
}
v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, onlineDDL.SQL)
v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, onlineDDL.SQL, onlineDDL.StrategySetting().IsAnalyzeTableFlag())
return v, nil
}

Expand Down Expand Up @@ -1370,7 +1370,7 @@ func (e *Executor) initVreplicationRevertMigration(ctx context.Context, onlineDD
if err := e.updateArtifacts(ctx, onlineDDL.UUID, vreplTableName); err != nil {
return v, err
}
v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, "")
v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, "", false)
v.pos = revertStream.pos
return v, nil
}
Expand Down Expand Up @@ -4207,6 +4207,7 @@ func (e *Executor) updateMigrationProgress(ctx context.Context, uuid string, pro

func (e *Executor) updateMigrationProgressByRowsCopied(ctx context.Context, uuid string, rowsCopied int64) error {
query, err := sqlparser.ParseAndBind(sqlUpdateMigrationProgressByRowsCopied,
sqltypes.Int64BindVariable(rowsCopied),
sqltypes.Int64BindVariable(rowsCopied),
sqltypes.StringBindVariable(uuid),
)
Expand Down
2 changes: 2 additions & 0 deletions go/vt/vttablet/onlineddl/schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ const (
`
sqlUpdateMigrationProgressByRowsCopied = `UPDATE _vt.schema_migrations
SET
table_rows=GREATEST(table_rows, %a),
progress=CASE
WHEN table_rows=0 THEN 100
ELSE LEAST(100, 100*%a/table_rows)
Expand Down Expand Up @@ -520,6 +521,7 @@ const (
sqlDropTableIfExists = "DROP TABLE IF EXISTS `%a`"
sqlShowColumnsFrom = "SHOW COLUMNS FROM `%a`"
sqlShowTableStatus = "SHOW TABLE STATUS LIKE '%a'"
sqlAnalyzeTable = "ANALYZE NO_WRITE_TO_BINLOG TABLE `%a`"
sqlShowCreateTable = "SHOW CREATE TABLE `%a`"
sqlGetAutoIncrement = `
SELECT
Expand Down
17 changes: 16 additions & 1 deletion go/vt/vttablet/onlineddl/vrepl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ type VRepl struct {
alterQuery string
tableRows int64

analyzeTable bool

sourceSharedColumns *vrepl.ColumnList
targetSharedColumns *vrepl.ColumnList
droppedSourceNonGeneratedColumns *vrepl.ColumnList
Expand All @@ -130,7 +132,7 @@ type VRepl struct {
}

// NewVRepl creates a VReplication handler for Online DDL
func NewVRepl(workflow, keyspace, shard, dbName, sourceTable, targetTable, alterQuery string) *VRepl {
func NewVRepl(workflow, keyspace, shard, dbName, sourceTable, targetTable, alterQuery string, analyzeTable bool) *VRepl {
return &VRepl{
workflow: workflow,
keyspace: keyspace,
Expand All @@ -139,6 +141,7 @@ func NewVRepl(workflow, keyspace, shard, dbName, sourceTable, targetTable, alter
sourceTable: sourceTable,
targetTable: targetTable,
alterQuery: alterQuery,
analyzeTable: analyzeTable,
parser: vrepl.NewAlterTableParser(),
enumToTextMap: map[string]string{},
intToEnumMap: map[string]bool{},
Expand Down Expand Up @@ -226,6 +229,13 @@ func (v *VRepl) readTableUniqueKeys(ctx context.Context, conn *dbconnpool.DBConn
return uniqueKeys, nil
}

// executeAnalyzeTable runs an ANALYZE TABLE command
func (v *VRepl) executeAnalyzeTable(ctx context.Context, conn *dbconnpool.DBConnection, tableName string) error {
parsed := sqlparser.BuildParsedQuery(sqlAnalyzeTable, tableName)
_, err := conn.ExecuteFetch(parsed.Query, 1, false)
return err
}

// readTableStatus reads table status information
func (v *VRepl) readTableStatus(ctx context.Context, conn *dbconnpool.DBConnection, tableName string) (tableRows int64, err error) {
parsed := sqlparser.BuildParsedQuery(sqlShowTableStatus, tableName)
Expand Down Expand Up @@ -335,6 +345,11 @@ func (v *VRepl) analyzeAlter(ctx context.Context) error {
}

func (v *VRepl) analyzeTables(ctx context.Context, conn *dbconnpool.DBConnection) (err error) {
if v.analyzeTable {
if err := v.executeAnalyzeTable(ctx, conn, v.sourceTable); err != nil {
return err
}
}
v.tableRows, err = v.readTableStatus(ctx, conn, v.sourceTable)
if err != nil {
return err
Expand Down

0 comments on commit a960b91

Please sign in to comment.