Online DDL: improved row estimation via ANALYE TABLE with --analyze-table strategy flag

.github/workflows/cluster_endtoend_vreplication_partial_movetables_sequences.yml

@@ -64,7 +64,7 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       uses: actions/setup-go@v4
       with:
-        go-version: 1.20.4
+        go-version: 1.20.5
 
     - name: Set up python
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

.github/workflows/cluster_endtoend_vreplication_partial_movetables_simple.yml

@@ -64,7 +64,7 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       uses: actions/setup-go@v4
       with:
-        go-version: 1.20.4
+        go-version: 1.20.5
 
     - name: Set up python
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

go/vt/schema/ddl_strategy.go

@@ -45,6 +45,7 @@ const (
 	cutOverThresholdFlag   = "cut-over-threshold"
 	vreplicationTestSuite  = "vreplication-test-suite"
 	allowForeignKeysFlag   = "unsafe-allow-foreign-keys"
+	analyzeTableFlag       = "analyze-table"
 )
 
 // DDLStrategy suggests how an ALTER TABLE should run (e.g. "direct", "online", "gh-ost" or "pt-osc")
@@ -221,6 +222,11 @@ func (setting *DDLStrategySetting) IsAllowForeignKeysFlag() bool {
 	return setting.hasFlag(allowForeignKeysFlag)
 }
 
+// IsAnalyzeTableFlag checks if strategy options include --analyze-table
+func (setting *DDLStrategySetting) IsAnalyzeTableFlag() bool {
+	return setting.hasFlag(analyzeTableFlag)
+}
+
 // RuntimeOptions returns the options used as runtime flags for given strategy, removing any internal hint options
 func (setting *DDLStrategySetting) RuntimeOptions() []string {
 	opts, _ := shlex.Split(setting.Options)
@@ -243,6 +249,7 @@ func (setting *DDLStrategySetting) RuntimeOptions() []string {
 		case isFlag(opt, fastRangeRotationFlag):
 		case isFlag(opt, vreplicationTestSuite):
 		case isFlag(opt, allowForeignKeysFlag):
+		case isFlag(opt, analyzeTableFlag):
 		default:
 			validOpts = append(validOpts, opt)
 		}

go/vt/schema/ddl_strategy_test.go

@@ -116,6 +116,7 @@ func TestParseDDLStrategy(t *testing.T) {
 		fastOverRevertible   bool
 		fastRangeRotation    bool
 		allowForeignKeys     bool
+		analyzeTable         bool
 		cutOverThreshold     time.Duration
 		runtimeOptions       string
 		err                  error
@@ -238,6 +239,13 @@ func TestParseDDLStrategy(t *testing.T) {
 			runtimeOptions:   "",
 			cutOverThreshold: 5 * time.Minute,
 		},
+		{
+			strategyVariable: "vitess --analyze-table",
+			strategy:         DDLStrategyVitess,
+			options:          "--analyze-table",
+			runtimeOptions:   "",
+			analyzeTable:     true,
+		},
 	}
 	for _, ts := range tt {
 		t.Run(ts.strategyVariable, func(t *testing.T) {
@@ -253,6 +261,7 @@ func TestParseDDLStrategy(t *testing.T) {
 			assert.Equal(t, ts.fastOverRevertible, setting.IsPreferInstantDDL())
 			assert.Equal(t, ts.fastRangeRotation, setting.IsFastRangeRotationFlag())
 			assert.Equal(t, ts.allowForeignKeys, setting.IsAllowForeignKeysFlag())
+			assert.Equal(t, ts.analyzeTable, setting.IsAnalyzeTableFlag())
 			cutOverThreshold, err := setting.CutOverThreshold()
 			assert.NoError(t, err)
 			assert.Equal(t, ts.cutOverThreshold, cutOverThreshold)

go/vt/vttablet/onlineddl/executor.go

@@ -1316,7 +1316,7 @@ func (e *Executor) initVreplicationOriginalMigration(ctx context.Context, online
 			}
 		}
 	}
-	v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, onlineDDL.SQL)
+	v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, onlineDDL.SQL, onlineDDL.StrategySetting().IsAnalyzeTableFlag())
 	return v, nil
 }
 
@@ -1370,7 +1370,7 @@ func (e *Executor) initVreplicationRevertMigration(ctx context.Context, onlineDD
 	if err := e.updateArtifacts(ctx, onlineDDL.UUID, vreplTableName); err != nil {
 		return v, err
 	}
-	v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, "")
+	v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, "", false)
 	v.pos = revertStream.pos
 	return v, nil
 }
@@ -4207,6 +4207,7 @@ func (e *Executor) updateMigrationProgress(ctx context.Context, uuid string, pro
 
 func (e *Executor) updateMigrationProgressByRowsCopied(ctx context.Context, uuid string, rowsCopied int64) error {
 	query, err := sqlparser.ParseAndBind(sqlUpdateMigrationProgressByRowsCopied,
+		sqltypes.Int64BindVariable(rowsCopied),
 		sqltypes.Int64BindVariable(rowsCopied),
 		sqltypes.StringBindVariable(uuid),
 	)

go/vt/vttablet/onlineddl/schema.go

@@ -213,6 +213,7 @@ const (
 	`
 	sqlUpdateMigrationProgressByRowsCopied = `UPDATE _vt.schema_migrations
 			SET
+				table_rows=GREATEST(table_rows, %a),
 				progress=CASE
 					WHEN table_rows=0 THEN 100
 					ELSE LEAST(100, 100*%a/table_rows)
@@ -520,6 +521,7 @@ const (
 	sqlDropTableIfExists = "DROP TABLE IF EXISTS `%a`"
 	sqlShowColumnsFrom   = "SHOW COLUMNS FROM `%a`"
 	sqlShowTableStatus   = "SHOW TABLE STATUS LIKE '%a'"
+	sqlAnalyzeTable      = "ANALYZE NO_WRITE_TO_BINLOG TABLE `%a`"
 	sqlShowCreateTable   = "SHOW CREATE TABLE `%a`"
 	sqlGetAutoIncrement  = `
 		SELECT

go/vt/vttablet/onlineddl/vrepl.go

@@ -104,6 +104,8 @@ type VRepl struct {
 	alterQuery  string
 	tableRows   int64
 
+	analyzeTable bool
+
 	sourceSharedColumns              *vrepl.ColumnList
 	targetSharedColumns              *vrepl.ColumnList
 	droppedSourceNonGeneratedColumns *vrepl.ColumnList
@@ -130,7 +132,7 @@ type VRepl struct {
 }
 
 // NewVRepl creates a VReplication handler for Online DDL
-func NewVRepl(workflow, keyspace, shard, dbName, sourceTable, targetTable, alterQuery string) *VRepl {
+func NewVRepl(workflow, keyspace, shard, dbName, sourceTable, targetTable, alterQuery string, analyzeTable bool) *VRepl {
 	return &VRepl{
 		workflow:       workflow,
 		keyspace:       keyspace,
@@ -139,6 +141,7 @@ func NewVRepl(workflow, keyspace, shard, dbName, sourceTable, targetTable, alter
 		sourceTable:    sourceTable,
 		targetTable:    targetTable,
 		alterQuery:     alterQuery,
+		analyzeTable:   analyzeTable,
 		parser:         vrepl.NewAlterTableParser(),
 		enumToTextMap:  map[string]string{},
 		intToEnumMap:   map[string]bool{},
@@ -226,6 +229,13 @@ func (v *VRepl) readTableUniqueKeys(ctx context.Context, conn *dbconnpool.DBConn
 	return uniqueKeys, nil
 }
 
+// executeAnalyzeTable runs an ANALYZE TABLE command
+func (v *VRepl) executeAnalyzeTable(ctx context.Context, conn *dbconnpool.DBConnection, tableName string) error {
+	parsed := sqlparser.BuildParsedQuery(sqlAnalyzeTable, tableName)
+	_, err := conn.ExecuteFetch(parsed.Query, 1, false)
+	return err
+}
+
 // readTableStatus reads table status information
 func (v *VRepl) readTableStatus(ctx context.Context, conn *dbconnpool.DBConnection, tableName string) (tableRows int64, err error) {
 	parsed := sqlparser.BuildParsedQuery(sqlShowTableStatus, tableName)
@@ -335,6 +345,11 @@ func (v *VRepl) analyzeAlter(ctx context.Context) error {
 }
 
 func (v *VRepl) analyzeTables(ctx context.Context, conn *dbconnpool.DBConnection) (err error) {
+	if v.analyzeTable {
+		if err := v.executeAnalyzeTable(ctx, conn, v.sourceTable); err != nil {
+			return err
+		}
+	}
 	v.tableRows, err = v.readTableStatus(ctx, conn, v.sourceTable)
 	if err != nil {
 		return err