pkg/filter(ticdc): ignore ddl in ddl_manager #10518
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -111,6 +111,7 @@ type ddlManager struct { | |
| redoMetaManager redo.MetaManager | ||
| // ddlSink is used to ddlSink DDL events to the downstream | ||
| ddlSink DDLSink | ||
| filter filter.Filter | ||
|
|
||
| // pendingDDLs store the pending DDL events of all tables | ||
| // the DDL events in the same table are ordered by commitTs. | ||
|
|
@@ -136,6 +137,7 @@ func newDDLManager( | |
| startTs model.Ts, | ||
| checkpointTs model.Ts, | ||
| ddlSink DDLSink, | ||
| filter filter.Filter, | ||
| ddlPuller puller.DDLPuller, | ||
| schema entry.SchemaStorage, | ||
| redoManager redo.DDLManager, | ||
|
|
@@ -152,6 +154,7 @@ func newDDLManager( | |
| return &ddlManager{ | ||
| changfeedID: changefeedID, | ||
| ddlSink: ddlSink, | ||
| filter: filter, | ||
| ddlPuller: ddlPuller, | ||
| schema: schema, | ||
| redoDDLManager: redoManager, | ||
|
|
@@ -208,7 +211,7 @@ func (m *ddlManager) tick( | |
|
|
||
| log.Info("handle a ddl job", | ||
| zap.String("namespace", m.changfeedID.Namespace), | ||
| zap.String("changefeed", m.changfeedID.ID), | ||
| zap.Int64("tableID", job.TableID), | ||
| zap.Int64("jobID", job.ID), | ||
| zap.String("query", job.Query), | ||
|
|
@@ -244,8 +247,14 @@ func (m *ddlManager) tick( | |
| // Send DDL events to redo log. | ||
| if m.redoDDLManager.Enabled() { | ||
| for _, event := range events { | ||
| skip, _, err := m.shouldSkipDDL(event) | ||
| if err != nil { | ||
| return nil, nil, errors.Trace(err) | ||
| } | ||
| if skip { | ||
| continue | ||
| } | ||
| if err := m.redoDDLManager.EmitDDLEvent(ctx, event); err != nil { | ||
| return nil, nil, err | ||
| } | ||
| } | ||
|
|
@@ -327,29 +336,34 @@ func (m *ddlManager) shouldExecDDL(nextDDL *model.DDLEvent) bool { | |
| return checkpointReachBarrier && redoCheckpointReachBarrier && redoDDLResolvedTsExceedBarrier | ||
| } | ||
|
|
||
| func (m *ddlManager) shouldSkipDDL(ddl *model.DDLEvent) (bool, string, error) { | ||
| ignored, err := m.filter.ShouldIgnoreDDLEvent(ddl) | ||
| if err != nil { | ||
| return false, "", errors.Trace(err) | ||
| } | ||
| if ignored { | ||
| return true, "ddl is ignored by event filter rule, skip it", nil | ||
| } | ||
|
|
||
| // In a BDR mode cluster, TiCDC can receive DDLs from all roles of TiDB. | ||
| // However, CDC only executes the DDLs from the TiDB that has BDRRolePrimary role. | ||
| if m.BDRMode && ddl.BDRRole != string(ast.BDRRolePrimary) { | ||
| return true, "changefeed is in BDRMode and the DDL is not executed by Primary Cluster, skip it", nil | ||
| } | ||
| return false, "", nil | ||
| } | ||
|
|
||
| // executeDDL executes ddlManager.executingDDL. | ||
| func (m *ddlManager) executeDDL(ctx context.Context) error { | ||
| if m.executingDDL == nil { | ||
| return nil | ||
| } | ||
| skip, cleanMsg, err := m.shouldSkipDDL(m.executingDDL) | ||
|
There was a problem hiding this comment. Why do we decide whether to skip this ddl here, but not in where we append ddls into pendingDDLs in "ddlManager.tick"? There was a problem hiding this comment. Because all ddls must be cached in pengingDDLs until |
||
| if err != nil { | ||
| return errors.Trace(err) | ||
| } | ||
| if skip { | ||
| m.cleanCache(cleanMsg) | ||
| return nil | ||
| } | ||
|
|
||
|
|
@@ -371,23 +385,10 @@ func (m *ddlManager) executeDDL(ctx context.Context) error { | |
|
|
||
| done, err := m.ddlSink.emitDDLEvent(ctx, m.executingDDL) | ||
| if err != nil { | ||
| return errors.Trace(err) | ||
| } | ||
| if done { | ||
| m.cleanCache("execute a ddl event successfully") | ||
| } | ||
| return nil | ||
| } | ||
|
|
@@ -555,9 +556,22 @@ func (m *ddlManager) getSnapshotTs() (ts uint64) { | |
| } | ||
|
|
||
| // cleanCache cleans the tableInfoCache and physicalTablesCache. | ||
| // It should be called after a DDL is skipped or sent to downstream successfully. | ||
| func (m *ddlManager) cleanCache(msg string) { | ||
| tableName := m.executingDDL.TableInfo.TableName | ||
| log.Info(msg, zap.String("ddl", m.executingDDL.Query), | ||
| zap.String("namespace", m.changfeedID.Namespace), | ||
| zap.String("changefeed", m.changfeedID.ID), | ||
| zap.String("bdrRole", m.executingDDL.BDRRole), | ||
| zap.Any("ddlEvent", m.executingDDL)) | ||
|
|
||
| // Set it to nil first to accelerate GC. | ||
| m.pendingDDLs[tableName][0] = nil | ||
| m.pendingDDLs[tableName] = m.pendingDDLs[tableName][1:] | ||
| m.schema.DoGC(m.executingDDL.CommitTs - 1) | ||
| m.justSentDDL = m.executingDDL | ||
| m.executingDDL = nil | ||
|
|
||
| m.tableInfoCache = nil | ||
| m.physicalTablesCache = nil | ||
| } | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the second returned value looks used to print log, can we just print the log here, instead of return it ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If redo is enabled, shouldSkipDDL will be called twice, so we use the return value to avoid printing duplicate logs.