From 24e959c25945c99369f8e44a0d8716bf045452b0 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 4 Nov 2020 17:50:34 +0800 Subject: [PATCH 01/59] support TLS in relay --- dm/worker/relay.go | 52 +++------------------- relay/config.go | 47 +++++++++++++------- relay/relay.go | 103 +++++++++++++++++++++++--------------------- relay/relay_test.go | 12 ++---- 4 files changed, 95 insertions(+), 119 deletions(-) diff --git a/dm/worker/relay.go b/dm/worker/relay.go index 73f6b4455d..1c3881ed3a 100644 --- a/dm/worker/relay.go +++ b/dm/worker/relay.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/relay" "github.com/pingcap/dm/relay/purger" - rr "github.com/pingcap/dm/relay/retry" ) // RelayHolder for relay unit @@ -77,36 +76,13 @@ type realRelayHolder struct { } // NewRealRelayHolder creates a new RelayHolder -func NewRealRelayHolder(cfg *config.SourceConfig) RelayHolder { - clone := cfg.DecryptPassword() - relayCfg := &relay.Config{ - EnableGTID: clone.EnableGTID, - AutoFixGTID: clone.AutoFixGTID, - Flavor: clone.Flavor, - RelayDir: clone.RelayDir, - ServerID: clone.ServerID, - Charset: clone.Charset, - From: relay.DBConfig{ - Host: clone.From.Host, - Port: clone.From.Port, - User: clone.From.User, - Password: clone.From.Password, - }, - BinLogName: clone.RelayBinLogName, - BinlogGTID: clone.RelayBinlogGTID, - ReaderRetry: rr.ReaderRetryConfig{ // we use config from TaskChecker now - BackoffRollback: cfg.Checker.BackoffRollback.Duration, - BackoffMax: cfg.Checker.BackoffMax.Duration, - BackoffMin: cfg.Checker.BackoffMin.Duration, - BackoffJitter: cfg.Checker.BackoffJitter, - BackoffFactor: cfg.Checker.BackoffFactor, - }, - } +func NewRealRelayHolder(sourceCfg *config.SourceConfig) RelayHolder { + cfg := relay.FromSourceCfg(sourceCfg) h := &realRelayHolder{ - cfg: cfg, + cfg: sourceCfg, stage: pb.Stage_New, - relay: relay.NewRelay(relayCfg), + relay: relay.NewRelay(cfg), l: log.With(zap.String("component", "relay holder")), } h.closed.Set(closedTrue) @@ -307,24 +283,8 @@ func (h *realRelayHolder) Result() *pb.ProcessResult { } // Update update relay config online -func (h *realRelayHolder) Update(ctx context.Context, cfg *config.SourceConfig) error { - relayCfg := &relay.Config{ - AutoFixGTID: cfg.AutoFixGTID, - Charset: cfg.Charset, - From: relay.DBConfig{ - Host: cfg.From.Host, - Port: cfg.From.Port, - User: cfg.From.User, - Password: cfg.From.Password, - }, - ReaderRetry: rr.ReaderRetryConfig{ // we use config from TaskChecker now - BackoffRollback: cfg.Checker.BackoffRollback.Duration, - BackoffMax: cfg.Checker.BackoffMax.Duration, - BackoffMin: cfg.Checker.BackoffMin.Duration, - BackoffJitter: cfg.Checker.BackoffJitter, - BackoffFactor: cfg.Checker.BackoffFactor, - }, - } +func (h *realRelayHolder) Update(ctx context.Context, sourceCfg *config.SourceConfig) error { + relayCfg := relay.FromSourceCfg(sourceCfg) stage := h.Stage() diff --git a/relay/config.go b/relay/config.go index d982ba81fc..ce4c84c8d2 100644 --- a/relay/config.go +++ b/relay/config.go @@ -16,19 +16,20 @@ package relay import ( "encoding/json" + "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/relay/retry" ) // Config is the configuration for Relay. type Config struct { - EnableGTID bool `toml:"enable-gtid" json:"enable-gtid"` - AutoFixGTID bool `toml:"auto-fix-gtid" json:"auto-fix-gtid"` - RelayDir string `toml:"relay-dir" json:"relay-dir"` - ServerID uint32 `toml:"server-id" json:"server-id"` - Flavor string `toml:"flavor" json:"flavor"` - Charset string `toml:"charset" json:"charset"` - From DBConfig `toml:"data-source" json:"data-source"` + EnableGTID bool `toml:"enable-gtid" json:"enable-gtid"` + AutoFixGTID bool `toml:"auto-fix-gtid" json:"auto-fix-gtid"` + RelayDir string `toml:"relay-dir" json:"relay-dir"` + ServerID uint32 `toml:"server-id" json:"server-id"` + Flavor string `toml:"flavor" json:"flavor"` + Charset string `toml:"charset" json:"charset"` + From config.DBConfig `toml:"data-source" json:"data-source"` // synchronous start point (if no meta saved before) // do not need to specify binlog-pos, because relay will fetch the whole file @@ -39,14 +40,6 @@ type Config struct { ReaderRetry retry.ReaderRetryConfig `toml:"reader-retry" json:"reader-retry"` } -// DBConfig is the DB configuration. -type DBConfig struct { - Host string `toml:"host" json:"host"` - User string `toml:"user" json:"user"` - Password string `toml:"password" json:"-"` // omit it for privacy - Port int `toml:"port" json:"port"` -} - func (c *Config) String() string { cfg, err := json.Marshal(c) if err != nil { @@ -54,3 +47,27 @@ func (c *Config) String() string { } return string(cfg) } + +// FromSourceCfg gen relay config from source config +func FromSourceCfg(sourceCfg *config.SourceConfig) *Config { + clone := sourceCfg.DecryptPassword() + cfg := &Config{ + EnableGTID: clone.EnableGTID, + AutoFixGTID: clone.AutoFixGTID, + Flavor: clone.Flavor, + RelayDir: clone.RelayDir, + ServerID: clone.ServerID, + Charset: clone.Charset, + From: clone.From, + BinLogName: clone.RelayBinLogName, + BinlogGTID: clone.RelayBinlogGTID, + ReaderRetry: retry.ReaderRetryConfig{ // we use config from TaskChecker now + BackoffRollback: clone.Checker.BackoffRollback.Duration, + BackoffMax: clone.Checker.BackoffMax.Duration, + BackoffMin: clone.Checker.BackoffMin.Duration, + BackoffJitter: clone.Checker.BackoffJitter, + BackoffFactor: clone.Checker.BackoffFactor, + }, + } + return cfg +} diff --git a/relay/relay.go b/relay/relay.go index 6ee5a24973..bad5d081f2 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -15,6 +15,7 @@ package relay import ( "context" + "crypto/tls" "database/sql" "fmt" "os" @@ -36,6 +37,7 @@ import ( "github.com/pingcap/dm/dm/unit" "github.com/pingcap/dm/pkg/binlog" "github.com/pingcap/dm/pkg/binlog/common" + "github.com/pingcap/dm/pkg/conn" tcontext "github.com/pingcap/dm/pkg/context" fr "github.com/pingcap/dm/pkg/func-rollback" "github.com/pingcap/dm/pkg/gtid" @@ -47,6 +49,7 @@ import ( "github.com/pingcap/dm/relay/retry" "github.com/pingcap/dm/relay/transformer" "github.com/pingcap/dm/relay/writer" + toolutils "github.com/pingcap/tidb-tools/pkg/utils" ) var ( @@ -113,34 +116,10 @@ type Relay struct { // NewRealRelay creates an instance of Relay. func NewRealRelay(cfg *Config) Process { - syncerCfg := replication.BinlogSyncerConfig{ - ServerID: uint32(cfg.ServerID), - Flavor: cfg.Flavor, - Host: cfg.From.Host, - Port: uint16(cfg.From.Port), - User: cfg.From.User, - Password: cfg.From.Password, - Charset: cfg.Charset, - } - common.SetDefaultReplicationCfg(&syncerCfg, common.MaxBinlogSyncerReconnect) - - if !cfg.EnableGTID { - // for rawMode(true), we only parse FormatDescriptionEvent and RotateEvent - // if not need to support GTID mode, we can enable rawMode - syncerCfg.RawModeEnabled = true - } - - if cfg.Flavor == mysql.MariaDBFlavor { - // ref: https://mariadb.com/kb/en/library/annotate_rows_log_event/#slave-option-replicate-annotate-row-events - // ref: https://github.com/MariaDB/server/blob/bf71d263621c90cbddc7bde9bf071dae503f333f/sql/sql_repl.cc#L1809 - syncerCfg.DumpCommandFlag |= dumpFlagSendAnnotateRowsEvent - } - return &Relay{ - cfg: cfg, - syncerCfg: syncerCfg, - meta: NewLocalMeta(cfg.Flavor, cfg.RelayDir), - tctx: tcontext.Background().WithLogger(log.With(zap.String("component", "relay log"))), + cfg: cfg, + meta: NewLocalMeta(cfg.Flavor, cfg.RelayDir), + tctx: tcontext.Background().WithLogger(log.With(zap.String("component", "relay log"))), } } @@ -153,13 +132,17 @@ func (r *Relay) Init(ctx context.Context) (err error) { } }() - cfg := r.cfg.From - dbDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=utf8mb4&interpolateParams=true&readTimeout=%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, showStatusConnectionTimeout) - db, err := sql.Open("mysql", dbDSN) + err = r.setSyncConfig() if err != nil { - return terror.WithScope(terror.DBErrorAdapt(err, terror.ErrDBDriverError), terror.ScopeUpstream) + return err } - r.db = db + + db, err := conn.DefaultDBProvider.Apply(r.cfg.From) + if err != nil { + return terror.WithScope(err, terror.ScopeUpstream) + } + + r.db = db.DB rollbackHolder.Add(fr.FuncRollback{Name: "close-DB", Fn: r.closeDB}) if err2 := os.MkdirAll(r.cfg.RelayDir, 0755); err2 != nil { @@ -790,25 +773,10 @@ func (r *Relay) Reload(newCfg *Config) error { } r.db = db - syncerCfg := replication.BinlogSyncerConfig{ - ServerID: uint32(r.cfg.ServerID), - Flavor: r.cfg.Flavor, - Host: newCfg.From.Host, - Port: uint16(newCfg.From.Port), - User: newCfg.From.User, - Password: newCfg.From.Password, - Charset: newCfg.Charset, - } - common.SetDefaultReplicationCfg(&syncerCfg, common.MaxBinlogSyncerReconnect) - - if !newCfg.EnableGTID { - // for rawMode(true), we only parse FormatDescriptionEvent and RotateEvent - // if not need to support GTID mode, we can enable rawMode - syncerCfg.RawModeEnabled = true + if err := r.setSyncConfig(); err != nil { + return err } - r.syncerCfg = syncerCfg - r.tctx.L().Info("relay unit is updated") return nil @@ -835,3 +803,40 @@ func (r *Relay) ActiveRelayLog() *pkgstreamer.RelayLogInfo { defer r.activeRelayLog.RUnlock() return r.activeRelayLog.info } + +func (r *Relay) setSyncConfig() error { + var tlsConfig *tls.Config + var err error + if r.cfg.From.Security != nil { + tlsConfig, err = toolutils.ToTLSConfig(r.cfg.From.Security.SSLCA, r.cfg.From.Security.SSLCert, r.cfg.From.Security.SSLKey) + if err != nil { + return terror.ErrConnInvalidTLSConfig.Delegate(err) + } + if tlsConfig != nil { + tlsConfig.InsecureSkipVerify = true + } + } + + syncerCfg := replication.BinlogSyncerConfig{ + ServerID: uint32(r.cfg.ServerID), + Flavor: r.cfg.Flavor, + Host: r.cfg.From.Host, + Port: uint16(r.cfg.From.Port), + User: r.cfg.From.User, + Password: r.cfg.From.Password, + Charset: r.cfg.Charset, + TLSConfig: tlsConfig, + } + common.SetDefaultReplicationCfg(&syncerCfg, common.MaxBinlogSyncerReconnect) + + if !r.cfg.EnableGTID { + syncerCfg.RawModeEnabled = true + } + + if r.cfg.Flavor == mysql.MariaDBFlavor { + syncerCfg.DumpCommandFlag |= dumpFlagSendAnnotateRowsEvent + } + + r.syncerCfg = syncerCfg + return nil +} diff --git a/relay/relay_test.go b/relay/relay_test.go index 56b961dd64..e06486bc1b 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -525,7 +525,7 @@ func (t *testRelaySuite) TestProcess(c *C) { Flavor: gmysql.MySQLFlavor, RelayDir: c.MkDir(), ServerID: 12321, - From: DBConfig{ + From: config.DBConfig{ Host: dbCfg.Host, Port: dbCfg.Port, User: dbCfg.User, @@ -541,16 +541,10 @@ func (t *testRelaySuite) TestProcess(c *C) { } r = NewRelay(relayCfg).(*Relay) ) - db, err := openDBForTest() - c.Assert(err, IsNil) - r.db = db - defer func() { - r.db.Close() - r.db = nil - }() - ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup + err := r.Init(ctx) + c.Assert(err, IsNil) wg.Add(1) go func() { defer wg.Done() From c62da6bfd7bf451f37aac9f278c8bee87159521a Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 9 Nov 2020 17:29:25 +0800 Subject: [PATCH 02/59] support gtid in relay --- _utils/terror_gen/errors_release.txt | 2 + errors.toml | 12 ++ pkg/streamer/reader.go | 284 ++++++++++++++++++++++++++- pkg/streamer/reader_test.go | 12 +- pkg/terror/error_list.go | 6 + relay/relay.go | 6 - syncer/streamer_controller.go | 10 +- 7 files changed, 307 insertions(+), 25 deletions(-) diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt index cd004450f1..d4bfd73fbf 100644 --- a/_utils/terror_gen/errors_release.txt +++ b/_utils/terror_gen/errors_release.txt @@ -45,6 +45,8 @@ ErrParserParseRelayLog,[code=11038:class=functional:scope=internal:level=high], ErrNoSubdirToSwitch,[code=11039:class=functional:scope=internal:level=high], "Message: parse for previous sub relay directory finished, but no next sub directory need to switch" ErrNeedSyncAgain,[code=11040:class=functional:scope=internal:level=high], "Message: Last sync error or closed, try sync and get event again" ErrSyncClosed,[code=11041:class=functional:scope=internal:level=high], "Message: Sync was closed" +ErrNoUUIDDirMatchGTID,[code=11120:class=functional:scope=internal:level=high], "Message: no relay subdir match gtid %s" +ErrNoRelayPosMatchGTID,[code=11121:class=functional:scope=internal:level=high], "Message: no relay pos match gtid %s" ErrSchemaTableNameNotValid,[code=11042:class=functional:scope=internal:level=high], "Message: table name %s not valid" ErrGenTableRouter,[code=11043:class=functional:scope=internal:level=high], "Message: generate table router, Workaround: Please check `routes` config in task configuration file." ErrEncryptSecretKeyNotValid,[code=11044:class=functional:scope=internal:level=high], "Message: key size should be 16, 24 or 32, but input key's size is %d" diff --git a/errors.toml b/errors.toml index cc5762d951..285d18be47 100644 --- a/errors.toml +++ b/errors.toml @@ -748,6 +748,18 @@ description = "" workaround = "" tags = ["internal", "high"] +[error.DM-functional-11120] +message = "no relay subdir match gtid %s" +description = "" +workaround = "" +tags = ["internal", "high"] + +[error.DM-functional-11121] +message = "no relay pos match gtid %s" +description = "" +workaround = "" +tags = ["internal", "high"] + [error.DM-config-20001] message = "checking item %s is not supported\n%s" description = "" diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index c5a9beccd9..5fbeb6eb73 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -15,6 +15,7 @@ package streamer import ( "context" + "fmt" "os" "path" "path/filepath" @@ -22,7 +23,9 @@ import ( "sync" "time" + "github.com/BurntSushi/toml" "github.com/pingcap/errors" + uuid "github.com/satori/go.uuid" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "go.uber.org/zap" @@ -34,6 +37,13 @@ import ( "github.com/pingcap/dm/pkg/utils" ) +// Meta represents binlog meta information in relay.meta +type Meta struct { + BinLogName string `toml:"binlog-name" json:"binlog-name"` + BinLogPos uint32 `toml:"binlog-pos" json:"binlog-pos"` + BinlogGTID string `toml:"binlog-gtid" json:"binlog-gtid"` +} + var ( // polling interval for watcher watcherInterval = 100 * time.Millisecond @@ -43,6 +53,7 @@ var ( type BinlogReaderConfig struct { RelayDir string Timezone *time.Location + Flavor string } // BinlogReader is a binlog reader. @@ -60,6 +71,8 @@ type BinlogReader struct { cancel context.CancelFunc tctx *tcontext.Context + + prevGset, currGset mysql.GTIDSet } // NewBinlogReader creates a new BinlogReader @@ -103,9 +116,172 @@ func (r *BinlogReader) checkRelayPos(pos mysql.Position) error { return nil } -// StartSync start syncon +// getUUIDByGTID gets uuid subdir which contain the gtid set +func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { + for _, uuid := range r.uuids { + filename := path.Join(r.cfg.RelayDir, uuid, utils.MetaFilename) + fd, err := os.Open(filename) + if err != nil { + return "", terror.ErrRelayLoadMetaData.Delegate(err) + } + defer fd.Close() + + var meta Meta + _, err = toml.DecodeReader(fd, &meta) + if err != nil { + return "", terror.ErrRelayLoadMetaData.Delegate(err) + } + + gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, meta.BinlogGTID) + if err != nil { + return "", terror.ErrRelayLoadMetaData.Delegate(err) + } + if gs.Contain(gset) { + r.tctx.L().Info("get uuid subdir by gtid", zap.Stringer("GTID Set", gset), zap.String("uuid", uuid)) + return uuid, nil + } + } + + // TODO: use a better mechanism to call relay.meta.Flush + // use the newest uuid subdir + if len(r.uuids) > 0 { + return r.uuids[len(r.uuids)-1], nil + } + return "", terror.ErrNoUUIDDirMatchGTID.Generate(gset.String()) +} + +// GetFilePosByGTID tries to get Pos by GTID for file +func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet) (uint32, error) { + s := newLocalStreamer() + + ctx2, cancel := context.WithCancel(ctx) + defer cancel() + + r.wg.Add(1) + go func() { + defer r.wg.Done() + + offset := int64(4) + onEventFunc := func(e *replication.BinlogEvent) error { + offset = int64(e.Header.LogPos) + + select { + case s.ch <- e: + case <-ctx2.Done(): + } + return nil + } + + err := r.parser.ParseFile(filePath, offset, onEventFunc) + if err != nil { + r.tctx.L().Error("parse file stopped", zap.Error(err)) + } + // reach end of file + s.ch <- nil + }() + + lastPos := uint32(0) + for { + select { + case <-ctx.Done(): + return 0, nil + default: + } + + e, err := s.GetEvent(ctx) + if err != nil { + return 0, err + } + // gset is the last txn in file + if e == nil { + return lastPos, nil + } + switch ev := e.Event.(type) { + case *replication.PreviousGTIDsEvent: + gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, ev.GTIDSets) + if err != nil { + return 0, err + } + // if PreviousGITDsEvent contain the gset, go to previous file + if gs.Contain(gset) { + return 0, nil + } + case *replication.RotateEvent: + if e.Header.Timestamp != 0 && e.Header.LogPos != 0 { + // gset is the last txn in file + return lastPos, nil + } + case *replication.XIDEvent: + lastPos = e.Header.LogPos + case *replication.QueryEvent: + lastPos = e.Header.LogPos + case *replication.GTIDEvent: + u, _ := uuid.FromBytes(ev.SID) + gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, fmt.Sprintf("%s:%d", u.String(), ev.GNO)) + if err != nil { + return 0, err + } + // meet first gtid event greater than gset + if !gset.Contain(gs) { + return lastPos, nil + } + case *replication.MariadbGTIDEvent: + GTID := ev.GTID + gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, fmt.Sprintf("%d-%d-%d", GTID.DomainID, GTID.ServerID, GTID.SequenceNumber)) + if err != nil { + return 0, err + } + // meet first gtid event greater than gset + if !gset.Contain(gs) { + return lastPos, nil + } + default: + } + } +} + +// getPosByGTID gets position by gtid +func (r *BinlogReader) getPosByGTID(gset mysql.GTIDSet) (*mysql.Position, error) { + uuid, err := r.getUUIDByGTID(gset) + if err != nil { + return nil, err + } + _, suffix, err := utils.ParseSuffixForUUID(uuid) + if err != nil { + return nil, err + } + + uuidDir := path.Join(r.cfg.RelayDir, uuid) + allFiles, err := CollectAllBinlogFiles(uuidDir) + if err != nil { + return nil, err + } + + // iterate files from the newest one + for i := len(allFiles) - 1; i >= 0; i-- { + file := allFiles[i] + filePath := path.Join(uuidDir, file) + pos, err := r.GetFilePosByGTID(r.tctx.Ctx, filePath, gset) + if err != nil { + return nil, err + } + if pos != 0 { + fileName, err := binlog.ParseFilename(file) + if err != nil { + return nil, err + } + return &mysql.Position{ + Name: binlog.ConstructFilenameWithUUIDSuffix(fileName, utils.SuffixIntToStr(suffix)), + Pos: uint32(pos), + }, nil + } + } + return nil, terror.ErrNoRelayPosMatchGTID.Generate(gset.String()) +} + +// StartSyncByPos start sync by pos // TODO: thread-safe? -func (r *BinlogReader) StartSync(pos mysql.Position) (Streamer, error) { +func (r *BinlogReader) StartSyncByPos(pos mysql.Position) (Streamer, error) { if pos.Name == "" { return nil, terror.ErrBinlogFileNotSpecified.Generate() } @@ -144,6 +320,48 @@ func (r *BinlogReader) StartSync(pos mysql.Position) (Streamer, error) { return s, nil } +// StartSyncByGTID start sync by gtid +func (r *BinlogReader) StartSyncByGTID(gset mysql.GTIDSet) (Streamer, error) { + r.tctx.L().Info("begin to sync binlog", zap.Stringer("GTID Set", gset)) + + if r.running { + return nil, terror.ErrReaderAlreadyRunning.Generate() + } + + err := r.updateUUIDs() + if err != nil { + return nil, err + } + + pos, err := r.getPosByGTID(gset) + if err != nil { + return nil, err + } + r.tctx.L().Info("get pos by gtid", zap.Stringer("GTID Set", gset), zap.Stringer("Position", pos)) + + r.prevGset = gset + r.currGset = nil + + r.latestServerID = 0 + r.running = true + s := newLocalStreamer() + + r.wg.Add(1) + go func() { + defer r.wg.Done() + r.tctx.L().Info("start reading", zap.Stringer("position", pos)) + err = r.parseRelay(r.tctx.Context(), s, *pos) + if errors.Cause(err) == r.tctx.Context().Err() { + r.tctx.L().Warn("parse relay finished", log.ShortError(err)) + } else if err != nil { + s.closeWithError(err) + r.tctx.L().Error("parse relay stopped", zap.Error(err)) + } + }() + + return s, nil +} + // parseRelay parses relay root directory, it support master-slave switch (switching to next sub directory) func (r *BinlogReader) parseRelay(ctx context.Context, s *LocalStreamer, pos mysql.Position) error { var ( @@ -292,15 +510,14 @@ func (r *BinlogReader) parseFile( r.tctx.L().Debug("read event", zap.Reflect("header", e.Header)) r.latestServerID = e.Header.ServerID // record server_id - switch e.Header.EventType { - case replication.FORMAT_DESCRIPTION_EVENT: + switch ev := e.Event.(type) { + case *replication.FormatDescriptionEvent: // ignore FORMAT_DESCRIPTION event, because go-mysql will send this fake event - case replication.ROTATE_EVENT: + case *replication.RotateEvent: // add master UUID suffix to pos.Name - env := e.Event.(*replication.RotateEvent) - parsed, _ := binlog.ParseFilename(string(env.NextLogName)) + parsed, _ := binlog.ParseFilename(string(ev.NextLogName)) nameWithSuffix := binlog.ConstructFilenameWithUUIDSuffix(parsed, uuidSuffix) - env.NextLogName = []byte(nameWithSuffix) + ev.NextLogName = []byte(nameWithSuffix) if e.Header.Timestamp != 0 && e.Header.LogPos != 0 { // not fake rotate event, update file pos @@ -314,10 +531,36 @@ func (r *BinlogReader) parseFile( // and we *try* to switch to the next when `needReParse` is false. // so this `currentPos` only used for log now. currentPos := mysql.Position{ - Name: string(env.NextLogName), - Pos: uint32(env.Position), + Name: string(ev.NextLogName), + Pos: uint32(ev.Position), } r.tctx.L().Info("rotate binlog", zap.Stringer("position", currentPos)) + case *replication.GTIDEvent: + if r.prevGset == nil { + break + } + u, _ := uuid.FromBytes(ev.SID) + err2 := r.advanceCurrentGtidSet(fmt.Sprintf("%s:%d", u.String(), ev.GNO)) + if err2 != nil { + return errors.Trace(err2) + } + latestPos = int64(e.Header.LogPos) + case *replication.MariadbGTIDEvent: + if r.prevGset == nil { + break + } + GTID := ev.GTID + err2 := r.advanceCurrentGtidSet(fmt.Sprintf("%d-%d-%d", GTID.DomainID, GTID.ServerID, GTID.SequenceNumber)) + if err2 != nil { + return errors.Trace(err2) + } + latestPos = int64(e.Header.LogPos) + case *replication.XIDEvent: + ev.GSet = r.getCurrentGtidSet() + latestPos = int64(e.Header.LogPos) + case *replication.QueryEvent: + ev.GSet = r.getCurrentGtidSet() + latestPos = int64(e.Header.LogPos) default: // update file pos latestPos = int64(e.Header.LogPos) @@ -419,3 +662,24 @@ func (r *BinlogReader) GetUUIDs() []string { uuids = append(uuids, r.uuids...) return uuids } + +func (r *BinlogReader) getCurrentGtidSet() mysql.GTIDSet { + if r.currGset == nil { + return nil + } + return r.currGset.Clone() +} + +func (r *BinlogReader) advanceCurrentGtidSet(gtid string) error { + if r.currGset == nil { + r.currGset = r.prevGset.Clone() + } + prev := r.currGset.Clone() + err := r.currGset.Update(gtid) + if err == nil { + if !r.currGset.Equal(prev) { + r.prevGset = prev + } + } + return err +} diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 26aca4ee0c..486f405d5b 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -522,7 +522,7 @@ func (t *testReaderSuite) TestStartSync(c *C) { } // start the reader - s, err := r.StartSync(startPos) + s, err := r.StartSyncByPos(startPos) c.Assert(err, IsNil) // get events from the streamer @@ -620,12 +620,12 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(err, ErrorMatches, ".*empty UUIDs not valid.*") // no startup pos specified - s, err := r.StartSync(gmysql.Position{}) + s, err := r.StartSyncByPos(gmysql.Position{}) c.Assert(terror.ErrBinlogFileNotSpecified.Equal(err), IsTrue) c.Assert(s, IsNil) // empty UUIDs - s, err = r.StartSync(startPos) + s, err = r.StartSyncByPos(startPos) c.Assert(err, ErrorMatches, ".*empty UUIDs not valid.*") c.Assert(s, IsNil) @@ -636,13 +636,13 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(err, IsNil) // the startup relay log file not found - s, err = r.StartSync(startPos) + s, err = r.StartSyncByPos(startPos) c.Assert(err, ErrorMatches, fmt.Sprintf(".*%s.*not found.*", startPos.Name)) c.Assert(s, IsNil) // can not re-start the reader r.running = true - s, err = r.StartSync(startPos) + s, err = r.StartSyncByPos(startPos) c.Assert(terror.ErrReaderAlreadyRunning.Equal(err), IsTrue) c.Assert(s, IsNil) r.Close() @@ -656,7 +656,7 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { err = ioutil.WriteFile(relayLogFilePath, make([]byte, 100), 0600) c.Assert(err, IsNil) startPos.Pos = 10000 - s, err = r.StartSync(startPos) + s, err = r.StartSyncByPos(startPos) c.Assert(terror.ErrRelayLogGivenPosTooBig.Equal(err), IsTrue) c.Assert(s, IsNil) } diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index f63f7f2564..e3ff900f6e 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -174,6 +174,10 @@ const ( // pkg/parser codeRewriteSQL + + // pkg/streamer + codeNoUUIDDirMatchGTID + codeNoRelayPosMatchGTID ) // Config related error code list @@ -653,6 +657,8 @@ var ( ErrNoSubdirToSwitch = New(codeNoSubdirToSwitch, ClassFunctional, ScopeInternal, LevelHigh, "parse for previous sub relay directory finished, but no next sub directory need to switch", "") ErrNeedSyncAgain = New(codeNeedSyncAgain, ClassFunctional, ScopeInternal, LevelHigh, "Last sync error or closed, try sync and get event again", "") ErrSyncClosed = New(codeSyncClosed, ClassFunctional, ScopeInternal, LevelHigh, "Sync was closed", "") + ErrNoUUIDDirMatchGTID = New(codeNoUUIDDirMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay subdir match gtid %s", "") + ErrNoRelayPosMatchGTID = New(codeNoRelayPosMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay pos match gtid %s", "") // pkg/utils ErrSchemaTableNameNotValid = New(codeSchemaTableNameNotValid, ClassFunctional, ScopeInternal, LevelHigh, "table name %s not valid", "") ErrGenTableRouter = New(codeGenTableRouter, ClassFunctional, ScopeInternal, LevelHigh, "generate table router", "Please check `routes` config in task configuration file.") diff --git a/relay/relay.go b/relay/relay.go index bad5d081f2..5e08e99b1a 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -194,12 +194,6 @@ func (r *Relay) process(parentCtx context.Context) error { } if isNew { - // purge old relay log - err = r.purgeRelayDir() - if err != nil { - return err - } - // re-setup meta for new server err = r.reSetupMeta() if err != nil { diff --git a/syncer/streamer_controller.go b/syncer/streamer_controller.go index 587e455eaf..1f6217350f 100644 --- a/syncer/streamer_controller.go +++ b/syncer/streamer_controller.go @@ -46,11 +46,15 @@ type StreamerProducer interface { // Read local relay log type localBinlogReader struct { - reader *streamer.BinlogReader + reader *streamer.BinlogReader + EnableGTID bool } func (l *localBinlogReader) generateStreamer(location binlog.Location) (streamer.Streamer, error) { - return l.reader.StartSync(location.Position) + if l.EnableGTID { + return l.reader.StartSyncByGTID(location.GetGTID().Origin().Clone()) + } + return l.reader.StartSyncByPos(location.Position) } // Read remote binlog @@ -199,7 +203,7 @@ func (c *StreamerController) resetReplicationSyncer(tctx *tcontext.Context, loca if c.currentBinlogType == RemoteBinlog { c.streamerProducer = &remoteBinlogReader{replication.NewBinlogSyncer(c.syncCfg), tctx, c.syncCfg.Flavor, c.enableGTID} } else { - c.streamerProducer = &localBinlogReader{streamer.NewBinlogReader(tctx, &streamer.BinlogReaderConfig{RelayDir: c.localBinlogDir, Timezone: c.timezone})} + c.streamerProducer = &localBinlogReader{streamer.NewBinlogReader(tctx, &streamer.BinlogReaderConfig{RelayDir: c.localBinlogDir, Timezone: c.timezone, Flavor: c.syncCfg.Flavor}), c.enableGTID} } c.streamer, err = c.streamerProducer.generateStreamer(location) From ec050eefeb397869b3b98e5c5f250ad0739bc141 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 9 Nov 2020 17:47:50 +0800 Subject: [PATCH 03/59] add relay hub for reader --- dm/worker/relay_test.go | 7 ++++++ pkg/streamer/hub.go | 53 +++++++++++++++++++++++++++++++++++++++-- pkg/streamer/reader.go | 33 ++++++++++++++++++++----- relay/relay.go | 34 ++++++++++++++++++++++---- relay/relay_test.go | 6 ++--- 5 files changed, 117 insertions(+), 16 deletions(-) diff --git a/dm/worker/relay_test.go b/dm/worker/relay_test.go index cf3b3a7d70..9f392d71e6 100644 --- a/dm/worker/relay_test.go +++ b/dm/worker/relay_test.go @@ -19,10 +19,12 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/siddontang/go-mysql/mysql" "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/dm/unit" + "github.com/pingcap/dm/pkg/gtid" pkgstreamer "github.com/pingcap/dm/pkg/streamer" "github.com/pingcap/dm/pkg/utils" "github.com/pingcap/dm/relay" @@ -114,6 +116,11 @@ func (d *DummyRelay) Close() {} // IsClosed implements Process interface func (d *DummyRelay) IsClosed() bool { return false } +// SaveMeta implements Process interface +func (d *DummyRelay) SaveMeta(pos mysql.Position, gset gtid.Set) error { + return nil +} + func (t *testRelay) TestRelay(c *C) { originNewRelay := relay.NewRelay relay.NewRelay = NewDummyRelay diff --git a/pkg/streamer/hub.go b/pkg/streamer/hub.go index 994f270f40..acd58100dc 100644 --- a/pkg/streamer/hub.go +++ b/pkg/streamer/hub.go @@ -19,13 +19,17 @@ import ( "sync" "github.com/pingcap/dm/pkg/binlog" + "github.com/pingcap/dm/pkg/gtid" "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" + "github.com/siddontang/go-mysql/mysql" ) var ( - readerHub *ReaderHub // singleton instance - once sync.Once + readerHub *ReaderHub // singleton instance + relayMetaHub *RelayMetaHub + relayMetaOnce sync.Once + once sync.Once ) // RelayLogInfo represents information for relay log @@ -137,3 +141,48 @@ func (h *ReaderHub) EarliestActiveRelayLog() *RelayLogInfo { _, rli := h.rlih.earliest() return rli } + +// RelayMetaHub holds information for relay metas +type RelayMetaHub struct { + mu sync.RWMutex + meta Meta +} + +// GetRelayMetaHub gets singleton instance of RelayMetaHub +func GetRelayMetaHub() *RelayMetaHub { + relayMetaOnce.Do(func() { + relayMetaHub = &RelayMetaHub{} + }) + return relayMetaHub +} + +// GetMeta gets all metas +func (r *RelayMetaHub) GetMeta() Meta { + r.mu.Lock() + defer r.mu.Unlock() + return r.meta +} + +// SetMeta sets meta +func (r *RelayMetaHub) SetMeta(uuid string, pos mysql.Position, gset gtid.Set) { + gs := "" + if gset != nil { + gs = gset.String() + } + meta := Meta{ + BinLogPos: pos.Pos, + BinLogName: pos.Name, + BinlogGTID: gs, + UUID: uuid, + } + r.mu.Lock() + defer r.mu.Unlock() + r.meta = meta +} + +// ClearMeta clears meta +func (r *RelayMetaHub) ClearMeta() { + r.mu.Lock() + defer r.mu.Unlock() + r.meta = Meta{} +} diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index 5fbeb6eb73..1e03e51702 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -42,6 +42,7 @@ type Meta struct { BinLogName string `toml:"binlog-name" json:"binlog-name"` BinLogPos uint32 `toml:"binlog-pos" json:"binlog-pos"` BinlogGTID string `toml:"binlog-gtid" json:"binlog-gtid"` + UUID string `toml:"-" json:"-"` } var ( @@ -118,6 +119,12 @@ func (r *BinlogReader) checkRelayPos(pos mysql.Position) error { // getUUIDByGTID gets uuid subdir which contain the gtid set func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { + // TODO: use a better mechanism to call relay.meta.Flush + // get the meta save in memory + relayMetaHub := GetRelayMetaHub() + meta := relayMetaHub.GetMeta() + + // get flush logs from oldest to newest for _, uuid := range r.uuids { filename := path.Join(r.cfg.RelayDir, uuid, utils.MetaFilename) fd, err := os.Open(filename) @@ -142,10 +149,16 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { } } - // TODO: use a better mechanism to call relay.meta.Flush - // use the newest uuid subdir - if len(r.uuids) > 0 { - return r.uuids[len(r.uuids)-1], nil + // use memory meta + if len(meta.UUID) > 0 { + gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, meta.BinlogGTID) + if err != nil { + return "", terror.ErrRelayLoadMetaData.Delegate(err) + } + if gs.Contain(gset) { + r.tctx.L().Info("get uuid subdir by gtid", zap.Stringer("GTID Set", gset), zap.String("uuid", meta.UUID)) + return meta.UUID, nil + } } return "", terror.ErrNoUUIDDirMatchGTID.Generate(gset.String()) } @@ -175,9 +188,17 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs err := r.parser.ParseFile(filePath, offset, onEventFunc) if err != nil { r.tctx.L().Error("parse file stopped", zap.Error(err)) + select { + case s.ech <- err: + case <-ctx2.Done(): + } + } else { + // reach end of file + select { + case s.ch <- nil: + case <-ctx2.Done(): + } } - // reach end of file - s.ch <- nil }() lastPos := uint32(0) diff --git a/relay/relay.go b/relay/relay.go index 5e08e99b1a..d8b6c71516 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -94,6 +94,8 @@ type Process interface { Close() // IsClosed returns whether relay log process unit was closed IsClosed() bool + // SaveMeta save relay meta + SaveMeta(pos mysql.Position, gset gtid.Set) error } // Relay relays mysql binlog to local file. @@ -112,6 +114,8 @@ type Relay struct { sync.RWMutex info *pkgstreamer.RelayLogInfo } + + relayMetaHub *pkgstreamer.RelayMetaHub } // NewRealRelay creates an instance of Relay. @@ -154,6 +158,8 @@ func (r *Relay) Init(ctx context.Context) (err error) { return err } + r.relayMetaHub = pkgstreamer.GetRelayMetaHub() + return reportRelayLogSpaceInBackground(r.cfg.RelayDir) } @@ -353,7 +359,7 @@ func (r *Relay) tryRecoverLatestFile(parser2 *parser.Parser) error { } else if err = latestGTID.Truncate(result.LatestGTIDs); err != nil { return err } - err = r.meta.Save(result.LatestPos, latestGTID) + err = r.SaveMeta(result.LatestPos, latestGTID) if err != nil { return terror.Annotatef(err, "save position %s, GTID sets %v after recovered", result.LatestPos, result.LatestGTIDs) } @@ -479,7 +485,7 @@ func (r *Relay) handleEvents(ctx context.Context, reader2 reader.Reader, transfo } if needSavePos { - err = r.meta.Save(lastPos, lastGTID) + err = r.SaveMeta(lastPos, lastGTID) if err != nil { return terror.Annotatef(err, "save position %s, GTID sets %v into meta", lastPos, lastGTID) } @@ -563,7 +569,7 @@ func (r *Relay) doIntervalOps(ctx context.Context) { select { case <-flushTicker.C: if r.meta.Dirty() { - err := r.meta.Flush() + err := r.FlushMeta() if err != nil { r.tctx.L().Error("flush meta", zap.Error(err)) } else { @@ -646,10 +652,28 @@ func (r *Relay) IsClosed() bool { return r.closed.Get() } +// SaveMeta save relay meta and update meta in RelayLogInfo +func (r *Relay) SaveMeta(pos mysql.Position, gset gtid.Set) error { + if err := r.meta.Save(pos, gset); err != nil { + return err + } + r.relayMetaHub.SetMeta(r.meta.UUID(), pos, gset) + return nil +} + +// FlushMeta flush relay meta and clear all metas in RelayLogInfo +func (r *Relay) FlushMeta() error { + if err := r.meta.Flush(); err != nil { + return err + } + r.relayMetaHub.ClearMeta() + return nil +} + // stopSync stops syncing, now it used by Close and Pause func (r *Relay) stopSync() { - if err := r.meta.Flush(); err != nil { - r.tctx.L().Error("flush checkpoint", zap.Error(err)) + if err := r.FlushMeta(); err != nil { + r.tctx.L().Error("flush meta", zap.Error(err)) } } diff --git a/relay/relay_test.go b/relay/relay_test.go index e06486bc1b..29da166913 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -205,7 +205,7 @@ func (t *testRelaySuite) TestTryRecoverLatestFile(c *C) { // write a greater GTID sets in meta greaterGITDSet, err := gtid.ParserGTID(relayCfg.Flavor, greaterGITDSetStr) c.Assert(err, IsNil) - c.Assert(r.meta.Save(startPos, greaterGITDSet), IsNil) + c.Assert(r.SaveMeta(startPos, greaterGITDSet), IsNil) // invalid data truncated, meta updated c.Assert(r.tryRecoverLatestFile(parser2), IsNil) @@ -217,7 +217,7 @@ func (t *testRelaySuite) TestTryRecoverLatestFile(c *C) { c.Assert(latestGTIDs.Equal(recoverGTIDSet), IsTrue) // verifyMetadata is not enough // no relay log file need to recover - c.Assert(r.meta.Save(minCheckpoint, latestGTIDs), IsNil) + c.Assert(r.SaveMeta(minCheckpoint, latestGTIDs), IsNil) c.Assert(r.tryRecoverLatestFile(parser2), IsNil) _, latestPos = r.meta.Pos() c.Assert(latestPos, DeepEquals, minCheckpoint) @@ -283,7 +283,7 @@ func (t *testRelaySuite) TestTryRecoverMeta(c *C) { f.Close() // recover with the subset of GTIDs (previous GTID set). - c.Assert(r.meta.Save(startPos, previousGTIDSet), IsNil) + c.Assert(r.SaveMeta(startPos, previousGTIDSet), IsNil) c.Assert(r.tryRecoverLatestFile(parser2), IsNil) _, latestPos = r.meta.Pos() c.Assert(latestPos, DeepEquals, gmysql.Position{Name: filename, Pos: g.LatestPos}) From b8b9c7c0d0d164f6a7694ce5070b6e2f2e875248 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 11 Nov 2020 09:47:56 +0800 Subject: [PATCH 04/59] temp commit --- pkg/streamer/reader.go | 2 + pkg/streamer/reader_test.go | 159 +++++++++++++++++++++--------------- 2 files changed, 95 insertions(+), 66 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index 1e03e51702..b750827275 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -558,6 +558,7 @@ func (r *BinlogReader) parseFile( r.tctx.L().Info("rotate binlog", zap.Stringer("position", currentPos)) case *replication.GTIDEvent: if r.prevGset == nil { + latestPos = int64(e.Header.LogPos) break } u, _ := uuid.FromBytes(ev.SID) @@ -568,6 +569,7 @@ func (r *BinlogReader) parseFile( latestPos = int64(e.Header.LogPos) case *replication.MariadbGTIDEvent: if r.prevGset == nil { + latestPos = int64(e.Header.LogPos) break } GTID := ev.GTID diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 486f405d5b..7980b67aca 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -24,15 +24,18 @@ import ( "path/filepath" "strconv" "sync" + "testing" "time" . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/siddontang/go-mysql/mysql" gmysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "github.com/pingcap/dm/pkg/binlog/event" tcontext "github.com/pingcap/dm/pkg/context" + "github.com/pingcap/dm/pkg/gtid" "github.com/pingcap/dm/pkg/terror" ) @@ -41,17 +44,28 @@ var parseFileTimeout = 10 * time.Second var _ = Suite(&testReaderSuite{}) type testReaderSuite struct { + lastPos uint32 + lastGset gtid.Set +} + +func TestReader(t *testing.T) { + TestingT(t) +} + +func (t *testReaderSuite) SetUpSuite(c *C) { + t.lastPos = 0 + t.lastGset, _ = gtid.ParserGTID(mysql.MySQLFlavor, "ba8f633f-1f15-11eb-b1c7-0242ac110002:30") } func (t *testReaderSuite) TestParseFileBase(c *C) { var ( - filename = "test-mysql-bin.000001" - baseDir = c.MkDir() - offset int64 - firstParse = true - possibleLast = false - baseEvents = t.genBinlogEvents(c, 0) - s = newLocalStreamer() + filename = "test-mysql-bin.000001" + baseDir = c.MkDir() + offset int64 + firstParse = true + possibleLast = false + baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGset) + s = newLocalStreamer() ) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -227,21 +241,21 @@ func (t *testReaderSuite) TestParseFileBase(c *C) { func (t *testReaderSuite) TestParseFileRelaySubDirUpdated(c *C) { var ( - filename = "test-mysql-bin.000001" - nextFilename = "test-mysql-bin.000002" - baseDir = c.MkDir() - offset int64 - firstParse = true - possibleLast = true - baseEvents = t.genBinlogEvents(c, 0) - currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" - relayDir = filepath.Join(baseDir, currentUUID) - fullPath = filepath.Join(relayDir, filename) - nextPath = filepath.Join(relayDir, nextFilename) - s = newLocalStreamer() - cfg = &BinlogReaderConfig{RelayDir: baseDir} - tctx = tcontext.Background() - r = NewBinlogReader(tctx, cfg) + filename = "test-mysql-bin.000001" + nextFilename = "test-mysql-bin.000002" + baseDir = c.MkDir() + offset int64 + firstParse = true + possibleLast = true + baseEvents, lastPos, lastGTID = t.genBinlogEvents(c, t.lastPos, t.lastGset) + currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" + relayDir = filepath.Join(baseDir, currentUUID) + fullPath = filepath.Join(relayDir, filename) + nextPath = filepath.Join(relayDir, nextFilename) + s = newLocalStreamer() + cfg = &BinlogReaderConfig{RelayDir: baseDir} + tctx = tcontext.Background() + r = NewBinlogReader(tctx, cfg) ) // create the current relay log file and write some events @@ -273,7 +287,7 @@ func (t *testReaderSuite) TestParseFileRelaySubDirUpdated(c *C) { // current relay log file updated, need to re-parse it var wg sync.WaitGroup wg.Add(1) - extraEvents := t.genBinlogEvents(c, baseEvents[len(baseEvents)-1].Header.LogPos) + extraEvents, _, _ := t.genBinlogEvents(c, lastPos, lastGTID) go func() { defer wg.Done() time.Sleep(500 * time.Millisecond) // wait parseFile started @@ -317,23 +331,23 @@ func (t *testReaderSuite) TestParseFileRelaySubDirUpdated(c *C) { func (t *testReaderSuite) TestParseFileRelayNeedSwitchSubDir(c *C) { var ( - filename = "test-mysql-bin.000001" - nextFilename = "test-mysql-bin.666888" - baseDir = c.MkDir() - offset int64 - firstParse = true - possibleLast = true - baseEvents = t.genBinlogEvents(c, 0) - currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" - switchedUUID = "b60868af-5a6f-11e9-9ea3-0242ac160007.000002" - relayDir = filepath.Join(baseDir, currentUUID) - nextRelayDir = filepath.Join(baseDir, switchedUUID) - fullPath = filepath.Join(relayDir, filename) - nextFullPath = filepath.Join(nextRelayDir, nextFilename) - s = newLocalStreamer() - cfg = &BinlogReaderConfig{RelayDir: baseDir} - tctx = tcontext.Background() - r = NewBinlogReader(tctx, cfg) + filename = "test-mysql-bin.000001" + nextFilename = "test-mysql-bin.666888" + baseDir = c.MkDir() + offset int64 + firstParse = true + possibleLast = true + baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGset) + currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" + switchedUUID = "b60868af-5a6f-11e9-9ea3-0242ac160007.000002" + relayDir = filepath.Join(baseDir, currentUUID) + nextRelayDir = filepath.Join(baseDir, switchedUUID) + fullPath = filepath.Join(relayDir, filename) + nextFullPath = filepath.Join(nextRelayDir, nextFilename) + s = newLocalStreamer() + cfg = &BinlogReaderConfig{RelayDir: baseDir} + tctx = tcontext.Background() + r = NewBinlogReader(tctx, cfg) ) // create the current relay log file and write some events @@ -389,19 +403,19 @@ func (t *testReaderSuite) TestParseFileRelayNeedSwitchSubDir(c *C) { func (t *testReaderSuite) TestParseFileRelayWithIgnorableError(c *C) { var ( - filename = "test-mysql-bin.000001" - baseDir = c.MkDir() - offset int64 - firstParse = true - possibleLast = true - baseEvents = t.genBinlogEvents(c, 0) - currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" - relayDir = filepath.Join(baseDir, currentUUID) - fullPath = filepath.Join(relayDir, filename) - s = newLocalStreamer() - cfg = &BinlogReaderConfig{RelayDir: baseDir} - tctx = tcontext.Background() - r = NewBinlogReader(tctx, cfg) + filename = "test-mysql-bin.000001" + baseDir = c.MkDir() + offset int64 + firstParse = true + possibleLast = true + baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGset) + currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" + relayDir = filepath.Join(baseDir, currentUUID) + fullPath = filepath.Join(relayDir, filename) + s = newLocalStreamer() + cfg = &BinlogReaderConfig{RelayDir: baseDir} + tctx = tcontext.Background() + r = NewBinlogReader(tctx, cfg) ) // create the current relay log file and write some events @@ -475,11 +489,11 @@ func (t *testReaderSuite) TestUpdateUUIDs(c *C) { func (t *testReaderSuite) TestStartSync(c *C) { var ( - filenamePrefix = "test-mysql-bin.00000" - baseDir = c.MkDir() - baseEvents = t.genBinlogEvents(c, 0) - eventsBuf bytes.Buffer - UUIDs = []string{ + filenamePrefix = "test-mysql-bin.00000" + baseDir = c.MkDir() + baseEvents, lastPos, lastGset = t.genBinlogEvents(c, t.lastPos, t.lastGset) + eventsBuf bytes.Buffer + UUIDs = []string{ "b60868af-5a6f-11e9-9ea3-0242ac160006.000001", "b60868af-5a6f-11e9-9ea3-0242ac160007.000002", "b60868af-5a6f-11e9-9ea3-0242ac160008.000003", @@ -549,7 +563,7 @@ func (t *testReaderSuite) TestStartSync(c *C) { // 2. write more events to the last file lastFilename := filepath.Join(baseDir, UUIDs[2], filenamePrefix+strconv.Itoa(3)) - extraEvents := t.genBinlogEvents(c, baseEvents[len(baseEvents)-1].Header.LogPos) + extraEvents, _, _ := t.genBinlogEvents(c, lastPos, lastGset) lastF, err := os.OpenFile(lastFilename, os.O_WRONLY|os.O_APPEND, 0600) c.Assert(err, IsNil) defer lastF.Close() @@ -629,6 +643,10 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(err, ErrorMatches, ".*empty UUIDs not valid.*") c.Assert(s, IsNil) + s, err = r.StartSyncByGTID(t.lastGset.Origin().Clone()) + c.Assert(err, ErrorMatches, ".*no relay subdir match gtid.*") + c.Assert(s, IsNil) + // write UUIDs into index file r = NewBinlogReader(tctx, cfg) // create a new reader uuidBytes := t.uuidListToBytes(c, UUIDs) @@ -640,6 +658,10 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(err, ErrorMatches, fmt.Sprintf(".*%s.*not found.*", startPos.Name)) c.Assert(s, IsNil) + s, err = r.StartSyncByGTID(t.lastGset.Origin().Clone()) + c.Assert(err, ErrorMatches, ".*load meta data.*no such file or directory.*") + c.Assert(s, IsNil) + // can not re-start the reader r.running = true s, err = r.StartSyncByPos(startPos) @@ -647,6 +669,12 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(s, IsNil) r.Close() + r.running = true + s, err = r.StartSyncByGTID(t.lastGset.Origin().Clone()) + c.Assert(terror.ErrReaderAlreadyRunning.Equal(err), IsTrue) + c.Assert(s, IsNil) + r.Close() + // too big startPos uuid := UUIDs[0] err = os.MkdirAll(filepath.Join(baseDir, uuid), 0700) @@ -661,7 +689,7 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(s, IsNil) } -func (t *testReaderSuite) genBinlogEvents(c *C, latestPos uint32) []*replication.BinlogEvent { +func (t *testReaderSuite) genBinlogEvents(c *C, latestPos uint32, latestGset gtid.Set) ([]*replication.BinlogEvent, uint32, gtid.Set) { var ( header = &replication.EventHeader{ Timestamp: uint32(time.Now().Unix()), @@ -680,15 +708,14 @@ func (t *testReaderSuite) genBinlogEvents(c *C, latestPos uint32) []*replication // for these tests, generates some DDL events is enough count := 5 + rand.Intn(5) for i := 0; i < count; i++ { - schema := []byte(fmt.Sprintf("db_%d", i)) - query := []byte(fmt.Sprintf("CREATE TABLE %d (c1 INT)", i)) - ev, err := event.GenQueryEvent(header, latestPos, 0, 0, 0, nil, schema, query) + evs, err := event.GenDDLEvents(mysql.MySQLFlavor, 1, latestPos, latestGset, fmt.Sprintf("db_%d", i), fmt.Sprintf("CREATE TABLE %d (c1 INT)", i)) c.Assert(err, IsNil) - latestPos = ev.Header.LogPos - events = append(events, ev) + events = append(events, evs.Events...) + latestPos = evs.LatestPos + latestGset = evs.LatestGTID } - return events + return events, latestPos, latestGset } func (t *testReaderSuite) purgeStreamer(c *C, s Streamer) { From 0dec4d96749433994eef83abf7d21b179c23ee66 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 11 Nov 2020 15:49:13 +0800 Subject: [PATCH 05/59] fix nil previous gtid event --- dm/pb/dmworker.pb.go | 301 +++++++++++++++++++++++----------------- dm/proto/dmworker.proto | 1 + dm/worker/subtask.go | 18 ++- loader/loader.go | 2 + pkg/streamer/reader.go | 15 +- 5 files changed, 206 insertions(+), 131 deletions(-) diff --git a/dm/pb/dmworker.pb.go b/dm/pb/dmworker.pb.go index 3809c12128..4dfabe291a 100644 --- a/dm/pb/dmworker.pb.go +++ b/dm/pb/dmworker.pb.go @@ -600,10 +600,11 @@ var xxx_messageInfo_DumpStatus proto.InternalMessageInfo // LoadStatus represents status for load unit type LoadStatus struct { - FinishedBytes int64 `protobuf:"varint,1,opt,name=finishedBytes,proto3" json:"finishedBytes,omitempty"` - TotalBytes int64 `protobuf:"varint,2,opt,name=totalBytes,proto3" json:"totalBytes,omitempty"` - Progress string `protobuf:"bytes,3,opt,name=progress,proto3" json:"progress,omitempty"` - MetaBinlog string `protobuf:"bytes,4,opt,name=metaBinlog,proto3" json:"metaBinlog,omitempty"` + FinishedBytes int64 `protobuf:"varint,1,opt,name=finishedBytes,proto3" json:"finishedBytes,omitempty"` + TotalBytes int64 `protobuf:"varint,2,opt,name=totalBytes,proto3" json:"totalBytes,omitempty"` + Progress string `protobuf:"bytes,3,opt,name=progress,proto3" json:"progress,omitempty"` + MetaBinlog string `protobuf:"bytes,4,opt,name=metaBinlog,proto3" json:"metaBinlog,omitempty"` + MetaBinlogGTID string `protobuf:"bytes,5,opt,name=metaBinlogGTID,proto3" json:"metaBinlogGTID,omitempty"` } func (m *LoadStatus) Reset() { *m = LoadStatus{} } @@ -667,6 +668,13 @@ func (m *LoadStatus) GetMetaBinlog() string { return "" } +func (m *LoadStatus) GetMetaBinlogGTID() string { + if m != nil { + return m.MetaBinlogGTID + } + return "" +} + // ShardingGroup represents a DDL sharding group, this is used by SyncStatus, and is differ from ShardingGroup in syncer pkg // target: target table name // DDL: in syncing DDL @@ -2419,127 +2427,127 @@ func init() { func init() { proto.RegisterFile("dmworker.proto", fileDescriptor_51a1b9e17fd67b10) } var fileDescriptor_51a1b9e17fd67b10 = []byte{ - // 1907 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0x41, 0x93, 0xdb, 0x58, - 0x11, 0xb6, 0x24, 0xdb, 0x63, 0xb7, 0xed, 0x89, 0xf2, 0x92, 0x5d, 0xcc, 0x10, 0xcc, 0x94, 0xb2, - 0xb5, 0x0c, 0x73, 0x98, 0x22, 0xc3, 0x52, 0x4b, 0x6d, 0x15, 0x10, 0x32, 0x93, 0x4d, 0x16, 0x1c, - 0x92, 0xc8, 0xc9, 0x72, 0xa4, 0x9e, 0xa5, 0x17, 0x8f, 0x6a, 0x64, 0x49, 0xd1, 0x93, 0x66, 0xca, - 0x07, 0xfe, 0x02, 0x70, 0xe1, 0x40, 0x15, 0x37, 0x8a, 0x2b, 0xc5, 0xaf, 0x00, 0x8e, 0x5b, 0x9c, - 0x38, 0x52, 0xc9, 0x89, 0xff, 0xc0, 0x81, 0xea, 0x7e, 0x4f, 0xd2, 0x73, 0xc6, 0x4e, 0xc8, 0x81, - 0x9b, 0xfa, 0xeb, 0x7e, 0xfd, 0xfa, 0xf5, 0xfb, 0xba, 0x5b, 0x12, 0xec, 0x86, 0xcb, 0xcb, 0x34, - 0x3f, 0x17, 0xf9, 0x51, 0x96, 0xa7, 0x45, 0xca, 0xec, 0x6c, 0xee, 0x1d, 0x00, 0x7b, 0x5a, 0x8a, - 0x7c, 0x35, 0x2b, 0x78, 0x51, 0x4a, 0x5f, 0xbc, 0x2c, 0x85, 0x2c, 0x18, 0x83, 0x76, 0xc2, 0x97, - 0x62, 0x6c, 0xed, 0x5b, 0x07, 0x7d, 0x9f, 0x9e, 0xbd, 0x0c, 0x6e, 0x9e, 0xa4, 0xcb, 0x65, 0x9a, - 0xfc, 0x82, 0x7c, 0xf8, 0x42, 0x66, 0x69, 0x22, 0x05, 0xfb, 0x10, 0xba, 0xb9, 0x90, 0x65, 0x5c, - 0x90, 0x75, 0xcf, 0xd7, 0x12, 0x73, 0xc1, 0x59, 0xca, 0xc5, 0xd8, 0x26, 0x17, 0xf8, 0x88, 0x96, - 0x32, 0x2d, 0xf3, 0x40, 0x8c, 0x1d, 0x02, 0xb5, 0x84, 0xb8, 0x8a, 0x6b, 0xdc, 0x56, 0xb8, 0x92, - 0xbc, 0x3f, 0x5b, 0x70, 0x63, 0x2d, 0xb8, 0xf7, 0xde, 0xf1, 0x13, 0x18, 0xaa, 0x3d, 0x94, 0x07, - 0xda, 0x77, 0x70, 0xec, 0x1e, 0x65, 0xf3, 0xa3, 0x99, 0x81, 0xfb, 0x6b, 0x56, 0xec, 0x53, 0x18, - 0xc9, 0x72, 0xfe, 0x8c, 0xcb, 0x73, 0xbd, 0xac, 0xbd, 0xef, 0x1c, 0x0c, 0x8e, 0xaf, 0xd3, 0x32, - 0x53, 0xe1, 0xaf, 0xdb, 0x79, 0x7f, 0xb2, 0x60, 0x70, 0x72, 0x26, 0x02, 0x2d, 0x63, 0xa0, 0x19, - 0x97, 0x52, 0x84, 0x55, 0xa0, 0x4a, 0x62, 0x37, 0xa1, 0x53, 0xa4, 0x05, 0x8f, 0x29, 0xd4, 0x8e, - 0xaf, 0x04, 0x36, 0x01, 0x90, 0x65, 0x10, 0x08, 0x29, 0x5f, 0x94, 0x31, 0x85, 0xda, 0xf1, 0x0d, - 0x04, 0xbd, 0xbd, 0xe0, 0x51, 0x2c, 0x42, 0x4a, 0x53, 0xc7, 0xd7, 0x12, 0x1b, 0xc3, 0xce, 0x25, - 0xcf, 0x93, 0x28, 0x59, 0x8c, 0x3b, 0xa4, 0xa8, 0x44, 0x5c, 0x11, 0x8a, 0x82, 0x47, 0xf1, 0xb8, - 0xbb, 0x6f, 0x1d, 0x0c, 0x7d, 0x2d, 0x79, 0x43, 0x80, 0xd3, 0x72, 0x99, 0xe9, 0xa8, 0x7f, 0x6d, - 0x01, 0x4c, 0x53, 0x1e, 0xea, 0xa0, 0x3f, 0x82, 0xd1, 0x8b, 0x28, 0x89, 0xe4, 0x99, 0x08, 0xef, - 0xad, 0x0a, 0x21, 0x29, 0x76, 0xc7, 0x5f, 0x07, 0x31, 0x58, 0x8a, 0x5a, 0x99, 0xd8, 0x64, 0x62, - 0x20, 0x6c, 0x0f, 0x7a, 0x59, 0x9e, 0x2e, 0x72, 0x21, 0xa5, 0xbe, 0xed, 0x5a, 0xc6, 0xb5, 0x4b, - 0x51, 0xf0, 0x7b, 0x51, 0x12, 0xa7, 0x0b, 0x7d, 0xe7, 0x06, 0xe2, 0xfd, 0xce, 0x82, 0xd1, 0xec, - 0x8c, 0xe7, 0x61, 0x94, 0x2c, 0x1e, 0xe4, 0x69, 0x99, 0xe1, 0x41, 0x0a, 0x9e, 0x2f, 0x44, 0xa1, - 0x19, 0xa9, 0x25, 0xe4, 0xe9, 0xe9, 0xe9, 0x14, 0xf7, 0x77, 0x90, 0xa7, 0xf8, 0xac, 0xe2, 0xcf, - 0x65, 0x31, 0x4d, 0x03, 0x5e, 0x44, 0x69, 0xa2, 0xb7, 0x5f, 0x07, 0x89, 0x8b, 0xab, 0x24, 0xa0, - 0x64, 0x3a, 0xc4, 0x45, 0x92, 0x30, 0xee, 0x32, 0xd1, 0x9a, 0x0e, 0x69, 0x6a, 0xd9, 0xfb, 0xa3, - 0x03, 0x30, 0x5b, 0x25, 0x81, 0x4e, 0xd4, 0x3e, 0x0c, 0xe8, 0xc0, 0xf7, 0x2f, 0x44, 0x52, 0x54, - 0x69, 0x32, 0x21, 0x74, 0x46, 0xe2, 0xb3, 0xac, 0x4a, 0x51, 0x2d, 0xb3, 0x5b, 0xd0, 0xcf, 0x45, - 0x20, 0x92, 0x02, 0x95, 0x0e, 0x29, 0x1b, 0x80, 0x79, 0x30, 0x5c, 0x72, 0x59, 0x88, 0x7c, 0x2d, - 0x49, 0x6b, 0x18, 0x3b, 0x04, 0xd7, 0x94, 0x1f, 0x14, 0x51, 0x48, 0x04, 0xe8, 0xfb, 0x57, 0x70, - 0xf4, 0x47, 0x87, 0xa8, 0xfc, 0x75, 0x95, 0x3f, 0x13, 0x43, 0x7f, 0xa6, 0x4c, 0xfe, 0x76, 0x94, - 0xbf, 0x37, 0x71, 0xf4, 0x37, 0x8f, 0xd3, 0xe0, 0x3c, 0x4a, 0x16, 0x74, 0x01, 0x3d, 0x4a, 0xd5, - 0x1a, 0xc6, 0x7e, 0x08, 0x6e, 0x99, 0xe4, 0x42, 0xa6, 0xf1, 0x85, 0x08, 0xe9, 0x1e, 0xe5, 0xb8, - 0x6f, 0x54, 0x92, 0x79, 0xc3, 0xfe, 0x15, 0x53, 0xe3, 0x86, 0x40, 0x15, 0x8f, 0xbe, 0xa1, 0x09, - 0xc0, 0x9c, 0x02, 0x79, 0xb6, 0xca, 0xc4, 0x78, 0xa0, 0xd8, 0xd3, 0x20, 0xde, 0x1f, 0x2c, 0x18, - 0x9a, 0xc5, 0x6d, 0xb4, 0x1d, 0x6b, 0x4b, 0xdb, 0xb1, 0xcd, 0xb6, 0xc3, 0xbe, 0x53, 0xb7, 0x17, - 0xd5, 0x2e, 0x28, 0xda, 0x27, 0x79, 0x8a, 0x75, 0xe8, 0x93, 0xa2, 0xee, 0x38, 0x77, 0x60, 0x90, - 0x8b, 0x98, 0xaf, 0xea, 0x3e, 0x81, 0xf6, 0xd7, 0xd0, 0xde, 0x6f, 0x60, 0xdf, 0xb4, 0xf1, 0xfe, - 0x66, 0xc3, 0xc0, 0x50, 0x5e, 0xb9, 0x69, 0xeb, 0x7f, 0xbc, 0x69, 0x7b, 0xcb, 0x4d, 0xef, 0x57, - 0x21, 0x95, 0xf3, 0xd3, 0x28, 0xd7, 0xe4, 0x37, 0xa1, 0xda, 0x62, 0x8d, 0x5a, 0x26, 0xc4, 0x0e, - 0xe0, 0x9a, 0x21, 0x1a, 0xc4, 0x7a, 0x13, 0x66, 0x47, 0xc0, 0x08, 0x3a, 0xe1, 0x45, 0x70, 0xf6, - 0x3c, 0x7b, 0x44, 0xd1, 0x10, 0xbb, 0x7a, 0xfe, 0x06, 0x0d, 0xfb, 0x16, 0x74, 0x64, 0xc1, 0x17, - 0x82, 0x88, 0xb5, 0x7b, 0xdc, 0x27, 0x22, 0x20, 0xe0, 0x2b, 0xdc, 0x48, 0x7e, 0xef, 0x1d, 0xc9, - 0xf7, 0xfe, 0x63, 0xc3, 0x68, 0xad, 0x1d, 0x6f, 0x1a, 0x5b, 0xcd, 0x8e, 0xf6, 0x96, 0x1d, 0xf7, - 0xa1, 0x5d, 0x26, 0x91, 0xba, 0xec, 0xdd, 0xe3, 0x21, 0xea, 0x9f, 0x27, 0x51, 0x81, 0x5c, 0xf2, - 0x49, 0x63, 0xc4, 0xd4, 0x7e, 0x17, 0x21, 0xbe, 0x0b, 0x37, 0x1a, 0x22, 0x9f, 0x9e, 0x4e, 0xa7, - 0x69, 0x70, 0xfe, 0xc5, 0xa9, 0xce, 0xde, 0x26, 0x15, 0x63, 0x6a, 0x68, 0x51, 0x41, 0x3e, 0x6c, - 0xa9, 0xb1, 0xf5, 0x6d, 0xe8, 0x04, 0x38, 0x46, 0x28, 0x4b, 0x9a, 0x50, 0xc6, 0x5c, 0x79, 0xd8, - 0xf2, 0x95, 0x9e, 0x7d, 0x04, 0xed, 0xb0, 0x5c, 0x66, 0x3a, 0x57, 0xbb, 0x68, 0xd7, 0x34, 0xf6, - 0x87, 0x2d, 0x9f, 0xb4, 0x68, 0x15, 0xa7, 0x3c, 0x1c, 0xf7, 0x1b, 0xab, 0xa6, 0xdf, 0xa3, 0x15, - 0x6a, 0xd1, 0x0a, 0x2b, 0x8c, 0xaa, 0x4d, 0x5b, 0x35, 0xcd, 0x0e, 0xad, 0x50, 0x7b, 0xaf, 0x07, - 0x5d, 0xa9, 0x88, 0xfc, 0x23, 0xb8, 0xbe, 0x96, 0xfd, 0x69, 0x24, 0x29, 0x55, 0x4a, 0x3d, 0xb6, - 0xb6, 0xcd, 0xcc, 0x6a, 0xfd, 0x04, 0x80, 0xce, 0x74, 0x3f, 0xcf, 0xd3, 0xbc, 0x9a, 0xdd, 0x56, - 0x3d, 0xbb, 0xbd, 0x6f, 0x42, 0x1f, 0xcf, 0xf2, 0x16, 0x35, 0x1e, 0x62, 0x9b, 0x3a, 0x83, 0x21, - 0x45, 0xff, 0x74, 0xba, 0xc5, 0x82, 0x1d, 0xc3, 0x4d, 0x35, 0x40, 0x15, 0x9d, 0x9f, 0xa4, 0x32, - 0xa2, 0x71, 0xa1, 0x0a, 0x6b, 0xa3, 0x0e, 0x1b, 0xba, 0x40, 0x77, 0xb3, 0xa7, 0xd3, 0x6a, 0xaa, - 0x55, 0xb2, 0xf7, 0x7d, 0xe8, 0xe3, 0x8e, 0x6a, 0xbb, 0x03, 0xe8, 0x92, 0xa2, 0xca, 0x83, 0x5b, - 0xa7, 0x53, 0x07, 0xe4, 0x6b, 0xbd, 0xf7, 0x1b, 0x0b, 0x06, 0xaa, 0x5d, 0xa9, 0x95, 0xef, 0xdb, - 0xad, 0xf6, 0xd7, 0x96, 0x57, 0xf5, 0x6e, 0x7a, 0x3c, 0x02, 0xa0, 0x86, 0xa3, 0x0c, 0xda, 0xcd, - 0xf5, 0x36, 0xa8, 0x6f, 0x58, 0xe0, 0xc5, 0x34, 0xd2, 0x86, 0xd4, 0xfe, 0xde, 0x86, 0xa1, 0xbe, - 0x52, 0x65, 0xf2, 0x7f, 0x2a, 0x3b, 0x5d, 0x19, 0x6d, 0xb3, 0x32, 0x3e, 0xae, 0x2a, 0xa3, 0xd3, - 0x1c, 0xa3, 0x61, 0x51, 0x53, 0x18, 0xb7, 0x75, 0x61, 0x74, 0xc9, 0x6c, 0x54, 0x15, 0x46, 0x65, - 0xa5, 0xea, 0xe2, 0xb6, 0xae, 0x8b, 0x9d, 0xc6, 0xa8, 0xa6, 0x54, 0x5d, 0x16, 0xb7, 0x75, 0x59, - 0xf4, 0x1a, 0xa3, 0xfa, 0x9a, 0xeb, 0xaa, 0xd8, 0x81, 0x0e, 0x5d, 0xa7, 0xf7, 0x19, 0xb8, 0x66, - 0x6a, 0xa8, 0x26, 0x3e, 0xd6, 0xca, 0x35, 0x2a, 0x18, 0x46, 0xbe, 0x5e, 0xfb, 0x12, 0x46, 0x6b, - 0x4d, 0x05, 0x27, 0x5d, 0x24, 0x4f, 0x78, 0x12, 0x88, 0xb8, 0x7e, 0x85, 0x34, 0x10, 0x83, 0x64, - 0x76, 0xe3, 0x59, 0xbb, 0x58, 0x23, 0x99, 0xf1, 0x22, 0xe8, 0xac, 0xbd, 0x08, 0xfe, 0xc3, 0x82, - 0xa1, 0xb9, 0x00, 0xdf, 0x25, 0xef, 0xe7, 0xf9, 0x49, 0x1a, 0xaa, 0xdb, 0xec, 0xf8, 0x95, 0x88, - 0xd4, 0xc7, 0xc7, 0x98, 0x4b, 0xa9, 0x19, 0x58, 0xcb, 0x5a, 0x37, 0x0b, 0xd2, 0xac, 0x7a, 0xb5, - 0xaf, 0x65, 0xad, 0x9b, 0x8a, 0x0b, 0x11, 0xeb, 0x51, 0x53, 0xcb, 0xb8, 0xdb, 0x23, 0x21, 0x25, - 0xd2, 0x44, 0x75, 0xc8, 0x4a, 0xc4, 0x55, 0x3e, 0xbf, 0x3c, 0xe1, 0xa5, 0x14, 0xfa, 0x5d, 0xa5, - 0x96, 0x31, 0x2d, 0xf8, 0x09, 0xc2, 0xf3, 0xb4, 0x4c, 0xaa, 0x37, 0x14, 0x03, 0xf1, 0x2e, 0xe1, - 0xfa, 0x93, 0x32, 0x5f, 0x08, 0x22, 0x71, 0xf5, 0x45, 0xb3, 0x07, 0xbd, 0x28, 0xe1, 0x41, 0x11, - 0x5d, 0x08, 0x9d, 0xc9, 0x5a, 0x46, 0xfe, 0x16, 0xd1, 0x52, 0xe8, 0x57, 0x34, 0x7a, 0x46, 0xfb, - 0x17, 0x51, 0x2c, 0x88, 0xd7, 0xfa, 0x48, 0x95, 0x4c, 0x25, 0xaa, 0xa6, 0xab, 0xfe, 0x5e, 0x51, - 0x92, 0xf7, 0x17, 0x0b, 0xf6, 0x1e, 0x67, 0x22, 0xe7, 0x85, 0x50, 0xdf, 0x48, 0xb3, 0xe0, 0x4c, - 0x2c, 0x79, 0x15, 0xc2, 0x2d, 0xb0, 0xd3, 0x8c, 0x36, 0xd7, 0x7c, 0x57, 0xea, 0xc7, 0x99, 0x6f, - 0xa7, 0x19, 0x05, 0xc1, 0xe5, 0xb9, 0xce, 0x2d, 0x3d, 0x6f, 0xfd, 0x60, 0xda, 0x83, 0x5e, 0xc8, - 0x0b, 0x3e, 0xe7, 0x52, 0x54, 0x39, 0xad, 0x64, 0xfa, 0xb6, 0xe0, 0xf3, 0xb8, 0xca, 0xa8, 0x12, - 0xc8, 0x13, 0xed, 0xa6, 0xb3, 0xa9, 0x25, 0xaf, 0x80, 0xd1, 0x97, 0x77, 0x34, 0x19, 0x1f, 0x89, - 0x82, 0xb3, 0x3d, 0x23, 0x48, 0xc0, 0x20, 0x51, 0xa3, 0x43, 0x7c, 0x67, 0x4d, 0x57, 0x8d, 0xc0, - 0x31, 0x1a, 0x41, 0x75, 0xae, 0x36, 0x11, 0x8f, 0x9e, 0xbd, 0x4f, 0xe0, 0xa6, 0xce, 0xd3, 0x97, - 0x77, 0x70, 0xd7, 0xad, 0x19, 0x52, 0x6a, 0xb5, 0xbd, 0xf7, 0x57, 0x0b, 0x3e, 0x78, 0x63, 0xd9, - 0x7b, 0x7f, 0x10, 0x7e, 0x0a, 0x6d, 0xfc, 0xd0, 0x18, 0x3b, 0x54, 0x30, 0xb7, 0x71, 0x8f, 0x8d, - 0x2e, 0x8f, 0x50, 0xb8, 0x9f, 0x14, 0xf9, 0xca, 0xa7, 0x05, 0x7b, 0x3f, 0x85, 0x7e, 0x0d, 0xa1, - 0xdf, 0x73, 0xb1, 0xaa, 0x7a, 0xe2, 0xb9, 0x58, 0xe1, 0xc4, 0xbe, 0xe0, 0x71, 0xa9, 0x52, 0xa3, - 0xc7, 0xde, 0x5a, 0x62, 0x7d, 0xa5, 0xff, 0xcc, 0xfe, 0x81, 0xe5, 0xfd, 0x0a, 0xc6, 0x0f, 0x79, - 0x12, 0xc6, 0x9a, 0x25, 0xaa, 0x54, 0x75, 0x0a, 0xbe, 0x61, 0xa4, 0x60, 0x80, 0x5e, 0x48, 0xfb, - 0x16, 0x8e, 0xdc, 0x82, 0xfe, 0xbc, 0x1a, 0x52, 0x3a, 0xf1, 0x0d, 0x80, 0x2b, 0xe4, 0xcb, 0x58, - 0xea, 0x8f, 0x1c, 0x7a, 0x3e, 0xfc, 0x25, 0x74, 0xd5, 0xa5, 0xb2, 0x11, 0xf4, 0xbf, 0x48, 0x2e, - 0x78, 0x1c, 0x85, 0x8f, 0x33, 0xb7, 0xc5, 0x7a, 0xd0, 0x9e, 0x15, 0x69, 0xe6, 0x5a, 0xac, 0x0f, - 0x9d, 0x27, 0x58, 0x6b, 0xae, 0xcd, 0x00, 0xba, 0xd8, 0x8e, 0x96, 0xc2, 0x75, 0x10, 0x9e, 0x15, - 0x3c, 0x2f, 0xdc, 0x36, 0xc2, 0xcf, 0xb3, 0x90, 0x17, 0xc2, 0xed, 0xb0, 0x5d, 0x80, 0x9f, 0x94, - 0x45, 0xaa, 0xcd, 0xba, 0x87, 0x2f, 0xc9, 0x6c, 0x21, 0x98, 0x0b, 0x43, 0xed, 0x9f, 0x64, 0xb7, - 0xc5, 0x76, 0xc0, 0xf9, 0xb9, 0xb8, 0x74, 0x2d, 0x36, 0x80, 0x1d, 0xbf, 0x4c, 0xf0, 0x2b, 0x55, - 0xed, 0x41, 0xdb, 0x85, 0xae, 0x83, 0x0a, 0x0c, 0x22, 0x13, 0xa1, 0xdb, 0x66, 0x43, 0xe8, 0x7d, - 0xae, 0x3f, 0x3b, 0xdd, 0x0e, 0xaa, 0xd0, 0x0c, 0xd7, 0x74, 0x51, 0x45, 0x1b, 0xa2, 0xb4, 0x73, - 0xf8, 0x18, 0x7a, 0xd5, 0xf4, 0x60, 0xd7, 0x60, 0xa0, 0x77, 0x45, 0xc8, 0x6d, 0x61, 0xd8, 0x34, - 0x23, 0x5c, 0x0b, 0x8f, 0x88, 0x73, 0xc0, 0xb5, 0xf1, 0x09, 0x9b, 0xbd, 0xeb, 0xd0, 0xb1, 0x57, - 0x49, 0xe0, 0xb6, 0xd1, 0x90, 0x9a, 0x86, 0x1b, 0x1e, 0x3e, 0x82, 0x1d, 0x7a, 0x7c, 0x8c, 0x59, - 0xdf, 0xd5, 0xfe, 0x34, 0xe2, 0xb6, 0x30, 0x73, 0x18, 0xa5, 0xb2, 0xb6, 0x30, 0x03, 0x74, 0x00, - 0x25, 0xdb, 0x18, 0x82, 0xca, 0x86, 0x02, 0x1c, 0x8c, 0xaf, 0xaa, 0x76, 0x76, 0x03, 0xae, 0x55, - 0x59, 0xd1, 0x90, 0x72, 0xf8, 0x40, 0x14, 0x0a, 0x70, 0x2d, 0xf2, 0x5f, 0x8b, 0x36, 0x26, 0xd2, - 0x17, 0xcb, 0xf4, 0x42, 0x68, 0xc4, 0x39, 0xbc, 0x0b, 0xbd, 0xaa, 0x38, 0x0c, 0x87, 0x15, 0x54, - 0x3b, 0x54, 0x80, 0x6b, 0x35, 0x1e, 0x34, 0x62, 0x1f, 0xde, 0xa5, 0x56, 0x8f, 0xdc, 0x32, 0x4e, - 0xa8, 0x11, 0x4d, 0x86, 0xf3, 0x28, 0xd3, 0x57, 0x25, 0xb2, 0x98, 0x07, 0x35, 0x1d, 0x2e, 0x44, - 0x5e, 0xb8, 0xce, 0xf1, 0xbf, 0x6d, 0xe8, 0x2a, 0x0a, 0xb3, 0xbb, 0x30, 0x30, 0xfe, 0xd4, 0xb0, - 0x0f, 0x91, 0xb9, 0x57, 0xff, 0x2b, 0xed, 0x7d, 0xed, 0x0a, 0xae, 0xca, 0xcd, 0x6b, 0xb1, 0x1f, - 0x03, 0x34, 0x5d, 0x9b, 0x7d, 0x40, 0xa3, 0xec, 0xcd, 0x2e, 0xbe, 0x37, 0xa6, 0x79, 0xbf, 0xe1, - 0x2f, 0x94, 0xd7, 0x62, 0x3f, 0x83, 0x91, 0x2e, 0x65, 0x95, 0x24, 0x36, 0x31, 0xaa, 0x7b, 0x43, - 0x3f, 0x7e, 0xab, 0xb3, 0xcf, 0x6b, 0x67, 0x2a, 0x5f, 0x6c, 0xbc, 0xa1, 0x55, 0x28, 0x37, 0x5f, - 0xdf, 0xda, 0x44, 0xbc, 0x16, 0x7b, 0x00, 0x03, 0x55, 0xea, 0x6a, 0xbc, 0xde, 0x42, 0xdb, 0x6d, - 0xb5, 0xff, 0xb6, 0x80, 0xee, 0x8d, 0xff, 0xfe, 0x6a, 0x62, 0x7d, 0xf5, 0x6a, 0x62, 0xfd, 0xeb, - 0xd5, 0xc4, 0xfa, 0xed, 0xeb, 0x49, 0xeb, 0xab, 0xd7, 0x93, 0xd6, 0x3f, 0x5f, 0x4f, 0x5a, 0xf3, - 0x2e, 0xfd, 0xcc, 0xfb, 0xde, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc2, 0x78, 0xee, 0x8f, 0xde, - 0x13, 0x00, 0x00, + // 1919 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0x4f, 0x73, 0x1b, 0x49, + 0x15, 0xd7, 0xcc, 0x48, 0xb2, 0xf4, 0x24, 0x3b, 0x93, 0x4e, 0x76, 0x11, 0x26, 0x08, 0xd7, 0x64, + 0x2b, 0x18, 0x1f, 0x5c, 0xc4, 0x2c, 0xb5, 0xd4, 0x56, 0x01, 0x21, 0x76, 0xd6, 0x59, 0x70, 0x70, + 0x32, 0x4a, 0x96, 0x23, 0xd5, 0x9a, 0xe9, 0xc8, 0x53, 0x1e, 0xcd, 0x4c, 0xa6, 0x67, 0xec, 0xd2, + 0x81, 0xcf, 0x00, 0x17, 0x0e, 0x54, 0x71, 0xa3, 0xb8, 0x52, 0x9c, 0xf8, 0x08, 0xc0, 0x31, 0xc5, + 0x89, 0x23, 0x95, 0x9c, 0xf8, 0x0e, 0x1c, 0xa8, 0xf7, 0xba, 0x67, 0xa6, 0x65, 0x4b, 0x09, 0x39, + 0x70, 0x9b, 0xf7, 0x7b, 0xaf, 0x5f, 0xbf, 0xfe, 0xf5, 0xfb, 0xa3, 0x16, 0x6c, 0x85, 0xf3, 0xcb, + 0x34, 0x3f, 0x17, 0xf9, 0x7e, 0x96, 0xa7, 0x45, 0xca, 0xec, 0x6c, 0xea, 0xed, 0x02, 0x7b, 0x56, + 0x8a, 0x7c, 0x31, 0x29, 0x78, 0x51, 0x4a, 0x5f, 0xbc, 0x2a, 0x85, 0x2c, 0x18, 0x83, 0x76, 0xc2, + 0xe7, 0x62, 0x64, 0xed, 0x58, 0xbb, 0x7d, 0x9f, 0xbe, 0xbd, 0x0c, 0x6e, 0x1f, 0xa6, 0xf3, 0x79, + 0x9a, 0xfc, 0x82, 0x7c, 0xf8, 0x42, 0x66, 0x69, 0x22, 0x05, 0xfb, 0x18, 0xba, 0xb9, 0x90, 0x65, + 0x5c, 0x90, 0x75, 0xcf, 0xd7, 0x12, 0x73, 0xc1, 0x99, 0xcb, 0xd9, 0xc8, 0x26, 0x17, 0xf8, 0x89, + 0x96, 0x32, 0x2d, 0xf3, 0x40, 0x8c, 0x1c, 0x02, 0xb5, 0x84, 0xb8, 0x8a, 0x6b, 0xd4, 0x56, 0xb8, + 0x92, 0xbc, 0x3f, 0x59, 0x70, 0x6b, 0x29, 0xb8, 0x0f, 0xde, 0xf1, 0x53, 0x18, 0xaa, 0x3d, 0x94, + 0x07, 0xda, 0x77, 0x70, 0xe0, 0xee, 0x67, 0xd3, 0xfd, 0x89, 0x81, 0xfb, 0x4b, 0x56, 0xec, 0x33, + 0xd8, 0x94, 0xe5, 0xf4, 0x39, 0x97, 0xe7, 0x7a, 0x59, 0x7b, 0xc7, 0xd9, 0x1d, 0x1c, 0xdc, 0xa4, + 0x65, 0xa6, 0xc2, 0x5f, 0xb6, 0xf3, 0xfe, 0x68, 0xc1, 0xe0, 0xf0, 0x4c, 0x04, 0x5a, 0xc6, 0x40, + 0x33, 0x2e, 0xa5, 0x08, 0xab, 0x40, 0x95, 0xc4, 0x6e, 0x43, 0xa7, 0x48, 0x0b, 0x1e, 0x53, 0xa8, + 0x1d, 0x5f, 0x09, 0x6c, 0x0c, 0x20, 0xcb, 0x20, 0x10, 0x52, 0xbe, 0x2c, 0x63, 0x0a, 0xb5, 0xe3, + 0x1b, 0x08, 0x7a, 0x7b, 0xc9, 0xa3, 0x58, 0x84, 0x44, 0x53, 0xc7, 0xd7, 0x12, 0x1b, 0xc1, 0xc6, + 0x25, 0xcf, 0x93, 0x28, 0x99, 0x8d, 0x3a, 0xa4, 0xa8, 0x44, 0x5c, 0x11, 0x8a, 0x82, 0x47, 0xf1, + 0xa8, 0xbb, 0x63, 0xed, 0x0e, 0x7d, 0x2d, 0x79, 0x43, 0x80, 0xa3, 0x72, 0x9e, 0xe9, 0xa8, 0xff, + 0x62, 0x01, 0x9c, 0xa4, 0x3c, 0xd4, 0x41, 0x7f, 0x02, 0x9b, 0x2f, 0xa3, 0x24, 0x92, 0x67, 0x22, + 0x7c, 0xb8, 0x28, 0x84, 0xa4, 0xd8, 0x1d, 0x7f, 0x19, 0xc4, 0x60, 0x29, 0x6a, 0x65, 0x62, 0x93, + 0x89, 0x81, 0xb0, 0x6d, 0xe8, 0x65, 0x79, 0x3a, 0xcb, 0x85, 0x94, 0xfa, 0xb6, 0x6b, 0x19, 0xd7, + 0xce, 0x45, 0xc1, 0x1f, 0x46, 0x49, 0x9c, 0xce, 0xf4, 0x9d, 0x1b, 0x08, 0xbb, 0x07, 0x5b, 0x8d, + 0x74, 0xfc, 0xfc, 0xcb, 0x23, 0x3a, 0x57, 0xdf, 0xbf, 0x82, 0x7a, 0xbf, 0xb5, 0x60, 0x73, 0x72, + 0xc6, 0xf3, 0x30, 0x4a, 0x66, 0xc7, 0x79, 0x5a, 0x66, 0x78, 0xe0, 0x82, 0xe7, 0x33, 0x51, 0xe8, + 0xcc, 0xd5, 0x12, 0xe6, 0xf3, 0xd1, 0xd1, 0x09, 0xc6, 0xe9, 0x60, 0x3e, 0xe3, 0xb7, 0x3a, 0x67, + 0x2e, 0x8b, 0x93, 0x34, 0xe0, 0x45, 0x94, 0x26, 0x3a, 0xcc, 0x65, 0x90, 0x72, 0x76, 0x91, 0x04, + 0x44, 0xba, 0x43, 0x39, 0x4b, 0x12, 0x9e, 0xaf, 0x4c, 0xb4, 0xa6, 0x43, 0x9a, 0x5a, 0xf6, 0xfe, + 0xe0, 0x00, 0x4c, 0x16, 0x49, 0xa0, 0x09, 0xdd, 0x81, 0x01, 0x11, 0xf3, 0xe8, 0x42, 0x24, 0x45, + 0x45, 0xa7, 0x09, 0xa1, 0x33, 0x12, 0x9f, 0x67, 0x15, 0x95, 0xb5, 0xcc, 0xee, 0x40, 0x3f, 0x17, + 0x81, 0x48, 0x0a, 0x54, 0x3a, 0xa4, 0x6c, 0x00, 0xe6, 0xc1, 0x70, 0xce, 0x65, 0x21, 0xf2, 0x25, + 0x32, 0x97, 0x30, 0xb6, 0x07, 0xae, 0x29, 0x1f, 0x17, 0x51, 0xa8, 0x09, 0xbd, 0x86, 0xa3, 0x3f, + 0x3a, 0x44, 0xe5, 0xaf, 0xab, 0xfc, 0x99, 0x18, 0xfa, 0x33, 0x65, 0xf2, 0xb7, 0xa1, 0xfc, 0x5d, + 0xc5, 0xd1, 0xdf, 0x34, 0x4e, 0x83, 0xf3, 0x28, 0x99, 0xd1, 0x05, 0xf4, 0x88, 0xaa, 0x25, 0x8c, + 0xfd, 0x10, 0xdc, 0x32, 0xc9, 0x85, 0x4c, 0xe3, 0x0b, 0x11, 0xd2, 0x3d, 0xca, 0x51, 0xdf, 0xa8, + 0x38, 0xf3, 0x86, 0xfd, 0x6b, 0xa6, 0xc6, 0x0d, 0x81, 0x2a, 0x32, 0x7d, 0x43, 0x63, 0x80, 0x29, + 0x05, 0xf2, 0x7c, 0x91, 0x89, 0xd1, 0x40, 0x65, 0x59, 0x83, 0x78, 0xbf, 0xb7, 0x60, 0x68, 0x36, + 0x01, 0xa3, 0x3d, 0x59, 0x6b, 0xda, 0x93, 0x6d, 0xb6, 0x27, 0xf6, 0x9d, 0xba, 0x0d, 0xa9, 0xb6, + 0x42, 0xd1, 0x3e, 0xcd, 0x53, 0xac, 0x57, 0x9f, 0x14, 0x75, 0x67, 0xba, 0x0f, 0x83, 0x5c, 0xc4, + 0x7c, 0x51, 0xf7, 0x13, 0xb4, 0xbf, 0x81, 0xf6, 0x7e, 0x03, 0xfb, 0xa6, 0x8d, 0xf7, 0x37, 0x1b, + 0x06, 0x86, 0xf2, 0xda, 0x4d, 0x5b, 0xff, 0xe3, 0x4d, 0xdb, 0x6b, 0x6e, 0x7a, 0xa7, 0x0a, 0xa9, + 0x9c, 0x1e, 0x45, 0xb9, 0x4e, 0x7e, 0x13, 0xaa, 0x2d, 0x96, 0x52, 0xcb, 0x84, 0xd8, 0x2e, 0xdc, + 0x30, 0x44, 0x23, 0xb1, 0xae, 0xc2, 0x6c, 0x1f, 0x18, 0x41, 0x87, 0xbc, 0x08, 0xce, 0x5e, 0x64, + 0x4f, 0x28, 0x1a, 0xca, 0xae, 0x9e, 0xbf, 0x42, 0xc3, 0xbe, 0x05, 0x1d, 0x59, 0xf0, 0x99, 0xa0, + 0xc4, 0xda, 0x3a, 0xe8, 0x53, 0x22, 0x20, 0xe0, 0x2b, 0xdc, 0x20, 0xbf, 0xf7, 0x1e, 0xf2, 0xbd, + 0xff, 0xd8, 0xb0, 0xb9, 0xd4, 0xb6, 0x57, 0x8d, 0xb7, 0x66, 0x47, 0x7b, 0xcd, 0x8e, 0x3b, 0xd0, + 0x2e, 0x93, 0x48, 0x5d, 0xf6, 0xd6, 0xc1, 0x10, 0xf5, 0x2f, 0x92, 0xa8, 0xc0, 0x5c, 0xf2, 0x49, + 0x63, 0xc4, 0xd4, 0x7e, 0x5f, 0x42, 0x7c, 0x17, 0x6e, 0x35, 0x89, 0x7c, 0x74, 0x74, 0x72, 0x92, + 0x06, 0xe7, 0x75, 0x9f, 0x5b, 0xa5, 0x62, 0x4c, 0x0d, 0x37, 0x2a, 0xc8, 0xc7, 0x2d, 0x35, 0xde, + 0xbe, 0x0d, 0x9d, 0x00, 0xc7, 0x0d, 0xb1, 0xa4, 0x13, 0xca, 0x98, 0x3f, 0x8f, 0x5b, 0xbe, 0xd2, + 0xb3, 0x4f, 0xa0, 0x1d, 0x96, 0xf3, 0x4c, 0x73, 0xb5, 0x85, 0x76, 0xcd, 0x00, 0x78, 0xdc, 0xf2, + 0x49, 0x8b, 0x56, 0x71, 0xca, 0xc3, 0x51, 0xbf, 0xb1, 0x6a, 0xe6, 0x02, 0x5a, 0xa1, 0x16, 0xad, + 0xb0, 0xc2, 0xa8, 0xda, 0xb4, 0x55, 0xd3, 0xec, 0xd0, 0x0a, 0xb5, 0x0f, 0x7b, 0xd0, 0x95, 0x2a, + 0x91, 0x7f, 0x04, 0x37, 0x97, 0xd8, 0x3f, 0x89, 0x24, 0x51, 0xa5, 0xd4, 0x23, 0x6b, 0xdd, 0x6c, + 0xad, 0xd6, 0x8f, 0x01, 0xe8, 0x4c, 0x8f, 0xf2, 0x3c, 0xcd, 0xab, 0x19, 0x6f, 0xd5, 0x33, 0xde, + 0xfb, 0x26, 0xf4, 0xf1, 0x2c, 0xef, 0x50, 0xe3, 0x21, 0xd6, 0xa9, 0x33, 0x18, 0x52, 0xf4, 0xcf, + 0x4e, 0xd6, 0x58, 0xb0, 0x03, 0xb8, 0xad, 0x06, 0xad, 0x4a, 0xe7, 0xa7, 0xa9, 0x8c, 0x68, 0x5c, + 0xa8, 0xc2, 0x5a, 0xa9, 0xc3, 0x86, 0x2e, 0xd0, 0xdd, 0xe4, 0xd9, 0x49, 0x35, 0xfd, 0x2a, 0xd9, + 0xfb, 0x3e, 0xf4, 0x71, 0x47, 0xb5, 0xdd, 0x2e, 0x74, 0x49, 0x51, 0xf1, 0xe0, 0xd6, 0x74, 0xea, + 0x80, 0x7c, 0xad, 0xf7, 0x7e, 0x6d, 0xc1, 0x40, 0xb5, 0x2b, 0xb5, 0xf2, 0x43, 0xbb, 0xd5, 0xce, + 0xd2, 0xf2, 0xaa, 0xde, 0x4d, 0x8f, 0xfb, 0x00, 0xd4, 0x70, 0x94, 0x41, 0xbb, 0xb9, 0xde, 0x06, + 0xf5, 0x0d, 0x0b, 0xbc, 0x98, 0x46, 0x5a, 0x41, 0xed, 0xef, 0x6c, 0x18, 0xea, 0x2b, 0x55, 0x26, + 0xff, 0xa7, 0xb2, 0xd3, 0x95, 0xd1, 0x36, 0x2b, 0xe3, 0x5e, 0x55, 0x19, 0x9d, 0xe6, 0x18, 0x4d, + 0x16, 0x35, 0x85, 0x71, 0x57, 0x17, 0x46, 0x97, 0xcc, 0x36, 0xab, 0xc2, 0xa8, 0xac, 0x54, 0x5d, + 0xdc, 0xd5, 0x75, 0xb1, 0xd1, 0x18, 0xd5, 0x29, 0x55, 0x97, 0xc5, 0x5d, 0x5d, 0x16, 0xbd, 0xc6, + 0xa8, 0xbe, 0xe6, 0xba, 0x2a, 0x36, 0xa0, 0x43, 0xd7, 0xe9, 0x7d, 0x0e, 0xae, 0x49, 0x0d, 0xd5, + 0xc4, 0x3d, 0xad, 0x5c, 0x4a, 0x05, 0xc3, 0xc8, 0xd7, 0x6b, 0x5f, 0xc1, 0xe6, 0x52, 0x53, 0xc1, + 0x49, 0x17, 0xc9, 0x43, 0x9e, 0x04, 0x22, 0xae, 0x7f, 0x6a, 0x1a, 0x88, 0x91, 0x64, 0x76, 0xe3, + 0x59, 0xbb, 0x58, 0x4a, 0x32, 0xe3, 0x07, 0xa3, 0xb3, 0xf4, 0x83, 0xf1, 0x1f, 0x16, 0x0c, 0xcd, + 0x05, 0xf8, 0x9b, 0xf3, 0x51, 0x9e, 0x1f, 0xa6, 0xa1, 0xba, 0xcd, 0x8e, 0x5f, 0x89, 0x98, 0xfa, + 0xf8, 0x19, 0x73, 0x29, 0x75, 0x06, 0xd6, 0xb2, 0xd6, 0x4d, 0x82, 0x34, 0xab, 0x9e, 0x00, 0xb5, + 0xac, 0x75, 0x27, 0xe2, 0x42, 0xc4, 0x7a, 0xd4, 0xd4, 0x32, 0xee, 0xf6, 0x44, 0x48, 0x89, 0x69, + 0xa2, 0x3a, 0x64, 0x25, 0xe2, 0x2a, 0x9f, 0x5f, 0x1e, 0xf2, 0x52, 0x0a, 0xfd, 0x5b, 0xa5, 0x96, + 0x91, 0x16, 0x7c, 0xaa, 0xf0, 0x3c, 0x2d, 0x93, 0xea, 0x17, 0x8a, 0x81, 0x78, 0x97, 0x70, 0xf3, + 0x69, 0x99, 0xcf, 0x04, 0x25, 0x71, 0xf5, 0xf2, 0xd9, 0x86, 0x5e, 0x94, 0xf0, 0xa0, 0x88, 0x2e, + 0x84, 0x66, 0xb2, 0x96, 0x31, 0x7f, 0x8b, 0x68, 0x2e, 0xf4, 0x4f, 0x34, 0xfa, 0x46, 0xfb, 0x97, + 0x51, 0x2c, 0x28, 0xaf, 0xf5, 0x91, 0x2a, 0x99, 0x4a, 0x54, 0x4d, 0x57, 0xfd, 0xae, 0x51, 0x92, + 0xf7, 0x67, 0x0b, 0xb6, 0x4f, 0x33, 0x91, 0xf3, 0x42, 0xa8, 0xb7, 0xd4, 0x24, 0x38, 0x13, 0x73, + 0x5e, 0x85, 0x70, 0x07, 0xec, 0x34, 0xa3, 0xcd, 0x75, 0xbe, 0x2b, 0xf5, 0x69, 0xe6, 0xdb, 0x69, + 0x46, 0x41, 0x70, 0x79, 0xae, 0xb9, 0xa5, 0xef, 0xb5, 0x0f, 0xab, 0x6d, 0xe8, 0x85, 0xbc, 0xe0, + 0x53, 0x2e, 0x45, 0xc5, 0x69, 0x25, 0xd3, 0x1b, 0x84, 0x4f, 0xe3, 0x8a, 0x51, 0x25, 0x90, 0x27, + 0xda, 0x4d, 0xb3, 0xa9, 0x25, 0xaf, 0x80, 0xcd, 0xaf, 0xee, 0xeb, 0x64, 0x7c, 0x22, 0x0a, 0xce, + 0xb6, 0x8d, 0x20, 0x01, 0x83, 0x44, 0x8d, 0x0e, 0xf1, 0xbd, 0x35, 0x5d, 0x35, 0x02, 0xc7, 0x68, + 0x04, 0xd5, 0xb9, 0xda, 0x94, 0x78, 0xf4, 0xed, 0x7d, 0x0a, 0xb7, 0x35, 0x4f, 0x5f, 0xdd, 0xc7, + 0x5d, 0xd7, 0x32, 0xa4, 0xd4, 0x6a, 0x7b, 0xef, 0xaf, 0x16, 0x7c, 0x74, 0x65, 0xd9, 0x07, 0x3f, + 0x1c, 0x3f, 0x83, 0x36, 0x3e, 0x36, 0x46, 0x0e, 0x15, 0xcc, 0x5d, 0xdc, 0x63, 0xa5, 0xcb, 0x7d, + 0x14, 0x1e, 0x25, 0x45, 0xbe, 0xf0, 0x69, 0xc1, 0xf6, 0x4f, 0xa1, 0x5f, 0x43, 0xe8, 0xf7, 0x5c, + 0x2c, 0xaa, 0x9e, 0x78, 0x2e, 0x16, 0x38, 0xb1, 0x2f, 0x78, 0x5c, 0x2a, 0x6a, 0xf4, 0xd8, 0x5b, + 0x22, 0xd6, 0x57, 0xfa, 0xcf, 0xed, 0x1f, 0x58, 0xde, 0xaf, 0x60, 0xf4, 0x98, 0x27, 0x61, 0xac, + 0xb3, 0x44, 0x95, 0xaa, 0xa6, 0xe0, 0x1b, 0x06, 0x05, 0x03, 0xf4, 0x42, 0xda, 0x77, 0xe4, 0xc8, + 0x1d, 0xe8, 0x4f, 0xab, 0x21, 0xa5, 0x89, 0x6f, 0x00, 0x5c, 0x21, 0x5f, 0xc5, 0x52, 0x3f, 0x72, + 0xe8, 0x7b, 0xef, 0x97, 0xd0, 0x55, 0x97, 0xca, 0x36, 0xa1, 0xff, 0x65, 0x72, 0xc1, 0xe3, 0x28, + 0x3c, 0xcd, 0xdc, 0x16, 0xeb, 0x41, 0x7b, 0x52, 0xa4, 0x99, 0x6b, 0xb1, 0x3e, 0x74, 0x9e, 0x62, + 0xad, 0xb9, 0x36, 0x03, 0xe8, 0x62, 0x3b, 0x9a, 0x0b, 0xd7, 0x41, 0x78, 0x52, 0xf0, 0xbc, 0x70, + 0xdb, 0x08, 0xbf, 0xc8, 0x42, 0x5e, 0x08, 0xb7, 0xc3, 0xb6, 0x00, 0x7e, 0x52, 0x16, 0xa9, 0x36, + 0xeb, 0xee, 0xbd, 0x22, 0xb3, 0x99, 0x60, 0x2e, 0x0c, 0xb5, 0x7f, 0x92, 0xdd, 0x16, 0xdb, 0x00, + 0xe7, 0xe7, 0xe2, 0xd2, 0xb5, 0xd8, 0x00, 0x36, 0xfc, 0x32, 0xc1, 0xd7, 0xac, 0xda, 0x83, 0xb6, + 0x0b, 0x5d, 0x07, 0x15, 0x18, 0x44, 0x26, 0x42, 0xb7, 0xcd, 0x86, 0xd0, 0xfb, 0x42, 0x3f, 0x4f, + 0xdd, 0x0e, 0xaa, 0xd0, 0x0c, 0xd7, 0x74, 0x51, 0x45, 0x1b, 0xa2, 0xb4, 0xb1, 0x77, 0x0a, 0xbd, + 0x6a, 0x7a, 0xb0, 0x1b, 0x30, 0xd0, 0xbb, 0x22, 0xe4, 0xb6, 0x30, 0x6c, 0x9a, 0x11, 0xae, 0x85, + 0x47, 0xc4, 0x39, 0xe0, 0xda, 0xf8, 0x85, 0xcd, 0xde, 0x75, 0xe8, 0xd8, 0x8b, 0x24, 0x70, 0xdb, + 0x68, 0x48, 0x4d, 0xc3, 0x0d, 0xf7, 0x9e, 0xc0, 0x06, 0x7d, 0x9e, 0x22, 0xeb, 0x5b, 0xda, 0x9f, + 0x46, 0xdc, 0x16, 0x32, 0x87, 0x51, 0x2a, 0x6b, 0x0b, 0x19, 0xa0, 0x03, 0x28, 0xd9, 0xc6, 0x10, + 0x14, 0x1b, 0x0a, 0x70, 0x30, 0xbe, 0xaa, 0xda, 0xd9, 0x2d, 0xb8, 0x51, 0xb1, 0xa2, 0x21, 0xe5, + 0xf0, 0x58, 0x14, 0x0a, 0x70, 0x2d, 0xf2, 0x5f, 0x8b, 0x36, 0x12, 0xe9, 0x8b, 0x79, 0x7a, 0x21, + 0x34, 0xe2, 0xec, 0x3d, 0x80, 0x5e, 0x55, 0x1c, 0x86, 0xc3, 0x0a, 0xaa, 0x1d, 0x2a, 0xc0, 0xb5, + 0x1a, 0x0f, 0x1a, 0xb1, 0xf7, 0x1e, 0x50, 0xab, 0xc7, 0xdc, 0x32, 0x4e, 0xa8, 0x11, 0x9d, 0x0c, + 0xe7, 0x51, 0xa6, 0xaf, 0x4a, 0x64, 0x31, 0x0f, 0xea, 0x74, 0xb8, 0x10, 0x79, 0xe1, 0x3a, 0x07, + 0xff, 0xb6, 0xa1, 0xab, 0x52, 0x98, 0x3d, 0x80, 0x81, 0xf1, 0x8f, 0x0e, 0xfb, 0x18, 0x33, 0xf7, + 0xfa, 0xff, 0x4f, 0xdb, 0x5f, 0xbb, 0x86, 0xab, 0x72, 0xf3, 0x5a, 0xec, 0xc7, 0x00, 0x4d, 0xd7, + 0x66, 0x1f, 0xd1, 0x28, 0xbb, 0xda, 0xc5, 0xb7, 0x47, 0x34, 0xef, 0x57, 0xfc, 0x5b, 0xe5, 0xb5, + 0xd8, 0xcf, 0x60, 0x53, 0x97, 0xb2, 0x22, 0x89, 0x8d, 0x8d, 0xea, 0x5e, 0xd1, 0x8f, 0xdf, 0xe9, + 0xec, 0x8b, 0xda, 0x99, 0xe2, 0x8b, 0x8d, 0x56, 0xb4, 0x0a, 0xe5, 0xe6, 0xeb, 0x6b, 0x9b, 0x88, + 0xd7, 0x62, 0xc7, 0x30, 0x50, 0xa5, 0xae, 0xc6, 0xeb, 0x1d, 0xb4, 0x5d, 0x57, 0xfb, 0xef, 0x0a, + 0xe8, 0xe1, 0xe8, 0xef, 0x6f, 0xc6, 0xd6, 0xeb, 0x37, 0x63, 0xeb, 0x5f, 0x6f, 0xc6, 0xd6, 0x6f, + 0xde, 0x8e, 0x5b, 0xaf, 0xdf, 0x8e, 0x5b, 0xff, 0x7c, 0x3b, 0x6e, 0x4d, 0xbb, 0xf4, 0xa7, 0xdf, + 0xf7, 0xfe, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x50, 0x16, 0x36, 0x3c, 0x06, 0x14, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -3027,6 +3035,13 @@ func (m *LoadStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.MetaBinlogGTID) > 0 { + i -= len(m.MetaBinlogGTID) + copy(dAtA[i:], m.MetaBinlogGTID) + i = encodeVarintDmworker(dAtA, i, uint64(len(m.MetaBinlogGTID))) + i-- + dAtA[i] = 0x2a + } if len(m.MetaBinlog) > 0 { i -= len(m.MetaBinlog) copy(dAtA[i:], m.MetaBinlog) @@ -4576,6 +4591,10 @@ func (m *LoadStatus) Size() (n int) { if l > 0 { n += 1 + l + sovDmworker(uint64(l)) } + l = len(m.MetaBinlogGTID) + if l > 0 { + n += 1 + l + sovDmworker(uint64(l)) + } return n } @@ -6047,6 +6066,38 @@ func (m *LoadStatus) Unmarshal(dAtA []byte) error { } m.MetaBinlog = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MetaBinlogGTID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDmworker + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDmworker + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDmworker + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MetaBinlogGTID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipDmworker(dAtA[iNdEx:]) diff --git a/dm/proto/dmworker.proto b/dm/proto/dmworker.proto index 91c31c3f4c..d84a9e9176 100644 --- a/dm/proto/dmworker.proto +++ b/dm/proto/dmworker.proto @@ -106,6 +106,7 @@ message LoadStatus { int64 totalBytes = 2; string progress = 3; string metaBinlog = 4; + string metaBinlogGTID = 5; } // ShardingGroup represents a DDL sharding group, this is used by SyncStatus, and is differ from ShardingGroup in syncer pkg diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index 6e146e62da..eb2566a05d 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -28,6 +28,8 @@ import ( "github.com/pingcap/dm/dm/unit" "github.com/pingcap/dm/dumpling" "github.com/pingcap/dm/loader" + "github.com/pingcap/dm/pkg/binlog" + "github.com/pingcap/dm/pkg/gtid" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/shardddl/pessimism" "github.com/pingcap/dm/pkg/terror" @@ -600,10 +602,24 @@ func (st *SubTask) unitTransWaitCondition(subTaskCtx context.Context) error { if err != nil { return terror.WithClass(err, terror.ClassDMWorker) } + if st.cfg.EnableGTID { + gset1, err := gtid.ParserGTID(st.cfg.Flavor, loadStatus.MetaBinlogGTID) + if err != nil { + st.l.Error("fail to parse gtid", log.ShortError(err)) + } + gset2, err := gtid.ParserGTID(st.cfg.Flavor, relayStatus.RelayBinlogGtid) + if err != nil { + st.l.Error("fail to parse gtid", log.ShortError(err)) + } + rc, ok := binlog.CompareGTID(gset1, gset2) + if ok && rc <= 0 { + break + } + } if pos1.Compare(*pos2) <= 0 { break } - st.l.Debug("wait relay to catchup", zap.Stringer("load end position", pos1), zap.Stringer("relay position", pos2)) + st.l.Debug("wait relay to catchup", zap.Stringer("load end position", pos1), zap.String("load end gtid", loadStatus.MetaBinlogGTID), zap.Stringer("relay position", pos2), zap.String("relay gtid", relayStatus.RelayBinlogGtid)) select { case <-ctx.Done(): diff --git a/loader/loader.go b/loader/loader.go index 923b4f9cf8..16f2556bda 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -407,6 +407,7 @@ type Loader struct { totalFileCount sync2.AtomicInt64 // schema + table + data finishedDataSize sync2.AtomicInt64 metaBinlog sync2.AtomicString + metaBinlogGTID sync2.AtomicString // record process error rather than log.Fatal runFatalChan chan *pb.ProcessError @@ -1313,6 +1314,7 @@ func (l *Loader) getMydumpMetadata() error { } l.metaBinlog.Set(loc.Position.String()) + l.metaBinlogGTID.Set(loc.GTIDSetStr()) return nil } diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index b750827275..f1dd7ac4b5 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -122,7 +122,7 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { // TODO: use a better mechanism to call relay.meta.Flush // get the meta save in memory relayMetaHub := GetRelayMetaHub() - meta := relayMetaHub.GetMeta() + relayMeta := relayMetaHub.GetMeta() // get flush logs from oldest to newest for _, uuid := range r.uuids { @@ -150,14 +150,14 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { } // use memory meta - if len(meta.UUID) > 0 { - gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, meta.BinlogGTID) + if len(relayMeta.UUID) > 0 { + gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, relayMeta.BinlogGTID) if err != nil { return "", terror.ErrRelayLoadMetaData.Delegate(err) } if gs.Contain(gset) { - r.tctx.L().Info("get uuid subdir by gtid", zap.Stringer("GTID Set", gset), zap.String("uuid", meta.UUID)) - return meta.UUID, nil + r.tctx.L().Info("get uuid subdir by gtid", zap.Stringer("GTID Set", gset), zap.String("uuid", relayMeta.UUID)) + return relayMeta.UUID, nil } } return "", terror.ErrNoUUIDDirMatchGTID.Generate(gset.String()) @@ -219,6 +219,11 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs } switch ev := e.Event.(type) { case *replication.PreviousGTIDsEvent: + // nil previous gtid event + if len(ev.GTIDSets) == 0 { + lastPos = e.Header.LogPos + continue + } gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, ev.GTIDSets) if err != nil { return 0, err From 78fa598bbada14ea10cbda3fbf5275313d24b378 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 12 Nov 2020 10:20:41 +0800 Subject: [PATCH 06/59] fix nil panic for relay_test --- relay/relay_test.go | 49 ++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/relay/relay_test.go b/relay/relay_test.go index 29da166913..6af68badd6 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -29,6 +29,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/parser" + "github.com/siddontang/go-mysql/mysql" gmysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" @@ -58,6 +59,29 @@ func (t *testRelaySuite) SetUpSuite(c *C) { c.Assert(log.InitLogger(&log.Config{}), IsNil) } +func newRelayCfg(c *C, flavor string) *Config { + dbCfg := getDBConfigForTest() + return &Config{ + EnableGTID: false, // position mode, so auto-positioning can work + Flavor: flavor, + RelayDir: c.MkDir(), + ServerID: 12321, + From: config.DBConfig{ + Host: dbCfg.Host, + Port: dbCfg.Port, + User: dbCfg.User, + Password: dbCfg.Password, + }, + ReaderRetry: retry.ReaderRetryConfig{ + BackoffRollback: 200 * time.Millisecond, + BackoffMax: 1 * time.Second, + BackoffMin: 1 * time.Millisecond, + BackoffJitter: true, + BackoffFactor: 2, + }, + } +} + func getDBConfigForTest() config.DBConfig { host := os.Getenv("MYSQL_HOST") if host == "" { @@ -151,12 +175,10 @@ func (t *testRelaySuite) TestTryRecoverLatestFile(c *C) { startPos = gmysql.Position{Name: filename, Pos: 123} parser2 = parser.New() - relayCfg = &Config{ - RelayDir: c.MkDir(), - Flavor: gmysql.MySQLFlavor, - } - r = NewRelay(relayCfg).(*Relay) + relayCfg = newRelayCfg(c, mysql.MySQLFlavor) + r = NewRelay(relayCfg).(*Relay) ) + c.Assert(r.Init(context.Background()), IsNil) // purge old relay dir f, err := os.Create(filepath.Join(r.cfg.RelayDir, "old_relay_log")) c.Assert(err, IsNil) @@ -236,13 +258,10 @@ func (t *testRelaySuite) TestTryRecoverMeta(c *C) { startPos = gmysql.Position{Name: filename, Pos: 123} parser2 = parser.New() - relayCfg = &Config{ - RelayDir: c.MkDir(), - Flavor: gmysql.MySQLFlavor, - } - r = NewRelay(relayCfg).(*Relay) + relayCfg = newRelayCfg(c, mysql.MySQLFlavor) + r = NewRelay(relayCfg).(*Relay) ) - + c.Assert(r.Init(context.Background()), IsNil) recoverGTIDSet, err := gtid.ParserGTID(relayCfg.Flavor, recoverGTIDSetStr) c.Assert(err, IsNil) @@ -362,11 +381,8 @@ func (t *testRelaySuite) TestHandleEvent(c *C) { reader2 = &mockReader{} transformer2 = transformer.NewTransformer(parser.New()) writer2 = &mockWriter{} - relayCfg = &Config{ - RelayDir: c.MkDir(), - Flavor: gmysql.MariaDBFlavor, - } - r = NewRelay(relayCfg).(*Relay) + relayCfg = newRelayCfg(c, mysql.MariaDBFlavor) + r = NewRelay(relayCfg).(*Relay) eventHeader = &replication.EventHeader{ Timestamp: uint32(time.Now().Unix()), @@ -376,6 +392,7 @@ func (t *testRelaySuite) TestHandleEvent(c *C) { rotateEv, _ = event.GenRotateEvent(eventHeader, 123, []byte(binlogPos.Name), uint64(binlogPos.Pos)) queryEv, _ = event.GenQueryEvent(eventHeader, 123, 0, 0, 0, nil, nil, []byte("CREATE DATABASE db_relay_test")) ) + c.Assert(r.Init(context.Background()), IsNil) // NOTE: we can mock meta later. c.Assert(r.meta.Load(), IsNil) c.Assert(r.meta.AddDir("24ecd093-8cec-11e9-aa0d-0242ac170002", nil, nil), IsNil) From 34d85df61afd578f018bf11f7259e0b3a80212a1 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 12 Nov 2020 16:43:16 +0800 Subject: [PATCH 07/59] add ut from relay gtid --- pkg/binlog/event/event.go | 2 +- pkg/streamer/reader.go | 12 +- pkg/streamer/reader_test.go | 315 +++++++++++++++++++++++++++++++++--- 3 files changed, 301 insertions(+), 28 deletions(-) diff --git a/pkg/binlog/event/event.go b/pkg/binlog/event/event.go index 3d449b26f2..eb3995358d 100644 --- a/pkg/binlog/event/event.go +++ b/pkg/binlog/event/event.go @@ -201,7 +201,7 @@ func GenRotateEvent(header *replication.EventHeader, latestPos uint32, nextLogNa // a. https://github.com/vitessio/vitess/blob/28e7e5503a6c3d3b18d4925d95f23ebcb6f25c8e/go/mysql/binlog_event_mysql56.go#L56 // b. https://dev.mysql.com/doc/internals/en/com-binlog-dump-gtid.html func GenPreviousGTIDsEvent(header *replication.EventHeader, latestPos uint32, gSet gtid.Set) (*replication.BinlogEvent, error) { - if gSet == nil || len(gSet.String()) == 0 { + if gSet == nil { return nil, terror.ErrBinlogEmptyGTID.Generate() } diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index f1dd7ac4b5..914b22616e 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -165,11 +165,17 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { // GetFilePosByGTID tries to get Pos by GTID for file func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet) (uint32, error) { - s := newLocalStreamer() - ctx2, cancel := context.WithCancel(ctx) defer cancel() + s := newLocalStreamer() + parser := replication.NewBinlogParser() + parser.SetVerifyChecksum(true) + parser.SetUseDecimal(true) + if r.cfg.Timezone != nil { + parser.SetTimestampStringLocation(r.cfg.Timezone) + } + r.wg.Add(1) go func() { defer r.wg.Done() @@ -185,7 +191,7 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs return nil } - err := r.parser.ParseFile(filePath, offset, onEventFunc) + err := parser.ParseFile(filePath, offset, onEventFunc) if err != nil { r.tctx.L().Error("parse file stopped", zap.Error(err)) select { diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 7980b67aca..242c0366f3 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -21,14 +21,17 @@ import ( "io/ioutil" "math/rand" "os" + "path" "path/filepath" "strconv" "sync" "testing" "time" + "github.com/BurntSushi/toml" . "github.com/pingcap/check" "github.com/pingcap/errors" + uuid "github.com/satori/go.uuid" "github.com/siddontang/go-mysql/mysql" gmysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" @@ -37,6 +40,7 @@ import ( tcontext "github.com/pingcap/dm/pkg/context" "github.com/pingcap/dm/pkg/gtid" "github.com/pingcap/dm/pkg/terror" + "github.com/pingcap/dm/pkg/utils" ) var parseFileTimeout = 10 * time.Second @@ -45,7 +49,7 @@ var _ = Suite(&testReaderSuite{}) type testReaderSuite struct { lastPos uint32 - lastGset gtid.Set + lastGTID gtid.Set } func TestReader(t *testing.T) { @@ -53,8 +57,10 @@ func TestReader(t *testing.T) { } func (t *testReaderSuite) SetUpSuite(c *C) { + var err error t.lastPos = 0 - t.lastGset, _ = gtid.ParserGTID(mysql.MySQLFlavor, "ba8f633f-1f15-11eb-b1c7-0242ac110002:30") + t.lastGTID, err = gtid.ParserGTID(mysql.MySQLFlavor, "ba8f633f-1f15-11eb-b1c7-0242ac110002:0") + c.Assert(err, IsNil) } func (t *testReaderSuite) TestParseFileBase(c *C) { @@ -64,7 +70,7 @@ func (t *testReaderSuite) TestParseFileBase(c *C) { offset int64 firstParse = true possibleLast = false - baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGset) + baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGTID) s = newLocalStreamer() ) @@ -76,7 +82,7 @@ func (t *testReaderSuite) TestParseFileBase(c *C) { // no valid currentUUID provide, failed currentUUID := "invalid-current-uuid" relayDir := filepath.Join(baseDir, currentUUID) - cfg := &BinlogReaderConfig{RelayDir: baseDir} + cfg := &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} r := NewBinlogReader(tctx, cfg) needSwitch, needReParse, latestPos, nextUUID, nextBinlogName, err := r.parseFile( ctx, s, filename, offset, relayDir, firstParse, currentUUID, possibleLast) @@ -91,7 +97,7 @@ func (t *testReaderSuite) TestParseFileBase(c *C) { currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" relayDir = filepath.Join(baseDir, currentUUID) fullPath := filepath.Join(relayDir, filename) - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} r = NewBinlogReader(tctx, cfg) // relay log file not exists, failed @@ -247,13 +253,13 @@ func (t *testReaderSuite) TestParseFileRelaySubDirUpdated(c *C) { offset int64 firstParse = true possibleLast = true - baseEvents, lastPos, lastGTID = t.genBinlogEvents(c, t.lastPos, t.lastGset) + baseEvents, lastPos, lastGTID = t.genBinlogEvents(c, t.lastPos, t.lastGTID) currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" relayDir = filepath.Join(baseDir, currentUUID) fullPath = filepath.Join(relayDir, filename) nextPath = filepath.Join(relayDir, nextFilename) s = newLocalStreamer() - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} tctx = tcontext.Background() r = NewBinlogReader(tctx, cfg) ) @@ -337,7 +343,7 @@ func (t *testReaderSuite) TestParseFileRelayNeedSwitchSubDir(c *C) { offset int64 firstParse = true possibleLast = true - baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGset) + baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGTID) currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" switchedUUID = "b60868af-5a6f-11e9-9ea3-0242ac160007.000002" relayDir = filepath.Join(baseDir, currentUUID) @@ -345,7 +351,7 @@ func (t *testReaderSuite) TestParseFileRelayNeedSwitchSubDir(c *C) { fullPath = filepath.Join(relayDir, filename) nextFullPath = filepath.Join(nextRelayDir, nextFilename) s = newLocalStreamer() - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} tctx = tcontext.Background() r = NewBinlogReader(tctx, cfg) ) @@ -408,12 +414,12 @@ func (t *testReaderSuite) TestParseFileRelayWithIgnorableError(c *C) { offset int64 firstParse = true possibleLast = true - baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGset) + baseEvents, _, _ = t.genBinlogEvents(c, t.lastPos, t.lastGTID) currentUUID = "b60868af-5a6f-11e9-9ea3-0242ac160006.000001" relayDir = filepath.Join(baseDir, currentUUID) fullPath = filepath.Join(relayDir, filename) s = newLocalStreamer() - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} tctx = tcontext.Background() r = NewBinlogReader(tctx, cfg) ) @@ -462,7 +468,7 @@ func (t *testReaderSuite) TestParseFileRelayWithIgnorableError(c *C) { func (t *testReaderSuite) TestUpdateUUIDs(c *C) { var ( baseDir = c.MkDir() - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} tctx = tcontext.Background() r = NewBinlogReader(tctx, cfg) ) @@ -487,18 +493,18 @@ func (t *testReaderSuite) TestUpdateUUIDs(c *C) { c.Assert(r.uuids, DeepEquals, UUIDs) } -func (t *testReaderSuite) TestStartSync(c *C) { +func (t *testReaderSuite) TestStartSyncByPos(c *C) { var ( filenamePrefix = "test-mysql-bin.00000" baseDir = c.MkDir() - baseEvents, lastPos, lastGset = t.genBinlogEvents(c, t.lastPos, t.lastGset) + baseEvents, lastPos, lastGTID = t.genBinlogEvents(c, t.lastPos, t.lastGTID) eventsBuf bytes.Buffer UUIDs = []string{ "b60868af-5a6f-11e9-9ea3-0242ac160006.000001", "b60868af-5a6f-11e9-9ea3-0242ac160007.000002", "b60868af-5a6f-11e9-9ea3-0242ac160008.000003", } - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} tctx = tcontext.Background() r = NewBinlogReader(tctx, cfg) startPos = gmysql.Position{Name: "test-mysql-bin|000001.000001"} // from the first relay log file in the first sub directory @@ -563,7 +569,7 @@ func (t *testReaderSuite) TestStartSync(c *C) { // 2. write more events to the last file lastFilename := filepath.Join(baseDir, UUIDs[2], filenamePrefix+strconv.Itoa(3)) - extraEvents, _, _ := t.genBinlogEvents(c, lastPos, lastGset) + extraEvents, _, _ := t.genBinlogEvents(c, lastPos, lastGTID) lastF, err := os.OpenFile(lastFilename, os.O_WRONLY|os.O_APPEND, 0600) c.Assert(err, IsNil) defer lastF.Close() @@ -618,13 +624,204 @@ func (t *testReaderSuite) TestStartSync(c *C) { r.Close() } +func (t *testReaderSuite) TestStartSyncByGTID(c *C) { + var ( + baseDir = c.MkDir() + events []*replication.BinlogEvent + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} + tctx = tcontext.Background() + r = NewBinlogReader(tctx, cfg) + lastPos uint32 + lastGTID gtid.Set + previousGset, _ = gtid.ParserGTID(mysql.MySQLFlavor, "") + ) + type FileEventType struct { + filename string + eventTypes []replication.EventType + } + + testCase := []struct { + uuid string + gtidStr string + fileEventTypes []FileEventType + }{ + { + "ba8f633f-1f15-11eb-b1c7-0242ac110002.000001", + "ba8f633f-1f15-11eb-b1c7-0242ac110002:0", + []FileEventType{ + { + "mysql.000001", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.ROTATE_EVENT, + }, + }, + { + "mysql.000002", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.ROTATE_EVENT, + }, + }, + { + "mysql.000003", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + }, + }, + }, + }, + { + "bf6227a7-1f15-11eb-9afb-0242ac110004.000002", + "bf6227a7-1f15-11eb-9afb-0242ac110004:20", + []FileEventType{ + { + "mysql.000001", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.ROTATE_EVENT, + }, + }, { + "mysql.000002", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.ROTATE_EVENT, + }, + }, { + "mysql.000003", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + }, + }, + }, + }, { + "bcbf9d42-1f15-11eb-a41c-0242ac110003.000003", + "bcbf9d42-1f15-11eb-a41c-0242ac110003:30", + []FileEventType{ + { + "mysql.000001", + []replication.EventType{ + replication.PREVIOUS_GTIDS_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.QUERY_EVENT, + replication.XID_EVENT, + replication.ROTATE_EVENT, + }, + }, + }, + }, + } + + for _, subDir := range testCase { + r.uuids = append(r.uuids, subDir.uuid) + } + + // wirte index file + uuidBytes := t.uuidListToBytes(c, r.uuids) + err := ioutil.WriteFile(r.indexPath, uuidBytes, 0600) + c.Assert(err, IsNil) + + var allEvents []*replication.BinlogEvent + for _, subDir := range testCase { + lastPos = 4 + lastGTID, err = gtid.ParserGTID(mysql.MySQLFlavor, subDir.gtidStr) + c.Assert(err, IsNil) + uuidDir := path.Join(baseDir, subDir.uuid) + err := os.MkdirAll(uuidDir, 0700) + c.Assert(err, IsNil) + + for _, fileEventType := range subDir.fileEventTypes { + events, lastPos, lastGTID, previousGset = t.genEvents(c, fileEventType.eventTypes, lastPos, lastGTID, previousGset) + allEvents = append(allEvents, events...) + + // write binlog file + f, err := os.OpenFile(path.Join(uuidDir, fileEventType.filename), os.O_CREATE|os.O_WRONLY, 0600) + c.Assert(err, IsNil) + _, err = f.Write(replication.BinLogFileHeader) + c.Assert(err, IsNil) + for _, ev := range events { + _, err = f.Write(ev.RawData) + c.Assert(err, IsNil) + } + f.Close() + + // write meta file + meta := Meta{BinLogName: fileEventType.filename, BinLogPos: lastPos, BinlogGTID: previousGset.String()} + metaFile, err := os.Create(path.Join(uuidDir, utils.MetaFilename)) + c.Assert(err, IsNil) + err = toml.NewEncoder(metaFile).Encode(meta) + c.Assert(err, IsNil) + metaFile.Close() + } + } + + startGTID, err := gtid.ParserGTID(mysql.MySQLFlavor, "") + c.Assert(err, IsNil) + s, err := r.StartSyncByGTID(startGTID.Origin().Clone()) + c.Assert(err, IsNil) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var obtainBaseEvents []*replication.BinlogEvent + for { + ev, err2 := s.GetEvent(ctx) + c.Assert(err2, IsNil) + obtainBaseEvents = append(obtainBaseEvents, ev) + // get event from second event(skip the nil previous event) + if len(obtainBaseEvents) == len(allEvents)-1 { + break + } + } + + for i, event := range obtainBaseEvents { + if i == 0 { + continue + } + c.Assert(event.Header, DeepEquals, allEvents[i+1].Header) + switch ev := event.Event.(type) { + case *replication.QueryEvent: + if bytes.Equal(ev.Query, []byte("BEGIN")) { + continue + } + pos, err := r.getPosByGTID(ev.GSet) + c.Assert(err, IsNil) + c.Assert(pos.Pos, Equals, event.Header.LogPos) + case *replication.XIDEvent: + pos, err := r.getPosByGTID(ev.GSet) + c.Assert(err, IsNil) + c.Assert(pos.Pos, Equals, event.Header.LogPos) + } + } +} + func (t *testReaderSuite) TestStartSyncError(c *C) { var ( baseDir = c.MkDir() UUIDs = []string{ "b60868af-5a6f-11e9-9ea3-0242ac160006.000001", } - cfg = &BinlogReaderConfig{RelayDir: baseDir} + cfg = &BinlogReaderConfig{RelayDir: baseDir, Flavor: mysql.MySQLFlavor} startPos = gmysql.Position{Name: "test-mysql-bin|000001.000001"} // from the first relay log file in the first sub directory ) @@ -643,7 +840,7 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(err, ErrorMatches, ".*empty UUIDs not valid.*") c.Assert(s, IsNil) - s, err = r.StartSyncByGTID(t.lastGset.Origin().Clone()) + s, err = r.StartSyncByGTID(t.lastGTID.Origin().Clone()) c.Assert(err, ErrorMatches, ".*no relay subdir match gtid.*") c.Assert(s, IsNil) @@ -658,7 +855,7 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(err, ErrorMatches, fmt.Sprintf(".*%s.*not found.*", startPos.Name)) c.Assert(s, IsNil) - s, err = r.StartSyncByGTID(t.lastGset.Origin().Clone()) + s, err = r.StartSyncByGTID(t.lastGTID.Origin().Clone()) c.Assert(err, ErrorMatches, ".*load meta data.*no such file or directory.*") c.Assert(s, IsNil) @@ -670,7 +867,7 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { r.Close() r.running = true - s, err = r.StartSyncByGTID(t.lastGset.Origin().Clone()) + s, err = r.StartSyncByGTID(t.lastGTID.Origin().Clone()) c.Assert(terror.ErrReaderAlreadyRunning.Equal(err), IsTrue) c.Assert(s, IsNil) r.Close() @@ -689,7 +886,7 @@ func (t *testReaderSuite) TestStartSyncError(c *C) { c.Assert(s, IsNil) } -func (t *testReaderSuite) genBinlogEvents(c *C, latestPos uint32, latestGset gtid.Set) ([]*replication.BinlogEvent, uint32, gtid.Set) { +func (t *testReaderSuite) genBinlogEvents(c *C, latestPos uint32, latestGTID gtid.Set) ([]*replication.BinlogEvent, uint32, gtid.Set) { var ( header = &replication.EventHeader{ Timestamp: uint32(time.Now().Unix()), @@ -708,14 +905,84 @@ func (t *testReaderSuite) genBinlogEvents(c *C, latestPos uint32, latestGset gti // for these tests, generates some DDL events is enough count := 5 + rand.Intn(5) for i := 0; i < count; i++ { - evs, err := event.GenDDLEvents(mysql.MySQLFlavor, 1, latestPos, latestGset, fmt.Sprintf("db_%d", i), fmt.Sprintf("CREATE TABLE %d (c1 INT)", i)) + evs, err := event.GenDDLEvents(mysql.MySQLFlavor, 1, latestPos, latestGTID, fmt.Sprintf("db_%d", i), fmt.Sprintf("CREATE TABLE %d (c1 INT)", i)) c.Assert(err, IsNil) events = append(events, evs.Events...) latestPos = evs.LatestPos - latestGset = evs.LatestGTID + latestGTID = evs.LatestGTID } - return events, latestPos, latestGset + return events, latestPos, latestGTID +} + +func (t *testReaderSuite) genEvents(c *C, eventTypes []replication.EventType, latestPos uint32, latestGTID gtid.Set, previousGset gtid.Set) ([]*replication.BinlogEvent, uint32, gtid.Set, gtid.Set) { + var ( + header = &replication.EventHeader{ + Timestamp: uint32(time.Now().Unix()), + ServerID: 11, + } + events = make([]*replication.BinlogEvent, 0, 10) + pGset = previousGset.Clone() + originSet = pGset.Origin() + ) + + if latestPos <= 4 { // generate a FormatDescriptionEvent if needed + ev, err := event.GenFormatDescriptionEvent(header, 4) + c.Assert(err, IsNil) + latestPos = ev.Header.LogPos + events = append(events, ev) + } + + for i, eventType := range eventTypes { + switch eventType { + case replication.QUERY_EVENT: + evs, err := event.GenDDLEvents(mysql.MySQLFlavor, 1, latestPos, latestGTID, fmt.Sprintf("db_%d", i), fmt.Sprintf("CREATE TABLE %d (c1 int)", i)) + c.Assert(err, IsNil) + events = append(events, evs.Events...) + latestPos = evs.LatestPos + latestGTID = evs.LatestGTID + ev, ok := evs.Events[0].Event.(*replication.GTIDEvent) + c.Assert(ok, IsTrue) + u, _ := uuid.FromBytes(ev.SID) + gs := fmt.Sprintf("%s:%d", u.String(), ev.GNO) + err = originSet.Update(gs) + c.Assert(err, IsNil) + case replication.XID_EVENT: + insertDMLData := []*event.DMLData{ + { + TableID: uint64(i), + Schema: fmt.Sprintf("db_%d", i), + Table: strconv.Itoa(i), + ColumnType: []byte{gmysql.MYSQL_TYPE_INT24}, + Rows: [][]interface{}{{int32(1)}, {int32(2)}}, + }, + } + evs, err := event.GenDMLEvents(mysql.MySQLFlavor, 1, latestPos, latestGTID, replication.WRITE_ROWS_EVENTv2, 10, insertDMLData) + c.Assert(err, IsNil) + events = append(events, evs.Events...) + latestPos = evs.LatestPos + latestGTID = evs.LatestGTID + ev, ok := evs.Events[0].Event.(*replication.GTIDEvent) + c.Assert(ok, IsTrue) + u, _ := uuid.FromBytes(ev.SID) + gs := fmt.Sprintf("%s:%d", u.String(), ev.GNO) + err = originSet.Update(gs) + c.Assert(err, IsNil) + case replication.ROTATE_EVENT: + ev, err := event.GenRotateEvent(header, latestPos, []byte("next_log"), 4) + c.Assert(err, IsNil) + events = append(events, ev) + latestPos = 4 + case replication.PREVIOUS_GTIDS_EVENT: + ev, err := event.GenPreviousGTIDsEvent(header, latestPos, pGset) + c.Assert(err, IsNil) + events = append(events, ev) + latestPos = ev.Header.LogPos + } + } + err := pGset.Set(originSet) + c.Assert(err, IsNil) + return events, latestPos, latestGTID, pGset } func (t *testReaderSuite) purgeStreamer(c *C, s Streamer) { From d788eef46e74112b4b9f17345ba1dd15f2564fb5 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 12 Nov 2020 18:06:07 +0800 Subject: [PATCH 08/59] change source config for test --- loader/status.go | 9 +++++---- tests/full_mode/conf/source1.yaml | 3 ++- tests/full_mode/conf/source2.yaml | 1 + tests/handle_error/conf/source1.yaml | 2 +- tests/handle_error/conf/source2.yaml | 2 +- tests/handle_error/run.sh | 12 ++++++------ tests/shardddl1/conf/source1.yaml | 2 +- tests/shardddl1/conf/source2.yaml | 2 +- tests/shardddl2/conf/source1.yaml | 2 +- tests/shardddl2/conf/source2.yaml | 2 +- tests/shardddl3/conf/source1.yaml | 2 +- tests/shardddl3/conf/source2.yaml | 2 +- 12 files changed, 22 insertions(+), 19 deletions(-) diff --git a/loader/status.go b/loader/status.go index 57ca5b1021..660524ffc0 100644 --- a/loader/status.go +++ b/loader/status.go @@ -33,10 +33,11 @@ func (l *Loader) Status() interface{} { totalSize := l.totalDataSize.Get() progress := percent(finishedSize, totalSize, l.finish.Get()) s := &pb.LoadStatus{ - FinishedBytes: finishedSize, - TotalBytes: totalSize, - Progress: progress, - MetaBinlog: l.metaBinlog.Get(), + FinishedBytes: finishedSize, + TotalBytes: totalSize, + Progress: progress, + MetaBinlog: l.metaBinlog.Get(), + MetaBinlogGTID: l.metaBinlogGTID.Get(), } return s } diff --git a/tests/full_mode/conf/source1.yaml b/tests/full_mode/conf/source1.yaml index c1cc021476..f4438825eb 100644 --- a/tests/full_mode/conf/source1.yaml +++ b/tests/full_mode/conf/source1.yaml @@ -1,7 +1,8 @@ server-id: 101 source-id: mysql-replica-01 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/full_mode/conf/source2.yaml b/tests/full_mode/conf/source2.yaml index 8b08f3b082..d84a86821d 100644 --- a/tests/full_mode/conf/source2.yaml +++ b/tests/full_mode/conf/source2.yaml @@ -2,6 +2,7 @@ server-id: 102 source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/handle_error/conf/source1.yaml b/tests/handle_error/conf/source1.yaml index 175e07df7a..692ce72eab 100644 --- a/tests/handle_error/conf/source1.yaml +++ b/tests/handle_error/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/handle_error/conf/source2.yaml b/tests/handle_error/conf/source2.yaml index 8850ed1849..28c0d00f6f 100644 --- a/tests/handle_error/conf/source2.yaml +++ b/tests/handle_error/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/handle_error/run.sh b/tests/handle_error/run.sh index ec86427348..874af45881 100644 --- a/tests/handle_error/run.sh +++ b/tests/handle_error/run.sh @@ -44,7 +44,7 @@ function DM_SKIP_ERROR_CASE() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 1 \ + "\"stage\": \"Running\"" 3 \ "\"stage\": \"Paused\"" 1 # skip all sources @@ -55,14 +55,14 @@ function DM_SKIP_ERROR_CASE() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # '11' -> 11, '22' -> 22, no error run_sql_source1 "insert into ${db}.${tb1} values('111',7)" run_sql_source2 "insert into ${db}.${tb2} values('222',8)" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_sql_tidb_with_retry "select count(1) from ${db}.${tb1} where id=111;" "count(1): 1" run_sql_tidb_with_retry "select count(1) from ${db}.${tb2} where id=222;" "count(1): 1" @@ -130,7 +130,7 @@ function DM_SKIP_ERROR_SHARDING_CASE() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_sql_tidb_with_retry "select count(1) from ${db}.${tb}" "count(1): 8" } @@ -290,7 +290,7 @@ function DM_REPLACE_ERROR_SHARDING_CASE() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 \ + "\"stage\": \"Running\"" 4 \ run_sql_tidb_with_retry "select count(1) from ${db}.${tb}" "count(1): 8" } @@ -359,7 +359,7 @@ function DM_REPLACE_ERROR_MULTIPLE_CASE() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 \ + "\"stage\": \"Running\"" 4 \ run_sql_tidb_with_retry "select count(1) from ${db}.${tb};" "count(1): 2" } diff --git a/tests/shardddl1/conf/source1.yaml b/tests/shardddl1/conf/source1.yaml index 01660cf685..8b2f8c4319 100644 --- a/tests/shardddl1/conf/source1.yaml +++ b/tests/shardddl1/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/shardddl1/conf/source2.yaml b/tests/shardddl1/conf/source2.yaml index 100a7bd6b3..31bd5de10e 100644 --- a/tests/shardddl1/conf/source2.yaml +++ b/tests/shardddl1/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/shardddl2/conf/source1.yaml b/tests/shardddl2/conf/source1.yaml index 01660cf685..8b2f8c4319 100644 --- a/tests/shardddl2/conf/source1.yaml +++ b/tests/shardddl2/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/shardddl2/conf/source2.yaml b/tests/shardddl2/conf/source2.yaml index 100a7bd6b3..31bd5de10e 100644 --- a/tests/shardddl2/conf/source2.yaml +++ b/tests/shardddl2/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/shardddl3/conf/source1.yaml b/tests/shardddl3/conf/source1.yaml index 01660cf685..8b2f8c4319 100644 --- a/tests/shardddl3/conf/source1.yaml +++ b/tests/shardddl3/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/shardddl3/conf/source2.yaml b/tests/shardddl3/conf/source2.yaml index 100a7bd6b3..31bd5de10e 100644 --- a/tests/shardddl3/conf/source2.yaml +++ b/tests/shardddl3/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root From 39238007851683809aa0158a9dcf6268fac2af1e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 12 Nov 2020 18:11:13 +0800 Subject: [PATCH 09/59] change error order --- _utils/terror_gen/errors_release.txt | 4 ++-- pkg/terror/error_list.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt index d4bfd73fbf..8653bb4506 100644 --- a/_utils/terror_gen/errors_release.txt +++ b/_utils/terror_gen/errors_release.txt @@ -45,8 +45,6 @@ ErrParserParseRelayLog,[code=11038:class=functional:scope=internal:level=high], ErrNoSubdirToSwitch,[code=11039:class=functional:scope=internal:level=high], "Message: parse for previous sub relay directory finished, but no next sub directory need to switch" ErrNeedSyncAgain,[code=11040:class=functional:scope=internal:level=high], "Message: Last sync error or closed, try sync and get event again" ErrSyncClosed,[code=11041:class=functional:scope=internal:level=high], "Message: Sync was closed" -ErrNoUUIDDirMatchGTID,[code=11120:class=functional:scope=internal:level=high], "Message: no relay subdir match gtid %s" -ErrNoRelayPosMatchGTID,[code=11121:class=functional:scope=internal:level=high], "Message: no relay pos match gtid %s" ErrSchemaTableNameNotValid,[code=11042:class=functional:scope=internal:level=high], "Message: table name %s not valid" ErrGenTableRouter,[code=11043:class=functional:scope=internal:level=high], "Message: generate table router, Workaround: Please check `routes` config in task configuration file." ErrEncryptSecretKeyNotValid,[code=11044:class=functional:scope=internal:level=high], "Message: key size should be 16, 24 or 32, but input key's size is %d" @@ -125,6 +123,8 @@ ErrFailUpdateV1DBSchema,[code=11116:class=functional:scope=internal:level=medium ErrBinlogStatusVarsParse,[code=11117:class=functional:scope=internal:level=medium], "Message: fail to parse binglog status_vars: %v, offset: %d" ErrVerifyHandleErrorArgs,[code=11118:class=functional:scope=internal:level=low], "Workaround: Please make sure the args are correct." ErrRewriteSQL,[code=11119:class=functional:scope=internal:level=high], "Message: failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v" +ErrNoUUIDDirMatchGTID,[code=11120:class=functional:scope=internal:level=high], "Message: no relay subdir match gtid %s" +ErrNoRelayPosMatchGTID,[code=11121:class=functional:scope=internal:level=high], "Message: no relay pos match gtid %s" ErrConfigCheckItemNotSupport,[code=20001:class=config:scope=internal:level=medium], "Message: checking item %s is not supported\n%s, Workaround: Please check `ignore-checking-items` config in task configuration file, which can be set including `all`/`dump_privilege`/`replication_privilege`/`version`/`binlog_enable`/`binlog_format`/`binlog_row_image`/`table_schema`/`schema_of_shard_tables`/`auto_increment_ID`." ErrConfigTomlTransform,[code=20002:class=config:scope=internal:level=medium], "Message: %s, Workaround: Please check the configuration file has correct TOML format." ErrConfigYamlTransform,[code=20003:class=config:scope=internal:level=medium], "Message: %s, Workaround: Please check the configuration file has correct YAML format." diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index e3ff900f6e..7310bcda1a 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -657,8 +657,6 @@ var ( ErrNoSubdirToSwitch = New(codeNoSubdirToSwitch, ClassFunctional, ScopeInternal, LevelHigh, "parse for previous sub relay directory finished, but no next sub directory need to switch", "") ErrNeedSyncAgain = New(codeNeedSyncAgain, ClassFunctional, ScopeInternal, LevelHigh, "Last sync error or closed, try sync and get event again", "") ErrSyncClosed = New(codeSyncClosed, ClassFunctional, ScopeInternal, LevelHigh, "Sync was closed", "") - ErrNoUUIDDirMatchGTID = New(codeNoUUIDDirMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay subdir match gtid %s", "") - ErrNoRelayPosMatchGTID = New(codeNoRelayPosMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay pos match gtid %s", "") // pkg/utils ErrSchemaTableNameNotValid = New(codeSchemaTableNameNotValid, ClassFunctional, ScopeInternal, LevelHigh, "table name %s not valid", "") ErrGenTableRouter = New(codeGenTableRouter, ClassFunctional, ScopeInternal, LevelHigh, "generate table router", "Please check `routes` config in task configuration file.") @@ -767,6 +765,10 @@ var ( // pkg/parser ErrRewriteSQL = New(codeRewriteSQL, ClassFunctional, ScopeInternal, LevelHigh, "failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v", "") + // pkg/streamer + ErrNoUUIDDirMatchGTID = New(codeNoUUIDDirMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay subdir match gtid %s", "") + ErrNoRelayPosMatchGTID = New(codeNoRelayPosMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay pos match gtid %s", "") + // Config related error ErrConfigCheckItemNotSupport = New(codeConfigCheckItemNotSupport, ClassConfig, ScopeInternal, LevelMedium, "checking item %s is not supported\n%s", "Please check `ignore-checking-items` config in task configuration file, which can be set including `all`/`dump_privilege`/`replication_privilege`/`version`/`binlog_enable`/`binlog_format`/`binlog_row_image`/`table_schema`/`schema_of_shard_tables`/`auto_increment_ID`.") ErrConfigTomlTransform = New(codeConfigTomlTransform, ClassConfig, ScopeInternal, LevelMedium, "%s", "Please check the configuration file has correct TOML format.") From 0f5433b7c801a4221bb402ef71169a11b80bc9ab Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 13 Nov 2020 09:51:43 +0800 Subject: [PATCH 10/59] revert full-mode config --- tests/full_mode/conf/source1.yaml | 3 +-- tests/full_mode/conf/source2.yaml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/full_mode/conf/source1.yaml b/tests/full_mode/conf/source1.yaml index f4438825eb..c1cc021476 100644 --- a/tests/full_mode/conf/source1.yaml +++ b/tests/full_mode/conf/source1.yaml @@ -1,8 +1,7 @@ server-id: 101 source-id: mysql-replica-01 flavor: '' -enable-gtid: true -enable-relay: true +enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/full_mode/conf/source2.yaml b/tests/full_mode/conf/source2.yaml index d84a86821d..8b08f3b082 100644 --- a/tests/full_mode/conf/source2.yaml +++ b/tests/full_mode/conf/source2.yaml @@ -2,7 +2,6 @@ server-id: 102 source-id: mysql-replica-02 flavor: '' enable-gtid: false -enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: From 83457ca656afa2ec779a48a52cccf6e3088dd5b4 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 13 Nov 2020 10:43:33 +0800 Subject: [PATCH 11/59] fix ut --- pkg/streamer/reader_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 242c0366f3..388a57dc02 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -748,7 +748,7 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { lastGTID, err = gtid.ParserGTID(mysql.MySQLFlavor, subDir.gtidStr) c.Assert(err, IsNil) uuidDir := path.Join(baseDir, subDir.uuid) - err := os.MkdirAll(uuidDir, 0700) + err = os.MkdirAll(uuidDir, 0700) c.Assert(err, IsNil) for _, fileEventType := range subDir.fileEventTypes { @@ -787,15 +787,21 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { for { ev, err2 := s.GetEvent(ctx) c.Assert(err2, IsNil) + if ev.Header.Timestamp == 0 || ev.Header.LogPos == 0 { + continue // ignore fake event + } obtainBaseEvents = append(obtainBaseEvents, ev) - // get event from second event(skip the nil previous event) + // start after the first nil previous event if len(obtainBaseEvents) == len(allEvents)-1 { break } } + // allEvents: [FORMAT_DESCRIPTION_EVENT, PREVIOUS_GTIDS_EVENT, GTID_EVENT, QUERY_EVENT...] + // obtainBaseEvents: [FORMAT_DESCRIPTION_EVENT(generated), GTID_EVENT, QUERY_EVENT...] for i, event := range obtainBaseEvents { if i == 0 { + c.Assert(event.Header.EventType, Equals, replication.FORMAT_DESCRIPTION_EVENT) continue } c.Assert(event.Header, DeepEquals, allEvents[i+1].Header) From f0e7e8c9489ee8702bd58e5b53077730de7d1238 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 16 Nov 2020 14:09:55 +0800 Subject: [PATCH 12/59] address comment --- pkg/streamer/reader.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index 914b22616e..b5259ea651 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -243,9 +243,7 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs // gset is the last txn in file return lastPos, nil } - case *replication.XIDEvent: - lastPos = e.Header.LogPos - case *replication.QueryEvent: + case *replication.XIDEvent, *replication.QueryEvent: lastPos = e.Header.LogPos case *replication.GTIDEvent: u, _ := uuid.FromBytes(ev.SID) From 664b70f4b6788381b00d9dd2f81addc6006a5ba5 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 16 Nov 2020 18:21:14 +0800 Subject: [PATCH 13/59] fix sync gtid from oldest file --- pkg/streamer/reader.go | 9 +++++++-- pkg/streamer/reader_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index b5259ea651..6f0ecc24d4 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -164,7 +164,7 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { } // GetFilePosByGTID tries to get Pos by GTID for file -func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet) (uint32, error) { +func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet, oldest bool) (uint32, error) { ctx2, cancel := context.WithCancel(ctx) defer cancel() @@ -236,6 +236,11 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs } // if PreviousGITDsEvent contain the gset, go to previous file if gs.Contain(gset) { + // if it's the oldest file and gs equals gset, continue + if oldest && gset.Contain(gs) { + lastPos = e.Header.LogPos + continue + } return 0, nil } case *replication.RotateEvent: @@ -291,7 +296,7 @@ func (r *BinlogReader) getPosByGTID(gset mysql.GTIDSet) (*mysql.Position, error) for i := len(allFiles) - 1; i >= 0; i-- { file := allFiles[i] filePath := path.Join(uuidDir, file) - pos, err := r.GetFilePosByGTID(r.tctx.Ctx, filePath, gset) + pos, err := r.GetFilePosByGTID(r.tctx.Ctx, filePath, gset, i == 0) if err != nil { return nil, err } diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 388a57dc02..9600598eff 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -756,8 +756,8 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { allEvents = append(allEvents, events...) // write binlog file - f, err := os.OpenFile(path.Join(uuidDir, fileEventType.filename), os.O_CREATE|os.O_WRONLY, 0600) - c.Assert(err, IsNil) + f, err2 := os.OpenFile(path.Join(uuidDir, fileEventType.filename), os.O_CREATE|os.O_WRONLY, 0600) + c.Assert(err2, IsNil) _, err = f.Write(replication.BinLogFileHeader) c.Assert(err, IsNil) for _, ev := range events { @@ -768,8 +768,8 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { // write meta file meta := Meta{BinLogName: fileEventType.filename, BinLogPos: lastPos, BinlogGTID: previousGset.String()} - metaFile, err := os.Create(path.Join(uuidDir, utils.MetaFilename)) - c.Assert(err, IsNil) + metaFile, err2 := os.Create(path.Join(uuidDir, utils.MetaFilename)) + c.Assert(err2, IsNil) err = toml.NewEncoder(metaFile).Encode(meta) c.Assert(err, IsNil) metaFile.Close() From 57c83e1d73f1b74cefe84e46022835d56391e037 Mon Sep 17 00:00:00 2001 From: GMHDBJD <35025882+GMHDBJD@users.noreply.github.com> Date: Mon, 16 Nov 2020 18:29:04 +0800 Subject: [PATCH 14/59] Update pkg/streamer/reader_test.go Co-authored-by: lance6716 --- pkg/streamer/reader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 9600598eff..906763fe54 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -737,7 +737,7 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { r.uuids = append(r.uuids, subDir.uuid) } - // wirte index file + // write index file uuidBytes := t.uuidListToBytes(c, r.uuids) err := ioutil.WriteFile(r.indexPath, uuidBytes, 0600) c.Assert(err, IsNil) From e4b6b7769805cb532c8a63bd63da01b29b6e3c7d Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 17 Nov 2020 13:57:42 +0800 Subject: [PATCH 15/59] remove oldest check --- pkg/streamer/reader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index 6f0ecc24d4..4622ad0159 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -164,7 +164,7 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { } // GetFilePosByGTID tries to get Pos by GTID for file -func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet, oldest bool) (uint32, error) { +func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet) (uint32, error) { ctx2, cancel := context.WithCancel(ctx) defer cancel() @@ -237,7 +237,7 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs // if PreviousGITDsEvent contain the gset, go to previous file if gs.Contain(gset) { // if it's the oldest file and gs equals gset, continue - if oldest && gset.Contain(gs) { + if gset.Contain(gs) { lastPos = e.Header.LogPos continue } @@ -296,7 +296,7 @@ func (r *BinlogReader) getPosByGTID(gset mysql.GTIDSet) (*mysql.Position, error) for i := len(allFiles) - 1; i >= 0; i-- { file := allFiles[i] filePath := path.Join(uuidDir, file) - pos, err := r.GetFilePosByGTID(r.tctx.Ctx, filePath, gset, i == 0) + pos, err := r.GetFilePosByGTID(r.tctx.Ctx, filePath, gset) if err != nil { return nil, err } From cf32bc618af10e18621468ba3f23502a9288f228 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 17 Nov 2020 19:54:19 +0800 Subject: [PATCH 16/59] address comment --- pkg/streamer/reader.go | 19 ++++++++----------- pkg/streamer/reader_test.go | 36 ++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index 4622ad0159..b26ded0064 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -225,31 +225,28 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs } switch ev := e.Event.(type) { case *replication.PreviousGTIDsEvent: - // nil previous gtid event + // nil previous gtid event, continue to parse file if len(ev.GTIDSets) == 0 { - lastPos = e.Header.LogPos - continue + break } gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, ev.GTIDSets) if err != nil { return 0, err } - // if PreviousGITDsEvent contain the gset, go to previous file + // if PreviousGITDsEvent contain but not equal the gset, go to previous file if gs.Contain(gset) { - // if it's the oldest file and gs equals gset, continue + // continue to parse file if gset equals gs if gset.Contain(gs) { - lastPos = e.Header.LogPos - continue + break } return 0, nil } case *replication.RotateEvent: + // should not happen if e.Header.Timestamp != 0 && e.Header.LogPos != 0 { - // gset is the last txn in file return lastPos, nil } - case *replication.XIDEvent, *replication.QueryEvent: - lastPos = e.Header.LogPos + continue case *replication.GTIDEvent: u, _ := uuid.FromBytes(ev.SID) gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, fmt.Sprintf("%s:%d", u.String(), ev.GNO)) @@ -270,8 +267,8 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs if !gset.Contain(gs) { return lastPos, nil } - default: } + lastPos = e.Header.LogPos } } diff --git a/pkg/streamer/reader_test.go b/pkg/streamer/reader_test.go index 906763fe54..7c99b982a2 100644 --- a/pkg/streamer/reader_test.go +++ b/pkg/streamer/reader_test.go @@ -24,6 +24,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "sync" "testing" "time" @@ -641,11 +642,13 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { } testCase := []struct { + serverUUID string uuid string gtidStr string fileEventTypes []FileEventType }{ { + "ba8f633f-1f15-11eb-b1c7-0242ac110002", "ba8f633f-1f15-11eb-b1c7-0242ac110002.000001", "ba8f633f-1f15-11eb-b1c7-0242ac110002:0", []FileEventType{ @@ -684,6 +687,7 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { }, }, { + "bf6227a7-1f15-11eb-9afb-0242ac110004", "bf6227a7-1f15-11eb-9afb-0242ac110004.000002", "bf6227a7-1f15-11eb-9afb-0242ac110004:20", []FileEventType{ @@ -715,6 +719,7 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { }, }, }, { + "bcbf9d42-1f15-11eb-a41c-0242ac110003", "bcbf9d42-1f15-11eb-a41c-0242ac110003.000003", "bcbf9d42-1f15-11eb-a41c-0242ac110003:30", []FileEventType{ @@ -726,7 +731,6 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { replication.XID_EVENT, replication.QUERY_EVENT, replication.XID_EVENT, - replication.ROTATE_EVENT, }, }, }, @@ -743,7 +747,12 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { c.Assert(err, IsNil) var allEvents []*replication.BinlogEvent + + // prePosMap record previous uuid's last file's size + // when master switch, we get the previous uuid's last pos now + prePosMap := make(map[string]uint32) for _, subDir := range testCase { + prePosMap[subDir.serverUUID] = lastPos lastPos = 4 lastGTID, err = gtid.ParserGTID(mysql.MySQLFlavor, subDir.gtidStr) c.Assert(err, IsNil) @@ -797,6 +806,8 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { } } + preGset, err := gtid.ParserGTID(mysql.MySQLFlavor, "") + c.Assert(err, IsNil) // allEvents: [FORMAT_DESCRIPTION_EVENT, PREVIOUS_GTIDS_EVENT, GTID_EVENT, QUERY_EVENT...] // obtainBaseEvents: [FORMAT_DESCRIPTION_EVENT(generated), GTID_EVENT, QUERY_EVENT...] for i, event := range obtainBaseEvents { @@ -806,17 +817,22 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *C) { } c.Assert(event.Header, DeepEquals, allEvents[i+1].Header) switch ev := event.Event.(type) { - case *replication.QueryEvent: - if bytes.Equal(ev.Query, []byte("BEGIN")) { - continue + case *replication.GTIDEvent: + pos, err2 := r.getPosByGTID(preGset.Origin().Clone()) + u, _ := uuid.FromBytes(ev.SID) + c.Assert(err2, IsNil) + // new uuid dir + if len(preGset.String()) != 0 && !strings.Contains(preGset.String(), u.String()) { + c.Assert(pos.Pos, Equals, prePosMap[u.String()], Commentf("a %d", i)) + } else { + c.Assert(pos.Pos, Equals, event.Header.LogPos-event.Header.EventSize, Commentf("b %d", i)) } - pos, err := r.getPosByGTID(ev.GSet) - c.Assert(err, IsNil) - c.Assert(pos.Pos, Equals, event.Header.LogPos) + case *replication.QueryEvent: + err2 := preGset.Set(ev.GSet) + c.Assert(err2, IsNil) case *replication.XIDEvent: - pos, err := r.getPosByGTID(ev.GSet) - c.Assert(err, IsNil) - c.Assert(pos.Pos, Equals, event.Header.LogPos) + err2 := preGset.Set(ev.GSet) + c.Assert(err2, IsNil) } } } From 241e27882e4522aa720c8701763a1e7bf86eea49 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 19 Nov 2020 15:38:44 +0800 Subject: [PATCH 17/59] support source switch between worker in relay --- dm/config/source_config.go | 2 ++ dm/ctl/ctl.go | 4 +-- dm/worker/relay_test.go | 8 +++++ dm/worker/server.go | 38 ++++++++++---------- dm/worker/server_test.go | 59 +++++++++++++++++++++++--------- dm/worker/worker_test.go | 4 +-- pkg/binlog/position.go | 14 ++++++++ pkg/utils/relay.go | 5 +++ relay/config.go | 2 ++ relay/meta.go | 12 ++++--- relay/meta_test.go | 4 +-- relay/relay.go | 39 ++++++++++++++++++--- relay/relay_test.go | 8 ++--- relay/transformer/transformer.go | 2 ++ 14 files changed, 149 insertions(+), 52 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 0d192064d1..b2002af9fc 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -59,6 +59,8 @@ type SourceConfig struct { // relay synchronous starting point (if specified) RelayBinLogName string `yaml:"relay-binlog-name" toml:"relay-binlog-name" json:"relay-binlog-name"` RelayBinlogGTID string `yaml:"relay-binlog-gtid" toml:"relay-binlog-gtid" json:"relay-binlog-gtid"` + // only use when worker bound source, do not marsh it + UUIDSuffix string `yaml:"-" toml:"-" json:"-"` SourceID string `yaml:"source-id" toml:"source-id" json:"source-id"` From DBConfig `yaml:"from" toml:"from" json:"from"` diff --git a/dm/ctl/ctl.go b/dm/ctl/ctl.go index 6752fe2fd4..71fc621b4b 100644 --- a/dm/ctl/ctl.go +++ b/dm/ctl/ctl.go @@ -64,8 +64,8 @@ func NewRootCmd() *cobra.Command { master.NewQueryStatusCmd(), master.NewShowDDLLocksCmd(), master.NewUnlockDDLLockCmd(), - // master.NewPauseRelayCmd(), - // master.NewResumeRelayCmd(), + master.NewPauseRelayCmd(), + master.NewResumeRelayCmd(), // master.NewPurgeRelayCmd(), master.NewOperateSourceCmd(), master.NewOfflineMemberCmd(), diff --git a/dm/worker/relay_test.go b/dm/worker/relay_test.go index 9f392d71e6..10b96c4c3f 100644 --- a/dm/worker/relay_test.go +++ b/dm/worker/relay_test.go @@ -121,6 +121,14 @@ func (d *DummyRelay) SaveMeta(pos mysql.Position, gset gtid.Set) error { return nil } +// ResetMeta implements Process interface +func (d *DummyRelay) ResetMeta() {} + +// PurgeRelayDir implements Process interface +func (d *DummyRelay) PurgeRelayDir() error { + return nil +} + func (t *testRelay) TestRelay(c *C) { originNewRelay := relay.NewRelay relay.NewRelay = NewDummyRelay diff --git a/dm/worker/server.go b/dm/worker/server.go index 588530b8aa..2679a16b6b 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -49,7 +49,7 @@ var ( keepaliveTime = 3 * time.Second retryConnectSleepTime = time.Second syncMasterEndpointsTime = 3 * time.Second - getMinPosForSubTaskFunc = getMinPosForSubTask + getMinLocForSubTaskFunc = getMinLocForSubTask ) // Server accepts RPC requests @@ -546,16 +546,18 @@ func (s *Server) startWorker(cfg *config.SourceConfig) error { if cfg.EnableRelay { dctx, dcancel := context.WithTimeout(s.etcdClient.Ctx(), time.Duration(len(subTaskCfgs))*3*time.Second) defer dcancel() - minPos, err1 := getMinPosInAllSubTasks(dctx, subTaskCfgs) + minLoc, err1 := getMinLocInAllSubTasks(dctx, subTaskCfgs) if err1 != nil { return err1 } - // TODO: support GTID - // don't contain GTID information in checkpoint table, just set it to empty - if minPos != nil { - cfg.RelayBinLogName = binlog.AdjustPosition(*minPos).Name - cfg.RelayBinlogGTID = "" + if minLoc != nil { + cfg.RelayBinLogName = binlog.AdjustPosition(minLoc.Position).Name + cfg.RelayBinlogGTID = minLoc.GTIDSetStr() + cfg.UUIDSuffix, err = binlog.ExtractSuffix(minLoc.Position.Name) + if err != nil { + return err + } } } @@ -634,32 +636,32 @@ func makeCommonWorkerResponse(reqErr error) *pb.CommonWorkerResponse { } // all subTask in subTaskCfgs should have same source -// this function return the min position in all subtasks, used for relay's position +// this function return the min location in all subtasks, used for relay's location // TODO: get min gtidSet -func getMinPosInAllSubTasks(ctx context.Context, subTaskCfgs []*config.SubTaskConfig) (minPos *mysql.Position, err error) { +func getMinLocInAllSubTasks(ctx context.Context, subTaskCfgs []*config.SubTaskConfig) (minLoc *binlog.Location, err error) { for _, subTaskCfg := range subTaskCfgs { - pos, err := getMinPosForSubTaskFunc(ctx, subTaskCfg) + loc, err := getMinLocForSubTaskFunc(ctx, subTaskCfg) if err != nil { return nil, err } - if pos == nil { + if loc == nil { continue } - if minPos == nil { - minPos = pos + if minLoc == nil { + minLoc = loc } else { - if minPos.Compare(*pos) >= 1 { - minPos = pos + if binlog.CompareLocation(*minLoc, *loc, subTaskCfg.EnableGTID) >= 1 { + minLoc = loc } } } - return minPos, nil + return minLoc, nil } -func getMinPosForSubTask(ctx context.Context, subTaskCfg *config.SubTaskConfig) (minPos *mysql.Position, err error) { +func getMinLocForSubTask(ctx context.Context, subTaskCfg *config.SubTaskConfig) (minLoc *binlog.Location, err error) { if subTaskCfg.Mode == config.ModeFull { return nil, nil } @@ -682,7 +684,7 @@ func getMinPosForSubTask(ctx context.Context, subTaskCfg *config.SubTaskConfig) } location := checkpoint.GlobalPoint() - return &location.Position, nil + return &location, nil } // unifyMasterBinlogPos eliminates different masterBinlog in one response diff --git a/dm/worker/server_test.go b/dm/worker/server_test.go index 9c5f1a255c..f5a2a23399 100644 --- a/dm/worker/server_test.go +++ b/dm/worker/server_test.go @@ -33,6 +33,8 @@ import ( "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/dm/pb" "github.com/pingcap/dm/dm/unit" + "github.com/pingcap/dm/pkg/binlog" + "github.com/pingcap/dm/pkg/gtid" "github.com/pingcap/dm/pkg/ha" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" @@ -60,11 +62,11 @@ func (t *testServer) SetUpSuite(c *C) { err := log.InitLogger(&log.Config{}) c.Assert(err, IsNil) - getMinPosForSubTaskFunc = getFakePosForSubTask + getMinLocForSubTaskFunc = getFakeLocForSubTask } func (t *testServer) TearDownSuite(c *C) { - getMinPosForSubTaskFunc = getMinPosForSubTask + getMinLocForSubTaskFunc = getMinLocForSubTask } func createMockETCD(dir string, host string) (*embed.Etcd, error) { @@ -414,10 +416,19 @@ func (t *testServer) TestGetMinPosInAllSubTasks(c *C) { Name: "test1", }, } - minPos, err := getMinPosInAllSubTasks(context.Background(), subTaskCfg) + minLoc, err := getMinLocInAllSubTasks(context.Background(), subTaskCfg) c.Assert(err, IsNil) - c.Assert(minPos.Name, Equals, "mysql-binlog.00001") - c.Assert(minPos.Pos, Equals, uint32(12)) + c.Assert(minLoc.Position.Name, Equals, "mysql-binlog.00001") + c.Assert(minLoc.Position.Pos, Equals, uint32(12)) + + for _, subtask := range subTaskCfg { + subtask.EnableGTID = true + } + + minLoc, err = getMinLocInAllSubTasks(context.Background(), subTaskCfg) + c.Assert(err, IsNil) + c.Assert(minLoc.Position.Name, Equals, "mysql-binlog.00001") + c.Assert(minLoc.Position.Pos, Equals, uint32(123)) } func (t *testServer) TestUnifyMasterBinlogPos(c *C) { @@ -530,22 +541,38 @@ func (t *testServer) TestUnifyMasterBinlogPos(c *C) { c.Assert(relay.RelayCatchUpMaster, IsTrue) } -func getFakePosForSubTask(ctx context.Context, subTaskCfg *config.SubTaskConfig) (minPos *mysql.Position, err error) { - switch subTaskCfg.Name { - case "test1": - return &mysql.Position{ +func getFakeLocForSubTask(ctx context.Context, subTaskCfg *config.SubTaskConfig) (minLoc *binlog.Location, err error) { + gset1, _ := gtid.ParserGTID(mysql.MySQLFlavor, "ba8f633f-1f15-11eb-b1c7-0242ac110001:1-30") + gset2, _ := gtid.ParserGTID(mysql.MySQLFlavor, "ba8f633f-1f15-11eb-b1c7-0242ac110001:1-50") + gset3, _ := gtid.ParserGTID(mysql.MySQLFlavor, "ba8f633f-1f15-11eb-b1c7-0242ac110001:1-50,ba8f633f-1f15-11eb-b1c7-0242ac110002:1") + loc1 := binlog.InitLocation( + mysql.Position{ Name: "mysql-binlog.00001", Pos: 123, - }, nil - case "test2": - return &mysql.Position{ + }, + gset1, + ) + loc2 := binlog.InitLocation( + mysql.Position{ Name: "mysql-binlog.00001", Pos: 12, - }, nil - case "test3": - return &mysql.Position{ + }, + gset2, + ) + loc3 := binlog.InitLocation( + mysql.Position{ Name: "mysql-binlog.00003", - }, nil + }, + gset3, + ) + + switch subTaskCfg.Name { + case "test1": + return &loc1, nil + case "test2": + return &loc2, nil + case "test3": + return &loc3, nil default: return nil, nil } diff --git a/dm/worker/worker_test.go b/dm/worker/worker_test.go index 8d6ad889d9..23d9d5a942 100644 --- a/dm/worker/worker_test.go +++ b/dm/worker/worker_test.go @@ -92,11 +92,11 @@ func (t *testServer2) SetUpSuite(c *C) { err := log.InitLogger(&log.Config{}) c.Assert(err, IsNil) - getMinPosForSubTaskFunc = getFakePosForSubTask + getMinLocForSubTaskFunc = getFakeLocForSubTask } func (t *testServer2) TearDownSuite(c *C) { - getMinPosForSubTaskFunc = getMinPosForSubTask + getMinLocForSubTaskFunc = getMinLocForSubTask } func (t *testServer2) TestTaskAutoResume(c *C) { diff --git a/pkg/binlog/position.go b/pkg/binlog/position.go index edd9d7a0af..19bce2c515 100644 --- a/pkg/binlog/position.go +++ b/pkg/binlog/position.go @@ -36,6 +36,7 @@ const ( // eg. mysql-bin.000003 in c6ae5afe-c7a3-11e8-a19d-0242ac130006.000002 => mysql-bin|000002.000003 // where `000002` in `c6ae5afe-c7a3-11e8-a19d-0242ac130006.000002` is the UUIDSuffix posUUIDSuffixSeparator = "|" + minUUIDSuffix = 1 ) var ( @@ -110,6 +111,19 @@ func RealMySQLPos(pos gmysql.Position) (gmysql.Position, error) { return pos, nil } +// ExtractSuffix extracts uuidSuffix from input name +func ExtractSuffix(name string) (string, error) { + filename, err := ParseFilename(name) + if err != nil { + return "", err + } + sepIdx := strings.LastIndex(filename.BaseName, posUUIDSuffixSeparator) + if sepIdx > 0 && sepIdx+len(posUUIDSuffixSeparator) < len(filename.BaseName) { + return filename.BaseName[sepIdx+len(posUUIDSuffixSeparator):], nil + } + return fmt.Sprintf("%06d", minUUIDSuffix), nil +} + // ExtractPos extracts (uuidWithSuffix, uuidSuffix, originalPos) from input pos (originalPos or convertedPos) func ExtractPos(pos gmysql.Position, uuids []string) (uuidWithSuffix string, uuidSuffix string, realPos gmysql.Position, err error) { if len(uuids) == 0 { diff --git a/pkg/utils/relay.go b/pkg/utils/relay.go index ce1f31ee7e..a6a4cf8002 100644 --- a/pkg/utils/relay.go +++ b/pkg/utils/relay.go @@ -72,6 +72,11 @@ func AddSuffixForUUID(uuid string, ID int) string { return fmt.Sprintf("%s%s%06d", uuid, uuidIndexSeparator, ID) // eg. 53ea0ed1-9bf8-11e6-8bea-64006a897c73.000001 } +// AddSuffixStrForUUID adds a suffix string for UUID +func AddSuffixStrForUUID(uuid string, suffix string) string { + return fmt.Sprintf("%s%s%s", uuid, uuidIndexSeparator, suffix) // eg. 53ea0ed1-9bf8-11e6-8bea-64006a897c73.000001 +} + // SuffixIntToStr convert int-represented suffix to string-represented func SuffixIntToStr(ID int) string { return fmt.Sprintf("%06d", ID) diff --git a/relay/config.go b/relay/config.go index ce4c84c8d2..294e688739 100644 --- a/relay/config.go +++ b/relay/config.go @@ -35,6 +35,7 @@ type Config struct { // do not need to specify binlog-pos, because relay will fetch the whole file BinLogName string `toml:"binlog-name" json:"binlog-name"` BinlogGTID string `toml:"binlog-gtid" json:"binlog-gtid"` + UUIDSuffix string `toml:"-" json:"-"` // for binlog reader retry ReaderRetry retry.ReaderRetryConfig `toml:"reader-retry" json:"reader-retry"` @@ -61,6 +62,7 @@ func FromSourceCfg(sourceCfg *config.SourceConfig) *Config { From: clone.From, BinLogName: clone.RelayBinLogName, BinlogGTID: clone.RelayBinlogGTID, + UUIDSuffix: clone.UUIDSuffix, ReaderRetry: retry.ReaderRetryConfig{ // we use config from TaskChecker now BackoffRollback: clone.Checker.BackoffRollback.Duration, BackoffMax: clone.Checker.BackoffMax.Duration, diff --git a/relay/meta.go b/relay/meta.go index 5560c1beba..b9c24fe1e1 100644 --- a/relay/meta.go +++ b/relay/meta.go @@ -63,7 +63,7 @@ type Meta interface { // and binlog pos will be set again when new binlog events received // @serverUUID should be a server_uuid for MySQL or MariaDB // if set @newPos / @newGTID, old value will be replaced - AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set) error + AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set, uuidSuffix string) error // Pos returns current (UUID with suffix, Position) pair Pos() (string, mysql.Position) @@ -268,7 +268,7 @@ func (lm *LocalMeta) Dir() string { } // AddDir implements Meta.AddDir -func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set) error { +func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set, uuidSuffix string) error { lm.Lock() defer lm.Unlock() @@ -276,7 +276,11 @@ func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID g if len(lm.currentUUID) == 0 { // no UUID exists yet, simply add it - newUUID = utils.AddSuffixForUUID(serverUUID, minUUIDSufix) + if len(uuidSuffix) == 0 { + newUUID = utils.AddSuffixForUUID(serverUUID, minUUIDSufix) + } else { + newUUID = utils.AddSuffixStrForUUID(serverUUID, uuidSuffix) + } } else { _, suffix, err := utils.ParseSuffixForUUID(lm.currentUUID) if err != nil { @@ -343,7 +347,7 @@ func (lm *LocalMeta) GTID() (string, gtid.Set) { lm.RLock() defer lm.RUnlock() - return lm.currentUUID, lm.gset.Clone() + return lm.currentUUID, lm.gset } // UUID implements Meta.UUID diff --git a/relay/meta_test.go b/relay/meta_test.go index df55d6831c..082394a34a 100644 --- a/relay/meta_test.go +++ b/relay/meta_test.go @@ -143,7 +143,7 @@ func (r *testMetaSuite) TestLocalMeta(c *C) { c.Assert(gset.String(), Equals, latestGTIDStr) for _, cs := range cases { - err = lm.AddDir(cs.uuid, nil, nil) + err = lm.AddDir(cs.uuid, nil, nil, "") c.Assert(err, IsNil) err = lm.Save(cs.pos, cs.gset) @@ -203,7 +203,7 @@ func (r *testMetaSuite) TestLocalMeta(c *C) { pos: mysql.Position{Name: "mysql-bin.000005", Pos: 789}, gset: gset5, } - err = lm.AddDir(cs.uuid, &cs.pos, cs.gset) + err = lm.AddDir(cs.uuid, &cs.pos, cs.gset, "") c.Assert(err, IsNil) dirty = lm.Dirty() diff --git a/relay/relay.go b/relay/relay.go index d8b6c71516..cb4715febd 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -96,6 +96,10 @@ type Process interface { IsClosed() bool // SaveMeta save relay meta SaveMeta(pos mysql.Position, gset gtid.Set) error + // ResetMeta reset relay meta + ResetMeta() + // PurgeRelayDir will clear all contents under w.cfg.RelayDir + PurgeRelayDir() error } // Relay relays mysql binlog to local file. @@ -200,7 +204,7 @@ func (r *Relay) process(parentCtx context.Context) error { } if isNew { - // re-setup meta for new server + // re-setup meta for new server or new source err = r.reSetupMeta() if err != nil { return err @@ -270,8 +274,8 @@ func (r *Relay) process(parentCtx context.Context) error { } } -// purgeRelayDir will clear all contents under w.cfg.RelayDir -func (r *Relay) purgeRelayDir() error { +// PurgeRelayDir implements the dm.Unit interface +func (r *Relay) PurgeRelayDir() error { dir := r.cfg.RelayDir d, err := os.Open(dir) // fail to open dir, return directly @@ -509,7 +513,29 @@ func (r *Relay) reSetupMeta() error { if err != nil { return err } - err = r.meta.AddDir(uuid, nil, nil) + + var newPos *mysql.Position + var newGset gtid.Set + var newUUIDSufiix string + if len(r.cfg.UUIDSuffix) != 0 { + if err = r.PurgeRelayDir(); err != nil { + return err + } + r.ResetMeta() + + newUUIDSufiix = r.cfg.UUIDSuffix + r.cfg.UUIDSuffix = "" + if len(r.cfg.BinLogName) != 0 { + newPos = &mysql.Position{Name: r.cfg.BinLogName, Pos: binlog.MinPosition.Pos} + } + if len(r.cfg.BinlogGTID) != 0 { + newGset, err = gtid.ParserGTID(r.cfg.Flavor, r.cfg.BinlogGTID) + if err != nil { + return err + } + } + } + err = r.meta.AddDir(uuid, newPos, newGset, newUUIDSufiix) if err != nil { return err } @@ -661,6 +687,11 @@ func (r *Relay) SaveMeta(pos mysql.Position, gset gtid.Set) error { return nil } +// ResetMeta reset relay meta +func (r *Relay) ResetMeta() { + r.meta = NewLocalMeta(r.cfg.Flavor, r.cfg.RelayDir) +} + // FlushMeta flush relay meta and clear all metas in RelayLogInfo func (r *Relay) FlushMeta() error { if err := r.meta.Flush(); err != nil { diff --git a/relay/relay_test.go b/relay/relay_test.go index 6af68badd6..9ac2b5b2ee 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -183,7 +183,7 @@ func (t *testRelaySuite) TestTryRecoverLatestFile(c *C) { f, err := os.Create(filepath.Join(r.cfg.RelayDir, "old_relay_log")) c.Assert(err, IsNil) f.Close() - c.Assert(r.purgeRelayDir(), IsNil) + c.Assert(r.PurgeRelayDir(), IsNil) files, err := ioutil.ReadDir(r.cfg.RelayDir) c.Assert(err, IsNil) c.Assert(files, HasLen, 0) @@ -194,7 +194,7 @@ func (t *testRelaySuite) TestTryRecoverLatestFile(c *C) { c.Assert(r.tryRecoverLatestFile(parser2), IsNil) // save position into meta - c.Assert(r.meta.AddDir(uuid, &startPos, nil), IsNil) + c.Assert(r.meta.AddDir(uuid, &startPos, nil, ""), IsNil) // relay log file does not exists, no need to recover c.Assert(r.tryRecoverLatestFile(parser2), IsNil) @@ -265,7 +265,7 @@ func (t *testRelaySuite) TestTryRecoverMeta(c *C) { recoverGTIDSet, err := gtid.ParserGTID(relayCfg.Flavor, recoverGTIDSetStr) c.Assert(err, IsNil) - c.Assert(r.meta.AddDir(uuid, &startPos, nil), IsNil) + c.Assert(r.meta.AddDir(uuid, &startPos, nil, ""), IsNil) c.Assert(r.meta.Load(), IsNil) // use a generator to generate some binlog events @@ -395,7 +395,7 @@ func (t *testRelaySuite) TestHandleEvent(c *C) { c.Assert(r.Init(context.Background()), IsNil) // NOTE: we can mock meta later. c.Assert(r.meta.Load(), IsNil) - c.Assert(r.meta.AddDir("24ecd093-8cec-11e9-aa0d-0242ac170002", nil, nil), IsNil) + c.Assert(r.meta.AddDir("24ecd093-8cec-11e9-aa0d-0242ac170002", nil, nil, ""), IsNil) // attach GTID sets to QueryEv queryEv2 := queryEv.Event.(*replication.QueryEvent) diff --git a/relay/transformer/transformer.go b/relay/transformer/transformer.go index 3fde2beec0..6682883212 100644 --- a/relay/transformer/transformer.go +++ b/relay/transformer/transformer.go @@ -67,6 +67,8 @@ func (t *transformer) Transform(e *replication.BinlogEvent) Result { } switch ev := e.Event.(type) { + case *replication.PreviousGTIDsEvent: + result.CanSaveGTID = true case *replication.RotateEvent: result.LogPos = uint32(ev.Position) // next event's position result.NextLogName = string(ev.NextLogName) // for RotateEvent, update binlog name From 5c6ea462deb7fdad88c5434c177d310bbc9e296e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 19 Nov 2020 17:58:11 +0800 Subject: [PATCH 18/59] use random server-id for relay reader --- relay/relay.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/relay/relay.go b/relay/relay.go index cb4715febd..414159b57f 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -630,6 +630,13 @@ func (r *Relay) doIntervalOps(ctx context.Context) { // setUpReader setups the underlying reader used to read binlog events from the upstream master server. func (r *Relay) setUpReader() (reader.Reader, error) { + randomServerID, err := utils.GetRandomServerID(r.tctx.Context(), r.db) + if err != nil { + // should never happened unless the master has too many slave + return nil, terror.Annotate(err, "fail to get random server id for relay reader") + } + r.syncerCfg.ServerID = randomServerID + uuid, pos := r.meta.Pos() _, gs := r.meta.GTID() cfg := &reader.Config{ @@ -641,7 +648,7 @@ func (r *Relay) setUpReader() (reader.Reader, error) { } reader2 := reader.NewReader(cfg) - err := reader2.Start() + err = reader2.Start() if err != nil { // do not log the whole config to protect the password in `SyncConfig`. // and other config items should already logged before or included in `err`. From 3bff78bb25d817a003f27295a9d9124bf9735813 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 19 Nov 2020 18:04:49 +0800 Subject: [PATCH 19/59] use relay log for ha_cases --- tests/ha_cases/conf/source1.yaml | 2 +- tests/ha_cases/conf/source2.yaml | 2 +- tests/ha_cases/run.sh | 40 ++++++++++++++++---------------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/ha_cases/conf/source1.yaml b/tests/ha_cases/conf/source1.yaml index 01660cf685..8b2f8c4319 100644 --- a/tests/ha_cases/conf/source1.yaml +++ b/tests/ha_cases/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/ha_cases/conf/source2.yaml b/tests/ha_cases/conf/source2.yaml index 100a7bd6b3..31bd5de10e 100644 --- a/tests/ha_cases/conf/source2.yaml +++ b/tests/ha_cases/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/ha_cases/run.sh b/tests/ha_cases/run.sh index abfc3fe566..e579e7a9fb 100755 --- a/tests/ha_cases/run.sh +++ b/tests/ha_cases/run.sh @@ -19,7 +19,7 @@ function test_running() { # make sure task to step in "Sync" stage run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 \ + "\"stage\": \"Running\"" 4 \ "\"unit\": \"Sync\"" 2 echo "use sync_diff_inspector to check full dump loader" @@ -49,11 +49,11 @@ function test_multi_task_running() { # make sure task to step in "Sync" stage run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 \ + "\"stage\": \"Running\"" 4 \ "\"unit\": \"Sync\"" 2 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test2" \ - "\"stage\": \"Running\"" 2 \ + "\"stage\": \"Running\"" 4 \ "\"unit\": \"Sync\"" 2 echo "use sync_diff_inspector to check full dump loader" @@ -158,11 +158,11 @@ function test_kill_master() { echo "check master2,3 are running" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_sql_file_withdb $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 $ha_test run_sql_file_withdb $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 $ha_test @@ -211,33 +211,33 @@ function test_kill_and_isolate_worker() { isolate_worker 4 "isolate" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 echo "isolate dm-worker3" isolate_worker 3 "isolate" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 1 \ + "\"stage\": \"Running\"" 2 \ "\"result\": false" 1 echo "disable isolate dm-worker4" isolate_worker 4 "disable_isolate" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 echo "query-status from all dm-master" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "pause-task test"\ @@ -281,7 +281,7 @@ function test_kill_master_in_sync() { check_http_alive 127.0.0.1:$MASTER_PORT2/apis/${API_VERSION}/status/test '"stage": "Running"' 10 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # waiting for syncing wait @@ -319,15 +319,15 @@ function test_kill_worker_in_sync() { echo "query-status from all dm-master" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # waiting for syncing wait @@ -404,7 +404,7 @@ function test_standalone_running() { # test should still running run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test"\ - "\"stage\": \"Running\"" 1 + "\"stage\": \"Running\"" 2 echo "[$(date)] <<<<<< finish test_standalone_running >>>>>>" } @@ -451,7 +451,7 @@ function test_pause_task() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $name"\ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 done # waiting for syncing @@ -499,7 +499,7 @@ function test_stop_task() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status ${task_name[$idx]}"\ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 done # waiting for syncing @@ -561,7 +561,7 @@ function test_multi_task_reduce_and_restart_worker() { for name in ${task_name[@]}; do run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $name"\ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 done # waiting for syncing @@ -578,7 +578,7 @@ function test_multi_task_reduce_and_restart_worker() { done search_str="\"stage\": \"Running\"" running_count=$(echo $status_str | sed "s/$search_str/$search_str\n/g" | grep -c "$search_str") - if [ $running_count != 4 ]; then + if [ $running_count != 8 ]; then echo "error running worker" echo $status_str exit 1 From 79cb20ea0342401cb3576cea207dec85ac2f9713 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 10:33:13 +0800 Subject: [PATCH 20/59] auto resume for relay --- dm/worker/task_checker.go | 72 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/dm/worker/task_checker.go b/dm/worker/task_checker.go index 22ae9c5601..a860034605 100644 --- a/dm/worker/task_checker.go +++ b/dm/worker/task_checker.go @@ -116,6 +116,11 @@ type backoffController struct { // task name -> the latest auto resume time latestResumeTime map[string]time.Time + + latestRelayPausedTime time.Time + latestRelayBlockTime time.Time + latestRelayResumeTime time.Time + relayBackoff *backoff.Backoff } // newBackoffController returns a new backoffController instance @@ -272,7 +277,65 @@ func (tsc *realTaskStatusChecker) getResumeStrategy(stStatus *pb.SubTaskStatus, return ResumeDispatch } -func (tsc *realTaskStatusChecker) check() { +func (tsc *realTaskStatusChecker) getRelayResumeStrategy(relayStatus *pb.RelayStatus, duration time.Duration) ResumeStrategy { + // relay that is not paused or paused manually, just ignore it + if relayStatus == nil || relayStatus.Stage != pb.Stage_Paused || relayStatus.Result == nil || relayStatus.Result.IsCanceled { + return ResumeIgnore + } + + // TODO: use different strategies based on the error detail + + if time.Since(tsc.bc.latestRelayResumeTime) < duration { + return ResumeSkip + } + + return ResumeDispatch +} + +func (tsc *realTaskStatusChecker) checkRelayStatus() { + relayStatus := tsc.w.relayHolder.Status() + if tsc.bc.relayBackoff == nil { + tsc.bc.relayBackoff, _ = backoff.NewBackoff(tsc.cfg.BackoffFactor, tsc.cfg.BackoffJitter, tsc.cfg.BackoffMin.Duration, tsc.cfg.BackoffMax.Duration) + tsc.bc.latestRelayPausedTime = time.Now() + tsc.bc.latestRelayResumeTime = time.Now() + } + rbf := tsc.bc.relayBackoff + duration := rbf.Current() + strategy := tsc.getRelayResumeStrategy(relayStatus, duration) + switch strategy { + case ResumeIgnore: + if time.Since(tsc.bc.latestRelayPausedTime) > tsc.cfg.BackoffRollback.Duration { + rbf.Rollback() + // after each rollback, reset this timer + tsc.bc.latestRelayPausedTime = time.Now() + } + case ResumeNoSense: + // this strategy doesn't forward or rollback backoff + tsc.bc.latestRelayPausedTime = time.Now() + blockTime := tsc.bc.latestRelayBlockTime + if !blockTime.IsZero() { + tsc.l.Warn("relay can't auto resume", zap.Duration("paused duration", time.Since(blockTime))) + } else { + tsc.bc.latestRelayBlockTime = time.Now() + tsc.l.Warn("relay can't auto resume") + } + case ResumeSkip: + tsc.l.Warn("backoff skip auto resume relay", zap.Time("latestResumeTime", tsc.bc.latestRelayResumeTime), zap.Duration("duration", duration)) + tsc.bc.latestRelayPausedTime = time.Now() + case ResumeDispatch: + tsc.bc.latestRelayPausedTime = time.Now() + err := tsc.w.operateRelay(tsc.ctx, pb.RelayOp_ResumeRelay) + if err != nil { + tsc.l.Error("dispatch auto resume relay failed", zap.Error(err)) + } else { + tsc.l.Info("dispatch auto resume task") + tsc.bc.latestRelayResumeTime = time.Now() + rbf.BoundaryForward() + } + } +} + +func (tsc *realTaskStatusChecker) checkTaskStatus() { allSubTaskStatus := tsc.w.getAllSubTaskStatus() defer func() { @@ -332,3 +395,10 @@ func (tsc *realTaskStatusChecker) check() { } } } + +func (tsc *realTaskStatusChecker) check() { + if tsc.w.cfg.EnableRelay { + tsc.checkRelayStatus() + } + tsc.checkTaskStatus() +} From c07ee7d301c54898eec0b11b7af4649cb7e52f40 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 10:33:21 +0800 Subject: [PATCH 21/59] fix some bugs --- dm/worker/relay.go | 3 ++- dm/worker/server.go | 2 ++ pkg/binlog/position.go | 3 +++ relay/relay.go | 4 +++- syncer/streamer_controller.go | 1 + 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dm/worker/relay.go b/dm/worker/relay.go index 1c3881ed3a..2b787063eb 100644 --- a/dm/worker/relay.go +++ b/dm/worker/relay.go @@ -226,11 +226,12 @@ func (h *realRelayHolder) resumeRelay(ctx context.Context, op pb.RelayOp) error func (h *realRelayHolder) stopRelay(ctx context.Context, op pb.RelayOp) error { h.Lock() - defer h.Unlock() if h.stage == pb.Stage_Stopped { + h.Unlock() return terror.ErrWorkerRelayStageNotValid.Generatef("current stage is already stopped not valid, relayop %s", op) } h.stage = pb.Stage_Stopped + h.Unlock() // now, when try to stop relay unit, we close relay holder h.Close() diff --git a/dm/worker/server.go b/dm/worker/server.go index 2679a16b6b..59f3dab066 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -552,8 +552,10 @@ func (s *Server) startWorker(cfg *config.SourceConfig) error { } if minLoc != nil { + log.L().Info("get min location in all subtasks", zap.Stringer("location", *minLoc)) cfg.RelayBinLogName = binlog.AdjustPosition(minLoc.Position).Name cfg.RelayBinlogGTID = minLoc.GTIDSetStr() + // set UUIDSuffix when bound to a source cfg.UUIDSuffix, err = binlog.ExtractSuffix(minLoc.Position.Name) if err != nil { return err diff --git a/pkg/binlog/position.go b/pkg/binlog/position.go index 19bce2c515..0a82cf9e0c 100644 --- a/pkg/binlog/position.go +++ b/pkg/binlog/position.go @@ -113,6 +113,9 @@ func RealMySQLPos(pos gmysql.Position) (gmysql.Position, error) { // ExtractSuffix extracts uuidSuffix from input name func ExtractSuffix(name string) (string, error) { + if len(name) == 0 { + return fmt.Sprintf("%06d", minUUIDSuffix), nil + } filename, err := ParseFilename(name) if err != nil { return "", err diff --git a/relay/relay.go b/relay/relay.go index 414159b57f..92cf54a351 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -203,7 +203,7 @@ func (r *Relay) process(parentCtx context.Context) error { return err } - if isNew { + if isNew || len(r.cfg.UUIDSuffix) > 0 { // re-setup meta for new server or new source err = r.reSetupMeta() if err != nil { @@ -524,7 +524,9 @@ func (r *Relay) reSetupMeta() error { r.ResetMeta() newUUIDSufiix = r.cfg.UUIDSuffix + // reset the UUIDSuffix r.cfg.UUIDSuffix = "" + if len(r.cfg.BinLogName) != 0 { newPos = &mysql.Position{Name: r.cfg.BinLogName, Pos: binlog.MinPosition.Pos} } diff --git a/syncer/streamer_controller.go b/syncer/streamer_controller.go index 1f6217350f..ef2a66a557 100644 --- a/syncer/streamer_controller.go +++ b/syncer/streamer_controller.go @@ -235,6 +235,7 @@ func (c *StreamerController) GetEvent(tctx *tcontext.Context) (event *replicatio failpoint.Inject("SyncerGetEventError", func(_ failpoint.Value) { if !mockRestarted { mockRestarted = true + c.meetError = true tctx.L().Info("mock upstream instance restart", zap.String("failpoint", "SyncerGetEventError")) failpoint.Return(nil, terror.ErrDBBadConn.Generate()) } From 632cbf012ef1655f81aa3e27964c9f59bccf320f Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 10:33:44 +0800 Subject: [PATCH 22/59] enable relay in integration test --- tests/all_mode/conf/source1.yaml | 1 + tests/all_mode/conf/source2.yaml | 3 ++- tests/compatibility/conf/source1.yaml | 1 + tests/compatibility/conf/source2.yaml | 3 ++- tests/dm_syncer/conf/source1.yaml | 1 + tests/dm_syncer/conf/source2.yaml | 3 ++- tests/dmctl_basic/conf/source2.yaml | 2 +- tests/dmctl_command/conf/source1.yaml | 1 + tests/dmctl_command/conf/source2.yaml | 3 ++- tests/dmctl_command/run.sh | 2 +- tests/drop_column_with_index/conf/source1.yaml | 1 + tests/drop_column_with_index/run.sh | 3 ++- tests/full_mode/conf/source1.yaml | 1 + tests/full_mode/conf/source2.yaml | 3 ++- tests/full_mode/run.sh | 2 +- tests/ha/conf/dm-worker3.toml | 2 +- tests/ha/conf/source1.yaml | 2 +- tests/ha/conf/source2.yaml | 2 +- tests/ha/run.sh | 12 ++++++------ tests/ha_master/conf/source1.yaml | 2 +- tests/ha_master/conf/source2.yaml | 2 +- tests/ha_master/run.sh | 4 ++-- tests/http_apis/conf/source1.yaml | 1 + tests/http_apis/run.sh | 4 ++-- tests/import_goroutine_leak/conf/source1.yaml | 1 + tests/import_goroutine_leak/run.sh | 2 +- tests/import_v10x/conf/source1.yaml | 1 + tests/import_v10x/conf/source2.yaml | 1 + tests/import_v10x/run.sh | 2 +- tests/incremental_mode/conf/source1.yaml | 1 + tests/incremental_mode/conf/source2.yaml | 1 + tests/load_interrupt/conf/source1.yaml | 1 + tests/online_ddl/conf/source1.yaml | 1 + tests/online_ddl/conf/source2.yaml | 1 + tests/print_status/conf/source1.yaml | 1 + tests/safe_mode/conf/source1.yaml | 1 + tests/safe_mode/conf/source2.yaml | 1 + tests/sequence_safe_mode/conf/source1.yaml | 1 + tests/sequence_safe_mode/conf/source2.yaml | 1 + tests/sequence_sharding/conf/source1.yaml | 1 + tests/sequence_sharding/conf/source2.yaml | 1 + tests/sequence_sharding_optimistic/conf/source1.yaml | 1 + tests/sequence_sharding_optimistic/conf/source2.yaml | 3 ++- tests/sequence_sharding_optimistic/run.sh | 4 ++-- tests/sequence_sharding_removemeta/conf/source1.yaml | 1 + tests/sequence_sharding_removemeta/conf/source2.yaml | 1 + tests/sharding/conf/source1.yaml | 1 + tests/sharding/conf/source2.yaml | 1 + tests/sharding2/conf/source1.yaml | 1 + tests/sharding2/conf/source2.yaml | 1 + tests/sharding2/run.sh | 2 +- tests/start_task/conf/source1.yaml | 1 + tests/tiup/conf/source1.yaml | 3 ++- tests/tiup/conf/source2.yaml | 1 + tests/tls/conf/source1.yaml | 1 + 55 files changed, 71 insertions(+), 31 deletions(-) diff --git a/tests/all_mode/conf/source1.yaml b/tests/all_mode/conf/source1.yaml index 948af37465..49b830cced 100644 --- a/tests/all_mode/conf/source1.yaml +++ b/tests/all_mode/conf/source1.yaml @@ -3,6 +3,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/all_mode/conf/source2.yaml b/tests/all_mode/conf/source2.yaml index 812917e6f2..3836a3353f 100644 --- a/tests/all_mode/conf/source2.yaml +++ b/tests/all_mode/conf/source2.yaml @@ -2,7 +2,8 @@ source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/compatibility/conf/source1.yaml b/tests/compatibility/conf/source1.yaml index 810b0af3d1..7d67feb8cf 100644 --- a/tests/compatibility/conf/source1.yaml +++ b/tests/compatibility/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/compatibility/conf/source2.yaml b/tests/compatibility/conf/source2.yaml index bd68886439..cc4bc2f3a4 100644 --- a/tests/compatibility/conf/source2.yaml +++ b/tests/compatibility/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dm_syncer/conf/source1.yaml b/tests/dm_syncer/conf/source1.yaml index 7cefc966e4..5a8554d591 100644 --- a/tests/dm_syncer/conf/source1.yaml +++ b/tests/dm_syncer/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dm_syncer/conf/source2.yaml b/tests/dm_syncer/conf/source2.yaml index bd68886439..cc4bc2f3a4 100644 --- a/tests/dm_syncer/conf/source2.yaml +++ b/tests/dm_syncer/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dmctl_basic/conf/source2.yaml b/tests/dmctl_basic/conf/source2.yaml index a3d54dbfc6..af8099984e 100644 --- a/tests/dmctl_basic/conf/source2.yaml +++ b/tests/dmctl_basic/conf/source2.yaml @@ -1,6 +1,6 @@ source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true relay-binlog-name: '' relay-binlog-gtid: '' enable-relay: true diff --git a/tests/dmctl_command/conf/source1.yaml b/tests/dmctl_command/conf/source1.yaml index 3cf5da4388..a124c8fc21 100644 --- a/tests/dmctl_command/conf/source1.yaml +++ b/tests/dmctl_command/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dmctl_command/conf/source2.yaml b/tests/dmctl_command/conf/source2.yaml index bd68886439..cc4bc2f3a4 100644 --- a/tests/dmctl_command/conf/source2.yaml +++ b/tests/dmctl_command/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dmctl_command/run.sh b/tests/dmctl_command/run.sh index 236e70e3b3..87ed15f621 100644 --- a/tests/dmctl_command/run.sh +++ b/tests/dmctl_command/run.sh @@ -6,7 +6,7 @@ cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source $cur/../_utils/test_prepare WORK_DIR=$TEST_DIR/$TEST_NAME -help_cnt=31 +help_cnt=33 function run() { # check dmctl alone output diff --git a/tests/drop_column_with_index/conf/source1.yaml b/tests/drop_column_with_index/conf/source1.yaml index 573489a847..1f01c40998 100644 --- a/tests/drop_column_with_index/conf/source1.yaml +++ b/tests/drop_column_with_index/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/drop_column_with_index/run.sh b/tests/drop_column_with_index/run.sh index 242505c190..8c4f7b77a9 100755 --- a/tests/drop_column_with_index/run.sh +++ b/tests/drop_column_with_index/run.sh @@ -29,7 +29,8 @@ function run() { check_sync_diff $WORK_DIR $cur/conf/diff_config.toml check_log_contain_with_retry "mock upstream instance restart" $WORK_DIR/worker1/log/dm-worker.log - check_log_contain_with_retry "dispatch auto resume task" $WORK_DIR/worker1/log/dm-worker.log + # check_log_contain_with_retry "dispatch auto resume task" $WORK_DIR/worker1/log/dm-worker.log + check_log_contain_with_retry "meet error when read from local binlog, will switch to remote binlog" $WORK_DIR/worker1/log/dm-worker.log run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 diff --git a/tests/full_mode/conf/source1.yaml b/tests/full_mode/conf/source1.yaml index c1cc021476..9c9acf4f8a 100644 --- a/tests/full_mode/conf/source1.yaml +++ b/tests/full_mode/conf/source1.yaml @@ -2,6 +2,7 @@ server-id: 101 source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/full_mode/conf/source2.yaml b/tests/full_mode/conf/source2.yaml index 8b08f3b082..f393837885 100644 --- a/tests/full_mode/conf/source2.yaml +++ b/tests/full_mode/conf/source2.yaml @@ -1,7 +1,8 @@ server-id: 102 source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh index 987a3ff305..5e714c5e72 100755 --- a/tests/full_mode/run.sh +++ b/tests/full_mode/run.sh @@ -52,7 +52,7 @@ function fail_acquire_global_lock() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Paused\"" 2 \ + "\"stage\": \"Paused\"" 4 \ "you need (at least one of) the RELOAD privilege(s) for this operation" 2 cleanup_data full_mode diff --git a/tests/ha/conf/dm-worker3.toml b/tests/ha/conf/dm-worker3.toml index ceecd87470..ab7e1b9cb3 100644 --- a/tests/ha/conf/dm-worker3.toml +++ b/tests/ha/conf/dm-worker3.toml @@ -1,2 +1,2 @@ name = "worker3" -join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561,127.0.0.1:8661" +join = "127.0.0.1:8261" diff --git a/tests/ha/conf/source1.yaml b/tests/ha/conf/source1.yaml index c2b659d3fb..c9d618cfdd 100644 --- a/tests/ha/conf/source1.yaml +++ b/tests/ha/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/ha/conf/source2.yaml b/tests/ha/conf/source2.yaml index fb1985ca35..d6f0846831 100644 --- a/tests/ha/conf/source2.yaml +++ b/tests/ha/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/ha/run.sh b/tests/ha/run.sh index 8e195ae70b..fcc6cb978c 100755 --- a/tests/ha/run.sh +++ b/tests/ha/run.sh @@ -91,22 +91,22 @@ function run() { echo "query-status from all dm-master" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 echo "join new dm-master and query-status" run_dm_master $WORK_DIR/master4 $MASTER_PORT4 $cur/conf/dm-master4.toml check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT4" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # may join failed with error `fail to join embed etcd: add member http://127.0.0.1:8295: etcdserver: unhealthy cluster`, and dm-master will exit. so just sleep some seconds. sleep 5 @@ -115,14 +115,14 @@ function run() { check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT5 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT5" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 sleep 5 run_dm_master $WORK_DIR/master6 $MASTER_PORT6 $cur/conf/dm-master6.toml check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT6 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT6" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 sleep 5 echo "kill dm-master1" diff --git a/tests/ha_master/conf/source1.yaml b/tests/ha_master/conf/source1.yaml index 38c58c28cb..3c4efbef52 100644 --- a/tests/ha_master/conf/source1.yaml +++ b/tests/ha_master/conf/source1.yaml @@ -3,7 +3,7 @@ flavor: '' enable-gtid: true relay-binlog-name: '' relay-binlog-gtid: '' -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/ha_master/conf/source2.yaml b/tests/ha_master/conf/source2.yaml index 92d2ee77b6..a3d54dbfc6 100644 --- a/tests/ha_master/conf/source2.yaml +++ b/tests/ha_master/conf/source2.yaml @@ -3,7 +3,7 @@ flavor: '' enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/ha_master/run.sh b/tests/ha_master/run.sh index 7d1b713a5e..223d1b2609 100755 --- a/tests/ha_master/run.sh +++ b/tests/ha_master/run.sh @@ -319,7 +319,7 @@ function run() { check_http_alive 127.0.0.1:$MASTER_PORT3/apis/${API_VERSION}/status/test '"stage": "Running"' 10 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 @@ -343,7 +343,7 @@ function run() { # the last two masters should elect a new leader and serve service run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT4" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # run master3 again run_dm_master $WORK_DIR/master3 $MASTER_PORT3 $cur/conf/dm-master3.toml diff --git a/tests/http_apis/conf/source1.yaml b/tests/http_apis/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/http_apis/conf/source1.yaml +++ b/tests/http_apis/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/http_apis/run.sh b/tests/http_apis/run.sh index 09087cc40b..c49f8ddaf2 100644 --- a/tests/http_apis/run.sh +++ b/tests/http_apis/run.sh @@ -50,7 +50,7 @@ function run() { sleep 1 curl -X GET 127.0.0.1:$MASTER_PORT/apis/${API_VERSION}/status/test > $WORK_DIR/status.log - check_log_contains $WORK_DIR/status.log "\"stage\": \"Running\"" 1 + check_log_contains $WORK_DIR/status.log "\"stage\": \"Running\"" 2 check_log_contains $WORK_DIR/status.log "\"name\": \"test\"" 1 echo "get sub task configs" @@ -76,7 +76,7 @@ function run() { sleep 1 curl -X GET 127.0.0.1:$MASTER_PORT/apis/${API_VERSION}/status/test > $WORK_DIR/status.log - check_log_contains $WORK_DIR/status.log "\"stage\": \"Running\"" 1 + check_log_contains $WORK_DIR/status.log "\"stage\": \"Running\"" 2 check_log_contains $WORK_DIR/status.log "\"name\": \"test\"" 1 sleep 1 diff --git a/tests/import_goroutine_leak/conf/source1.yaml b/tests/import_goroutine_leak/conf/source1.yaml index 810b0af3d1..7d67feb8cf 100644 --- a/tests/import_goroutine_leak/conf/source1.yaml +++ b/tests/import_goroutine_leak/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/import_goroutine_leak/run.sh b/tests/import_goroutine_leak/run.sh index a192007b96..a38aaf3dca 100644 --- a/tests/import_goroutine_leak/run.sh +++ b/tests/import_goroutine_leak/run.sh @@ -111,7 +111,7 @@ function run() { # wait until the task running run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - '"stage": "Running"' 1 + '"stage": "Running"' 2 sleep 2 # wait to be blocked # check to be blocked diff --git a/tests/import_v10x/conf/source1.yaml b/tests/import_v10x/conf/source1.yaml index 6d078d2523..b221f0c3e3 100644 --- a/tests/import_v10x/conf/source1.yaml +++ b/tests/import_v10x/conf/source1.yaml @@ -3,6 +3,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/import_v10x/conf/source2.yaml b/tests/import_v10x/conf/source2.yaml index 0a99dec60b..d0f4e7bc6f 100644 --- a/tests/import_v10x/conf/source2.yaml +++ b/tests/import_v10x/conf/source2.yaml @@ -3,6 +3,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/import_v10x/run.sh b/tests/import_v10x/run.sh index 3146bfafe2..22b316c7c0 100644 --- a/tests/import_v10x/run.sh +++ b/tests/import_v10x/run.sh @@ -71,7 +71,7 @@ function run() { # check task running. run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # check task config, just a simple match run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ diff --git a/tests/incremental_mode/conf/source1.yaml b/tests/incremental_mode/conf/source1.yaml index 6e0aa87543..0018c1b61e 100644 --- a/tests/incremental_mode/conf/source1.yaml +++ b/tests/incremental_mode/conf/source1.yaml @@ -3,6 +3,7 @@ flavor: '' enable-gtid: true relay-binlog-name: '' relay-binlog-gtid: '' +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/incremental_mode/conf/source2.yaml b/tests/incremental_mode/conf/source2.yaml index eb3f32f965..e21304db8e 100644 --- a/tests/incremental_mode/conf/source2.yaml +++ b/tests/incremental_mode/conf/source2.yaml @@ -3,6 +3,7 @@ flavor: '' enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' +enable-relay: true from: host: 127.0.0.1 user: root diff --git a/tests/load_interrupt/conf/source1.yaml b/tests/load_interrupt/conf/source1.yaml index 810b0af3d1..7d67feb8cf 100644 --- a/tests/load_interrupt/conf/source1.yaml +++ b/tests/load_interrupt/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/online_ddl/conf/source1.yaml b/tests/online_ddl/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/online_ddl/conf/source1.yaml +++ b/tests/online_ddl/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/online_ddl/conf/source2.yaml b/tests/online_ddl/conf/source2.yaml index bd68886439..3df6bdf955 100644 --- a/tests/online_ddl/conf/source2.yaml +++ b/tests/online_ddl/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/print_status/conf/source1.yaml b/tests/print_status/conf/source1.yaml index 810b0af3d1..7d67feb8cf 100644 --- a/tests/print_status/conf/source1.yaml +++ b/tests/print_status/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/safe_mode/conf/source1.yaml b/tests/safe_mode/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/safe_mode/conf/source1.yaml +++ b/tests/safe_mode/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/safe_mode/conf/source2.yaml b/tests/safe_mode/conf/source2.yaml index bd68886439..3df6bdf955 100644 --- a/tests/safe_mode/conf/source2.yaml +++ b/tests/safe_mode/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_safe_mode/conf/source1.yaml b/tests/sequence_safe_mode/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/sequence_safe_mode/conf/source1.yaml +++ b/tests/sequence_safe_mode/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_safe_mode/conf/source2.yaml b/tests/sequence_safe_mode/conf/source2.yaml index bd68886439..3df6bdf955 100644 --- a/tests/sequence_safe_mode/conf/source2.yaml +++ b/tests/sequence_safe_mode/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding/conf/source1.yaml b/tests/sequence_sharding/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/sequence_sharding/conf/source1.yaml +++ b/tests/sequence_sharding/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding/conf/source2.yaml b/tests/sequence_sharding/conf/source2.yaml index bd68886439..3df6bdf955 100644 --- a/tests/sequence_sharding/conf/source2.yaml +++ b/tests/sequence_sharding/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding_optimistic/conf/source1.yaml b/tests/sequence_sharding_optimistic/conf/source1.yaml index 082837b755..1ce2e303bc 100644 --- a/tests/sequence_sharding_optimistic/conf/source1.yaml +++ b/tests/sequence_sharding_optimistic/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding_optimistic/conf/source2.yaml b/tests/sequence_sharding_optimistic/conf/source2.yaml index eb3f32f965..17e3479acb 100644 --- a/tests/sequence_sharding_optimistic/conf/source2.yaml +++ b/tests/sequence_sharding_optimistic/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' -enable-gtid: false +enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding_optimistic/run.sh b/tests/sequence_sharding_optimistic/run.sh index fd382bde38..f299084838 100755 --- a/tests/sequence_sharding_optimistic/run.sh +++ b/tests/sequence_sharding_optimistic/run.sh @@ -74,7 +74,7 @@ run() { "\"result\": true" 3 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $task_name" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # now, for optimistic shard DDL, different sources will reach a stage often not at the same time, # in order to simply the check and resume flow, only enable the failpoint for one DM-worker. @@ -198,7 +198,7 @@ run() { "\"result\": true" 3 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $task_name" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 # use sync_diff_inspector to check data now! check_sync_diff $WORK_DIR $cur/conf/diff_config.toml diff --git a/tests/sequence_sharding_removemeta/conf/source1.yaml b/tests/sequence_sharding_removemeta/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/sequence_sharding_removemeta/conf/source1.yaml +++ b/tests/sequence_sharding_removemeta/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding_removemeta/conf/source2.yaml b/tests/sequence_sharding_removemeta/conf/source2.yaml index bd68886439..3df6bdf955 100644 --- a/tests/sequence_sharding_removemeta/conf/source2.yaml +++ b/tests/sequence_sharding_removemeta/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding/conf/source1.yaml b/tests/sharding/conf/source1.yaml index 6e0aa87543..78e96409ee 100644 --- a/tests/sharding/conf/source1.yaml +++ b/tests/sharding/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding/conf/source2.yaml b/tests/sharding/conf/source2.yaml index eb3f32f965..4271c2b9f2 100644 --- a/tests/sharding/conf/source2.yaml +++ b/tests/sharding/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding2/conf/source1.yaml b/tests/sharding2/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/sharding2/conf/source1.yaml +++ b/tests/sharding2/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding2/conf/source2.yaml b/tests/sharding2/conf/source2.yaml index 31f2a552d2..cc4bc2f3a4 100644 --- a/tests/sharding2/conf/source2.yaml +++ b/tests/sharding2/conf/source2.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding2/run.sh b/tests/sharding2/run.sh index 99b0ae7c6c..a75817eb4f 100755 --- a/tests/sharding2/run.sh +++ b/tests/sharding2/run.sh @@ -81,7 +81,7 @@ function run() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 \ + "\"stage\": \"Running\"" 4 \ "\"unit\": \"Sync\"" 2 } diff --git a/tests/start_task/conf/source1.yaml b/tests/start_task/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/start_task/conf/source1.yaml +++ b/tests/start_task/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/tiup/conf/source1.yaml b/tests/tiup/conf/source1.yaml index 999c5decff..695d33f8a1 100644 --- a/tests/tiup/conf/source1.yaml +++ b/tests/tiup/conf/source1.yaml @@ -1,5 +1,6 @@ source-id: mysql-replica-01 -enable-gtid: true +enable-gtid: false +enable-relay: true from: host: mysql1 user: root diff --git a/tests/tiup/conf/source2.yaml b/tests/tiup/conf/source2.yaml index 0cdcfefe87..cf0a47d14e 100644 --- a/tests/tiup/conf/source2.yaml +++ b/tests/tiup/conf/source2.yaml @@ -1,5 +1,6 @@ source-id: mysql-replica-02 enable-gtid: true +enable-relay: true from: host: mysql2 user: root diff --git a/tests/tls/conf/source1.yaml b/tests/tls/conf/source1.yaml index 1894302d70..fae9f5bd7e 100644 --- a/tests/tls/conf/source1.yaml +++ b/tests/tls/conf/source1.yaml @@ -1,6 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true +enable-relay: true relay-binlog-name: '' relay-binlog-gtid: '' from: From 3ead31a5b3ca6db56566af2304fe01b8cab02c24 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 11:32:58 +0800 Subject: [PATCH 23/59] address comment --- _utils/terror_gen/errors_release.txt | 2 +- dm/worker/subtask.go | 51 ++++++++++++++++++---------- errors.toml | 2 +- pkg/terror/error_list.go | 2 +- tests/shardddl1/conf/source1.yaml | 2 +- tests/shardddl2/conf/source1.yaml | 2 +- tests/shardddl3/conf/source1.yaml | 2 +- 7 files changed, 40 insertions(+), 23 deletions(-) diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt index 9ef93e7599..8b0349cc65 100644 --- a/_utils/terror_gen/errors_release.txt +++ b/_utils/terror_gen/errors_release.txt @@ -430,7 +430,7 @@ ErrWorkerCacheDDLInfoExists,[code=40063:class=dm-worker:scope=internal:level=hig ErrWorkerExecSkipDDLConflict,[code=40064:class=dm-worker:scope=internal:level=high], "Message: execDDL and skipDDL can not specify both at the same time" ErrWorkerExecDDLSyncerOnly,[code=40065:class=dm-worker:scope=internal:level=high], "Message: only syncer support ExecuteDDL, but current unit is %s" ErrWorkerExecDDLTimeout,[code=40066:class=dm-worker:scope=internal:level=high], "Message: ExecuteDDL timeout (exceeding %s), Workaround: Please try use `query-status` to query whether the DDL is still blocking." -ErrWorkerWaitRelayCatchupTimeout,[code=40067:class=dm-worker:scope=internal:level=high], "Message: waiting for relay binlog pos to catch up with loader end binlog pos is timeout (exceeding %s), loader end binlog pos: %s, relay binlog pos: %s" +ErrWorkerWaitRelayCatchupTimeout,[code=40067:class=dm-worker:scope=internal:level=high], "Message: waiting for relay to catch up with loader is timeout (exceeding %s), loader: %s, relay: %s" ErrWorkerRelayIsPurging,[code=40068:class=dm-worker:scope=internal:level=high], "Message: relay log purger is purging, cannot start sub task %s, Workaround: Please try again later." ErrWorkerHostPortNotValid,[code=40069:class=dm-worker:scope=internal:level=high], "Message: host:port '%s' not valid, Workaround: Please check configs in worker configuration file." ErrWorkerNoStart,[code=40070:class=dm-worker:scope=internal:level=high], "Message: no mysql source is being handled in the worker" diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index 45b4dc6414..3a051be4bc 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -19,6 +19,7 @@ import ( "time" "github.com/pingcap/failpoint" + "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go/sync2" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" @@ -584,6 +585,13 @@ func (st *SubTask) ShardDDLOperation() *pessimism.Operation { // Currently there is only one wait condition // from Load unit to Sync unit, wait for relay-log catched up with mydumper binlog position. func (st *SubTask) unitTransWaitCondition(subTaskCtx context.Context) error { + var ( + gset1 gtid.Set + gset2 gtid.Set + pos1 *mysql.Position + pos2 *mysql.Position + err error + ) pu := st.PrevUnit() cu := st.CurrUnit() if pu != nil && pu.Type() == pb.UnitType_Load && cu.Type() == pb.UnitType_Sync { @@ -601,40 +609,49 @@ func (st *SubTask) unitTransWaitCondition(subTaskCtx context.Context) error { loadStatus := pu.Status(ctxStatus).(*pb.LoadStatus) cancelStatus() - pos1, err := utils.DecodeBinlogPosition(loadStatus.MetaBinlog) - if err != nil { - return terror.WithClass(err, terror.ClassDMWorker) + if st.cfg.EnableGTID { + gset1, err = gtid.ParserGTID(st.cfg.Flavor, loadStatus.MetaBinlogGTID) + if err != nil { + return terror.WithClass(err, terror.ClassDMWorker) + } + } else { + pos1, err = utils.DecodeBinlogPosition(loadStatus.MetaBinlog) + if err != nil { + return terror.WithClass(err, terror.ClassDMWorker) + } } + for { ctxStatus, cancelStatus = context.WithTimeout(ctxWait, utils.DefaultDBTimeout) relayStatus := hub.w.relayHolder.Status(ctxStatus) cancelStatus() - pos2, err := utils.DecodeBinlogPosition(relayStatus.RelayBinlog) - if err != nil { - return terror.WithClass(err, terror.ClassDMWorker) - } if st.cfg.EnableGTID { - gset1, err := gtid.ParserGTID(st.cfg.Flavor, loadStatus.MetaBinlogGTID) - if err != nil { - st.l.Error("fail to parse gtid", log.ShortError(err)) - } - gset2, err := gtid.ParserGTID(st.cfg.Flavor, relayStatus.RelayBinlogGtid) + gset2, err = gtid.ParserGTID(st.cfg.Flavor, relayStatus.RelayBinlogGtid) if err != nil { - st.l.Error("fail to parse gtid", log.ShortError(err)) + return terror.WithClass(err, terror.ClassDMWorker) } rc, ok := binlog.CompareGTID(gset1, gset2) if ok && rc <= 0 { break } + } else { + pos2, err = utils.DecodeBinlogPosition(relayStatus.RelayBinlog) + if err != nil { + return terror.WithClass(err, terror.ClassDMWorker) + } + if pos1.Compare(*pos2) <= 0 { + break + } } - if pos1.Compare(*pos2) <= 0 { - break - } - st.l.Debug("wait relay to catchup", zap.Stringer("load end position", pos1), zap.String("load end gtid", loadStatus.MetaBinlogGTID), zap.Stringer("relay position", pos2), zap.String("relay gtid", relayStatus.RelayBinlogGtid)) + + st.l.Debug("wait relay to catchup", zap.Bool("enableGTID", st.cfg.EnableGTID), zap.Stringer("load end position", pos1), zap.String("load end gtid", loadStatus.MetaBinlogGTID), zap.Stringer("relay position", pos2), zap.String("relay gtid", relayStatus.RelayBinlogGtid)) select { case <-ctxWait.Done(): + if st.cfg.EnableGTID { + return terror.ErrWorkerWaitRelayCatchupTimeout.Generate(waitRelayCatchupTimeout, loadStatus.MetaBinlogGTID, relayStatus.RelayBinlogGtid) + } return terror.ErrWorkerWaitRelayCatchupTimeout.Generate(waitRelayCatchupTimeout, pos1, pos2) case <-subTaskCtx.Done(): return nil diff --git a/errors.toml b/errors.toml index 793e33839d..872a6e80eb 100644 --- a/errors.toml +++ b/errors.toml @@ -2591,7 +2591,7 @@ workaround = "Please try use `query-status` to query whether the DDL is still bl tags = ["internal", "high"] [error.DM-dm-worker-40067] -message = "waiting for relay binlog pos to catch up with loader end binlog pos is timeout (exceeding %s), loader end binlog pos: %s, relay binlog pos: %s" +message = "waiting for relay to catch up with loader is timeout (exceeding %s), loader: %s, relay: %s" description = "" workaround = "" tags = ["internal", "high"] diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index f3c7f2ff63..ba2ec75500 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -1100,7 +1100,7 @@ var ( ErrWorkerExecSkipDDLConflict = New(codeWorkerExecSkipDDLConflict, ClassDMWorker, ScopeInternal, LevelHigh, "execDDL and skipDDL can not specify both at the same time", "") ErrWorkerExecDDLSyncerOnly = New(codeWorkerExecDDLSyncerOnly, ClassDMWorker, ScopeInternal, LevelHigh, "only syncer support ExecuteDDL, but current unit is %s", "") ErrWorkerExecDDLTimeout = New(codeWorkerExecDDLTimeout, ClassDMWorker, ScopeInternal, LevelHigh, "ExecuteDDL timeout (exceeding %s)", "Please try use `query-status` to query whether the DDL is still blocking.") - ErrWorkerWaitRelayCatchupTimeout = New(codeWorkerWaitRelayCatchupTimeout, ClassDMWorker, ScopeInternal, LevelHigh, "waiting for relay binlog pos to catch up with loader end binlog pos is timeout (exceeding %s), loader end binlog pos: %s, relay binlog pos: %s", "") + ErrWorkerWaitRelayCatchupTimeout = New(codeWorkerWaitRelayCatchupTimeout, ClassDMWorker, ScopeInternal, LevelHigh, "waiting for relay to catch up with loader is timeout (exceeding %s), loader: %s, relay: %s", "") ErrWorkerRelayIsPurging = New(codeWorkerRelayIsPurging, ClassDMWorker, ScopeInternal, LevelHigh, "relay log purger is purging, cannot start sub task %s", "Please try again later.") ErrWorkerHostPortNotValid = New(codeWorkerHostPortNotValid, ClassDMWorker, ScopeInternal, LevelHigh, "host:port '%s' not valid", "Please check configs in worker configuration file.") ErrWorkerNoStart = New(codeWorkerNoStart, ClassDMWorker, ScopeInternal, LevelHigh, "no mysql source is being handled in the worker", "") diff --git a/tests/shardddl1/conf/source1.yaml b/tests/shardddl1/conf/source1.yaml index 8b2f8c4319..01660cf685 100644 --- a/tests/shardddl1/conf/source1.yaml +++ b/tests/shardddl1/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/shardddl2/conf/source1.yaml b/tests/shardddl2/conf/source1.yaml index 8b2f8c4319..01660cf685 100644 --- a/tests/shardddl2/conf/source1.yaml +++ b/tests/shardddl2/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/shardddl3/conf/source1.yaml b/tests/shardddl3/conf/source1.yaml index 8b2f8c4319..01660cf685 100644 --- a/tests/shardddl3/conf/source1.yaml +++ b/tests/shardddl3/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root From 8b024b728641fa885fb059ae1dd62eeb18eb0c01 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 11:46:26 +0800 Subject: [PATCH 24/59] return error if cannot compare gtid --- _utils/terror_gen/errors_release.txt | 1 + dm/worker/subtask.go | 5 ++++- errors.toml | 6 ++++++ pkg/terror/error_list.go | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt index 8b0349cc65..93ef439944 100644 --- a/_utils/terror_gen/errors_release.txt +++ b/_utils/terror_gen/errors_release.txt @@ -436,6 +436,7 @@ ErrWorkerHostPortNotValid,[code=40069:class=dm-worker:scope=internal:level=high] ErrWorkerNoStart,[code=40070:class=dm-worker:scope=internal:level=high], "Message: no mysql source is being handled in the worker" ErrWorkerAlreadyStart,[code=40071:class=dm-worker:scope=internal:level=high], "Message: mysql source handler worker already started" ErrWorkerSourceNotMatch,[code=40072:class=dm-worker:scope=internal:level=high], "Message: source of request does not match with source in worker" +ErrWorkerWaitRelayCatchupGTID,[code=40078:class=dm-worker:scope=internal:level=high], "Message: cannot compare gtid between loader and relay, loader gtid: %s, relay gtid: %s" ErrWorkerFailToGetSubtaskConfigFromEtcd,[code=40073:class=dm-worker:scope=internal:level=medium], "Message: there is no relative subtask config for task %s in etcd" ErrWorkerFailToGetSourceConfigFromEtcd,[code=40074:class=dm-worker:scope=internal:level=medium], "Message: there is no relative source config for source %s in etcd" ErrWorkerDDLLockOpNotFound,[code=40075:class=dm-worker:scope=internal:level=high], "Message: missing shard DDL lock operation for shard DDL info (%s)" diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index 3a051be4bc..14ead538bd 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -632,7 +632,10 @@ func (st *SubTask) unitTransWaitCondition(subTaskCtx context.Context) error { return terror.WithClass(err, terror.ClassDMWorker) } rc, ok := binlog.CompareGTID(gset1, gset2) - if ok && rc <= 0 { + if !ok { + return terror.ErrWorkerWaitRelayCatchupGTID.Generate(loadStatus.MetaBinlogGTID, relayStatus.RelayBinlogGtid) + } + if rc <= 0 { break } } else { diff --git a/errors.toml b/errors.toml index 872a6e80eb..5e9a98b7de 100644 --- a/errors.toml +++ b/errors.toml @@ -2656,6 +2656,12 @@ description = "" workaround = "Please check network connection of worker" tags = ["internal", "high"] +[error.DM-dm-worker-40078] +message = "cannot compare gtid between loader and relay, loader gtid: %s, relay gtid: %s" +description = "" +workaround = "" +tags = ["internal", "high"] + [error.DM-dm-tracer-42001] message = "parse dm-tracer config flag set" description = "" diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index ba2ec75500..fd6273eee5 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -538,6 +538,7 @@ const ( codeWorkerDDLLockOpNotFound codeWorkerTLSConfigNotValid codeWorkerFailConnectMaster + codeWorkerWaitRelayCatchupGTID ) // DM-tracer error code @@ -1106,6 +1107,7 @@ var ( ErrWorkerNoStart = New(codeWorkerNoStart, ClassDMWorker, ScopeInternal, LevelHigh, "no mysql source is being handled in the worker", "") ErrWorkerAlreadyStart = New(codeWorkerAlreadyStarted, ClassDMWorker, ScopeInternal, LevelHigh, "mysql source handler worker already started", "") ErrWorkerSourceNotMatch = New(codeWorkerSourceNotMatch, ClassDMWorker, ScopeInternal, LevelHigh, "source of request does not match with source in worker", "") + ErrWorkerWaitRelayCatchupGTID = New(codeWorkerWaitRelayCatchupGTID, ClassDMWorker, ScopeInternal, LevelHigh, "cannot compare gtid between loader and relay, loader gtid: %s, relay gtid: %s", "") ErrWorkerFailToGetSubtaskConfigFromEtcd = New(codeWorkerFailToGetSubtaskConfigFromEtcd, ClassDMWorker, ScopeInternal, LevelMedium, "there is no relative subtask config for task %s in etcd", "") ErrWorkerFailToGetSourceConfigFromEtcd = New(codeWorkerFailToGetSourceConfigFromEtcd, ClassDMWorker, ScopeInternal, LevelMedium, "there is no relative source config for source %s in etcd", "") From bf75c71707d9f6bdbc03f1c82714a414d08ccbd3 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 16:17:47 +0800 Subject: [PATCH 25/59] use fileReader instead --- pkg/streamer/reader.go | 69 ++++++++++-------------------------------- 1 file changed, 16 insertions(+), 53 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index b2b7fe011b..ec33c0f699 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -31,6 +31,7 @@ import ( "go.uber.org/zap" "github.com/pingcap/dm/pkg/binlog" + "github.com/pingcap/dm/pkg/binlog/reader" tcontext "github.com/pingcap/dm/pkg/context" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" @@ -127,14 +128,8 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { // get flush logs from oldest to newest for _, uuid := range r.uuids { filename := path.Join(r.cfg.RelayDir, uuid, utils.MetaFilename) - fd, err := os.Open(filename) - if err != nil { - return "", terror.ErrRelayLoadMetaData.Delegate(err) - } - defer fd.Close() - var meta Meta - _, err = toml.DecodeReader(fd, &meta) + _, err := toml.DecodeFile(filename, &meta) if err != nil { return "", terror.ErrRelayLoadMetaData.Delegate(err) } @@ -144,7 +139,7 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { return "", terror.ErrRelayLoadMetaData.Delegate(err) } if gs.Contain(gset) { - r.tctx.L().Info("get uuid subdir by gtid", zap.Stringer("GTID Set", gset), zap.String("uuid", uuid)) + r.tctx.L().Info("get uuid subdir by gtid", zap.Stringer("GTID Set", gset), zap.String("uuid", uuid), zap.Stringer("latest GTID Set in subdir", gs)) return uuid, nil } } @@ -165,48 +160,13 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { // GetFilePosByGTID tries to get Pos by GTID for file func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gset mysql.GTIDSet) (uint32, error) { - ctx2, cancel := context.WithCancel(ctx) - defer cancel() - - s := newLocalStreamer() - parser := replication.NewBinlogParser() - parser.SetVerifyChecksum(true) - parser.SetUseDecimal(true) - if r.cfg.Timezone != nil { - parser.SetTimestampStringLocation(r.cfg.Timezone) + fileReader := reader.NewFileReader(&reader.FileReaderConfig{Timezone: r.cfg.Timezone}) + defer fileReader.Close() + err := fileReader.StartSyncByPos(mysql.Position{Name: filePath, Pos: 4}) + if err != nil { + return 0, err } - r.wg.Add(1) - go func() { - defer r.wg.Done() - - offset := int64(4) - onEventFunc := func(e *replication.BinlogEvent) error { - offset = int64(e.Header.LogPos) - - select { - case s.ch <- e: - case <-ctx2.Done(): - } - return nil - } - - err := parser.ParseFile(filePath, offset, onEventFunc) - if err != nil { - r.tctx.L().Error("parse file stopped", zap.Error(err)) - select { - case s.ech <- err: - case <-ctx2.Done(): - } - } else { - // reach end of file - select { - case s.ch <- nil: - case <-ctx2.Done(): - } - } - }() - lastPos := uint32(0) for { select { @@ -215,14 +175,17 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs default: } - e, err := s.GetEvent(ctx) + ctx2, cancel := context.WithTimeout(ctx, time.Second) + e, err := fileReader.GetEvent(ctx2) + cancel() if err != nil { + // reach end of file + if errors.Cause(err) == context.DeadlineExceeded { + return lastPos, nil + } return 0, err } - // gset is the last txn in file - if e == nil { - return lastPos, nil - } + switch ev := e.Event.(type) { case *replication.PreviousGTIDsEvent: // nil previous gtid event, continue to parse file From e65307bdad5a5d48e1def524521ca266a5b00cc4 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 16:27:18 +0800 Subject: [PATCH 26/59] clear memory relay meta only when init --- pkg/streamer/reader.go | 11 +++++------ relay/relay.go | 9 +++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index ec33c0f699..df93eab03a 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -120,11 +120,6 @@ func (r *BinlogReader) checkRelayPos(pos mysql.Position) error { // getUUIDByGTID gets uuid subdir which contain the gtid set func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { - // TODO: use a better mechanism to call relay.meta.Flush - // get the meta save in memory - relayMetaHub := GetRelayMetaHub() - relayMeta := relayMetaHub.GetMeta() - // get flush logs from oldest to newest for _, uuid := range r.uuids { filename := path.Join(r.cfg.RelayDir, uuid, utils.MetaFilename) @@ -144,7 +139,11 @@ func (r *BinlogReader) getUUIDByGTID(gset mysql.GTIDSet) (string, error) { } } - // use memory meta + // TODO: use a better mechanism to call relay.meta.Flush + // get the meta save in memory + relayMetaHub := GetRelayMetaHub() + relayMeta := relayMetaHub.GetMeta() + if len(relayMeta.UUID) > 0 { gs, err := mysql.ParseGTIDSet(r.cfg.Flavor, relayMeta.BinlogGTID) if err != nil { diff --git a/relay/relay.go b/relay/relay.go index e7b07ae7f6..c874d057a2 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -158,6 +158,7 @@ func (r *Relay) Init(ctx context.Context) (err error) { } r.relayMetaHub = pkgstreamer.GetRelayMetaHub() + r.relayMetaHub.ClearMeta() return reportRelayLogSpaceInBackground(r.cfg.RelayDir) } @@ -663,13 +664,9 @@ func (r *Relay) SaveMeta(pos mysql.Position, gset gtid.Set) error { return nil } -// FlushMeta flush relay meta and clear all metas in RelayLogInfo +// FlushMeta flush relay meta. func (r *Relay) FlushMeta() error { - if err := r.meta.Flush(); err != nil { - return err - } - r.relayMetaHub.ClearMeta() - return nil + return r.meta.Flush() } // stopSync stops syncing, now it used by Close and Pause From 513821e6e167cf85da9ce2eb548ca3c6fc9f9bfd Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 16:33:52 +0800 Subject: [PATCH 27/59] minor change --- relay/relay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/relay.go b/relay/relay.go index c874d057a2..7098ceb46e 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -664,7 +664,7 @@ func (r *Relay) SaveMeta(pos mysql.Position, gset gtid.Set) error { return nil } -// FlushMeta flush relay meta. +// FlushMeta flush relay meta func (r *Relay) FlushMeta() error { return r.meta.Flush() } From 2df59e4fa52a36094151a582e7d44856bc6c9ef4 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 23 Nov 2020 16:41:08 +0800 Subject: [PATCH 28/59] clear meta when reset --- relay/relay.go | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/relay.go b/relay/relay.go index d01670c603..8c42b090d8 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -704,6 +704,7 @@ func (r *Relay) SaveMeta(pos mysql.Position, gset gtid.Set) error { // ResetMeta reset relay meta func (r *Relay) ResetMeta() { r.meta = NewLocalMeta(r.cfg.Flavor, r.cfg.RelayDir) + r.relayMetaHub.ClearMeta() } // FlushMeta flush relay meta From de66815dc47400b95b89fc27efad203639750caa Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 25 Nov 2020 15:51:34 +0800 Subject: [PATCH 29/59] return error if reach end of file in reader --- _utils/terror_gen/errors_release.txt | 1 + errors.toml | 6 ++++++ pkg/binlog/reader/file.go | 7 +++++++ pkg/binlog/reader/file_test.go | 3 ++- pkg/streamer/reader.go | 2 +- pkg/terror/error_list.go | 5 +++++ 6 files changed, 22 insertions(+), 2 deletions(-) diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt index 93ef439944..44ec755007 100644 --- a/_utils/terror_gen/errors_release.txt +++ b/_utils/terror_gen/errors_release.txt @@ -125,6 +125,7 @@ ErrVerifyHandleErrorArgs,[code=11118:class=functional:scope=internal:level=low], ErrRewriteSQL,[code=11119:class=functional:scope=internal:level=high], "Message: failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v" ErrNoUUIDDirMatchGTID,[code=11120:class=functional:scope=internal:level=high], "Message: no relay subdir match gtid %s" ErrNoRelayPosMatchGTID,[code=11121:class=functional:scope=internal:level=high], "Message: no relay pos match gtid %s" +ErrReaderReachEndOfFile,[code=11122:class=functional:scope=internal:level=low] ErrConfigCheckItemNotSupport,[code=20001:class=config:scope=internal:level=medium], "Message: checking item %s is not supported\n%s, Workaround: Please check `ignore-checking-items` config in task configuration file, which can be set including `all`/`dump_privilege`/`replication_privilege`/`version`/`binlog_enable`/`binlog_format`/`binlog_row_image`/`table_schema`/`schema_of_shard_tables`/`auto_increment_ID`." ErrConfigTomlTransform,[code=20002:class=config:scope=internal:level=medium], "Message: %s, Workaround: Please check the configuration file has correct TOML format." ErrConfigYamlTransform,[code=20003:class=config:scope=internal:level=medium], "Message: %s, Workaround: Please check the configuration file has correct YAML format." diff --git a/errors.toml b/errors.toml index 5e9a98b7de..181a84bdf2 100644 --- a/errors.toml +++ b/errors.toml @@ -760,6 +760,12 @@ description = "" workaround = "" tags = ["internal", "high"] +[error.DM-functional-11122] +message = "" +description = "" +workaround = "" +tags = ["internal", "low"] + [error.DM-config-20001] message = "checking item %s is not supported\n%s" description = "" diff --git a/pkg/binlog/reader/file.go b/pkg/binlog/reader/file.go index a59a697cbe..c8c7948e2d 100644 --- a/pkg/binlog/reader/file.go +++ b/pkg/binlog/reader/file.go @@ -45,6 +45,7 @@ type FileReader struct { parser *replication.BinlogParser ch chan *replication.BinlogEvent ech chan error + endCh chan struct{} logger log.Logger @@ -89,6 +90,7 @@ func NewFileReader(cfg *FileReaderConfig) Reader { parser: parser, ch: make(chan *replication.BinlogEvent, cfg.ChBufferSize), ech: make(chan error, cfg.EchBufferSize), + endCh: make(chan struct{}), logger: log.With(zap.String("component", "binlog file reader")), } } @@ -115,6 +117,9 @@ func (r *FileReader) StartSyncByPos(pos gmysql.Position) error { case r.ech <- err: case <-r.ctx.Done(): } + } else { + r.logger.Info("parse end of binlog file", zap.Stringer("pos", pos)) + close(r.endCh) } }() @@ -159,6 +164,8 @@ func (r *FileReader) GetEvent(ctx context.Context) (*replication.BinlogEvent, er return ev, nil case err := <-r.ech: return nil, err + case <-r.endCh: + return nil, terror.ErrReaderReachEndOfFile.Generate() case <-ctx.Done(): return nil, ctx.Err() } diff --git a/pkg/binlog/reader/file_test.go b/pkg/binlog/reader/file_test.go index af7be451bb..712a566369 100644 --- a/pkg/binlog/reader/file_test.go +++ b/pkg/binlog/reader/file_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/dm/pkg/binlog/common" "github.com/pingcap/dm/pkg/binlog/event" "github.com/pingcap/dm/pkg/gtid" + "github.com/pingcap/dm/pkg/terror" ) var ( @@ -138,7 +139,7 @@ func (t *testFileReaderSuite) TestGetEvent(c *C) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() e, err = r.GetEvent(ctx) - c.Assert(errors.Cause(err), Equals, context.DeadlineExceeded) + c.Assert(terror.ErrReaderReachEndOfFile.Equal(err), IsTrue) c.Assert(e, IsNil) c.Assert(r.Close(), IsNil) // close the reader diff --git a/pkg/streamer/reader.go b/pkg/streamer/reader.go index df93eab03a..e4db581218 100644 --- a/pkg/streamer/reader.go +++ b/pkg/streamer/reader.go @@ -179,7 +179,7 @@ func (r *BinlogReader) GetFilePosByGTID(ctx context.Context, filePath string, gs cancel() if err != nil { // reach end of file - if errors.Cause(err) == context.DeadlineExceeded { + if terror.ErrReaderReachEndOfFile.Equal(err) { return lastPos, nil } return 0, err diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index fd6273eee5..c5681be640 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -178,6 +178,9 @@ const ( // pkg/streamer codeNoUUIDDirMatchGTID codeNoRelayPosMatchGTID + + // pkg/reader + codeReaderReachEndOfFile ) // Config related error code list @@ -770,6 +773,8 @@ var ( ErrNoUUIDDirMatchGTID = New(codeNoUUIDDirMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay subdir match gtid %s", "") ErrNoRelayPosMatchGTID = New(codeNoRelayPosMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay pos match gtid %s", "") + ErrReaderReachEndOfFile = New(codeReaderReachEndOfFile, ClassFunctional, ScopeInternal, LevelLow, "", "") + // Config related error ErrConfigCheckItemNotSupport = New(codeConfigCheckItemNotSupport, ClassConfig, ScopeInternal, LevelMedium, "checking item %s is not supported\n%s", "Please check `ignore-checking-items` config in task configuration file, which can be set including `all`/`dump_privilege`/`replication_privilege`/`version`/`binlog_enable`/`binlog_format`/`binlog_row_image`/`table_schema`/`schema_of_shard_tables`/`auto_increment_ID`.") ErrConfigTomlTransform = New(codeConfigTomlTransform, ClassConfig, ScopeInternal, LevelMedium, "%s", "Please check the configuration file has correct TOML format.") From cdb3d1f408e3801952df51a777d2fb03be1781c4 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 25 Nov 2020 16:16:04 +0800 Subject: [PATCH 30/59] enable purge relay cmd --- dm/ctl/ctl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/ctl/ctl.go b/dm/ctl/ctl.go index 71fc621b4b..f5738d36c0 100644 --- a/dm/ctl/ctl.go +++ b/dm/ctl/ctl.go @@ -66,7 +66,7 @@ func NewRootCmd() *cobra.Command { master.NewUnlockDDLLockCmd(), master.NewPauseRelayCmd(), master.NewResumeRelayCmd(), - // master.NewPurgeRelayCmd(), + master.NewPurgeRelayCmd(), master.NewOperateSourceCmd(), master.NewOfflineMemberCmd(), master.NewOperateLeaderCmd(), From 55b876164b6302c87366f8e88aaef05252912d5e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 25 Nov 2020 16:27:20 +0800 Subject: [PATCH 31/59] change help_cnt --- tests/dmctl_command/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dmctl_command/run.sh b/tests/dmctl_command/run.sh index 87ed15f621..14469b3f23 100644 --- a/tests/dmctl_command/run.sh +++ b/tests/dmctl_command/run.sh @@ -6,7 +6,7 @@ cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source $cur/../_utils/test_prepare WORK_DIR=$TEST_DIR/$TEST_NAME -help_cnt=33 +help_cnt=34 function run() { # check dmctl alone output From d3c4689967e1abba972b20c5c3900c80c973bce7 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 25 Nov 2020 17:00:03 +0800 Subject: [PATCH 32/59] remove error comment --- pkg/terror/error_list.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index c5681be640..0ca7fd00dd 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -178,8 +178,6 @@ const ( // pkg/streamer codeNoUUIDDirMatchGTID codeNoRelayPosMatchGTID - - // pkg/reader codeReaderReachEndOfFile ) @@ -770,9 +768,8 @@ var ( ErrRewriteSQL = New(codeRewriteSQL, ClassFunctional, ScopeInternal, LevelHigh, "failed to rewrite SQL for target DB, stmt: %+v, targetTableNames: %+v", "") // pkg/streamer - ErrNoUUIDDirMatchGTID = New(codeNoUUIDDirMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay subdir match gtid %s", "") - ErrNoRelayPosMatchGTID = New(codeNoRelayPosMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay pos match gtid %s", "") - + ErrNoUUIDDirMatchGTID = New(codeNoUUIDDirMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay subdir match gtid %s", "") + ErrNoRelayPosMatchGTID = New(codeNoRelayPosMatchGTID, ClassFunctional, ScopeInternal, LevelHigh, "no relay pos match gtid %s", "") ErrReaderReachEndOfFile = New(codeReaderReachEndOfFile, ClassFunctional, ScopeInternal, LevelLow, "", "") // Config related error From 32de02eaa854d5b5f4c4a9c2833deb5b78568ca0 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 25 Nov 2020 17:25:30 +0800 Subject: [PATCH 33/59] fix some tests --- tests/incremental_mode/run.sh | 2 +- tests/online_ddl/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/incremental_mode/run.sh b/tests/incremental_mode/run.sh index 6667bb91dd..2c9246a7dd 100755 --- a/tests/incremental_mode/run.sh +++ b/tests/incremental_mode/run.sh @@ -110,7 +110,7 @@ function run() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "Running" 2 + "Running" 4 # we use failpoint to let worker sleep 8 second when executeSQLs, to increase possibility of # meeting an error of context cancel. # when below check pass, it means we filter out that error, or that error doesn't happen. diff --git a/tests/online_ddl/run.sh b/tests/online_ddl/run.sh index d970a23d43..8c4d5f9e2b 100755 --- a/tests/online_ddl/run.sh +++ b/tests/online_ddl/run.sh @@ -52,7 +52,7 @@ function real_run() { echo "wait and check task running" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 run_sql_file $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 From 4fd723fd51a36fd45a63aa9dd0841bbd203ccfbe Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 25 Nov 2020 18:19:47 +0800 Subject: [PATCH 34/59] fix compatibility test --- tests/compatibility/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compatibility/start.sh b/tests/compatibility/start.sh index 7f940c4926..d392debda9 100755 --- a/tests/compatibility/start.sh +++ b/tests/compatibility/start.sh @@ -68,7 +68,7 @@ function run() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 2 + "\"stage\": \"Running\"" 4 echo "use sync_diff_inspector to check data second time" check_sync_diff $WORK_DIR $cur/conf/diff_config.toml From e4448891a9cce186b52135ce4bc89c14663de278 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 26 Nov 2020 17:42:03 +0800 Subject: [PATCH 35/59] add test for resetup meta --- dm/worker/server.go | 1 - dm/worker/server_test.go | 2 +- relay/relay_test.go | 19 ++++++++++++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/dm/worker/server.go b/dm/worker/server.go index d492c15407..bb79450bbe 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -639,7 +639,6 @@ func makeCommonWorkerResponse(reqErr error) *pb.CommonWorkerResponse { // all subTask in subTaskCfgs should have same source // this function return the min location in all subtasks, used for relay's location -// TODO: get min gtidSet func getMinLocInAllSubTasks(ctx context.Context, subTaskCfgs []*config.SubTaskConfig) (minLoc *binlog.Location, err error) { for _, subTaskCfg := range subTaskCfgs { loc, err := getMinLocForSubTaskFunc(ctx, subTaskCfg) diff --git a/dm/worker/server_test.go b/dm/worker/server_test.go index f5a2a23399..a5505a9faa 100644 --- a/dm/worker/server_test.go +++ b/dm/worker/server_test.go @@ -406,7 +406,7 @@ func (t *testServer) testStopWorkerWhenLostConnect(c *C, s *Server, ETCD *embed. c.Assert(s.getWorker(true), IsNil) } -func (t *testServer) TestGetMinPosInAllSubTasks(c *C) { +func (t *testServer) TestGetMinLocInAllSubTasks(c *C) { subTaskCfg := []*config.SubTaskConfig{ { Name: "test2", diff --git a/relay/relay_test.go b/relay/relay_test.go index 3f2c0441f0..573cfbf050 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -487,12 +487,11 @@ func (t *testRelaySuite) TestReSetupMeta(c *C) { defer cancel() var ( - relayCfg = &Config{ - RelayDir: c.MkDir(), - Flavor: gmysql.MySQLFlavor, - } - r = NewRelay(relayCfg).(*Relay) + relayCfg = newRelayCfg(c, mysql.MySQLFlavor) + r = NewRelay(relayCfg).(*Relay) ) + c.Assert(r.Init(context.Background()), IsNil) + // empty metadata c.Assert(r.meta.Load(), IsNil) t.verifyMetadata(c, r, "", minCheckpoint, "", nil) @@ -521,6 +520,16 @@ func (t *testRelaySuite) TestReSetupMeta(c *C) { uuid002 := fmt.Sprintf("%s.000002", uuid) t.verifyMetadata(c, r, uuid002, minCheckpoint, r.cfg.BinlogGTID, []string{uuid001, uuid002}) + r.cfg.BinLogName = "mysql-bin.000002" + r.cfg.BinlogGTID = "24ecd093-8cec-11e9-aa0d-0242ac170002:1-50,24ecd093-8cec-11e9-aa0d-0242ac170003:1-50" + r.cfg.UUIDSuffix = "000002" + c.Assert(r.reSetupMeta(ctx), IsNil) + t.verifyMetadata(c, r, uuid002, gmysql.Position{Name: r.cfg.BinLogName, Pos: 4}, r.cfg.BinlogGTID, []string{uuid002}) + + // re-setup meta again, often happen when connecting a server behind a VIP. + c.Assert(r.reSetupMeta(ctx), IsNil) + uuid003 := fmt.Sprintf("%s.000003", uuid) + t.verifyMetadata(c, r, uuid003, minCheckpoint, r.cfg.BinlogGTID, []string{uuid002, uuid003}) } func (t *testRelaySuite) verifyMetadata(c *C, r *Relay, uuidExpected string, From 16a7558ce437dfa212c2e12dcd6c39c97f7a305c Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 26 Nov 2020 17:59:49 +0800 Subject: [PATCH 36/59] add more comment --- dm/worker/relay.go | 2 +- relay/meta.go | 3 ++- relay/relay.go | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dm/worker/relay.go b/dm/worker/relay.go index e733fa2b48..0028d09980 100644 --- a/dm/worker/relay.go +++ b/dm/worker/relay.go @@ -232,7 +232,7 @@ func (h *realRelayHolder) stopRelay(ctx context.Context, op pb.RelayOp) error { return terror.ErrWorkerRelayStageNotValid.Generatef("current stage is already stopped not valid, relayop %s", op) } h.stage = pb.Stage_Stopped - h.Unlock() + h.Unlock() // unlock to make `run` can return // now, when try to stop relay unit, we close relay holder h.Close() diff --git a/relay/meta.go b/relay/meta.go index b9c24fe1e1..1fcaa9443e 100644 --- a/relay/meta.go +++ b/relay/meta.go @@ -58,7 +58,8 @@ type Meta interface { Dirty() bool // AddDir adds sub relay directory for server UUID (without suffix) - // the added sub relay directory's suffix is incremented + // if uuidSuffix not nil, add sub relay directory with uuidSuffix (bount to a new source) + // otherwise the added sub relay directory's suffix is incremented (master/slave switch) // after sub relay directory added, the internal binlog pos should be reset // and binlog pos will be set again when new binlog events received // @serverUUID should be a server_uuid for MySQL or MariaDB diff --git a/relay/relay.go b/relay/relay.go index 8c42b090d8..34b1fc90e0 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -519,6 +519,7 @@ func (r *Relay) reSetupMeta(ctx context.Context) error { var newGset gtid.Set var newUUIDSufiix string if len(r.cfg.UUIDSuffix) != 0 { + // if bound or rebound to a source, clear all relay log and meta if err = r.PurgeRelayDir(); err != nil { return err } From 5e875abc2098b5321c69a78e7964af5618eee99c Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 26 Nov 2020 18:25:29 +0800 Subject: [PATCH 37/59] fix fmt --- dm/worker/relay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/worker/relay.go b/dm/worker/relay.go index 0028d09980..6dcbe0d859 100644 --- a/dm/worker/relay.go +++ b/dm/worker/relay.go @@ -232,7 +232,7 @@ func (h *realRelayHolder) stopRelay(ctx context.Context, op pb.RelayOp) error { return terror.ErrWorkerRelayStageNotValid.Generatef("current stage is already stopped not valid, relayop %s", op) } h.stage = pb.Stage_Stopped - h.Unlock() // unlock to make `run` can return + h.Unlock() // unlock to make `run` can return // now, when try to stop relay unit, we close relay holder h.Close() From e54f231cf941d31581cc56551ec0a643307e87a9 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Fri, 27 Nov 2020 11:51:23 +0800 Subject: [PATCH 38/59] fix test --- relay/relay_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/relay/relay_test.go b/relay/relay_test.go index 573cfbf050..befb280257 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -536,9 +536,11 @@ func (t *testRelaySuite) verifyMetadata(c *C, r *Relay, uuidExpected string, posExpected gmysql.Position, gsStrExpected string, UUIDsExpected []string) { uuid, pos := r.meta.Pos() _, gs := r.meta.GTID() + gsExpected, err := gtid.ParserGTID(mysql.MySQLFlavor, gsStrExpected) + c.Assert(err, IsNil) c.Assert(uuid, Equals, uuidExpected) c.Assert(pos, DeepEquals, posExpected) - c.Assert(gs.String(), Equals, gsStrExpected) + c.Assert(gs.Equal(gsExpected), IsTrue) indexFile := filepath.Join(r.cfg.RelayDir, utils.UUIDIndexFilename) UUIDs, err := utils.ParseUUIDIndex(indexFile) From 151e97c1407282ea4e2772e0718932caa0e4448a Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 30 Nov 2020 18:10:12 +0800 Subject: [PATCH 39/59] relay test for chaos --- chaos/cases/conf/source1.yaml | 2 ++ chaos/cases/conf/source2.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/chaos/cases/conf/source1.yaml b/chaos/cases/conf/source1.yaml index 686897f149..f474f85131 100644 --- a/chaos/cases/conf/source1.yaml +++ b/chaos/cases/conf/source1.yaml @@ -1,4 +1,6 @@ source-id: "mysql-replica-01" +enable-gtid: false +enable-relay: true from: host: "mysql-0.mysql" # same namespace with MySQL diff --git a/chaos/cases/conf/source2.yaml b/chaos/cases/conf/source2.yaml index b9c01ef575..3d561d71c6 100644 --- a/chaos/cases/conf/source2.yaml +++ b/chaos/cases/conf/source2.yaml @@ -1,4 +1,6 @@ source-id: "mysql-replica-02" +enable-gtid: true +enable-relay: true from: host: "mysql-1.mysql" # same namespace with MySQL From 67b928fab79503decef4872f63a04914c5318e41 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 2 Dec 2020 09:54:11 +0800 Subject: [PATCH 40/59] minor change --- dm/worker/task_checker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/worker/task_checker.go b/dm/worker/task_checker.go index 80a6ef70ae..fbacbfca09 100644 --- a/dm/worker/task_checker.go +++ b/dm/worker/task_checker.go @@ -333,7 +333,7 @@ func (tsc *realTaskStatusChecker) checkRelayStatus() { if err != nil { tsc.l.Error("dispatch auto resume relay failed", zap.Error(err)) } else { - tsc.l.Info("dispatch auto resume task") + tsc.l.Info("dispatch auto resume relay") tsc.bc.latestRelayResumeTime = time.Now() rbf.BoundaryForward() } From 76f338d5627a6981da1c50395993925f05e84db3 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 2 Dec 2020 11:53:43 +0800 Subject: [PATCH 41/59] fix default relay-dir --- dm/config/source_config.go | 6 +++++- tests/dmctl_command/conf/source2.yaml | 1 + tests/dmctl_command/run.sh | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 8e4c3ac0f4..0895863ba5 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -24,6 +24,7 @@ import ( const ( // the default base(min) server id generated by random defaultBaseServerID = math.MaxUint32 / 10 + defaultRelayDir = "relay-dir" ) var getAllServerIDFunc = utils.GetAllServerID @@ -75,7 +76,6 @@ type SourceConfig struct { // NewSourceConfig creates a new base config for upstream MySQL/MariaDB source. func NewSourceConfig() *SourceConfig { c := &SourceConfig{ - RelayDir: "relay-dir", Purge: PurgeConfig{ Interval: 60 * 60, Expires: 0, @@ -244,6 +244,10 @@ func (c *SourceConfig) Adjust(ctx context.Context, db *sql.DB) (err error) { } } + if c.EnableRelay && len(c.RelayDir) == 0 { + c.RelayDir = defaultRelayDir + } + return nil } diff --git a/tests/dmctl_command/conf/source2.yaml b/tests/dmctl_command/conf/source2.yaml index cc4bc2f3a4..c7b63092c6 100644 --- a/tests/dmctl_command/conf/source2.yaml +++ b/tests/dmctl_command/conf/source2.yaml @@ -2,6 +2,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true enable-relay: true +relay-dir: '' relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dmctl_command/run.sh b/tests/dmctl_command/run.sh index 14469b3f23..4bdcd80b28 100644 --- a/tests/dmctl_command/run.sh +++ b/tests/dmctl_command/run.sh @@ -93,7 +93,6 @@ function run() { cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml - sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 From 92ef267d276101ff18abd47aa07790bbaee13920 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 2 Dec 2020 14:46:53 +0800 Subject: [PATCH 42/59] fix error log --- pkg/binlog/reader/file.go | 5 ++++- relay/relay.go | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/binlog/reader/file.go b/pkg/binlog/reader/file.go index c8c7948e2d..b55651f2eb 100644 --- a/pkg/binlog/reader/file.go +++ b/pkg/binlog/reader/file.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/pingcap/errors" gmysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "github.com/siddontang/go/sync2" @@ -112,7 +113,9 @@ func (r *FileReader) StartSyncByPos(pos gmysql.Position) error { defer r.wg.Done() err := r.parser.ParseFile(pos.Name, int64(pos.Pos), r.onEvent) if err != nil { - r.logger.Error("fail to parse binlog file", zap.Error(err)) + if errors.Cause(err) != context.Canceled { + r.logger.Error("fail to parse binlog file", zap.Error(err)) + } select { case r.ech <- err: case <-r.ctx.Done(): diff --git a/relay/relay.go b/relay/relay.go index 34b1fc90e0..b263d95522 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -386,8 +386,15 @@ func (r *Relay) tryRecoverLatestFile(ctx context.Context, parser2 *parser.Parser func (r *Relay) handleEvents(ctx context.Context, reader2 reader.Reader, transformer2 transformer.Transformer, writer2 writer.Writer) error { var ( _, lastPos = r.meta.Pos() - _, lastGTID = r.meta.GTID() + _, metaGTID = r.meta.GTID() + lastGTID gtid.Set + err error ) + if metaGTID != nil { + lastGTID = metaGTID.Clone() + } else if lastGTID, err = gtid.ParserGTID(r.cfg.Flavor, ""); err != nil { + return err + } for { // 1. read events from upstream server From f14659f7b40cb8e131e4877599eb45e05e077914 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 2 Dec 2020 17:06:55 +0800 Subject: [PATCH 43/59] fix unsupported table option --- tests/_utils/run_sql_file_online_ddl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_utils/run_sql_file_online_ddl b/tests/_utils/run_sql_file_online_ddl index ee5179c31a..6a137656c9 100755 --- a/tests/_utils/run_sql_file_online_ddl +++ b/tests/_utils/run_sql_file_online_ddl @@ -45,7 +45,7 @@ do --recursion-method=none --print --execute \ >> $TEST_DIR/pt-osc.log $ptosc_bin --user=root --host=$host --port=$port --password=$password \ - --alter "row_format=compressed key_block_size=8" D=$schema,t=$table \ + --alter "COMMENT='string' CHARACTER SET='utf8mb4'" D=$schema,t=$table \ --recursion-method=none --print --execute \ >> $TEST_DIR/pt-osc.log else From 67337852fc7dd80247a41315dfa281ebe31d5197 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 7 Dec 2020 10:12:05 +0800 Subject: [PATCH 44/59] better arg name --- relay/relay_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay/relay_test.go b/relay/relay_test.go index befb280257..2732681261 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -533,7 +533,7 @@ func (t *testRelaySuite) TestReSetupMeta(c *C) { } func (t *testRelaySuite) verifyMetadata(c *C, r *Relay, uuidExpected string, - posExpected gmysql.Position, gsStrExpected string, UUIDsExpected []string) { + posExpected gmysql.Position, gsStrExpected string, uuidsExpected []string) { uuid, pos := r.meta.Pos() _, gs := r.meta.GTID() gsExpected, err := gtid.ParserGTID(mysql.MySQLFlavor, gsStrExpected) @@ -545,7 +545,7 @@ func (t *testRelaySuite) verifyMetadata(c *C, r *Relay, uuidExpected string, indexFile := filepath.Join(r.cfg.RelayDir, utils.UUIDIndexFilename) UUIDs, err := utils.ParseUUIDIndex(indexFile) c.Assert(err, IsNil) - c.Assert(UUIDs, DeepEquals, UUIDsExpected) + c.Assert(UUIDs, DeepEquals, uuidsExpected) } func (t *testRelaySuite) TestProcess(c *C) { From b6c19f97f366c7d8b8b1531ca056b4d214c63a22 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 9 Dec 2020 10:40:26 +0800 Subject: [PATCH 45/59] address comment --- dm/config/source_config.go | 2 +- pkg/binlog/position.go | 12 +++++++----- pkg/utils/relay.go | 5 ----- relay/config.go | 2 +- relay/meta.go | 10 +++++----- relay/meta_test.go | 4 ++-- relay/relay.go | 8 ++++---- relay/relay_test.go | 8 ++++---- tests/dmctl_basic/run.sh | 36 ++++++++++++++++++------------------ 9 files changed, 42 insertions(+), 45 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 0895863ba5..306e158a6c 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -55,7 +55,7 @@ type SourceConfig struct { RelayBinLogName string `yaml:"relay-binlog-name" toml:"relay-binlog-name" json:"relay-binlog-name"` RelayBinlogGTID string `yaml:"relay-binlog-gtid" toml:"relay-binlog-gtid" json:"relay-binlog-gtid"` // only use when worker bound source, do not marsh it - UUIDSuffix string `yaml:"-" toml:"-" json:"-"` + UUIDSuffix int `yaml:"-" toml:"-" json:"-"` SourceID string `yaml:"source-id" toml:"source-id" json:"source-id"` From DBConfig `yaml:"from" toml:"from" json:"from"` diff --git a/pkg/binlog/position.go b/pkg/binlog/position.go index 0a82cf9e0c..0df9989dbc 100644 --- a/pkg/binlog/position.go +++ b/pkg/binlog/position.go @@ -112,19 +112,21 @@ func RealMySQLPos(pos gmysql.Position) (gmysql.Position, error) { } // ExtractSuffix extracts uuidSuffix from input name -func ExtractSuffix(name string) (string, error) { +func ExtractSuffix(name string) (int, error) { if len(name) == 0 { - return fmt.Sprintf("%06d", minUUIDSuffix), nil + return minUUIDSuffix, nil } filename, err := ParseFilename(name) if err != nil { - return "", err + return 0, err } sepIdx := strings.LastIndex(filename.BaseName, posUUIDSuffixSeparator) if sepIdx > 0 && sepIdx+len(posUUIDSuffixSeparator) < len(filename.BaseName) { - return filename.BaseName[sepIdx+len(posUUIDSuffixSeparator):], nil + suffix := filename.BaseName[sepIdx+len(posUUIDSuffixSeparator):] + v, err := strconv.ParseInt(suffix, 10, 64) + return int(v), err } - return fmt.Sprintf("%06d", minUUIDSuffix), nil + return minUUIDSuffix, nil } // ExtractPos extracts (uuidWithSuffix, uuidSuffix, originalPos) from input pos (originalPos or convertedPos) diff --git a/pkg/utils/relay.go b/pkg/utils/relay.go index a6a4cf8002..ce1f31ee7e 100644 --- a/pkg/utils/relay.go +++ b/pkg/utils/relay.go @@ -72,11 +72,6 @@ func AddSuffixForUUID(uuid string, ID int) string { return fmt.Sprintf("%s%s%06d", uuid, uuidIndexSeparator, ID) // eg. 53ea0ed1-9bf8-11e6-8bea-64006a897c73.000001 } -// AddSuffixStrForUUID adds a suffix string for UUID -func AddSuffixStrForUUID(uuid string, suffix string) string { - return fmt.Sprintf("%s%s%s", uuid, uuidIndexSeparator, suffix) // eg. 53ea0ed1-9bf8-11e6-8bea-64006a897c73.000001 -} - // SuffixIntToStr convert int-represented suffix to string-represented func SuffixIntToStr(ID int) string { return fmt.Sprintf("%06d", ID) diff --git a/relay/config.go b/relay/config.go index 294e688739..63b150329d 100644 --- a/relay/config.go +++ b/relay/config.go @@ -35,7 +35,7 @@ type Config struct { // do not need to specify binlog-pos, because relay will fetch the whole file BinLogName string `toml:"binlog-name" json:"binlog-name"` BinlogGTID string `toml:"binlog-gtid" json:"binlog-gtid"` - UUIDSuffix string `toml:"-" json:"-"` + UUIDSuffix int `toml:"-" json:"-"` // for binlog reader retry ReaderRetry retry.ReaderRetryConfig `toml:"reader-retry" json:"reader-retry"` diff --git a/relay/meta.go b/relay/meta.go index 1fcaa9443e..200db0ca8b 100644 --- a/relay/meta.go +++ b/relay/meta.go @@ -58,13 +58,13 @@ type Meta interface { Dirty() bool // AddDir adds sub relay directory for server UUID (without suffix) - // if uuidSuffix not nil, add sub relay directory with uuidSuffix (bount to a new source) + // if uuidSuffix is not zero value, add sub relay directory with uuidSuffix (bount to a new source) // otherwise the added sub relay directory's suffix is incremented (master/slave switch) // after sub relay directory added, the internal binlog pos should be reset // and binlog pos will be set again when new binlog events received // @serverUUID should be a server_uuid for MySQL or MariaDB // if set @newPos / @newGTID, old value will be replaced - AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set, uuidSuffix string) error + AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set, uuidSuffix int) error // Pos returns current (UUID with suffix, Position) pair Pos() (string, mysql.Position) @@ -269,7 +269,7 @@ func (lm *LocalMeta) Dir() string { } // AddDir implements Meta.AddDir -func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set, uuidSuffix string) error { +func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID gtid.Set, uuidSuffix int) error { lm.Lock() defer lm.Unlock() @@ -277,10 +277,10 @@ func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID g if len(lm.currentUUID) == 0 { // no UUID exists yet, simply add it - if len(uuidSuffix) == 0 { + if uuidSuffix == 0 { newUUID = utils.AddSuffixForUUID(serverUUID, minUUIDSufix) } else { - newUUID = utils.AddSuffixStrForUUID(serverUUID, uuidSuffix) + newUUID = utils.AddSuffixForUUID(serverUUID, uuidSuffix) } } else { _, suffix, err := utils.ParseSuffixForUUID(lm.currentUUID) diff --git a/relay/meta_test.go b/relay/meta_test.go index 082394a34a..9e5ca98073 100644 --- a/relay/meta_test.go +++ b/relay/meta_test.go @@ -143,7 +143,7 @@ func (r *testMetaSuite) TestLocalMeta(c *C) { c.Assert(gset.String(), Equals, latestGTIDStr) for _, cs := range cases { - err = lm.AddDir(cs.uuid, nil, nil, "") + err = lm.AddDir(cs.uuid, nil, nil, 0) c.Assert(err, IsNil) err = lm.Save(cs.pos, cs.gset) @@ -203,7 +203,7 @@ func (r *testMetaSuite) TestLocalMeta(c *C) { pos: mysql.Position{Name: "mysql-bin.000005", Pos: 789}, gset: gset5, } - err = lm.AddDir(cs.uuid, &cs.pos, cs.gset, "") + err = lm.AddDir(cs.uuid, &cs.pos, cs.gset, 0) c.Assert(err, IsNil) dirty = lm.Dirty() diff --git a/relay/relay.go b/relay/relay.go index b263d95522..dbefc2dc6b 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -203,7 +203,7 @@ func (r *Relay) process(ctx context.Context) error { return err } - if isNew || len(r.cfg.UUIDSuffix) > 0 { + if isNew || r.cfg.UUIDSuffix > 0 { // re-setup meta for new server or new source err = r.reSetupMeta(ctx) if err != nil { @@ -524,8 +524,8 @@ func (r *Relay) reSetupMeta(ctx context.Context) error { var newPos *mysql.Position var newGset gtid.Set - var newUUIDSufiix string - if len(r.cfg.UUIDSuffix) != 0 { + var newUUIDSufiix int + if r.cfg.UUIDSuffix > 0 { // if bound or rebound to a source, clear all relay log and meta if err = r.PurgeRelayDir(); err != nil { return err @@ -534,7 +534,7 @@ func (r *Relay) reSetupMeta(ctx context.Context) error { newUUIDSufiix = r.cfg.UUIDSuffix // reset the UUIDSuffix - r.cfg.UUIDSuffix = "" + r.cfg.UUIDSuffix = 0 if len(r.cfg.BinLogName) != 0 { newPos = &mysql.Position{Name: r.cfg.BinLogName, Pos: binlog.MinPosition.Pos} diff --git a/relay/relay_test.go b/relay/relay_test.go index 2732681261..4aa5517b98 100644 --- a/relay/relay_test.go +++ b/relay/relay_test.go @@ -194,7 +194,7 @@ func (t *testRelaySuite) TestTryRecoverLatestFile(c *C) { c.Assert(r.tryRecoverLatestFile(context.Background(), parser2), IsNil) // save position into meta - c.Assert(r.meta.AddDir(uuid, &startPos, nil, ""), IsNil) + c.Assert(r.meta.AddDir(uuid, &startPos, nil, 0), IsNil) // relay log file does not exists, no need to recover c.Assert(r.tryRecoverLatestFile(context.Background(), parser2), IsNil) @@ -265,7 +265,7 @@ func (t *testRelaySuite) TestTryRecoverMeta(c *C) { recoverGTIDSet, err := gtid.ParserGTID(relayCfg.Flavor, recoverGTIDSetStr) c.Assert(err, IsNil) - c.Assert(r.meta.AddDir(uuid, &startPos, nil, ""), IsNil) + c.Assert(r.meta.AddDir(uuid, &startPos, nil, 0), IsNil) c.Assert(r.meta.Load(), IsNil) // use a generator to generate some binlog events @@ -395,7 +395,7 @@ func (t *testRelaySuite) TestHandleEvent(c *C) { c.Assert(r.Init(context.Background()), IsNil) // NOTE: we can mock meta later. c.Assert(r.meta.Load(), IsNil) - c.Assert(r.meta.AddDir("24ecd093-8cec-11e9-aa0d-0242ac170002", nil, nil, ""), IsNil) + c.Assert(r.meta.AddDir("24ecd093-8cec-11e9-aa0d-0242ac170002", nil, nil, 0), IsNil) // attach GTID sets to QueryEv queryEv2 := queryEv.Event.(*replication.QueryEvent) @@ -522,7 +522,7 @@ func (t *testRelaySuite) TestReSetupMeta(c *C) { r.cfg.BinLogName = "mysql-bin.000002" r.cfg.BinlogGTID = "24ecd093-8cec-11e9-aa0d-0242ac170002:1-50,24ecd093-8cec-11e9-aa0d-0242ac170003:1-50" - r.cfg.UUIDSuffix = "000002" + r.cfg.UUIDSuffix = 2 c.Assert(r.reSetupMeta(ctx), IsNil) t.verifyMetadata(c, r, uuid002, gmysql.Position{Name: r.cfg.BinLogName, Pos: 4}, r.cfg.BinlogGTID, []string{uuid002}) diff --git a/tests/dmctl_basic/run.sh b/tests/dmctl_basic/run.sh index 9071f0f3ae..c5d22656c0 100755 --- a/tests/dmctl_basic/run.sh +++ b/tests/dmctl_basic/run.sh @@ -16,13 +16,13 @@ function usage_and_arg_test() { check_task_wrong_arg check_task_wrong_config_file -# echo "pause_relay_wrong_arg" -# pause_relay_wrong_arg -# pause_relay_wihout_worker -# -# echo "resume_relay_wrong_arg" -# resume_relay_wrong_arg -# resume_relay_wihout_worker + echo "pause_relay_wrong_arg" + pause_relay_wrong_arg + pause_relay_wihout_worker + + echo "resume_relay_wrong_arg" + resume_relay_wrong_arg + resume_relay_wihout_worker echo "pause_task_wrong_arg" pause_task_wrong_arg @@ -57,10 +57,10 @@ function usage_and_arg_test() { # update_master_config_wrong_arg # update_master_config_wrong_config_file # -# echo "purge_relay_wrong_arg" -# purge_relay_wrong_arg -# purge_relay_wihout_worker -# purge_relay_filename_with_multi_workers + echo "purge_relay_wrong_arg" + purge_relay_wrong_arg + purge_relay_wihout_worker + purge_relay_filename_with_multi_workers echo "operate_source_empty_arg" operate_source_empty_arg @@ -147,13 +147,13 @@ function run() { '"worker": "worker1"' 1 \ '"worker": "worker2"' 1 -# echo "pause_relay_success" -# pause_relay_success -# query_status_stopped_relay -# # pause twice won't receive an error now -# # pause_relay_fail -# resume_relay_success -# query_status_with_no_tasks + echo "pause_relay_success" + pause_relay_success + query_status_stopped_relay + # pause twice won't receive an error now + # pause_relay_fail + resume_relay_success + query_status_with_no_tasks echo "dmctl_check_task" check_task_pass $TASK_CONF From 76b784101ad34a257cab9fa8370788eb4987609d Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 9 Dec 2020 11:40:42 +0800 Subject: [PATCH 46/59] fix data race --- relay/relay.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/relay/relay.go b/relay/relay.go index dbefc2dc6b..e68cdba2f1 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -772,8 +772,12 @@ func (r *Relay) Status(ctx context.Context) interface{} { rs.RelayBinlogGtid = relayGTIDSet.String() } if r.cfg.EnableGTID { - if masterGTID != nil && relayGTIDSet != nil && relayGTIDSet.Equal(masterGTID) { - rs.RelayCatchUpMaster = true + if masterGTID != nil && relayGTIDSet != nil { + // avoid data race + clone := relayGTIDSet.Clone() + if clone.Equal(masterGTID) { + rs.RelayCatchUpMaster = true + } } } else { rs.RelayCatchUpMaster = masterPos.Compare(relayPos) == 0 From 8f7cf97c49ad3749f4d7bbd4e9fc075d0957b6b1 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 9 Dec 2020 14:37:02 +0800 Subject: [PATCH 47/59] fix worker not exit --- tests/ha_cases/lib.sh | 4 ++-- tests/ha_cases/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ha_cases/lib.sh b/tests/ha_cases/lib.sh index 9850905a89..e5afc4bb3d 100755 --- a/tests/ha_cases/lib.sh +++ b/tests/ha_cases/lib.sh @@ -20,8 +20,8 @@ function load_data() { run_sql "CREATE DATABASE if not exists ${db};" $port $pswd run_sql "DROP TABLE if exists ${db}.t${i};" $port $pswd run_sql "CREATE TABLE ${db}.t${i}(i TINYINT, j INT UNIQUE KEY);" $port $pswd - for j in $(seq 80); do - run_sql "INSERT INTO ${db}.t${i} VALUES ($j,${j}000$j),($j,${j}001$j);" $port $pswd + for j in $(seq 800); do + run_sql "INSERT INTO ${db}.t${i} VALUES ($j,${j}00$j),($j,${j}01$j);" $port $pswd sleep 0.1 done } diff --git a/tests/ha_cases/run.sh b/tests/ha_cases/run.sh index e579e7a9fb..d7c05fa060 100755 --- a/tests/ha_cases/run.sh +++ b/tests/ha_cases/run.sh @@ -542,7 +542,7 @@ function test_multi_task_reduce_and_restart_worker() { echo "try to kill worker port ${worker_ports[$[ $idx - 1 ] ]}" ps aux | grep dm-worker${idx} |awk '{print $2}'|xargs kill || true - check_port_offline ${worker_ports[$[ $idx - 1] ]} 20 + run_dm_ctl_with_retry $WORK_DIR 127.0.0.1:$MASTER_PORT2 "list-member --worker --name=worker$idx" '"stage": "offline"' 1 echo "start dm-worker${idx}" run_dm_worker $WORK_DIR/worker${idx} ${worker_ports[$[ $idx - 1] ]} $cur/conf/dm-worker${idx}.toml From faf371a9d91919fc0ffe5289f174e0b2d442aac7 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 17 Dec 2020 15:31:00 +0800 Subject: [PATCH 48/59] address comment --- pkg/binlog/position_test.go | 28 ++++++++++++++++++++++++++++ pkg/utils/db.go | 27 +++++++++++++++++++++++++++ relay/meta.go | 7 +++++-- relay/relay.go | 19 +++++++++---------- 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/pkg/binlog/position_test.go b/pkg/binlog/position_test.go index f551569330..197a3a6bf1 100644 --- a/pkg/binlog/position_test.go +++ b/pkg/binlog/position_test.go @@ -765,3 +765,31 @@ func (t *testPositionSuite) TestSetGTID(c *C) { c.Assert(loc2.gtidSet.String(), Equals, GTIDSetStr) c.Assert(CompareLocation(loc, loc2, true), Equals, 1) } + +func (t *testPositionSuite) TestExtractSuffix(c *C) { + testCases := []struct { + name string + suffix int + }{ + { + "", + minUUIDSuffix, + }, { + "mysql-bin.00005", + minUUIDSuffix, + }, + { + "mysql-bin|000001.000001", + 1, + }, { + "mysql-bin|000005.000004", + 5, + }, + } + + for _, tc := range testCases { + suffix, err := ExtractSuffix(tc.name) + c.Assert(err, IsNil) + c.Assert(suffix, Equals, tc.suffix) + } +} diff --git a/pkg/utils/db.go b/pkg/utils/db.go index 8c491108a3..25fb57921a 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -78,6 +78,33 @@ func GetAllServerID(ctx context.Context, db *sql.DB) (map[uint32]struct{}, error return serverIDs, nil } +// ReuseServerID reuse given server ID or get a new server ID +func ReuseServerID(ctx context.Context, serverID uint32, db *sql.DB) (uint32, error) { + serverIDs, err := GetAllServerID(ctx, db) + if err != nil { + return 0, err + } + + if _, ok := serverIDs[serverID]; !ok && serverID > 0 { + // reuse given server ID + return serverID, nil + } + + rand.Seed(time.Now().UnixNano()) + for i := 0; i < 99999; i++ { + randomValue := uint32(rand.Intn(100000)) + randomServerID := uint32(defaultBaseServerID) + randomValue + if _, ok := serverIDs[randomServerID]; ok { + continue + } + + return randomServerID, nil + } + + // should never happened unless the master has too many slave. + return 0, terror.ErrInvalidServerID.Generatef("can't find a random available server ID") +} + // GetRandomServerID gets a random server ID which is not used func GetRandomServerID(ctx context.Context, db *sql.DB) (uint32, error) { rand.Seed(time.Now().UnixNano()) diff --git a/relay/meta.go b/relay/meta.go index 200db0ca8b..08368fa8a0 100644 --- a/relay/meta.go +++ b/relay/meta.go @@ -58,7 +58,7 @@ type Meta interface { Dirty() bool // AddDir adds sub relay directory for server UUID (without suffix) - // if uuidSuffix is not zero value, add sub relay directory with uuidSuffix (bount to a new source) + // if uuidSuffix is not zero value, add sub relay directory with uuidSuffix (bound to a new source) // otherwise the added sub relay directory's suffix is incremented (master/slave switch) // after sub relay directory added, the internal binlog pos should be reset // and binlog pos will be set again when new binlog events received @@ -348,7 +348,10 @@ func (lm *LocalMeta) GTID() (string, gtid.Set) { lm.RLock() defer lm.RUnlock() - return lm.currentUUID, lm.gset + if lm.gset != nil { + return lm.currentUUID, lm.gset.Clone() + } + return lm.currentUUID, nil } // UUID implements Meta.UUID diff --git a/relay/relay.go b/relay/relay.go index e68cdba2f1..38d7b4edeb 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -386,14 +386,13 @@ func (r *Relay) tryRecoverLatestFile(ctx context.Context, parser2 *parser.Parser func (r *Relay) handleEvents(ctx context.Context, reader2 reader.Reader, transformer2 transformer.Transformer, writer2 writer.Writer) error { var ( _, lastPos = r.meta.Pos() - _, metaGTID = r.meta.GTID() - lastGTID gtid.Set + _, lastGTID = r.meta.GTID() err error ) - if metaGTID != nil { - lastGTID = metaGTID.Clone() - } else if lastGTID, err = gtid.ParserGTID(r.cfg.Flavor, ""); err != nil { - return err + if lastGTID == nil { + if lastGTID, err = gtid.ParserGTID(r.cfg.Flavor, ""); err != nil { + return err + } } for { @@ -645,12 +644,14 @@ func (r *Relay) doIntervalOps(ctx context.Context) { func (r *Relay) setUpReader(ctx context.Context) (reader.Reader, error) { ctx2, cancel := context.WithTimeout(ctx, utils.DefaultDBTimeout) defer cancel() - randomServerID, err := utils.GetRandomServerID(ctx2, r.db) + + randomServerID, err := utils.ReuseServerID(ctx2, r.cfg.ServerID, r.db) if err != nil { // should never happened unless the master has too many slave return nil, terror.Annotate(err, "fail to get random server id for relay reader") } r.syncerCfg.ServerID = randomServerID + r.cfg.ServerID = randomServerID uuid, pos := r.meta.Pos() _, gs := r.meta.GTID() @@ -773,9 +774,7 @@ func (r *Relay) Status(ctx context.Context) interface{} { } if r.cfg.EnableGTID { if masterGTID != nil && relayGTIDSet != nil { - // avoid data race - clone := relayGTIDSet.Clone() - if clone.Equal(masterGTID) { + if relayGTIDSet.Equal(masterGTID) { rs.RelayCatchUpMaster = true } } From c45ba2ba7cedd6c0dc127031a4fef17101119811 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 17 Dec 2020 15:31:10 +0800 Subject: [PATCH 49/59] unresume some errors --- dm/worker/task_checker.go | 6 +++++- pkg/retry/errors.go | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dm/worker/task_checker.go b/dm/worker/task_checker.go index fbacbfca09..69c150ea4d 100644 --- a/dm/worker/task_checker.go +++ b/dm/worker/task_checker.go @@ -285,7 +285,11 @@ func (tsc *realTaskStatusChecker) getRelayResumeStrategy(relayStatus *pb.RelaySt return ResumeIgnore } - // TODO: use different strategies based on the error detail + for _, err := range relayStatus.Result.Errors { + if _, ok := retry.UnresumableRelayErrCodes[err.ErrCode]; ok { + return ResumeNoSense + } + } if time.Since(tsc.bc.latestRelayResumeTime) < duration { return ResumeSkip diff --git a/pkg/retry/errors.go b/pkg/retry/errors.go index 2ea5027212..63e7eaa5f6 100644 --- a/pkg/retry/errors.go +++ b/pkg/retry/errors.go @@ -68,6 +68,15 @@ var ( int32(terror.ErrDumpUnitRuntime.Code()): {}, int32(terror.ErrSyncerUnitDMLColumnNotMatch.Code()): {}, } + + // UnresumableRelayErrCodes is a set of unresumeable relay unit err codes. + UnresumableRelayErrCodes = map[int32]struct{}{ + int32(terror.ErrRelayUUIDSuffixNotValid.Code()): {}, + int32(terror.ErrRelayUUIDSuffixLessThanPrev.Code()): {}, + int32(terror.ErrRelayBinlogNameNotValid.Code()): {}, + int32(terror.ErrRelayNoCurrentUUID.Code()): {}, + int32(terror.ErrRelayLogDirpathEmpty.Code()): {}, + } ) // IsConnectionError tells whether this error should reconnect to Database From 3543055bbb354cd1037cc825bb4be6f53f6d70bd Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 17 Dec 2020 16:16:29 +0800 Subject: [PATCH 50/59] fix fmt --- pkg/retry/errors.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/retry/errors.go b/pkg/retry/errors.go index 63e7eaa5f6..06cb3bd95d 100644 --- a/pkg/retry/errors.go +++ b/pkg/retry/errors.go @@ -71,11 +71,11 @@ var ( // UnresumableRelayErrCodes is a set of unresumeable relay unit err codes. UnresumableRelayErrCodes = map[int32]struct{}{ - int32(terror.ErrRelayUUIDSuffixNotValid.Code()): {}, + int32(terror.ErrRelayUUIDSuffixNotValid.Code()): {}, int32(terror.ErrRelayUUIDSuffixLessThanPrev.Code()): {}, - int32(terror.ErrRelayBinlogNameNotValid.Code()): {}, - int32(terror.ErrRelayNoCurrentUUID.Code()): {}, - int32(terror.ErrRelayLogDirpathEmpty.Code()): {}, + int32(terror.ErrRelayBinlogNameNotValid.Code()): {}, + int32(terror.ErrRelayNoCurrentUUID.Code()): {}, + int32(terror.ErrRelayLogDirpathEmpty.Code()): {}, } ) From ef4d21ff479836891ed5f567beba5095fdd8bbbd Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 21 Dec 2020 16:46:26 +0800 Subject: [PATCH 51/59] revert clone change --- relay/relay.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/relay/relay.go b/relay/relay.go index 38d7b4edeb..214c560edb 100755 --- a/relay/relay.go +++ b/relay/relay.go @@ -773,10 +773,8 @@ func (r *Relay) Status(ctx context.Context) interface{} { rs.RelayBinlogGtid = relayGTIDSet.String() } if r.cfg.EnableGTID { - if masterGTID != nil && relayGTIDSet != nil { - if relayGTIDSet.Equal(masterGTID) { - rs.RelayCatchUpMaster = true - } + if masterGTID != nil && relayGTIDSet != nil && relayGTIDSet.Equal(masterGTID) { + rs.RelayCatchUpMaster = true } } else { rs.RelayCatchUpMaster = masterPos.Compare(relayPos) == 0 From db85f8fe49151c4e971e151833b48f5dc73a9669 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 21 Dec 2020 18:28:09 +0800 Subject: [PATCH 52/59] revert change of integration test --- chaos/cases/conf/source1.yaml | 2 +- tests/all_mode/conf/source2.yaml | 2 +- tests/compatibility/conf/source1.yaml | 2 +- tests/compatibility/start.sh | 2 +- tests/dm_syncer/conf/source1.yaml | 2 +- tests/dmctl_basic/conf/get_source1.yaml | 2 +- tests/dmctl_basic/conf/source1.yaml | 2 +- tests/dmctl_command/conf/source1.yaml | 2 +- tests/full_mode/conf/source1.yaml | 2 +- tests/full_mode/run.sh | 2 +- tests/ha/conf/source1.yaml | 2 +- tests/ha/run.sh | 12 ++++++------ tests/ha_master/conf/source1.yaml | 2 +- tests/ha_master/run.sh | 4 ++-- tests/import_v10x/conf/source1.yaml | 2 +- tests/import_v10x/run.sh | 2 +- tests/incremental_mode/conf/source1.yaml | 2 +- tests/incremental_mode/run.sh | 2 +- tests/online_ddl/conf/source1.yaml | 2 +- tests/online_ddl/run.sh | 2 +- tests/safe_mode/conf/source2.yaml | 2 +- tests/sequence_safe_mode/conf/source1.yaml | 2 +- tests/sequence_sharding/conf/source2.yaml | 2 +- tests/sequence_sharding_optimistic/conf/source1.yaml | 2 +- tests/sequence_sharding_optimistic/run.sh | 4 ++-- tests/sequence_sharding_removemeta/conf/source1.yaml | 2 +- tests/sharding/conf/source1.yaml | 2 +- tests/sharding2/conf/source1.yaml | 2 +- tests/sharding2/run.sh | 2 +- tests/tiup/conf/source1.yaml | 2 +- 30 files changed, 37 insertions(+), 37 deletions(-) diff --git a/chaos/cases/conf/source1.yaml b/chaos/cases/conf/source1.yaml index f474f85131..796c553a90 100644 --- a/chaos/cases/conf/source1.yaml +++ b/chaos/cases/conf/source1.yaml @@ -1,6 +1,6 @@ source-id: "mysql-replica-01" enable-gtid: false -enable-relay: true +enable-relay: false from: host: "mysql-0.mysql" # same namespace with MySQL diff --git a/tests/all_mode/conf/source2.yaml b/tests/all_mode/conf/source2.yaml index 3836a3353f..32f44eb948 100644 --- a/tests/all_mode/conf/source2.yaml +++ b/tests/all_mode/conf/source2.yaml @@ -3,7 +3,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/compatibility/conf/source1.yaml b/tests/compatibility/conf/source1.yaml index 7d67feb8cf..2c7dc924ef 100644 --- a/tests/compatibility/conf/source1.yaml +++ b/tests/compatibility/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/compatibility/start.sh b/tests/compatibility/start.sh index d392debda9..53a1fe3466 100755 --- a/tests/compatibility/start.sh +++ b/tests/compatibility/start.sh @@ -68,7 +68,7 @@ function run() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 echo "use sync_diff_inspector to check data second time" check_sync_diff $WORK_DIR $cur/conf/diff_config.toml diff --git a/tests/dm_syncer/conf/source1.yaml b/tests/dm_syncer/conf/source1.yaml index 5a8554d591..00754d9626 100644 --- a/tests/dm_syncer/conf/source1.yaml +++ b/tests/dm_syncer/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/dmctl_basic/conf/get_source1.yaml b/tests/dmctl_basic/conf/get_source1.yaml index e059f6098c..7eba6be50d 100644 --- a/tests/dmctl_basic/conf/get_source1.yaml +++ b/tests/dmctl_basic/conf/get_source1.yaml @@ -4,7 +4,7 @@ relay-dir: /tmp/dm_test/dmctl_basic/worker1/relay_log meta-dir: "" flavor: mysql charset: "" -enable-relay: true +enable-relay: false relay-binlog-name: "" relay-binlog-gtid: "" source-id: mysql-replica-01 diff --git a/tests/dmctl_basic/conf/source1.yaml b/tests/dmctl_basic/conf/source1.yaml index 4bbcf0f019..6bab334388 100644 --- a/tests/dmctl_basic/conf/source1.yaml +++ b/tests/dmctl_basic/conf/source1.yaml @@ -3,7 +3,7 @@ server-id: 123456 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/dmctl_command/conf/source1.yaml b/tests/dmctl_command/conf/source1.yaml index a124c8fc21..6cb9c1ac65 100644 --- a/tests/dmctl_command/conf/source1.yaml +++ b/tests/dmctl_command/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/full_mode/conf/source1.yaml b/tests/full_mode/conf/source1.yaml index 9c9acf4f8a..901727c15b 100644 --- a/tests/full_mode/conf/source1.yaml +++ b/tests/full_mode/conf/source1.yaml @@ -2,7 +2,7 @@ server-id: 101 source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh index 5e714c5e72..1f5bcd12bc 100755 --- a/tests/full_mode/run.sh +++ b/tests/full_mode/run.sh @@ -52,7 +52,7 @@ function fail_acquire_global_lock() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Paused\"" 4 \ + "\"stage\": \"Paused\"" 3 \ "you need (at least one of) the RELOAD privilege(s) for this operation" 2 cleanup_data full_mode diff --git a/tests/ha/conf/source1.yaml b/tests/ha/conf/source1.yaml index c9d618cfdd..c2b659d3fb 100644 --- a/tests/ha/conf/source1.yaml +++ b/tests/ha/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/ha/run.sh b/tests/ha/run.sh index fcc6cb978c..647783c6da 100755 --- a/tests/ha/run.sh +++ b/tests/ha/run.sh @@ -91,22 +91,22 @@ function run() { echo "query-status from all dm-master" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 echo "join new dm-master and query-status" run_dm_master $WORK_DIR/master4 $MASTER_PORT4 $cur/conf/dm-master4.toml check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT4 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT4" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 # may join failed with error `fail to join embed etcd: add member http://127.0.0.1:8295: etcdserver: unhealthy cluster`, and dm-master will exit. so just sleep some seconds. sleep 5 @@ -115,14 +115,14 @@ function run() { check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT5 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT5" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 sleep 5 run_dm_master $WORK_DIR/master6 $MASTER_PORT6 $cur/conf/dm-master6.toml check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT6 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT6" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 sleep 5 echo "kill dm-master1" diff --git a/tests/ha_master/conf/source1.yaml b/tests/ha_master/conf/source1.yaml index 3c4efbef52..38c58c28cb 100644 --- a/tests/ha_master/conf/source1.yaml +++ b/tests/ha_master/conf/source1.yaml @@ -3,7 +3,7 @@ flavor: '' enable-gtid: true relay-binlog-name: '' relay-binlog-gtid: '' -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/ha_master/run.sh b/tests/ha_master/run.sh index 47168be979..cba066a069 100755 --- a/tests/ha_master/run.sh +++ b/tests/ha_master/run.sh @@ -327,7 +327,7 @@ function run() { check_http_alive 127.0.0.1:$MASTER_PORT3/apis/${API_VERSION}/status/test '"stage": "Running"' 10 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 @@ -351,7 +351,7 @@ function run() { # the last two masters should elect a new leader and serve service run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT4" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 # run master3 again run_dm_master $WORK_DIR/master3 $MASTER_PORT3 $cur/conf/dm-master3.toml diff --git a/tests/import_v10x/conf/source1.yaml b/tests/import_v10x/conf/source1.yaml index b221f0c3e3..4bb3139236 100644 --- a/tests/import_v10x/conf/source1.yaml +++ b/tests/import_v10x/conf/source1.yaml @@ -3,7 +3,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/import_v10x/run.sh b/tests/import_v10x/run.sh index e10247fe94..f5455337dc 100644 --- a/tests/import_v10x/run.sh +++ b/tests/import_v10x/run.sh @@ -71,7 +71,7 @@ function run() { # check task running. run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 # check task config, just a simple match run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ diff --git a/tests/incremental_mode/conf/source1.yaml b/tests/incremental_mode/conf/source1.yaml index 0018c1b61e..2bc3933f51 100644 --- a/tests/incremental_mode/conf/source1.yaml +++ b/tests/incremental_mode/conf/source1.yaml @@ -3,7 +3,7 @@ flavor: '' enable-gtid: true relay-binlog-name: '' relay-binlog-gtid: '' -enable-relay: true +enable-relay: false from: host: 127.0.0.1 user: root diff --git a/tests/incremental_mode/run.sh b/tests/incremental_mode/run.sh index 223d435c25..05dedd05e0 100755 --- a/tests/incremental_mode/run.sh +++ b/tests/incremental_mode/run.sh @@ -110,7 +110,7 @@ function run() { run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "Running" 4 + "Running" 3 # we use failpoint to let worker sleep 8 second when executeSQLs, to increase possibility of # meeting an error of context cancel. # when below check pass, it means we filter out that error, or that error doesn't happen. diff --git a/tests/online_ddl/conf/source1.yaml b/tests/online_ddl/conf/source1.yaml index fae9f5bd7e..49390fefa4 100644 --- a/tests/online_ddl/conf/source1.yaml +++ b/tests/online_ddl/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/online_ddl/run.sh b/tests/online_ddl/run.sh index 8c4d5f9e2b..59f4a42546 100755 --- a/tests/online_ddl/run.sh +++ b/tests/online_ddl/run.sh @@ -52,7 +52,7 @@ function real_run() { echo "wait and check task running" run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 run_sql_file $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 diff --git a/tests/safe_mode/conf/source2.yaml b/tests/safe_mode/conf/source2.yaml index 3df6bdf955..cc24bfedbc 100644 --- a/tests/safe_mode/conf/source2.yaml +++ b/tests/safe_mode/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_safe_mode/conf/source1.yaml b/tests/sequence_safe_mode/conf/source1.yaml index fae9f5bd7e..49390fefa4 100644 --- a/tests/sequence_safe_mode/conf/source1.yaml +++ b/tests/sequence_safe_mode/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding/conf/source2.yaml b/tests/sequence_sharding/conf/source2.yaml index 3df6bdf955..cc24bfedbc 100644 --- a/tests/sequence_sharding/conf/source2.yaml +++ b/tests/sequence_sharding/conf/source2.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-02 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding_optimistic/conf/source1.yaml b/tests/sequence_sharding_optimistic/conf/source1.yaml index 1ce2e303bc..bca70d80c0 100644 --- a/tests/sequence_sharding_optimistic/conf/source1.yaml +++ b/tests/sequence_sharding_optimistic/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: false -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sequence_sharding_optimistic/run.sh b/tests/sequence_sharding_optimistic/run.sh index f299084838..998d2b31b5 100755 --- a/tests/sequence_sharding_optimistic/run.sh +++ b/tests/sequence_sharding_optimistic/run.sh @@ -74,7 +74,7 @@ run() { "\"result\": true" 3 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $task_name" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 # now, for optimistic shard DDL, different sources will reach a stage often not at the same time, # in order to simply the check and resume flow, only enable the failpoint for one DM-worker. @@ -198,7 +198,7 @@ run() { "\"result\": true" 3 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status $task_name" \ - "\"stage\": \"Running\"" 4 + "\"stage\": \"Running\"" 3 # use sync_diff_inspector to check data now! check_sync_diff $WORK_DIR $cur/conf/diff_config.toml diff --git a/tests/sequence_sharding_removemeta/conf/source1.yaml b/tests/sequence_sharding_removemeta/conf/source1.yaml index fae9f5bd7e..49390fefa4 100644 --- a/tests/sequence_sharding_removemeta/conf/source1.yaml +++ b/tests/sequence_sharding_removemeta/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding/conf/source1.yaml b/tests/sharding/conf/source1.yaml index 78e96409ee..a3b176d0da 100644 --- a/tests/sharding/conf/source1.yaml +++ b/tests/sharding/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding2/conf/source1.yaml b/tests/sharding2/conf/source1.yaml index fae9f5bd7e..49390fefa4 100644 --- a/tests/sharding2/conf/source1.yaml +++ b/tests/sharding2/conf/source1.yaml @@ -1,7 +1,7 @@ source-id: mysql-replica-01 flavor: '' enable-gtid: true -enable-relay: true +enable-relay: false relay-binlog-name: '' relay-binlog-gtid: '' from: diff --git a/tests/sharding2/run.sh b/tests/sharding2/run.sh index a75817eb4f..f0b95343b5 100755 --- a/tests/sharding2/run.sh +++ b/tests/sharding2/run.sh @@ -81,7 +81,7 @@ function run() { run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "query-status test" \ - "\"stage\": \"Running\"" 4 \ + "\"stage\": \"Running\"" 3 \ "\"unit\": \"Sync\"" 2 } diff --git a/tests/tiup/conf/source1.yaml b/tests/tiup/conf/source1.yaml index 695d33f8a1..c49cc78ae2 100644 --- a/tests/tiup/conf/source1.yaml +++ b/tests/tiup/conf/source1.yaml @@ -1,6 +1,6 @@ source-id: mysql-replica-01 enable-gtid: false -enable-relay: true +enable-relay: false from: host: mysql1 user: root From 41ef863e0d00254ffd5e1bebf0096b12c7a6d141 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 21 Dec 2020 18:28:28 +0800 Subject: [PATCH 53/59] fix ci --- dm/worker/server.go | 2 ++ pkg/binlog/position.go | 7 ++++--- pkg/binlog/position_test.go | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dm/worker/server.go b/dm/worker/server.go index 909ddaad6f..c9153c63c7 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -560,6 +560,8 @@ func (s *Server) startWorker(cfg *config.SourceConfig) error { if err != nil { return err } + } else { + cfg.UUIDSuffix = binlog.MinUUIDSuffix } } diff --git a/pkg/binlog/position.go b/pkg/binlog/position.go index 0df9989dbc..6f302e1a59 100644 --- a/pkg/binlog/position.go +++ b/pkg/binlog/position.go @@ -36,7 +36,8 @@ const ( // eg. mysql-bin.000003 in c6ae5afe-c7a3-11e8-a19d-0242ac130006.000002 => mysql-bin|000002.000003 // where `000002` in `c6ae5afe-c7a3-11e8-a19d-0242ac130006.000002` is the UUIDSuffix posUUIDSuffixSeparator = "|" - minUUIDSuffix = 1 + // MinUUIDSuffix is same as relay.MinUUIDSuffix + MinUUIDSuffix = 1 ) var ( @@ -114,7 +115,7 @@ func RealMySQLPos(pos gmysql.Position) (gmysql.Position, error) { // ExtractSuffix extracts uuidSuffix from input name func ExtractSuffix(name string) (int, error) { if len(name) == 0 { - return minUUIDSuffix, nil + return MinUUIDSuffix, nil } filename, err := ParseFilename(name) if err != nil { @@ -126,7 +127,7 @@ func ExtractSuffix(name string) (int, error) { v, err := strconv.ParseInt(suffix, 10, 64) return int(v), err } - return minUUIDSuffix, nil + return MinUUIDSuffix, nil } // ExtractPos extracts (uuidWithSuffix, uuidSuffix, originalPos) from input pos (originalPos or convertedPos) diff --git a/pkg/binlog/position_test.go b/pkg/binlog/position_test.go index 197a3a6bf1..24d8526075 100644 --- a/pkg/binlog/position_test.go +++ b/pkg/binlog/position_test.go @@ -773,10 +773,10 @@ func (t *testPositionSuite) TestExtractSuffix(c *C) { }{ { "", - minUUIDSuffix, + MinUUIDSuffix, }, { "mysql-bin.00005", - minUUIDSuffix, + MinUUIDSuffix, }, { "mysql-bin|000001.000001", From 273949537c6ad279157a4547de0c3e6e89c2d922 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 21 Dec 2020 18:29:59 +0800 Subject: [PATCH 54/59] fix ci --- tests/dmctl_basic/check_list/get_config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dmctl_basic/check_list/get_config.sh b/tests/dmctl_basic/check_list/get_config.sh index c7f76d1eb5..e271b243dc 100644 --- a/tests/dmctl_basic/check_list/get_config.sh +++ b/tests/dmctl_basic/check_list/get_config.sh @@ -45,12 +45,12 @@ function diff_get_config() { "\"result\": true" 1 diff $dm_master_conf $cur/conf/get_master1.toml || exit 1 - run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "get-config worker worker1 --file $dm_worker1_conf" \ "\"result\": true" 1 diff $dm_worker1_conf $cur/conf/get_worker1.toml || exit 1 - run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ + run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "get-config worker worker2 --file $dm_worker2_conf" \ "\"result\": true" 1 diff $dm_worker2_conf $cur/conf/get_worker2.toml || exit 1 From 61fb37b4fca812ecf53d82d14cec84a169b9901d Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 21 Dec 2020 18:33:05 +0800 Subject: [PATCH 55/59] add comment --- dm/worker/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dm/worker/server.go b/dm/worker/server.go index c9153c63c7..f539ce1042 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -561,6 +561,8 @@ func (s *Server) startWorker(cfg *config.SourceConfig) error { return err } } else { + // set UUIDSuffix even not checkpoint exist + // so we will still remove relay dir cfg.UUIDSuffix = binlog.MinUUIDSuffix } } From 3ceabf8b2bb92b1b003721e14a59eeb45ca6de81 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 21 Dec 2020 18:49:29 +0800 Subject: [PATCH 56/59] fix ci --- tests/dmctl_basic/conf/source1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dmctl_basic/conf/source1.yaml b/tests/dmctl_basic/conf/source1.yaml index 6bab334388..4bbcf0f019 100644 --- a/tests/dmctl_basic/conf/source1.yaml +++ b/tests/dmctl_basic/conf/source1.yaml @@ -3,7 +3,7 @@ server-id: 123456 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' -enable-relay: false +enable-relay: true from: host: 127.0.0.1 user: root From c6bdfd3476f94a08aa9d57bd0e44092502b72d45 Mon Sep 17 00:00:00 2001 From: GMHDBJD <35025882+GMHDBJD@users.noreply.github.com> Date: Mon, 21 Dec 2020 19:41:24 +0800 Subject: [PATCH 57/59] fix ci --- tests/dmctl_basic/conf/get_source1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dmctl_basic/conf/get_source1.yaml b/tests/dmctl_basic/conf/get_source1.yaml index 7eba6be50d..e059f6098c 100644 --- a/tests/dmctl_basic/conf/get_source1.yaml +++ b/tests/dmctl_basic/conf/get_source1.yaml @@ -4,7 +4,7 @@ relay-dir: /tmp/dm_test/dmctl_basic/worker1/relay_log meta-dir: "" flavor: mysql charset: "" -enable-relay: false +enable-relay: true relay-binlog-name: "" relay-binlog-gtid: "" source-id: mysql-replica-01 From b6876f9e16746224fb10be53a3ba6d7974adf91a Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 22 Dec 2020 14:11:25 +0800 Subject: [PATCH 58/59] fix chaos --- chaos/cases/source.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chaos/cases/source.go b/chaos/cases/source.go index 64150d691f..053c4fc551 100644 --- a/chaos/cases/source.go +++ b/chaos/cases/source.go @@ -39,8 +39,8 @@ func createSources(ctx context.Context, cli pb.MasterClient, cfg *config) error return err } - var cfg1 config2.SourceConfig - var cfg2 config2.SourceConfig + cfg1 := config2.NewSourceConfig() + cfg2 := config2.NewSourceConfig() if err = cfg1.ParseYaml(string(s1Content)); err != nil { return err } From f9c1b1ad3ebafc4e6812ed933382538e18f82939 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 22 Dec 2020 14:26:01 +0800 Subject: [PATCH 59/59] increase diff count --- chaos/cases/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chaos/cases/task.go b/chaos/cases/task.go index 6a56cb8b77..3ff1d2800f 100644 --- a/chaos/cases/task.go +++ b/chaos/cases/task.go @@ -37,7 +37,7 @@ import ( const ( tableCount = 10 // tables count in schema. fullInsertCount = 100 // `INSERT INTO` count (not rows count) for each table in full stage. - diffCount = 20 // diff data check count + diffCount = 30 // diff data check count diffInterval = 10 * time.Second // diff data check interval incrRoundTime = 20 * time.Second // time to generate incremental data in one round )