From a3b02443d957444a4ea14f3f16f94dafe4a65eaf Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 13 May 2021 19:37:39 +0800 Subject: [PATCH] Support migration data from source with different time_zone by default (#1670) --- chaos/cases/conf/task-optimistic.yaml | 1 - chaos/cases/conf/task-pessimistic.yaml | 1 - chaos/cases/conf/task-single.yaml | 1 - chaos/cases/source.go | 12 ++--- cmd/dm-syncer/config.go | 22 +++++--- dm/config/source_config.go | 29 ++++++---- dm/config/source_config_test.go | 51 ++++++++++-------- dm/config/subtask.go | 21 ++++---- dm/config/subtask_test.go | 10 ---- dm/config/task.go | 53 +++++++++++------- dm/config/task_test.go | 28 ++++------ dm/dm-ansible/conf/task_advanced.yaml.example | 1 - dm/dm-ansible/scripts/dm_instances.json | 1 - dm/master/bootstrap.go | 10 ++-- dm/master/bootstrap_test.go | 23 ++++---- dm/master/scheduler/scheduler.go | 10 ++-- dm/master/scheduler/scheduler_test.go | 54 +++++++++---------- dm/master/server.go | 11 ++-- dm/master/server_test.go | 9 ++-- dm/master/task_advanced.yaml | 1 - dm/worker/relay_test.go | 2 +- dm/worker/server.go | 8 +-- dm/worker/server_test.go | 11 ++-- dm/worker/subtask.toml | 4 -- dm/worker/task_checker_test.go | 4 +- dm/worker/worker_test.go | 16 +++--- dumpling/dumpling.go | 5 ++ loader/loader.go | 1 + pkg/ha/bound.go | 4 +- pkg/ha/bound_test.go | 13 +++-- pkg/ha/ops_test.go | 4 +- pkg/ha/relay.go | 2 +- pkg/ha/relay_test.go | 6 +-- pkg/ha/source.go | 12 ++--- pkg/ha/source_test.go | 11 ++-- pkg/utils/common.go | 6 +-- syncer/db_test.go | 12 +---- syncer/syncer.go | 14 +---- tests/all_mode/conf/dm-task.yaml | 1 - tests/all_mode/data/db1.increment.sql | 9 ++-- tests/all_mode/data/db1.increment0.sql | 3 +- tests/all_mode/data/db1.prepare.sql | 9 +++- tests/all_mode/data/db2.increment.sql | 1 - tests/all_mode/data/db2.prepare.sql | 8 ++- tests/all_mode/run.sh | 4 +- tests/compatibility/conf/dm-task.yaml | 1 - tests/dm_syncer/conf/dm-syncer-1.toml | 4 -- tests/dm_syncer/conf/dm-syncer-2.toml | 4 -- tests/dm_syncer/conf/dm-task.yaml | 1 - tests/dmctl_basic/conf/dm-task.yaml | 1 - tests/dmctl_basic/conf/dm-task3.yaml | 1 - tests/dmctl_basic/conf/get_source1.yaml | 3 +- tests/dmctl_basic/conf/get_source2.yaml | 3 +- tests/dmctl_basic/conf/get_task.yaml | 3 +- tests/dmctl_command/conf/dm-task.yaml | 1 - .../drop_column_with_index/conf/dm-task.yaml | 1 - tests/full_mode/conf/dm-task.yaml | 1 - tests/full_mode/data/db1.prepare.sql | 8 ++- tests/full_mode/data/db2.prepare.sql | 7 ++- tests/full_mode/run.sh | 5 +- tests/ha/conf/dm-task.yaml | 1 - tests/ha_cases/conf/dm-task.yaml | 1 - tests/ha_cases/conf/dm-task2.yaml | 1 - tests/ha_cases/conf/standalone-task.yaml | 1 - tests/ha_cases/conf/standalone-task2.yaml | 1 - tests/ha_cases2/conf/dm-task.yaml | 1 - tests/ha_cases2/conf/dm-task2.yaml | 1 - tests/ha_cases2/conf/standalone-task.yaml | 1 - tests/ha_cases2/conf/standalone-task2.yaml | 1 - tests/ha_master/conf/dm-task.yaml | 1 - .../conf/double-source-no-sharding.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-no-sharding.yaml | 1 - tests/http_apis/conf/dm-task.yaml | 1 - tests/import_goroutine_leak/conf/dm-task.yaml | 1 - tests/import_v10x/conf/task.yaml | 3 +- tests/incremental_mode/conf/dm-task.yaml | 1 - tests/initial_unit/conf/dm-task.yaml | 1 - tests/load_interrupt/conf/dm-task.yaml | 1 - tests/many_tables/conf/dm-task.yaml | 1 - tests/new_relay/conf/dm-task.yaml | 1 - tests/online_ddl/conf/dm-task.yaml | 1 - tests/only_dml/conf/dm-task.yaml | 1 - tests/print_status/conf/dm-task.yaml | 1 - tests/relay_interrupt/conf/dm-task.yaml | 1 - tests/retry_cancel/conf/dm-task.yaml | 1 - tests/safe_mode/conf/dm-task.yaml | 1 - tests/safe_mode/run.sh | 2 +- tests/sequence_safe_mode/conf/dm-task.yaml | 1 - tests/sequence_sharding/conf/dm-task.yaml | 1 - .../conf/dm-task.yaml | 1 - .../conf/dm-task.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-no-routes.yaml | 1 - .../conf/single-source-no-sharding.yaml | 1 - .../conf/single-source-optimistic.yaml | 1 - .../conf/single-source-pessimistic.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-optimistic-2.yaml | 1 - .../conf/single-source-optimistic.yaml | 1 - .../conf/single-source-pessimistic-2.yaml | 1 - .../conf/single-source-pessimistic.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-optimistic-2.yaml | 1 - .../conf/single-source-optimistic.yaml | 1 - .../conf/single-source-pessimistic-2.yaml | 1 - .../conf/single-source-pessimistic.yaml | 1 - tests/sharding/conf/dm-task.yaml | 1 - tests/sharding2/conf/dm-task.yaml | 1 - tests/start_task/conf/dm-task.yaml | 1 - tests/tls/conf/dm-task.yaml | 1 - 117 files changed, 288 insertions(+), 335 deletions(-) diff --git a/chaos/cases/conf/task-optimistic.yaml b/chaos/cases/conf/task-optimistic.yaml index 9c77787274..84113731be 100644 --- a/chaos/cases/conf/task-optimistic.yaml +++ b/chaos/cases/conf/task-optimistic.yaml @@ -7,7 +7,6 @@ shard-mode: optimistic # so ignore sharding table check, seems has no effect on sync-diff-inspector # see https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-19.html ignore-checking-items: ["schema_of_shard_tables","auto_increment_ID"] # tables generated by go-sqlsmith may have auto increment ID -timezone: "UTC" target-database: host: "tidb-0.tidb" diff --git a/chaos/cases/conf/task-pessimistic.yaml b/chaos/cases/conf/task-pessimistic.yaml index 4fb63cfbdb..0af0c2b7a2 100644 --- a/chaos/cases/conf/task-pessimistic.yaml +++ b/chaos/cases/conf/task-pessimistic.yaml @@ -7,7 +7,6 @@ shard-mode: pessimistic # so ignore sharding table check, seems has no effect on sync-diff-inspector # see https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-19.html ignore-checking-items: ["schema_of_shard_tables","auto_increment_ID"] # tables generated by go-sqlsmith may have auto increment ID -timezone: "UTC" target-database: host: "tidb-0.tidb" diff --git a/chaos/cases/conf/task-single.yaml b/chaos/cases/conf/task-single.yaml index 31d91b60c7..e8b725b7d5 100644 --- a/chaos/cases/conf/task-single.yaml +++ b/chaos/cases/conf/task-single.yaml @@ -1,7 +1,6 @@ --- name: "task_single" task-mode: all -timezone: "UTC" target-database: host: "tidb-0.tidb" diff --git a/chaos/cases/source.go b/chaos/cases/source.go index 545be2bb01..46e2f11780 100644 --- a/chaos/cases/source.go +++ b/chaos/cases/source.go @@ -45,16 +45,16 @@ func createSources(ctx context.Context, cli pb.MasterClient, cfg *config) error return err } - cfg1 := config2.NewSourceConfig() - cfg2 := config2.NewSourceConfig() - cfg3 := config2.NewSourceConfig() - if err = cfg1.ParseYaml(string(s1Content)); err != nil { + cfg1, err := config2.ParseYaml(string(s1Content)) + if err != nil { return err } - if err = cfg2.ParseYaml(string(s2Content)); err != nil { + cfg2, err := config2.ParseYaml(string(s2Content)) + if err != nil { return err } - if err = cfg3.ParseYaml(string(s3Content)); err != nil { + cfg3, err := config2.ParseYaml(string(s3Content)) + if err != nil { return err } diff --git a/cmd/dm-syncer/config.go b/cmd/dm-syncer/config.go index a5780680ee..85a879bfe3 100644 --- a/cmd/dm-syncer/config.go +++ b/cmd/dm-syncer/config.go @@ -17,7 +17,6 @@ import ( "flag" "fmt" "os" - "time" "github.com/BurntSushi/toml" "github.com/go-mysql-org/go-mysql/mysql" @@ -27,6 +26,7 @@ import ( router "github.com/pingcap/tidb-tools/pkg/table-router" "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" ) @@ -54,6 +54,7 @@ type commonConfig struct { SafeMode bool MaxRetry int + // deprecated TimezoneStr string SyncerConfigFormat bool @@ -76,13 +77,13 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask EnableGTID: c.EnableGTID, SafeMode: c.SafeMode, MaxRetry: c.MaxRetry, - TimezoneStr: c.TimezoneStr, } cfg.FlagSet = flag.NewFlagSet("dm-syncer", flag.ContinueOnError) fs := cfg.FlagSet var SyncerConfigFormat bool + var timezoneStr string fs.BoolVar(&cfg.printVersion, "V", false, "prints version and exit") fs.StringVar(&cfg.Name, "name", "", "the task name") @@ -101,7 +102,7 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") - fs.StringVar(&cfg.TimezoneStr, "timezone", "", "target database timezone location string") + fs.StringVar(&timezoneStr, "timezone", "", "target database timezone location string") fs.BoolVar(&SyncerConfigFormat, "syncer-config-format", false, "read syncer config format") if err := fs.Parse(args); err != nil { @@ -119,6 +120,10 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask return nil, errors.Trace(err) } + if timezoneStr != "" { + log.L().Warn("'--timezone' is deprecated, needn't set it anymore.") + } + return cfg.convertToNewFormat() } @@ -146,6 +151,7 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e var syncerConfigFormat bool var printVersion bool var serverID uint + var timezoneStr string fs.BoolVar(&printVersion, "V", false, "prints version and exit") fs.StringVar(&cfg.Name, "name", "", "the task name") @@ -163,7 +169,7 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") - fs.StringVar(&cfg.Timezone, "timezone", "", "target database timezone location string") + fs.StringVar(&timezoneStr, "timezone", "", "target database timezone location string") fs.StringVar(&cfg.Name, "cp-table-prefix", "dm-syncer", "the prefix of the checkpoint table name") fs.BoolVar(&syncerConfigFormat, "syncer-config-format", false, "read syncer config format") @@ -172,6 +178,9 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e if err := cfg.Parse(args, false); err != nil { return nil, errors.Trace(err) } + if timezoneStr != "" { + log.L().Warn("'--timezone' is deprecated, needn't set it anymore.") + } if serverID != 101 { cfg.ServerID = uint32(serverID) @@ -202,7 +211,6 @@ func newCommonConfig() *commonConfig { fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") - fs.StringVar(&cfg.TimezoneStr, "timezone", "", "target database timezone location string") fs.BoolVar(&cfg.SyncerConfigFormat, "syncer-config-format", false, "read syncer config format") return cfg @@ -265,8 +273,7 @@ type syncerConfig struct { // MaxDMLConnectionTimeout string `toml:"execute-dml-timeout" json:"execute-dml-timeout"` // ExecutionQueueLength int `toml:"execute-queue-length" json:"execute-queue-length"` - TimezoneStr string `toml:"timezone" json:"timezone"` - Timezone *time.Location `json:"-"` + TimezoneStr string `toml:"timezone" json:"timezone"` printVersion bool } @@ -343,7 +350,6 @@ func (oc *syncerConfig) convertToNewFormat() (*config.SubTaskConfig, error) { }, ConfigFile: oc.ConfigFile, - Timezone: oc.TimezoneStr, From: oc.From, To: oc.To, } diff --git a/dm/config/source_config.go b/dm/config/source_config.go index d2ed9a2825..a4303d5c83 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -82,6 +82,13 @@ type SourceConfig struct { // NewSourceConfig creates a new base config for upstream MySQL/MariaDB source. func NewSourceConfig() *SourceConfig { + c := newSourceConfig() + c.adjust() + return c +} + +// NewSourceConfig creates a new base config without adjust. +func newSourceConfig() *SourceConfig { c := &SourceConfig{ Purge: PurgeConfig{ Interval: 60 * 60, @@ -94,7 +101,6 @@ func NewSourceConfig() *SourceConfig { BackoffMax: Duration{DefaultBackoffMax}, }, } - c.adjust() return c } @@ -140,12 +146,13 @@ func (c *SourceConfig) Parse(content string) error { } // ParseYaml parses flag definitions from the argument list, content should be yaml format. -func (c *SourceConfig) ParseYaml(content string) error { +func ParseYaml(content string) (*SourceConfig, error) { + c := newSourceConfig() if err := yaml.UnmarshalStrict([]byte(content), c); err != nil { - return terror.ErrConfigYamlTransform.Delegate(err, "decode source config") + return nil, terror.ErrConfigYamlTransform.Delegate(err, "decode source config") } c.adjust() - return nil + return c, nil } // EncodeToml encodes config. @@ -311,16 +318,20 @@ func (c *SourceConfig) AdjustServerID(ctx context.Context, db *sql.DB) error { } // LoadFromFile loads config from file. -func (c *SourceConfig) LoadFromFile(path string) error { +func LoadFromFile(path string) (*SourceConfig, error) { + c := newSourceConfig() content, err := ioutil.ReadFile(path) if err != nil { - return terror.ErrConfigReadCfgFromFile.Delegate(err, path) + return nil, terror.ErrConfigReadCfgFromFile.Delegate(err, path) } - if err := yaml.UnmarshalStrict(content, c); err != nil { - return terror.ErrConfigYamlTransform.Delegate(err, "decode source config") + if err = yaml.UnmarshalStrict(content, c); err != nil { + return nil, terror.ErrConfigYamlTransform.Delegate(err, "decode source config") } c.adjust() - return c.Verify() + if err = c.Verify(); err != nil { + return nil, err + } + return c, nil } func (c *SourceConfig) check(metaData *toml.MetaData, err error) error { diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index 793c0b18c0..3aaddb3653 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -32,9 +32,8 @@ import ( const sourceSampleFile = "../worker/source.yaml" func (t *testConfig) TestConfig(c *C) { - cfg := NewSourceConfig() - - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" c.Assert(cfg.RelayDir, Equals, "./xx") c.Assert(cfg.ServerID, Equals, uint32(101)) @@ -63,18 +62,24 @@ func (t *testConfig) TestConfig(c *C) { // test update config file and reload c.Assert(cfg.Parse(tomlStr), IsNil) c.Assert(cfg.ServerID, Equals, uint32(100)) - c.Assert(cfg.ParseYaml(yamlStr), IsNil) - c.Assert(cfg.ServerID, Equals, uint32(100)) - c.Assert(cfg.Parse(originCfgStr), IsNil) - c.Assert(cfg.ServerID, Equals, uint32(101)) - c.Assert(cfg.ParseYaml(originCfgYamlStr), IsNil) - c.Assert(cfg.ServerID, Equals, uint32(101)) + cfg1, err := ParseYaml(yamlStr) + c.Assert(err, IsNil) + c.Assert(cfg1.ServerID, Equals, uint32(100)) + cfg.Filters = []*bf.BinlogEventRule{} + cfg.Tracer = map[string]interface{}{} + + var cfg2 SourceConfig + c.Assert(cfg2.Parse(originCfgStr), IsNil) + c.Assert(cfg2.ServerID, Equals, uint32(101)) + + cfg3, err := ParseYaml(originCfgYamlStr) + c.Assert(err, IsNil) + c.Assert(cfg3.ServerID, Equals, uint32(101)) // test decrypt password clone1.From.Password = "1234" - clone1.ServerID = 101 // fix empty map after marshal/unmarshal becomes nil - clone1.From.Session = map[string]string{} + clone1.From.Session = cfg.From.Session clone1.Tracer = map[string]interface{}{} clone1.Filters = []*bf.BinlogEventRule{} clone2 := cfg.DecryptPassword() @@ -96,6 +101,7 @@ func (t *testConfig) TestConfig(c *C) { c.Assert(err, IsNil) c.Assert(clone4toml, Matches, "(.|\n)*backoff-rollback = \"5m(.|\n)*") c.Assert(clone4toml, Matches, "(.|\n)*backoff-max = \"5m(.|\n)*") + var clone5 SourceConfig c.Assert(clone5.Parse(clone4toml), IsNil) c.Assert(clone5, DeepEquals, *clone4) @@ -103,9 +109,10 @@ func (t *testConfig) TestConfig(c *C) { c.Assert(err, IsNil) c.Assert(clone4yaml, Matches, "(.|\n)*backoff-rollback: 5m(.|\n)*") c.Assert(clone4yaml, Matches, "(.|\n)*backoff-max: 5m(.|\n)*") - var clone6 SourceConfig - c.Assert(clone6.ParseYaml(clone4yaml), IsNil) - c.Assert(clone6, DeepEquals, *clone4) + + clone6, err := ParseYaml(clone4yaml) + c.Assert(err, IsNil) + c.Assert(clone6, DeepEquals, clone4) // test invalid config dir2 := c.MkDir() @@ -116,15 +123,15 @@ aaa: xxx `) err = ioutil.WriteFile(configFile, configContent, 0o644) c.Assert(err, IsNil) - err = cfg.LoadFromFile(configFile) + _, err = LoadFromFile(configFile) c.Assert(err, NotNil) c.Assert(err, ErrorMatches, "(.|\n)*field aaa not found in type config.SourceConfig(.|\n)*") } func (t *testConfig) TestConfigVerify(c *C) { newConfig := func() *SourceConfig { - cfg := NewSourceConfig() - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" return cfg } @@ -247,12 +254,12 @@ func subtestFlavor(c *C, cfg *SourceConfig, sqlInfo, expectedFlavor, expectedErr } func (t *testConfig) TestAdjustFlavor(c *C) { - cfg := NewSourceConfig() - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" cfg.Flavor = "mariadb" - err := cfg.AdjustFlavor(context.Background(), nil) + err = cfg.AdjustFlavor(context.Background(), nil) c.Assert(err, IsNil) c.Assert(cfg.Flavor, Equals, mysql.MariaDBFlavor) cfg.Flavor = "MongoDB" @@ -270,8 +277,8 @@ func (t *testConfig) TestAdjustServerID(c *C) { }() getAllServerIDFunc = getMockServerIDs - cfg := NewSourceConfig() - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" c.Assert(cfg.AdjustServerID(context.Background(), nil), IsNil) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index d534b83c85..f52219fb7d 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -19,11 +19,6 @@ import ( "flag" "fmt" "strings" - "time" - - "github.com/pingcap/dm/pkg/log" - "github.com/pingcap/dm/pkg/terror" - "github.com/pingcap/dm/pkg/utils" "github.com/BurntSushi/toml" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" @@ -31,6 +26,10 @@ import ( "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" "go.uber.org/zap" + + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/terror" + "github.com/pingcap/dm/pkg/utils" ) // task modes. @@ -118,6 +117,8 @@ func (db *DBConfig) Decode(data string) error { // Adjust adjusts the config. func (db *DBConfig) Adjust() { + // force set session time zone to UTC here. + AdjustTargetDBTimeZone(db) } // SubTaskConfig is the configuration for SubTask. @@ -150,8 +151,8 @@ type SubTaskConfig struct { HeartbeatReportInterval int `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` EnableHeartbeat bool `toml:"enable-heartbeat" json:"enable-heartbeat"` Meta *Meta `toml:"meta" json:"meta"` - Timezone string `toml:"timezone" josn:"timezone"` - + // deprecated + Timezone string `toml:"timezone" json:"timezone"` // RelayDir get value from dm-worker config RelayDir string `toml:"relay-dir" json:"relay-dir"` @@ -274,10 +275,8 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error { } if c.Timezone != "" { - _, err := time.LoadLocation(c.Timezone) - if err != nil { - return terror.ErrConfigInvalidTimezone.Delegate(err, c.Timezone) - } + log.L().Warn("'timezone' is deprecated, please remove this field.") + c.Timezone = "" } dirSuffix := "." + c.Name diff --git a/dm/config/subtask_test.go b/dm/config/subtask_test.go index 101d9b7512..b0241206fb 100644 --- a/dm/config/subtask_test.go +++ b/dm/config/subtask_test.go @@ -25,7 +25,6 @@ func (t *testConfig) TestSubTask(c *C) { ShardMode: "optimistic", SourceID: "mysql-instance-01", OnlineDDLScheme: "pt", - Timezone: "Asia/Shanghai", From: DBConfig{ Host: "127.0.0.1", Port: 3306, @@ -74,7 +73,6 @@ func (t *testConfig) TestSubTaskAdjustFail(c *C) { Name: "test-task", SourceID: "mysql-instance-01", OnlineDDLScheme: "pt", - Timezone: "Asia/Shanghai", From: DBConfig{ Host: "127.0.0.1", Port: 3306, @@ -133,14 +131,6 @@ func (t *testConfig) TestSubTaskAdjustFail(c *C) { }, "\\[.*\\], Message: online scheme rtc not supported.*", }, - { - func() *SubTaskConfig { - cfg := newSubTaskConfig() - cfg.Timezone = "my-house" - return cfg - }, - "\\[.*\\], Message: invalid timezone string: my-house.*", - }, } for _, tc := range testCases { diff --git a/dm/config/task.go b/dm/config/task.go index 99af958b70..e133cc04c4 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -20,20 +20,18 @@ import ( "io/ioutil" "sort" "strings" - "time" + "github.com/coreos/go-semver/semver" + "github.com/dustin/go-humanize" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/column-mapping" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "go.uber.org/zap" + "gopkg.in/yaml.v2" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" - - "github.com/coreos/go-semver/semver" - "github.com/dustin/go-humanize" - "go.uber.org/zap" - yaml "gopkg.in/yaml.v2" ) // Online DDL Scheme. @@ -71,6 +69,8 @@ var ( defaultBatch = 100 defaultQueueSize = 1024 // do not give too large default value to avoid OOM defaultCheckpointFlushInterval = 30 // in seconds + // force use UTC time_zone. + defaultTimeZone = "+00:00" // TargetDBConfig. defaultSessionCfg = []struct { @@ -282,10 +282,11 @@ type TaskConfig struct { // don't save configuration into it MetaSchema string `yaml:"meta-schema" toml:"meta-schema" json:"meta-schema"` - EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` - HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` - HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` - Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` + EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` + HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` + HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` + // deprecated + Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` // handle schema/table name mode, and only for schema/table name // if case insensitive, we would convert schema/table name to lower case @@ -593,14 +594,10 @@ func (c *TaskConfig) adjust() error { sort.Strings(unusedConfigs) return terror.ErrConfigGlobalConfigsUnused.Generate(unusedConfigs) } - if c.Timezone != "" { - _, err := time.LoadLocation(c.Timezone) - if err != nil { - return terror.ErrConfigInvalidTimezone.Delegate(err, c.Timezone) - } + log.L().Warn("`timezone` is deprecated and useless anymore, please remove it.") + c.Timezone = "" } - if c.RemoveMeta { log.L().Warn("`remove-meta` in task config is deprecated, please use `start-task ... --remove-meta` instead") } @@ -632,7 +629,6 @@ func (c *TaskConfig) SubTaskConfigs(sources map[string]DBConfig) ([]*SubTaskConf } cfg.HeartbeatUpdateInterval = c.HeartbeatUpdateInterval cfg.HeartbeatReportInterval = c.HeartbeatReportInterval - cfg.Timezone = c.Timezone cfg.Meta = inst.Meta cfg.From = dbCfg @@ -710,7 +706,6 @@ func FromSubTaskConfigs(stCfgs ...*SubTaskConfig) *TaskConfig { c.EnableHeartbeat = stCfg0.EnableHeartbeat c.HeartbeatUpdateInterval = stCfg0.HeartbeatUpdateInterval c.HeartbeatReportInterval = stCfg0.HeartbeatReportInterval - c.Timezone = stCfg0.Timezone c.CaseSensitive = stCfg0.CaseSensitive c.TargetDB = &stCfg0.To // just ref c.OnlineDDLScheme = stCfg0.OnlineDDLScheme @@ -815,5 +810,27 @@ func AdjustTargetDBSessionCfg(dbConfig *DBConfig, version *semver.Version) { lowerMap[cfg.key] = cfg.val } } + // force set time zone to UTC + if tz, ok := lowerMap["time_zone"]; ok { + log.L().Warn("session variable 'time_zone' is overwritten with UTC timezone.", + zap.String("time_zone", tz)) + } + lowerMap["time_zone"] = defaultTimeZone dbConfig.Session = lowerMap } + +// AdjustTargetDBTimeZone force adjust session `time_zone` to UTC. +func AdjustTargetDBTimeZone(config *DBConfig) { + for k := range config.Session { + if strings.ToLower(k) == "time_zone" { + log.L().Warn("session variable 'time_zone' is overwritten by default UTC timezone.", + zap.String("time_zone", config.Session[k])) + config.Session[k] = defaultTimeZone + return + } + } + if config.Session == nil { + config.Session = make(map[string]string, 1) + } + config.Session["time_zone"] = defaultTimeZone +} diff --git a/dm/config/task_test.go b/dm/config/task_test.go index 9fe4a4b6bb..ba09f6df46 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -35,7 +35,6 @@ name: test task-mode: all shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" case-sensitive: false online-ddl-scheme: "gh-ost" clean-dump-file: true @@ -137,7 +136,6 @@ name: test task-mode: all shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" case-sensitive: false online-ddl-scheme: "gh-ost" clean-dump-file: true @@ -244,8 +242,8 @@ name: test task-mode: all is-sharding: true meta-schema: "dm_meta" -enable-heartbeat: true timezone: "Asia/Shanghai" +enable-heartbeat: true ignore-checking-items: ["all"] target-database: @@ -308,7 +306,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] `) err = ioutil.WriteFile(filepath, configContent, 0o644) @@ -325,7 +322,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] `) err = ioutil.WriteFile(filepath, configContent, 0o644) @@ -340,7 +336,6 @@ name: test is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] `) err = ioutil.WriteFile(filepath, configContent, 0o644) @@ -358,7 +353,6 @@ meta-schema: "dm_meta" enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" @@ -436,7 +430,6 @@ meta-schema: "dm_meta" enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" @@ -543,13 +536,14 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { metaSchema = "meta-sub-tasks" heartbeatUI = 12 heartbeatRI = 21 - timezone = "Asia/Shanghai" maxAllowedPacket = 10244201 fromSession = map[string]string{ - "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "time_zone": "+00:00", } toSession = map[string]string{ - "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "time_zone": "+00:00", } security = Security{ SSLCA: "/path/to/ca", @@ -648,8 +642,7 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { BinLogPos: 456, BinLogGTID: "1-1-12,4-4-4", }, - Timezone: timezone, - From: source1DBCfg, + From: source1DBCfg, To: DBConfig{ Host: "127.0.0.1", Port: 4000, @@ -717,7 +710,6 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { EnableHeartbeat: stCfg1.EnableHeartbeat, HeartbeatUpdateInterval: heartbeatUI, HeartbeatReportInterval: heartbeatRI, - Timezone: timezone, CaseSensitive: stCfg1.CaseSensitive, TargetDB: &stCfg1.To, MySQLInstances: []*MySQLInstance{ @@ -889,22 +881,22 @@ func (t *testConfig) TestAdjustTargetDBConfig(c *C) { }{ { DBConfig{}, - DBConfig{Session: map[string]string{}}, + DBConfig{Session: map[string]string{"time_zone": "+00:00"}}, semver.New("0.0.0"), }, { DBConfig{Session: map[string]string{"SQL_MODE": "ANSI_QUOTES"}}, - DBConfig{Session: map[string]string{"sql_mode": "ANSI_QUOTES"}}, + DBConfig{Session: map[string]string{"sql_mode": "ANSI_QUOTES", "time_zone": "+00:00"}}, semver.New("2.0.7"), }, { DBConfig{}, - DBConfig{Session: map[string]string{tidbTxnMode: tidbTxnOptimistic}}, + DBConfig{Session: map[string]string{tidbTxnMode: tidbTxnOptimistic, "time_zone": "+00:00"}}, semver.New("3.0.1"), }, { DBConfig{Session: map[string]string{"SQL_MODE": "", tidbTxnMode: "pessimistic"}}, - DBConfig{Session: map[string]string{"sql_mode": "", tidbTxnMode: "pessimistic"}}, + DBConfig{Session: map[string]string{"sql_mode": "", tidbTxnMode: "pessimistic", "time_zone": "+00:00"}}, semver.New("4.0.0-beta.2"), }, } diff --git a/dm/dm-ansible/conf/task_advanced.yaml.example b/dm/dm-ansible/conf/task_advanced.yaml.example index 6b1fbe2c66..9b56179388 100644 --- a/dm/dm-ansible/conf/task_advanced.yaml.example +++ b/dm/dm-ansible/conf/task_advanced.yaml.example @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # meta schema in downstreaming database to store meta in enable-heartbeat: false # whether to enable heartbeat for calculating lag between master and syncer # heartbeat-update-interval: 1 # interval to do heartbeat and save timestamp, default 1s # heartbeat-report-interval: 10 # interval to report time lap to prometheus, default 10s -# timezone: "Asia/Shanghai" # target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone target-database: host: "192.168.0.1" diff --git a/dm/dm-ansible/scripts/dm_instances.json b/dm/dm-ansible/scripts/dm_instances.json index 8862183488..95c6fef83d 100644 --- a/dm/dm-ansible/scripts/dm_instances.json +++ b/dm/dm-ansible/scripts/dm_instances.json @@ -1504,7 +1504,6 @@ "30d" ] }, - "timezone": "", "title": "Test-Cluster-DM-worker-instances", "uid": "le1FDN_Wz", "version": 1 diff --git a/dm/master/bootstrap.go b/dm/master/bootstrap.go index 539fabd35d..81bb5ffc04 100644 --- a/dm/master/bootstrap.go +++ b/dm/master/bootstrap.go @@ -162,13 +162,13 @@ func (s *Server) importFromV10x(ctx context.Context) error { } // collectSourceConfigFilesV1Import tries to collect source config files for v1.0.x importing. -func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[string]config.SourceConfig, error) { +func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[string]*config.SourceConfig, error) { files, err := ioutil.ReadDir(s.cfg.V1SourcesPath) if err != nil { return nil, err } - cfgs := make(map[string]config.SourceConfig) + cfgs := make(map[string]*config.SourceConfig) for _, f := range files { if f.IsDir() { continue // ignore sub directories. @@ -186,7 +186,7 @@ func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[s return nil, err } - cfgs[cfgs2[0].SourceID] = *cfgs2[0] + cfgs[cfgs2[0].SourceID] = cfgs2[0] tctx.Logger.Info("collected source config", zap.Stringer("config", cfgs2[0])) } @@ -195,7 +195,7 @@ func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[s // waitWorkersReadyV1Import waits for DM-worker instances ready for v1.0.x importing. // NOTE: in v1.0.x, `count of DM-worker instances` equals `count of source config files`. -func (s *Server) waitWorkersReadyV1Import(tctx *tcontext.Context, sourceCfgs map[string]config.SourceConfig) error { +func (s *Server) waitWorkersReadyV1Import(tctx *tcontext.Context, sourceCfgs map[string]*config.SourceConfig) error { // now, we simply check count repeatedly. count := len(sourceCfgs) ctx2, cancel2 := context.WithTimeout(context.Background(), waitWorkerV1Timeout) @@ -291,7 +291,7 @@ func (s *Server) getSubtaskCfgsStagesV1Import(tctx *tcontext.Context) ( } // addSourcesV1Import tries to add source config into the cluster for v1.0.x importing. -func (s *Server) addSourcesV1Import(tctx *tcontext.Context, cfgs map[string]config.SourceConfig) error { +func (s *Server) addSourcesV1Import(tctx *tcontext.Context, cfgs map[string]*config.SourceConfig) error { var ( added []string err error diff --git a/dm/master/bootstrap_test.go b/dm/master/bootstrap_test.go index 66bf4b0b03..bf1797c4b8 100644 --- a/dm/master/bootstrap_test.go +++ b/dm/master/bootstrap_test.go @@ -68,13 +68,12 @@ func (t *testMaster) TestCollectSourceConfigFilesV1Import(c *C) { } password := os.Getenv("MYSQL_PSWD") - // load a valid source file. - cfg1 := config.NewSourceConfig() + cfg1, err := config.LoadFromFile("./source.yaml") + c.Assert(err, IsNil) // fix empty map after marshal/unmarshal becomes nil - cfg1.From.Session = map[string]string{} + cfg1.From.Adjust() cfg1.Tracer = map[string]interface{}{} cfg1.Filters = []*filter.BinlogEventRule{} - c.Assert(cfg1.LoadFromFile("./source.yaml"), IsNil) cfg1.From.Host = host cfg1.From.Port = port cfg1.From.User = user @@ -95,8 +94,8 @@ func (t *testMaster) TestCollectSourceConfigFilesV1Import(c *C) { cfgs, err = s.collectSourceConfigFilesV1Import(tctx) c.Assert(err, IsNil) c.Assert(cfgs, HasLen, 2) - c.Assert(cfgs[cfg1.SourceID], DeepEquals, *cfg1) - c.Assert(cfgs[cfg2.SourceID], DeepEquals, *cfg2) + c.Assert(cfgs[cfg1.SourceID], DeepEquals, cfg1) + c.Assert(cfgs[cfg2.SourceID], DeepEquals, cfg2) // put a invalid source file. c.Assert(ioutil.WriteFile(filepath.Join(s.cfg.V1SourcesPath, "invalid.yaml"), []byte("invalid-source-data"), 0o644), IsNil) @@ -122,17 +121,17 @@ func (t *testMaster) TestWaitWorkersReadyV1Import(c *C) { s.cfg.V1SourcesPath = c.MkDir() c.Assert(s.scheduler.Start(ctx, etcdTestCli), IsNil) - cfg1 := config.NewSourceConfig() - c.Assert(cfg1.LoadFromFile("./source.yaml"), IsNil) + cfg1, err := config.LoadFromFile("./source.yaml") + c.Assert(err, IsNil) cfg2 := cfg1.Clone() cfg2.SourceID = "mysql-replica-02" - cfgs := map[string]config.SourceConfig{ - cfg1.SourceID: *cfg1, - cfg2.SourceID: *cfg2, + cfgs := map[string]*config.SourceConfig{ + cfg1.SourceID: cfg1, + cfg2.SourceID: cfg2, } // no worker registered, timeout. - err := s.waitWorkersReadyV1Import(tctx, cfgs) + err = s.waitWorkersReadyV1Import(tctx, cfgs) c.Assert(err, ErrorMatches, ".*wait for DM-worker instances timeout.*") // register one worker. diff --git a/dm/master/scheduler/scheduler.go b/dm/master/scheduler/scheduler.go index 7c66cc4fca..a27ef81afd 100644 --- a/dm/master/scheduler/scheduler.go +++ b/dm/master/scheduler/scheduler.go @@ -76,7 +76,7 @@ type Scheduler struct { // - recover from etcd (calling `recoverSources`). // delete: // - remove source by user request (calling `RemoveSourceCfg`). - sourceCfgs map[string]config.SourceConfig + sourceCfgs map[string]*config.SourceConfig // all subtask configs, task name -> source ID -> subtask config. // add: @@ -151,7 +151,7 @@ type Scheduler struct { func NewScheduler(pLogger *log.Logger, securityCfg config.Security) *Scheduler { return &Scheduler{ logger: pLogger.WithFields(zap.String("component", "scheduler")), - sourceCfgs: make(map[string]config.SourceConfig), + sourceCfgs: make(map[string]*config.SourceConfig), subTaskCfgs: make(map[string]map[string]config.SubTaskConfig), workers: make(map[string]*Worker), bounds: make(map[string]*Worker), @@ -255,7 +255,7 @@ func (s *Scheduler) CloseAllWorkers() { // AddSourceCfg adds the upstream source config to the cluster. // NOTE: please verify the config before call this. -func (s *Scheduler) AddSourceCfg(cfg config.SourceConfig) error { +func (s *Scheduler) AddSourceCfg(cfg *config.SourceConfig) error { s.mu.Lock() defer s.mu.Unlock() @@ -380,7 +380,7 @@ func (s *Scheduler) GetSourceCfgByID(source string) *config.SourceConfig { if !ok { return nil } - clone := cfg + clone := *cfg return &clone } @@ -1745,7 +1745,7 @@ func (s *Scheduler) updateStatusForUnbound(source string) { // reset resets the internal status. func (s *Scheduler) reset() { - s.sourceCfgs = make(map[string]config.SourceConfig) + s.sourceCfgs = make(map[string]*config.SourceConfig) s.subTaskCfgs = make(map[string]map[string]config.SubTaskConfig) s.workers = make(map[string]*Worker) s.bounds = make(map[string]*Worker) diff --git a/dm/master/scheduler/scheduler_test.go b/dm/master/scheduler/scheduler_test.go index 75581797d4..6b818e5597 100644 --- a/dm/master/scheduler/scheduler_test.go +++ b/dm/master/scheduler/scheduler_test.go @@ -100,7 +100,6 @@ func (t *testScheduler) testSchedulerProgress(c *C, restart int) { taskName2 = "task-2" workerInfo1 = ha.NewWorkerInfo(workerName1, workerAddr1) workerInfo2 = ha.NewWorkerInfo(workerName2, workerAddr2) - sourceCfg1 config.SourceConfig subtaskCfg1 config.SubTaskConfig keepAliveTTL = int64(5) // NOTE: this should be >= minLeaseTTL, in second. @@ -116,10 +115,11 @@ func (t *testScheduler) testSchedulerProgress(c *C, restart int) { } } ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 sourceCfg1.EnableRelay = true - sourceCfg2 := sourceCfg1 + sourceCfg2 := *sourceCfg1 sourceCfg2.SourceID = sourceID2 c.Assert(subtaskCfg1.DecodeFile(subTaskSampleFile, true), IsNil) @@ -375,9 +375,9 @@ func (t *testScheduler) testSchedulerProgress(c *C, restart int) { // source2 not exists before. t.sourceCfgNotExist(c, s, sourceID2) // add source2. - c.Assert(s.AddSourceCfg(sourceCfg2), IsNil) + c.Assert(s.AddSourceCfg(&sourceCfg2), IsNil) // source2 added. - t.sourceCfgExist(c, s, sourceCfg2) + t.sourceCfgExist(c, s, &sourceCfg2) // source2 should bound to worker2. t.workerBound(c, s, ha.NewSourceBound(sourceID2, workerName2)) t.sourceBounds(c, s, []string{sourceID1, sourceID2}, []string{}) @@ -433,7 +433,7 @@ func (t *testScheduler) testSchedulerProgress(c *C, restart int) { }), IsTrue) c.Assert(terror.ErrSchedulerSourceOpTaskExist.Equal(s.RemoveSourceCfg(sourceID2)), IsTrue) // source2 keep there. - t.sourceCfgExist(c, s, sourceCfg2) + t.sourceCfgExist(c, s, &sourceCfg2) // source2 still bound to worker2. t.workerBound(c, s, ha.NewSourceBound(sourceID2, workerName2)) t.sourceBounds(c, s, []string{sourceID1, sourceID2}, []string{}) @@ -550,9 +550,9 @@ func (t *testScheduler) sourceCfgNotExist(c *C, s *Scheduler, source string) { c.Assert(scm, HasLen, 0) } -func (t *testScheduler) sourceCfgExist(c *C, s *Scheduler, expectCfg config.SourceConfig) { +func (t *testScheduler) sourceCfgExist(c *C, s *Scheduler, expectCfg *config.SourceConfig) { cfgP := s.GetSourceCfgByID(expectCfg.SourceID) - c.Assert(cfgP, DeepEquals, &expectCfg) + c.Assert(cfgP, DeepEquals, expectCfg) scm, _, err := ha.GetSourceCfg(etcdTestCli, expectCfg.SourceID, 0) c.Assert(err, IsNil) cfgV := scm[expectCfg.SourceID] @@ -708,11 +708,11 @@ func (t *testScheduler) TestRestartScheduler(c *C) { workerAddr1 = "127.0.0.1:8262" workerInfo1 = ha.NewWorkerInfo(workerName1, workerAddr1) sourceBound1 = ha.NewSourceBound(sourceID1, workerName1) - sourceCfg1 config.SourceConfig wg sync.WaitGroup keepAliveTTL = int64(2) // NOTE: this should be >= minLeaseTTL, in second. ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 s := NewScheduler(&logger, config.Security{}) @@ -822,12 +822,12 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { workerAddr2 = "127.0.0.1:18262" workerAddr3 = "127.0.0.1:18362" workerAddr4 = "127.0.0.1:18462" - sourceCfg1 config.SourceConfig keepAliveTTL = int64(2) // NOTE: this should be >= minLeaseTTL, in second. ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 - sourceCfg2 := sourceCfg1 + sourceCfg2 := *sourceCfg1 sourceCfg2.SourceID = sourceID2 ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -839,7 +839,7 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { // step 2: add two sources and register four workers c.Assert(s.AddSourceCfg(sourceCfg1), IsNil) - c.Assert(s.AddSourceCfg(sourceCfg2), IsNil) + c.Assert(s.AddSourceCfg(&sourceCfg2), IsNil) c.Assert(s.unbounds, HasLen, 2) c.Assert(s.unbounds, HasKey, sourceID1) c.Assert(s.unbounds, HasKey, sourceID2) @@ -868,8 +868,8 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { c.Assert(ha.KeepAlive(ctx1, etcdTestCli, workerName2, keepAliveTTL), IsNil) }() c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { - kam, _, err := ha.GetKeepAliveWorkers(etcdTestCli) - return err == nil && len(kam) == 2 + kam, _, e := ha.GetKeepAliveWorkers(etcdTestCli) + return e == nil && len(kam) == 2 }), IsTrue) cancel1() wg.Wait() @@ -877,14 +877,14 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { time.Sleep(time.Duration(keepAliveTTL) * time.Second) var rev int64 c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { - kam, rev1, err := ha.GetKeepAliveWorkers(etcdTestCli) + kam, rev1, e := ha.GetKeepAliveWorkers(etcdTestCli) rev = rev1 - return err == nil && len(kam) == 0 + return e == nil && len(kam) == 0 }), IsTrue) // step 4: trigger etcd compaction and check whether we can receive it through watcher var startRev int64 = 1 - _, err := etcdTestCli.Compact(ctx, rev) + _, err = etcdTestCli.Compact(ctx, rev) c.Assert(err, IsNil) workerEvCh := make(chan ha.WorkerEvent, 10) workerErrCh := make(chan error, 10) @@ -962,10 +962,10 @@ func (t *testScheduler) TestLastBound(c *C) { workerName2 = "dm-worker-2" workerName3 = "dm-worker-3" workerName4 = "dm-worker-4" - sourceCfg1 config.SourceConfig ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 sourceCfg2 := sourceCfg1 sourceCfg2.SourceID = sourceID2 @@ -1052,8 +1052,8 @@ func (t *testScheduler) TestTransferSource(c *C) { s.workers[workerName2] = worker2 s.workers[workerName3] = worker3 s.workers[workerName4] = worker4 - s.sourceCfgs[sourceID1] = config.SourceConfig{} - s.sourceCfgs[sourceID2] = config.SourceConfig{} + s.sourceCfgs[sourceID1] = &config.SourceConfig{} + s.sourceCfgs[sourceID2] = &config.SourceConfig{} worker1.ToFree() c.Assert(s.boundSourceToWorker(sourceID1, worker1), IsNil) @@ -1075,7 +1075,7 @@ func (t *testScheduler) TestTransferSource(c *C) { c.Assert(worker1.Stage(), Equals, WorkerFree) // test valid transfer: source -> worker = unbound -> free - s.sourceCfgs[sourceID3] = config.SourceConfig{} + s.sourceCfgs[sourceID3] = &config.SourceConfig{} s.unbounds[sourceID3] = struct{}{} c.Assert(s.TransferSource(sourceID3, workerName3), IsNil) c.Assert(s.bounds[sourceID3], DeepEquals, worker3) @@ -1095,7 +1095,7 @@ func (t *testScheduler) TestTransferSource(c *C) { c.Assert(s.bounds[sourceID1], DeepEquals, worker4) // test invalid transfer: source -> worker = unbound -> bound - s.sourceCfgs[sourceID4] = config.SourceConfig{} + s.sourceCfgs[sourceID4] = &config.SourceConfig{} s.unbounds[sourceID4] = struct{}{} c.Assert(s.TransferSource(sourceID4, workerName3), NotNil) c.Assert(s.bounds[sourceID3], DeepEquals, worker3) @@ -1147,8 +1147,8 @@ func (t *testScheduler) TestStartStopSource(c *C) { s.workers[workerName2] = worker2 s.workers[workerName3] = worker3 s.workers[workerName4] = worker4 - s.sourceCfgs[sourceID1] = config.SourceConfig{} - s.sourceCfgs[sourceID2] = config.SourceConfig{} + s.sourceCfgs[sourceID1] = &config.SourceConfig{} + s.sourceCfgs[sourceID2] = &config.SourceConfig{} worker1.ToFree() c.Assert(s.boundSourceToWorker(sourceID1, worker1), IsNil) diff --git a/dm/master/server.go b/dm/master/server.go index f158b79f8f..77f4b94948 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1090,8 +1090,8 @@ func (s *Server) CheckTask(ctx context.Context, req *pb.CheckTaskRequest) (*pb.C func parseAndAdjustSourceConfig(ctx context.Context, contents []string) ([]*config.SourceConfig, error) { cfgs := make([]*config.SourceConfig, len(contents)) for i, content := range contents { - cfg := config.NewSourceConfig() - if err := cfg.ParseYaml(content); err != nil { + cfg, err := config.ParseYaml(content) + if err != nil { return cfgs, err } @@ -1123,8 +1123,8 @@ func parseAndAdjustSourceConfig(ctx context.Context, contents []string) ([]*conf func parseSourceConfig(contents []string) ([]*config.SourceConfig, error) { cfgs := make([]*config.SourceConfig, len(contents)) for i, content := range contents { - cfg := config.NewSourceConfig() - if err := cfg.ParseYaml(content); err != nil { + cfg, err := config.ParseYaml(content) + if err != nil { return cfgs, err } cfgs[i] = cfg @@ -1155,6 +1155,7 @@ func adjustTargetDB(ctx context.Context, dbConfig *config.DBConfig) error { config.AdjustTargetDBSessionCfg(dbConfig, version) } else { log.L().Warn("get tidb version", log.ShortError(err)) + config.AdjustTargetDBTimeZone(dbConfig) } return nil } @@ -1201,7 +1202,7 @@ func (s *Server) OperateSource(ctx context.Context, req *pb.OperateSourceRequest err error ) for _, cfg := range cfgs { - err = s.scheduler.AddSourceCfg(*cfg) + err = s.scheduler.AddSourceCfg(cfg) // return first error and try to revert, so user could copy-paste same start command after error if err != nil { resp.Msg = err.Error() diff --git a/dm/master/server_test.go b/dm/master/server_test.go index 997b1a94d1..1dd27ddff7 100644 --- a/dm/master/server_test.go +++ b/dm/master/server_test.go @@ -72,7 +72,6 @@ is-sharding: true shard-mode: "" meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] target-database: @@ -276,7 +275,7 @@ func testMockScheduler(ctx context.Context, wg *sync.WaitGroup, c *check.C, sour cfg := config.NewSourceConfig() cfg.SourceID = sources[i] cfg.From.Password = password - c.Assert(scheduler2.AddSourceCfg(*cfg), check.IsNil, check.Commentf("all sources: %v", sources)) + c.Assert(scheduler2.AddSourceCfg(cfg), check.IsNil, check.Commentf("all sources: %v", sources)) wg.Add(1) ctx1, cancel1 := context.WithCancel(ctx) cancels = append(cancels, cancel1) @@ -308,7 +307,7 @@ func testMockSchedulerForRelay(ctx context.Context, wg *sync.WaitGroup, c *check cfg := config.NewSourceConfig() cfg.SourceID = sources[i] cfg.From.Password = password - c.Assert(scheduler2.AddSourceCfg(*cfg), check.IsNil, check.Commentf("all sources: %v", sources)) + c.Assert(scheduler2.AddSourceCfg(cfg), check.IsNil, check.Commentf("all sources: %v", sources)) wg.Add(1) ctx1, cancel1 := context.WithCancel(ctx) cancels = append(cancels, cancel1) @@ -1504,8 +1503,8 @@ func (t *testMaster) TestOperateSource(c *check.C) { s1.leader.Store(oneselfLeader) c.Assert(s1.Start(ctx), check.IsNil) defer s1.Close() - mysqlCfg := config.NewSourceConfig() - c.Assert(mysqlCfg.LoadFromFile("./source.yaml"), check.IsNil) + mysqlCfg, err := config.LoadFromFile("./source.yaml") + c.Assert(err, check.IsNil) mysqlCfg.From.Password = os.Getenv("MYSQL_PSWD") task, err := mysqlCfg.Yaml() c.Assert(err, check.IsNil) diff --git a/dm/master/task_advanced.yaml b/dm/master/task_advanced.yaml index 6b1fbe2c66..9b56179388 100644 --- a/dm/master/task_advanced.yaml +++ b/dm/master/task_advanced.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # meta schema in downstreaming database to store meta in enable-heartbeat: false # whether to enable heartbeat for calculating lag between master and syncer # heartbeat-update-interval: 1 # interval to do heartbeat and save timestamp, default 1s # heartbeat-report-interval: 10 # interval to report time lap to prometheus, default 10s -# timezone: "Asia/Shanghai" # target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone target-database: host: "192.168.0.1" diff --git a/dm/worker/relay_test.go b/dm/worker/relay_test.go index 4b31463567..4bbec4a6b9 100644 --- a/dm/worker/relay_test.go +++ b/dm/worker/relay_test.go @@ -145,7 +145,7 @@ func (t *testRelay) TestRelay(c *C) { cfg.RelayDir = dir cfg.MetaDir = dir - relayHolder := NewRealRelayHolder(&cfg) + relayHolder := NewRealRelayHolder(cfg) c.Assert(relayHolder, NotNil) holder, ok := relayHolder.(*realRelayHolder) diff --git a/dm/worker/server.go b/dm/worker/server.go index 3ceb65cbbc..3efd1fe58b 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -156,7 +156,7 @@ func (s *Server) Start() error { } if !bound.IsEmpty() { log.L().Warn("worker has been assigned source before keepalive", zap.Stringer("bound", bound), zap.Bool("is deleted", bound.IsDeleted)) - if err2 := s.enableHandleSubtasks(&sourceCfg, true); err2 != nil { + if err2 := s.enableHandleSubtasks(sourceCfg, true); err2 != nil { return err2 } log.L().Info("started to handle mysql source", zap.String("sourceCfg", sourceCfg.String())) @@ -452,7 +452,7 @@ func (s *Server) observeSourceBound(ctx context.Context, rev int64) error { } log.L().Info("will recover observeSourceBound", zap.String("relay source", cfg.SourceID)) - return s.enableHandleSubtasks(&cfg, false) + return s.enableHandleSubtasks(cfg, false) }() if err2 != nil { return err2 @@ -653,7 +653,7 @@ func (s *Server) operateSourceBound(bound ha.SourceBound) error { if !ok { return terror.ErrWorkerFailToGetSourceConfigFromEtcd.Generate(bound.Source) } - return s.enableHandleSubtasks(&sourceCfg, true) + return s.enableHandleSubtasks(sourceCfg, true) } func (s *Server) enableHandleSubtasks(sourceCfg *config.SourceConfig, needLock bool) error { @@ -704,7 +704,7 @@ func (s *Server) operateRelaySource(relaySource ha.RelaySource) error { if !ok { return terror.ErrWorkerFailToGetSourceConfigFromEtcd.Generate(relaySource.Source) } - return s.enableRelay(&sourceCfg, true) + return s.enableRelay(sourceCfg, true) } func (s *Server) enableRelay(sourceCfg *config.SourceConfig, needLock bool) error { diff --git a/dm/worker/server_test.go b/dm/worker/server_test.go index 1d078a3b88..ed9525bd5a 100644 --- a/dm/worker/server_test.go +++ b/dm/worker/server_test.go @@ -386,7 +386,7 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *C) { rev, err := ha.DeleteSourceBound(etcdCli, cfg.Name) c.Assert(err, IsNil) // step 2: start source at this worker - w, err := s.getOrStartWorker(&sourceCfg, true) + w, err := s.getOrStartWorker(sourceCfg, true) c.Assert(err, IsNil) c.Assert(w.EnableHandleSubtasks(), IsNil) // step 3: trigger etcd compaction and check whether we can receive it through watcher @@ -420,7 +420,7 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *C) { return s.getWorker(true) != nil }), IsTrue) cfg2 := s.getWorker(true).cfg - c.Assert(*cfg2, DeepEquals, sourceCfg) + c.Assert(cfg2, DeepEquals, sourceCfg) cancel1() wg.Wait() c.Assert(s.stopWorker(sourceCfg.SourceID, true), IsNil) @@ -435,7 +435,7 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *C) { return s.getWorker(true) != nil }), IsTrue) cfg2 = s.getWorker(true).cfg - c.Assert(*cfg2, DeepEquals, sourceCfg) + c.Assert(cfg2, DeepEquals, sourceCfg) cancel2() wg.Wait() } @@ -633,9 +633,8 @@ func checkRelayStatus(cli pb.WorkerClient, expect pb.Stage) bool { return status.SourceStatus.RelayStatus.Stage == expect } -func loadSourceConfigWithoutPassword(c *C) config.SourceConfig { - var sourceCfg config.SourceConfig - err := sourceCfg.LoadFromFile(sourceSampleFile) +func loadSourceConfigWithoutPassword(c *C) *config.SourceConfig { + sourceCfg, err := config.LoadFromFile(sourceSampleFile) c.Assert(err, IsNil) sourceCfg.From.Password = "" // no password set return sourceCfg diff --git a/dm/worker/subtask.toml b/dm/worker/subtask.toml index f6b7e6c802..b46f8f0aa4 100644 --- a/dm/worker/subtask.toml +++ b/dm/worker/subtask.toml @@ -53,10 +53,6 @@ meta-file = "./syncer.subTaskA.meta" worker-count = 16 batch = 1000 - -# target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone -# timezone = "Asia/Shanghai" - # filter # block allow list provides a library to filter replicate on schema/table by given rules diff --git a/dm/worker/task_checker_test.go b/dm/worker/task_checker_test.go index 466536e4e7..ac3db40e38 100644 --- a/dm/worker/task_checker_test.go +++ b/dm/worker/task_checker_test.go @@ -89,7 +89,7 @@ func (s *testTaskCheckerSuite) TestCheck(c *check.C) { cfg := loadSourceConfigWithoutPassword(c) cfg.RelayDir = dir cfg.MetaDir = dir - w, err := NewWorker(&cfg, nil, "") + w, err := NewWorker(cfg, nil, "") c.Assert(err, check.IsNil) w.closed.Store(false) @@ -204,7 +204,7 @@ func (s *testTaskCheckerSuite) TestCheckTaskIndependent(c *check.C) { cfg := loadSourceConfigWithoutPassword(c) cfg.RelayDir = dir cfg.MetaDir = dir - w, err := NewWorker(&cfg, nil, "") + w, err := NewWorker(cfg, nil, "") c.Assert(err, check.IsNil) w.closed.Store(false) diff --git a/dm/worker/worker_test.go b/dm/worker/worker_test.go index 878bba480c..0dc7aa9fdb 100644 --- a/dm/worker/worker_test.go +++ b/dm/worker/worker_test.go @@ -71,12 +71,12 @@ func (t *testServer) testWorker(c *C) { defer func() { NewRelayHolder = NewRealRelayHolder }() - w, err := NewWorker(&cfg, etcdCli, "") + w, err := NewWorker(cfg, etcdCli, "") c.Assert(err, IsNil) c.Assert(w.EnableRelay(), ErrorMatches, "init error") NewRelayHolder = NewDummyRelayHolder - w, err = NewWorker(&cfg, etcdCli, "") + w, err = NewWorker(cfg, etcdCli, "") c.Assert(err, IsNil) c.Assert(w.StatusJSON(""), HasLen, emptyWorkerStatusInfoJSONLength) @@ -175,7 +175,7 @@ func (t *testServer2) TestTaskAutoResume(c *C) { if s.closed.Load() { return false } - w, err2 := s.getOrStartWorker(&sourceConfig, true) + w, err2 := s.getOrStartWorker(sourceConfig, true) c.Assert(err2, IsNil) // we set sourceConfig.EnableRelay = true above c.Assert(w.EnableRelay(), IsNil) @@ -279,7 +279,7 @@ func (t *testWorkerFunctionalities) TestWorkerFunctionalities(c *C) { c.Assert(err, IsNil) // start worker - w, err := NewWorker(&sourceCfg, etcdCli, "") + w, err := NewWorker(sourceCfg, etcdCli, "") c.Assert(err, IsNil) defer w.Close() go func() { @@ -349,7 +349,7 @@ func (t *testWorkerFunctionalities) TestWorkerFunctionalities(c *C) { } func (t *testWorkerFunctionalities) testEnableRelay(c *C, w *Worker, etcdCli *clientv3.Client, - sourceCfg config.SourceConfig, cfg *Config) { + sourceCfg *config.SourceConfig, cfg *Config) { c.Assert(w.EnableRelay(), IsNil) c.Assert(w.relayEnabled.Load(), IsTrue) @@ -379,7 +379,7 @@ func (t *testWorkerFunctionalities) testDisableRelay(c *C, w *Worker) { } func (t *testWorkerFunctionalities) testEnableHandleSubtasks(c *C, w *Worker, etcdCli *clientv3.Client, - subtaskCfg config.SubTaskConfig, sourceCfg config.SourceConfig) { + subtaskCfg config.SubTaskConfig, sourceCfg *config.SourceConfig) { c.Assert(w.EnableHandleSubtasks(), IsNil) c.Assert(w.subTaskEnabled.Load(), IsTrue) @@ -469,7 +469,7 @@ func (t *testWorkerEtcdCompact) TestWatchSubtaskStageEtcdCompact(c *C) { sourceCfg.EnableRelay = false // step 1: start worker - w, err := NewWorker(&sourceCfg, etcdCli, "") + w, err := NewWorker(sourceCfg, etcdCli, "") c.Assert(err, IsNil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -585,7 +585,7 @@ func (t *testWorkerEtcdCompact) TestWatchRelayStageEtcdCompact(c *C) { sourceCfg.MetaDir = c.MkDir() // step 1: start worker - w, err := NewWorker(&sourceCfg, etcdCli, "") + w, err := NewWorker(sourceCfg, etcdCli, "") c.Assert(err, IsNil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/dumpling/dumpling.go b/dumpling/dumpling.go index b19a0f9b8d..975945f1eb 100644 --- a/dumpling/dumpling.go +++ b/dumpling/dumpling.go @@ -60,6 +60,7 @@ func (m *Dumpling) Init(ctx context.Context) error { var err error m.dumpConfig, err = m.constructArgs() m.detectSQLMode(ctx) + m.dumpConfig.SessionParams["time_zone"] = "+00:00" m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig)) return err } @@ -219,6 +220,10 @@ func (m *Dumpling) constructArgs() (*export.Config, error) { dumpConfig.TableFilter = tableFilter dumpConfig.CompleteInsert = true // always keep column name in `INSERT INTO` statements. dumpConfig.Logger = m.logger.Logger + // force using UTC timezone + dumpConfig.SessionParams = map[string]interface{}{ + "time_zone": "+00:00", + } if cfg.Threads > 0 { dumpConfig.Threads = cfg.Threads diff --git a/loader/loader.go b/loader/loader.go index 3fd2a3c270..da571e4d8f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -494,6 +494,7 @@ func (l *Loader) Init(ctx context.Context) (err error) { if lcfg.To.Session == nil { lcfg.To.Session = make(map[string]string) } + lcfg.To.Session["time_zone"] = "+00:00" hasSQLMode := false for k := range l.cfg.To.Session { diff --git a/pkg/ha/bound.go b/pkg/ha/bound.go index 1d7eb67516..1aff76700b 100644 --- a/pkg/ha/bound.go +++ b/pkg/ha/bound.go @@ -185,11 +185,11 @@ func GetLastSourceBounds(cli *clientv3.Client) (map[string]SourceBound, int64, e // if source bound is empty, will return an empty sourceBound and an empty source config // if source bound is not empty but sourceConfig is empty, will return an error // if the source bound is different for over retryNum times, will return an error. -func GetSourceBoundConfig(cli *clientv3.Client, worker string) (SourceBound, config.SourceConfig, int64, error) { +func GetSourceBoundConfig(cli *clientv3.Client, worker string) (SourceBound, *config.SourceConfig, int64, error) { var ( bound SourceBound newBound SourceBound - cfg config.SourceConfig + cfg *config.SourceConfig ok bool retryNum = defaultGetSourceBoundConfigRetry ) diff --git a/pkg/ha/bound_test.go b/pkg/ha/bound_test.go index fa95f6c0b7..5a40b81e4d 100644 --- a/pkg/ha/bound_test.go +++ b/pkg/ha/bound_test.go @@ -125,20 +125,19 @@ func (t *testForEtcd) TestGetSourceBoundConfigEtcd(c *C) { defer clearTestInfoOperation(c) var ( - worker = "dm-worker-1" - source = "mysql-replica-1" - bound = NewSourceBound(source, worker) - cfg config.SourceConfig - emptyCfg config.SourceConfig + worker = "dm-worker-1" + source = "mysql-replica-1" + bound = NewSourceBound(source, worker) ) - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.SourceID = source // no source bound and config bound1, cfg1, rev1, err := GetSourceBoundConfig(etcdTestCli, worker) c.Assert(err, IsNil) c.Assert(rev1, Greater, int64(0)) c.Assert(bound1.IsEmpty(), IsTrue) - c.Assert(cfg1, DeepEquals, emptyCfg) + c.Assert(cfg1, IsNil) rev2, err := PutSourceBound(etcdTestCli, bound) c.Assert(err, IsNil) diff --git a/pkg/ha/ops_test.go b/pkg/ha/ops_test.go index 3ad51caaff..80f129ae54 100644 --- a/pkg/ha/ops_test.go +++ b/pkg/ha/ops_test.go @@ -34,11 +34,11 @@ func (t *testForEtcd) TestOpsEtcd(c *C) { bound = NewSourceBound(source, worker) emptyStage Stage - sourceCfg config.SourceConfig subtaskCfg1 config.SubTaskConfig ) - c.Assert(sourceCfg.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg.SourceID = source c.Assert(subtaskCfg1.DecodeFile(subTaskSampleFile, true), IsNil) subtaskCfg1.SourceID = source diff --git a/pkg/ha/relay.go b/pkg/ha/relay.go index b75268aa3a..9b041573a6 100644 --- a/pkg/ha/relay.go +++ b/pkg/ha/relay.go @@ -173,7 +173,7 @@ func GetRelayConfig(cli *clientv3.Client, worker string) (*config.SourceConfig, return nil, 0, terror.ErrConfigMissingForBound.Generate(source) } - return &cfg, rev2, nil + return cfg, rev2, nil } return nil, 0, terror.ErrWorkerRelayConfigChanging.Generate(worker, source, newSource) } diff --git a/pkg/ha/relay_test.go b/pkg/ha/relay_test.go index 08d62aca46..bd357cde2b 100644 --- a/pkg/ha/relay_test.go +++ b/pkg/ha/relay_test.go @@ -25,9 +25,9 @@ func (t *testForEtcd) TestGetRelayConfigEtcd(c *C) { var ( worker = "dm-worker-1" source = "mysql-replica-1" - cfg config.SourceConfig ) - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.SourceID = source // no relay source and config cfg1, rev1, err := GetRelayConfig(etcdTestCli, worker) @@ -50,7 +50,7 @@ func (t *testForEtcd) TestGetRelayConfigEtcd(c *C) { cfg2, rev4, err := GetRelayConfig(etcdTestCli, worker) c.Assert(err, IsNil) c.Assert(rev4, Equals, rev3) - c.Assert(*cfg2, DeepEquals, cfg) + c.Assert(cfg2, DeepEquals, cfg) rev5, err := DeleteRelayConfig(etcdTestCli, worker) c.Assert(err, IsNil) diff --git a/pkg/ha/source.go b/pkg/ha/source.go index de0d7c661f..bb9cc49684 100644 --- a/pkg/ha/source.go +++ b/pkg/ha/source.go @@ -27,7 +27,7 @@ import ( // PutSourceCfg puts the config of the upstream source into etcd. // k/v: sourceID -> source config. -func PutSourceCfg(cli *clientv3.Client, cfg config.SourceConfig) (int64, error) { +func PutSourceCfg(cli *clientv3.Client, cfg *config.SourceConfig) (int64, error) { value, err := cfg.Toml() if err != nil { return 0, err @@ -42,12 +42,12 @@ func PutSourceCfg(cli *clientv3.Client, cfg config.SourceConfig) (int64, error) // if the source config for the sourceID not exist, return with `err == nil`. // if the source name is "", it will return all source configs as a map{sourceID: config}. // if the source name is given, it will return a map{sourceID: config} whose length is 1. -func GetSourceCfg(cli *clientv3.Client, source string, rev int64) (map[string]config.SourceConfig, int64, error) { +func GetSourceCfg(cli *clientv3.Client, source string, rev int64) (map[string]*config.SourceConfig, int64, error) { ctx, cancel := context.WithTimeout(cli.Ctx(), etcdutil.DefaultRequestTimeout) defer cancel() var ( - scm = make(map[string]config.SourceConfig) + scm = make(map[string]*config.SourceConfig) resp *clientv3.GetResponse err error ) @@ -77,8 +77,8 @@ func deleteSourceCfgOp(source string) clientv3.Op { return clientv3.OpDelete(common.UpstreamConfigKeyAdapter.Encode(source)) } -func sourceCfgFromResp(source string, resp *clientv3.GetResponse) (map[string]config.SourceConfig, error) { - scm := make(map[string]config.SourceConfig) +func sourceCfgFromResp(source string, resp *clientv3.GetResponse) (map[string]*config.SourceConfig, error) { + scm := make(map[string]*config.SourceConfig) if resp.Count == 0 { return scm, nil } else if source != "" && resp.Count > 1 { @@ -92,7 +92,7 @@ func sourceCfgFromResp(source string, resp *clientv3.GetResponse) (map[string]co if err != nil { return scm, terror.ErrConfigEtcdParse.Delegate(err, kv.Key) } - scm[cfg.SourceID] = cfg + scm[cfg.SourceID] = &cfg } return scm, nil } diff --git a/pkg/ha/source_test.go b/pkg/ha/source_test.go index af7568c1e3..42cc9944bf 100644 --- a/pkg/ha/source_test.go +++ b/pkg/ha/source_test.go @@ -52,11 +52,10 @@ var _ = Suite(&testForEtcd{}) func (t *testForEtcd) TestSourceEtcd(c *C) { defer clearTestInfoOperation(c) - var cfg config.SourceConfig - - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) source := cfg.SourceID - cfgExtra := cfg + cfgExtra := *cfg cfgExtra.SourceID = "mysql-replica-2" // no source config exist. @@ -81,7 +80,7 @@ func (t *testForEtcd) TestSourceEtcd(c *C) { c.Assert(cfg2, DeepEquals, cfg) // put another source config. - rev2, err = PutSourceCfg(etcdTestCli, cfgExtra) + rev2, err = PutSourceCfg(etcdTestCli, &cfgExtra) c.Assert(err, IsNil) // get all two config. @@ -90,7 +89,7 @@ func (t *testForEtcd) TestSourceEtcd(c *C) { c.Assert(rev3, Equals, rev2) c.Assert(cfgM, HasLen, 2) c.Assert(cfgM[source], DeepEquals, cfg) - c.Assert(cfgM[cfgExtra.SourceID], DeepEquals, cfgExtra) + c.Assert(cfgM[cfgExtra.SourceID], DeepEquals, &cfgExtra) // delete the config. deleteOp := deleteSourceCfgOp(source) diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 922fc40005..65ef2711f7 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -20,15 +20,15 @@ import ( "regexp" "strings" - "github.com/pingcap/dm/pkg/log" - "github.com/pingcap/dm/pkg/terror" - "github.com/pingcap/failpoint" tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" "go.uber.org/zap" + + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/terror" ) // TrimCtrlChars returns a slice of the string s with all leading diff --git a/syncer/db_test.go b/syncer/db_test.go index 104ede3b45..93af69963a 100644 --- a/syncer/db_test.go +++ b/syncer/db_test.go @@ -82,11 +82,7 @@ func (s *testDBSuite) resetBinlogSyncer(c *C) { UseDecimal: true, VerifyChecksum: true, } - if s.cfg.Timezone != "" { - timezone, err2 := time.LoadLocation(s.cfg.Timezone) - c.Assert(err2, IsNil) - cfg.TimestampStringLocation = timezone - } + cfg.TimestampStringLocation = time.UTC if s.syncer != nil { s.syncer.Close() @@ -236,7 +232,6 @@ func (s *testDBSuite) TestTimezone(c *C) { } for _, testCase := range testCases { - s.cfg.Timezone = testCase.timezone syncer := NewSyncer(s.cfg, nil) c.Assert(syncer.genRouter(), IsNil) s.resetBinlogSyncer(c) @@ -260,9 +255,6 @@ func (s *testDBSuite) TestTimezone(c *C) { err = txn.Commit() c.Assert(err, IsNil) - location, err := time.LoadLocation(testCase.timezone) - c.Assert(err, IsNil) - idx := 0 for { if idx >= len(testCase.sqls) { @@ -285,7 +277,7 @@ func (s *testDBSuite) TestTimezone(c *C) { c.Assert(ts.Valid, IsTrue) raw := ev.Rows[0][1].(string) - data, err := time.ParseInLocation("2006-01-02 15:04:05", raw, location) + data, err := time.ParseInLocation("2006-01-02 15:04:05", raw, time.UTC) c.Assert(err, IsNil) c.Assert(data.Unix(), DeepEquals, ts.Int64) idx++ diff --git a/syncer/syncer.go b/syncer/syncer.go index 989f50c294..1b7dc2ca51 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2716,7 +2716,6 @@ func (s *Syncer) Update(cfg *config.SubTaskConfig) error { s.cfg.RouteRules = cfg.RouteRules s.cfg.FilterRules = cfg.FilterRules s.cfg.ColumnMappingRules = cfg.ColumnMappingRules - s.cfg.Timezone = cfg.Timezone // update timezone s.setTimezone() @@ -2769,17 +2768,8 @@ func (s *Syncer) UpdateFromConfig(cfg *config.SubTaskConfig) error { } func (s *Syncer) setTimezone() { - var loc *time.Location - - if s.cfg.Timezone != "" { - loc, _ = time.LoadLocation(s.cfg.Timezone) - } - if loc == nil { - loc = time.Now().Location() - s.tctx.L().Warn("use system default time location") - } - s.tctx.L().Info("use timezone", log.WrapStringerField("location", loc)) - s.timezone = loc + s.tctx.L().Info("use timezone", log.WrapStringerField("location", time.UTC)) + s.timezone = time.UTC } func (s *Syncer) setSyncCfg() error { diff --git a/tests/all_mode/conf/dm-task.yaml b/tests/all_mode/conf/dm-task.yaml index c1de2ff48b..3d303f76a5 100644 --- a/tests/all_mode/conf/dm-task.yaml +++ b/tests/all_mode/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/all_mode/data/db1.increment.sql b/tests/all_mode/data/db1.increment.sql index a172eb5aa9..4509ea3a3a 100644 --- a/tests/all_mode/data/db1.increment.sql +++ b/tests/all_mode/data/db1.increment.sql @@ -6,10 +6,10 @@ update t1 set name = 'Catelyn Stark' where name = 'catelyn'; -- test multi column index with generated column alter table t1 add column info json; alter table t1 add column gen_id int as (info->'$.id'); -alter table t1 add index multi_col(`id`, `gen_id`); +alter table t1 add index multi_col(`id`, `gen_id`, ts); insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}'); insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}'); -update t1 set info = '{"id": 120}' where id = 1; +update t1 set info = '{"id": 120}', ts = '2021-05-11 12:02:05' where id = 1; update t1 set info = '{"id": 121}' where id = 2; update t1 set info = '{"id": 122}' where id = 3; @@ -20,7 +20,7 @@ alter table t1 add unique key gen_idx(`gen_id`); update t1 set name = 'gentestxx' where gen_id = 123; insert into t1 (id, name, info) values (7, 'gentest', '{"id": 126}'); -update t1 set name = 'gentestxxxxxx' where gen_id = 124; +update t1 set name = 'gentestxxxxxx', dt = '2021-05-11 12:03:05', ts = '2021-05-11 12:03:05' where gen_id = 124; -- delete with unique key delete from t1 where gen_id > 124; @@ -43,3 +43,6 @@ alter table t1 add column big2 bigint unsigned; insert into t1 (id, name, info, lat, big1, big2) values (11, 'gentest', '{"id":130}', '123.123', -9223372036854775808, 0); insert into t1 (id, name, info, lat, big1, big2) values (12, 'gentest', '{"id":131}', '123.123', 9223372036854775807, 18446744073709551615); +-- test with different session time_zone +SET @@session.time_zone = '+07:00'; +insert into t1 (id, name, info) values (13, 'tztest', '{"id": 132}'); diff --git a/tests/all_mode/data/db1.increment0.sql b/tests/all_mode/data/db1.increment0.sql index 99e094c474..fa406496ae 100644 --- a/tests/all_mode/data/db1.increment0.sql +++ b/tests/all_mode/data/db1.increment0.sql @@ -1,2 +1,3 @@ +SET @@SESSION.TIME_ZONE = "+03:00"; use all_mode; -insert into t1 (id, name) values (100, 'Eddard Stark'); +insert into t1 (id, name, ts) values (100, 'Eddard Stark', '2021-05-11 12:01:05'); diff --git a/tests/all_mode/data/db1.prepare.sql b/tests/all_mode/data/db1.prepare.sql index 631062d40e..eef8893e84 100644 --- a/tests/all_mode/data/db1.prepare.sql +++ b/tests/all_mode/data/db1.prepare.sql @@ -1,9 +1,14 @@ drop database if exists `all_mode`; create database `all_mode`; use `all_mode`; -create table t1 (id int NOT NULL AUTO_INCREMENT, name varchar(20), PRIMARY KEY (id)); +create table t1 ( + id int NOT NULL AUTO_INCREMENT, + name varchar(20), + dt datetime, + ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id)); -- test ANSI_QUOTES works with quote in string -insert into t1 (id, name) values (1, 'ar"ya'), (2, 'catelyn'); +insert into t1 (id, name, dt, ts) values (1, 'ar"ya', now(), now()), (2, 'catelyn', '2021-05-11 10:01:05', '2021-05-11 10:01:05'); -- test sql_mode=NO_AUTO_VALUE_ON_ZERO insert into t1 (id, name) values (0, 'lalala'); diff --git a/tests/all_mode/data/db2.increment.sql b/tests/all_mode/data/db2.increment.sql index 524f4cacd3..bce1755e76 100644 --- a/tests/all_mode/data/db2.increment.sql +++ b/tests/all_mode/data/db2.increment.sql @@ -1,6 +1,5 @@ use all_mode; delete from t2 where name = 'Sansa'; - -- test sql_mode=NO_AUTO_VALUE_ON_ZERO insert into t2 (id, name) values (0,'haha') \ No newline at end of file diff --git a/tests/all_mode/data/db2.prepare.sql b/tests/all_mode/data/db2.prepare.sql index c7688c5a7c..fc8900ae73 100644 --- a/tests/all_mode/data/db2.prepare.sql +++ b/tests/all_mode/data/db2.prepare.sql @@ -1,8 +1,12 @@ drop database if exists `all_mode`; create database `all_mode`; use `all_mode`; -create table t2 (id int auto_increment, name varchar(20), primary key (`id`)); -insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa'); +create table t2 ( + id int NOT NULL AUTO_INCREMENT, + name varchar(20), + ts timestamp, + PRIMARY KEY (id));; +insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'), ('Sansa', NULL); -- test block-allow-list drop database if exists `ignore_db`; diff --git a/tests/all_mode/run.sh b/tests/all_mode/run.sh index 05b497988f..07af11af3b 100755 --- a/tests/all_mode/run.sh +++ b/tests/all_mode/run.sh @@ -167,7 +167,8 @@ function test_stop_task_before_checkpoint() { function run() { run_sql_both_source "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES,NO_AUTO_VALUE_ON_ZERO'" - + run_sql_source1 "SET @@global.time_zone = '+01:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" test_session_config test_query_timeout @@ -345,6 +346,7 @@ function run() { export GO_FAILPOINTS='' run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" + run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" } cleanup_data all_mode diff --git a/tests/compatibility/conf/dm-task.yaml b/tests/compatibility/conf/dm-task.yaml index 17b0a9809c..5abe754c52 100644 --- a/tests/compatibility/conf/dm-task.yaml +++ b/tests/compatibility/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/dm_syncer/conf/dm-syncer-1.toml b/tests/dm_syncer/conf/dm-syncer-1.toml index 4826a33c03..2707f65d0d 100644 --- a/tests/dm_syncer/conf/dm-syncer-1.toml +++ b/tests/dm_syncer/conf/dm-syncer-1.toml @@ -52,10 +52,6 @@ binlog-pos = binlog-pos-placeholder-1 worker-count = 16 batch = 1000 - -# target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone -# timezone = "Asia/Shanghai" - # filter # block allow list provides a library to filter replicate on schema/table by given rules diff --git a/tests/dm_syncer/conf/dm-syncer-2.toml b/tests/dm_syncer/conf/dm-syncer-2.toml index b3ab30e2d7..825746bab6 100644 --- a/tests/dm_syncer/conf/dm-syncer-2.toml +++ b/tests/dm_syncer/conf/dm-syncer-2.toml @@ -12,10 +12,6 @@ flavor = "mysql" worker-count = 16 batch = 1000 - -# target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone -# timezone = "Asia/Shanghai" - # filter # block allow list provides a library to filter replicate on schema/table by given rules diff --git a/tests/dm_syncer/conf/dm-task.yaml b/tests/dm_syncer/conf/dm-task.yaml index 53bb403668..1a8d9e122a 100644 --- a/tests/dm_syncer/conf/dm-task.yaml +++ b/tests/dm_syncer/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" clean-dump-file: false target-database: diff --git a/tests/dmctl_basic/conf/dm-task.yaml b/tests/dmctl_basic/conf/dm-task.yaml index fe29e72e69..a048ca36b3 100644 --- a/tests/dmctl_basic/conf/dm-task.yaml +++ b/tests/dmctl_basic/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/dmctl_basic/conf/dm-task3.yaml b/tests/dmctl_basic/conf/dm-task3.yaml index 13a759e3f4..a36d82e80c 100644 --- a/tests/dmctl_basic/conf/dm-task3.yaml +++ b/tests/dmctl_basic/conf/dm-task3.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/dmctl_basic/conf/get_source1.yaml b/tests/dmctl_basic/conf/get_source1.yaml index 6e921fcf73..c73cc1ebb2 100644 --- a/tests/dmctl_basic/conf/get_source1.yaml +++ b/tests/dmctl_basic/conf/get_source1.yaml @@ -14,7 +14,8 @@ from: user: root password: '******' max-allowed-packet: null - session: {} + session: + time_zone: "+00:00" security: null purge: interval: 3600 diff --git a/tests/dmctl_basic/conf/get_source2.yaml b/tests/dmctl_basic/conf/get_source2.yaml index 2948f2f878..114d04acc6 100644 --- a/tests/dmctl_basic/conf/get_source2.yaml +++ b/tests/dmctl_basic/conf/get_source2.yaml @@ -14,7 +14,8 @@ from: user: root password: '******' max-allowed-packet: null - session: {} + session: + time_zone: "+00:00" security: null purge: interval: 3600 diff --git a/tests/dmctl_basic/conf/get_task.yaml b/tests/dmctl_basic/conf/get_task.yaml index 2c2c812fc0..8ce3b7dae3 100644 --- a/tests/dmctl_basic/conf/get_task.yaml +++ b/tests/dmctl_basic/conf/get_task.yaml @@ -7,7 +7,7 @@ meta-schema: dm_meta enable-heartbeat: false heartbeat-update-interval: 1 heartbeat-report-interval: 10 -timezone: Asia/Shanghai +timezone: "" case-sensitive: false target-database: host: 127.0.0.1 @@ -17,6 +17,7 @@ target-database: max-allowed-packet: null session: tidb_txn_mode: optimistic + time_zone: "+00:00" security: null mysql-instances: - source-id: mysql-replica-01 diff --git a/tests/dmctl_command/conf/dm-task.yaml b/tests/dmctl_command/conf/dm-task.yaml index 4c5473c5f5..21b6540b32 100644 --- a/tests/dmctl_command/conf/dm-task.yaml +++ b/tests/dmctl_command/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" remove-meta: true target-database: diff --git a/tests/drop_column_with_index/conf/dm-task.yaml b/tests/drop_column_with_index/conf/dm-task.yaml index e29507a35e..54ae455a51 100644 --- a/tests/drop_column_with_index/conf/dm-task.yaml +++ b/tests/drop_column_with_index/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/full_mode/conf/dm-task.yaml b/tests/full_mode/conf/dm-task.yaml index 84c3c528f5..75c21d4f8b 100644 --- a/tests/full_mode/conf/dm-task.yaml +++ b/tests/full_mode/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/full_mode/data/db1.prepare.sql b/tests/full_mode/data/db1.prepare.sql index 77592b929f..98ea937311 100644 --- a/tests/full_mode/data/db1.prepare.sql +++ b/tests/full_mode/data/db1.prepare.sql @@ -1,8 +1,12 @@ drop database if exists `full_mode`; create database `full_mode`; use `full_mode`; -create table t1 (id int, name varchar(20), primary key(`id`)); -insert into t1 (id, name) values (1, 'arya'), (2, 'catelyn'); +create table t1 ( + id int, + name varchar(20), + ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + primary key(`id`)); +insert into t1 (id, name, ts) values (1, 'arya', now()), (2, 'catelyn', '2021-05-11 10:01:03'); insert into t1 (id, name) values (3, 'Eddard Stark'); update t1 set name = 'Arya S\\\\tark' where id = 1; diff --git a/tests/full_mode/data/db2.prepare.sql b/tests/full_mode/data/db2.prepare.sql index 0eb75594ac..1f66dfc0fd 100644 --- a/tests/full_mode/data/db2.prepare.sql +++ b/tests/full_mode/data/db2.prepare.sql @@ -1,5 +1,8 @@ drop database if exists `full_mode`; create database `full_mode`; use `full_mode`; -create table t2 (id int auto_increment, name varchar(20), primary key (`id`)); -insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa'); +create table t2 (id int auto_increment, name varchar(20), dt datetime, ts timestamp, primary key (`id`)); +insert into t2 (name, dt, ts) values + ('Arya', now(), now()), + ('Bran', '2021-05-11 10:09:03', '2021-05-11 10:09:03'), + ('Sansa', NULL, NULL); diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh index 2e8558d6ac..2c0bc53ddf 100755 --- a/tests/full_mode/run.sh +++ b/tests/full_mode/run.sh @@ -42,7 +42,7 @@ function fail_acquire_global_lock() { "\"result\": true" 1 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml - sed -i '/timezone/i\ignore-checking-items: ["dump_privilege"]' $WORK_DIR/dm-task.yaml + sed -i '/heartbeat-report-interval/i\ignore-checking-items: ["dump_privilege"]' $WORK_DIR/dm-task.yaml run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "start-task $WORK_DIR/dm-task.yaml --remove-meta" @@ -147,6 +147,8 @@ function run() { empty_data run_sql_both_source "SET @@GLOBAL.SQL_MODE='NO_BACKSLASH_ESCAPES'" + run_sql_source1 "SET @@global.time_zone = '+01:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 check_contains 'Query OK, 2 rows affected' @@ -196,6 +198,7 @@ function run() { check_metric_not_contains $WORKER1_PORT 'dm_worker_task_state{source_id="mysql-replica-01",task="test"}' 3 check_metric_not_contains $WORKER2_PORT 'dm_worker_task_state{source_id="mysql-replica-02",task="test"}' 3 run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" + run_sql_both_source "SET @@GLOBAL.TIME_ZONE='SYSTEM';" } cleanup_data full_mode diff --git a/tests/ha/conf/dm-task.yaml b/tests/ha/conf/dm-task.yaml index e818fbf085..2610bdc6fa 100644 --- a/tests/ha/conf/dm-task.yaml +++ b/tests/ha/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/dm-task.yaml b/tests/ha_cases/conf/dm-task.yaml index e818fbf085..2610bdc6fa 100644 --- a/tests/ha_cases/conf/dm-task.yaml +++ b/tests/ha_cases/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/dm-task2.yaml b/tests/ha_cases/conf/dm-task2.yaml index dc233a8af1..ea23a367ff 100644 --- a/tests/ha_cases/conf/dm-task2.yaml +++ b/tests/ha_cases/conf/dm-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/standalone-task.yaml b/tests/ha_cases/conf/standalone-task.yaml index a201efcb43..be9c235e84 100644 --- a/tests/ha_cases/conf/standalone-task.yaml +++ b/tests/ha_cases/conf/standalone-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/standalone-task2.yaml b/tests/ha_cases/conf/standalone-task2.yaml index 4de5d77670..d51f51df00 100644 --- a/tests/ha_cases/conf/standalone-task2.yaml +++ b/tests/ha_cases/conf/standalone-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/dm-task.yaml b/tests/ha_cases2/conf/dm-task.yaml index e818fbf085..2610bdc6fa 100644 --- a/tests/ha_cases2/conf/dm-task.yaml +++ b/tests/ha_cases2/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/dm-task2.yaml b/tests/ha_cases2/conf/dm-task2.yaml index dc233a8af1..ea23a367ff 100644 --- a/tests/ha_cases2/conf/dm-task2.yaml +++ b/tests/ha_cases2/conf/dm-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/standalone-task.yaml b/tests/ha_cases2/conf/standalone-task.yaml index a201efcb43..be9c235e84 100644 --- a/tests/ha_cases2/conf/standalone-task.yaml +++ b/tests/ha_cases2/conf/standalone-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/standalone-task2.yaml b/tests/ha_cases2/conf/standalone-task2.yaml index 4de5d77670..d51f51df00 100644 --- a/tests/ha_cases2/conf/standalone-task2.yaml +++ b/tests/ha_cases2/conf/standalone-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_master/conf/dm-task.yaml b/tests/ha_master/conf/dm-task.yaml index 16093491f9..e4f597edfa 100644 --- a/tests/ha_master/conf/dm-task.yaml +++ b/tests/ha_master/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "localhost" diff --git a/tests/handle_error/conf/double-source-no-sharding.yaml b/tests/handle_error/conf/double-source-no-sharding.yaml index d2116075a0..c483994ec3 100644 --- a/tests/handle_error/conf/double-source-no-sharding.yaml +++ b/tests/handle_error/conf/double-source-no-sharding.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/handle_error/conf/double-source-optimistic.yaml b/tests/handle_error/conf/double-source-optimistic.yaml index d1b6dda25e..f92e748d31 100644 --- a/tests/handle_error/conf/double-source-optimistic.yaml +++ b/tests/handle_error/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/handle_error/conf/double-source-pessimistic.yaml b/tests/handle_error/conf/double-source-pessimistic.yaml index 739ac8581c..85875049de 100644 --- a/tests/handle_error/conf/double-source-pessimistic.yaml +++ b/tests/handle_error/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/handle_error/conf/single-source-no-sharding.yaml b/tests/handle_error/conf/single-source-no-sharding.yaml index cf75a92fe5..e3c6c18da9 100644 --- a/tests/handle_error/conf/single-source-no-sharding.yaml +++ b/tests/handle_error/conf/single-source-no-sharding.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/http_apis/conf/dm-task.yaml b/tests/http_apis/conf/dm-task.yaml index dfc6f8e108..4ffcab07cb 100644 --- a/tests/http_apis/conf/dm-task.yaml +++ b/tests/http_apis/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/import_goroutine_leak/conf/dm-task.yaml b/tests/import_goroutine_leak/conf/dm-task.yaml index 232e2b27ab..210bca31e2 100644 --- a/tests/import_goroutine_leak/conf/dm-task.yaml +++ b/tests/import_goroutine_leak/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: full is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/import_v10x/conf/task.yaml b/tests/import_v10x/conf/task.yaml index fddd0483cb..0524883740 100644 --- a/tests/import_v10x/conf/task.yaml +++ b/tests/import_v10x/conf/task.yaml @@ -7,7 +7,7 @@ meta-schema: dm_meta enable-heartbeat: false heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: Asia/Shanghai +timezone: "" case-sensitive: false target-database: host: 127.0.0.1 @@ -19,6 +19,7 @@ target-database: tidb_disable_txn_auto_retry: "off" tidb_retry_limit: "10" tidb_skip_utf8_check: "1" + time_zone: "+00:00" security: null mysql-instances: - source-id: mysql-replica-01 diff --git a/tests/incremental_mode/conf/dm-task.yaml b/tests/incremental_mode/conf/dm-task.yaml index d21aeb8000..a9464ea524 100644 --- a/tests/incremental_mode/conf/dm-task.yaml +++ b/tests/incremental_mode/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" clean-dump-file: false target-database: diff --git a/tests/initial_unit/conf/dm-task.yaml b/tests/initial_unit/conf/dm-task.yaml index f54d4e918a..5166d1f672 100644 --- a/tests/initial_unit/conf/dm-task.yaml +++ b/tests/initial_unit/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/load_interrupt/conf/dm-task.yaml b/tests/load_interrupt/conf/dm-task.yaml index 0c91d590ea..d09b9d0434 100644 --- a/tests/load_interrupt/conf/dm-task.yaml +++ b/tests/load_interrupt/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: full is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/many_tables/conf/dm-task.yaml b/tests/many_tables/conf/dm-task.yaml index 7e6fb62aa3..6637f03585 100644 --- a/tests/many_tables/conf/dm-task.yaml +++ b/tests/many_tables/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/new_relay/conf/dm-task.yaml b/tests/new_relay/conf/dm-task.yaml index 3abc7a87f8..b083e8f2d2 100644 --- a/tests/new_relay/conf/dm-task.yaml +++ b/tests/new_relay/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" clean-dump-file: false target-database: diff --git a/tests/online_ddl/conf/dm-task.yaml b/tests/online_ddl/conf/dm-task.yaml index 57decfc1cb..05be92be76 100644 --- a/tests/online_ddl/conf/dm-task.yaml +++ b/tests/online_ddl/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" online-ddl-scheme: online-ddl-scheme-placeholder target-database: diff --git a/tests/only_dml/conf/dm-task.yaml b/tests/only_dml/conf/dm-task.yaml index 14a890a045..ecf826dfb0 100644 --- a/tests/only_dml/conf/dm-task.yaml +++ b/tests/only_dml/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/print_status/conf/dm-task.yaml b/tests/print_status/conf/dm-task.yaml index d7c9e4077f..d5a483479f 100644 --- a/tests/print_status/conf/dm-task.yaml +++ b/tests/print_status/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/relay_interrupt/conf/dm-task.yaml b/tests/relay_interrupt/conf/dm-task.yaml index 25286d2653..b71ab8dace 100644 --- a/tests/relay_interrupt/conf/dm-task.yaml +++ b/tests/relay_interrupt/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/retry_cancel/conf/dm-task.yaml b/tests/retry_cancel/conf/dm-task.yaml index 1f8f3cc5c9..92d93803cf 100644 --- a/tests/retry_cancel/conf/dm-task.yaml +++ b/tests/retry_cancel/conf/dm-task.yaml @@ -2,7 +2,6 @@ name: test task-mode: all # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/safe_mode/conf/dm-task.yaml b/tests/safe_mode/conf/dm-task.yaml index 84d3de1161..a77684ff97 100644 --- a/tests/safe_mode/conf/dm-task.yaml +++ b/tests/safe_mode/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/safe_mode/run.sh b/tests/safe_mode/run.sh index 9de35a6ac7..16d1d49186 100755 --- a/tests/safe_mode/run.sh +++ b/tests/safe_mode/run.sh @@ -27,7 +27,7 @@ function consistency_none() { dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml - sed -i "/timezone/i\clean-dump-file: false" $WORK_DIR/dm-task.yaml + sed -i "/enable-heartbeat/i\clean-dump-file: false" $WORK_DIR/dm-task.yaml sed -i "s/extra-args: \"\"/extra-args: \"--consistency none\"/g" $WORK_DIR/dm-task.yaml dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta" check_sync_diff $WORK_DIR $cur/conf/diff_config.toml diff --git a/tests/sequence_safe_mode/conf/dm-task.yaml b/tests/sequence_safe_mode/conf/dm-task.yaml index 1ee51c3d72..7074d98535 100644 --- a/tests/sequence_safe_mode/conf/dm-task.yaml +++ b/tests/sequence_safe_mode/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sequence_sharding/conf/dm-task.yaml b/tests/sequence_sharding/conf/dm-task.yaml index 3203afcf6e..4dd936aac5 100644 --- a/tests/sequence_sharding/conf/dm-task.yaml +++ b/tests/sequence_sharding/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sequence_sharding_optimistic/conf/dm-task.yaml b/tests/sequence_sharding_optimistic/conf/dm-task.yaml index 44f4a9b815..3a8afdfe5d 100644 --- a/tests/sequence_sharding_optimistic/conf/dm-task.yaml +++ b/tests/sequence_sharding_optimistic/conf/dm-task.yaml @@ -5,7 +5,6 @@ is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sequence_sharding_removemeta/conf/dm-task.yaml b/tests/sequence_sharding_removemeta/conf/dm-task.yaml index b9d0f13617..1f5395c2c2 100644 --- a/tests/sequence_sharding_removemeta/conf/dm-task.yaml +++ b/tests/sequence_sharding_removemeta/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/double-source-optimistic.yaml b/tests/shardddl1/conf/double-source-optimistic.yaml index ffee908e20..4ea5f0aa25 100644 --- a/tests/shardddl1/conf/double-source-optimistic.yaml +++ b/tests/shardddl1/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/double-source-pessimistic.yaml b/tests/shardddl1/conf/double-source-pessimistic.yaml index fa62c11612..07f8c0c121 100644 --- a/tests/shardddl1/conf/double-source-pessimistic.yaml +++ b/tests/shardddl1/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-no-routes.yaml b/tests/shardddl1/conf/single-source-no-routes.yaml index 9fbb7c596f..ea36ff2058 100644 --- a/tests/shardddl1/conf/single-source-no-routes.yaml +++ b/tests/shardddl1/conf/single-source-no-routes.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-no-sharding.yaml b/tests/shardddl1/conf/single-source-no-sharding.yaml index b65cdb73fb..90987e1399 100644 --- a/tests/shardddl1/conf/single-source-no-sharding.yaml +++ b/tests/shardddl1/conf/single-source-no-sharding.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-optimistic.yaml b/tests/shardddl1/conf/single-source-optimistic.yaml index 2ddd12e9ca..bf8501bbf0 100644 --- a/tests/shardddl1/conf/single-source-optimistic.yaml +++ b/tests/shardddl1/conf/single-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-pessimistic.yaml b/tests/shardddl1/conf/single-source-pessimistic.yaml index 0e966f76d5..1884130992 100644 --- a/tests/shardddl1/conf/single-source-pessimistic.yaml +++ b/tests/shardddl1/conf/single-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl2/conf/double-source-optimistic.yaml b/tests/shardddl2/conf/double-source-optimistic.yaml index ffee908e20..4ea5f0aa25 100644 --- a/tests/shardddl2/conf/double-source-optimistic.yaml +++ b/tests/shardddl2/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl2/conf/double-source-pessimistic.yaml b/tests/shardddl2/conf/double-source-pessimistic.yaml index fa62c11612..07f8c0c121 100644 --- a/tests/shardddl2/conf/double-source-pessimistic.yaml +++ b/tests/shardddl2/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/double-source-optimistic.yaml b/tests/shardddl3/conf/double-source-optimistic.yaml index 871b1fc378..c87c6448c3 100644 --- a/tests/shardddl3/conf/double-source-optimistic.yaml +++ b/tests/shardddl3/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl3/conf/double-source-pessimistic.yaml b/tests/shardddl3/conf/double-source-pessimistic.yaml index 9821f1d649..db256a06ed 100644 --- a/tests/shardddl3/conf/double-source-pessimistic.yaml +++ b/tests/shardddl3/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl3/conf/single-source-optimistic-2.yaml b/tests/shardddl3/conf/single-source-optimistic-2.yaml index 831ed9bdad..2439cce00a 100644 --- a/tests/shardddl3/conf/single-source-optimistic-2.yaml +++ b/tests/shardddl3/conf/single-source-optimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/single-source-optimistic.yaml b/tests/shardddl3/conf/single-source-optimistic.yaml index 2ddd12e9ca..bf8501bbf0 100644 --- a/tests/shardddl3/conf/single-source-optimistic.yaml +++ b/tests/shardddl3/conf/single-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/single-source-pessimistic-2.yaml b/tests/shardddl3/conf/single-source-pessimistic-2.yaml index 7801725515..c468c59ca1 100644 --- a/tests/shardddl3/conf/single-source-pessimistic-2.yaml +++ b/tests/shardddl3/conf/single-source-pessimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/single-source-pessimistic.yaml b/tests/shardddl3/conf/single-source-pessimistic.yaml index 0e966f76d5..1884130992 100644 --- a/tests/shardddl3/conf/single-source-pessimistic.yaml +++ b/tests/shardddl3/conf/single-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/double-source-optimistic.yaml b/tests/shardddl4/conf/double-source-optimistic.yaml index 871b1fc378..c87c6448c3 100644 --- a/tests/shardddl4/conf/double-source-optimistic.yaml +++ b/tests/shardddl4/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl4/conf/double-source-pessimistic.yaml b/tests/shardddl4/conf/double-source-pessimistic.yaml index 9821f1d649..db256a06ed 100644 --- a/tests/shardddl4/conf/double-source-pessimistic.yaml +++ b/tests/shardddl4/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl4/conf/single-source-optimistic-2.yaml b/tests/shardddl4/conf/single-source-optimistic-2.yaml index 831ed9bdad..2439cce00a 100644 --- a/tests/shardddl4/conf/single-source-optimistic-2.yaml +++ b/tests/shardddl4/conf/single-source-optimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/single-source-optimistic.yaml b/tests/shardddl4/conf/single-source-optimistic.yaml index 2ddd12e9ca..bf8501bbf0 100644 --- a/tests/shardddl4/conf/single-source-optimistic.yaml +++ b/tests/shardddl4/conf/single-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/single-source-pessimistic-2.yaml b/tests/shardddl4/conf/single-source-pessimistic-2.yaml index 7801725515..c468c59ca1 100644 --- a/tests/shardddl4/conf/single-source-pessimistic-2.yaml +++ b/tests/shardddl4/conf/single-source-pessimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/single-source-pessimistic.yaml b/tests/shardddl4/conf/single-source-pessimistic.yaml index 0e966f76d5..1884130992 100644 --- a/tests/shardddl4/conf/single-source-pessimistic.yaml +++ b/tests/shardddl4/conf/single-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sharding/conf/dm-task.yaml b/tests/sharding/conf/dm-task.yaml index 7e72a97cf3..fa225c0785 100644 --- a/tests/sharding/conf/dm-task.yaml +++ b/tests/sharding/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sharding2/conf/dm-task.yaml b/tests/sharding2/conf/dm-task.yaml index 60b8b87b68..11a406fb9c 100644 --- a/tests/sharding2/conf/dm-task.yaml +++ b/tests/sharding2/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/start_task/conf/dm-task.yaml b/tests/start_task/conf/dm-task.yaml index be785ce8e9..1114b35704 100644 --- a/tests/start_task/conf/dm-task.yaml +++ b/tests/start_task/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/tls/conf/dm-task.yaml b/tests/tls/conf/dm-task.yaml index 89ffbf6b1c..990666658d 100644 --- a/tests/tls/conf/dm-task.yaml +++ b/tests/tls/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1"