From d95d60d47eaf6bad6bac55aa712634e6c7a08f28 Mon Sep 17 00:00:00 2001 From: luancheng Date: Mon, 30 Mar 2020 18:00:18 +0800 Subject: [PATCH 1/4] restore: set max-index-length to max --- pkg/gluetidb/glue.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/gluetidb/glue.go b/pkg/gluetidb/glue.go index 5f4aff6fa..d8770dc9d 100644 --- a/pkg/gluetidb/glue.go +++ b/pkg/gluetidb/glue.go @@ -8,6 +8,7 @@ import ( "github.com/pingcap/parser/model" pd "github.com/pingcap/pd/v4/client" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" @@ -38,6 +39,10 @@ func (Glue) CreateSession(store kv.Storage) (glue.Session, error) { if err != nil { return nil, err } + // set max index length to DefaultMaxIndexLength + conf := config.GetGlobalConfig() + conf.MaxIndexLength = config.DefMaxOfMaxIndexLength + config.StoreGlobalConfig(conf) return &tidbSession{se: se}, nil } From fbf3de918051d2fee1df36be535fcce048d85ca6 Mon Sep 17 00:00:00 2001 From: luancheng Date: Mon, 30 Mar 2020 18:31:49 +0800 Subject: [PATCH 2/4] restore:add max-index-length params --- pkg/gluetidb/glue.go | 5 ----- pkg/task/restore.go | 23 +++++++++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/gluetidb/glue.go b/pkg/gluetidb/glue.go index d8770dc9d..5f4aff6fa 100644 --- a/pkg/gluetidb/glue.go +++ b/pkg/gluetidb/glue.go @@ -8,7 +8,6 @@ import ( "github.com/pingcap/parser/model" pd "github.com/pingcap/pd/v4/client" - "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" @@ -39,10 +38,6 @@ func (Glue) CreateSession(store kv.Storage) (glue.Session, error) { if err != nil { return nil, err } - // set max index length to DefaultMaxIndexLength - conf := config.GetGlobalConfig() - conf.MaxIndexLength = config.DefMaxOfMaxIndexLength - config.StoreGlobalConfig(conf) return &tidbSession{se: se}, nil } diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 336758dd7..1e753ee9a 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -10,6 +10,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/config" "github.com/spf13/pflag" "go.uber.org/zap" @@ -23,8 +24,9 @@ import ( ) const ( - flagOnline = "online" - flagNoSchema = "no-schema" + flagOnline = "online" + flagNoSchema = "no-schema" + flagMaxIndexLength = "max-index-length" ) var schedulers = map[string]struct{}{ @@ -46,8 +48,9 @@ const ( type RestoreConfig struct { Config - Online bool `json:"online" toml:"online"` - NoSchema bool `json:"no-schema" toml:"no-schema"` + Online bool `json:"online" toml:"online"` + NoSchema bool `json:"no-schema" toml:"no-schema"` + MaxIndexLength int `json:"max-index-length" toml:"max-index-length"` } // DefineRestoreFlags defines common flags for the restore command. @@ -55,9 +58,11 @@ func DefineRestoreFlags(flags *pflag.FlagSet) { // TODO remove experimental tag if it's stable flags.Bool(flagOnline, false, "(experimental) Whether online when restore") flags.Bool(flagNoSchema, false, "skip creating schemas and tables, reuse existing empty ones") + flags.Int(flagMaxIndexLength, 3072, "need keep this as same as tidb max-index-length") // Do not expose this flag _ = flags.MarkHidden(flagNoSchema) + _ = flags.MarkHidden(flagMaxIndexLength) } // ParseFromFlags parses the restore-related flags from the flag set. @@ -71,6 +76,10 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error { if err != nil { return errors.Trace(err) } + cfg.MaxIndexLength, err = flags.GetInt(flagMaxIndexLength) + if err != nil { + return errors.Trace(err) + } err = cfg.Config.ParseFromFlags(flags) if err != nil { return errors.Trace(err) @@ -148,6 +157,12 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf return err } // execute DDL first + + // set max index length before execute DDLs and create tables + conf := config.GetGlobalConfig() + conf.MaxIndexLength = cfg.MaxIndexLength + config.StoreGlobalConfig(conf) + err = client.ExecDDLs(ddlJobs) if err != nil { return errors.Trace(err) From 1a38cddd567a274cace71af9f04f87247d18647e Mon Sep 17 00:00:00 2001 From: luancheng Date: Mon, 30 Mar 2020 22:14:42 +0800 Subject: [PATCH 3/4] address comment --- pkg/task/restore.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 1e753ee9a..ad1e20a3d 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -24,9 +24,8 @@ import ( ) const ( - flagOnline = "online" - flagNoSchema = "no-schema" - flagMaxIndexLength = "max-index-length" + flagOnline = "online" + flagNoSchema = "no-schema" ) var schedulers = map[string]struct{}{ @@ -48,9 +47,8 @@ const ( type RestoreConfig struct { Config - Online bool `json:"online" toml:"online"` - NoSchema bool `json:"no-schema" toml:"no-schema"` - MaxIndexLength int `json:"max-index-length" toml:"max-index-length"` + Online bool `json:"online" toml:"online"` + NoSchema bool `json:"no-schema" toml:"no-schema"` } // DefineRestoreFlags defines common flags for the restore command. @@ -58,11 +56,9 @@ func DefineRestoreFlags(flags *pflag.FlagSet) { // TODO remove experimental tag if it's stable flags.Bool(flagOnline, false, "(experimental) Whether online when restore") flags.Bool(flagNoSchema, false, "skip creating schemas and tables, reuse existing empty ones") - flags.Int(flagMaxIndexLength, 3072, "need keep this as same as tidb max-index-length") // Do not expose this flag _ = flags.MarkHidden(flagNoSchema) - _ = flags.MarkHidden(flagMaxIndexLength) } // ParseFromFlags parses the restore-related flags from the flag set. @@ -76,10 +72,6 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error { if err != nil { return errors.Trace(err) } - cfg.MaxIndexLength, err = flags.GetInt(flagMaxIndexLength) - if err != nil { - return errors.Trace(err) - } err = cfg.Config.ParseFromFlags(flags) if err != nil { return errors.Trace(err) @@ -158,9 +150,11 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } // execute DDL first - // set max index length before execute DDLs and create tables + // set max-index-length before execute DDLs and create tables + // we set this value to max(3072*4), otherwise we might not restore table + // when upstream and downstream both set this value greater than default(3072) conf := config.GetGlobalConfig() - conf.MaxIndexLength = cfg.MaxIndexLength + conf.MaxIndexLength = config.DefMaxOfMaxIndexLength config.StoreGlobalConfig(conf) err = client.ExecDDLs(ddlJobs) From 465b31785b5343634a6aeb8d007605de049b4386 Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 1 Apr 2020 20:59:51 +0800 Subject: [PATCH 4/4] address comment --- pkg/task/restore.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/task/restore.go b/pkg/task/restore.go index ad1e20a3d..0af77132c 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -156,6 +156,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf conf := config.GetGlobalConfig() conf.MaxIndexLength = config.DefMaxOfMaxIndexLength config.StoreGlobalConfig(conf) + log.Warn("set max-index-length to max(3072*4) to skip check index length in DDL") err = client.ExecDDLs(ddlJobs) if err != nil {