From 5fa8c41320e3bd8f33f59f2d11bafee609d02e6e Mon Sep 17 00:00:00 2001 From: wjHuang Date: Fri, 12 May 2023 22:30:21 +0800 Subject: [PATCH 1/3] This is an automated cherry-pick of #43781 Signed-off-by: ti-chi-bot --- ddl/metadatalocktest/BUILD.bazel | 5 +++++ ddl/metadatalocktest/mdl_test.go | 16 ++++++++++++++++ ddl/mock.go | 5 +++++ domain/domain.go | 10 ++++++++++ 4 files changed, 36 insertions(+) diff --git a/ddl/metadatalocktest/BUILD.bazel b/ddl/metadatalocktest/BUILD.bazel index 7f1fc31f9987a..63c8833897539 100644 --- a/ddl/metadatalocktest/BUILD.bazel +++ b/ddl/metadatalocktest/BUILD.bazel @@ -7,6 +7,10 @@ go_test( "mdl_test.go", ], flaky = True, +<<<<<<< HEAD +======= + shard_count = 32, +>>>>>>> cde395dc597 (ddl, domain: fix a bug which causes MDL blocking (#43781)) deps = [ "//config", "//ddl", @@ -14,6 +18,7 @@ go_test( "//server", "//testkit", "//testkit/testsetup", + "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", ], diff --git a/ddl/metadatalocktest/mdl_test.go b/ddl/metadatalocktest/mdl_test.go index 64bdf77d55707..2732073251c07 100644 --- a/ddl/metadatalocktest/mdl_test.go +++ b/ddl/metadatalocktest/mdl_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" mysql "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/server" "github.com/pingcap/tidb/testkit" @@ -1118,3 +1119,18 @@ func TestMDLPrepareFail(t *testing.T) { tk2.MustExec("alter table test.t add column c int") } + +func TestMDLUpdateEtcdFail(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(a int);") + + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockUpdateMDLToETCDError", `3*return(true)`)) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockUpdateMDLToETCDError")) + }() + + tk.MustExec("alter table test.t add column c int") +} diff --git a/ddl/mock.go b/ddl/mock.go index 57a60794f514a..05d09b522bd99 100644 --- a/ddl/mock.go +++ b/ddl/mock.go @@ -66,6 +66,11 @@ func (s *MockSchemaSyncer) WatchGlobalSchemaVer(context.Context) {} // UpdateSelfVersion implements SchemaSyncer.UpdateSelfVersion interface. func (s *MockSchemaSyncer) UpdateSelfVersion(ctx context.Context, jobID int64, version int64) error { + failpoint.Inject("mockUpdateMDLToETCDError", func(val failpoint.Value) { + if val.(bool) { + failpoint.Return(errors.New("mock update mdl to etcd error")) + } + }) if variable.EnableMDL.Load() { s.mdlSchemaVersions.Store(jobID, version) } else { diff --git a/domain/domain.go b/domain/domain.go index 05133cd905cc1..a88bfe410c2cb 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -677,6 +677,7 @@ func (do *Domain) mdlCheckLoop() { if !variable.EnableMDL.Load() { continue } +<<<<<<< HEAD do.mdlCheckTableInfo.mu.Lock() maxVer := do.mdlCheckTableInfo.newestVer @@ -686,6 +687,15 @@ func (do *Domain) mdlCheckLoop() { // Schema doesn't change, and no job to check in the last run. do.mdlCheckTableInfo.mu.Unlock() continue +======= + logutil.BgLogger().Info("mdl gets lock, update to owner", zap.Int64("jobID", jobID), zap.Int64("version", ver)) + err := do.ddl.SchemaSyncer().UpdateSelfVersion(context.Background(), jobID, ver) + if err != nil { + jobNeedToSync = true + logutil.BgLogger().Warn("update self version failed", zap.Error(err)) + } else { + jobCache[jobID] = ver +>>>>>>> cde395dc597 (ddl, domain: fix a bug which causes MDL blocking (#43781)) } jobNeedToCheckCnt := len(do.mdlCheckTableInfo.jobsVerMap) From 9af45520d79f54f22cd52e2beb31228b0c5ac643 Mon Sep 17 00:00:00 2001 From: wjHuang Date: Mon, 15 May 2023 15:39:58 +0800 Subject: [PATCH 2/3] Update domain.go --- domain/domain.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/domain/domain.go b/domain/domain.go index a88bfe410c2cb..47d8225f1f1bb 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -677,7 +677,6 @@ func (do *Domain) mdlCheckLoop() { if !variable.EnableMDL.Load() { continue } -<<<<<<< HEAD do.mdlCheckTableInfo.mu.Lock() maxVer := do.mdlCheckTableInfo.newestVer @@ -687,15 +686,6 @@ func (do *Domain) mdlCheckLoop() { // Schema doesn't change, and no job to check in the last run. do.mdlCheckTableInfo.mu.Unlock() continue -======= - logutil.BgLogger().Info("mdl gets lock, update to owner", zap.Int64("jobID", jobID), zap.Int64("version", ver)) - err := do.ddl.SchemaSyncer().UpdateSelfVersion(context.Background(), jobID, ver) - if err != nil { - jobNeedToSync = true - logutil.BgLogger().Warn("update self version failed", zap.Error(err)) - } else { - jobCache[jobID] = ver ->>>>>>> cde395dc597 (ddl, domain: fix a bug which causes MDL blocking (#43781)) } jobNeedToCheckCnt := len(do.mdlCheckTableInfo.jobsVerMap) @@ -741,6 +731,7 @@ func (do *Domain) mdlCheckLoop() { logutil.BgLogger().Info("mdl gets lock, update to owner", zap.Int64("jobID", jobID), zap.Int64("version", ver)) err := do.ddl.SchemaSyncer().UpdateSelfVersion(context.Background(), jobID, ver) if err != nil { + jobNeedToSync = true logutil.BgLogger().Warn("update self version failed", zap.Error(err)) } else { jobCache[jobID] = ver From b39832e1613777e98d1323715b95eba1a1c11b2d Mon Sep 17 00:00:00 2001 From: wjHuang Date: Mon, 15 May 2023 15:40:26 +0800 Subject: [PATCH 3/3] Update BUILD.bazel --- ddl/metadatalocktest/BUILD.bazel | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ddl/metadatalocktest/BUILD.bazel b/ddl/metadatalocktest/BUILD.bazel index 63c8833897539..9679742c5121a 100644 --- a/ddl/metadatalocktest/BUILD.bazel +++ b/ddl/metadatalocktest/BUILD.bazel @@ -7,10 +7,6 @@ go_test( "mdl_test.go", ], flaky = True, -<<<<<<< HEAD -======= - shard_count = 32, ->>>>>>> cde395dc597 (ddl, domain: fix a bug which causes MDL blocking (#43781)) deps = [ "//config", "//ddl",