ddl, domain: fix a bug which causes MDL blocking (#43781)

close #43755
pingcap · May 12, 2023 · cde395d · cde395d
1 parent 4d011d4
commit cde395d
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 1 deletion.
diff --git a/ddl/metadatalocktest/BUILD.bazel b/ddl/metadatalocktest/BUILD.bazel
@@ -8,14 +8,15 @@ go_test(
         "mdl_test.go",
     ],
     flaky = True,
-    shard_count = 31,
+    shard_count = 32,
     deps = [
         "//config",
         "//ddl",
         "//errno",
         "//server",
         "//testkit",
         "//testkit/testsetup",
+        "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",
     ],

diff --git a/ddl/metadatalocktest/mdl_test.go b/ddl/metadatalocktest/mdl_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pingcap/failpoint"
 	mysql "github.com/pingcap/tidb/errno"
 	"github.com/pingcap/tidb/server"
 	"github.com/pingcap/tidb/testkit"
@@ -1165,3 +1166,18 @@ func TestMDLPrepareFail(t *testing.T) {
 
 	tk2.MustExec("alter table test.t add column c int")
 }
+
+func TestMDLUpdateEtcdFail(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t(a int);")
+
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/mockUpdateMDLToETCDError", `3*return(true)`))
+	defer func() {
+		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/mockUpdateMDLToETCDError"))
+	}()
+
+	tk.MustExec("alter table test.t add column c int")
+}
diff --git a/ddl/mock.go b/ddl/mock.go
@@ -67,6 +67,11 @@ func (s *MockSchemaSyncer) WatchGlobalSchemaVer(context.Context) {}
 
 // UpdateSelfVersion implements SchemaSyncer.UpdateSelfVersion interface.
 func (s *MockSchemaSyncer) UpdateSelfVersion(ctx context.Context, jobID int64, version int64) error {
+	failpoint.Inject("mockUpdateMDLToETCDError", func(val failpoint.Value) {
+		if val.(bool) {
+			failpoint.Return(errors.New("mock update mdl to etcd error"))
+		}
+	})
 	if variable.EnableMDL.Load() {
 		s.mdlSchemaVersions.Store(jobID, version)
 	} else {

diff --git a/domain/domain.go b/domain/domain.go
@@ -833,6 +833,7 @@ func (do *Domain) mdlCheckLoop() {
 			logutil.BgLogger().Info("mdl gets lock, update to owner", zap.Int64("jobID", jobID), zap.Int64("version", ver))
 			err := do.ddl.SchemaSyncer().UpdateSelfVersion(context.Background(), jobID, ver)
 			if err != nil {
+				jobNeedToSync = true
 				logutil.BgLogger().Warn("update self version failed", zap.Error(err))
 			} else {
 				jobCache[jobID] = ver