diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel index 79a316d794fbb..0ca18ce55587b 100644 --- a/pkg/ddl/BUILD.bazel +++ b/pkg/ddl/BUILD.bazel @@ -155,7 +155,11 @@ go_library( "//pkg/util/sqlexec", "//pkg/util/sqlkiller", "//pkg/util/stringutil", +<<<<<<< HEAD "//pkg/util/syncutil", +======= + "//pkg/util/tiflash", +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)) "//pkg/util/timeutil", "//pkg/util/topsql", "//pkg/util/topsql/state", @@ -328,6 +332,7 @@ go_test( "@com_github_tikv_client_go_v2//util", "@io_etcd_go_etcd_client_v3//:client", "@org_golang_google_grpc//:grpc", + "@org_golang_x_sync//errgroup", "@org_uber_go_atomic//:atomic", "@org_uber_go_goleak//:goleak", "@org_uber_go_zap//:zap", diff --git a/pkg/ddl/ddl.go b/pkg/ddl/ddl.go index a34ef9aac5690..069bef60e0243 100644 --- a/pkg/ddl/ddl.go +++ b/pkg/ddl/ddl.go @@ -64,7 +64,11 @@ import ( "github.com/pingcap/tidb/pkg/util/dbterror" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/gcutil" +<<<<<<< HEAD "github.com/pingcap/tidb/pkg/util/syncutil" +======= + "github.com/pingcap/tidb/pkg/util/generic" +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)) "github.com/tikv/client-go/v2/tikvrpc" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" @@ -383,11 +387,6 @@ type ddlCtx struct { // reorgCtx is used for reorganization. reorgCtx reorgContexts - // backfillCtx is used for backfill workers. - backfillCtx struct { - syncutil.RWMutex - jobCtxMap map[int64]*JobContext - } jobCtx struct { sync.RWMutex diff --git a/pkg/ddl/ddl_api_test.go b/pkg/ddl/ddl_api_test.go index e849b43b3ff17..a24b8e54fc931 100644 --- a/pkg/ddl/ddl_api_test.go +++ b/pkg/ddl/ddl_api_test.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestGetDDLJobs(t *testing.T) { @@ -151,6 +152,46 @@ func enQueueDDLJobs(t *testing.T, sess sessiontypes.Session, txn kv.Transaction, } } +func TestCreateViewConcurrently(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + tk.MustExec("create table t (a int);") + tk.MustExec("create view v as select * from t;") + var ( + counterErr error + counter int + ) + failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/onDDLCreateView", func(job *model.Job) { + counter++ + if counter > 1 { + counterErr = fmt.Errorf("create view job should not run concurrently") + return + } + }) + failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/afterDelivery2Worker", func(job *model.Job) { + if job.Type == model.ActionCreateView { + counter-- + } + }) + var eg errgroup.Group + for i := 0; i < 5; i++ { + eg.Go(func() error { + newTk := testkit.NewTestKit(t, store) + _, err := newTk.Exec("use test") + if err != nil { + return err + } + _, err = newTk.Exec("create or replace view v as select * from t;") + return err + }) + } + err := eg.Wait() + require.NoError(t, err) + require.NoError(t, counterErr) +} + func TestCreateDropCreateTable(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) diff --git a/pkg/ddl/job_table.go b/pkg/ddl/job_table.go index aa2ebc3d516d0..5467ca04c316b 100644 --- a/pkg/ddl/job_table.go +++ b/pkg/ddl/job_table.go @@ -421,8 +421,14 @@ func (d *ddl) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) { d.wg.Run(func() { metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc() defer func() { +<<<<<<< HEAD d.runningJobs.remove(job) asyncNotify(d.ddlJobCh) +======= + failpoint.InjectCall("afterDelivery2Worker", job) + s.runningJobs.remove(job) + asyncNotify(s.ddlJobNotifyCh) +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)) metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec() }() ownerID := d.ownerManager.ID() diff --git a/pkg/ddl/placement_policy.go b/pkg/ddl/placement_policy.go index 4ab6495754f12..44e01b3fecd13 100644 --- a/pkg/ddl/placement_policy.go +++ b/pkg/ddl/placement_policy.go @@ -119,7 +119,7 @@ func getPlacementPolicyByName(d *ddlCtx, t *meta.Meta, policyName model.CIStr) ( } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { // Use cached policy. policy, ok := is.PolicyByName(policyName) if ok { @@ -320,7 +320,7 @@ func checkPlacementPolicyNotInUse(d *ddlCtx, t *meta.Meta, policy *model.PolicyI return err } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { err = CheckPlacementPolicyNotInUseFromInfoSchema(is, policy) } else { err = CheckPlacementPolicyNotInUseFromMeta(t, policy) diff --git a/pkg/ddl/schema.go b/pkg/ddl/schema.go index 3be32a14a26de..412169b08e11f 100644 --- a/pkg/ddl/schema.go +++ b/pkg/ddl/schema.go @@ -76,7 +76,7 @@ func checkSchemaNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, dbInfo *model return err } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { return checkSchemaNotExistsFromInfoSchema(is, schemaID, dbInfo) } return checkSchemaNotExistsFromStore(t, schemaID, dbInfo) diff --git a/pkg/ddl/table.go b/pkg/ddl/table.go index d602117acdb48..1aa11c2f25731 100644 --- a/pkg/ddl/table.go +++ b/pkg/ddl/table.go @@ -300,14 +300,19 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) schemaID := job.SchemaID tbInfo := &model.TableInfo{} var orReplace bool - var oldTbInfoID int64 - if err := job.DecodeArgs(tbInfo, &orReplace, &oldTbInfoID); err != nil { + var _placeholder int64 // oldTblInfoID + if err := job.DecodeArgs(tbInfo, &orReplace, &_placeholder); err != nil { // Invalid arguments, cancel this job. job.State = model.JobStateCancelled return ver, errors.Trace(err) } tbInfo.State = model.StateNone - err := checkTableNotExists(d, t, schemaID, tbInfo.Name.L) + + oldTableID, err := findTableIDByName(d, t, schemaID, tbInfo.Name.L) + if infoschema.ErrTableNotExists.Equal(err) { + err = nil + } + failpoint.InjectCall("onDDLCreateView", job) if err != nil { if infoschema.ErrDatabaseNotExists.Equal(err) { job.State = model.JobStateCancelled @@ -329,13 +334,13 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) // none -> public tbInfo.State = model.StatePublic tbInfo.UpdateTS = t.StartTS - if oldTbInfoID > 0 && orReplace { - err = t.DropTableOrView(schemaID, job.SchemaName, oldTbInfoID, tbInfo.Name.L) + if oldTableID > 0 && orReplace { + err = t.DropTableOrView(schemaID, job.SchemaName, oldTableID, tbInfo.Name.L) if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) } - err = t.GetAutoIDAccessors(schemaID, oldTbInfoID).Del() + err = t.GetAutoIDAccessors(schemaID, oldTableID).Del() if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) @@ -1505,7 +1510,7 @@ func checkTableNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, tableName stri return err } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { return checkTableNotExistsFromInfoSchema(is, schemaID, tableName) } @@ -1519,7 +1524,7 @@ func checkTableNotExistsByName(d *ddlCtx, t *meta.Meta, schemaID int64, schemaNa return err } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { return checkTableNotExistsFromInfoSchema(is, schemaID, tableName) } return t.CheckTableNameNotExists(t.TableNameKey(schemaName, tableName)) @@ -1593,6 +1598,48 @@ func checkTableNotExistsFromStore(t *meta.Meta, schemaID int64, tableName string return nil } +func findTableIDByName(d *ddlCtx, t *meta.Meta, schemaID int64, tableName string) (int64, error) { + // Try to use memory schema info to check first. + currVer, err := t.GetSchemaVersion() + if err != nil { + return 0, err + } + is := d.infoCache.GetLatest() + if is != nil && is.SchemaMetaVersion() == currVer { + return findTableIDFromInfoSchema(is, schemaID, tableName) + } + + return findTableIDFromStore(t, schemaID, tableName) +} + +func findTableIDFromInfoSchema(is infoschema.InfoSchema, schemaID int64, tableName string) (int64, error) { + schema, ok := is.SchemaByID(schemaID) + if !ok { + return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("") + } + tbl, err := is.TableByName(schema.Name, model.NewCIStr(tableName)) + if err != nil { + return 0, err + } + return tbl.Meta().ID, nil +} + +func findTableIDFromStore(t *meta.Meta, schemaID int64, tableName string) (int64, error) { + tbls, err := t.ListSimpleTables(schemaID) + if err != nil { + if meta.ErrDBNotExists.Equal(err) { + return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("") + } + return 0, errors.Trace(err) + } + for _, tbl := range tbls { + if tbl.Name.L == tableName { + return tbl.ID, nil + } + } + return 0, infoschema.ErrTableNotExists.FastGenByArgs(tableName) +} + // updateVersionAndTableInfoWithCheck checks table info validate and updates the schema version and the table information func updateVersionAndTableInfoWithCheck(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) ( ver int64, err error) { diff --git a/pkg/ddl/table_test.go b/pkg/ddl/table_test.go index 61fc39a943de4..13d37572e5cef 100644 --- a/pkg/ddl/table_test.go +++ b/pkg/ddl/table_test.go @@ -309,7 +309,8 @@ func TestCreateView(t *testing.T) { } ctx.SetValue(sessionctx.QueryString, "skip") err = d.DoDDLJob(ctx, job) - require.Error(t, err) + // The non-existing table id in job args will not be considered anymore. + require.NoError(t, err) } func checkTableCacheTest(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) {