unexpected assertion failed using global index

[wjhuang2016]

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

Here is the log from our internal test schrddl.

2024/10/09 16:48:19 ddl.go:212: �[0;31m[fatal] [error] [instance 16] ERROR: Error 8141: assertion failed: key: 7480000000000000ba5f69800000000000000307800029e4698b0200, assertion: Exist, start_ts: 453113916671131764, existing start ts: 0, existing commit ts: 0
/build/ddl/dml_ops.go:52: Error when executing SQL: delete from `b5d472af` where not( IsNull( `b5d472af`.`56b0971e` ) ) or `b5d472af`.`56b0971e` in ( 1444517 ,824880 ) 

2. What did you expect to see? (Required)

No error

3. What did you see instead (Required)

assertion failed

4. What is your TiDB version? (Required)

master

[Defined2014]

Do DMLs during add global index (ingest mode) while trigger this problem

[Defined2014]

Found a new bug, it seems like the global index key is double deleted.

[2024/10/17 00:29:24.592 +08:00] [Error] [session.go:660] ["assertion failed"] [conn=1008730372] [session_alias=] [message="[tikv:8141]assertion failed: key: 7480000000000000a55f69800000000000000305bff034009418482e, assertion: Exist, start_ts: 453272184960843862, existing start ts: 453272184934891627, existing commit ts: 453272184947998730"] ["mvcc history"="{\"decoded\":{\"453272182195224667\":{\"handle\":\"800\"},\"453272182221439084\":{\"handle\":\"807\"},\"453272182313189387\":{\"handle\":\"842\"},\"453272182313189519\":{\"handle\":\"845\"},\"453272182366666865\":{\"handle\":\"873\"},\"453272182444261488\":{\"handle\":\"889\"},\"453272182536011947\":{\"handle\":\"929\"},\"453272182562488352\":{\"handle\":\"943\"},\"453272182601810026\":{\"handle\":\"949\"},\"453272182630383724\":{\"handle\":\"955\"},\"453272182654238811\":{\"handle\":\"957\"},\"453272182680453129\":{\"handle\":\"970\"},\"453272182680453146\":{\"handle\":\"976\"},\"453272182850584611\":{\"handle\":\"1032\"},\"453272182890168332\":{\"handle\":\"1039\"},\"453272182903013399\":{\"handle\":\"1046\"},\"453272182968811620\":{\"handle\":\"1053\"},\"453272182968811638\":{\"handle\":\"1054\"},\"453272183047454785\":{\"handle\":\"1079\"},\"453272183060562048\":{\"handle\":\"1083\"},\"453272183191371819\":{\"handle\":\"1131\"},\"453272183283384372\":{\"handle\":\"1139\"},\"453272183322705973\":{\"handle\":\"1148\"},\"453272183335813145\":{\"handle\":\"1152\"},\"453272183349444717\":{\"handle\":\"1157\"},\"453272183493099668\":{\"handle\":\"1204\"},\"453272183519314043\":{\"handle\":\"1213\"},\"453272183545528422\":{\"handle\":\"1230\"},\"453272183702814775\":{\"handle\":\"1289\"},\"453272183728766981\":{\"handle\":\"1290\"},\"453272183742136374\":{\"handle\":\"1294\"},\"453272183742136409\":{\"handle\":\"1295\"},\"453272183807410235\":{\"handle\":\"1315\"},\"453272183846993949\":{\"handle\":\"1329\"},\"453272183938744360\":{\"handle\":\"1342\"},\"453272183951589463\":{\"handle\":\"1345\"},\"453272184096292880\":{\"handle\":\"1378\"},\"453272184161566817\":{\"handle\":\"1390\"},\"453272184410341557\":{\"handle\":\"1430\"},\"453272184620318724\":{\"handle\":\"1480\"},\"453272184738021449\":{\"handle\":\"1501\"},\"453272184895570033\":{\"handle\":\"1525\"},\"453272184921784395\":{\"handle\":\"1531\"},\"453272184934891553\":{\"handle\":\"1532\"}},\"key\":\"7480000000000000A55F69800000000000000305BFF034009418482E\",\"mvcc\":{\"info\":{\"writes\":[{\"type\":1,\"start_ts\":453272184934891627,\"commit_ts\":453272184947998730},{\"start_ts\":453272184934891553,\"commit_ts\":453272184934891610,\"short_value\":\"CH6AAAAAAAAAqQAAAAAAAAX8\"},{\"type\":1,\"start_ts\":453272184921784451,\"commit_ts\":453272184934891537},{\"start_ts\":453272184921784395,\"commit_ts\":453272184921784431,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAX7\"},{\"start_ts\":453272184895570033,\"commit_ts\":453272184895570042,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAX1\"},{\"type\":1,\"start_ts\":453272184751390760,\"commit_ts\":453272184751390784},{\"start_ts\":453272184738021449,\"commit_ts\":453272184738021481,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAXd\"},{\"type\":1,\"start_ts\":453272184620318816,\"commit_ts\":453272184620318840},{\"start_ts\":453272184620318724,\"commit_ts\":453272184620318779,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAXI\"},{\"type\":1,\"start_ts\":453272184423710812,\"commit_ts\":453272184436818002},{\"start_ts\":453272184410341557,\"commit_ts\":453272184423710783,\"short_value\":\"CH6AAAAAAAAApwAAAAAAAAWW\"},{\"type\":1,\"start_ts\":453272184174411808,\"commit_ts\":453272184174411832},{\"start_ts\":453272184161566817,\"commit_ts\":453272184174411801,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAVu\"},{\"type\":1,\"start_ts\":453272184109138010,\"commit_ts\":453272184122245127},{\"start_ts\":453272184096292880,\"commit_ts\":453272184096292918,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAVi\"},{\"type\":1,\"start_ts\":453272183951589516,\"commit_ts\":453272183964696613},{\"start_ts\":453272183951589463,\"commit_ts\":453272183951589477,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAVB\"},{\"start_ts\":453272183938744360,\"commit_ts\":453272183938744367,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAU+\"},{\"type\":1,\"start_ts\":453272183846994041,\"commit_ts\":453272183860101139},{\"start_ts\":453272183846993949,\"commit_ts\":453272183846993957,\"short_value\":\"CH6AAAAAAAAApwAAAAAAAAUx\"},{\"type\":1,\"start_ts\":453272183807410249,\"commit_ts\":453272183807410272},{\"start_ts\":453272183807410235,\"commit_ts\":453272183807410243,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAUj\"},{\"type\":1,\"start_ts\":453272183755243555,\"commit_ts\":453272183755243562},{\"start_ts\":453272183742136409,\"commit_ts\":453272183742136437,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAUP\"},{\"start_ts\":453272183742136374,\"commit_ts\":453272183742136398,\"short_value\":\"CH6AAAAAAAAAqAAAAAAAAAUO\"},{\"type\":1,\"start_ts\":453272183728767023,\"commit_ts\":453272183728767051},{\"start_ts\":453272183728766981,\"commit_ts\":453272183728766994,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAUK\"},{\"type\":1,\"start_ts\":453272183715921942,\"commit_ts\":453272183715921965},{\"start_ts\":453272183702814775,\"commit_ts\":453272183702814801,\"short_value\":\"CH6AAAAAAAAAqAAAAAAAAAUJ\"},{\"type\":1,\"start_ts\":453272183545528478,\"commit_ts\":453272183545528516},{\"start_ts\":453272183545528422,\"commit_ts\":453272183545528437,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAATO\"},{\"type\":1,\"start_ts\":453272183532421127,\"commit_ts\":453272183532421145},{\"start_ts\":453272183519314043,\"commit_ts\":453272183519314051,\"short_value\":\"CH6AAAAAAAAApgAAAAAAAAS9\"},{\"type\":1,\"start_ts\":453272183519313994,\"commit_ts\":453272183519314007},{\"start_ts\":453272183493099668,\"commit_ts\":453272183493099683,\"short_value\":\"CH6AAAAAAAAApwAAAAAAAAS0\"},{\"type\":1,\"start_ts\":453272183375134754,\"commit_ts\":4532721833751[truncated]..."]

[Defined2014]

func TestAddGlobalIndexInIngest(t *testing.T) {
	store := testkit.CreateMockStore(t)
	defer ingesttestutil.InjectMockBackendMgr(t, store)()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")

	tk.MustExec("drop table if exists t")
	tk.MustExec("create table t (a int, b int) partition by hash(a) partitions 5")
	tk.MustExec("insert into t (a, b) values (1, 1), (2, 2), (3, 3)")
	var i atomic.Int32
	i.Store(3)
	testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onJobUpdated", func(job *model.Job) {
		tk2 := testkit.NewTestKit(t, store)
		tmp := i.Add(1)
		_, err := tk2.Exec(fmt.Sprintf("insert into test.t values (%d, %d)", tmp, tmp))
		assert.Nil(t, err)

		_, err = tk2.Exec(fmt.Sprintf("update test.t set b = b + 11, a = b where b = %d", tmp-1))
		assert.Nil(t, err)

		// _, err = tk2.Exec(fmt.Sprintf("delete from test.t where b = %d", tmp-1))
		// assert.Nil(t, err)
	})
	tk.MustExec("alter table t add unique index idx_2(b) global")
	rsGlobalIndex := tk.MustQuery("select * from t use index(idx_2)").Sort()
	rsTable := tk.MustQuery("select * from t use index()").Sort()
	// rsNormalIndex := tk.MustQuery("select * from t use index(idx_1)").Sort()
	// num := len(rsGlobalIndex.Rows())
	// require.Greater(t, num, 3)
	require.Equal(t, rsGlobalIndex.String(), rsTable.String())
	// require.Equal(t, rsGlobalIndex.String(), rsNormalIndex.String())
}