From e30753e3ed7d5b98763711441779946b2843be0a Mon Sep 17 00:00:00 2001
From: EasonBall <592838129@qq.com>
Date: Wed, 1 Nov 2023 10:30:37 +0800
Subject: [PATCH] disttask/ddl: fail task for panic (#48135)

close pingcap/tidb#48136
---
 pkg/ddl/backfilling_operators.go                | 17 +++++++++++++++++
 .../realtikvtest/addindextest1/disttask_test.go |  9 +++++++++
 2 files changed, 26 insertions(+)

diff --git a/pkg/ddl/backfilling_operators.go b/pkg/ddl/backfilling_operators.go
index 694c71997fb4b..6d6190c2d1ff1 100644
--- a/pkg/ddl/backfilling_operators.go
+++ b/pkg/ddl/backfilling_operators.go
@@ -36,6 +36,7 @@ import (
 	util2 "github.com/pingcap/tidb/pkg/ddl/util"
 	"github.com/pingcap/tidb/pkg/disttask/operator"
 	"github.com/pingcap/tidb/pkg/kv"
+	"github.com/pingcap/tidb/pkg/metrics"
 	"github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/parser/terror"
 	"github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool"
@@ -45,6 +46,7 @@ import (
 	"github.com/pingcap/tidb/pkg/table"
 	"github.com/pingcap/tidb/pkg/table/tables"
 	"github.com/pingcap/tidb/pkg/tablecodec"
+	tidbutil "github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/chunk"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/memory"
@@ -402,6 +404,13 @@ type tableScanWorker struct {
 }
 
 func (w *tableScanWorker) HandleTask(task TableScanTask, sender func(IndexRecordChunk)) {
+	defer tidbutil.Recover(metrics.LblAddIndex, "handleTableScanTaskWithRecover", func() {
+		w.ctx.onError(errors.New("met panic in tableScanWorker"))
+	}, false)
+
+	failpoint.Inject("injectPanicForTableScan", func() {
+		panic("mock panic")
+	})
 	if w.se == nil {
 		sessCtx, err := w.sessPool.Get()
 		if err != nil {
@@ -604,6 +613,14 @@ func (w *indexIngestWorker) HandleTask(rs IndexRecordChunk, send func(IndexWrite
 			w.srcChunkPool <- rs.Chunk
 		}
 	}()
+	defer tidbutil.Recover(metrics.LblAddIndex, "handleIndexIngtestTaskWithRecover", func() {
+		w.ctx.onError(errors.New("met panic in indexIngestWorker"))
+	}, false)
+
+	failpoint.Inject("injectPanicForIndexIngest", func() {
+		panic("mock panic")
+	})
+
 	result := IndexWriteResult{
 		ID: rs.ID,
 	}
diff --git a/tests/realtikvtest/addindextest1/disttask_test.go b/tests/realtikvtest/addindextest1/disttask_test.go
index 568cd4744afe2..fa5c07861ad9d 100644
--- a/tests/realtikvtest/addindextest1/disttask_test.go
+++ b/tests/realtikvtest/addindextest1/disttask_test.go
@@ -66,6 +66,15 @@ func TestAddIndexDistBasic(t *testing.T) {
 	tk.MustExec("alter table t1 add index idx1(a);")
 	tk.MustExec("admin check index t1 idx1;")
 	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/MockRunSubtaskContextCanceled"))
+
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan", "return()"))
+	tk.MustExecToErr("alter table t1 add index idx2(a);")
+	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan"))
+
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest", "return()"))
+	tk.MustExecToErr("alter table t1 add index idx2(a);")
+	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest"))
+
 	tk.MustExec(`set global tidb_enable_dist_task=0;`)
 }