Implement terminate failpoint

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>

tests/robustness/failpoint/failpoint.go

@@ -37,7 +37,7 @@ const (
 
 var (
 	allFailpoints = []Failpoint{
-		KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
+		ForcefulShutdown, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
 		DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
 		BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
 		BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
@@ -50,6 +50,7 @@ var (
 		DropPeerNetwork,
 		RaftBeforeSaveSleep,
 		RaftAfterSaveSleep,
+		GracefulShutdown,
 	}
 )
 

tests/robustness/failpoint/kill.go

@@ -27,12 +27,13 @@ import (
 )
 
 var (
-	KillFailpoint Failpoint = killFailpoint{}
+	ForcefulShutdown Failpoint = forcefulShutdownFailpoint{}
+	GracefulShutdown Failpoint = gracefulShutdownFailpoint{}
 )
 
-type killFailpoint struct{}
+type forcefulShutdownFailpoint struct{}
 
-func (f killFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
+func (f forcefulShutdownFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
 	member := clus.Procs[rand.Int()%len(clus.Procs)]
 
 	for member.IsRunning() {
@@ -60,10 +61,25 @@ func (f killFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger,
 	return nil
 }
 
-func (f killFailpoint) Name() string {
-	return "Kill"
+func (f forcefulShutdownFailpoint) Name() string {
+	return "forceful-shutdown"
 }
 
-func (f killFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess) bool {
+func (f forcefulShutdownFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess) bool {
+	return true
+}
+
+type gracefulShutdownFailpoint struct{}
+
+func (f gracefulShutdownFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
+	member := clus.Procs[rand.Int()%len(clus.Procs)]
+	return member.Restart(ctx)
+}
+
+func (f gracefulShutdownFailpoint) Name() string {
+	return "graceful-shutdown"
+}
+
+func (f gracefulShutdownFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess) bool {
 	return true
 }

tests/robustness/scenarios.go

@@ -136,7 +136,7 @@ func scenarios(t *testing.T) []testScenario {
 	})
 	scenarios = append(scenarios, testScenario{
 		name:      "Issue13766",
-		failpoint: failpoint.KillFailpoint,
+		failpoint: failpoint.ForcefulShutdown,
 		profile:   traffic.HighTrafficProfile,
 		traffic:   traffic.EtcdPut,
 		cluster: *e2e.NewConfig(