kvserver: add DisableQuiescence testing knob

pkg/kv/kvserver/raft_log_queue_test.go

@@ -449,22 +449,18 @@ func TestNewTruncateDecision(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	// Unquiescing can add spurious empty log entries. Just disable it.
-	testingDisableQuiescence = true
-	defer func() {
-		testingDisableQuiescence = false
-	}()
-
 	ctx := context.Background()
 	stopper := stop.NewStopper()
 	defer stopper.Stop(ctx)
-	store, _ := createTestStore(ctx, t,
-		testStoreOpts{
-			// This test was written before test stores could start with more than one
-			// range and was not adapted.
-			createSystemRanges: false,
-		},
-		stopper)
+
+	opts := testStoreOpts{
+		// This test was written before test stores could start with more than one
+		// range and was not adapted.
+		createSystemRanges: false,
+	}
+	cfg := TestStoreConfig(nil /* clock */)
+	cfg.TestingKnobs.DisableQuiescence = true // quiescence adds spurious empty log entries
+	store := createTestStoreWithConfig(ctx, t, stopper, opts, &cfg)
 	store.SetRaftLogQueueActive(false)
 
 	r, err := store.GetReplica(1)

pkg/kv/kvserver/replica_raft.go

@@ -35,6 +35,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/buildutil"
 	"github.com/cockroachdb/cockroach/pkg/util/encoding"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -69,6 +70,10 @@ var (
 	// ~16 bytes, so Pebble point deletion batches will be bounded at ~1.6MB.
 	raftLogTruncationClearRangeThreshold = kvpb.RaftIndex(util.ConstantWithMetamorphicTestRange(
 		"raft-log-truncation-clearrange-threshold", 100000 /* default */, 1 /* min */, 1e6 /* max */))
+
+	// raftDisableLeaderFollowsLeaseholder disables lease/leader colocation.
+	raftDisableLeaderFollowsLeaseholder = envutil.EnvOrDefaultBool(
+		"COCKROACH_DISABLE_LEADER_FOLLOWS_LEASEHOLDER", false)
 )
 
 func makeIDKey() kvserverbase.CmdIDKey {

pkg/kv/kvserver/replica_raft_quiesce.go

@@ -34,8 +34,8 @@ import (
 // e.g. on every tick.
 var quiesceAfterTicks = envutil.EnvOrDefaultInt("COCKROACH_QUIESCE_AFTER_TICKS", 6)
 
-// testingDisableQuiescence disables replica quiescence.
-var testingDisableQuiescence = envutil.EnvOrDefaultBool("COCKROACH_DISABLE_QUIESCENCE", false)
+// raftDisableQuiescence disables raft quiescence.
+var raftDisableQuiescence = envutil.EnvOrDefaultBool("COCKROACH_DISABLE_QUIESCENCE", false)
 
 func (r *Replica) quiesceLocked(ctx context.Context, lagging laggingReplicaSet) {
 	if !r.mu.quiescent {
@@ -195,6 +195,9 @@ func (r *Replica) canUnquiesceRLocked() bool {
 func (r *Replica) maybeQuiesceRaftMuLockedReplicaMuLocked(
 	ctx context.Context, leaseStatus kvserverpb.LeaseStatus, livenessMap livenesspb.IsLiveMap,
 ) bool {
+	if r.store.cfg.TestingKnobs.DisableQuiescence {
+		return false
+	}
 	status, lagging, ok := shouldReplicaQuiesce(ctx, r, leaseStatus, livenessMap, r.mu.pausedFollowers)
 	if !ok {
 		return false
@@ -291,9 +294,6 @@ func shouldReplicaQuiesce(
 	livenessMap livenesspb.IsLiveMap,
 	pausedFollowers map[roachpb.ReplicaID]struct{},
 ) (*raftSparseStatus, laggingReplicaSet, bool) {
-	if testingDisableQuiescence {
-		return nil, nil, false
-	}
 	if !q.isRaftLeaderRLocked() { // fast path
 		if log.V(4) {
 			log.Infof(ctx, "not quiescing: not leader")

pkg/kv/kvserver/store.go

@@ -1226,10 +1226,13 @@ func (sc *StoreConfig) SetDefaults(numStores int) {
 	if sc.RaftEntryCacheSize == 0 {
 		sc.RaftEntryCacheSize = defaultRaftEntryCacheSize
 	}
-	if envutil.EnvOrDefaultBool("COCKROACH_DISABLE_LEADER_FOLLOWS_LEASEHOLDER", false) {
+	if raftDisableLeaderFollowsLeaseholder {
 		sc.TestingKnobs.DisableLeaderFollowsLeaseholder = true
 		sc.TestingKnobs.AllowLeaseRequestProposalsWhenNotLeader = true // otherwise lease requests fail
 	}
+	if raftDisableQuiescence {
+		sc.TestingKnobs.DisableQuiescence = true
+	}
 }
 
 // GetStoreConfig exposes the config used for this store.

pkg/kv/kvserver/testing_knobs.go

@@ -217,6 +217,9 @@ type StoreTestingKnobs struct {
 	DisableConsistencyQueue bool
 	// DisableScanner disables the replica scanner.
 	DisableScanner bool
+	// DisableQuiescence disables replica quiescence. This can also be
+	// set via COCKROACH_DISABLE_QUIESCENCE.
+	DisableQuiescence bool
 	// DisableLeaderFollowsLeaseholder disables attempts to transfer raft
 	// leadership when it diverges from the range's leaseholder. This can
 	// also be set via COCKROACH_DISABLE_LEADER_FOLLOWS_LEASEHOLDER.