Tasks: 102 total, 2 running, 98 sleeping, 2 stopped, 0 zombie %Cpu(s): 0.6 us, 2.3 sy, 0.0 ni, 2.6 id, 94.4 wa, 0.0 hi, 0.1 si, 0.0 st KiB Mem: 7679804 total, 7547980 used, 131824 free, 60 buffers KiB Swap: 0 total, 0 used, 0 free. 64140 cached Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 8680 margolin 20 0 7301336 6.720g 0 S 4.6 91.8 1:30.38 cockroach 32 root 20 0 0 0 0 S 4.1 0.0 2:22.78 kswapd0 I161207 13:50:40.685924 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:50:40.686081 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 W161207 13:50:44.201760 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 236.526641ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xe191c31) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:44.203061 52 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick for 120.471714ms (>100ms): goroutine 52 [running]: runtime/debug.Stack(0x1e83718, 0x1e83758, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x72e40a2) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick(0xc420302380, 0x75, 0xc44efa9f10) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3314 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:225 +0x2b6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:50:45.462058 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:50:45.669025 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 W161207 13:50:46.038777 50 storage/replica.go:582 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.741977927s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x67d47947) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e2978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206e2900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x31) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:46.040951 50 storage/store.go:3289 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] handle raft ready: 1.8s [processed=1] W161207 13:50:46.078292 53 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.801118331s (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6b5ae27b) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:46.078321 53 storage/store.go:3289 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] handle raft ready: 1.8s [processed=1] I161207 13:50:46.144271 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:50:46.144496 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:50:46.159729 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:50:46.159984 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:50:47.494928 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 60 goroutines, 501 MiB/5.5 GiB/6.3 GiB GO alloc/idle/total, 455 MiB/508 MiB CGO alloc/total, 1684.23cgo/sec, 0.13/0.04 %(u/s)time, 0.00 %gc (3x) I161207 13:50:47.895470 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE messages SET time_heard=CURRENT_TIMESTAMP() WHERE id=$1 I161207 13:50:48.013653 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE messages SET time_heard=CURRENT_TIMESTAMP() WHERE id=$1 I161207 13:50:48.096321 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE messages SET time_heard = current_timestamp() WHERE id = $1 W161207 13:50:53.359928 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 250.19146ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xee99e64) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:54.853708 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 216.634483ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xce99473) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:54.869005 50 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tick for 887.943589ms (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6faf0, 0x1e6fb30, 0x3c) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x34ecf1a5) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tick(0xc42025a480, 0xc420439600, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2538 +0xa3 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick(0xc420302380, 0x1, 0xc45ad9bf00) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3319 +0xfb github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:225 +0x2b6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:54.954741 51 storage/engine/rocksdb.go:1030 batch [3/324/0] commit took 557.57725ms (>500ms): goroutine 51 [running]: runtime/debug.Stack(0xecfda7b3d, 0xc41dd58de5, 0x29eeaa0) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc4615bac40, 0xc5088f78c0, 0x253b000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1030 +0x1d2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4206d0900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2435 +0x4e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206d0900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1d) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:55.431560 31489 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyTimestampCache for 695.065813ms (>500ms): goroutine 31489 [running]: runtime/debug.Stack(0x1e6e278, 0x1e6e2b8, 0x4b) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x296ddcd5) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyTimestampCache(0xc42025a480, 0xc420cf4380, 0xc460b17800, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1567 +0x251 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc460b178c0, 0x148e17c229342c20, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc460a09a40, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1845 +0x9da github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc460b178c0, 0x148e17c229342c20, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc460a09a40, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc460b178c0, 0x148e17c229342c20, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc460a09a40, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc460b17860, 0x148e17c229342c20, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc460a09a40, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc460b175f0, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc460a09a40, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc45e4edcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc42a3dd340, 0xc4bc352f28, 0xc42a3dd340, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc42a3dd340, 0xc4bc352f28, 0xc45ff5d590, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc53603c6e0, 0xc4bc352f00, 0xc4aeea3260) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:50:55.578833 51 storage/replica.go:582 [n1,s1,r29/1:/System/tsd/cr.store.mutex.sc…,@c4206d0900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.090224227s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7c964a63) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d0978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206d0900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1d) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:50:55.578869 51 storage/store.go:3289 [n1,s1,r29/1:/System/tsd/cr.store.mutex.sc…,@c4206d0900] handle raft ready: 2.1s [processed=1] I161207 13:50:56.484155 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 60 goroutines, 831 MiB/5.2 GiB/6.3 GiB GO alloc/idle/total, 456 MiB/509 MiB CGO alloc/total, 1368.22cgo/sec, 0.08/0.02 %(u/s)time, 0.00 %gc (1x) W161207 13:51:00.955583 52 storage/replica.go:582 [n1,s1,r86/1:/System/tsd/cr.node.gossip.…,@c42025b680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 560.624828ms (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x216a74bc) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025b6f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025b680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x56) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:00.955615 52 storage/store.go:3289 [n1,s1,r86/1:/System/tsd/cr.node.gossip.…,@c42025b680] handle raft ready: 0.6s [processed=1] W161207 13:51:06.530131 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 203.371725ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xc1f34cd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:06.556384 50 storage/replica.go:594 [n1,s1,r53/1:/System/tsd/cr.node.sql.mon.c…,@c42008a900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 1.30595414s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x4dd7475c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42008a9e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42008a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2447 +0x594 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42008a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x35) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:06.570988 66 storage/replica.go:594 [n1,s1,r115/1:/System/tsd/cr.store.queue.…,@c4206e3680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RaftStatus for 988.02899ms (>500ms): goroutine 66 [running]: runtime/debug.Stack(0x1e6d7d0, 0x1e6d810, 0x42) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ae4203e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e3768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RaftStatus(0xc4206e3680, 0xc4638d2540) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1120 +0x85 github.com/cockroachdb/cockroach/pkg/storage.(*replicaGCQueue).shouldQueue(0xc420285720, 0x2533aa0, 0xc45c452c00, 0x148e17c588f3b558, 0xc400000000, 0xc4206e3680, 0xc421005000, 0x21, 0x40, 0x2542e558, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_gc_queue.go:115 +0xf4 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).MaybeAdd(0xc4200c0960, 0xc4206e3680, 0x148e17c588f3b558, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:334 +0x227 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420436280, 0x2533aa0, 0xc420769590, 0xecfda7b43, 0xa46b1d2, 0x29eeaa0, 0xc42033b440, 0xc420412ab0, 0xc4206e3680, 0xecfda7b48) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:221 +0x324 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4206e3680, 0xc420439701) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420439710, 0xc4601ae460) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:321 +0x303 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2a0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772d60) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:06.592924 51 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick for 227.883879ms (>100ms): goroutine 51 [running]: runtime/debug.Stack(0x1e83718, 0x1e83758, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xd953b67) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick(0xc420302380, 0x1, 0xc4f56bff00) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3314 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:225 +0x2b6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:07.366220 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 285.614807ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x110622d7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:51:08.557891 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 65 goroutines, 889 MiB/5.2 GiB/6.3 GiB GO alloc/idle/total, 458 MiB/510 MiB CGO alloc/total, 528.50cgo/sec, 0.04/0.01 %(u/s)time, 0.00 %gc (0x) W161207 13:51:09.618335 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 425.563542ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x195d9596) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:10.804676 51 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).checkIfTxnAborted for 1.143316759s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e711a0, 0x1e711e0, 0x49) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x4425a117) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).checkIfTxnAborted(0xc42025a480, 0x2533aa0, 0xc45c3f25a0, 0x25382c0, 0xc42042cbe0, 0xc45d20c6a0, 0x0, 0xc45d20c6b0, 0xc, 0x10, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3633 +0x41a github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommandInBatch(0xc42025a480, 0x2533aa0, 0xc45c3f25a0, 0xc45dc5fa90, 0x8, 0x148e17c57d3f8448, 0x0, 0x100000001, 0x1, 0x1, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3534 +0x59f github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateProposal(0xc42025a480, 0x2533aa0, 0xc45c3f25a0, 0xc45dc5fa90, 0x8, 0x100000001, 0x1, 0x148e17c57d3f8448, 0x0, 0x100000001, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1994 +0x138 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc42025a480, 0x2533aa0, 0xc45c3f25a0, 0xc45dc5fa90, 0x8, 0x3ca9f9, 0x100000001, 0x1, 0x3748bf, 0xc44f495730, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3243 +0xb66 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:10.817896 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 160.51889ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x99152ea) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:11.569915 50 storage/replica.go:582 [n1,s1,r53/1:/System/tsd/cr.node.sql.mon.c…,@c42008a900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 6.936439408s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x19d71aa70) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42008a978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42008a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x35) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:11.570559 50 storage/store.go:3289 [n1,s1,r53/1:/System/tsd/cr.node.sql.mon.c…,@c42008a900] handle raft ready: 7.0s [processed=1] W161207 13:51:11.601788 51 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 4.958222002s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1278876b2) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:11.601812 51 storage/store.go:3289 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] handle raft ready: 5.0s [processed=1] W161207 13:51:13.380027 31875 storage/replica.go:594 [n1,s1,r224/1:/System/tsd/cr.node.sql.mon.i…,@c420433200] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.747471713s (>500ms): goroutine 31875 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x68284d61) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4204332e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc45b6e70b0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45b6e7050, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:13.380205 31875 storage/replica.go:582 [n1,s1,r224/1:/System/tsd/cr.node.sql.mon.i…,@c420433200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.755419822s (>500ms): goroutine 31875 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x68a194ae) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420433278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420433200, 0x2533aa0, 0xc45b6e70e0, 0x148e17c72a26ea74, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc45b6e70b0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45b6e7050, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:13.415185 31834 storage/replica.go:594 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.799781663s (>500ms): goroutine 31834 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6b467d1f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d09e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc45b7f48a0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45b7f4840, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:13.415305 31834 storage/replica.go:582 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.799940226s (>500ms): goroutine 31834 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6b48e882) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0900, 0x2533aa0, 0xc45b7f48d0, 0x148e17c72996f3af, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc45b7f48a0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45b7f4840, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:13.456668 31733 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 889.946186ms (>500ms): goroutine 31733 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x350b804a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc45c452cc0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45c3f2630, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:13.456782 31733 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.799809838s (>500ms): goroutine 31733 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6b46eb2e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc45c3f2a80, 0x148e17c594e3379f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc45c452cc0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45c3f2630, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:13.469645 50 storage/replica.go:594 [n1,s1,r53/1:/System/tsd/cr.node.sql.mon.c…,@c42008a900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState for 1.244569058s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6da00, 0x1e6da40, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x4a2e9de2) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42008a9e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState(0xc42008a900, 0x25382c0, 0xc42042cbe0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1149 +0x86 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResult(0xc42008a900, 0x2533aa0, 0xc45c452900, 0x100000001, 0x1, 0xc45d647410, 0x8, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:701 +0x16a github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc42008a900, 0x2533aa0, 0xc45c452900, 0xc45d647410, 0x8, 0x69263, 0x100000001, 0x1, 0x3c5b7, 0xc517f9db90, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3333 +0x762 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42008a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42008a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x35) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:13.469757 50 storage/replica.go:582 [n1,s1,r53/1:/System/tsd/cr.node.sql.mon.c…,@c42008a900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.899094897s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7131e371) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42008a978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42008a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x35) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:13.469789 50 storage/store.go:3289 [n1,s1,r53/1:/System/tsd/cr.node.sql.mon.c…,@c42008a900] handle raft ready: 1.9s [processed=1] W161207 13:51:14.026761 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 227.166001ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xd8a4731) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:14.075249 53 storage/replica.go:582 [n1,s1,r58/1:/System/tsd/cr.node.{go…-sq…},@c42025bb00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.470842553s (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x934610b9) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025bb78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025bb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x3a) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:14.075276 53 storage/store.go:3289 [n1,s1,r58/1:/System/tsd/cr.node.{go…-sq…},@c42025bb00] handle raft ready: 2.5s [processed=1] W161207 13:51:14.102206 52 storage/replica.go:582 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.484360395s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x941454cb) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x66) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:14.102233 52 storage/store.go:3289 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] handle raft ready: 2.5s [processed=1] W161207 13:51:14.122569 51 storage/replica.go:582 [n1,s1,r84/1:/System/tsd/cr.node.sql.mon.c…,@c420150480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.484668822s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x94190996) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201504f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420150480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x54) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:14.122597 51 storage/store.go:3289 [n1,s1,r84/1:/System/tsd/cr.node.sql.mon.c…,@c420150480] handle raft ready: 2.5s [processed=1] W161207 13:51:14.130658 50 storage/replica.go:582 [n1,s1,r56/1:/System/tsd/cr.node.sql.mon.c…,@c420150900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 660.747182ms (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x276233ae) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420150978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420150900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x38) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:14.130687 50 storage/store.go:3289 [n1,s1,r56/1:/System/tsd/cr.node.sql.mon.c…,@c420150900] handle raft ready: 0.7s [processed=1] W161207 13:51:16.808589 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 100.022201ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x5f637b9) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:51:16.907747 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 60 goroutines, 945 MiB/5.1 GiB/6.3 GiB GO alloc/idle/total, 459 MiB/510 MiB CGO alloc/total, 490.18cgo/sec, 0.06/0.04 %(u/s)time, 0.00 %gc (0x) W161207 13:51:18.365755 32030 storage/replica.go:594 [n1,s1,r224/1:/System/tsd/cr.node.sql.mon.i…,@c420433200] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 635.686859ms (>500ms): goroutine 32030 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x25e3cfcb) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4204332e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4576ad290, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4576ad230, 0x0, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4360b5cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc45b776a40, 0xc456f6af28, 0xc45b776a40, 0x62ef23, 0xc420783ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc45b776a40, 0xc456f6af28, 0x253cde0, 0xc457088780, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45bb33a40, 0xc456f6af00, 0xc45f241980) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:18.365905 32030 storage/replica.go:582 [n1,s1,r224/1:/System/tsd/cr.node.sql.mon.i…,@c420433200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 639.545403ms (>500ms): goroutine 32030 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x261eb03b) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420433278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420433200, 0x2533aa0, 0xc4576ad320, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4576ad290, 0x148e17c895d47743, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4576ad230, 0x0, 0x0, 0x100000001, 0x1, 0xe0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4360b5cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc45b776a40, 0xc456f6af28, 0xc45b776a40, 0x62ef23, 0xc420783ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc45b776a40, 0xc456f6af28, 0x253cde0, 0xc457088780, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45bb33a40, 0xc456f6af00, 0xc45f241980) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:18.370135 32031 storage/replica.go:594 [n1,s1,r103/1:/System/tsd/cr.node.sql.mon.i…,@c420433680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 630.465655ms (>500ms): goroutine 32031 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x25942477) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420433768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4576ad530, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4576ad4d0, 0x0, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc44513fcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc45b776d40, 0xc456f6b108, 0xc45b776d40, 0x62ef23, 0xc4207816f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc45b776d40, 0xc456f6b108, 0x253cde0, 0xc457088820, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45bb33e50, 0xc456f6b0e0, 0xc45f241c20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:18.370231 32031 storage/replica.go:582 [n1,s1,r103/1:/System/tsd/cr.node.sql.mon.i…,@c420433680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 630.609178ms (>500ms): goroutine 32031 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2596551a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4204336f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420433680, 0x2533aa0, 0xc4576ad5c0, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4576ad530, 0x148e17c8969f4cb9, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4576ad4d0, 0x0, 0x0, 0x100000001, 0x1, 0x67, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc44513fcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc45b776d40, 0xc456f6b108, 0xc45b776d40, 0x62ef23, 0xc4207816f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc45b776d40, 0xc456f6b108, 0x253cde0, 0xc457088820, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45bb33e50, 0xc456f6b0e0, 0xc45f241c20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:18.373769 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 144.848985ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x8a23859) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:18.388844 52 storage/engine/rocksdb.go:1030 batch [3/2140/0] commit took 577.635769ms (>500ms): goroutine 52 [running]: runtime/debug.Stack(0xecfda7b55, 0xc42c9f4d9f, 0x29eeaa0) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc43f3a3c00, 0xc4525c8000, 0x253b000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1030 +0x1d2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42025a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2435 +0x4e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x10f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:18.409925 52 storage/replica.go:582 [n1,s1,r271/1:/System/tsd/cr.node.exec.late…,@c42025a900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 870.501645ms (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x33e2cd0d) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x10f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:18.409954 52 storage/store.go:3289 [n1,s1,r271/1:/System/tsd/cr.node.exec.late…,@c42025a900] handle raft ready: 0.9s [processed=1] W161207 13:51:18.415517 50 storage/replica.go:582 [n1,s1,r242/1:/System/tsd/cr.node.sql.mon.c…,@c42008ad80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 633.409444ms (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x25c10fa4) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42008adf8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42008ad80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0xf2) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:18.415544 50 storage/store.go:3289 [n1,s1,r242/1:/System/tsd/cr.node.sql.mon.c…,@c42008ad80] handle raft ready: 0.6s [processed=1] W161207 13:51:18.420411 53 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 882.236005ms (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3495da65) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:18.420433 53 storage/store.go:3289 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] handle raft ready: 0.9s [processed=2] W161207 13:51:18.452004 51 storage/replica.go:582 [n1,s1,r85/1:/System/tsd/cr.node.sql.mon.i…,@c420433b00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 682.362877ms (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x28ac07fd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420433b78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420433b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x55) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:18.452040 51 storage/store.go:3289 [n1,s1,r85/1:/System/tsd/cr.node.sql.mon.i…,@c420433b00] handle raft ready: 0.7s [processed=1] W161207 13:51:24.564176 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 551.45717ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x20de9192) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:27.452860 51 storage/replica.go:594 [n1,s1,r84/1:/System/tsd/cr.node.sql.mon.c…,@c420150480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RequestLease for 940.214239ms (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e74d78, 0x1e74db8, 0x44) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x380a87df) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420150568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RequestLease(0xc420150480, 0x2533aa0, 0xc456b95b00, 0x7f2b1df08e18, 0xc4e157c8c0, 0xc4e157b1f0, 0x148e17c9b02b3751, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:1678 +0x448 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeCmd(0xc420150480, 0x2533aa0, 0xc456b95b00, 0xc459c48a38, 0x8, 0x0, 0x7f2b1df08e18, 0xc4e157c8c0, 0xc4e157b1f0, 0x148e17c9b02b3751, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:155 +0x31cd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeBatch(0xc420150480, 0x2533aa0, 0xc456b95b00, 0xc459c48a38, 0x8, 0x7f2b1df08e18, 0xc4e157c8c0, 0xc4e157b1f0, 0x148e17c9b02b3751, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3867 +0x426 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc420150480, 0x2533aa0, 0xc456b95b00, 0xc459c48a38, 0x8, 0x148e17c9b02b3751, 0x0, 0x0, 0x0, 0x54, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3659 +0x1e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommandInBatch(0xc420150480, 0x2533aa0, 0xc456b95b00, 0xc459c48a38, 0x8, 0x148e17c9b02b3751, 0x0, 0x0, 0x0, 0x54, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3555 +0x127 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateProposal(0xc420150480, 0x2533aa0, 0xc456b95b00, 0xc459c48a38, 0x8, 0x100000001, 0x1, 0x148e17c9b02b3751, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1994 +0x138 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc420150480, 0x2533aa0, 0xc456b95b00, 0xc459c48a38, 0x8, 0x52595, 0x100000001, 0x1, 0x2e604, 0xc4e157b030, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3243 +0xb66 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420150480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420150480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x54) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:28.043304 52 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RequestLease for 1.024503671s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e74d78, 0x1e74db8, 0x44) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3d10af77) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RequestLease(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0x7f2b1df08e18, 0xc54ae20e00, 0xc452d44b60, 0x148e17ca300e5790, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:1678 +0x448 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeCmd(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0xc459c48a90, 0x8, 0x0, 0x7f2b1df08e18, 0xc54ae20e00, 0xc452d44b60, 0x148e17ca300e5790, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:155 +0x31cd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeBatch(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0xc459c48a90, 0x8, 0x7f2b1df08e18, 0xc54ae20e00, 0xc452d44b60, 0x148e17ca300e5790, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3867 +0x426 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0xc459c48a90, 0x8, 0x148e17ca300e5790, 0x0, 0x0, 0x0, 0x1, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3659 +0x1e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommandInBatch(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0xc459c48a90, 0x8, 0x148e17ca300e5790, 0x0, 0x0, 0x0, 0x1, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3555 +0x127 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateProposal(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0xc459c48a90, 0x8, 0x100000001, 0x1, 0x148e17ca300e5790, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1994 +0x138 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc42025a480, 0x2533aa0, 0xc456b00d20, 0xc459c48a90, 0x8, 0x3caa06, 0x100000001, 0x1, 0x3748c9, 0xc4e157b0a0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3243 +0xb66 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:28.168246 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 180.844797ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xac778fd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:51:28.478100 69 gossip/gossip.go:435 [n1] gossip status (ok, 1 node) gossip client (0/3 cur/max conns) gossip server (0/3 cur/max conns, infos 0/0 sent/received, bytes 0B/0B sent/received) W161207 13:51:28.945512 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 242.435769ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xe7346b9) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:51:29.255418 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT username, testonly FROM users I161207 13:51:29.697306 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 63 goroutines, 996 MiB/5.1 GiB/6.3 GiB GO alloc/idle/total, 460 MiB/511 MiB CGO alloc/total, 176.43cgo/sec, 0.05/0.04 %(u/s)time, 0.00 %gc (0x) W161207 13:51:30.053471 38 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica for 174.872595ms (>100ms): goroutine 38 [running]: runtime/debug.Stack(0x1e80390, 0x1e803d0, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xa6c5813) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica(0xc420302380, 0xc42024f433, 0xb, 0xb, 0x0, 0x0, 0x0, 0xc42025a480) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1548 +0x1f6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeGossipSystemData(0xc420302380, 0x2533aa0, 0xc4207691d0, 0xb696e8, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1317 +0xe1 github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.maybeGossipSystemData)-fm(0x2533aa0, 0xc4207691d0, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1253 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1276 +0x209 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc4202b6450) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:30.053535 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 238.066103ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xe3099b7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:30.053573 37 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica for 734.038562ms (>100ms): goroutine 37 [running]: runtime/debug.Stack(0x1e80390, 0x1e803d0, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2bc08a22) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica(0xc420302380, 0x2a40040, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42025a480) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1548 +0x1f6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeGossipFirstRange(0xc420302380, 0x2533aa0, 0xc4207691a0, 0xb696e8, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1299 +0x17b github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.maybeGossipFirstRange)-fm(0x2533aa0, 0xc4207691a0, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1248 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1276 +0x209 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc4202b6420) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:30.129963 51 storage/replica.go:582 [n1,s1,r84/1:/System/tsd/cr.node.sql.mon.c…,@c420150480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 5.215275981s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x136dacbcd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201504f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420150480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x54) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:30.129997 51 storage/store.go:3289 [n1,s1,r84/1:/System/tsd/cr.node.sql.mon.c…,@c420150480] handle raft ready: 5.2s [processed=1] W161207 13:51:30.130023 52 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 5.215289779s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x136db01b3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:30.130035 52 storage/store.go:3289 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] handle raft ready: 5.2s [processed=1] I161207 13:51:30.214919 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT username, testonly FROM users W161207 13:51:32.062447 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 192.196771ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xb74b0a3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:32.217504 53 storage/replica.go:594 [n1,s1,r98/1:/System/tsd/cr.node.sql.mon.c…,@c420151b00] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 938.440276ms (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x37ef7654) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420151be8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420151b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2447 +0x594 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420151b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x62) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:32.237387 32199 storage/replica.go:594 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyTimestampCache for 766.767643ms (>500ms): goroutine 32199 [running]: runtime/debug.Stack(0x1e6e278, 0x1e6e2b8, 0x4b) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2db3f21b) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyTimestampCache(0xc4201d0d80, 0xc4ad002700, 0xc45584a900, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1567 +0x251 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1845 +0x9da github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc45584a990, 0x0, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4ad00dcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc456cd81c0, 0xc4503a8d48, 0xc456cd81c0, 0x208020002300208, 0x20001f8020401f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc456cd81c0, 0xc4503a8d48, 0x80070004c0070, 0xffd0fffc00200050, 0xff7cff54ffb4ff94) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a3a40f0, 0xc4503a8d20, 0xc45aea90e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:32.645089 52 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick for 247.948142ms (>100ms): goroutine 52 [running]: runtime/debug.Stack(0x1e83718, 0x1e83758, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xec7636e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick(0xc420302380, 0x31, 0xc44ecf7f00) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3314 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:225 +0x2b6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:34.002271 32135 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).GetReplica for 137.99479ms (>100ms): goroutine 32135 [running]: runtime/debug.Stack(0x1e802d8, 0x1e80318, 0x40) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x839a226) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).GetReplica(0xc420302380, 0x1, 0xc42025a480, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1520 +0xd2 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4558a2180, 0x148e17cb799e62ec, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2425 +0x6a9 github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4558a2120, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4acfdfcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc46564b8c0, 0xc45549ce88, 0xc46564b8c0, 0x62ef23, 0xc438bd76f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc46564b8c0, 0xc45549ce88, 0x253cde0, 0xc45590b680, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a3e4690, 0xc45549ce60, 0xc44ed8ecc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:34.011645 50 storage/engine/rocksdb.go:1030 batch [3/231/3] commit took 971.413244ms (>500ms): goroutine 50 [running]: runtime/debug.Stack(0xecfda7b64, 0x21d786c5, 0x29eeaa0) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc4568df180, 0xc455572060, 0x253b000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1030 +0x1d2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommand(0xc42070f200, 0x2533aa0, 0xc455572060, 0xc456effd30, 0x8, 0x0, 0x43e54, 0x25669, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3501 +0x42f github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc42070f200, 0x2533aa0, 0xc455572060, 0xc456effd30, 0x8, 0x43e54, 0x100000001, 0x1, 0x25669, 0xc456556ee0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3292 +0x578 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42070f200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42070f200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x5b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:35.070359 52 storage/replica.go:594 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 580.184605ms (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2294ea1d) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e29e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4206e2900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2342 +0x233 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206e2900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x31) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:35.461047 50 storage/replica.go:582 [n1,s1,r91/1:/System/tsd/cr.store.replicas…,@c42070f200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 4.915546503s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x124fd4987) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42070f278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42070f200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x5b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:36.246062 32251 storage/replica.go:594 [n1,s1,r50/1:/System/tsd/cr.node.s{ql…-ys…},@c420308900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 998.684713ms (>500ms): goroutine 32251 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3b86b829) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4203089e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc45572e360, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc45572e300, 0x0, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc561eabcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc455f1d180, 0xc44f05d888, 0xc455f1d180, 0x62ef23, 0xc4203f7ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc455f1d180, 0xc44f05d888, 0x253cde0, 0xc4509cf040, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc459a96dc0, 0xc44f05d860, 0xc459d90000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:36.660604 51 storage/engine/rocksdb.go:1030 batch [3/231/3] commit took 646.59488ms (>500ms): goroutine 51 [running]: runtime/debug.Stack(0xecfda7b66, 0x21a6f08d, 0x29eeaa0) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc4e157d500, 0xc4557466f0, 0x253b000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1030 +0x1d2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommand(0xc420432000, 0x2533aa0, 0xc4557466f0, 0xc45968f010, 0x8, 0x0, 0x79a5e, 0x46cc7, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3501 +0x42f github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc420432000, 0x2533aa0, 0xc4557466f0, 0xc45968f010, 0x8, 0x79a5e, 0x100000001, 0x1, 0x46cc7, 0xc49fa0e2a0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3292 +0x578 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420432000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420432000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x2f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:36.757664 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 177.490609ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xa944ab1) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:36.757696 32196 storage/replica.go:607 [n1,s1,r33/1:/System/{tsd/cr.s…-"update-…},@c420722900] cmdQMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds for 510.087697ms (>500ms): goroutine 32196 [running]: runtime/debug.Stack(0x1e6dfe8, 0x1e6e028, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e675211) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4207229a8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc420722900, 0x2533aa0, 0xc455012c30, 0xc49fa0ea10, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x934 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420722900, 0x2533aa0, 0xc455012c30, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1811 +0xbfd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420722900, 0x2533aa0, 0xc455012c30, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420722900, 0x2533aa0, 0xc455012c30, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4559176b0, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4559175f0, 0x0, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4acf9bcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc456d598c0, 0xc4503a87a8, 0xc456d598c0, 0xffdc009000b00170, 0xff28fec4ff1cff54) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc456d598c0, 0xc4503a87a8, 0xfd68fdc0fde4fdd4, 0xfd50fd38fd38fd18, 0xfc8cfd2cfce8fd64) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a4c9770, 0xc4503a8780, 0xc45aea8540) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:36.776286 50 storage/store.go:3289 [n1,s1,r91/1:/System/tsd/cr.store.replicas…,@c42070f200] handle raft ready: 5.1s [processed=1] W161207 13:51:37.274495 32251 storage/replica.go:582 [n1,s1,r50/1:/System/tsd/cr.node.s{ql…-ys…},@c420308900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 4.154294374s (>500ms): goroutine 32251 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xf79d8066) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420308978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420308900, 0x2533aa0, 0xc45572e3f0, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc45572e360, 0x148e17cb7aedd658, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc45572e300, 0x0, 0x0, 0x100000001, 0x1, 0x32, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc561eabcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc455f1d180, 0xc44f05d888, 0xc455f1d180, 0x62ef23, 0xc4203f7ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc455f1d180, 0xc44f05d888, 0x253cde0, 0xc4509cf040, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc459a96dc0, 0xc44f05d860, 0xc459d90000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:37.481120 32199 storage/replica.go:594 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 3.49939962s (>500ms): goroutine 32199 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xd09499c4) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc45584a990, 0x0, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4ad00dcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc456cd81c0, 0xc4503a8d48, 0xc456cd81c0, 0x208020002300208, 0x20001f8020401f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc456cd81c0, 0xc4503a8d48, 0x80070004c0070, 0xffd0fffc00200050, 0xff7cff54ffb4ff94) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a3a40f0, 0xc4503a8d20, 0xc45aea90e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:37.676131 32199 storage/replica.go:582 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 5.240756728s (>500ms): goroutine 32199 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1385f99f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0d80, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc45584a990, 0x148e17cb7a30b25d, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc45584a990, 0x0, 0x0, 0x100000001, 0x1, 0x66, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4ad00dcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc456cd81c0, 0xc4503a8d48, 0xc456cd81c0, 0x208020002300208, 0x20001f8020401f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc456cd81c0, 0xc4503a8d48, 0x80070004c0070, 0xffd0fffc00200050, 0xff7cff54ffb4ff94) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a3a40f0, 0xc4503a8d20, 0xc45aea90e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:37.848431 32115 storage/replica.go:607 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] cmdQMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds for 948.458371ms (>500ms): goroutine 32115 [running]: runtime/debug.Stack(0x1e6dfe8, 0x1e6e028, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x38885383) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a528) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc42025a480, 0x2533aa0, 0xc455012c00, 0xc49fa0e9a0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x934 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc455012c00, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1811 +0xbfd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc455012c00, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc455012c00, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc455b03ec0, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc455b03e60, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4c5683cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4573949c0, 0xc45590b7e8, 0xc4573949c0, 0x62ef23, 0xc438bd36f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4573949c0, 0xc45590b7e8, 0x253cde0, 0xc457088b40, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45ac6dea0, 0xc45590b7c0, 0xc45bf3b8c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:38.148981 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 266.614271ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xfe435ff) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:38.657777 32196 storage/replica.go:594 [n1,s1,r33/1:/System/{tsd/cr.s…-"update-…},@c420722900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 703.929081ms (>500ms): goroutine 32196 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x29f51af9) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4207229e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc420722900, 0x148e17cd046caf58, 0x0, 0x2533aa0, 0xc455012c30, 0xc4acf9aa98, 0xc455305020, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc420722900, 0x2533aa0, 0xc455012c30, 0xc4acf9aff0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420722900, 0x2533aa0, 0xc455012c30, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420722900, 0x2533aa0, 0xc455012c30, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420722900, 0x2533aa0, 0xc455012c30, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4559176b0, 0x148e17cb79943696, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4559175f0, 0x0, 0x0, 0x100000001, 0x1, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4acf9bcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc456d598c0, 0xc4503a87a8, 0xc456d598c0, 0xffdc009000b00170, 0xff28fec4ff1cff54) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc456d598c0, 0xc4503a87a8, 0xfd68fdc0fde4fdd4, 0xfd50fd38fd38fd18, 0xfc8cfd2cfce8fd64) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a4c9770, 0xc4503a8780, 0xc45aea8540) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:38.881580 51 storage/replica.go:582 [n1,s1,r47/1:/System/tsd/cr.node.sql.mon.…,@c420432000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 8.181837937s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e7acf071) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420432078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420432000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x2f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:38.884860 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 263.55272ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xfb57ed0) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:38.886375 51 storage/store.go:3289 [n1,s1,r47/1:/System/tsd/cr.node.sql.mon.…,@c420432000] handle raft ready: 8.6s [processed=1] W161207 13:51:40.364381 32134 storage/replica.go:594 [n1,s1,r271/1:/System/tsd/cr.node.exec.late…,@c42025a900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 2.395931916s (>500ms): goroutine 32134 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x8ecf050c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a9e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc42025a900, 0x148e17ccf342b80c, 0x0, 0x2533aa0, 0xc453c09770, 0xc42ede0a98, 0xc455158300, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc42025a900, 0x2533aa0, 0xc453c09770, 0xc42ede0ff0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a900, 0x2533aa0, 0xc453c09770, 0x148e17cb7944e0ab, 0x0, 0x100000001, 0x1, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a900, 0x2533aa0, 0xc453c09770, 0x148e17cb7944e0ab, 0x0, 0x100000001, 0x1, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a900, 0x2533aa0, 0xc453c09770, 0x148e17cb7944e0ab, 0x0, 0x100000001, 0x1, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc455b03a10, 0x148e17cb7944e0ab, 0x0, 0x100000001, 0x1, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc455b03980, 0x0, 0x0, 0x100000001, 0x1, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc42ede1cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc46564b480, 0xc45549cc08, 0xc46564b480, 0x62ef23, 0xc438bd6ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc46564b480, 0xc45549cc08, 0x253cde0, 0xc45549caa0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a3e4140, 0xc45549cbe0, 0xc44ed8eae0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:40.819398 50 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady for 206.832765ms (>100ms): goroutine 50 [running]: runtime/debug.Stack(0x1e83620, 0x1e83660, 0x42) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xc54047d) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x90) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3273 +0xe2 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:41.783482 52 storage/replica.go:594 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RequestLease for 1.997208901s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e74d78, 0x1e74db8, 0x44) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x770afd45) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e29e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RequestLease(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0x7f2b1df08e18, 0xc4568df500, 0xc55f57e5b0, 0x148e17cb7c21022e, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:1678 +0x448 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeCmd(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0xc45968f900, 0x8, 0x0, 0x7f2b1df08e18, 0xc4568df500, 0xc55f57e5b0, 0x148e17cb7c21022e, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:155 +0x31cd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeBatch(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0xc45968f900, 0x8, 0x7f2b1df08e18, 0xc4568df500, 0xc55f57e5b0, 0x148e17cb7c21022e, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3867 +0x426 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0xc45968f900, 0x8, 0x148e17cb7c21022e, 0x0, 0x0, 0x0, 0x31, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3659 +0x1e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommandInBatch(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0xc45968f900, 0x8, 0x148e17cb7c21022e, 0x0, 0x0, 0x0, 0x31, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3555 +0x127 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateProposal(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0xc45968f900, 0x8, 0x100000001, 0x1, 0x148e17cb7c21022e, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1994 +0x138 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc4206e2900, 0x2533aa0, 0xc45565bd10, 0xc45968f900, 0x8, 0x7c8cc, 0x100000001, 0x1, 0x49dd0, 0xc49fa0ecb0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3243 +0xb66 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4206e2900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206e2900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x31) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:42.093068 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 182.364477ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xadea93d) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:42.405681 51 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady for 473.564663ms (>100ms): goroutine 51 [running]: runtime/debug.Stack(0x1e83620, 0x1e83660, 0x42) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1c3a05f7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0xdf) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3273 +0xe2 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:42.515558 32135 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 3.214476535s (>500ms): goroutine 32135 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xbf9904f7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc42025a480, 0x148e17ccb62dcb53, 0x0, 0x2533aa0, 0xc453c098f0, 0xc4acfdea98, 0xc4551581e0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc42025a480, 0x2533aa0, 0xc453c098f0, 0xc4acfdeff0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc453c098f0, 0x148e17cb799e62ec, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc453c098f0, 0x148e17cb799e62ec, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc453c098f0, 0x148e17cb799e62ec, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4558a2180, 0x148e17cb799e62ec, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4558a2120, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4acfdfcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc46564b8c0, 0xc45549ce88, 0xc46564b8c0, 0x62ef23, 0xc438bd76f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc46564b8c0, 0xc45549ce88, 0x253cde0, 0xc45590b680, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45a3e4690, 0xc45549ce60, 0xc44ed8ecc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:42.605041 32115 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 1.000731944s (>500ms): goroutine 32115 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ba5f528) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc42025a480, 0x148e17cd52b77dae, 0x0, 0x2533aa0, 0xc455012c00, 0xc4c5682a98, 0xc455305140, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc42025a480, 0x2533aa0, 0xc455012c00, 0xc4c5682ff0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc455012c00, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc455012c00, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc455012c00, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc455b03ec0, 0x148e17cb7948091f, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc455b03e60, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4c5683cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4573949c0, 0xc45590b7e8, 0xc4573949c0, 0x62ef23, 0xc438bd36f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4573949c0, 0xc45590b7e8, 0x253cde0, 0xc457088b40, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45ac6dea0, 0xc45590b7c0, 0xc45bf3b8c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 I161207 13:51:42.613079 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 92 goroutines, 1.0 GiB/5.0 GiB/6.3 GiB GO alloc/idle/total, 460 MiB/512 MiB CGO alloc/total, 260.89cgo/sec, 0.02/0.03 %(u/s)time, 0.00 %gc (0x) W161207 13:51:44.135503 50 storage/replica.go:594 [n1,s1,r144/1:/System/tsd/cr.store.raft.…,@c4206fc480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 563.02919ms (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x218f24c6) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206fc568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4206fc480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2342 +0x233 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206fc480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x90) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:44.150199 32136 storage/replica.go:594 [n1,s1,r33/1:/System/{tsd/cr.s…-"update-…},@c420722900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 2.191747074s (>500ms): goroutine 32136 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x82a36802) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4207229e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc453c09f50) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc453c09e90, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:44.150334 32136 storage/replica.go:582 [n1,s1,r33/1:/System/{tsd/cr.s…-"update-…},@c420722900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 3.960475908s (>500ms): goroutine 32136 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xec101104) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420722978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420722900, 0x2533aa0, 0xc45399a0f0, 0x148e17cd2ec420d7, 0x0, 0x0, 0x0, 0x21, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc453c09f50) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc453c09e90, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:44.157153 51 storage/replica.go:582 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.751339152s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x68635090) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0xdf) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:44.157183 51 storage/store.go:3289 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] handle raft ready: 5.0s [processed=1] W161207 13:51:44.167516 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 319.72591ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x130ea156) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:44.167804 32323 storage/replica.go:594 [n1,s1,r271/1:/System/tsd/cr.node.exec.late…,@c42025a900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 2.054782526s (>500ms): goroutine 32323 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7a797e3e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a9e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc4537ca000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45399a060, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:44.167905 32323 storage/replica.go:582 [n1,s1,r271/1:/System/tsd/cr.node.exec.late…,@c42025a900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 2.117031092s (>500ms): goroutine 32323 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7e2f54b4) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a900, 0x2533aa0, 0xc45399a5d0, 0x148e17cd82cb54f2, 0x0, 0x0, 0x0, 0x10f, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc4537ca000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc45399a060, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:44.168708 32322 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.15468329s (>500ms): goroutine 32322 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x44d3119a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc455013da0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc455013560, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:44.168807 32322 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 1.285366205s (>500ms): goroutine 32322 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x4c9d21bd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc45399aa80, 0x148e17cd7c53788c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc455013da0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc455013560, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:51:44.170122 52 storage/replica.go:582 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 10.701408595s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x27dda8953) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e2978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206e2900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x31) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:44.170145 52 storage/store.go:3289 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] handle raft ready: 10.7s [processed=1] W161207 13:51:44.176481 53 storage/replica.go:582 [n1,s1,r98/1:/System/tsd/cr.node.sql.mon.c…,@c420151b00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 13.841197302s (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x338ffe8f6) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420151b78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420151b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x62) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:44.176508 53 storage/store.go:3289 [n1,s1,r98/1:/System/tsd/cr.node.sql.mon.c…,@c420151b00] handle raft ready: 13.9s [processed=1] W161207 13:51:44.178407 50 storage/replica.go:582 [n1,s1,r144/1:/System/tsd/cr.store.raft.…,@c4206fc480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.86907019s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xab02896e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206fc4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206fc480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x90) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:44.178424 50 storage/store.go:3289 [n1,s1,r144/1:/System/tsd/cr.store.raft.…,@c4206fc480] handle raft ready: 7.2s [processed=1] I161207 13:51:44.941388 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: SELECT COUNT(*) FROM messages I161207 13:51:45.384676 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: SELECT COUNT(*) FROM messages I161207 13:51:47.778716 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 60 goroutines, 1.1 GiB/5.0 GiB/6.3 GiB GO alloc/idle/total, 460 MiB/512 MiB CGO alloc/total, 783.07cgo/sec, 0.05/0.02 %(u/s)time, 0.00 %gc (0x) W161207 13:51:49.608130 32533 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 503.533988ms (>500ms): goroutine 32533 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e0351a4) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc4507db440, 0x148e17cf3c7fae08, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc44d615080, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc4507db440, 0x148e17cf3c7fae08, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc44d615080, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc4507db440, 0x148e17cf3c7fae08, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc44d615080, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc4507db440, 0x148e17cf3c7fae08, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc44d615080, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc45082f770, 0x148e17cf3c7fae08, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc44d615080, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4507db050, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc44d615080, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc462f93cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc44bb66b00, 0xc440086f28, 0xc44bb66b00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc44bb66b00, 0xc440086f28, 0xc450b7fd10, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc455499540, 0xc440086f00, 0xc4360140c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:51:49.691051 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 136.528661ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x8234315) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:56.592835 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 298.579687ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x11cbf6e7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:56.695487 51 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 6.678907279s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x18e18098f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:51:56.695589 51 storage/store.go:3289 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] handle raft ready: 7.1s [processed=1] W161207 13:51:57.577621 50 storage/replica.go:594 [n1,s1,r111/1:/System/tsd/cr.store.mutex.co…,@c420366900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState for 6.271924168s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6da00, 0x1e6da40, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x175d5f7c8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4203669e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState(0xc420366900, 0x25382c0, 0xc42042cbe0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1149 +0x86 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResult(0xc420366900, 0x2533aa0, 0xc450589a10, 0x100000001, 0x1, 0xc4526895d8, 0x8, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:701 +0x16a github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc420366900, 0x2533aa0, 0xc450589a10, 0xc4526895d8, 0x8, 0x2dee, 0x100000001, 0x1, 0x12c7, 0xc445f04e00, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3333 +0x762 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420366900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420366900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x6f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:00.013741 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 194.961994ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xb9ee24a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:00.697900 50 storage/replica.go:582 [n1,s1,r111/1:/System/tsd/cr.store.mutex.co…,@c420366900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 9.796532562s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x247eb3952) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420366978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420366900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x6f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:52:00.728283 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 89 goroutines, 1.1 GiB/4.9 GiB/6.3 GiB GO alloc/idle/total, 460 MiB/512 MiB CGO alloc/total, 94.87cgo/sec, 0.03/0.02 %(u/s)time, 0.00 %gc (0x) W161207 13:52:00.733258 53 storage/replica.go:582 [n1,s1,r41/1:/System/tsd/cr.store.mutex.…,@c420366d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 8.055158095s (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e01ff54f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420366df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420366d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x29) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:00.733359 53 storage/store.go:3289 [n1,s1,r41/1:/System/tsd/cr.store.mutex.…,@c420366d80] handle raft ready: 11.1s [processed=1] W161207 13:52:01.221625 32643 storage/replica.go:594 [n1,s1,r41/1:/System/tsd/cr.store.mutex.…,@c420366d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 2.134478216s (>500ms): goroutine 32643 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7f398d88) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420366e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc43537fe30, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc43537fd70, 0x0, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4521bdcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc440a5fa40, 0xc4416bbc48, 0xc440a5fa40, 0x62ef23, 0xc56d1cbef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc440a5fa40, 0xc4416bbc48, 0x253cde0, 0xc43b8b88c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc4547f6c80, 0xc4416bbc20, 0xc440977bc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:52:01.224508 32643 storage/replica.go:582 [n1,s1,r41/1:/System/tsd/cr.store.mutex.…,@c420366d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 3.274289703s (>500ms): goroutine 32643 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xc329b227) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420366df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420366d80, 0x2533aa0, 0xc43537fec0, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc43537fe30, 0x148e17d004c6fa3f, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc43537fd70, 0x0, 0x0, 0x100000001, 0x1, 0x29, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4521bdcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc440a5fa40, 0xc4416bbc48, 0xc440a5fa40, 0x62ef23, 0xc56d1cbef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc440a5fa40, 0xc4416bbc48, 0x253cde0, 0xc43b8b88c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc4547f6c80, 0xc4416bbc20, 0xc440977bc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:52:01.225721 50 storage/store.go:3289 [n1,s1,r111/1:/System/tsd/cr.store.mutex.co…,@c420366900] handle raft ready: 11.0s [processed=1] W161207 13:52:01.289006 32603 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 4.28187681s (>500ms): goroutine 32603 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xff38414a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc435d7ccf0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc435d7cc00, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc44df71cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4362d5380, 0xc43b8b8de8, 0xc4362d5380, 0x62ef23, 0xc56d1cbef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4362d5380, 0xc43b8b8de8, 0x253cde0, 0xc43b8b88c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc4552554a0, 0xc43b8b8dc0, 0xc433466e40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:52:01.289110 32603 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 4.658943836s (>500ms): goroutine 32603 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x115b1d75c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025a480, 0x2533aa0, 0xc435d7cdb0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc435d7ccf0, 0x148e17d002f6bd7e, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc435d7cc00, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc44df71cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4362d5380, 0xc43b8b8de8, 0xc4362d5380, 0x62ef23, 0xc56d1cbef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4362d5380, 0xc43b8b8de8, 0x253cde0, 0xc43b8b88c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc4552554a0, 0xc43b8b8dc0, 0xc433466e40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:52:01.289693 52 storage/replica.go:582 [n1,s1,r30/1:/System/tsd/cr.store.raft.…,@c4206fd200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 11.542116908s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2aff6ba2c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206fd278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4206fd200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1e) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:01.289727 52 storage/store.go:3289 [n1,s1,r30/1:/System/tsd/cr.store.raft.…,@c4206fd200] handle raft ready: 11.5s [processed=1] W161207 13:52:08.539317 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 1.120296048s (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x42c65c70) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:16.363299 51 storage/scheduler.go:155 [n1] raftScheduler.mu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker for 539.785683ms (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e7db20, 0x1e7db60, 0x44) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x202c79d3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420274438) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:220 +0x169 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:19.165527 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 3.521284709s (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xd1e28a65) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:19.964891 32779 storage/scheduler.go:155 [n1] raftScheduler.mu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).enqueue1 for 556.204599ms (>500ms): goroutine 32779 [running]: runtime/debug.Stack(0x1e7dd00, 0x1e7dd40, 0x46) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x21270237) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420274438) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).enqueue1(0xc420274420, 0x2, 0xdf, 0x1b59030) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:278 +0x70 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).EnqueueRaftReady(0xc420274420, 0xdf) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:310 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).enqueueRaftUpdateCheck(0xc420302380, 0xdf) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3243 +0x3c github.com/cockroachdb/cockroach/pkg/storage.defaultSubmitProposalLocked(0xc4201d0900, 0xc54167a480, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2232 +0x6c7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).submitProposalLocked(0xc4201d0900, 0xc54167a480, 0x100000001, 0xc400000001) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2153 +0x77 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2129 +0x5ab github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441999f50) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441999ef0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:52:23.810496 52 storage/replica.go:594 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 851.387369ms (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x32bf23e9) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2342 +0x233 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x66) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:38.836788 68 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit for 294.371686ms (>100ms): goroutine 68 [running]: runtime/debug.Stack(0x1e7ea28, 0x1e7ea68, 0x49) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x118bc166) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc441b4ca50, 0xc449059a48) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:299 +0x1ae github.com/cockroachdb/cockroach/pkg/storage.(*Store).LeaseCount(0xc420302380, 0x6c) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2287 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Capacity(0xc420302380, 0x7253ad, 0x663561, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2190 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*Store).Descriptor(0xc420302380, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2217 +0x2f github.com/cockroachdb/cockroach/pkg/storage.(*Store).updateCapacityGauges(0xc420302380, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3674 +0x2f github.com/cockroachdb/cockroach/pkg/storage.(*Store).ComputeMetrics(0xc420302380, 0x51, 0x800000000000000, 0x40) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3880 +0x40 github.com/cockroachdb/cockroach/pkg/server.(*Node).computePeriodicMetrics.func1(0xc420302380, 0xc420438420, 0xc449059d90) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:666 +0x52 github.com/cockroachdb/cockroach/pkg/storage.(*Stores).VisitStores(0xc42000d6e0, 0xc449059e20, 0xc449059e38, 0x640915) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:131 +0x219 github.com/cockroachdb/cockroach/pkg/server.(*Node).computePeriodicMetrics(0xc4201ce000, 0x51, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:671 +0x60 github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1() /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:652 +0x17e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420773200) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:52:41.721720 69 gossip/gossip.go:435 [n1] gossip status (ok, 1 node) gossip client (0/3 cur/max conns) gossip server (0/3 cur/max conns, infos 0/0 sent/received, bytes 0B/0B sent/received) W161207 13:52:44.421598 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 1.677671855s (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x63ff3daf) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:44.831250 37 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica for 13.070583542s (>100ms): goroutine 37 [running]: runtime/debug.Stack(0x1e80390, 0x1e803d0, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x30b1146f6) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica(0xc420302380, 0x2a40040, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42025a480) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1548 +0x1f6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeGossipFirstRange(0xc420302380, 0x2533aa0, 0xc4207691a0, 0xb696e8, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1299 +0x17b github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.maybeGossipFirstRange)-fm(0x2533aa0, 0xc4207691a0, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1248 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1276 +0x209 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc4202b6420) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I161207 13:52:46.018778 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 124 goroutines, 1.2 GiB/4.8 GiB/6.3 GiB GO alloc/idle/total, 462 MiB/514 MiB CGO alloc/total, 89.09cgo/sec, 0.01/0.01 %(u/s)time, 0.00 %gc (0x) W161207 13:52:47.450909 38 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica for 3.963008743s (>100ms): goroutine 38 [running]: runtime/debug.Stack(0x1e80390, 0x1e803d0, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xec36b6e7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).LookupReplica(0xc420302380, 0xc42024f433, 0xb, 0xb, 0x0, 0x0, 0x0, 0xc42025a480) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1548 +0x1f6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeGossipSystemData(0xc420302380, 0x2533aa0, 0xc4207691d0, 0xb696e8, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1317 +0xe1 github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.maybeGossipSystemData)-fm(0x2533aa0, 0xc4207691d0, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1253 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1276 +0x209 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc4202b6450) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:48.960531 32688 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).GetReplica for 182.369817ms (>100ms): goroutine 32688 [running]: runtime/debug.Stack(0x1e802d8, 0x1e80318, 0x40) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xadebe19) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).GetReplica(0xc420302380, 0x1c, 0xc4201d0480, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1520 +0xd2 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc441b66120, 0x148e17da04b8251b, 0x0, 0x100000001, 0x1, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2425 +0x6a9 github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc441ae7dd0, 0x0, 0x0, 0x100000001, 0x1, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4f55bbcc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4418ec640, 0xc44793a2a8, 0xc4418ec640, 0x18, 0xffffffffffffffff) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4418ec640, 0xc44793a2a8, 0xffffffffffffffff, 0xc466cccf80, 0x708056) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc451f897c0, 0xc44793a280, 0xc438917920) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:52:50.617563 53 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 1.584224292s (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x5e6d5824) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2447 +0x594 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:56.289674 65 storage/replica.go:594 [n1,s1,r86/1:/System/tsd/cr.node.gossip.…,@c42025b680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 16.877919392s (>500ms): goroutine 65 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ee009ca0) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025b768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc42025b680, 0x148e17d6076521b1, 0xc400000000, 0x2533a60, 0xc434823440, 0xc4633bbaa0, 0xc438917e00, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc42025b680, 0x2533a60, 0xc434823440, 0x12) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc4200c0c30, 0x2533aa0, 0xc441b4c6f0, 0xc42025b680, 0xc42033b440, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:535 +0x656 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:474 +0xa2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc420412ab0, 0xc4633bbe58, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x100 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:478 +0x2f8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772d40) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:56.331966 51 storage/replica.go:594 [n1,s1,r97/1:/System/tsd/cr.node.sql.mon.a…,@c42022fb00] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 1.040828569s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3e09c899) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42022fbe8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42022fb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2342 +0x233 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42022fb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x61) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:56.449637 52 storage/engine/rocksdb.go:1030 batch [3/325/0] commit took 1.039731478s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0xecfda7bb0, 0xc4177bdaea, 0x29eeaa0) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc4f55221c0, 0xc441b4c600, 0x253b000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1030 +0x1d2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2435 +0x4e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x66) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:52:56.454067 32779 storage/replica.go:594 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 19.031801832s (>500ms): goroutine 32779 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x46e623fe8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d09e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441999f50) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441999ef0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:52:56.454371 32779 storage/replica.go:582 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 54.21600014s (>500ms): goroutine 32779 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xc9f86828c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0900, 0x2533aa0, 0xc441999f80, 0x148e17d2f2f32ca3, 0x0, 0x0, 0x0, 0xdf, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441999f50) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441999ef0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:52:59.457424 69 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit for 623.031576ms (>100ms): goroutine 69 [running]: runtime/debug.Stack(0x1e7ea28, 0x1e7ea68, 0x49) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2522b518) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc441b4db00, 0xc420846a20) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:299 +0x1ae github.com/cockroachdb/cockroach/pkg/storage.(*Store).LeaseCount(0xc420302380, 0x6c) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2287 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Capacity(0xc420302380, 0x61351c, 0xc42076d258, 0xc42000db58, 0x0, 0xc44778ba40) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2190 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*Store).Descriptor(0xc420302380, 0x0, 0xc42000db00, 0xc456001301) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2217 +0x2f github.com/cockroachdb/cockroach/pkg/storage.(*Store).GossipStore(0xc420302380, 0x2533aa0, 0xc42084c060, 0x6de599, 0xc420438420) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1356 +0x7b github.com/cockroachdb/cockroach/pkg/server.(*Node).gossipStores.func1(0xc420302380, 0xc420438420, 0xc420846cc8) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:628 +0x4b github.com/cockroachdb/cockroach/pkg/storage.(*Stores).VisitStores(0xc42000d6e0, 0xc420846d70, 0xc420846d88, 0x64027c) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:131 +0x219 github.com/cockroachdb/cockroach/pkg/server.(*Node).gossipStores(0xc4201ce000, 0x2533aa0, 0xc42084c060) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:635 +0x72 github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:613 +0x308 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420773220) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:00.272743 52 storage/replica.go:594 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 621.910445ms (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f7a0, 0x1e6f7e0, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x251199ad) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2447 +0x594 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x66) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:01.771011 66 storage/replica.go:594 [n1,s1,r28/1:/System/tsd/cr.node.sql.mon.…,@c4201d0480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RaftStatus for 3.932284069s (>500ms): goroutine 66 [running]: runtime/debug.Stack(0x1e6d7d0, 0x1e6d810, 0x42) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xea61e4a5) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).RaftStatus(0xc4201d0480, 0xc4456501e0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1120 +0x85 github.com/cockroachdb/cockroach/pkg/storage.getTruncatableIndexes(0x2533aa0, 0xc441b4dce0, 0xc4201d0480, 0x2533aa0, 0xc441b4dce0, 0x17ff9c0, 0x2a40528) /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:80 +0x4c github.com/cockroachdb/cockroach/pkg/storage.(*raftLogQueue).shouldQueue(0xc420285740, 0x2533aa0, 0xc441b4dce0, 0x148e17dd83e1dd0a, 0xc400000000, 0xc4201d0480, 0xc421005000, 0x21, 0x40, 0x24e9c70a, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:196 +0x46 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).MaybeAdd(0xc4200c0a50, 0xc4201d0480, 0x148e17dd83e1dd0a, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:334 +0x227 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420436280, 0x2533aa0, 0xc420769590, 0xecfda7b43, 0xa46b1d2, 0x29eeaa0, 0xc42033b440, 0xc420412ab0, 0xc4201d0480, 0xecfda7b82) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:221 +0x324 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4201d0480, 0xc420439701) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420439710, 0xc4601ae460) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:321 +0x303 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2a0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772d60) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:11.230782 37 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 935.718482ms (>500ms): goroutine 37 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x37c5ee52) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc42025a480, 0x148e17e05363b6a8, 0x0, 0x2533aa0, 0xc4207691a0, 0xc449053838, 0xc44585cc60, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc42025a480, 0x2533aa0, 0xc4207691a0, 0x659f70) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).getLeaseForGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3975 +0x6f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc420412ab0, 0xc449053ac0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x100 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).getLeaseForGossip(0xc42025a480, 0x2533aa0, 0xc4207691a0, 0xc449053b90, 0x10) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:4006 +0x112 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).maybeGossipFirstRange(0xc42025a480, 0x2533aa0, 0xc4207691a0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:4045 +0x30c github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeGossipFirstRange(0xc420302380, 0x2533aa0, 0xc4207691a0, 0xb696e8, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1301 +0x1ac github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.maybeGossipFirstRange)-fm(0x2533aa0, 0xc4207691a0, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1248 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1276 +0x209 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc4202b6420) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:14.111647 50 storage/replica.go:594 [n1,s1,r89/1:/Table/57/1/195{19438…-59251…},@c420746d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState for 57.462512127s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6da00, 0x1e6da40, 0x43) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xd610859ff) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420746e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState(0xc420746d80, 0x25382c0, 0xc42042cbe0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1149 +0x86 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResult(0xc420746d80, 0x2533aa0, 0xc4407c5d40, 0x100000001, 0x1, 0xc451ad5f20, 0x8, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:701 +0x16a github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc420746d80, 0x2533aa0, 0xc4407c5d40, 0xc451ad5f20, 0x8, 0x126dd, 0x100000001, 0x1, 0x79a, 0xc5430ec070, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3333 +0x762 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420746d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2485 +0x831 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420746d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x59) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:14.117646 50 storage/replica.go:582 [n1,s1,r89/1:/Table/57/1/195{19438…-59251…},@c420746d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1m12.044757772s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x10c633c30c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420746df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420746d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x59) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:14.117674 50 storage/store.go:3289 [n1,s1,r89/1:/Table/57/1/195{19438…-59251…},@c420746d80] handle raft ready: 72.0s [processed=1] W161207 13:53:14.117743 32792 storage/replica.go:594 [n1,s1,r83/1:/System/tsd/cr.store.queue.…,@c4206d1b00] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 7.285887455s (>500ms): goroutine 32792 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1b245d1df) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d1be8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6e300) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6e2a0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:14.117804 32792 storage/replica.go:582 [n1,s1,r83/1:/System/tsd/cr.store.queue.…,@c4206d1b00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 9.350937154s (>500ms): goroutine 32792 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x22d5bfa42) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d1b78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206d1b00, 0x2533aa0, 0xc441b66960, 0x148e17df8a1d57f5, 0x0, 0x0, 0x0, 0x53, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6e300) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6e2a0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 I161207 13:53:14.142831 72 server/status/runtime.go:228 runtime stats: 6.7 GiB RSS, 132 goroutines, 1.2 GiB/4.8 GiB/6.3 GiB GO alloc/idle/total, 462 MiB/514 MiB CGO alloc/total, 1.15cgo/sec, 0.03/0.04 %(u/s)time, 0.00 %gc (0x) W161207 13:53:15.503717 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 1.018873118s (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3cbac51e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:16.766781 38 storage/replica.go:594 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 2.5011957s (>500ms): goroutine 38 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x951537b4) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc42025a480, 0x148e17df8040c757, 0xc400000000, 0x2533aa0, 0xc4207691d0, 0xc4f565da38, 0xc44585c7e0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc42025a480, 0x2533aa0, 0xc4207691d0, 0x659f70) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).getLeaseForGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3975 +0x6f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc420412ab0, 0xc4f565dcc0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x100 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).getLeaseForGossip(0xc42025a480, 0x2533aa0, 0xc4207691d0, 0xb, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:4006 +0x112 github.com/cockroachdb/cockroach/pkg/storage.(*Store).maybeGossipSystemData(0xc420302380, 0x2533aa0, 0xc4207691d0, 0xb696e8, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1326 +0x160 github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.maybeGossipSystemData)-fm(0x2533aa0, 0xc4207691d0, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1253 +0x3e github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1276 +0x209 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc4202b6450) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:18.277605 32787 storage/replica.go:607 [n1,s1,r222/1:/System/tsd/cr.store.mutex.st…,@c4206d1200] cmdQMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds for 1.99170164s (>500ms): goroutine 32787 [running]: runtime/debug.Stack(0x1e6dfe8, 0x1e6e028, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x76b6f488) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d12a8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0xc5416d96c0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x934 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1811 +0xbfd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc441b4d6b0, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc441b66690, 0x0, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc426741cd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4418eddc0, 0xc44793a3e8, 0xc4418eddc0, 0x62ef23, 0xc4203f3ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4418eddc0, 0xc44793a3e8, 0x253cde0, 0xc43b6977c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45157d540, 0xc44793a3c0, 0xc434823140) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:20.507404 32788 storage/replica.go:607 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] cmdQMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds for 2.750781212s (>500ms): goroutine 32788 [running]: runtime/debug.Stack(0x1e6dfe8, 0x1e6e028, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xa3f5971c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e29a8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0xc42678a9a0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x934 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1811 +0xbfd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc441b67440, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc441b673e0, 0x0, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc426745cd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc47363ae80, 0xc44778b9c8, 0xc47363ae80, 0x62ef23, 0xc4203f36f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc47363ae80, 0xc44778b9c8, 0x253cde0, 0xc43b697860, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc451f89c20, 0xc44778b9a0, 0xc434823500) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.103439 51 storage/engine/rocksdb.go:1030 batch [3/325/0] commit took 882.287138ms (>500ms): goroutine 51 [running]: runtime/debug.Stack(0xecfda7bd5, 0xc40c1c1e13, 0x29eeaa0) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc4f5522540, 0xc441b4ccf0, 0x253b000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1030 +0x1d2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42022fb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2435 +0x4e0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42022fb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x61) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.134498 32689 storage/replica.go:594 [n1,s1,r86/1:/System/tsd/cr.node.gossip.…,@c42025b680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 27.469515306s (>500ms): goroutine 32689 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6654f8a2a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025b768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441b4d1d0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441ae7e00, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.134558 32689 storage/replica.go:582 [n1,s1,r86/1:/System/tsd/cr.node.gossip.…,@c42025b680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 29.650123054s (>500ms): goroutine 32689 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6e748f92e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025b6f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42025b680, 0x2533aa0, 0xc441b4db60, 0x148e17d9f689bc85, 0x0, 0x0, 0x0, 0x56, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441b4d1d0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441ae7e00, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.147205 66 storage/replica.go:594 [n1,s1,r28/1:/System/tsd/cr.node.sql.mon.…,@c4201d0480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.getTruncatableIndexes for 517.636344ms (>500ms): goroutine 66 [running]: runtime/debug.Stack(0x1e6a260, 0x1e6a2a0, 0x42) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1eda80f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.getTruncatableIndexes(0x2533aa0, 0xc441b4dce0, 0xc4201d0480, 0x2533aa0, 0xc441b4dce0, 0x17ff9c0, 0x2a40528) /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:109 +0x13d github.com/cockroachdb/cockroach/pkg/storage.(*raftLogQueue).shouldQueue(0xc420285740, 0x2533aa0, 0xc441b4dce0, 0x148e17dd83e1dd0a, 0xc400000000, 0xc4201d0480, 0xc421005000, 0x21, 0x40, 0x24e9c70a, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:196 +0x46 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).MaybeAdd(0xc4200c0a50, 0xc4201d0480, 0x148e17dd83e1dd0a, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:334 +0x227 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420436280, 0x2533aa0, 0xc420769590, 0xecfda7b43, 0xa46b1d2, 0x29eeaa0, 0xc42033b440, 0xc420412ab0, 0xc4201d0480, 0xecfda7b82) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:221 +0x324 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4201d0480, 0xc420439701) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420439710, 0xc4601ae460) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:321 +0x303 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2a0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772d60) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.149433 32795 storage/replica.go:594 [n1,s1,r82/1:/System/tsd/cr.node.txn.resta…,@c42032f200] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.748531284s (>500ms): goroutine 32795 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3e64a4e54) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42032f2e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6e990) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6e930, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.149486 32795 storage/replica.go:582 [n1,s1,r82/1:/System/tsd/cr.node.txn.resta…,@c42032f200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 18.226296303s (>500ms): goroutine 32795 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x43e5f35ef) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42032f278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42032f200, 0x2533aa0, 0xc441b66ba0, 0x148e17df9284b3b5, 0x0, 0x0, 0x0, 0x52, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6e990) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6e930, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.149501 68 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit for 1.498793383s (>100ms): goroutine 68 [running]: runtime/debug.Stack(0x1e7ea28, 0x1e7ea68, 0x49) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x5955c5a7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc439a6f7a0, 0xc449059af8) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:299 +0x1ae github.com/cockroachdb/cockroach/pkg/storage.(*Store).updateReplicationGauges(0xc420302380, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3790 +0x299 github.com/cockroachdb/cockroach/pkg/storage.(*Store).ComputeMetrics(0xc420302380, 0x51, 0x800000000000000, 0x40) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3884 +0x64 github.com/cockroachdb/cockroach/pkg/server.(*Node).computePeriodicMetrics.func1(0xc420302380, 0xc420438420, 0xc449059d90) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:666 +0x52 github.com/cockroachdb/cockroach/pkg/storage.(*Stores).VisitStores(0xc42000d6e0, 0xc449059e20, 0xc449059e38, 0x640915) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:131 +0x219 github.com/cockroachdb/cockroach/pkg/server.(*Node).computePeriodicMetrics(0xc4201ce000, 0x51, 0x0, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:671 +0x60 github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1() /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:652 +0x17e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420773200) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 E161207 13:53:26.198635 76 storage/replica_consistency_queue.go:70 [n1,replica consistency checker,s1,r36/1:/System/tsd/cr.store.{qu…-ra…},@c4206fc000] could not compute checksum for range [/System/tsd/cr.store.queue.split.pending/1/10s/2016-11-04T09:00:00Z, /System/tsd/cr.store.raft.heartbeats.pending/1/10s/2016-11-18T04:00:00Z]: while waiting for compute checksum (ID = 5edf4de2-1708-4dc6-a73f-db01fa3635e9): context deadline exceeded W161207 13:53:26.262392 55 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1 for 625.80866ms (>100ms): goroutine 55 [running]: runtime/debug.Stack(0x1e8fc98, 0x1e8fcd8, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x254d1514) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3371 +0x26a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc42077a0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.262796 32806 storage/replica.go:594 [n1,s1,r101/1:/System/tsd/cr.store.{mu…-qu…},@c4206d1680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 5.452610132s (>500ms): goroutine 32806 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x145003a54) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d1768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441b676e0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441b67680, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.262872 32806 storage/replica.go:582 [n1,s1,r101/1:/System/tsd/cr.store.{mu…-qu…},@c4206d1680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 15.029004895s (>500ms): goroutine 32806 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x37fcc6a5f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d16f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206d1680, 0x2533aa0, 0xc441b666c0, 0x148e17df8a0f8a7f, 0x0, 0x0, 0x0, 0x65, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441b676e0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441b67680, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.262959 32797 storage/replica.go:594 [n1,s1,r115/1:/System/tsd/cr.store.queue.…,@c4206e3680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.345491661s (>500ms): goroutine 32797 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ce4468cd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e3768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6eba0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6eb40, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.263010 32797 storage/replica.go:582 [n1,s1,r115/1:/System/tsd/cr.store.queue.…,@c4206e3680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.352585436s (>500ms): goroutine 32797 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ceb0a6dc) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e36f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206e3680, 0x2533aa0, 0xc439a6f590, 0x148e17df928fe1d0, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6eba0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6eb40, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.263114 32807 storage/replica.go:594 [n1,s1,r28/1:/System/tsd/cr.node.sql.mon.…,@c4201d0480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.34562429s (>500ms): goroutine 32807 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ce466ee2) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441b677a0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441b67740, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.263160 32807 storage/replica.go:582 [n1,s1,r28/1:/System/tsd/cr.node.sql.mon.…,@c4201d0480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.34568068s (>500ms): goroutine 32807 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ce474b28) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d04f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4201d0480, 0x2533aa0, 0xc439a6f440, 0x148e17df8dea8cb7, 0x0, 0x0, 0x0, 0x1c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc441b677a0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc441b67740, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.264583 32793 storage/replica.go:594 [n1,s1,r40/1:/System/tsd/cr.store.queue.r…,@c4206e3200] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.347110042s (>500ms): goroutine 32793 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3ce5d1a9a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e32e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6e630) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6e5d0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.264630 32793 storage/replica.go:582 [n1,s1,r40/1:/System/tsd/cr.store.queue.r…,@c4206e3200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 16.359309867s (>500ms): goroutine 32793 [running]: runtime/debug.Stack(0x1e6ef48, 0x1e6ef88, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3cf17422b) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e3278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2142 +0x6ef github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1879 +0x1c0 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206e3200, 0x2533aa0, 0xc441b665a0, 0x148e17df8b30d52d, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).InitOrJoinRequest.func1(0x2533aa0, 0xc439a6e630) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:133 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420412ab0, 0x21f2cfe, 0x1e, 0x9d, 0x0, 0x0, 0xc439a6e5d0, 0x2533a20, 0xc4200122b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234 W161207 13:53:26.264644 32796 storage/store.go:817 [n1] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).GetReplica for 616.269441ms (>100ms): goroutine 32796 [running]: runtime/debug.Stack(0x1e802d8, 0x1e80318, 0x40) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x24bb8681) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420302670) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).GetReplica(0xc420302380, 0x1e, 0xc4206fd200, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1520 +0xd2 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc439a6f710, 0x148e17e2a30dbab7, 0x0, 0x100000001, 0x1, 0x1e, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2425 +0x6a9 github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc439a6f290, 0x0, 0x0, 0x100000001, 0x1, 0x1e, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc5416c7cc8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc47363b9c0, 0xc4479de2a8, 0xc47363b9c0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc47363b9c0, 0xc4479de2a8, 0xc439a6e990, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45157dd10, 0xc4479de280, 0xc434823e00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.265211 32790 storage/replica.go:594 [n1,s1,r36/1:/System/tsd/cr.store.{qu…-ra…},@c4206fc000] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 9.651419155s (>500ms): goroutine 32790 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x23f44f813) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206fc0e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc4206fc000, 0x148e17e42a2335df, 0x0, 0x2533aa0, 0xc439a6fa40, 0xc4633baaa8, 0xc44585d2c0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc4206fc000, 0x2533aa0, 0xc439a6fa40, 0xc4633bb000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206fc000, 0x2533aa0, 0xc439a6fa40, 0x148e17e07ae11449, 0x0, 0x100000001, 0x1, 0x24, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206fc000, 0x2533aa0, 0xc439a6fa40, 0x148e17e07ae11449, 0x0, 0x100000001, 0x1, 0x24, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206fc000, 0x2533aa0, 0xc439a6fa40, 0x148e17e07ae11449, 0x0, 0x100000001, 0x1, 0x24, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc441b66480, 0x148e17e07ae11449, 0x0, 0x100000001, 0x1, 0x24, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc441b66390, 0x0, 0x0, 0x100000001, 0x1, 0x24, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc4633bbcd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc47363af40, 0xc44778b928, 0xc47363af40, 0x62ef23, 0xc4203f3ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc47363af40, 0xc44778b928, 0x253cde0, 0xc43b6977c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45157d590, 0xc44778b900, 0xc438917c20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.265314 32788 storage/replica.go:594 [n1,s1,r49/1:/System/tsd/cr.store.queue.…,@c4206e2900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 2.528709819s (>500ms): goroutine 32788 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x96b90cbb) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206e29e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc4206e2900, 0x148e17e5ddf4e0a3, 0x0, 0x2533aa0, 0xc439a6f2f0, 0xc426744aa8, 0xc445650cc0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0xc426745000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206e2900, 0x2533aa0, 0xc439a6f2f0, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc441b67440, 0x148e17df81bdb852, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc441b673e0, 0x0, 0x0, 0x100000001, 0x1, 0x31, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc426745cd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc47363ae80, 0xc44778b9c8, 0xc47363ae80, 0x62ef23, 0xc4203f36f0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc47363ae80, 0xc44778b9c8, 0x253cde0, 0xc43b697860, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc451f89c20, 0xc44778b9a0, 0xc434823500) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.265412 32672 storage/replica.go:594 [n1,s1,r89/1:/Table/57/1/195{19438…-59251…},@c420746d80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 7.335397465s (>500ms): goroutine 32672 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1b5394859) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420746e68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc420746d80, 0x148e17e4c19881fb, 0x0, 0x2533aa0, 0xc4407c5a70, 0xc4267509d0, 0xc445650b40, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc420746d80, 0x2533aa0, 0xc4407c5a70, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addReadOnlyCmd(0xc420746d80, 0x2533aa0, 0xc4407c5a70, 0x148e17cf0388c340, 0x0, 0x100000001, 0x1, 0x59, 0x0, 0xc445b7bb00, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1635 +0x753 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420746d80, 0x2533aa0, 0xc4407c5a70, 0x148e17cf0388c340, 0x0, 0x100000001, 0x1, 0x59, 0x0, 0xc445b7bb00, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1195 +0x48f github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc4407c59e0, 0x148e17cf0388c340, 0x0, 0x100000001, 0x1, 0x59, 0x0, 0xc445b7bb00, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc4407c5860, 0x0, 0x0, 0x100000001, 0x1, 0x59, 0x0, 0xc445b7bb00, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc426751cd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc437b42a00, 0xc43a0d2fc8, 0xc437b42a00, 0xc4203f2f28, 0x62fce1) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc437b42a00, 0xc43a0d2fc8, 0x21ed509, 0x13, 0x12d) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc451c9b090, 0xc43a0d2fa0, 0xc438b086c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.265510 32787 storage/replica.go:594 [n1,s1,r222/1:/System/tsd/cr.store.mutex.st…,@c4206d1200] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1 for 6.168430362s (>500ms): goroutine 32787 [running]: runtime/debug.Stack(0x1e8b198, 0x1e8b1d8, 0x56) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x16faac71a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206d12e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func1(0xc4206d1200, 0x148e17e4e04dfb40, 0x0, 0x2533aa0, 0xc439a6e090, 0xc426740aa8, 0xc44585d320, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:909 +0x27b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0xc426741000) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:930 +0xa6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1829 +0xabd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206d1200, 0x2533aa0, 0xc439a6e090, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc441b4d6b0, 0x148e17dca5c6a6df, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc441b66690, 0x0, 0x0, 0x100000001, 0x1, 0xde, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc426741cd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc4418eddc0, 0xc44793a3e8, 0xc4418eddc0, 0x62ef23, 0xc4203f3ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc4418eddc0, 0xc44793a3e8, 0x253cde0, 0xc43b6977c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45157d540, 0xc44793a3c0, 0xc434823140) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.276325 32789 storage/replica.go:607 [n1,s1,r144/1:/System/tsd/cr.store.raft.…,@c4206fc480] cmdQMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds for 5.43645813s (>500ms): goroutine 32789 [running]: runtime/debug.Stack(0x1e6dfe8, 0x1e6e028, 0x41) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x14409c492) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4206fc528) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4206fc480, 0x2533aa0, 0xc439d163f0, 0xc42678b490, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x934 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc4206fc480, 0x2533aa0, 0xc439d163f0, 0x148e17e0c50a8d39, 0x0, 0x100000001, 0x1, 0x90, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1811 +0xbfd github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc4206fc480, 0x2533aa0, 0xc439d163f0, 0x148e17e0c50a8d39, 0x0, 0x100000001, 0x1, 0x90, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1724 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206fc480, 0x2533aa0, 0xc439d163f0, 0x148e17e0c50a8d39, 0x0, 0x100000001, 0x1, 0x90, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1192 +0x222 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420302380, 0x2533aa0, 0xc439a6f410, 0x148e17e0c50a8d39, 0x0, 0x100000001, 0x1, 0x90, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2454 +0x75e github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42000d6e0, 0x2533aa0, 0xc439a6f080, 0x0, 0x0, 0x100000001, 0x1, 0x90, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:819 +0x352 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420412ab0, 0xc426749cd8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4201ce000, 0x25339e0, 0xc47363b140, 0xc44778b888, 0xc47363b140, 0x62ef23, 0xc4203f3ef0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:830 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4201ce000, 0x25339e0, 0xc47363b140, 0xc44778b888, 0x253cde0, 0xc43b6977c0, 0x11) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:852 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x251bd60, 0xc4201ce000, 0xc45148a140, 0xc44778b860, 0xc434823800) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:185 +0x64 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:188 +0x1d6 W161207 13:53:26.283063 52 storage/replica.go:582 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1m23.012555699s (>500ms): goroutine 52 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1353ef13b3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x66) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772940) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.283090 52 storage/store.go:3289 [n1,s1,r102/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0d80] handle raft ready: 83.0s [processed=1] W161207 13:53:26.297673 53 storage/replica.go:582 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1m23.952192693s (>500ms): goroutine 53 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x138bf0ccb5) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42025a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42025a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772960) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.297707 53 storage/store.go:3289 [n1,s1,r1/1:/{Min-System/tsd/c…},@c42025a480] handle raft ready: 84.0s [processed=1] W161207 13:53:26.299891 51 storage/replica.go:582 [n1,s1,r97/1:/System/tsd/cr.node.sql.mon.a…,@c42022fb00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1m8.116504661s (>500ms): goroutine 51 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0xfdc0f6055) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42022fb78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42022fb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0x61) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772920) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.299923 51 storage/store.go:3289 [n1,s1,r97/1:/System/tsd/cr.node.sql.mon.a…,@c42022fb00] handle raft ready: 68.4s [processed=1] I161207 13:53:26.319133 72 server/status/runtime.go:228 runtime stats: 2.1 GiB RSS, 155 goroutines, 1.1 GiB/309 MiB/1.7 GiB GO alloc/idle/total, 464 MiB/514 MiB CGO alloc/total, 11.66cgo/sec, 0.03/0.02 %(u/s)time, 0.02 %gc (1x) W161207 13:53:26.320589 50 storage/replica.go:582 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 12.082721165s (>500ms): goroutine 50 [running]: runtime/debug.Stack(0x1e6f6e8, 0x1e6f728, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2d02fb18d) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4201d0978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4201d0900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2318 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420302380, 0xdf) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3276 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420274420, 0xc420412ab0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420412ab0, 0xc420772900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W161207 13:53:26.320617 50 storage/store.go:3289 [n1,s1,r223/1:/System/tsd/cr.node.sql.mon.c…,@c4201d0900] handle raft ready: 12.2s [processed=1] I161207 13:53:26.445747 72 server/status/runtime.go:228 runtime stats: 2.1 GiB RSS, 125 goroutines, 462 MiB/1002 MiB/1.7 GiB GO alloc/idle/total, 468 MiB/515 MiB CGO alloc/total, 27714.85cgo/sec, 0.88/0.09 %(u/s)time, 0.00 %gc (0x) I161207 13:53:26.446145 69 gossip/gossip.go:435 [n1] gossip status (ok, 1 node) gossip client (0/3 cur/max conns) gossip server (0/3 cur/max conns, infos 0/0 sent/received, bytes 0B/0B sent/received) I161207 13:53:26.892820 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT id, sender, recipient, LENGTH(data) AS blobsize, is_group, transcription, fetches, deleted FROM messages ORDER BY created DESC LIMIT 20 I161207 13:53:26.950740 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT id, sender, recipient, LENGTH(data) AS blobsize, is_group, transcription, fetches, deleted FROM messages ORDER BY created DESC LIMIT 20 I161207 13:53:27.836292 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: SELECT schema_file FROM schema_history ORDER BY apply_time DESC LIMIT 1 I161207 13:53:27.836550 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: SELECT schema_file FROM schema_history ORDER BY apply_time DESC LIMIT 1 I161207 13:53:27.888841 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT key, value FROM stats ORDER BY key I161207 13:53:27.889033 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT key, value FROM stats ORDER BY key I161207 13:53:28.034855 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.035569 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.035761 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:28.043421 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.044074 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.044441 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:28.145505 33255 sql/executor.go:351 [n1,client=127.0.0.1:37437,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.146207 33255 sql/executor.go:488 [n1,client=127.0.0.1:37437,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.146398 33255 sql/executor.go:789 [n1,client=127.0.0.1:37437,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:28.222379 33260 sql/executor.go:351 [n1,client=127.0.0.1:37433,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.223049 33260 sql/executor.go:488 [n1,client=127.0.0.1:37433,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.223286 33260 sql/executor.go:789 [n1,client=127.0.0.1:37433,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:28.273942 33264 sql/executor.go:351 [n1,client=127.0.0.1:37435,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.274785 33264 sql/executor.go:488 [n1,client=127.0.0.1:37435,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.275032 33264 sql/executor.go:789 [n1,client=127.0.0.1:37435,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:28.294056 33263 sql/executor.go:351 [n1,client=127.0.0.1:37434,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.294866 33263 sql/executor.go:488 [n1,client=127.0.0.1:37434,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:28.295097 33263 sql/executor.go:789 [n1,client=127.0.0.1:37434,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:29.225932 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: SELECT data FROM messages WHERE id=$1 I161207 13:53:29.229108 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: SELECT data FROM messages WHERE id=$1 I161207 13:53:29.229303 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: SELECT data FROM messages WHERE id = $1 I161207 13:53:29.523041 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: SELECT data FROM messages WHERE id=$1 I161207 13:53:29.544169 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT data FROM messages WHERE id=$1 I161207 13:53:29.544393 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT data FROM messages WHERE id = $1 I161207 13:53:30.044156 33255 sql/executor.go:351 [n1,client=127.0.0.1:37437,user=root] preparing: SELECT data FROM messages WHERE id=$1 I161207 13:53:30.045245 33255 sql/executor.go:488 [n1,client=127.0.0.1:37437,user=root] execRequest: SELECT data FROM messages WHERE id=$1 I161207 13:53:30.045409 33255 sql/executor.go:789 [n1,client=127.0.0.1:37437,user=root] executing 1/1: SELECT data FROM messages WHERE id = $1 I161207 13:53:30.127407 33263 sql/executor.go:351 [n1,client=127.0.0.1:37434,user=root] preparing: SELECT data FROM messages WHERE id=$1 I161207 13:53:30.146205 33263 sql/executor.go:488 [n1,client=127.0.0.1:37434,user=root] execRequest: SELECT data FROM messages WHERE id=$1 I161207 13:53:30.146253 33260 sql/executor.go:351 [n1,client=127.0.0.1:37433,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:30.146379 33263 sql/executor.go:789 [n1,client=127.0.0.1:37434,user=root] executing 1/1: SELECT data FROM messages WHERE id = $1 I161207 13:53:30.146705 33260 sql/executor.go:488 [n1,client=127.0.0.1:37433,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:30.146815 33260 sql/executor.go:789 [n1,client=127.0.0.1:37433,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:30.170928 25189 sql/mon/mem_usage.go:510 [n1,client=127.0.0.1:37364,user=root] session: memory usage increases to 54 KiB (+54632) I161207 13:53:30.177608 33260 sql/executor.go:351 [n1,client=127.0.0.1:37433,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:30.217942 33260 sql/executor.go:488 [n1,client=127.0.0.1:37433,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:30.218231 33260 sql/executor.go:789 [n1,client=127.0.0.1:37433,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:30.248277 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:30.248967 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:30.249140 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:30.298526 33349 sql/executor.go:351 [n1,client=127.0.0.1:37439,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:30.299346 33349 sql/executor.go:488 [n1,client=127.0.0.1:37439,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:30.299584 33349 sql/executor.go:789 [n1,client=127.0.0.1:37439,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:30.327151 26671 sql/mon/mem_usage.go:510 [n1,client=127.0.0.1:37387,user=root] session: memory usage increases to 56 KiB (+56680) I161207 13:53:30.386726 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:30.387189 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:30.426081 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:30.426394 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:30.446216 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:30.446430 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:30.449082 33264 sql/executor.go:351 [n1,client=127.0.0.1:37435,user=root] preparing: SELECT data FROM messages WHERE id=$1 I161207 13:53:30.449527 33264 sql/executor.go:488 [n1,client=127.0.0.1:37435,user=root] execRequest: SELECT data FROM messages WHERE id=$1 I161207 13:53:30.449747 33264 sql/executor.go:789 [n1,client=127.0.0.1:37435,user=root] executing 1/1: SELECT data FROM messages WHERE id = $1 I161207 13:53:31.143293 33255 sql/mon/mem_usage.go:510 [n1,client=127.0.0.1:37437,user=root] session: memory usage increases to 88 KiB (+89616) I161207 13:53:31.176159 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.177059 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.177233 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:31.205826 33255 sql/executor.go:351 [n1,client=127.0.0.1:37437,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.234260 33255 sql/executor.go:488 [n1,client=127.0.0.1:37437,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.234695 33255 sql/executor.go:789 [n1,client=127.0.0.1:37437,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:31.538493 33263 sql/mon/mem_usage.go:510 [n1,client=127.0.0.1:37434,user=root] session: memory usage increases to 55 KiB (+56016) I161207 13:53:31.569716 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.570441 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.570626 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:31.573610 33263 sql/executor.go:351 [n1,client=127.0.0.1:37434,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.576278 33263 sql/executor.go:488 [n1,client=127.0.0.1:37434,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.576471 33263 sql/executor.go:789 [n1,client=127.0.0.1:37434,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:31.597404 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.598657 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.598922 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:31.609909 33263 sql/executor.go:351 [n1,client=127.0.0.1:37434,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.614189 33349 sql/executor.go:351 [n1,client=127.0.0.1:37439,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.614747 33349 sql/executor.go:488 [n1,client=127.0.0.1:37439,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.614942 33349 sql/executor.go:789 [n1,client=127.0.0.1:37439,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:31.663374 33349 sql/executor.go:351 [n1,client=127.0.0.1:37439,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.680335 33263 sql/executor.go:488 [n1,client=127.0.0.1:37434,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.680743 33263 sql/executor.go:789 [n1,client=127.0.0.1:37434,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:31.707349 33349 sql/executor.go:488 [n1,client=127.0.0.1:37439,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.707604 33349 sql/executor.go:789 [n1,client=127.0.0.1:37439,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:31.806741 33264 sql/mon/mem_usage.go:510 [n1,client=127.0.0.1:37435,user=root] session: memory usage increases to 30 KiB (+30416) I161207 13:53:31.855758 33264 sql/executor.go:351 [n1,client=127.0.0.1:37435,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.856345 33264 sql/executor.go:488 [n1,client=127.0.0.1:37435,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.856690 33264 sql/executor.go:789 [n1,client=127.0.0.1:37435,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:31.859896 33427 sql/executor.go:351 [n1,client=127.0.0.1:37441,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.860473 33427 sql/executor.go:488 [n1,client=127.0.0.1:37441,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.860719 33427 sql/executor.go:789 [n1,client=127.0.0.1:37441,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:31.863990 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.864953 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:31.865156 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:31.910150 33264 sql/executor.go:351 [n1,client=127.0.0.1:37435,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.910919 33264 sql/executor.go:488 [n1,client=127.0.0.1:37435,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:31.911127 33264 sql/executor.go:789 [n1,client=127.0.0.1:37435,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:32.133853 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:32.134584 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:32.134786 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:32.143923 33255 sql/executor.go:351 [n1,client=127.0.0.1:37437,user=root] preparing: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:32.204399 33255 sql/executor.go:488 [n1,client=127.0.0.1:37437,user=root] execRequest: SELECT id, sender, recipient, is_group, EXTRACT('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id=$1 I161207 13:53:32.204714 33255 sql/executor.go:789 [n1,client=127.0.0.1:37437,user=root] executing 1/1: SELECT id, sender, recipient, is_group, extract('epoch', created), LENGTH(data), transcription, deleted FROM MESSAGES WHERE id = $1 I161207 13:53:32.742968 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:32.743768 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:32.743910 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:32.837984 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:32.838757 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:32.838960 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:33.031154 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.032006 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.032160 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:33.066796 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.067622 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.067782 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:33.119287 26671 sql/executor.go:351 [n1,client=127.0.0.1:37387,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.120050 26671 sql/executor.go:488 [n1,client=127.0.0.1:37387,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.120203 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:33.123637 25189 sql/executor.go:351 [n1,client=127.0.0.1:37364,user=root] preparing: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.123997 25189 sql/executor.go:488 [n1,client=127.0.0.1:37364,user=root] execRequest: UPDATE stats SET value=value+$1 WHERE key=$2 I161207 13:53:33.124102 25189 sql/executor.go:789 [n1,client=127.0.0.1:37364,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:33.144378 26671 sql/executor.go:789 [n1,client=127.0.0.1:37387,user=root] executing 1/1: UPDATE stats SET value = value + $1 WHERE key = $2 I161207 13:53:36.434357 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 1.8 GiB/2.0 GiB/4.0 GiB GO alloc/idle/total, 503 MiB/553 MiB CGO alloc/total, 16880.47cgo/sec, 0.94/0.16 %(u/s)time, 0.00 %gc (10x) I161207 13:53:46.433622 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 1.8 GiB/1.9 GiB/4.0 GiB GO alloc/idle/total, 504 MiB/554 MiB CGO alloc/total, 648.34cgo/sec, 0.05/0.01 %(u/s)time, 0.00 %gc (0x) I161207 13:53:56.433608 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 1.9 GiB/1.9 GiB/4.0 GiB GO alloc/idle/total, 505 MiB/555 MiB CGO alloc/total, 652.60cgo/sec, 0.02/0.01 %(u/s)time, 0.00 %gc (0x) I161207 13:54:06.433708 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 1.9 GiB/1.8 GiB/4.0 GiB GO alloc/idle/total, 506 MiB/556 MiB CGO alloc/total, 638.19cgo/sec, 0.02/0.00 %(u/s)time, 0.00 %gc (0x) I161207 13:54:16.433975 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 2.0 GiB/1.8 GiB/4.0 GiB GO alloc/idle/total, 506 MiB/556 MiB CGO alloc/total, 599.09cgo/sec, 0.02/0.01 %(u/s)time, 0.00 %gc (0x) I161207 13:54:26.421853 69 gossip/gossip.go:435 [n1] gossip status (ok, 1 node) gossip client (0/3 cur/max conns) gossip server (0/3 cur/max conns, infos 0/0 sent/received, bytes 0B/0B sent/received) I161207 13:54:26.433987 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 2.1 GiB/1.7 GiB/4.0 GiB GO alloc/idle/total, 507 MiB/557 MiB CGO alloc/total, 722.60cgo/sec, 0.02/0.01 %(u/s)time, 0.00 %gc (0x) I161207 13:54:36.433732 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 2.1 GiB/1.6 GiB/4.0 GiB GO alloc/idle/total, 507 MiB/557 MiB CGO alloc/total, 580.61cgo/sec, 0.02/0.00 %(u/s)time, 0.00 %gc (0x) I161207 13:54:46.434056 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 2.2 GiB/1.6 GiB/4.0 GiB GO alloc/idle/total, 507 MiB/557 MiB CGO alloc/total, 609.88cgo/sec, 0.02/0.01 %(u/s)time, 0.00 %gc (0x) I161207 13:54:56.434460 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 61 goroutines, 2.2 GiB/1.5 GiB/4.0 GiB GO alloc/idle/total, 508 MiB/558 MiB CGO alloc/total, 595.08cgo/sec, 0.02/0.00 %(u/s)time, 0.00 %gc (0x) I161207 13:55:06.434365 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 2.3 GiB/1.5 GiB/4.0 GiB GO alloc/idle/total, 508 MiB/558 MiB CGO alloc/total, 642.61cgo/sec, 0.02/0.01 %(u/s)time, 0.00 %gc (0x) I161207 13:55:16.434261 72 server/status/runtime.go:228 runtime stats: 4.5 GiB RSS, 60 goroutines, 2.4 GiB/1.4 GiB/4.0 GiB GO alloc/idle/total, 509 MiB/559 MiB CGO alloc/total, 605.01cgo/sec, 0.02/0.01 %(u/s)time, 0.00 %gc (0x)