[Bug]: v2 simd panic on restart

[alpe]

Is there an existing issue for this?

• I have searched the existing issues

What happened?

12:48PM INF service start connection=consensus impl=localClient module=abci-client msg="Starting localClient service"
12:48PM INF service start impl=EventBus module=events msg="Starting EventBus service"
12:48PM INF service start impl=PubSub module=pubsub msg="Starting PubSub service"
12:48PM INF service start impl=IndexerService module=txindex msg="Starting IndexerService service"
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x2 addr=0x0 pc=0x105ed80e8]

goroutine 131 [running]:
database/sql.(*DB).conn(0x41?, {0x1085420b8?, 0x10a4fa440?}, 0xac?)
        database/sql/sql.go:1309 +0x28
database/sql.(*DB).prepare(0x0, {0x1085420b8, 0x10a4fa440}, {0x107679b39, 0x46}, 0x0?)
        database/sql/sql.go:1617 +0x34
database/sql.(*DB).PrepareContext.func1(0x10?)
        database/sql/sql.go:1591 +0x48
database/sql.(*DB).retry(0x14001f93360?, 0x1400208a968)
        database/sql/sql.go:1568 +0x4c
database/sql.(*DB).PrepareContext(0x10a4fa440?, {0x1085420b8?, 0x10a4fa440?}, {0x107679b39?, 0x1400088d420?})
        database/sql/sql.go:1590 +0x64
database/sql.(*DB).Prepare(...)
        database/sql/sql.go:1607
cosmossdk.io/store/v2/storage/sqlite.(*Database).GetLatestVersion(0x1400208aab8?)
        cosmossdk.io/store/v2@v2.0.0/storage/sqlite/db.go:106 +0x50
cosmossdk.io/store/v2/storage/sqlite.(*Database).VersionExists(0x140018a83c0, 0x0)
        cosmossdk.io/store/v2@v2.0.0/storage/sqlite/db.go:131 +0x24
cosmossdk.io/store/v2/storage.(*StorageStore).VersionExists(0x1400208ab08?, 0x104e61ad4?)
        cosmossdk.io/store/v2@v2.0.0/storage/store.go:89 +0x28
cosmossdk.io/store/v2/root.(*Store).getVersionedReader(0x140008c0300, 0x0)
        cosmossdk.io/store/v2@v2.0.0/root/store.go:116 +0x34
cosmossdk.io/store/v2/root.(*Store).StateLatest(0x140008c0300)
        cosmossdk.io/store/v2@v2.0.0/root/store.go:143 +0x3c
cosmossdk.io/server/v2/cometbft.(*consensus[...]).Info(0x1085a1e20, {0x108542010, 0x14002021540}, 0x1400208ac28)
        cosmossdk.io/server/v2/cometbft@v0.0.0-20241015140036-ee3d320eaa55/abci.go:128 +0x3c
github.com/cometbft/cometbft/abci/client.(*unsyncLocalClient).Info(0x0?, {0x108542010?, 0x14002021540?}, 0x1055afc70?)
        github.com/cometbft/cometbft@v1.0.0-rc1.0.20240908111210-ab0be101882f/abci/client/unsync_local_client.go:81 +0x30
github.com/cometbft/cometbft/proxy.(*appConnQuery).Info(0x14002baaa80, {0x108542010, 0x14002021540}, 0x10a30cb40)
        github.com/cometbft/cometbft@v1.0.0-rc1.0.20240908111210-ab0be101882f/proxy/app_conn.go:178 +0x120
github.com/cometbft/cometbft/internal/consensus.(*Handshaker).Handshake(0x1400208b1c8, {0x108542010, 0x14002021540}, {0x108569e18, 0x140005cdc00})
        github.com/cometbft/cometbft@v1.0.0-rc1.0.20240908111210-ab0be101882f/internal/consensus/replay.go:244 +0x60
github.com/cometbft/cometbft/node.doHandshake({_, _}, {_, _}, {{{0xb, 0x0}, {0x1075852f4, 0x9}}, {0x14002d0a8c0, 0x7}, ...}, ...)
        github.com/cometbft/cometbft@v1.0.0-rc1.0.20240908111210-ab0be101882f/node/setup.go:236 +0x118
github.com/cometbft/cometbft/node.NewNodeWithCliParams({0x108542010, 0x14002021540}, 0x14000606780, {0x108541d90, 0x14002008000}, 0x14000e1c000, {0x108542258, 0x14000e1c150}, 0x1400208bf18, 0x1084ebb68, ...)
        github.com/cometbft/cometbft@v1.0.0-rc1.0.20240908111210-ab0be101882f/node/node.go:405 +0x87c
github.com/cometbft/cometbft/node.NewNode(...)
        github.com/cometbft/cometbft@v1.0.0-rc1.0.20240908111210-ab0be101882f/node/node.go:286
cosmossdk.io/server/v2/cometbft.(*CometBFTServer[...]).Start(0x616c62223d726f6c, {0x108542010?, 0x14002021540})
        cosmossdk.io/server/v2/cometbft@v0.0.0-20241015140036-ee3d320eaa55/server.go:238 +0x338
cosmossdk.io/server/v2.(*Server[...]).Start.func1()
        cosmossdk.io/server/v2@v2.0.0-20240718121635-a877e3e8048a/server.go:96 +0x38
created by cosmossdk.io/server/v2.(*Server[...]).Start in goroutine 1
        cosmossdk.io/server/v2@v2.0.0-20240718121635-a877e3e8048a/server.go:95 +0x13c

Cosmos SDK Version

main

How to reproduce?

export COSMOS_BUILD_OPTIONS=v2
make test

[tac0turtle]

i was unable to reproduce this locally. can you do it consistently?

[julienrbrt]

Been able to reproduce by running COSMOS_BUILD_OPTIONS=v2 make test-system

[tac0turtle]

ah i was following the how to reproduce but got it with system-test