From 05fb492bd5c7c531bc3e00c40da68d17ba2b41e7 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Tue, 8 Oct 2024 12:12:06 -0500 Subject: [PATCH] refactor(server/v2): clean up storage use and config (#22008) --- runtime/v2/app.go | 20 ----- runtime/v2/builder.go | 27 ++++--- runtime/v2/module.go | 118 +++++++++++------------------- runtime/v2/store.go | 77 ++++++------------- server/v2/cometbft/server.go | 29 ++++++-- server/v2/cometbft/types/store.go | 8 +- server/v2/commands.go | 2 +- server/v2/config_test.go | 7 +- server/v2/server.go | 1 + server/v2/server_test.go | 16 +++- server/v2/store/config.go | 17 ----- server/v2/store/server.go | 55 ++++++++++---- server/v2/store/snapshot.go | 14 ++-- server/v2/types.go | 1 - simapp/v2/app_di.go | 53 +++++++------- simapp/v2/app_test.go | 2 + simapp/v2/simdv2/cmd/commands.go | 9 ++- simapp/v2/simdv2/cmd/root_di.go | 19 ++--- simapp/v2/simdv2/cmd/testnet.go | 15 ++-- store/v2/root/builder.go | 95 ++++++++++++++++++++++++ store/v2/root/config.go | 14 ++++ store/v2/root/factory.go | 2 +- store/v2/store.go | 20 +++-- 23 files changed, 344 insertions(+), 277 deletions(-) delete mode 100644 server/v2/store/config.go create mode 100644 store/v2/root/builder.go create mode 100644 store/v2/root/config.go diff --git a/runtime/v2/app.go b/runtime/v2/app.go index caf55b92ce01..f62795f53f3c 100644 --- a/runtime/v2/app.go +++ b/runtime/v2/app.go @@ -2,8 +2,6 @@ package runtime import ( "encoding/json" - "errors" - "slices" runtimev2 "cosmossdk.io/api/cosmos/app/runtime/v2" appmodulev2 "cosmossdk.io/core/appmodule/v2" @@ -36,8 +34,6 @@ type App[T transaction.Tx] struct { logger log.Logger config *runtimev2.Module - // modules configuration - storeKeys []string interfaceRegistrar registry.InterfaceRegistrar amino registry.AminoRegistrar moduleManager *MM[T] @@ -93,22 +89,6 @@ func (a *App[T]) Close() error { return nil } -// GetStoreKeys returns all the app store keys. -func (a *App[T]) GetStoreKeys() []string { - return a.storeKeys -} - -// UnsafeFindStoreKey fetches a registered StoreKey from the App in linear time. -// NOTE: This should only be used in testing. -func (a *App[T]) UnsafeFindStoreKey(storeKey string) (string, error) { - i := slices.IndexFunc(a.storeKeys, func(s string) bool { return s == storeKey }) - if i == -1 { - return "", errors.New("store key not found") - } - - return a.storeKeys[i], nil -} - // GetStore returns the app store. func (a *App[T]) GetStore() Store { return a.db diff --git a/runtime/v2/builder.go b/runtime/v2/builder.go index b28ddbc741b4..e74d6ff34f22 100644 --- a/runtime/v2/builder.go +++ b/runtime/v2/builder.go @@ -15,13 +15,15 @@ import ( "cosmossdk.io/server/v2/appmanager" "cosmossdk.io/server/v2/stf" "cosmossdk.io/server/v2/stf/branch" + "cosmossdk.io/store/v2/root" ) // AppBuilder is a type that is injected into a container by the runtime/v2 module // (as *AppBuilder) which can be used to create an app which is compatible with // the existing app.go initialization conventions. type AppBuilder[T transaction.Tx] struct { - app *App[T] + app *App[T] + storeBuilder root.Builder // the following fields are used to overwrite the default branch func(state store.ReaderMap) store.WriterMap @@ -62,14 +64,6 @@ func (a *AppBuilder[T]) RegisterModules(modules map[string]appmodulev2.AppModule return nil } -// RegisterStores registers the provided store keys. -// This method should only be used for registering extra stores -// which is necessary for modules that not registered using the app config. -// To be used in combination of RegisterModules. -func (a *AppBuilder[T]) RegisterStores(keys ...string) { - a.app.storeKeys = append(a.app.storeKeys, keys...) -} - // Build builds an *App instance. func (a *AppBuilder[T]) Build(opts ...AppBuilderOption[T]) (*App[T], error) { for _, opt := range opts { @@ -93,8 +87,9 @@ func (a *AppBuilder[T]) Build(opts ...AppBuilderOption[T]) (*App[T], error) { } } + a.app.db = a.storeBuilder.Get() if a.app.db == nil { - return nil, fmt.Errorf("app.db is not set, it is required to build the app") + return nil, fmt.Errorf("storeBuilder did not return a db") } if err := a.app.moduleManager.RegisterServices(a.app); err != nil { @@ -205,7 +200,11 @@ func AppBuilderWithBranch[T transaction.Tx](branch func(state store.ReaderMap) s // AppBuilderWithTxValidator sets the tx validator for the app. // It overrides all default tx validators defined by modules. -func AppBuilderWithTxValidator[T transaction.Tx](txValidators func(ctx context.Context, tx T) error) AppBuilderOption[T] { +func AppBuilderWithTxValidator[T transaction.Tx]( + txValidators func( + ctx context.Context, tx T, + ) error, +) AppBuilderOption[T] { return func(a *AppBuilder[T]) { a.txValidator = txValidators } @@ -213,7 +212,11 @@ func AppBuilderWithTxValidator[T transaction.Tx](txValidators func(ctx context.C // AppBuilderWithPostTxExec sets logic that will be executed after each transaction. // When not provided, a no-op function will be used. -func AppBuilderWithPostTxExec[T transaction.Tx](postTxExec func(ctx context.Context, tx T, success bool) error) AppBuilderOption[T] { +func AppBuilderWithPostTxExec[T transaction.Tx]( + postTxExec func( + ctx context.Context, tx T, success bool, + ) error, +) AppBuilderOption[T] { return func(a *AppBuilder[T]) { a.postTxExec = postTxExec } diff --git a/runtime/v2/module.go b/runtime/v2/module.go index a95e38f96ea5..11cd7037fff3 100644 --- a/runtime/v2/module.go +++ b/runtime/v2/module.go @@ -19,7 +19,6 @@ import ( "cosmossdk.io/core/event" "cosmossdk.io/core/header" "cosmossdk.io/core/registry" - "cosmossdk.io/core/server" "cosmossdk.io/core/store" "cosmossdk.io/core/transaction" "cosmossdk.io/depinject" @@ -27,7 +26,7 @@ import ( "cosmossdk.io/log" "cosmossdk.io/runtime/v2/services" "cosmossdk.io/server/v2/stf" - rootstore "cosmossdk.io/store/v2/root" + "cosmossdk.io/store/v2/root" ) var ( @@ -97,9 +96,9 @@ func init() { appconfig.Register(&runtimev2.Module{}, appconfig.Provide( ProvideAppBuilder[transaction.Tx], - ProvideEnvironment[transaction.Tx], ProvideModuleManager[transaction.Tx], - ProvideStoreBuilder, + ProvideEnvironment, + ProvideKVService, ), appconfig.Invoke(SetupAppBuilder), ) @@ -108,6 +107,7 @@ func init() { func ProvideAppBuilder[T transaction.Tx]( interfaceRegistrar registry.InterfaceRegistrar, amino registry.AminoRegistrar, + storeBuilder root.Builder, ) ( *AppBuilder[T], *stf.MsgRouterBuilder, @@ -127,7 +127,6 @@ func ProvideAppBuilder[T transaction.Tx]( msgRouterBuilder := stf.NewMsgRouterBuilder() app := &App[T]{ - storeKeys: nil, interfaceRegistrar: interfaceRegistrar, amino: amino, msgRouterBuilder: msgRouterBuilder, @@ -135,7 +134,7 @@ func ProvideAppBuilder[T transaction.Tx]( QueryHandlers: map[string]appmodulev2.Handler{}, storeLoader: DefaultStoreLoader, } - appBuilder := &AppBuilder[T]{app: app} + appBuilder := &AppBuilder[T]{app: app, storeBuilder: storeBuilder} return appBuilder, msgRouterBuilder, appModule[T]{app}, protoFiles, protoTypes } @@ -149,12 +148,7 @@ type AppInputs struct { InterfaceRegistrar registry.InterfaceRegistrar LegacyAmino registry.AminoRegistrar Logger log.Logger - // StoreBuilder is a builder for a store/v2 RootStore satisfying the Store interface - StoreBuilder *StoreBuilder - // StoreOptions are required as input for the StoreBuilder. If not provided, the default options are used. - StoreOptions *rootstore.Options `optional:"true"` - // DynamicConfig can be nil in client wiring, but is required in server wiring. - DynamicConfig server.DynamicConfig `optional:"true"` + StoreBuilder root.Builder } func SetupAppBuilder(inputs AppInputs) { @@ -164,24 +158,8 @@ func SetupAppBuilder(inputs AppInputs) { app.moduleManager = inputs.ModuleManager app.moduleManager.RegisterInterfaces(inputs.InterfaceRegistrar) app.moduleManager.RegisterLegacyAminoCodec(inputs.LegacyAmino) - - if inputs.DynamicConfig == nil { - return - } - storeOptions := rootstore.DefaultStoreOptions() - if inputs.StoreOptions != nil { - storeOptions = *inputs.StoreOptions - } - var err error - app.db, err = inputs.StoreBuilder.Build( - inputs.Logger, - app.storeKeys, - inputs.DynamicConfig, - storeOptions, - ) - if err != nil { - panic(err) - } + // STF requires some state to run + inputs.StoreBuilder.RegisterKey("stf") } func ProvideModuleManager[T transaction.Tx]( @@ -192,44 +170,47 @@ func ProvideModuleManager[T transaction.Tx]( return NewModuleManager[T](logger, config, modules) } -// ProvideEnvironment provides the environment for keeper modules, while maintaining backward compatibility and provide services directly as well. -func ProvideEnvironment[T transaction.Tx]( - logger log.Logger, +func ProvideKVService( config *runtimev2.Module, key depinject.ModuleKey, - appBuilder *AppBuilder[T], kvFactory store.KVStoreServiceFactory, - headerService header.Service, - eventService event.Service, -) ( - appmodulev2.Environment, - store.KVStoreService, - store.MemoryStoreService, -) { - var ( - kvService store.KVStoreService = failingStoreService{} - memKvService store.MemoryStoreService = failingStoreService{} - ) - + storeBuilder root.Builder, +) (store.KVStoreService, store.MemoryStoreService) { // skips modules that have no store - if !slices.Contains(config.SkipStoreKeys, key.Name()) { - var kvStoreKey string - storeKeyOverride := storeKeyOverride(config, key.Name()) - if storeKeyOverride != nil { - kvStoreKey = storeKeyOverride.KvStoreKey - } else { - kvStoreKey = key.Name() - } + if slices.Contains(config.SkipStoreKeys, key.Name()) { + return &failingStoreService{}, &failingStoreService{} + } + var kvStoreKey string + override := storeKeyOverride(config, key.Name()) + if override != nil { + kvStoreKey = override.KvStoreKey + } else { + kvStoreKey = key.Name() + } - registerStoreKey(appBuilder, kvStoreKey) - kvService = kvFactory([]byte(kvStoreKey)) + storeBuilder.RegisterKey(kvStoreKey) + return kvFactory([]byte(kvStoreKey)), stf.NewMemoryStoreService([]byte(fmt.Sprintf("memory:%s", kvStoreKey))) +} - memStoreKey := fmt.Sprintf("memory:%s", key.Name()) - registerStoreKey(appBuilder, memStoreKey) - memKvService = stf.NewMemoryStoreService([]byte(memStoreKey)) +func storeKeyOverride(config *runtimev2.Module, moduleName string) *runtimev2.StoreKeyConfig { + for _, cfg := range config.OverrideStoreKeys { + if cfg.ModuleName == moduleName { + return cfg + } } + return nil +} - env := appmodulev2.Environment{ +// ProvideEnvironment provides the environment for keeper modules, while maintaining backward compatibility and provide services directly as well. +func ProvideEnvironment( + logger log.Logger, + key depinject.ModuleKey, + kvService store.KVStoreService, + memKvService store.MemoryStoreService, + headerService header.Service, + eventService event.Service, +) appmodulev2.Environment { + return appmodulev2.Environment{ Logger: logger, BranchService: stf.BranchService{}, EventService: eventService, @@ -241,28 +222,13 @@ func ProvideEnvironment[T transaction.Tx]( KVStoreService: kvService, MemStoreService: memKvService, } - - return env, kvService, memKvService -} - -func registerStoreKey[T transaction.Tx](builder *AppBuilder[T], key string) { - builder.app.storeKeys = append(builder.app.storeKeys, key) -} - -func storeKeyOverride(config *runtimev2.Module, moduleName string) *runtimev2.StoreKeyConfig { - for _, cfg := range config.OverrideStoreKeys { - if cfg.ModuleName == moduleName { - return cfg - } - } - - return nil } // DefaultServiceBindings provides default services for the following service interfaces: // - store.KVStoreServiceFactory // - header.Service // - comet.Service +// - event.Service // // They are all required. For most use cases these default services bindings should be sufficient. // Power users (or tests) may wish to provide their own services bindings, in which case they must diff --git a/runtime/v2/store.go b/runtime/v2/store.go index 6c45dd5e72db..7f7b2135c809 100644 --- a/runtime/v2/store.go +++ b/runtime/v2/store.go @@ -3,18 +3,35 @@ package runtime import ( "errors" "fmt" - "path/filepath" + "sync" - "cosmossdk.io/core/server" "cosmossdk.io/core/store" - "cosmossdk.io/log" "cosmossdk.io/server/v2/stf" storev2 "cosmossdk.io/store/v2" - "cosmossdk.io/store/v2/db" "cosmossdk.io/store/v2/proof" "cosmossdk.io/store/v2/root" ) +var ( + storeBuilderSingleton root.Builder + storeBuilderSingletonOnce sync.Once +) + +// ProvideSingletonScopedStoreBuilder returns a store builder that is a singleton +// in the scope of the process lifetime. +func ProvideSingletonScopedStoreBuilder() root.Builder { + storeBuilderSingletonOnce.Do(func() { + storeBuilderSingleton = root.NewBuilder() + }) + return storeBuilderSingleton +} + +// ResetSingletonScopedStoreBuilder resets the singleton store builder. Applications +// should not ever need to call this, but it may be useful in tests. +func ResetSingletonScopedStoreBuilder() { + storeBuilderSingletonOnce = sync.Once{} +} + // NewKVStoreService creates a new KVStoreService. // This wrapper is kept for backwards compatibility. // When migrating from runtime to runtime/v2, use runtimev2.NewKVStoreService(storeKey.Name()) instead of runtime.NewKVStoreService(storeKey). @@ -64,58 +81,6 @@ type Store interface { LastCommitID() (proof.CommitID, error) } -// StoreBuilder is a builder for a store/v2 RootStore satisfying the Store interface. -type StoreBuilder struct { - store Store -} - -// Build creates a new store/v2 RootStore. -func (sb *StoreBuilder) Build( - logger log.Logger, - storeKeys []string, - config server.DynamicConfig, - options root.Options, -) (Store, error) { - if sb.store != nil { - return sb.store, nil - } - home := config.GetString(flagHome) - scRawDb, err := db.NewDB( - db.DBType(config.GetString("store.app-db-backend")), - "application", - filepath.Join(home, "data"), - nil, - ) - if err != nil { - return nil, fmt.Errorf("failed to create SCRawDB: %w", err) - } - - factoryOptions := &root.FactoryOptions{ - Logger: logger, - RootDir: home, - Options: options, - // STF needs to store a bit of state - StoreKeys: append(storeKeys, "stf"), - SCRawDB: scRawDb, - } - - rs, err := root.CreateRootStore(factoryOptions) - if err != nil { - return nil, fmt.Errorf("failed to create root store: %w", err) - } - sb.store = rs - return sb.store, nil -} - -// Get returns the Store. Build must be called before calling Get or the result will be nil. -func (sb *StoreBuilder) Get() Store { - return sb.store -} - -func ProvideStoreBuilder() *StoreBuilder { - return &StoreBuilder{} -} - // StoreLoader allows for custom loading of the store, this is useful when upgrading the store from a previous version type StoreLoader func(store Store) error diff --git a/server/v2/cometbft/server.go b/server/v2/cometbft/server.go index b15f5695cf94..7fa1a94d8ee6 100644 --- a/server/v2/cometbft/server.go +++ b/server/v2/cometbft/server.go @@ -8,6 +8,9 @@ import ( "os" "path/filepath" + "cosmossdk.io/server/v2/store" + "cosmossdk.io/store/v2/root" + abciserver "github.com/cometbft/cometbft/abci/server" cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" cmtcfg "github.com/cometbft/cometbft/config" @@ -23,7 +26,6 @@ import ( serverv2 "cosmossdk.io/server/v2" cometlog "cosmossdk.io/server/v2/cometbft/log" "cosmossdk.io/server/v2/cometbft/mempool" - "cosmossdk.io/server/v2/cometbft/types" "cosmossdk.io/store/v2/snapshots" genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types" @@ -43,14 +45,21 @@ type CometBFTServer[T transaction.Tx] struct { initTxCodec transaction.Codec[T] logger log.Logger + storeBuilder root.Builder serverOptions ServerOptions[T] config Config cfgOptions []CfgOption } -func New[T transaction.Tx](txCodec transaction.Codec[T], serverOptions ServerOptions[T], cfgOptions ...CfgOption) *CometBFTServer[T] { +func New[T transaction.Tx]( + txCodec transaction.Codec[T], + storeBuilder root.Builder, + serverOptions ServerOptions[T], + cfgOptions ...CfgOption, +) *CometBFTServer[T] { return &CometBFTServer[T]{ initTxCodec: txCodec, + storeBuilder: storeBuilder, serverOptions: serverOptions, cfgOptions: cfgOptions, } @@ -97,8 +106,16 @@ func (s *CometBFTServer[T]) Init(appI serverv2.AppI[T], cfg map[string]any, logg indexEvents[e] = struct{}{} } + storeCfg, err := store.UnmarshalConfig(cfg) + if err != nil { + return err + } + rs, err := s.storeBuilder.Build(logger, storeCfg) + if err != nil { + return err + } + s.logger = logger.With(log.ModuleKey, s.Name()) - store := appI.GetStore().(types.Store) consensus := NewConsensus( s.logger, appI.Name(), @@ -106,7 +123,7 @@ func (s *CometBFTServer[T]) Init(appI serverv2.AppI[T], cfg map[string]any, logg s.serverOptions.Mempool(cfg), indexEvents, appI.GetQueryHandlers(), - store, + rs, s.config, s.initTxCodec, chainID, @@ -119,8 +136,8 @@ func (s *CometBFTServer[T]) Init(appI serverv2.AppI[T], cfg map[string]any, logg consensus.addrPeerFilter = s.serverOptions.AddrPeerFilter consensus.idPeerFilter = s.serverOptions.IdPeerFilter - ss := store.GetStateStorage().(snapshots.StorageSnapshotter) - sc := store.GetStateCommitment().(snapshots.CommitSnapshotter) + ss := rs.GetStateStorage().(snapshots.StorageSnapshotter) + sc := rs.GetStateCommitment().(snapshots.CommitSnapshotter) snapshotStore, err := GetSnapshotStore(s.config.ConfigTomlConfig.RootDir) if err != nil { diff --git a/server/v2/cometbft/types/store.go b/server/v2/cometbft/types/store.go index ff06163af5ab..3e603ff18ced 100644 --- a/server/v2/cometbft/types/store.go +++ b/server/v2/cometbft/types/store.go @@ -7,6 +7,8 @@ import ( ) type Store interface { + storev2.Backend + // GetLatestVersion returns the latest version that consensus has been made on GetLatestVersion() (uint64, error) // StateLatest returns a readonly view over the latest @@ -30,10 +32,4 @@ type Store interface { // LastCommitID returns a CommitID pertaining to the last commitment. LastCommitID() (proof.CommitID, error) - - // GetStateStorage returns the SS backend. - GetStateStorage() storev2.VersionedDatabase - - // GetStateCommitment returns the SC backend. - GetStateCommitment() storev2.Committer } diff --git a/server/v2/commands.go b/server/v2/commands.go index da8e0ec6e537..0357fe180d93 100644 --- a/server/v2/commands.go +++ b/server/v2/commands.go @@ -33,7 +33,7 @@ func Execute(rootCmd *cobra.Command, envPrefix, defaultHome string) error { } // AddCommands add the server commands to the root command -// It configure the config handling and the logger handling +// It configures the config handling and the logger handling func AddCommands[T transaction.Tx]( rootCmd *cobra.Command, newApp AppCreator[T], diff --git a/server/v2/config_test.go b/server/v2/config_test.go index f69e2ed56769..3ddd7893a001 100644 --- a/server/v2/config_test.go +++ b/server/v2/config_test.go @@ -10,6 +10,7 @@ import ( serverv2 "cosmossdk.io/server/v2" grpc "cosmossdk.io/server/v2/api/grpc" store "cosmossdk.io/server/v2/store" + "cosmossdk.io/store/v2/root" ) func TestReadConfig(t *testing.T) { @@ -21,7 +22,7 @@ func TestReadConfig(t *testing.T) { require.NoError(t, err) require.Equal(t, v.GetString(grpc.FlagAddress), grpc.DefaultConfig().Address) - require.Equal(t, v.GetString(store.FlagAppDBBackend), store.DefaultConfig().AppDBBackend) + require.Equal(t, v.GetString(store.FlagAppDBBackend), root.DefaultConfig().AppDBBackend) } func TestUnmarshalSubConfig(t *testing.T) { @@ -40,8 +41,8 @@ func TestUnmarshalSubConfig(t *testing.T) { require.True(t, grpc.DefaultConfig().Enable) require.False(t, grpcConfig.Enable) - storeConfig := store.Config{} + storeConfig := root.Config{} err = serverv2.UnmarshalSubConfig(cfg, "store", &storeConfig) require.NoError(t, err) - require.Equal(t, *store.DefaultConfig(), storeConfig) + require.Equal(t, *root.DefaultConfig(), storeConfig) } diff --git a/server/v2/server.go b/server/v2/server.go index 7ee91e649982..a628fbdf1000 100644 --- a/server/v2/server.go +++ b/server/v2/server.go @@ -61,6 +61,7 @@ const ( var _ ServerComponent[transaction.Tx] = (*Server[transaction.Tx])(nil) +// Server is the top-level server component which contains all other server components. type Server[T transaction.Tx] struct { logger log.Logger components []ServerComponent[T] diff --git a/server/v2/server_test.go b/server/v2/server_test.go index 97afa40fdaa9..c216824fb652 100644 --- a/server/v2/server_test.go +++ b/server/v2/server_test.go @@ -19,6 +19,8 @@ import ( grpc "cosmossdk.io/server/v2/api/grpc" "cosmossdk.io/server/v2/appmanager" "cosmossdk.io/server/v2/store" + storev2 "cosmossdk.io/store/v2" + "cosmossdk.io/store/v2/root" ) type mockInterfaceRegistry struct{} @@ -48,6 +50,18 @@ func (*mockApp[T]) InterfaceRegistry() coreserver.InterfaceRegistry { return &mockInterfaceRegistry{} } +var _ root.Builder = &mockStoreBuilder{} + +type mockStoreBuilder struct{} + +func (m mockStoreBuilder) Build(logger log.Logger, config *root.Config) (storev2.RootStore, error) { + return nil, nil +} + +func (m mockStoreBuilder) RegisterKey(string) {} + +func (m mockStoreBuilder) Get() storev2.RootStore { return nil } + func TestServer(t *testing.T) { currentDir, err := os.Getwd() require.NoError(t, err) @@ -64,7 +78,7 @@ func TestServer(t *testing.T) { err = grpcServer.Init(&mockApp[transaction.Tx]{}, cfg, logger) require.NoError(t, err) - storeServer := store.New[transaction.Tx](nil /* nil appCreator as not using CLI commands */) + storeServer := store.New[transaction.Tx](&mockStoreBuilder{}) err = storeServer.Init(&mockApp[transaction.Tx]{}, cfg, logger) require.NoError(t, err) diff --git a/server/v2/store/config.go b/server/v2/store/config.go deleted file mode 100644 index 457bfa9383cb..000000000000 --- a/server/v2/store/config.go +++ /dev/null @@ -1,17 +0,0 @@ -package store - -import ( - "cosmossdk.io/store/v2/root" -) - -func DefaultConfig() *Config { - return &Config{ - AppDBBackend: "goleveldb", - Options: root.DefaultStoreOptions(), - } -} - -type Config struct { - AppDBBackend string `mapstructure:"app-db-backend" toml:"app-db-backend" comment:"The type of database for application and snapshots databases."` - Options root.Options `mapstructure:"options" toml:"options"` -} diff --git a/server/v2/store/server.go b/server/v2/store/server.go index c50a53dc6e24..a793a1912dba 100644 --- a/server/v2/store/server.go +++ b/server/v2/store/server.go @@ -9,6 +9,8 @@ import ( "cosmossdk.io/core/transaction" "cosmossdk.io/log" serverv2 "cosmossdk.io/server/v2" + storev2 "cosmossdk.io/store/v2" + "cosmossdk.io/store/v2/root" ) var ( @@ -21,24 +23,26 @@ const ServerName = "store" // Server manages store config and contains prune & snapshot commands type Server[T transaction.Tx] struct { - config *Config - // saving appCreator for only RestoreSnapshotCmd - appCreator serverv2.AppCreator[T] + config *root.Config + builder root.Builder + backend storev2.Backend } -func New[T transaction.Tx](appCreator serverv2.AppCreator[T]) *Server[T] { - return &Server[T]{appCreator: appCreator} +func New[T transaction.Tx](builder root.Builder) *Server[T] { + return &Server[T]{builder: builder} } -func (s *Server[T]) Init(appI serverv2.AppI[T], cfg map[string]any, logger log.Logger) error { - serverCfg := s.Config().(*Config) - if len(cfg) > 0 { - if err := serverv2.UnmarshalSubConfig(cfg, s.Name(), &serverCfg); err != nil { - return fmt.Errorf("failed to unmarshal config: %w", err) - } +func (s *Server[T]) Init(_ serverv2.AppI[T], cfg map[string]any, log log.Logger) error { + var err error + s.config, err = UnmarshalConfig(cfg) + if err != nil { + return fmt.Errorf("failed to unmarshal config: %w", err) + } + s.backend, err = s.builder.Build(log, s.config) + if err != nil { + return fmt.Errorf("failed to create store backend: %w", err) } - s.config = serverCfg return nil } @@ -46,11 +50,11 @@ func (s *Server[T]) Name() string { return ServerName } -func (s *Server[T]) Start(ctx context.Context) error { +func (s *Server[T]) Start(context.Context) error { return nil } -func (s *Server[T]) Stop(ctx context.Context) error { +func (s *Server[T]) Stop(context.Context) error { return nil } @@ -63,15 +67,34 @@ func (s *Server[T]) CLICommands() serverv2.CLIConfig { s.ListSnapshotsCmd(), s.DumpArchiveCmd(), s.LoadArchiveCmd(), - s.RestoreSnapshotCmd(s.appCreator), + s.RestoreSnapshotCmd(s.backend), }, } } func (s *Server[T]) Config() any { if s.config == nil || s.config.AppDBBackend == "" { - return DefaultConfig() + return root.DefaultConfig() } return s.config } + +// UnmarshalConfig unmarshals the store config from the given map. +// If the config is not found in the map, the default config is returned. +// If the home directory is found in the map, it sets the home directory in the config. +// An empty home directory *is* permitted at this stage, but attempting to build +// the store with an empty home directory will fail. +func UnmarshalConfig(cfg map[string]any) (*root.Config, error) { + config := &root.Config{ + Options: root.DefaultStoreOptions(), + } + if err := serverv2.UnmarshalSubConfig(cfg, ServerName, config); err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) + } + home := cfg[serverv2.FlagHome] + if home != nil { + config.Home = home.(string) + } + return config, nil +} diff --git a/server/v2/store/snapshot.go b/server/v2/store/snapshot.go index b804e34b71c8..e7958068e56a 100644 --- a/server/v2/store/snapshot.go +++ b/server/v2/store/snapshot.go @@ -76,7 +76,7 @@ func (s *Server[T]) ExportSnapshotCmd() *cobra.Command { } // RestoreSnapshotCmd returns a command to restore a snapshot -func (s *Server[T]) RestoreSnapshotCmd(newApp serverv2.AppCreator[T]) *cobra.Command { +func (s *Server[T]) RestoreSnapshotCmd(rootStore storev2.Backend) *cobra.Command { cmd := &cobra.Command{ Use: "restore ", Short: "Restore app state from local snapshot", @@ -95,8 +95,6 @@ func (s *Server[T]) RestoreSnapshotCmd(newApp serverv2.AppCreator[T]) *cobra.Com } logger := log.NewLogger(cmd.OutOrStdout()) - app := newApp(logger, v) - rootStore := app.GetStore().(storev2.RootStore) sm, err := createSnapshotsManager(cmd, v, logger, rootStore) if err != nil { @@ -350,7 +348,9 @@ func (s *Server[T]) LoadArchiveCmd() *cobra.Command { } } -func createSnapshotsManager(cmd *cobra.Command, v *viper.Viper, logger log.Logger, store storev2.RootStore) (*snapshots.Manager, error) { +func createSnapshotsManager( + cmd *cobra.Command, v *viper.Viper, logger log.Logger, store storev2.Backend, +) (*snapshots.Manager, error) { home := v.GetString(serverv2.FlagHome) snapshotStore, err := snapshots.NewStore(filepath.Join(home, "data", "snapshots")) if err != nil { @@ -371,7 +371,11 @@ func createSnapshotsManager(cmd *cobra.Command, v *viper.Viper, logger log.Logge } } - sm := snapshots.NewManager(snapshotStore, snapshots.NewSnapshotOptions(interval, uint32(keepRecent)), store.GetStateCommitment().(snapshots.CommitSnapshotter), store.GetStateStorage().(snapshots.StorageSnapshotter), nil, logger) + sm := snapshots.NewManager( + snapshotStore, snapshots.NewSnapshotOptions(interval, uint32(keepRecent)), + store.GetStateCommitment().(snapshots.CommitSnapshotter), + store.GetStateStorage().(snapshots.StorageSnapshotter), + nil, logger) return sm, nil } diff --git a/server/v2/types.go b/server/v2/types.go index 7cd15fd10307..a28385922178 100644 --- a/server/v2/types.go +++ b/server/v2/types.go @@ -17,5 +17,4 @@ type AppI[T transaction.Tx] interface { InterfaceRegistry() server.InterfaceRegistry GetAppManager() *appmanager.AppManager[T] GetQueryHandlers() map[string]appmodulev2.Handler - GetStore() any } diff --git a/simapp/v2/app_di.go b/simapp/v2/app_di.go index 43ca45f99aed..d66b17ddb8ca 100644 --- a/simapp/v2/app_di.go +++ b/simapp/v2/app_di.go @@ -12,6 +12,7 @@ import ( "cosmossdk.io/depinject" "cosmossdk.io/log" "cosmossdk.io/runtime/v2" + serverstore "cosmossdk.io/server/v2/store" "cosmossdk.io/store/v2/root" basedepinject "cosmossdk.io/x/accounts/defaults/base/depinject" lockupdepinject "cosmossdk.io/x/accounts/defaults/lockup/depinject" @@ -55,6 +56,18 @@ func init() { func AppConfig() depinject.Config { return depinject.Configs( appConfig, // Alternatively use appconfig.LoadYAML(AppConfigYAML) + runtime.DefaultServiceBindings(), + depinject.Provide( + codec.ProvideInterfaceRegistry, + codec.ProvideAddressCodec, + codec.ProvideProtoCodec, + codec.ProvideLegacyAmino, + runtime.ProvideSingletonScopedStoreBuilder, + ), + depinject.Invoke( + std.RegisterInterfaces, + std.RegisterLegacyAminoCodec, + ), ) } @@ -64,14 +77,14 @@ func NewSimApp[T transaction.Tx]( viper *viper.Viper, ) *SimApp[T] { var ( - app = &SimApp[T]{} - appBuilder *runtime.AppBuilder[T] - err error + app = &SimApp[T]{} + appBuilder *runtime.AppBuilder[T] + storeBuilder root.Builder + err error // merge the AppConfig and other configuration in one config appConfig = depinject.Configs( AppConfig(), - runtime.DefaultServiceBindings(), depinject.Supply( logger, viper, @@ -117,10 +130,6 @@ func NewSimApp[T transaction.Tx]( // interface. ), depinject.Provide( - codec.ProvideInterfaceRegistry, - codec.ProvideAddressCodec, - codec.ProvideProtoCodec, - codec.ProvideLegacyAmino, // inject desired account types: multisigdepinject.ProvideAccount, basedepinject.ProvideAccount, @@ -141,27 +150,11 @@ func NewSimApp[T transaction.Tx]( // } // }) ), - depinject.Invoke( - std.RegisterInterfaces, - std.RegisterLegacyAminoCodec, - ), ) ) - // the subsection of config that contains the store options (in app.toml [store.options] header) - // is unmarshaled into a store.Options struct and passed to the store builder. - // future work may move this specification and retrieval into store/v2. - // If these options are not specified then default values will be used. - if sub := viper.Sub("store.options"); sub != nil { - storeOptions := &root.Options{} - err := sub.Unmarshal(storeOptions) - if err != nil { - panic(err) - } - appConfig = depinject.Configs(appConfig, depinject.Supply(storeOptions)) - } - if err := depinject.Inject(appConfig, + &storeBuilder, &appBuilder, &app.appCodec, &app.legacyAmino, @@ -172,6 +165,16 @@ func NewSimApp[T transaction.Tx]( panic(err) } + // store/v2 follows a slightly more eager config life cycle than server components + storeConfig, err := serverstore.UnmarshalConfig(viper.AllSettings()) + if err != nil { + panic(err) + } + _, err = storeBuilder.Build(logger, storeConfig) + if err != nil { + panic(err) + } + app.App, err = appBuilder.Build() if err != nil { panic(err) diff --git a/simapp/v2/app_test.go b/simapp/v2/app_test.go index f634d2d67664..6ee3709ee552 100644 --- a/simapp/v2/app_test.go +++ b/simapp/v2/app_test.go @@ -18,6 +18,7 @@ import ( "cosmossdk.io/core/transaction" "cosmossdk.io/log" sdkmath "cosmossdk.io/math" + "cosmossdk.io/runtime/v2" serverv2 "cosmossdk.io/server/v2" comettypes "cosmossdk.io/server/v2/cometbft/types" serverv2store "cosmossdk.io/server/v2/store" @@ -40,6 +41,7 @@ func NewTestApp(t *testing.T) (*SimApp[transaction.Tx], context.Context) { vp.Set(serverv2store.FlagAppDBBackend, string(db.DBTypeGoLevelDB)) vp.Set(serverv2.FlagHome, t.TempDir()) + runtime.ResetSingletonScopedStoreBuilder() app := NewSimApp[transaction.Tx](logger, vp) genesis := app.ModuleManager().DefaultGenesis() diff --git a/simapp/v2/simdv2/cmd/commands.go b/simapp/v2/simdv2/cmd/commands.go index addc7eeb768b..6cc233cc9b19 100644 --- a/simapp/v2/simdv2/cmd/commands.go +++ b/simapp/v2/simdv2/cmd/commands.go @@ -17,8 +17,9 @@ import ( "cosmossdk.io/server/v2/api/grpc" "cosmossdk.io/server/v2/api/telemetry" "cosmossdk.io/server/v2/cometbft" - "cosmossdk.io/server/v2/store" + serverstore "cosmossdk.io/server/v2/store" "cosmossdk.io/simapp/v2" + "cosmossdk.io/store/v2/root" confixcmd "cosmossdk.io/tools/confix/cmd" "github.com/cosmos/cosmos-sdk/client" @@ -43,6 +44,7 @@ func newApp[T transaction.Tx](logger log.Logger, viper *viper.Viper) serverv2.Ap func initRootCmd[T transaction.Tx]( rootCmd *cobra.Command, txConfig client.TxConfig, + storeBuilder root.Builder, moduleManager *runtimev2.MM[T], ) { cfg := sdk.GetConfig() @@ -52,7 +54,7 @@ func initRootCmd[T transaction.Tx]( genutilcli.InitCmd(moduleManager), debug.Cmd(), confixcmd.ConfigCommand(), - NewTestnetCmd(moduleManager), + NewTestnetCmd(storeBuilder, moduleManager), ) logger, err := serverv2.NewLogger(viper.New(), rootCmd.OutOrStdout()) @@ -77,11 +79,12 @@ func initRootCmd[T transaction.Tx]( initServerConfig(), cometbft.New( &genericTxDecoder[T]{txConfig}, + storeBuilder, initCometOptions[T](), initCometConfig(), ), grpc.New[T](), - store.New[T](newApp), + serverstore.New[T](storeBuilder), telemetry.New[T](), ); err != nil { panic(err) diff --git a/simapp/v2/simdv2/cmd/root_di.go b/simapp/v2/simdv2/cmd/root_di.go index 1ad834b53a5a..30291b4ecdb3 100644 --- a/simapp/v2/simdv2/cmd/root_di.go +++ b/simapp/v2/simdv2/cmd/root_di.go @@ -14,13 +14,13 @@ import ( "cosmossdk.io/log" "cosmossdk.io/runtime/v2" "cosmossdk.io/simapp/v2" + "cosmossdk.io/store/v2/root" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/config" nodeservice "github.com/cosmos/cosmos-sdk/client/grpc/node" "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" - "github.com/cosmos/cosmos-sdk/std" "github.com/cosmos/cosmos-sdk/x/auth/tx" authtxconfig "github.com/cosmos/cosmos-sdk/x/auth/tx/config" "github.com/cosmos/cosmos-sdk/x/auth/types" @@ -32,25 +32,16 @@ func NewRootCmd[T transaction.Tx]() *cobra.Command { autoCliOpts autocli.AppOptions moduleManager *runtime.MM[T] clientCtx client.Context + storeBuilder root.Builder ) if err := depinject.Inject( depinject.Configs( simapp.AppConfig(), - runtime.DefaultServiceBindings(), + depinject.Provide(ProvideClientContext), depinject.Supply(log.NewNopLogger()), - depinject.Provide( - codec.ProvideInterfaceRegistry, - codec.ProvideAddressCodec, - codec.ProvideProtoCodec, - codec.ProvideLegacyAmino, - ProvideClientContext, - ), - depinject.Invoke( - std.RegisterInterfaces, - std.RegisterLegacyAminoCodec, - ), ), + &storeBuilder, &autoCliOpts, &moduleManager, &clientCtx, @@ -83,7 +74,7 @@ func NewRootCmd[T transaction.Tx]() *cobra.Command { }, } - initRootCmd(rootCmd, clientCtx.TxConfig, moduleManager) + initRootCmd(rootCmd, clientCtx.TxConfig, storeBuilder, moduleManager) nodeCmds := nodeservice.NewNodeCommands() autoCliOpts.ModuleOptions = make(map[string]*autocliv1.ModuleOptions) diff --git a/simapp/v2/simdv2/cmd/testnet.go b/simapp/v2/simdv2/cmd/testnet.go index ae01310f9151..4e24f7083654 100644 --- a/simapp/v2/simdv2/cmd/testnet.go +++ b/simapp/v2/simdv2/cmd/testnet.go @@ -23,6 +23,7 @@ import ( "cosmossdk.io/server/v2/api/grpc" "cosmossdk.io/server/v2/cometbft" "cosmossdk.io/server/v2/store" + "cosmossdk.io/store/v2/root" banktypes "cosmossdk.io/x/bank/types" bankv2types "cosmossdk.io/x/bank/v2/types" stakingtypes "cosmossdk.io/x/staking/types" @@ -87,7 +88,7 @@ func addTestnetFlagsToCmd(cmd *cobra.Command) { // NewTestnetCmd creates a root testnet command with subcommands to run an in-process testnet or initialize // validator configuration files for running a multi-validator testnet in a separate process -func NewTestnetCmd[T transaction.Tx](mm *runtimev2.MM[T]) *cobra.Command { +func NewTestnetCmd[T transaction.Tx](sb root.Builder, mm *runtimev2.MM[T]) *cobra.Command { testnetCmd := &cobra.Command{ Use: "testnet", Short: "subcommands for starting or configuring local testnets", @@ -96,13 +97,13 @@ func NewTestnetCmd[T transaction.Tx](mm *runtimev2.MM[T]) *cobra.Command { RunE: client.ValidateCmd, } - testnetCmd.AddCommand(testnetInitFilesCmd(mm)) + testnetCmd.AddCommand(testnetInitFilesCmd(sb, mm)) return testnetCmd } // testnetInitFilesCmd returns a cmd to initialize all files for CometBFT testnet and application -func testnetInitFilesCmd[T transaction.Tx](mm *runtimev2.MM[T]) *cobra.Command { +func testnetInitFilesCmd[T transaction.Tx](sb root.Builder, mm *runtimev2.MM[T]) *cobra.Command { cmd := &cobra.Command{ Use: "init-files", Short: "Initialize config directories & files for a multi-validator testnet running locally via separate processes (e.g. Docker Compose or similar)", @@ -143,7 +144,7 @@ Example: return err } - return initTestnetFiles(clientCtx, cmd, config, mm, args) + return initTestnetFiles(clientCtx, sb, cmd, config, mm, args) }, } @@ -165,6 +166,7 @@ const nodeDirPerm = 0o755 // initTestnetFiles initializes testnet files for a testnet to be run in a separate process func initTestnetFiles[T transaction.Tx]( clientCtx client.Context, + sb root.Builder, cmd *cobra.Command, nodeConfig *cmtconfig.Config, mm *runtimev2.MM[T], @@ -339,12 +341,13 @@ func initTestnetFiles[T transaction.Tx]( // Write server config cometServer := cometbft.New[T]( &genericTxDecoder[T]{clientCtx.TxConfig}, + sb, cometbft.ServerOptions[T]{}, cometbft.OverwriteDefaultConfigTomlConfig(nodeConfig), ) - storeServer := store.New[T](newApp) + storeServer := store.New[T](sb) grpcServer := grpc.New[T](grpc.OverwriteDefaultConfig(grpcConfig)) - server := serverv2.NewServer(log.NewNopLogger(), serverCfg, cometServer, grpcServer, storeServer) + server := serverv2.NewServer[T](log.NewNopLogger(), serverCfg, cometServer, grpcServer, storeServer) err = server.WriteConfig(filepath.Join(nodeDir, "config")) if err != nil { return err diff --git a/store/v2/root/builder.go b/store/v2/root/builder.go new file mode 100644 index 000000000000..885c41d24484 --- /dev/null +++ b/store/v2/root/builder.go @@ -0,0 +1,95 @@ +package root + +import ( + "fmt" + "path/filepath" + + "cosmossdk.io/log" + "cosmossdk.io/store/v2" + "cosmossdk.io/store/v2/db" +) + +// Builder is the interface for a store/v2 RootStore builder. +// RootStores built by the Cosmos SDK typically involve a 2 phase initialization: +// 1. Namespace registration +// 2. Configuration and loading +// +// The Builder interface is used to facilitate this pattern. Namespaces (store keys) are registered +// by calling RegisterKey before Build is called. Build is then called with a Config +// object and a RootStore is returned. Calls to Get may return the `RootStore` if Build +// was successful, but that's left up to the implementation. +type Builder interface { + // Build creates a new store/v2 RootStore from the given Config. + Build(log.Logger, *Config) (store.RootStore, error) + // RegisterKey registers a store key (namespace) to be used when building the RootStore. + RegisterKey(string) + // Get returns the Store. Build should be called before calling Get or the result will be nil. + Get() store.RootStore +} + +var _ Builder = (*builder)(nil) + +// builder is the default builder for a store/v2 RootStore satisfying the Store interface. +// Tangibly it combines store key registration and a top-level Config to create a RootStore by calling +// the CreateRootStore factory function. +type builder struct { + // input + storeKeys map[string]struct{} + + // output + store store.RootStore +} + +func NewBuilder() Builder { + return &builder{storeKeys: make(map[string]struct{})} +} + +// Build creates a new store/v2 RootStore. +func (sb *builder) Build( + logger log.Logger, + config *Config, +) (store.RootStore, error) { + if sb.store != nil { + return sb.store, nil + } + if config.Home == "" { + return nil, fmt.Errorf("home directory is required") + } + scRawDb, err := db.NewDB( + db.DBType(config.AppDBBackend), + "application", + filepath.Join(config.Home, "data"), + nil, + ) + if err != nil { + return nil, fmt.Errorf("failed to create SCRawDB: %w", err) + } + + var storeKeys []string + for key := range sb.storeKeys { + storeKeys = append(storeKeys, key) + } + + factoryOptions := &FactoryOptions{ + Logger: logger, + RootDir: config.Home, + Options: config.Options, + StoreKeys: storeKeys, + SCRawDB: scRawDb, + } + + rs, err := CreateRootStore(factoryOptions) + if err != nil { + return nil, fmt.Errorf("failed to create root store: %w", err) + } + sb.store = rs + return sb.store, nil +} + +func (sb *builder) Get() store.RootStore { + return sb.store +} + +func (sb *builder) RegisterKey(key string) { + sb.storeKeys[key] = struct{}{} +} diff --git a/store/v2/root/config.go b/store/v2/root/config.go new file mode 100644 index 000000000000..28f46a3879b7 --- /dev/null +++ b/store/v2/root/config.go @@ -0,0 +1,14 @@ +package root + +func DefaultConfig() *Config { + return &Config{ + AppDBBackend: "goleveldb", + Options: DefaultStoreOptions(), + } +} + +type Config struct { + Home string `toml:"-"` // this field is omitted in the TOML file + AppDBBackend string `mapstructure:"app-db-backend" toml:"app-db-backend" comment:"The type of database for application and snapshots databases."` + Options Options `mapstructure:"options" toml:"options"` +} diff --git a/store/v2/root/factory.go b/store/v2/root/factory.go index e6f86917ab0d..9e1f76a36ceb 100644 --- a/store/v2/root/factory.go +++ b/store/v2/root/factory.go @@ -33,7 +33,7 @@ const ( SCTypeIavlV2 SCType = "iavl-v2" ) -// app.toml config options +// Options are the options for creating a root store. type Options struct { SSType SSType `mapstructure:"ss-type" toml:"ss-type" comment:"SState storage database type. Currently we support: \"sqlite\", \"pebble\" and \"rocksdb\""` SCType SCType `mapstructure:"sc-type" toml:"sc-type" comment:"State commitment database type. Currently we support: \"iavl\" and \"iavl-v2\""` diff --git a/store/v2/store.go b/store/v2/store.go index fee3ad39dbf2..1adf44f0b89b 100644 --- a/store/v2/store.go +++ b/store/v2/store.go @@ -12,6 +12,9 @@ import ( // RootStore defines an abstraction layer containing a State Storage (SS) engine // and one or more State Commitment (SC) engines. type RootStore interface { + Pruner + Backend + // StateLatest returns a read-only version of the RootStore at the latest // height, alongside the associated version. StateLatest() (uint64, corestore.ReaderMap, error) @@ -21,12 +24,6 @@ type RootStore interface { // an error must be returned. StateAt(version uint64) (corestore.ReaderMap, error) - // GetStateStorage returns the SS backend. - GetStateStorage() VersionedDatabase - - // GetStateCommitment returns the SC backend. - GetStateCommitment() Committer - // Query performs a query on the RootStore for a given store key, version (height), // and key tuple. Queries should be routed to the underlying SS engine. Query(storeKey []byte, version uint64, key []byte, prove bool) (QueryResult, error) @@ -67,11 +64,18 @@ type RootStore interface { // SetMetrics sets the telemetry handler on the RootStore. SetMetrics(m metrics.Metrics) - Prune(version uint64) error - io.Closer } +// Backend defines the interface for the RootStore backends. +type Backend interface { + // GetStateStorage returns the SS backend. + GetStateStorage() VersionedDatabase + + // GetStateCommitment returns the SC backend. + GetStateCommitment() Committer +} + // UpgradeableStore defines the interface for upgrading store keys. type UpgradeableStore interface { // LoadVersionAndUpgrade behaves identically to LoadVersion except it also