Merge branch 'main' into eze/consensus_docs_fix

cosmos · Oct 4, 2024 · 3122409 · 3122409
2 parents 7635350 + dc14960
commit 3122409
Show file tree

Hide file tree

Showing 18 changed files with 156 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -52,6 +52,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 
 ### Improvements
 
+* (crypto/ledger) [#22116](https://github.com/cosmos/cosmos-sdk/pull/22116) Improve error message when deriving paths using index >100
 * (sims) [#21613](https://github.com/cosmos/cosmos-sdk/pull/21613) Add sims2 framework and factory methods for simpler message factories in modules
 * (modules) [#21963](https://github.com/cosmos/cosmos-sdk/pull/21963) Duplicatable metrics are no more collected in modules. They were unecessary overhead.
 

diff --git a/collections/go.mod b/collections/go.mod
@@ -3,7 +3,7 @@ module cosmossdk.io/collections
 go 1.23
 
 require (
-	cosmossdk.io/core v1.0.0-alpha.3
+	cosmossdk.io/core v1.0.0-alpha.4
 	cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29
 	cosmossdk.io/schema v0.3.0
 	github.com/stretchr/testify v1.9.0

diff --git a/collections/go.sum b/collections/go.sum
@@ -1,5 +1,5 @@
-cosmossdk.io/core v1.0.0-alpha.3 h1:pnxaYAas7llXgVz1lM7X6De74nWrhNKnB3yMKe4OUUA=
-cosmossdk.io/core v1.0.0-alpha.3/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
+cosmossdk.io/core v1.0.0-alpha.4 h1:9iuroT9ejDYETCsGkzkvs/wAY/5UFl7nCIINFRxyMJY=
+cosmossdk.io/core v1.0.0-alpha.4/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
 cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29 h1:NxxUo0GMJUbIuVg0R70e3cbn9eFTEuMr7ev1AFvypdY=
 cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29/go.mod h1:8s2tPeJtSiQuoyPmr2Ag7meikonISO4Fv4MoO8+ORrs=
 cosmossdk.io/schema v0.3.0 h1:01lcaM4trhzZ1HQTfTV8z6Ma1GziOZ/YmdzBN3F720c=

diff --git a/crypto/ledger/ledger_secp256k1.go b/crypto/ledger/ledger_secp256k1.go
@@ -341,6 +341,12 @@ func getPubKeyUnsafe(device SECP256K1, path hd.BIP44Params) (types.PubKey, error
 func getPubKeyAddrSafe(device SECP256K1, path hd.BIP44Params, hrp string) (types.PubKey, string, error) {
 	publicKey, addr, err := device.GetAddressPubKeySECP256K1(path.DerivationPath(), hrp)
 	if err != nil {
+		// Check special case if user is trying to use an index > 100
+		if path.AddressIndex > 100 {
+			return nil, "", fmt.Errorf("%w: cannot derive paths where index > 100: %s "+
+				"This is a security measure to avoid very hard to find derivation paths introduced by a possible attacker. "+
+				"You can disable this by setting expert mode in your ledger device. Do this at your own risk", err, path)
+		}
 		return nil, "", fmt.Errorf("%w: address rejected for path %s", err, path)
 	}
 

diff --git a/server/cmt_cmds.go b/server/cmt_cmds.go
@@ -368,7 +368,7 @@ func BootstrapStateCmd[T types.Application](appCreator types.AppCreator[T]) *cob
 		Use:     "bootstrap-state",
 		Short:   "Bootstrap CometBFT state at an arbitrary block height using a light client",
 		Args:    cobra.NoArgs,
-		Example: fmt.Sprintf("%s bootstrap-state --height 1000000", version.AppName),
+		Example: "bootstrap-state --height 1000000",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			serverCtx := GetServerContextFromCmd(cmd)
 			logger := log.NewLogger(cmd.OutOrStdout())

diff --git a/store/go.mod b/store/go.mod
@@ -3,7 +3,7 @@ module cosmossdk.io/store
 go 1.23
 
 require (
-	cosmossdk.io/core v1.0.0-alpha.3
+	cosmossdk.io/core v1.0.0-alpha.4
 	cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29
 	cosmossdk.io/errors v1.0.1
 	cosmossdk.io/log v1.4.1

diff --git a/store/go.sum b/store/go.sum
@@ -1,5 +1,5 @@
-cosmossdk.io/core v1.0.0-alpha.3 h1:pnxaYAas7llXgVz1lM7X6De74nWrhNKnB3yMKe4OUUA=
-cosmossdk.io/core v1.0.0-alpha.3/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
+cosmossdk.io/core v1.0.0-alpha.4 h1:9iuroT9ejDYETCsGkzkvs/wAY/5UFl7nCIINFRxyMJY=
+cosmossdk.io/core v1.0.0-alpha.4/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY=
 cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29 h1:NxxUo0GMJUbIuVg0R70e3cbn9eFTEuMr7ev1AFvypdY=
 cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29/go.mod h1:8s2tPeJtSiQuoyPmr2Ag7meikonISO4Fv4MoO8+ORrs=
 cosmossdk.io/errors v1.0.1 h1:bzu+Kcr0kS/1DuPBtUFdWjzLqyUuCiyHjyJB6srBV/0=

diff --git a/store/v2/README.md b/store/v2/README.md
@@ -1,7 +1,7 @@
 # Store
 
 The `store` package contains the implementation of store/v2, which is the SDK's
-abstraction around managing historical and committed state. See [ADR-065](../docs/architecture/adr-065-store-v2.md)
+abstraction around managing historical and committed state. See [ADR-065](../../docs/architecture/adr-065-store-v2.md)
 and [Store v2 Design](https://docs.google.com/document/d/1l6uXIjTPHOOWM5N4sUUmUfCZvePoa5SNfIEtmgvgQSU/edit#heading=h.nz8dqy6wa4g1) for a high-level overview of the design and rationale.
 
 ## Usage
@@ -42,21 +42,26 @@ sequenceDiagram
     end
 ```
 
-`Prune store keys` does not remove the data from the SC and SS instantly. It only
+`PruneStoreKeys` does not remove the data from the SC and SS instantly. It only
 marks the store keys as pruned. The actual data removal is done by the pruning
 process of the underlying SS and SC.
 
 ## Migration
 
-<!-- TODO -->
+The migration from store/v1 to store/v2 is supported by the `MigrationManager` in
+the `migration` package. See [Migration Manager](./migration/README.md) for more details.
 
 ## Pruning
 
 The `root.Store` is NOT responsible for pruning. Rather, pruning is the responsibility
 of the underlying SS and SC layers. This means pruning can be implementation specific,
-such as being synchronous or asynchronous.
+such as being synchronous or asynchronous. See [Pruning Manager](./pruning/README.md) for more details.
 
 
+## State Sync
+
+The `root.Store` is NOT responsible for state sync. See [Snapshots Manager](./snapshots/README.md)
+for more details.
 
 ## Test Coverage
 

diff --git a/store/v2/commitment/README.md b/store/v2/commitment/README.md
@@ -28,7 +28,11 @@ See this [section](https://docs.google.com/document/d/1l6uXIjTPHOOWM5N4sUUmUfCZv
 
 ## Pruning
 
-<!-- TODO -->
+Pruning is the process of efficiently managing and removing outdated data from the 
+State Commitment (SC). To facilitate this, the SC backend must implement the `Pruner` 
+interface, allowing the `PruningManager` to execute data pruning operations according 
+to the specified `PruningOption`. Optionally, the SC backend can implement the
+`PausablePruner` interface to pause pruning during a commit.
 
 ## State Sync
 

diff --git a/store/v2/migration/README.md b/store/v2/migration/README.md
@@ -0,0 +1,111 @@
+# Migration Manager
+
+The `migration` package contains the `migration.Manager`, which is responsible 
+for migrating data from `store/v1` to `store/v2`. To ensure a smooth transition, 
+the process is designed to **lazily** migrate data in the background without blocking 
+`root.Store` operations.
+
+## Overview
+
+The migration process involves several steps:
+
+1. **Create a snapshot** of the current state while `Commit` operations continue to 
+    function with `store/v1`.
+2. **Restore the snapshot** into the new StateStorage (SS) and StateCommitment (SC).
+3. **Sync recent state changes** from `store/v1` to the new SS and SC.
+4. After syncing, the `Commit` operation will be switched to the new `store/v2`.
+
+Taking a snapshot is a lightweight operation. The snapshot is not stored on disk but
+consumed by the `Restore` process, which replays state changes to the new SS and SC.
+
+> **Note:** After migration, `store/v2` does **not** support historical queries. 
+If historical data access is required, a full state migration to `store/v2` is necessary.
+
+## Usage
+
+You can create a new `migration.Manager` by calling the following function:
+
+```go
+func NewManager(
+    db corestore.KVStoreWithBatch, 
+    sm *snapshots.Manager, 
+    ss *storage.StorageStore, 
+    sc *commitment.CommitStore, 
+    logger log.Logger
+) *Manager
+```
+
+* `sc` (Commitment Store) can be `nil`. In that case, the Manager will migrate only 
+    the state storage.
+* The migration process is lazy, meaning data is migrated in the background while 
+    `root.Store` remains fully operational.
+
+To initiate the migration process, call the `Start` method:
+
+```go
+func (m *Manager) Start(ctx context.Context) error
+```
+
+> **Note:** It should be called by the RootStore, running in the background.
+
+## Migration Flow
+
+```mermaid
+sequenceDiagram
+    autonumber
+
+    participant A as RootStore
+    participant B as MigrationManager
+    participant C as SnapshotsManager
+    participant D as StateCommitment
+    participant E as StateStorage
+
+    A->>B: Start
+    loop Old Data Migration
+        B->>C: Create Snapshot
+        C->>B: Stream Snapshot
+        B->>D: State Sync (Restore)
+        B->>E: Write Changeset (Restore)
+    end
+
+    loop New Commit Data Sync
+        A->>B: Commit(Changeset)
+        B->>B: Store Changeset
+        B->>D: Commit Changeset
+        B->>E: Write Changeset
+    end
+
+    B->>A: Switch to new store/v2
+```
+
+## Key Considerations
+
+### Laziness and Background Operation
+
+The migration is performed lazily, meaning it occurs in the background without 
+interrupting the current operations on root.Store. This allows the chain to continue 
+running while data is gradually migrated to `store/v2`. State synchronization ensures 
+that any new state changes during the migration are also applied to `store/v2`.
+
+However, note that there may be a performance impact depending on the size of the data
+being migrated, and it’s essential to monitor the migration process in production 
+environments.
+
+### Handling Failures and Rollbacks
+
+It is important to consider how the migration manager handles errors or system failures 
+during the migration process:
+
+* If the migration fails, there is no impact on the existing `store/v1` operations, 
+    but need to restart the migration process from the scratch.
+* In the event of a critical failure after migration, a rollback may not be possible, 
+    and it is needed to keep the `store/v1` backup for a certain period.
+
+### Impact on Historical Queries
+
+After the migration, the new `store/v2` does not support historical queries. 
+This limitation should be clearly understood before starting the migration process, 
+especially if the node relies on historical data for any operations.
+
+If historical queries are required, users must fully migrate all historical data to `store/v2`.
+Alternatively, keeping store/v1 accessible for historical queries could be an option.
diff --git a/store/v2/storage/README.md b/store/v2/storage/README.md
@@ -70,13 +70,10 @@ Iterate/backend_rocksdb_versiondb_opts-10          778ms ± 0%
 
 ## Pruning
 
-Pruning is an implementation and responsibility of the underlying SS backend.
-Specifically, the `StorageStore` accepts `store.PruningOption` which defines the
-pruning configuration. During `ApplyChangeset`, the `StorageStore` will check if
-pruning should occur based on the current height being committed. If so, it will
-delegate a `Prune` call on the underlying SS backend, which can be defined specific
-to the implementation, e.g. asynchronous or synchronous.
-
+Pruning is the process of efficiently managing and removing outdated or redundant 
+data from the State Storage (SS). To facilitate this, the SS backend must implement 
+the `Pruner` interface, allowing the `PruningManager` to execute data pruning operations 
+according to the specified `PruningOption`.
 
 ## State Sync
 

diff --git a/tests/integration/tx/context_test.go b/tests/integration/tx/context_test.go
@@ -3,16 +3,18 @@ package tx
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"cosmossdk.io/depinject"
 	"cosmossdk.io/log"
 	_ "cosmossdk.io/x/accounts"
 	"cosmossdk.io/x/tx/signing"
+
 	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
 	"github.com/cosmos/cosmos-sdk/tests/integration/tx/internal"
 	"github.com/cosmos/cosmos-sdk/tests/integration/tx/internal/pulsar/testpb"
 	"github.com/cosmos/cosmos-sdk/testutil/configurator"
 	simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
-	"github.com/stretchr/testify/require"
 )
 
 func ProvideCustomGetSigner() signing.CustomGetSigner {

diff --git a/tests/integration/tx/internal/util.go b/tests/integration/tx/internal/util.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"testing"
 
+	gogoproto "github.com/cosmos/gogoproto/proto"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protoreflect"
@@ -26,7 +27,6 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx"
 	authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing"
 	"github.com/cosmos/cosmos-sdk/x/auth/tx"
-	gogoproto "github.com/cosmos/gogoproto/proto"
 )
 
 var TestRepeatedFieldsSigner = signing.CustomGetSigner{

diff --git a/tools/cosmovisor/testdata/download/cosmovisor/genesis/bin/autod b/tools/cosmovisor/testdata/download/cosmovisor/genesis/bin/autod
@@ -10,7 +10,7 @@ cat > "$3" <<EOL
 {
 	"name": "chain2",
 	"height": 49,
-	"info": "{\"binaries\":{\"linux/amd64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:339911508de5e20b573ce902c500ee670589073485216bee8b045e853f24bce8\",\"linux/arm64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:339911508de5e20b573ce902c500ee670589073485216bee8b045e853f24bce8\",\"darwin/amd64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:339911508de5e20b573ce902c500ee670589073485216bee8b045e853f24bce8\",\"darwin/arm64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:339911508de5e20b573ce902c500ee670589073485216bee8b045e853f24bce8\"}}"
+	"info": "{\"binaries\":{\"linux/amd64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:e84db844e123e3bb888c9ec52f6768ae29fbef7c17eaf8fc7431485a65dae6b9\",\"linux/arm64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:e84db844e123e3bb888c9ec52f6768ae29fbef7c17eaf8fc7431485a65dae6b9\",\"darwin/amd64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:e84db844e123e3bb888c9ec52f6768ae29fbef7c17eaf8fc7431485a65dae6b9\",\"darwin/arm64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip?checksum=sha256:e84db844e123e3bb888c9ec52f6768ae29fbef7c17eaf8fc7431485a65dae6b9\"}}"
 }
 EOL
 

diff --git a/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod b/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod
@@ -10,7 +10,7 @@ cat > "$3" <<EOL
 {
 	"name": "chain3",
 	"height": 936,
-	"info": "{\"binaries\":{\"linux/amd64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/ref_to_chain3-zip_dir.json?checksum=sha256:6758973f7404f6d34381029931b85826fc7a6315584ede03bad4c19e9b787f6c\",\"linux/arm64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/ref_to_chain3-zip_dir.json?checksum=sha256:6758973f7404f6d34381029931b85826fc7a6315584ede03bad4c19e9b787f6c\",\"darwin/amd64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/ref_to_chain3-zip_dir.json?checksum=sha256:6758973f7404f6d34381029931b85826fc7a6315584ede03bad4c19e9b787f6c\",\"darwin/arm64\": \"https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/ref_to_chain3-zip_dir.json?checksum=sha256:6758973f7404f6d34381029931b85826fc7a6315584ede03bad4c19e9b787f6c\"}}"
+	"info": "https://github.com/cosmos/cosmos-sdk/raw/main/tools/cosmovisor/testdata/repo/ref_to_chain3-zip_dir.json?checksum=sha256:6758973f7404f6d34381029931b85826fc7a6315584ede03bad4c19e9b787f6c"
 }
 EOL
 

diff --git a/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip b/tools/cosmovisor/testdata/repo/chain2-zip_bin/autod.zip
diff --git a/x/epochs/README.md b/x/epochs/README.md
@@ -68,22 +68,6 @@ The `epochs` module emits the following events:
 
 Epochs keeper module provides utility functions to manage epochs.
 
-``` go
-// Keeper is the interface for epochs module keeper
-type Keeper interface {
-  // GetEpochInfo returns epoch info by identifier
-  GetEpochInfo(ctx sdk.Context, identifier string) types.EpochInfo
-  // SetEpochInfo set epoch info
-  SetEpochInfo(ctx sdk.Context, epoch types.EpochInfo)
-  // DeleteEpochInfo delete epoch info
-  DeleteEpochInfo(ctx sdk.Context, identifier string)
-  // IterateEpochInfo iterate through epochs
-  IterateEpochInfo(ctx sdk.Context, fn func(index int64, epochInfo types.EpochInfo) (stop bool))
-  // Get all epoch infos
-  AllEpochInfos(ctx sdk.Context) []types.EpochInfo
-}
-```
-
 ## Hooks
 
 ```go

diff --git a/x/epochs/keeper/epoch.go b/x/epochs/keeper/epoch.go
@@ -5,8 +5,15 @@ import (
 	"fmt"
 
 	"cosmossdk.io/x/epochs/types"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
+// GetEpochInfo returns epoch info by identifier.
+func (k Keeper) GetEpochInfo(ctx sdk.Context, identifier string) (types.EpochInfo, error) {
+	return k.EpochInfo.Get(ctx, identifier)
+}
+
 // AddEpochInfo adds a new epoch info. Will return an error if the epoch fails validation,
 // or re-uses an existing identifier.
 // This method also sets the start time if left unset, and sets the epoch start height.