Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Cadence 1.0] Optionally check atree storage health before migration #5591

Merged
merged 7 commits into from
Mar 27, 2024
Merged

[Cadence 1.0] Optionally check atree storage health before migration #5591

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e22c007
optionally check atree storage health before migration
turbolent Mar 26, 2024
d9a1bcf
check health before migration
turbolent Mar 27, 2024
e7416f9
Merge branch 'feature/stable-cadence' into bastian/check-health-befor…
turbolent Mar 27, 2024
a706f2a
retrieve all slabs before checking atree storage health
turbolent Mar 27, 2024
3481138
Apply suggestions from code review
turbolent Mar 27, 2024
406c586
fix test
turbolent Mar 27, 2024
902b189
only check storage health before migration once in first migratio
turbolent Mar 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 31 additions & 21 deletions cmd/util/cmd/execution-state-extract/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package extract

Check failure on line 1 in cmd/util/cmd/execution-state-extract/cmd.go

View workflow job for this annotation

GitHub Actions / Lint (./) 

: # github.com/onflow/flow-go/cmd/util/cmd/execution-state-extract [github.com/onflow/flow-go/cmd/util/cmd/execution-state-extract.test]

import (
"encoding/hex"
Expand All @@ -22,27 +22,28 @@
)

var (
flagExecutionStateDir string
flagOutputDir string
flagBlockHash string
flagStateCommitment string
flagDatadir string
flagChain string
flagNWorker int
flagNoMigration bool
flagNoReport bool
flagValidateMigration bool
flagAllowPartialStateFromPayloads bool
flagSortPayloads bool
flagPrune bool
flagLogVerboseValidationError bool
flagDiffMigration bool
flagLogVerboseDiff bool
flagStagedContractsFile string
flagInputPayloadFileName string
flagOutputPayloadFileName string
flagOutputPayloadByAddresses string
flagMaxAccountSize uint64
flagExecutionStateDir string
flagOutputDir string
flagBlockHash string
flagStateCommitment string
flagDatadir string
flagChain string
flagNWorker int
flagNoMigration bool
flagNoReport bool
flagValidateMigration bool
flagAllowPartialStateFromPayloads bool
flagSortPayloads bool
flagPrune bool
flagLogVerboseValidationError bool
flagDiffMigration bool
flagLogVerboseDiff bool
flagCheckStorageHealthBeforeMigration bool
flagStagedContractsFile string
flagInputPayloadFileName string
flagOutputPayloadFileName string
flagOutputPayloadByAddresses string
flagMaxAccountSize uint64
)

var Cmd = &cobra.Command{
Expand Down Expand Up @@ -92,6 +93,9 @@
Cmd.Flags().BoolVar(&flagLogVerboseDiff, "log-verbose-diff", false,
"log entire Cadence values on diff (requires --diff flag)")

Cmd.Flags().BoolVar(&flagCheckStorageHealthBeforeMigration, "check-storage-health-before", false,
"check (atree) storage health before migration")

Cmd.Flags().StringVar(&flagStagedContractsFile, "staged-contracts", "",
"Staged contracts CSV file")

Expand Down Expand Up @@ -280,6 +284,10 @@
log.Warn().Msgf("--log-verbose-diff flag is enabled which may increase size of log")
}

if flagCheckStorageHealthBeforeMigration {
log.Warn().Msgf("--check-storage-health-before flag is enabled and will increase duration of migration")
}

var inputMsg string
if len(flagInputPayloadFileName) > 0 {
// Input is payloads
Expand Down Expand Up @@ -341,6 +349,7 @@
!flagNoMigration,
flagDiffMigration,
flagLogVerboseDiff,
flagCheckStorageHealthBeforeMigration,
chainID,
evmContractChange,
burnerContractChange,
Expand All @@ -362,6 +371,7 @@
!flagNoMigration,
flagDiffMigration,
flagLogVerboseDiff,
flagCheckStorageHealthBeforeMigration,
chainID,
evmContractChange,
burnerContractChange,
Expand Down
6 changes: 6 additions & 0 deletions cmd/util/cmd/execution-state-extract/execution_state_extract.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func extractExecutionState(
runMigrations bool,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
chainID flow.ChainID,
evmContractChange migrators.EVMContractChange,
burnerContractChange migrators.BurnerContractChange,
Expand Down Expand Up @@ -114,6 +115,7 @@ func extractExecutionState(
runMigrations,
diffMigrations,
logVerboseDiff,
checkStorageHealthBeforeMigration,
chainID,
evmContractChange,
burnerContractChange,
Expand Down Expand Up @@ -229,6 +231,7 @@ func extractExecutionStateFromPayloads(
runMigrations bool,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
chainID flow.ChainID,
evmContractChange migrators.EVMContractChange,
burnerContractChange migrators.BurnerContractChange,
Expand All @@ -255,6 +258,7 @@ func extractExecutionStateFromPayloads(
runMigrations,
diffMigrations,
logVerboseDiff,
checkStorageHealthBeforeMigration,
chainID,
evmContractChange,
burnerContractChange,
Expand Down Expand Up @@ -412,6 +416,7 @@ func newMigrations(
runMigrations bool,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
chainID flow.ChainID,
evmContractChange migrators.EVMContractChange,
burnerContractChange migrators.BurnerContractChange,
Expand All @@ -434,6 +439,7 @@ func newMigrations(
chainID,
diffMigrations,
logVerboseDiff,
checkStorageHealthBeforeMigration,
evmContractChange,
burnerContractChange,
stagedContracts,
Expand Down
6 changes: 6 additions & 0 deletions cmd/util/ledger/migrations/cadence.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ func NewCadence1ValueMigrations(
chainID flow.ChainID,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
) (migrations []NamedMigration) {

// Populated by CadenceLinkValueMigrator,
Expand Down Expand Up @@ -251,6 +252,7 @@ func NewCadence1ValueMigrations(
rwf,
diffMigrations,
logVerboseDiff,
checkStorageHealthBeforeMigration,
errorMessageHandler,
contracts,
NewCadence1CompositeStaticTypeConverter(chainID),
Expand All @@ -260,6 +262,7 @@ func NewCadence1ValueMigrations(
rwf,
diffMigrations,
logVerboseDiff,
checkStorageHealthBeforeMigration,
fxamacker marked this conversation as resolved.
Show resolved Hide resolved
errorMessageHandler,
contracts,
capabilityMapping,
Expand All @@ -268,6 +271,7 @@ func NewCadence1ValueMigrations(
rwf,
diffMigrations,
logVerboseDiff,
checkStorageHealthBeforeMigration,
turbolent marked this conversation as resolved.
Show resolved Hide resolved
errorMessageHandler,
contracts,
capabilityMapping,
Expand Down Expand Up @@ -357,6 +361,7 @@ func NewCadence1Migrations(
chainID flow.ChainID,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
evmContractChange EVMContractChange,
burnerContractChange BurnerContractChange,
stagedContracts []StagedContract,
Expand Down Expand Up @@ -417,6 +422,7 @@ func NewCadence1Migrations(
nWorker,
chainID,
diffMigrations,
checkStorageHealthBeforeMigration,
logVerboseDiff,
)...,
)
Expand Down
66 changes: 45 additions & 21 deletions cmd/util/ledger/migrations/cadence_values_migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,13 @@ import (
)

type CadenceBaseMigrator struct {
name string
log zerolog.Logger
reporter reporters.ReportWriter
diffReporter reporters.ReportWriter
logVerboseDiff bool
valueMigrations func(
name string
log zerolog.Logger
reporter reporters.ReportWriter
diffReporter reporters.ReportWriter
logVerboseDiff bool
checkStorageHealthBeforeMigration bool
valueMigrations func(
inter *interpreter.Interpreter,
accounts environment.Accounts,
reporter *cadenceValueMigrationReporter,
Expand Down Expand Up @@ -124,14 +125,31 @@ func (m *CadenceBaseMigrator) MigrateAccount(
),
)

var storageHealthErrorBefore error
if m.checkStorageHealthBeforeMigration {
storageHealthErrorBefore = storage.CheckHealth()
if storageHealthErrorBefore != nil {
m.log.Warn().
Err(storageHealthErrorBefore).
Str("account", address.Hex()).
Msg("storage health check before migration failed")
}
}

fxamacker marked this conversation as resolved.
Show resolved Hide resolved
err = migration.Commit()
if err != nil {
return nil, fmt.Errorf("failed to commit changes: %w", err)
}

err = storage.CheckHealth()
if err != nil {
m.log.Err(err).Msg("storage health check failed")
// Check storage health after migration.
// If the storage health check failed before the migration, we don't need to check it again.
if storageHealthErrorBefore == nil {
storageHealthErrorAfter := storage.CheckHealth()
if storageHealthErrorAfter != nil {
m.log.Err(storageHealthErrorAfter).
Str("account", address.Hex()).
Msg("storage health check after migration failed")
}
}

// finalize the transaction
Expand Down Expand Up @@ -202,6 +220,7 @@ func NewCadence1ValueMigrator(
rwf reporters.ReportWriterFactory,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
errorMessageHandler *errorMessageHandler,
contracts map[common.AddressLocation][]byte,
compositeTypeConverter statictypes.CompositeTypeConverterFunc,
Expand All @@ -214,10 +233,11 @@ func NewCadence1ValueMigrator(
}

return &CadenceBaseMigrator{
name: "cadence-value-migration",
reporter: rwf.ReportWriter("cadence-value-migrator"),
diffReporter: diffReporter,
logVerboseDiff: logVerboseDiff,
name: "cadence-value-migration",
reporter: rwf.ReportWriter("cadence-value-migrator"),
diffReporter: diffReporter,
logVerboseDiff: logVerboseDiff,
checkStorageHealthBeforeMigration: checkStorageHealthBeforeMigration,
valueMigrations: func(
inter *interpreter.Interpreter,
_ environment.Accounts,
Expand All @@ -243,6 +263,7 @@ func NewCadence1LinkValueMigrator(
rwf reporters.ReportWriterFactory,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
errorMessageHandler *errorMessageHandler,
contracts map[common.AddressLocation][]byte,
capabilityMapping *capcons.CapabilityMapping,
Expand All @@ -253,10 +274,11 @@ func NewCadence1LinkValueMigrator(
}

return &CadenceBaseMigrator{
name: "cadence-link-value-migration",
reporter: rwf.ReportWriter("cadence-link-value-migrator"),
diffReporter: diffReporter,
logVerboseDiff: logVerboseDiff,
name: "cadence-link-value-migration",
reporter: rwf.ReportWriter("cadence-link-value-migrator"),
diffReporter: diffReporter,
logVerboseDiff: logVerboseDiff,
checkStorageHealthBeforeMigration: checkStorageHealthBeforeMigration,
valueMigrations: func(
_ *interpreter.Interpreter,
accounts environment.Accounts,
Expand Down Expand Up @@ -288,6 +310,7 @@ func NewCadence1CapabilityValueMigrator(
rwf reporters.ReportWriterFactory,
diffMigrations bool,
logVerboseDiff bool,
checkStorageHealthBeforeMigration bool,
errorMessageHandler *errorMessageHandler,
contracts map[common.AddressLocation][]byte,
capabilityMapping *capcons.CapabilityMapping,
Expand All @@ -298,10 +321,11 @@ func NewCadence1CapabilityValueMigrator(
}

return &CadenceBaseMigrator{
name: "cadence-capability-value-migration",
reporter: rwf.ReportWriter("cadence-capability-value-migrator"),
diffReporter: diffReporter,
logVerboseDiff: logVerboseDiff,
name: "cadence-capability-value-migration",
reporter: rwf.ReportWriter("cadence-capability-value-migrator"),
diffReporter: diffReporter,
logVerboseDiff: logVerboseDiff,
checkStorageHealthBeforeMigration: checkStorageHealthBeforeMigration,
valueMigrations: func(
_ *interpreter.Interpreter,
_ environment.Accounts,
Expand Down
4 changes: 4 additions & 0 deletions cmd/util/ledger/migrations/cadence_values_migration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func TestCadenceValuesMigration(t *testing.T) {
chainID,
false,
false,
false,
evmContractChange,
burnerContractChange,
stagedContracts,
Expand Down Expand Up @@ -702,6 +703,7 @@ func TestBootstrappedStateMigration(t *testing.T) {
chainID,
false,
false,
false,
evmContractChange,
burnerContractChange,
nil,
Expand Down Expand Up @@ -828,6 +830,7 @@ func TestProgramParsingError(t *testing.T) {
chainID,
false,
false,
false,
evmContractChange,
burnerContractChange,
nil,
Expand Down Expand Up @@ -953,6 +956,7 @@ func TestCoreContractUsage(t *testing.T) {
chainID,
false,
false,
false,
evmContractChange,
burnerContractChange,
nil,
Expand Down
1 change: 1 addition & 0 deletions cmd/util/ledger/migrations/staged_contracts_migration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1469,6 +1469,7 @@ func TestConcurrentContractUpdate(t *testing.T) {
chainID,
false,
false,
false,
evmContractChange,
burnerContractChange,
stagedContracts,
Expand Down
Loading