diff --git a/go.mod b/go.mod index 9eb3a0c9..8093ebe8 100644 --- a/go.mod +++ b/go.mod @@ -123,6 +123,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -153,6 +154,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/cors v1.8.0 // indirect + github.com/schollz/progressbar/v3 v3.13.1 // indirect github.com/sethvargo/go-retry v0.2.3 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/slok/go-http-metrics v0.10.0 // indirect @@ -192,6 +194,7 @@ require ( golang.org/x/net v0.19.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.16.1 // indirect diff --git a/storage/migration/cadence1.go b/storage/migration/cadence1.go new file mode 100644 index 00000000..2e367b14 --- /dev/null +++ b/storage/migration/cadence1.go @@ -0,0 +1,69 @@ +/* + * Flow Emulator + * + * Copyright 2024 Dapper Labs, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package migration + +import ( + "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/model/flow" + "github.com/rs/zerolog" + + "github.com/onflow/flow-emulator/storage/sqlite" + + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/cmd/util/ledger/util" +) + +func MigrateCadence1( + store *sqlite.Store, + rwf reporters.ReportWriterFactory, + logger zerolog.Logger, +) error { + payloads, payloadInfo, _, err := util.PayloadsAndAccountsFromEmulatorSnapshot(store.DB()) + if err != nil { + return err + } + + // TODO: >1 breaks atree storage map iteration + // and requires LinkValueMigration.LinkValueMigration to be thread-safe + const nWorker = 1 + + // TODO: EVM contract is not deployed in snapshot yet, so can't update it + const evmContractChange = migrations.EVMContractChangeNone + + // TODO: + var stagedContracts []migrations.StagedContract + + cadence1Migrations := migrations.NewCadence1Migrations( + logger, + rwf, + nWorker, + flow.Emulator, + evmContractChange, + stagedContracts, + ) + + for _, migration := range cadence1Migrations { + payloads, err = migration(payloads) + if err != nil { + return err + } + } + + return WritePayloadsToSnapshot(store, payloads, payloadInfo) +} diff --git a/storage/migration/cadence1_test.go b/storage/migration/cadence1_test.go new file mode 100644 index 00000000..0d8baf69 --- /dev/null +++ b/storage/migration/cadence1_test.go @@ -0,0 +1,77 @@ +/* + * Flow Emulator + * + * Copyright 2024 Dapper Labs, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package migration + +import ( + "io" + "os" + "testing" + + "github.com/rs/zerolog" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-emulator/storage/sqlite" +) + +func TestCadence1Migration(t *testing.T) { + const emulatorStateFile = "test-data/emulator_state_cadence_v0.42.6" + + // Work on a temp copy of the state, + // since the migration will be updating the state. + tempEmulatorState, err := os.CreateTemp("test-data", "temp_emulator_state") + require.NoError(t, err) + + tempEmulatorStatePath := tempEmulatorState.Name() + + defer tempEmulatorState.Close() + defer os.Remove(tempEmulatorStatePath) + + content, err := os.ReadFile(emulatorStateFile) + require.NoError(t, err) + + _, err = tempEmulatorState.Write(content) + require.NoError(t, err) + + // Migrate + + store, err := sqlite.New(tempEmulatorStatePath) + require.NoError(t, err) + + logWriter := &writer{} + logger := zerolog.New(logWriter).Level(zerolog.ErrorLevel) + + // Then migrate the values. + rwf := &NOOPReportWriterFactory{} + err = MigrateCadence1(store, rwf, logger) + require.NoError(t, err) + + require.Empty(t, logWriter.logs) +} + +type writer struct { + logs []string +} + +var _ io.Writer = &writer{} + +func (w *writer) Write(p []byte) (n int, err error) { + w.logs = append(w.logs, string(p)) + return len(p), nil +} diff --git a/storage/migration/test-data/README.md b/storage/migration/test-data/README.md new file mode 100644 index 00000000..3c3688d4 --- /dev/null +++ b/storage/migration/test-data/README.md @@ -0,0 +1,3 @@ +This is the same emulator state used in https://github.com/onflow/flow-go/tree/feature/stable-cadence/cmd/util/ledger/migrations/test-data/cadence_values_migration + +Follow the instruction there to generate a new state, if needed. \ No newline at end of file diff --git a/storage/migration/test-data/emulator_state_cadence_v0.42.6 b/storage/migration/test-data/emulator_state_cadence_v0.42.6 new file mode 100644 index 00000000..198d0f9a Binary files /dev/null and b/storage/migration/test-data/emulator_state_cadence_v0.42.6 differ diff --git a/storage/migration/utils.go b/storage/migration/utils.go new file mode 100644 index 00000000..47687059 --- /dev/null +++ b/storage/migration/utils.go @@ -0,0 +1,117 @@ +/* + * Flow Emulator + * + * Copyright 2024 Dapper Labs, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package migration + +import ( + "context" + "os" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-emulator/storage" + "github.com/onflow/flow-emulator/storage/sqlite" + + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" +) + +func WritePayloadsToSnapshot( + store *sqlite.Store, + payloads []*ledger.Payload, + payloadInfoSet map[flow.RegisterID]util.PayloadMetaInfo, +) error { + + const storeName = storage.LedgerStoreName + + ctx := context.TODO() + + for _, payload := range payloads { + key, err := payload.Key() + if err != nil { + return err + } + + registerId, err := convert.LedgerKeyToRegisterID(key) + if err != nil { + return err + } + + registerIdBytes := []byte(registerId.String()) + + value := payload.Value() + + payloadInfo, ok := payloadInfoSet[registerId] + if ok { + // Insert the values back with the existing height and version. + err = store.SetBytesWithVersionAndHeight( + ctx, + storeName, + registerIdBytes, + value, + payloadInfo.Version, + payloadInfo.Height, + ) + } else { + // If this is a new payload, use the current block height, and default version. + err = store.SetBytes( + ctx, + storeName, + registerIdBytes, + value, + ) + } + + if err != nil { + return err + } + } + + return nil +} + +func NewConsoleLogger() zerolog.Logger { + writer := zerolog.ConsoleWriter{ + Out: os.Stdout, + } + + return zerolog.New(writer). + With(). + Timestamp(). + Logger(). + Level(zerolog.InfoLevel) +} + +type NOOPReportWriterFactory struct{} + +func (*NOOPReportWriterFactory) ReportWriter(_ string) reporters.ReportWriter { + return &NOOPWriter{} +} + +type NOOPWriter struct{} + +var _ reporters.ReportWriter = &NOOPWriter{} + +func (*NOOPWriter) Write(_ any) { + // NO-OP +} + +func (r *NOOPWriter) Close() {} diff --git a/storage/sqlite/store.go b/storage/sqlite/store.go index 4b153eef..a64a45cd 100644 --- a/storage/sqlite/store.go +++ b/storage/sqlite/store.go @@ -342,3 +342,30 @@ func (s *Store) Close() error { return nil } + +// Below are needed for the state migrations only. + +func (s *Store) DB() *sql.DB { + return s.db +} + +func (s *Store) SetBytesWithVersionAndHeight(_ context.Context, store string, key []byte, value []byte, version, height uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + + _, err := s.db.Exec( + fmt.Sprintf( + "INSERT INTO %s (key, version, value, height) VALUES (?, ?, ?, ?) ON CONFLICT(key, version, height) DO UPDATE SET value=excluded.value", + store, + ), + hex.EncodeToString(key), + version, + hex.EncodeToString(value), + height, + ) + if err != nil { + return err + } + + return nil +}