From fab98d696f7571bf386e751f6355e27070623d1d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 3 Aug 2023 14:57:41 +0300 Subject: [PATCH 01/87] Updated payload to include protocol state ID. Updated some tests and convertion methods --- .../state/bootstrap/bootstrap_test.go | 2 +- model/flow/payload.go | 16 +++++++++++----- model/flow/payload_test.go | 2 +- model/messages/consensus.go | 18 ++++++++++++------ utils/unittest/fixtures.go | 2 ++ 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 1d84b2938db..eda2a0a811f 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("986c540657fdb3b4154311069d901223a3268492f678ae706010cd537cc328ad") + expectedStateCommitmentBytes, _ := hex.DecodeString("47b188ee023c2646940f0ce31513df922a3722197849237d7b7f41db7e296433") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/model/flow/payload.go b/model/flow/payload.go index a6af04000a3..fd3d6fc5798 100644 --- a/model/flow/payload.go +++ b/model/flow/payload.go @@ -16,6 +16,11 @@ type Payload struct { Seals []*Seal Receipts ExecutionReceiptMetaList Results ExecutionResultList + // ProtocolStateID is the root hash of protocol state. + // Per convention, this is the resulting state after applying all identity-changing operations potentially contained in the block. + // The block payload itself is validated wrt to the protocol state committed to by its parent. Thereby, we are only + // accepting protocol states that have been certified by a valid QC. + ProtocolStateID Identifier } // EmptyPayload returns an empty block payload. @@ -51,16 +56,17 @@ func (p Payload) Hash() Identifier { sealHash := MerkleRoot(GetIDs(p.Seals)...) recHash := MerkleRoot(GetIDs(p.Receipts)...) resHash := MerkleRoot(GetIDs(p.Results)...) - return ConcatSum(collHash, sealHash, recHash, resHash) + return ConcatSum(collHash, sealHash, recHash, resHash, p.ProtocolStateID) } // Index returns the index for the payload. func (p Payload) Index() *Index { idx := &Index{ - CollectionIDs: GetIDs(p.Guarantees), - SealIDs: GetIDs(p.Seals), - ReceiptIDs: GetIDs(p.Receipts), - ResultIDs: GetIDs(p.Results), + CollectionIDs: GetIDs(p.Guarantees), + SealIDs: GetIDs(p.Seals), + ReceiptIDs: GetIDs(p.Receipts), + ResultIDs: GetIDs(p.Results), + ProtocolStateID: p.ProtocolStateID, } return idx } diff --git a/model/flow/payload_test.go b/model/flow/payload_test.go index 52bf8369b86..06ddf7dbab3 100644 --- a/model/flow/payload_test.go +++ b/model/flow/payload_test.go @@ -29,7 +29,7 @@ func TestPayloadEncodeEmptyJSON(t *testing.T) { payloadHash2 := payload.Hash() assert.Equal(t, payloadHash2, payloadHash1) encoded2, err := json.Marshal(payload) - assert.Equal(t, `{"Guarantees":null,"Seals":null,"Receipts":null,"Results":null}`, string(encoded2)) + assert.Equal(t, `{"Guarantees":null,"Seals":null,"Receipts":null,"Results":null,"ProtocolStateID":"0000000000000000000000000000000000000000000000000000000000000000"}`, string(encoded2)) assert.Equal(t, string(encoded1), string(encoded2)) require.NoError(t, err) err = json.Unmarshal(encoded2, &decoded) diff --git a/model/messages/consensus.go b/model/messages/consensus.go index 85c2730d1fc..eeaa34cf9b4 100644 --- a/model/messages/consensus.go +++ b/model/messages/consensus.go @@ -56,10 +56,11 @@ func UntrustedExecutionResultFromInternal(internal *flow.ExecutionResult) Untrus // Deprecated: Please update flow.Payload to use []flow.Guarantee etc., then // replace instances of this type with flow.Payload type UntrustedBlockPayload struct { - Guarantees []flow.CollectionGuarantee - Seals []flow.Seal - Receipts []flow.ExecutionReceiptMeta - Results []UntrustedExecutionResult + Guarantees []flow.CollectionGuarantee + Seals []flow.Seal + Receipts []flow.ExecutionReceiptMeta + Results []UntrustedExecutionResult + ProtocolStateID flow.Identifier } // UntrustedBlock is a duplicate of flow.Block used within @@ -76,8 +77,10 @@ type UntrustedBlock struct { // ToInternal returns the internal representation of the type. func (ub *UntrustedBlock) ToInternal() *flow.Block { block := flow.Block{ - Header: &ub.Header, - Payload: &flow.Payload{}, + Header: &ub.Header, + Payload: &flow.Payload{ + ProtocolStateID: ub.Payload.ProtocolStateID, + }, } for _, guarantee := range ub.Payload.Guarantees { guarantee := guarantee @@ -104,6 +107,9 @@ func (ub *UntrustedBlock) ToInternal() *flow.Block { func UntrustedBlockFromInternal(flowBlock *flow.Block) UntrustedBlock { block := UntrustedBlock{ Header: *flowBlock.Header, + Payload: UntrustedBlockPayload{ + ProtocolStateID: flowBlock.Payload.ProtocolStateID, + }, } for _, guarantee := range flowBlock.Payload.Guarantees { block.Payload.Guarantees = append(block.Payload.Guarantees, *guarantee) diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 10a72b53f28..729b564b353 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -200,6 +200,7 @@ func FullBlockFixture() flow.Block { ExecutionReceiptFixture(WithResult(payload.Results[0])).Meta(), ExecutionReceiptFixture(WithResult(payload.Results[1])).Meta(), } + payload.ProtocolStateID = IdentifierFixture() header := block.Header header.PayloadHash = payload.Hash() @@ -270,6 +271,7 @@ func WithAllTheFixins(payload *flow.Payload) { payload.Receipts = flow.ExecutionReceiptMetaList{receipt.Meta()} payload.Results = flow.ExecutionResultList{&receipt.ExecutionResult} } + payload.ProtocolStateID = IdentifierFixture() } func WithSeals(seals ...*flow.Seal) func(*flow.Payload) { From a3fd425a8c073d29ce415490b43602e6fc7b294c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 14 Aug 2023 14:03:17 +0300 Subject: [PATCH 02/87] Updated storage layer to store protocol state ID as part of payload --- model/flow/index.go | 9 +++++---- storage/badger/operation/prefix.go | 17 +++++++++-------- storage/badger/operation/seals.go | 8 ++++++++ storage/badger/payloads.go | 9 +++++---- storage/badger/procedure/index.go | 18 ++++++++++++++---- 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/model/flow/index.go b/model/flow/index.go index 6f71575aa51..6d98412dc46 100644 --- a/model/flow/index.go +++ b/model/flow/index.go @@ -1,8 +1,9 @@ package flow type Index struct { - CollectionIDs []Identifier - SealIDs []Identifier - ReceiptIDs []Identifier - ResultIDs []Identifier + CollectionIDs []Identifier + SealIDs []Identifier + ReceiptIDs []Identifier + ResultIDs []Identifier + ProtocolStateID Identifier } diff --git a/storage/badger/operation/prefix.go b/storage/badger/operation/prefix.go index a8a6fa6a9ad..a21a4433587 100644 --- a/storage/badger/operation/prefix.go +++ b/storage/badger/operation/prefix.go @@ -59,14 +59,15 @@ const ( // codes for indexing multiple identifiers by identifier // NOTE: 51 was used for identity indexes before epochs - codeBlockChildren = 50 // index mapping block ID to children blocks - codePayloadGuarantees = 52 // index mapping block ID to payload guarantees - codePayloadSeals = 53 // index mapping block ID to payload seals - codeCollectionBlock = 54 // index mapping collection ID to block ID - codeOwnBlockReceipt = 55 // index mapping block ID to execution receipt ID for execution nodes - codePayloadReceipts = 57 // index mapping block ID to payload receipts - codePayloadResults = 58 // index mapping block ID to payload results - codeAllBlockReceipts = 59 // index mapping of blockID to multiple receipts + codeBlockChildren = 50 // index mapping block ID to children blocks + codePayloadGuarantees = 52 // index mapping block ID to payload guarantees + codePayloadProtocolStateID = 51 // index mapping block ID to payload protocol state ID + codePayloadSeals = 53 // index mapping block ID to payload seals + codeCollectionBlock = 54 // index mapping collection ID to block ID + codeOwnBlockReceipt = 55 // index mapping block ID to execution receipt ID for execution nodes + codePayloadReceipts = 57 // index mapping block ID to payload receipts + codePayloadResults = 58 // index mapping block ID to payload results + codeAllBlockReceipts = 59 // index mapping of blockID to multiple receipts // codes related to protocol level information codeEpochSetup = 61 // EpochSetup service event, keyed by ID diff --git a/storage/badger/operation/seals.go b/storage/badger/operation/seals.go index 961f9826e34..91fc0488122 100644 --- a/storage/badger/operation/seals.go +++ b/storage/badger/operation/seals.go @@ -30,6 +30,14 @@ func IndexPayloadResults(blockID flow.Identifier, resultIDs []flow.Identifier) f return insert(makePrefix(codePayloadResults, blockID), resultIDs) } +func IndexPayloadProtocolStateID(blockID flow.Identifier, stateID flow.Identifier) func(*badger.Txn) error { + return insert(makePrefix(codePayloadProtocolStateID, blockID), stateID) +} + +func LookupPayloadProtocolStateID(blockID flow.Identifier, stateID *flow.Identifier) func(*badger.Txn) error { + return retrieve(makePrefix(codePayloadProtocolStateID, blockID), stateID) +} + func LookupPayloadReceipts(blockID flow.Identifier, receiptIDs *[]flow.Identifier) func(*badger.Txn) error { return retrieve(makePrefix(codePayloadReceipts, blockID), receiptIDs) } diff --git a/storage/badger/payloads.go b/storage/badger/payloads.go index ec75103cde3..c4d57277c72 100644 --- a/storage/badger/payloads.go +++ b/storage/badger/payloads.go @@ -144,10 +144,11 @@ func (p *Payloads) retrieveTx(blockID flow.Identifier) func(tx *badger.Txn) (*fl results = append(results, result) } payload := &flow.Payload{ - Seals: seals, - Guarantees: guarantees, - Receipts: receipts, - Results: results, + Seals: seals, + Guarantees: guarantees, + Receipts: receipts, + Results: results, + ProtocolStateID: idx.ProtocolStateID, } return payload, nil diff --git a/storage/badger/procedure/index.go b/storage/badger/procedure/index.go index 0b4e56c7fd2..32d066176d6 100644 --- a/storage/badger/procedure/index.go +++ b/storage/badger/procedure/index.go @@ -27,6 +27,10 @@ func InsertIndex(blockID flow.Identifier, index *flow.Index) func(tx *badger.Txn if err != nil { return fmt.Errorf("could not store results index: %w", err) } + err = operation.IndexPayloadProtocolStateID(blockID, index.ProtocolStateID)(tx) + if err != nil { + return fmt.Errorf("could not store protocol state id: %w", err) + } return nil } } @@ -53,12 +57,18 @@ func RetrieveIndex(blockID flow.Identifier, index *flow.Index) func(tx *badger.T if err != nil { return fmt.Errorf("could not retrieve receipts index: %w", err) } + var stateID flow.Identifier + err = operation.LookupPayloadProtocolStateID(blockID, &stateID)(tx) + if err != nil { + return fmt.Errorf("could not retrieve protocol state id: %w", err) + } *index = flow.Index{ - CollectionIDs: collIDs, - SealIDs: sealIDs, - ReceiptIDs: receiptIDs, - ResultIDs: resultsIDs, + CollectionIDs: collIDs, + SealIDs: sealIDs, + ReceiptIDs: receiptIDs, + ResultIDs: resultsIDs, + ProtocolStateID: stateID, } return nil } From 4ce82806bd2d53f3afc5d4a453cd741c9201db47 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 14 Aug 2023 14:04:00 +0300 Subject: [PATCH 03/87] Updated mutator to check for protocol state ID in blocks that are being applied. Updating test setups --- state/protocol/badger/mutator.go | 8 ++++---- state/protocol/badger/mutator_test.go | 4 ++-- state/protocol/inmem/convert.go | 27 +++++++++++++++------------ utils/unittest/fixtures.go | 14 ++++++++++++++ 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index 02e563c46ac..9e07d855ff7 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -524,10 +524,10 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi } updatedState, updatedStateID, hasChanges := protocolStateUpdater.Build() - // TODO: check if updatedStateID corresponds to the root protocol state ID stored in payload - // if updatedStateID != payload.ProtocolStateID { - // return state.NewInvalidExtension("invalid protocol state transition detected expected (%x) got %x", payload.ProtocolStateID, updatedStateID) - // } + if updatedStateID != candidate.Payload.ProtocolStateID { + return state.NewInvalidExtensionErrorf("invalid protocol state transition detected, "+ + "payload contains (%x) but after applying changes got %x", candidate.Payload.ProtocolStateID, updatedStateID) + } qc := candidate.Header.QuorumCertificate() diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index bbc6f43afbe..0da02519ee7 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -129,7 +129,7 @@ func TestExtendValid(t *testing.T) { require.NoError(t, err) // insert block1 on top of the root block - block1 := unittest.BlockWithParentFixture(block.Header) + block1 := unittest.BlockWithParentProtocolState(block) err = fullState.Extend(context.Background(), block1) require.NoError(t, err) @@ -143,7 +143,7 @@ func TestExtendValid(t *testing.T) { }) t.Run("BlockProcessable event should be emitted when any child of block1 is inserted", func(t *testing.T) { - block2 := unittest.BlockWithParentFixture(block1.Header) + block2 := unittest.BlockWithParentProtocolState(block1) consumer.On("BlockProcessable", block1.Header, mock.Anything).Once() err := fullState.Extend(context.Background(), block2) require.NoError(t, err) diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index 28e127a5522..bdd5531e7e9 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -342,17 +342,6 @@ func SnapshotFromBootstrapStateWithParams( EpochCommitSafetyThreshold: epochCommitSafetyThreshold, // see protocol.Params for details } - protocolState := &flow.ProtocolStateEntry{ - CurrentEpochEventIDs: flow.EventIDs{ - SetupID: setup.ID(), - CommitID: commit.ID(), - }, - PreviousEpochEventIDs: flow.EventIDs{}, - Identities: flow.DynamicIdentityEntryListFromIdentities(setup.Participants), - InvalidStateTransitionAttempted: false, - NextEpochProtocolState: nil, - } - snap := SnapshotFromEncodable(EncodableSnapshot{ Head: root.Header, LatestSeal: seal, @@ -367,8 +356,22 @@ func SnapshotFromBootstrapStateWithParams( QuorumCertificate: qc, Epochs: epochs, Params: params, - ProtocolState: protocolState, + ProtocolState: ProtocolStateForBootstrapState(setup, commit), SealedVersionBeacon: nil, }) return snap, nil } + +// ProtocolStateForBootstrapState generates a protocol.ProtocolStateEntry for a root protocol state which is used for bootstrapping. +func ProtocolStateForBootstrapState(setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.ProtocolStateEntry { + return &flow.ProtocolStateEntry{ + CurrentEpochEventIDs: flow.EventIDs{ + SetupID: setup.ID(), + CommitID: commit.ID(), + }, + PreviousEpochEventIDs: flow.EventIDs{}, + Identities: flow.DynamicIdentityEntryListFromIdentities(setup.Participants), + InvalidStateTransitionAttempted: false, + NextEpochProtocolState: nil, + } +} diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 5ba5662b252..441470c95c8 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -323,6 +323,18 @@ func BlockWithParentFixture(parent *flow.Header) *flow.Block { } } +func BlockWithParentProtocolState(parent *flow.Block) *flow.Block { + payload := PayloadFixture(func(p *flow.Payload) { + p.ProtocolStateID = parent.Payload.ProtocolStateID + }) + header := BlockHeaderWithParentFixture(parent.Header) + header.PayloadHash = payload.Hash() + return &flow.Block{ + Header: header, + Payload: &payload, + } +} + func BlockWithGuaranteesFixture(guarantees []*flow.CollectionGuarantee) *flow.Block { payload := PayloadFixture(WithGuarantees(guarantees...)) header := BlockHeaderFixture() @@ -2142,7 +2154,9 @@ func BootstrapFixtureWithChainID( WithDKGFromParticipants(participants), ) + root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState(setup, commit).ID() stateCommit := GenesisStateCommitmentByChainID(chainID) + result := BootstrapExecutionResultFixture(root, stateCommit) result.ServiceEvents = []flow.ServiceEvent{ setup.ServiceEvent(), From 13f57f4db3d6dfe7e0bd8614582368a24caf74a0 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 14 Aug 2023 14:33:33 +0300 Subject: [PATCH 04/87] Work in progress on updating tests to include correct protocol state ID --- state/protocol/badger/mutator_test.go | 34 +++++++++++++++++++++------ utils/unittest/fixtures.go | 6 +++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 0da02519ee7..9e36c773d51 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -439,6 +439,7 @@ func TestVersionBeaconIndex(t *testing.T) { func TestExtendSealedBoundary(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { head, err := rootSnapshot.Head() require.NoError(t, err) @@ -450,7 +451,7 @@ func TestExtendSealedBoundary(t *testing.T) { // Create a first block on top of the snapshot block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) @@ -458,8 +459,9 @@ func TestExtendSealedBoundary(t *testing.T) { block1Receipt := unittest.ReceiptForBlockFixture(block1) block2 := unittest.BlockWithParentFixture(block1.Header) block2.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()}, - Results: []*flow.ExecutionResult{&block1Receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()}, + Results: []*flow.ExecutionResult{&block1Receipt.ExecutionResult}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), block2) require.NoError(t, err) @@ -468,7 +470,8 @@ func TestExtendSealedBoundary(t *testing.T) { block1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) block3 := unittest.BlockWithParentFixture(block2.Header) block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{block1Seal}, + Seals: []*flow.Seal{block1Seal}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -1996,6 +1999,7 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { func TestHeaderExtendValid(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { head, err := rootSnapshot.Head() require.NoError(t, err) @@ -2003,7 +2007,7 @@ func TestHeaderExtendValid(t *testing.T) { require.NoError(t, err) extend := unittest.BlockWithParentFixture(head) - extend.SetPayload(flow.EmptyPayload()) + extend.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.ExtendCertified(context.Background(), extend, unittest.CertifyBlock(extend.Header)) require.NoError(t, err) @@ -2039,11 +2043,13 @@ func TestHeaderExtendMissingParent(t *testing.T) { func TestHeaderExtendHeightTooSmall(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { head, err := rootSnapshot.Head() require.NoError(t, err) block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) // create another block that points to the previous block `extend` as parent // but has _same_ height as parent. This violates the condition that a child's @@ -2087,11 +2093,13 @@ func TestExtendBlockProcessable(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) head, err := rootSnapshot.Head() require.NoError(t, err) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) consumer := mockprotocol.NewConsumer(t) util.RunWithFullProtocolStateAndConsumer(t, rootSnapshot, consumer, func(db *badger.DB, state *protocol.ParticipantState) { block := unittest.BlockWithParentFixture(head) - child := unittest.BlockWithParentFixture(block.Header) - grandChild := unittest.BlockWithParentFixture(child.Header) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) + child := unittest.BlockWithParentProtocolState(block) + grandChild := unittest.BlockWithParentProtocolState(child) // extend block using certifying QC, expect that BlockProcessable will be emitted once consumer.On("BlockProcessable", block.Header, child.Header.QuorumCertificate()).Once() @@ -2120,11 +2128,13 @@ func TestExtendBlockProcessable(t *testing.T) { // The Follower should accept this block since tracking of orphan blocks is implemented by another component. func TestFollowerHeaderExtendBlockNotConnected(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { head, err := rootSnapshot.Head() require.NoError(t, err) block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.ExtendCertified(context.Background(), block1, unittest.CertifyBlock(block1.Header)) require.NoError(t, err) @@ -2133,6 +2143,7 @@ func TestFollowerHeaderExtendBlockNotConnected(t *testing.T) { // create a fork at view/height 1 and try to connect it to root block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.ExtendCertified(context.Background(), block2, unittest.CertifyBlock(block2.Header)) require.NoError(t, err) @@ -2150,11 +2161,13 @@ func TestFollowerHeaderExtendBlockNotConnected(t *testing.T) { // The Participant should reject this block as an outdated chain extension func TestParticipantHeaderExtendBlockNotConnected(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { head, err := rootSnapshot.Head() require.NoError(t, err) block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) @@ -2163,6 +2176,7 @@ func TestParticipantHeaderExtendBlockNotConnected(t *testing.T) { // create a fork at view/height 1 and try to connect it to root block2 := unittest.BlockWithParentFixture(head) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block2) require.True(t, st.IsOutdatedExtensionError(err), err) @@ -2548,3 +2562,9 @@ func mockMetricsForRootSnapshot(metricsMock *mockmodule.ComplianceMetrics, rootS metricsMock.On("FinalizedHeight", mock.Anything) metricsMock.On("SealedHeight", mock.Anything) } + +func getRootProtocolStateID(t *testing.T, rootSnapshot *inmem.Snapshot) flow.Identifier { + rootProtocolState, err := rootSnapshot.ProtocolState() + require.NoError(t, err) + return rootProtocolState.Entry().ID() +} diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 441470c95c8..f0b47000681 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -295,6 +295,12 @@ func WithReceipts(receipts ...*flow.ExecutionReceipt) func(*flow.Payload) { } } +func WithProtocolStateID(stateID flow.Identifier) func(payload *flow.Payload) { + return func(payload *flow.Payload) { + payload.ProtocolStateID = stateID + } +} + // WithReceiptsAndNoResults will add receipt to payload only func WithReceiptsAndNoResults(receipts ...*flow.ExecutionReceipt) func(*flow.Payload) { return func(payload *flow.Payload) { From 9b9e9a823486046d264a2088900894e88dd48dd4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 15 Aug 2023 12:29:44 +0300 Subject: [PATCH 05/87] Updated many tests to properly include protocol state ID --- consensus/integration/nodes_test.go | 1 + consensus/recovery/protocol/state_test.go | 11 +- state/cluster/badger/mutator_test.go | 20 ++-- state/protocol/badger/mutator_test.go | 129 ++++++++++++++-------- state/protocol/badger/snapshot_test.go | 12 +- state/protocol/inmem/convert.go | 9 +- utils/unittest/fixtures.go | 6 +- 7 files changed, 115 insertions(+), 73 deletions(-) diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 9ff53e5f4f6..215105516d5 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -280,6 +280,7 @@ func createRootBlockData(participantData *run.ParticipantData) (*flow.Block, *fl }, ) + root.SetPayload(flow.Payload{ProtocolStateID: inmem.ProtocolStateForBootstrapState(setup, commit).ID()}) result := unittest.BootstrapExecutionResultFixture(root, unittest.GenesisStateCommitment) result.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent(), commit.ServiceEvent()} diff --git a/consensus/recovery/protocol/state_test.go b/consensus/recovery/protocol/state_test.go index d22b4ef53f9..8852efe3652 100644 --- a/consensus/recovery/protocol/state_test.go +++ b/consensus/recovery/protocol/state_test.go @@ -21,23 +21,24 @@ import ( func TestSaveBlockAsReplica(t *testing.T) { participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(participants) + protocolState, err := rootSnapshot.ProtocolState() + require.NoError(t, err) + rootProtocolStateID := protocolState.Entry().ID() b0, err := rootSnapshot.Head() require.NoError(t, err) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { b1 := unittest.BlockWithParentFixture(b0) - b1.SetPayload(flow.Payload{}) + b1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), b1) require.NoError(t, err) - b2 := unittest.BlockWithParentFixture(b1.Header) - b2.SetPayload(flow.Payload{}) + b2 := unittest.BlockWithParentProtocolState(b1) err = state.Extend(context.Background(), b2) require.NoError(t, err) - b3 := unittest.BlockWithParentFixture(b2.Header) - b3.SetPayload(flow.Payload{}) + b3 := unittest.BlockWithParentProtocolState(b2) err = state.Extend(context.Background(), b3) require.NoError(t, err) diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 4254b448e4c..3c9974fd8b9 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -43,7 +43,7 @@ type MutatorSuite struct { // protocol state for reference blocks for transactions protoState protocol.FollowerState - protoGenesis *flow.Header + protoGenesis *flow.Block state cluster.MutableState } @@ -66,15 +66,17 @@ func (suite *MutatorSuite) SetupTest() { // just bootstrap with a genesis block, we'll use this as reference genesis, result, seal := unittest.BootstrapFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) + // ensure we don't enter a new epoch for tests that build many blocks result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = genesis.Header.View + 100_000 + seal.ResultID = result.ID() qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) rootSnapshot, err := inmem.SnapshotFromBootstrapState(genesis, result, seal, qc) require.NoError(suite.T(), err) suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter - suite.protoGenesis = genesis.Header + suite.protoGenesis = genesis state, err := pbadger.Bootstrap( metrics, suite.db, @@ -366,14 +368,12 @@ func (suite *MutatorSuite) TestExtend_WithExpiredReferenceBlock() { // the collection to be expired parent := suite.protoGenesis for i := 0; i < flow.DefaultTransactionExpiry+1; i++ { - next := unittest.BlockWithParentFixture(parent) - next.Payload.Guarantees = nil - next.SetPayload(*next.Payload) + next := unittest.BlockWithParentProtocolState(parent) err := suite.protoState.ExtendCertified(context.Background(), next, unittest.CertifyBlock(next.Header)) suite.Require().Nil(err) err = suite.protoState.Finalize(context.Background(), next.ID()) suite.Require().Nil(err) - parent = next.Header + parent = next } block := suite.Block() @@ -417,9 +417,7 @@ func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromDifferentEpoch() { // should be considered an unverifiable extension. It's possible that this reference // block has been finalized, we just haven't processed it yet. func (suite *MutatorSuite) TestExtend_WithUnfinalizedReferenceBlock() { - unfinalized := unittest.BlockWithParentFixture(suite.protoGenesis) - unfinalized.Payload.Guarantees = nil - unfinalized.SetPayload(*unfinalized.Payload) + unfinalized := unittest.BlockWithParentProtocolState(suite.protoGenesis) err := suite.protoState.ExtendCertified(context.Background(), unfinalized, unittest.CertifyBlock(unfinalized.Header)) suite.Require().NoError(err) @@ -436,12 +434,12 @@ func (suite *MutatorSuite) TestExtend_WithUnfinalizedReferenceBlock() { // to only use finalized blocks as reference, the proposer knowingly generated an invalid func (suite *MutatorSuite) TestExtend_WithOrphanedReferenceBlock() { // create a block extending genesis which is not finalized - orphaned := unittest.BlockWithParentFixture(suite.protoGenesis) + orphaned := unittest.BlockWithParentProtocolState(suite.protoGenesis) err := suite.protoState.ExtendCertified(context.Background(), orphaned, unittest.CertifyBlock(orphaned.Header)) suite.Require().NoError(err) // create a block extending genesis (conflicting with previous) which is finalized - finalized := unittest.BlockWithParentFixture(suite.protoGenesis) + finalized := unittest.BlockWithParentProtocolState(suite.protoGenesis) finalized.Payload.Guarantees = nil finalized.SetPayload(*finalized.Payload) err = suite.protoState.ExtendCertified(context.Background(), finalized, unittest.CertifyBlock(finalized.Header)) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 9e36c773d51..557c14392f8 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -153,6 +153,7 @@ func TestExtendValid(t *testing.T) { func TestSealedIndex(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { rootHeader, err := rootSnapshot.Head() require.NoError(t, err) @@ -165,20 +166,22 @@ func TestSealedIndex(t *testing.T) { // block 1 b1 := unittest.BlockWithParentFixture(rootHeader) - b1.SetPayload(flow.EmptyPayload()) + b1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), b1) require.NoError(t, err) // block 2(result B1) b1Receipt := unittest.ReceiptForBlockFixture(b1) b2 := unittest.BlockWithParentFixture(b1.Header) - b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt))) + b2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(b1Receipt), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.Extend(context.Background(), b2) require.NoError(t, err) // block 3 - b3 := unittest.BlockWithParentFixture(b2.Header) - b3.SetPayload(flow.EmptyPayload()) + b3 := unittest.BlockWithParentProtocolState(b2) err = state.Extend(context.Background(), b3) require.NoError(t, err) @@ -187,8 +190,9 @@ func TestSealedIndex(t *testing.T) { b3Receipt := unittest.ReceiptForBlockFixture(b3) b4 := unittest.BlockWithParentFixture(b3.Header) b4.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, - Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, + Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), b4) require.NoError(t, err) @@ -197,7 +201,8 @@ func TestSealedIndex(t *testing.T) { b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult)) b5 := unittest.BlockWithParentFixture(b4.Header) b5.SetPayload(flow.Payload{ - Seals: []*flow.Seal{b1Seal}, + Seals: []*flow.Seal{b1Seal}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), b5) require.NoError(t, err) @@ -207,14 +212,14 @@ func TestSealedIndex(t *testing.T) { b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult)) b6 := unittest.BlockWithParentFixture(b5.Header) b6.SetPayload(flow.Payload{ - Seals: []*flow.Seal{b2Seal, b3Seal}, + Seals: []*flow.Seal{b2Seal, b3Seal}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), b6) require.NoError(t, err) // block 7 - b7 := unittest.BlockWithParentFixture(b6.Header) - b7.SetPayload(flow.EmptyPayload()) + b7 := unittest.BlockWithParentProtocolState(b6) err = state.Extend(context.Background(), b7) require.NoError(t, err) @@ -272,6 +277,7 @@ func TestSealedIndex(t *testing.T) { func TestVersionBeaconIndex(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { rootHeader, err := rootSnapshot.Head() require.NoError(t, err) @@ -284,7 +290,7 @@ func TestVersionBeaconIndex(t *testing.T) { // block 1 b1 := unittest.BlockWithParentFixture(rootHeader) - b1.SetPayload(flow.EmptyPayload()) + b1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), b1) require.NoError(t, err) @@ -340,13 +346,13 @@ func TestVersionBeaconIndex(t *testing.T) { b1Receipt := unittest.ReceiptForBlockFixture(b1) b1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb1.ServiceEvent()} b2 := unittest.BlockWithParentFixture(b1.Header) - b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt))) + b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt), + unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), b2) require.NoError(t, err) // block 3 - b3 := unittest.BlockWithParentFixture(b2.Header) - b3.SetPayload(flow.EmptyPayload()) + b3 := unittest.BlockWithParentProtocolState(b2) err = state.Extend(context.Background(), b3) require.NoError(t, err) @@ -359,8 +365,9 @@ func TestVersionBeaconIndex(t *testing.T) { b4 := unittest.BlockWithParentFixture(b3.Header) b4.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, - Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, + Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), b4) require.NoError(t, err) @@ -369,7 +376,8 @@ func TestVersionBeaconIndex(t *testing.T) { b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult)) b5 := unittest.BlockWithParentFixture(b4.Header) b5.SetPayload(flow.Payload{ - Seals: []*flow.Seal{b1Seal}, + Seals: []*flow.Seal{b1Seal}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), b5) require.NoError(t, err) @@ -379,7 +387,8 @@ func TestVersionBeaconIndex(t *testing.T) { b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult)) b6 := unittest.BlockWithParentFixture(b5.Header) b6.SetPayload(flow.Payload{ - Seals: []*flow.Seal{b2Seal, b3Seal}, + Seals: []*flow.Seal{b2Seal, b3Seal}, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), b6) require.NoError(t, err) @@ -528,12 +537,13 @@ func TestExtendMissingParent(t *testing.T) { func TestExtendHeightTooSmall(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { head, err := rootSnapshot.Head() require.NoError(t, err) extend := unittest.BlockFixture() - extend.SetPayload(flow.EmptyPayload()) + extend.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) extend.Header.Height = 1 extend.Header.View = 1 extend.Header.ParentID = head.ID() @@ -597,6 +607,7 @@ func TestExtendInconsistentParentView(t *testing.T) { func TestExtendBlockNotConnected(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { head, err := rootSnapshot.Head() @@ -604,7 +615,7 @@ func TestExtendBlockNotConnected(t *testing.T) { // add 2 blocks, the second finalizing/sealing the state of the first extend := unittest.BlockWithParentFixture(head) - extend.SetPayload(flow.EmptyPayload()) + extend.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), extend) require.NoError(t, err) @@ -686,8 +697,8 @@ func TestExtendReceiptsNotSorted(t *testing.T) { func TestExtendReceiptsInvalid(t *testing.T) { validator := mockmodule.NewReceiptValidator(t) - rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolStateAndValidator(t, rootSnapshot, validator, func(db *badger.DB, state *protocol.ParticipantState) { head, err := rootSnapshot.Head() require.NoError(t, err) @@ -696,7 +707,7 @@ func TestExtendReceiptsInvalid(t *testing.T) { // create block2 and block3 block2 := unittest.BlockWithParentFixture(head) - block2.SetPayload(flow.EmptyPayload()) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block2) require.NoError(t, err) @@ -705,8 +716,9 @@ func TestExtendReceiptsInvalid(t *testing.T) { block3 := unittest.BlockWithParentFixture(block2.Header) block3.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{receipt.Meta()}, - Results: []*flow.ExecutionResult{&receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{receipt.Meta()}, + Results: []*flow.ExecutionResult{&receipt.ExecutionResult}, + ProtocolStateID: rootProtocolStateID, }) // force the receipt validator to refuse this payload @@ -720,21 +732,20 @@ func TestExtendReceiptsInvalid(t *testing.T) { func TestExtendReceiptsValid(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { head, err := rootSnapshot.Head() require.NoError(t, err) block2 := unittest.BlockWithParentFixture(head) - block2.SetPayload(flow.EmptyPayload()) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block2) require.NoError(t, err) - block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(flow.EmptyPayload()) + block3 := unittest.BlockWithParentProtocolState(block2) err = state.Extend(context.Background(), block3) require.NoError(t, err) - block4 := unittest.BlockWithParentFixture(block3.Header) - block4.SetPayload(flow.EmptyPayload()) + block4 := unittest.BlockWithParentProtocolState(block3) err = state.Extend(context.Background(), block4) require.NoError(t, err) @@ -754,6 +765,7 @@ func TestExtendReceiptsValid(t *testing.T) { &receipt3b.ExecutionResult, &receipt3c.ExecutionResult, }, + ProtocolStateID: rootProtocolStateID, }) err = state.Extend(context.Background(), block5) require.NoError(t, err) @@ -785,7 +797,6 @@ func TestExtendEpochTransitionValid(t *testing.T) { consumer.On("BlockFinalized", mock.Anything) consumer.On("BlockProcessable", mock.Anything, mock.Anything) rootSnapshot := unittest.RootSnapshotFixture(participants) - unittest.RunWithBadgerDB(t, func(db *badger.DB) { // set up state and mock ComplianceMetrics object @@ -1923,6 +1934,7 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { consumer.On("BlockProcessable", mock.Anything, mock.Anything) rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) state, err := protocol.Bootstrap( metrics, @@ -1944,17 +1956,20 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { require.NoError(t, err) block1 := unittest.BlockWithParentFixture(head) - block1.Payload.Guarantees = nil - block1.Header.PayloadHash = block1.Payload.Hash() + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) block1Receipt := unittest.ReceiptForBlockFixture(block1) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block1Receipt))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(block1Receipt), + unittest.WithProtocolStateID(rootProtocolStateID), + )) block1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) block3 := unittest.BlockWithParentFixture(block2.Header) block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{block1Seal}, + Seals: []*flow.Seal{block1Seal}, + ProtocolStateID: rootProtocolStateID, }) sealValidator := mockmodule.NewSealValidator(t) @@ -2191,13 +2206,13 @@ func TestHeaderExtendHighestSeal(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) head, err := rootSnapshot.Head() require.NoError(t, err) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { // create block2 and block3 block2 := unittest.BlockWithParentFixture(head) - block2.SetPayload(flow.EmptyPayload()) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) - block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(flow.EmptyPayload()) + block3 := unittest.BlockWithParentProtocolState(block2) err := state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate()) require.NoError(t, err) @@ -2209,13 +2224,19 @@ func TestHeaderExtendHighestSeal(t *testing.T) { // include the seals in block4 block4 := unittest.BlockWithParentFixture(block3.Header) // include receipts and results - block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt3, receipt2))) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt3, receipt2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) // include the seals in block4 block5 := unittest.BlockWithParentFixture(block4.Header) // placing seals in the reversed order to test // Extend will pick the highest sealed block - block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal3, seal2))) + block5.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal3, seal2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.ExtendCertified(context.Background(), block3, block4.Header.QuorumCertificate()) require.NoError(t, err) @@ -2263,6 +2284,7 @@ func TestExtendCertifiedInvalidQC(t *testing.T) { // guarantees with invalid guarantors func TestExtendInvalidGuarantee(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { // create a valid block head, err := rootSnapshot.Head() @@ -2277,13 +2299,15 @@ func TestExtendInvalidGuarantee(t *testing.T) { require.NoError(t, err) block := unittest.BlockWithParentFixture(head) - payload := flow.EmptyPayload() - payload.Guarantees = []*flow.CollectionGuarantee{ - { - ChainID: cluster.ChainID(), - ReferenceBlockID: head.ID(), - SignerIndices: validSignerIndices, + payload := flow.Payload{ + Guarantees: []*flow.CollectionGuarantee{ + { + ChainID: cluster.ChainID(), + ReferenceBlockID: head.ID(), + SignerIndices: validSignerIndices, + }, }, + ProtocolStateID: rootProtocolStateID, } // now the valid block has a guarantee in the payload with valid signer indices. @@ -2368,18 +2392,23 @@ func TestExtendInvalidGuarantee(t *testing.T) { // If block B is finalized and contains a seal for block A, then A is the last sealed block func TestSealed(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { head, err := rootSnapshot.Head() require.NoError(t, err) // block 1 will be sealed block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) // block 2 contains receipt for block 1 block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate()) require.NoError(t, err) @@ -2389,7 +2418,8 @@ func TestSealed(t *testing.T) { // block 3 contains seal for block 1 block3 := unittest.BlockWithParentFixture(block2.Header) block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, + Seals: []*flow.Seal{seal1}, + ProtocolStateID: rootProtocolStateID, }) err = state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate()) @@ -2414,12 +2444,14 @@ func TestSealed(t *testing.T) { // A non atomic bug would be: header is found in DB, but payload index is not found func TestCacheAtomicity(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFollowerProtocolStateAndHeaders(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState, headers storage.Headers, index storage.Index) { head, err := rootSnapshot.Head() require.NoError(t, err) block := unittest.BlockWithParentFixture(head) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) blockID := block.ID() // check 100 times to see if either 1) or 2) satisfies @@ -2511,11 +2543,13 @@ func TestHeaderInvalidTimestamp(t *testing.T) { // where second extend doesn't result in an error and effectively is no-op. func TestProtocolStateIdempotent(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) head, err := rootSnapshot.Head() require.NoError(t, err) t.Run("follower", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) { block := unittest.BlockWithParentFixture(head) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err := state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) require.NoError(t, err) @@ -2527,6 +2561,7 @@ func TestProtocolStateIdempotent(t *testing.T) { t.Run("participant", func(t *testing.T) { util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { block := unittest.BlockWithParentFixture(head) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err := state.Extend(context.Background(), block) require.NoError(t, err) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 825e0d18d6c..01b7e847c3c 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -104,6 +104,7 @@ func TestHead(t *testing.T) { func TestSnapshot_Params(t *testing.T) { participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) expectedChainID := rootSnapshot.Params().ChainID() expectedSporkID := rootSnapshot.Params().SporkID() @@ -118,6 +119,7 @@ func TestSnapshot_Params(t *testing.T) { const nBlocks = 10 for i := 0; i < nBlocks; i++ { next := unittest.BlockWithParentFixture(head) + next.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, next) head = next.Header } @@ -905,6 +907,7 @@ func TestLatestSealedResult(t *testing.T) { func TestQuorumCertificate(t *testing.T) { identities := unittest.IdentityListFixture(5, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(identities) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -914,7 +917,7 @@ func TestQuorumCertificate(t *testing.T) { // create a block to query block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err := state.Extend(context.Background(), block1) require.NoError(t, err) @@ -944,7 +947,7 @@ func TestQuorumCertificate(t *testing.T) { // add a block so we aren't testing against root block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) certifyingQC := unittest.CertifyBlock(block1.Header) err := state.ExtendCertified(context.Background(), block1, certifyingQC) require.NoError(t, err) @@ -967,15 +970,14 @@ func TestQuorumCertificate(t *testing.T) { util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { // create a block to query block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err := state.Extend(context.Background(), block1) require.NoError(t, err) _, err = state.AtBlockID(block1.ID()).QuorumCertificate() assert.ErrorIs(t, err, storage.ErrNotFound) - block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(flow.EmptyPayload()) + block2 := unittest.BlockWithParentProtocolState(block1) err = state.Extend(context.Background(), block2) require.NoError(t, err) diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index bdd5531e7e9..e0ee0259090 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -342,6 +342,12 @@ func SnapshotFromBootstrapStateWithParams( EpochCommitSafetyThreshold: epochCommitSafetyThreshold, // see protocol.Params for details } + rootProtocolState := ProtocolStateForBootstrapState(setup, commit) + if rootProtocolState.ID() != root.Payload.ProtocolStateID { + return nil, fmt.Errorf("incorrect protocol state ID in root block, expected (%x) but got (%x)", + root.Payload.ProtocolStateID, rootProtocolState.ID()) + } + snap := SnapshotFromEncodable(EncodableSnapshot{ Head: root.Header, LatestSeal: seal, @@ -356,9 +362,10 @@ func SnapshotFromBootstrapStateWithParams( QuorumCertificate: qc, Epochs: epochs, Params: params, - ProtocolState: ProtocolStateForBootstrapState(setup, commit), + ProtocolState: rootProtocolState, SealedVersionBeacon: nil, }) + return snap, nil } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index f0b47000681..b91ca7637b8 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -330,9 +330,7 @@ func BlockWithParentFixture(parent *flow.Header) *flow.Block { } func BlockWithParentProtocolState(parent *flow.Block) *flow.Block { - payload := PayloadFixture(func(p *flow.Payload) { - p.ProtocolStateID = parent.Payload.ProtocolStateID - }) + payload := PayloadFixture(WithProtocolStateID(parent.Payload.ProtocolStateID)) header := BlockHeaderWithParentFixture(parent.Header) header.PayloadHash = payload.Hash() return &flow.Block{ @@ -2160,7 +2158,7 @@ func BootstrapFixtureWithChainID( WithDKGFromParticipants(participants), ) - root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState(setup, commit).ID() + root.SetPayload(flow.Payload{ProtocolStateID: inmem.ProtocolStateForBootstrapState(setup, commit).ID()}) stateCommit := GenesisStateCommitmentByChainID(chainID) result := BootstrapExecutionResultFixture(root, stateCommit) From d93b6599b5566e008837aa2a0533e924841f7a4a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 9 Oct 2023 11:21:48 +0300 Subject: [PATCH 06/87] Updated mutator and updater to be accessed from block builder --- module/builder/consensus/builder.go | 54 ++++---- state/protocol/protocol_state.go | 10 +- state/protocol/protocol_state/mutator.go | 156 ++++++++++++++++++++++- state/protocol/protocol_state/updater.go | 14 +- state/protocol/util.go | 8 +- 5 files changed, 198 insertions(+), 44 deletions(-) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index b9a279a0dcc..354ae94a3ce 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -25,20 +25,21 @@ import ( // Builder is the builder for consensus block payloads. Upon providing a payload // hash, it also memorizes which entities were included into the payload. type Builder struct { - metrics module.MempoolMetrics - tracer module.Tracer - db *badger.DB - state protocol.ParticipantState - seals storage.Seals - headers storage.Headers - index storage.Index - blocks storage.Blocks - resultsDB storage.ExecutionResults - receiptsDB storage.ExecutionReceipts - guarPool mempool.Guarantees - sealPool mempool.IncorporatedResultSeals - recPool mempool.ExecutionTree - cfg Config + metrics module.MempoolMetrics + tracer module.Tracer + db *badger.DB + state protocol.ParticipantState + seals storage.Seals + headers storage.Headers + index storage.Index + blocks storage.Blocks + resultsDB storage.ExecutionResults + receiptsDB storage.ExecutionReceipts + guarPool mempool.Guarantees + sealPool mempool.IncorporatedResultSeals + recPool mempool.ExecutionTree + protocolStateMutator protocol.StateMutator + cfg Config } // NewBuilder creates a new block builder. @@ -608,14 +609,6 @@ func (b *Builder) createProposal(parentID flow.Identifier, insertableReceipts *InsertableReceipts, setter func(*flow.Header) error) (*flow.Block, error) { - // build the payload so we can get the hash - payload := &flow.Payload{ - Guarantees: guarantees, - Seals: seals, - Receipts: insertableReceipts.receipts, - Results: insertableReceipts.results, - } - parent, err := b.headers.ByBlockID(parentID) if err != nil { return nil, fmt.Errorf("could not retrieve parent: %w", err) @@ -629,9 +622,15 @@ func (b *Builder) createProposal(parentID flow.Identifier, ParentID: parentID, Height: parent.Height + 1, Timestamp: timestamp, - PayloadHash: payload.Hash(), + PayloadHash: flow.ZeroID, } + updater, err := b.protocolStateMutator.CreateUpdater(header.View, header.ParentID) + + _, err = b.protocolStateMutator.ApplyServiceEvents(updater, payload.Seals) + + _, updatedStateID, _ := updater.Build() + // apply the custom fields setter of the consensus algorithm err = setter(header) if err != nil { @@ -639,9 +638,14 @@ func (b *Builder) createProposal(parentID flow.Identifier, } proposal := &flow.Block{ - Header: header, - Payload: payload, + Header: header, } + proposal.SetPayload(flow.Payload{ + Guarantees: guarantees, + Seals: seals, + Receipts: insertableReceipts.receipts, + Results: insertableReceipts.results, + }) return proposal, nil } diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 2a6a7ad0530..3258218b571 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -104,9 +104,9 @@ type StateUpdater interface { // - candidate block is in the next epoch. // No errors are expected during normal operations. TransitionToNextEpoch() error - // Block returns the block header that is associated with this state updater. - // StateUpdater is created for a specific block where protocol state changes are incorporated. - Block() *flow.Header + // View returns the view that is associated with this state updater. + // StateUpdater is created for a view where protocol state changes will be applied. + View() uint64 // ParentState returns parent protocol state that is associated with this state updater. ParentState() *flow.RichProtocolStateEntry } @@ -120,9 +120,11 @@ type StateMutator interface { // Has to be called for each block to correctly index the protocol state. // Expected errors during normal operations: // * `storage.ErrNotFound` if no protocol state for parent block is known. - CreateUpdater(candidate *flow.Header) (StateUpdater, error) + CreateUpdater(candidateView uint64, parentID flow.Identifier) (StateUpdater, error) // CommitProtocolState commits the protocol state to the database. // Has to be called for each block to correctly index the protocol state. // No errors are expected during normal operations. CommitProtocolState(updater StateUpdater) func(tx *transaction.Tx) error + + ApplyServiceEvents(updater StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 8db5bebdd44..d8476817569 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -3,6 +3,7 @@ package protocol_state import ( "errors" "fmt" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" @@ -16,6 +17,10 @@ import ( // is indexed by block ID, and we need to maintain such index. type Mutator struct { protocolStateDB storage.ProtocolState + headers storage.Headers + results storage.ExecutionResults + setups storage.EpochSetups + commits storage.EpochCommits } var _ protocol.StateMutator = (*Mutator)(nil) @@ -30,12 +35,12 @@ func NewMutator(protocolStateDB storage.ProtocolState) *Mutator { // Has to be called for each block to correctly index the protocol state. // Expected errors during normal operations: // - `storage.ErrNotFound` if no protocol state for parent block is known. -func (m *Mutator) CreateUpdater(candidate *flow.Header) (protocol.StateUpdater, error) { - parentState, err := m.protocolStateDB.ByBlockID(candidate.ParentID) +func (m *Mutator) CreateUpdater(candidateView uint64, parentID flow.Identifier) (protocol.StateUpdater, error) { + parentState, err := m.protocolStateDB.ByBlockID(parentID) if err != nil { - return nil, fmt.Errorf("could not retrieve protocol state for block (%v): %w", candidate.ParentID, err) + return nil, fmt.Errorf("could not retrieve protocol state for block (%v): %w", parentID, err) } - return NewUpdater(candidate, parentState), nil + return NewUpdater(candidateView, parentState), nil } // CommitProtocolState commits the protocol state updater as part of DB transaction. @@ -59,3 +64,146 @@ func (m *Mutator) CommitProtocolState(updater protocol.StateUpdater) func(tx *tr return nil } } + +// handleEpochServiceEvents handles applying state changes which occur as a result +// of service events being included in a block payload: +// - inserting incorporated service events +// - updating EpochStatus for the candidate block +// +// Consider a chain where a service event is emitted during execution of block A. +// Block B contains a receipt for A. Block C contains a seal for block A. +// +// A <- .. <- B(RA) <- .. <- C(SA) +// +// Service events are included within execution results, which are stored +// opaquely as part of the block payload in block B. We only validate and insert +// the typed service event to storage once we process C, the block containing the +// seal for block A. This is because we rely on the sealing subsystem to validate +// correctness of the service event before processing it. +// Consequently, any change to the protocol state introduced by a service event +// emitted during execution of block A would only become visible when querying +// C or its descendants. +// +// This method will only apply service-event-induced state changes when the +// input block has the form of block C (ie. contains a seal for a block in +// which a service event was emitted). +// +// Return values: +// - dbUpdates - If the service events are valid, or there are no service events, +// this method returns a slice of Badger operations to apply while storing the block. +// This includes an operation to index the epoch status for every block, and +// operations to insert service events for blocks that include them. +// +// No errors are expected during normal operation. +func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { + epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() + if err != nil { + return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) + } + + parentProtocolState := updater.ParentState() + epochStatus := parentProtocolState.EpochStatus() + activeSetup := parentProtocolState.CurrentEpochSetup + + // never process service events after epoch fallback is triggered + if epochStatus.InvalidServiceEventIncorporated || epochFallbackTriggered { + return dbUpdates, nil + } + + // perform protocol state transition to next epoch if next epoch is committed and we are at first block of epoch + phase, err := epochStatus.Phase() + if err != nil { + return nil, fmt.Errorf("could not determine epoch phase: %w", err) + } + if phase == flow.EpochPhaseCommitted { + if updater.View() > activeSetup.FinalView { + // TODO: this is a temporary workaround to allow for the epoch transition to be triggered + // most likely it will be not needed when we refactor protocol state entries and define strict safety rules. + err = updater.TransitionToNextEpoch() + if err != nil { + return nil, fmt.Errorf("could not transition protocol state to next epoch: %w", err) + } + } + } + + // We apply service events from blocks which are sealed by this candidate block. + // The block's payload might contain epoch preparation service events for the next + // epoch. In this case, we need to update the tentative protocol state. + // We need to validate whether all information is available in the protocol + // state to go to the next epoch when needed. In cases where there is a bug + // in the smart contract, it could be that this happens too late and the + // chain finalization should halt. + + // block payload may not specify seals in order, so order them by block height before processing + orderedSeals, err := protocol.OrderedSeals(seals, m.headers) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + for _, seal := range orderedSeals { + result, err := m.results.ByID(seal.ResultID) + if err != nil { + return nil, fmt.Errorf("could not get result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) + } + + for _, event := range result.ServiceEvents { + + switch ev := event.Event.(type) { + case *flow.EpochSetup: + // validate the service event + err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) + if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + updater.SetInvalidStateTransitionAttempted() + return dbUpdates, nil + } + return nil, fmt.Errorf("unexpected error validating EpochSetup service event: %w", err) + } + + err = updater.ProcessEpochSetup(ev) + if err != nil { + return nil, irrecoverable.NewExceptionf("could not process epoch setup event: %w", err) + } + + // we'll insert the setup event when we insert the block + dbUpdates = append(dbUpdates, m.setups.StoreTx(ev)) + + case *flow.EpochCommit: + // if we receive an EpochCommit event, we must have already observed an EpochSetup event + // => otherwise, we have observed an EpochCommit without corresponding EpochSetup, which triggers epoch fallback mode + if epochStatus.NextEpoch.SetupID == flow.ZeroID { + updater.SetInvalidStateTransitionAttempted() + return dbUpdates, nil + } + extendingSetup := parentProtocolState.NextEpochSetup + + // validate the service event + err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) + if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + updater.SetInvalidStateTransitionAttempted() + return dbUpdates, nil + } + return nil, fmt.Errorf("unexpected error validating EpochCommit service event: %w", err) + } + + err = updater.ProcessEpochCommit(ev) + if err != nil { + return nil, irrecoverable.NewExceptionf("could not process epoch commit event: %w", err) + } + + // we'll insert the commit event when we insert the block + dbUpdates = append(dbUpdates, m.commits.StoreTx(ev)) + case *flow.VersionBeacon: + // do nothing for now + default: + return nil, fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) + } + } + } + return +} diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 46b8fbeb415..8987fafae43 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -20,7 +20,7 @@ import ( type Updater struct { parentState *flow.RichProtocolStateEntry state *flow.ProtocolStateEntry - candidate *flow.Header + view uint64 // nextEpochIdentitiesLookup is a map from NodeID → DynamicIdentityEntry for the _current_ epoch, containing the // same identities as in the EpochStateContainer `state.CurrentEpoch.Identities`. Note that map values are pointers, @@ -36,11 +36,11 @@ type Updater struct { var _ protocol.StateUpdater = (*Updater)(nil) // NewUpdater creates a new protocol state updater. -func NewUpdater(candidate *flow.Header, parentState *flow.RichProtocolStateEntry) *Updater { +func NewUpdater(view uint64, parentState *flow.RichProtocolStateEntry) *Updater { updater := &Updater{ parentState: parentState, state: parentState.ProtocolStateEntry.Copy(), - candidate: candidate, + view: view, } return updater } @@ -241,10 +241,10 @@ func (u *Updater) TransitionToNextEpoch() error { return nil } -// Block returns the block header that is associated with this state updater. -// StateUpdater is created for a specific block where protocol state changes are incorporated. -func (u *Updater) Block() *flow.Header { - return u.candidate +// View returns the view that is associated with this state updater. +// StateUpdater is created for a view where protocol state changes will be applied. +func (u *Updater) View() uint64 { + return u.view } // ParentState returns parent protocol state that is associated with this state updater. diff --git a/state/protocol/util.go b/state/protocol/util.go index 0e74916a25c..af8b093182b 100644 --- a/state/protocol/util.go +++ b/state/protocol/util.go @@ -123,14 +123,14 @@ func FindGuarantors(state State, guarantee *flow.CollectionGuarantee) ([]flow.Id // - ErrMultipleSealsForSameHeight in case there are seals repeatedly sealing block at the same height // - ErrDiscontinuousSeals in case there are height-gaps in the sealed blocks // - storage.ErrNotFound if any of the seals references an unknown block -func OrderedSeals(payload *flow.Payload, headers storage.Headers) ([]*flow.Seal, error) { - numSeals := uint64(len(payload.Seals)) +func OrderedSeals(blockSeals []*flow.Seal, headers storage.Headers) ([]*flow.Seal, error) { + numSeals := uint64(len(blockSeals)) if numSeals == 0 { return nil, nil } heights := make([]uint64, numSeals) minHeight := uint64(math.MaxUint64) - for i, seal := range payload.Seals { + for i, seal := range blockSeals { header, err := headers.ByBlockID(seal.BlockID) if err != nil { return nil, fmt.Errorf("could not get block (id=%x) for seal: %w", seal.BlockID, err) // storage.ErrNotFound or exception @@ -143,7 +143,7 @@ func OrderedSeals(payload *flow.Payload, headers storage.Headers) ([]*flow.Seal, // As seals in a valid payload must have consecutive heights, we can populate // the ordered output by shifting by minHeight. seals := make([]*flow.Seal, numSeals) - for i, seal := range payload.Seals { + for i, seal := range blockSeals { idx := heights[i] - minHeight // (0) Per construction, `minHeight` is the smallest value in the `heights` slice. Hence, `idx ≥ 0` // (1) But if there are gaps in the heights of the sealed blocks (byzantine inputs), From 3a7f0c8bd7b368e6db8241af30b3aa090212ef5c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 9 Oct 2023 14:22:00 +0300 Subject: [PATCH 07/87] Updated usage of protocol state updater and mutator. Fixed some issues from last merge --- module/builder/consensus/builder.go | 14 +- state/protocol/badger/mutator.go | 168 +---------------------- state/protocol/badger/state.go | 18 +-- state/protocol/inmem/convert.go | 10 +- state/protocol/protocol_state.go | 4 +- state/protocol/protocol_state/mutator.go | 30 ++-- 6 files changed, 46 insertions(+), 198 deletions(-) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index 354ae94a3ce..d5c89fabbd5 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -626,10 +626,9 @@ func (b *Builder) createProposal(parentID flow.Identifier, } updater, err := b.protocolStateMutator.CreateUpdater(header.View, header.ParentID) + _, err = b.protocolStateMutator.ApplyServiceEvents(updater, seals) - _, err = b.protocolStateMutator.ApplyServiceEvents(updater, payload.Seals) - - _, updatedStateID, _ := updater.Build() + _, protocolStateID, _ := updater.Build() // apply the custom fields setter of the consensus algorithm err = setter(header) @@ -641,10 +640,11 @@ func (b *Builder) createProposal(parentID flow.Identifier, Header: header, } proposal.SetPayload(flow.Payload{ - Guarantees: guarantees, - Seals: seals, - Receipts: insertableReceipts.receipts, - Results: insertableReceipts.results, + Guarantees: guarantees, + Seals: seals, + Receipts: insertableReceipts.receipts, + Results: insertableReceipts.results, + ProtocolStateID: protocolStateID, }) return proposal, nil diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index 1960fac93c9..a3d0b6b77c0 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -18,7 +18,6 @@ import ( "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/state" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/badger/procedure" @@ -476,7 +475,7 @@ func (m *FollowerState) lastSealed(candidate *flow.Block) (*flow.Seal, error) { return last, nil } - ordered, err := protocol.OrderedSeals(payload, m.headers) + ordered, err := protocol.OrderedSeals(payload.Seals, m.headers) if err != nil { // all errors are unexpected - differentiation is for clearer error messages if errors.Is(err, storage.ErrNotFound) { @@ -511,30 +510,20 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi return fmt.Errorf("could not retrieve block header for %x: %w", parentID, err) } - parentProtocolState, err := m.protocolStateSnapshotsDB.ByBlockID(candidate.Header.ParentID) - if err != nil { - return fmt.Errorf("could not retrieve protocol state for block (%v): %w", candidate.Header.ParentID, err) - } - protocolStateUpdater := protocol_state.NewUpdater(candidate.Header, parentProtocolState) + protocolStateUpdater, err := m.protocolStateMutator.CreateUpdater(candidate.Header.View, parentID) // apply any state changes from service events sealed by this block - dbUpdates, err := m.handleEpochServiceEvents(candidate, protocolStateUpdater) + dbUpdates, err := m.protocolStateMutator.ApplyServiceEvents(protocolStateUpdater, candidate.Payload.Seals) if err != nil { return fmt.Errorf("could not process service events: %w", err) } - updatedState, updatedStateID, hasChanges := protocolStateUpdater.Build() + commitStateDbUpdate, updatedStateID := m.protocolStateMutator.CommitProtocolState(blockID, protocolStateUpdater) if updatedStateID != candidate.Payload.ProtocolStateID { return state.NewInvalidExtensionErrorf("invalid protocol state transition detected, "+ "payload contains (%x) but after applying changes got %x", candidate.Payload.ProtocolStateID, updatedStateID) } - - if hasChanges { - dbUpdates = append(dbUpdates, operation.SkipDuplicatesTx( - m.protocolStateSnapshotsDB.StoreTx(updatedStateID, updatedState), - )) - } - dbUpdates = append(dbUpdates, m.protocolStateSnapshotsDB.Index(blockID, updatedStateID)) + dbUpdates = append(dbUpdates, commitStateDbUpdate) // events is a queue of node-internal events (aka notifications) that are emitted after the database write succeeded var events []func() @@ -915,7 +904,7 @@ func (m *FollowerState) epochPhaseMetricsAndEventsOnBlockFinalized(block *flow.B ) { // block payload may not specify seals in order, so order them by block height before processing - orderedSeals, err := protocol.OrderedSeals(block.Payload, m.headers) + orderedSeals, err := protocol.OrderedSeals(block.Payload.Seals, m.headers) if err != nil { if errors.Is(err, storage.ErrNotFound) { return nil, nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) @@ -968,7 +957,7 @@ func (m *FollowerState) versionBeaconOnBlockFinalized( ) ([]*flow.SealedVersionBeacon, error) { var versionBeacons []*flow.SealedVersionBeacon - seals, err := protocol.OrderedSeals(finalized.Payload, m.headers) + seals, err := protocol.OrderedSeals(finalized.Payload.Seals, m.headers) if err != nil { if errors.Is(err, storage.ErrNotFound) { return nil, fmt.Errorf( @@ -1019,146 +1008,3 @@ func (m *FollowerState) versionBeaconOnBlockFinalized( return versionBeacons, nil } - -// handleEpochServiceEvents handles applying state changes which occur as a result -// of service events being included in a block payload: -// - inserting incorporated service events -// - updating EpochStatus for the candidate block -// -// Consider a chain where a service event is emitted during execution of block A. -// Block B contains a receipt for A. Block C contains a seal for block A. -// -// A <- .. <- B(RA) <- .. <- C(SA) -// -// Service events are included within execution results, which are stored -// opaquely as part of the block payload in block B. We only validate and insert -// the typed service event to storage once we process C, the block containing the -// seal for block A. This is because we rely on the sealing subsystem to validate -// correctness of the service event before processing it. -// Consequently, any change to the protocol state introduced by a service event -// emitted during execution of block A would only become visible when querying -// C or its descendants. -// -// This method will only apply service-event-induced state changes when the -// input block has the form of block C (ie. contains a seal for a block in -// which a service event was emitted). -// -// Return values: -// - dbUpdates - If the service events are valid, or there are no service events, -// this method returns a slice of Badger operations to apply while storing the block. -// This includes an operation to index the epoch status for every block, and -// operations to insert service events for blocks that include them. -// -// No errors are expected during normal operation. -func (m *FollowerState) handleEpochServiceEvents(candidate *flow.Block, updater protocol.StateUpdater) (dbUpdates []func(*transaction.Tx) error, err error) { - epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() - if err != nil { - return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) - } - - parentProtocolState := updater.ParentState() - epochStatus := parentProtocolState.EpochStatus() - activeSetup := parentProtocolState.CurrentEpochSetup - - // never process service events after epoch fallback is triggered - if epochStatus.InvalidServiceEventIncorporated || epochFallbackTriggered { - return dbUpdates, nil - } - - // perform protocol state transition to next epoch if next epoch is committed and we are at first block of epoch - phase, err := epochStatus.Phase() - if err != nil { - return nil, fmt.Errorf("could not determine epoch phase: %w", err) - } - if phase == flow.EpochPhaseCommitted { - if candidate.Header.View > activeSetup.FinalView { - // TODO: this is a temporary workaround to allow for the epoch transition to be triggered - // most likely it will be not needed when we refactor protocol state entries and define strict safety rules. - err = updater.TransitionToNextEpoch() - if err != nil { - return nil, fmt.Errorf("could not transition protocol state to next epoch: %w", err) - } - } - } - - // We apply service events from blocks which are sealed by this candidate block. - // The block's payload might contain epoch preparation service events for the next - // epoch. In this case, we need to update the tentative protocol state. - // We need to validate whether all information is available in the protocol - // state to go to the next epoch when needed. In cases where there is a bug - // in the smart contract, it could be that this happens too late and the - // chain finalization should halt. - - // block payload may not specify seals in order, so order them by block height before processing - orderedSeals, err := protocol.OrderedSeals(candidate.Payload, m.headers) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - return nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) - } - return nil, fmt.Errorf("unexpected error ordering seals: %w", err) - } - for _, seal := range orderedSeals { - result, err := m.results.ByID(seal.ResultID) - if err != nil { - return nil, fmt.Errorf("could not get result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) - } - - for _, event := range result.ServiceEvents { - - switch ev := event.Event.(type) { - case *flow.EpochSetup: - // validate the service event - err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) - if err != nil { - if protocol.IsInvalidServiceEventError(err) { - // we have observed an invalid service event, which triggers epoch fallback mode - updater.SetInvalidStateTransitionAttempted() - return dbUpdates, nil - } - return nil, fmt.Errorf("unexpected error validating EpochSetup service event: %w", err) - } - - err = updater.ProcessEpochSetup(ev) - if err != nil { - return nil, irrecoverable.NewExceptionf("could not process epoch setup event: %w", err) - } - - // we'll insert the setup event when we insert the block - dbUpdates = append(dbUpdates, m.epoch.setups.StoreTx(ev)) - - case *flow.EpochCommit: - // if we receive an EpochCommit event, we must have already observed an EpochSetup event - // => otherwise, we have observed an EpochCommit without corresponding EpochSetup, which triggers epoch fallback mode - if epochStatus.NextEpoch.SetupID == flow.ZeroID { - updater.SetInvalidStateTransitionAttempted() - return dbUpdates, nil - } - extendingSetup := parentProtocolState.NextEpochSetup - - // validate the service event - err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) - if err != nil { - if protocol.IsInvalidServiceEventError(err) { - // we have observed an invalid service event, which triggers epoch fallback mode - updater.SetInvalidStateTransitionAttempted() - return dbUpdates, nil - } - return nil, fmt.Errorf("unexpected error validating EpochCommit service event: %w", err) - } - - err = updater.ProcessEpochCommit(ev) - if err != nil { - return nil, irrecoverable.NewExceptionf("could not process epoch commit event: %w", err) - } - - // we'll insert the commit event when we insert the block - dbUpdates = append(dbUpdates, m.epoch.commits.StoreTx(ev)) - case *flow.VersionBeacon: - // do nothing for now - default: - return nil, fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) - } - } - } - return -} diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index 7e2472fd40c..a9f7bffe1a5 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -39,9 +39,9 @@ type State struct { setups storage.EpochSetups commits storage.EpochCommits } - protocolStateSnapshotsDB storage.ProtocolState - protocolState protocol.ProtocolState - versionBeacons storage.VersionBeacons + protocolStateMutator protocol.StateMutator + protocolState protocol.ProtocolState + versionBeacons storage.VersionBeacons // rootHeight marks the cutoff of the history this node knows about. We cache it in the state // because it cannot change over the lifecycle of a protocol state instance. It is frequently @@ -769,7 +769,7 @@ func newState( qcs storage.QuorumCertificates, setups storage.EpochSetups, commits storage.EpochCommits, - protocolState storage.ProtocolState, + protocolStateMutator protocol.StateMutator, protocolStateReader protocol.ProtocolState, versionBeacons storage.VersionBeacons, ) *State { @@ -788,11 +788,11 @@ func newState( setups: setups, commits: commits, }, - protocolStateSnapshotsDB: protocolState, - protocolState: protocolStateReader, - versionBeacons: versionBeacons, - cachedFinal: new(atomic.Pointer[cachedHeader]), - cachedSealed: new(atomic.Pointer[cachedHeader]), + protocolState: protocolStateReader, + protocolStateMutator: protocolStateMutator, + versionBeacons: versionBeacons, + cachedFinal: new(atomic.Pointer[cachedHeader]), + cachedSealed: new(atomic.Pointer[cachedHeader]), } } diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index 0d7312e91de..8a09c11839c 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -323,11 +323,11 @@ func SnapshotFromBootstrapStateWithParams( EpochCommitSafetyThreshold: epochCommitSafetyThreshold, // see protocol.Params for details } - rootProtocolState := ProtocolStateForBootstrapState(setup, commit) if rootProtocolState.ID() != root.Payload.ProtocolStateID { return nil, fmt.Errorf("incorrect protocol state ID in root block, expected (%x) but got (%x)", root.Payload.ProtocolStateID, rootProtocolState.ID()) + } snap := SnapshotFromEncodable(EncodableSnapshot{ Head: root.Header, @@ -355,10 +355,10 @@ func ProtocolStateForBootstrapState(setup *flow.EpochSetup, commit *flow.EpochCo return &flow.ProtocolStateEntry{ PreviousEpochEventIDs: flow.EventIDs{}, CurrentEpoch: flow.EpochStateContainer{ - SetupID: setup.ID(), - CommitID: commit.ID(), - ActiveIdentities: flow.DynamicIdentityEntryListFromIdentities(setup.Participants), - }, + SetupID: setup.ID(), + CommitID: commit.ID(), + ActiveIdentities: flow.DynamicIdentityEntryListFromIdentities(setup.Participants), + }, NextEpoch: nil, InvalidStateTransitionAttempted: false, } diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 3258218b571..6297a1c8b52 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -124,7 +124,7 @@ type StateMutator interface { // CommitProtocolState commits the protocol state to the database. // Has to be called for each block to correctly index the protocol state. // No errors are expected during normal operations. - CommitProtocolState(updater StateUpdater) func(tx *transaction.Tx) error + CommitProtocolState(blockID flow.Identifier, updater StateUpdater) (func(tx *transaction.Tx) error, flow.Identifier) - ApplyServiceEvents(updater StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { + ApplyServiceEvents(updater StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index d8476817569..72a83d28734 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -46,9 +46,9 @@ func (m *Mutator) CreateUpdater(candidateView uint64, parentID flow.Identifier) // CommitProtocolState commits the protocol state updater as part of DB transaction. // Has to be called for each block to correctly index the protocol state. // No errors are expected during normal operations. -func (m *Mutator) CommitProtocolState(updater protocol.StateUpdater) func(tx *transaction.Tx) error { +func (m *Mutator) CommitProtocolState(blockID flow.Identifier, updater protocol.StateUpdater) (func(tx *transaction.Tx) error, flow.Identifier) { + updatedState, updatedStateID, hasChanges := updater.Build() return func(tx *transaction.Tx) error { - updatedState, updatedStateID, hasChanges := updater.Build() if hasChanges { err := m.protocolStateDB.StoreTx(updatedStateID, updatedState)(tx) if err != nil && !errors.Is(err, storage.ErrAlreadyExists) { @@ -56,13 +56,13 @@ func (m *Mutator) CommitProtocolState(updater protocol.StateUpdater) func(tx *tr } } - err := m.protocolStateDB.Index(updater.Block().ID(), updatedStateID)(tx) + err := m.protocolStateDB.Index(blockID, updatedStateID)(tx) if err != nil { return fmt.Errorf("could not index protocol state (%v) for block (%v): %w", - updatedStateID, updater.Block().ID(), err) + updatedStateID, blockID, err) } return nil - } + }, updatedStateID } // handleEpochServiceEvents handles applying state changes which occur as a result @@ -96,10 +96,12 @@ func (m *Mutator) CommitProtocolState(updater protocol.StateUpdater) func(tx *tr // // No errors are expected during normal operation. func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { - epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() - if err != nil { - return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) - } + // TODO: hook up epoch fallback mode + epochFallbackTriggered := false + //epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() + //if err != nil { + // return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) + //} parentProtocolState := updater.ParentState() epochStatus := parentProtocolState.EpochStatus() @@ -152,8 +154,8 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo switch ev := event.Event.(type) { case *flow.EpochSetup: - // validate the service event - err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) + // TODO: validate the service event + //err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) if err != nil { if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode @@ -178,10 +180,10 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo updater.SetInvalidStateTransitionAttempted() return dbUpdates, nil } - extendingSetup := parentProtocolState.NextEpochSetup + //extendingSetup := parentProtocolState.NextEpochSetup - // validate the service event - err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) + // TODO: validate the service event + //err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) if err != nil { if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode From 031ebb3c85b8d3000f52052a4efd73effb2f5608 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 9 Oct 2023 14:29:40 +0300 Subject: [PATCH 08/87] Updated usages of state mutator --- state/protocol/badger/state.go | 6 ++++-- state/protocol/protocol_state/mutator.go | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index a9f7bffe1a5..561eb959470 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -110,6 +110,7 @@ func Bootstrap( return nil, fmt.Errorf("expected empty database") } + protocolStateMutator := protocol_state.NewMutator(headers, results, setups, commits, protocolStateSnapshotsDB) protocolState := protocol_state.NewProtocolState(protocolStateSnapshotsDB, root.Params()) state := newState( metrics, @@ -121,7 +122,7 @@ func Bootstrap( qcs, setups, commits, - protocolStateSnapshotsDB, + protocolStateMutator, protocolState, versionBeacons, ) @@ -648,6 +649,7 @@ func OpenState( if err != nil { return nil, fmt.Errorf("could not read global params") } + protocolStateMutator := protocol_state.NewMutator(headers, results, setups, commits, protocolState) protocolStateReader := protocol_state.NewProtocolState(protocolState, globalParams) state := newState( metrics, @@ -659,7 +661,7 @@ func OpenState( qcs, setups, commits, - protocolState, + protocolStateMutator, protocolStateReader, versionBeacons, ) // populate the protocol state cache diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 72a83d28734..f1d081b96a3 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -16,17 +16,27 @@ import ( // service events sealed in candidate block. This requirement is due to the fact that protocol state // is indexed by block ID, and we need to maintain such index. type Mutator struct { - protocolStateDB storage.ProtocolState headers storage.Headers results storage.ExecutionResults setups storage.EpochSetups commits storage.EpochCommits + protocolStateDB storage.ProtocolState } var _ protocol.StateMutator = (*Mutator)(nil) -func NewMutator(protocolStateDB storage.ProtocolState) *Mutator { +func NewMutator( + headers storage.Headers, + results storage.ExecutionResults, + setups storage.EpochSetups, + commits storage.EpochCommits, + protocolStateDB storage.ProtocolState, +) *Mutator { return &Mutator{ + headers: headers, + results: results, + setups: setups, + commits: commits, protocolStateDB: protocolStateDB, } } From e233f05283bb270c764af67d9b03e77c20793e63 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 9 Oct 2023 21:38:45 +0300 Subject: [PATCH 09/87] Fixed compilation issue in updater --- state/protocol/protocol_state/updater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 8987fafae43..45d254f6787 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -229,7 +229,7 @@ func (u *Updater) TransitionToNextEpoch() error { return fmt.Errorf("invalid state transition has been attempted, no transition is allowed") } // Check if we are at the next epoch, only then a transition is allowed - if u.candidate.View < u.parentState.NextEpochSetup.FirstView { + if u.view < u.parentState.NextEpochSetup.FirstView { return fmt.Errorf("protocol state transition is only allowed when enterring next epoch") } u.state = &flow.ProtocolStateEntry{ From df1939b10fa3a80af8327a0b9995ea4efb874957 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 9 Oct 2023 23:44:56 +0300 Subject: [PATCH 10/87] Updated mutator happy path test --- state/protocol/badger/mutator_test.go | 41 +++++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 557c14392f8..1265deb3373 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -5,6 +5,7 @@ package badger_test import ( "context" "errors" + "github.com/onflow/flow-go/state/protocol/protocol_state" "math/rand" "sync" "testing" @@ -797,6 +798,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { consumer.On("BlockFinalized", mock.Anything) consumer.On("BlockProcessable", mock.Anything, mock.Anything) rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) unittest.RunWithBadgerDB(t, func(db *badger.DB) { // set up state and mock ComplianceMetrics object @@ -859,6 +861,20 @@ func TestExtendEpochTransitionValid(t *testing.T) { ) require.NoError(t, err) + stateMutator := protocol_state.NewMutator(all.Headers, all.Results, all.Setups, all.EpochCommits, all.ProtocolState) + + // a utility function which makes easier to get expected protocol state ID after applying service events contained in seals. + calculateExpectedStateId := func(header *flow.Header, seals []*flow.Seal) flow.Identifier { + updater, err := stateMutator.CreateUpdater(header.View, header.ParentID) + require.NoError(t, err) + + _, err = stateMutator.ApplyServiceEvents(updater, seals) + require.NoError(t, err) + + _, expectedStateID, _ := updater.Build() + return expectedStateID + } + head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -871,7 +887,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) err = state.Finalize(context.Background(), block1.ID()) @@ -899,7 +915,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { // add a second block with the receipt for block 1 block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1), unittest.WithProtocolStateID(block1.Payload.ProtocolStateID))) err = state.Extend(context.Background(), block2) require.NoError(t, err) @@ -907,9 +923,11 @@ func TestExtendEpochTransitionValid(t *testing.T) { require.NoError(t, err) // block 3 contains the seal for block 1 + seals := []*flow.Seal{seal1} block3 := unittest.BlockWithParentFixture(block2.Header) block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, + Seals: seals, + ProtocolStateID: calculateExpectedStateId(block3.Header, seals), }) // insert the block sealing the EpochSetup event @@ -940,7 +958,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { require.Error(t, err) // insert B4 - block4 := unittest.BlockWithParentFixture(block3.Header) + block4 := unittest.BlockWithParentProtocolState(block3) err = state.Extend(context.Background(), block4) require.NoError(t, err) @@ -976,7 +994,8 @@ func TestExtendEpochTransitionValid(t *testing.T) { // block 5 contains the receipt for block 2 block5 := unittest.BlockWithParentFixture(block4.Header) - block5.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2))) + block5.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), + unittest.WithProtocolStateID(block4.Payload.ProtocolStateID))) err = state.Extend(context.Background(), block5) require.NoError(t, err) @@ -984,9 +1003,11 @@ func TestExtendEpochTransitionValid(t *testing.T) { require.NoError(t, err) // block 6 contains the seal for block 2 + seals = []*flow.Seal{seal2} block6 := unittest.BlockWithParentFixture(block5.Header) block6.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal2}, + Seals: seals, + ProtocolStateID: calculateExpectedStateId(block6.Header, seals), }) err = state.Extend(context.Background(), block6) @@ -1012,8 +1033,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { require.Equal(t, flow.EpochPhaseCommitted, phase) // block 7 has the final view of the epoch, insert it, finalized after finalizing block 6 - block7 := unittest.BlockWithParentFixture(block6.Header) - block7.SetPayload(flow.EmptyPayload()) + block7 := unittest.BlockWithParentProtocolState(block6) block7.Header.View = epoch1FinalView err = state.Extend(context.Background(), block7) require.NoError(t, err) @@ -1046,9 +1066,12 @@ func TestExtendEpochTransitionValid(t *testing.T) { // block 8 has a view > final view of epoch 1, it will be considered the first block of epoch 2 block8 := unittest.BlockWithParentFixture(block7.Header) - block8.SetPayload(flow.EmptyPayload()) // we should handle views that aren't exactly the first valid view of the epoch block8.Header.View = epoch1FinalView + uint64(1+rand.Intn(10)) + // need to update root protocol state since we enter new epoch + block8.SetPayload( + unittest.PayloadFixture( + unittest.WithProtocolStateID(calculateExpectedStateId(block8.Header, nil)))) err = state.Extend(context.Background(), block8) require.NoError(t, err) From e097f4f5bf4bdc0f4c0dc1b20ed5c1eeaf7846f4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 11:54:43 +0300 Subject: [PATCH 11/87] Added an utility function to make easier extend testing. Updated test duplicated epoch events --- state/protocol/badger/mutator_test.go | 59 +++++++++++++++------------ state/protocol/util/testing.go | 43 +++++++++++++++++++ 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 1265deb3373..0496d5e1adb 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -862,18 +862,7 @@ func TestExtendEpochTransitionValid(t *testing.T) { require.NoError(t, err) stateMutator := protocol_state.NewMutator(all.Headers, all.Results, all.Setups, all.EpochCommits, all.ProtocolState) - - // a utility function which makes easier to get expected protocol state ID after applying service events contained in seals. - calculateExpectedStateId := func(header *flow.Header, seals []*flow.Seal) flow.Identifier { - updater, err := stateMutator.CreateUpdater(header.View, header.ParentID) - require.NoError(t, err) - - _, err = stateMutator.ApplyServiceEvents(updater, seals) - require.NoError(t, err) - - _, expectedStateID, _ := updater.Build() - return expectedStateID - } + calculateExpectedStateId := calculateExpectedStateId(t, stateMutator) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -1124,7 +1113,9 @@ func TestExtendEpochTransitionValid(t *testing.T) { // \--B2<--B4(R2)<--B6(S2)<--B8 func TestExtendConflictingEpochEvents(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, stateMutator realprotocol.StateMutator) { + calculateExpectedStateId := calculateExpectedStateId(t, stateMutator) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -1133,12 +1124,12 @@ func TestExtendConflictingEpochEvents(t *testing.T) { // add two conflicting blocks for each service event to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) block2 := unittest.BlockWithParentFixture(head) - block2.SetPayload(flow.EmptyPayload()) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block2) require.NoError(t, err) @@ -1166,8 +1157,9 @@ func TestExtendConflictingEpochEvents(t *testing.T) { // add block 1 receipt to block 3 payload block3 := unittest.BlockWithParentFixture(block1.Header) block3.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()}, - Results: []*flow.ExecutionResult{&block1Receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()}, + Results: []*flow.ExecutionResult{&block1Receipt.ExecutionResult}, + ProtocolStateID: block1.Payload.ProtocolStateID, }) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -1179,22 +1171,24 @@ func TestExtendConflictingEpochEvents(t *testing.T) { // add block 2 receipt to block 4 payload block4 := unittest.BlockWithParentFixture(block2.Header) block4.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{block2Receipt.Meta()}, - Results: []*flow.ExecutionResult{&block2Receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{block2Receipt.Meta()}, + Results: []*flow.ExecutionResult{&block2Receipt.ExecutionResult}, + ProtocolStateID: block1.Payload.ProtocolStateID, }) err = state.Extend(context.Background(), block4) require.NoError(t, err) // seal for block 1 - seal1 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) + seals1 := []*flow.Seal{unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult))} // seal for block 2 - seal2 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult)) + seals2 := []*flow.Seal{unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult))} // block 5 builds on block 3, contains seal for block 1 block5 := unittest.BlockWithParentFixture(block3.Header) block5.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, + Seals: seals1, + ProtocolStateID: calculateExpectedStateId(block5.Header, seals1), }) err = state.Extend(context.Background(), block5) require.NoError(t, err) @@ -1202,18 +1196,19 @@ func TestExtendConflictingEpochEvents(t *testing.T) { // block 6 builds on block 4, contains seal for block 2 block6 := unittest.BlockWithParentFixture(block4.Header) block6.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal2}, + Seals: seals2, + ProtocolStateID: calculateExpectedStateId(block6.Header, seals2), }) err = state.Extend(context.Background(), block6) require.NoError(t, err) // block 7 builds on block 5, contains QC for block 7 - block7 := unittest.BlockWithParentFixture(block5.Header) + block7 := unittest.BlockWithParentProtocolState(block5) err = state.Extend(context.Background(), block7) require.NoError(t, err) // block 8 builds on block 6, contains QC for block 6 - block8 := unittest.BlockWithParentFixture(block6.Header) + block8 := unittest.BlockWithParentProtocolState(block6) err = state.Extend(context.Background(), block8) require.NoError(t, err) @@ -2626,3 +2621,17 @@ func getRootProtocolStateID(t *testing.T, rootSnapshot *inmem.Snapshot) flow.Ide require.NoError(t, err) return rootProtocolState.Entry().ID() } + +// calculateExpectedStateId is a utility function which makes easier to get expected protocol state ID after applying service events contained in seals. +func calculateExpectedStateId(t *testing.T, stateMutator realprotocol.StateMutator) func(header *flow.Header, seals []*flow.Seal) flow.Identifier { + return func(header *flow.Header, seals []*flow.Seal) flow.Identifier { + updater, err := stateMutator.CreateUpdater(header.View, header.ParentID) + require.NoError(t, err) + + _, err = stateMutator.ApplyServiceEvents(updater, seals) + require.NoError(t, err) + + _, expectedStateID, _ := updater.Build() + return expectedStateID + } +} diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index 16bbac99bfe..b5d2e07d3e0 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -1,6 +1,7 @@ package util import ( + "github.com/onflow/flow-go/state/protocol/protocol_state" "testing" "github.com/dgraph-io/badger/v2" @@ -359,3 +360,45 @@ func RunWithFollowerProtocolStateAndHeaders(t testing.TB, rootSnapshot protocol. f(db, followerState, all.Headers, all.Index) }) } + +func RunWithFullProtocolStateAndMutator(t testing.TB, rootSnapshot protocol.Snapshot, f func(*badger.DB, *pbadger.ParticipantState, protocol.StateMutator)) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + tracer := trace.NewNoopTracer() + log := zerolog.Nop() + consumer := events.NewNoop() + all := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.ProtocolState, + all.VersionBeacons, + rootSnapshot, + ) + require.NoError(t, err) + receiptValidator := MockReceiptValidator() + sealValidator := MockSealValidator(all.Seals) + mockTimer := MockBlockTimer() + fullState, err := pbadger.NewFullConsensusState( + log, + tracer, + consumer, + state, + all.Index, + all.Payloads, + mockTimer, + receiptValidator, + sealValidator, + ) + require.NoError(t, err) + mutator := protocol_state.NewMutator(all.Headers, all.Results, all.Setups, all.EpochCommits, all.ProtocolState) + f(db, fullState, mutator) + }) +} From f97e06adbf678ce0b4abd9e331fdd3dfe2717d0e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 12:25:19 +0300 Subject: [PATCH 12/87] Extracted epoch validity functions into a different module. Updated related code --- state/protocol/badger/state.go | 12 +- state/protocol/badger/validity.go | 180 --------------------- state/protocol/badger/validity_test.go | 16 +- state/protocol/protocol_state/mutator.go | 8 +- state/protocol/validity.go | 189 +++++++++++++++++++++++ state/protocol/validity_test.go | 104 +++++++++++++ 6 files changed, 310 insertions(+), 199 deletions(-) create mode 100644 state/protocol/validity.go create mode 100644 state/protocol/validity_test.go diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index 561eb959470..d24427bd2a7 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -468,10 +468,10 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get previous epoch commit event: %w", err) } - if err := verifyEpochSetup(setup, verifyNetworkAddress); err != nil { + if err := protocol.VerifyEpochSetup(setup, verifyNetworkAddress); err != nil { return fmt.Errorf("invalid setup: %w", err) } - if err := isValidEpochCommit(commit, setup); err != nil { + if err := protocol.IsValidEpochCommit(commit, setup); err != nil { return fmt.Errorf("invalid commit: %w", err) } @@ -498,10 +498,10 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get current epoch commit event: %w", err) } - if err := verifyEpochSetup(setup, verifyNetworkAddress); err != nil { + if err := protocol.VerifyEpochSetup(setup, verifyNetworkAddress); err != nil { return fmt.Errorf("invalid setup: %w", err) } - if err := isValidEpochCommit(commit, setup); err != nil { + if err := protocol.IsValidEpochCommit(commit, setup); err != nil { return fmt.Errorf("invalid commit: %w", err) } @@ -524,7 +524,7 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get next epoch setup event: %w", err) } - if err := verifyEpochSetup(setup, verifyNetworkAddress); err != nil { + if err := protocol.VerifyEpochSetup(setup, verifyNetworkAddress); err != nil { return fmt.Errorf("invalid setup: %w", err) } @@ -535,7 +535,7 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get next epoch commit event: %w", err) } if err == nil { - if err := isValidEpochCommit(commit, setup); err != nil { + if err := protocol.IsValidEpochCommit(commit, setup); err != nil { return fmt.Errorf("invalid commit") } commits = append(commits, commit) diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 06d14483744..a88c3f940a9 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -8,191 +8,11 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/state/protocol" ) -// isValidExtendingEpochSetup checks whether an epoch setup service being -// added to the state is valid. In addition to intrinsic validity, we also -// check that it is valid w.r.t. the previous epoch setup event, and the -// current epoch status. -// Assumes all inputs besides extendingSetup are already validated. -// Expected errors during normal operations: -// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status -func isValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { - // We should only have a single epoch setup event per epoch. - if status.NextEpoch.SetupID != flow.ZeroID { - // true iff EpochSetup event for NEXT epoch was already included before - return protocol.NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) - } - - // The setup event should have the counter increased by one. - if extendingSetup.Counter != activeSetup.Counter+1 { - return protocol.NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) - } - - // The first view needs to be exactly one greater than the current epoch final view - if extendingSetup.FirstView != activeSetup.FinalView+1 { - return protocol.NewInvalidServiceEventErrorf( - "next epoch first view must be exactly 1 more than current epoch final view (%d != %d+1)", - extendingSetup.FirstView, - activeSetup.FinalView, - ) - } - - // Finally, the epoch setup event must contain all necessary information. - err := verifyEpochSetup(extendingSetup, true) - if err != nil { - return protocol.NewInvalidServiceEventErrorf("invalid epoch setup: %w", err) - } - - return nil -} - -// verifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. -// The boolean parameter `verifyNetworkAddress` controls, whether we want to permit -// nodes to share a networking address. -// This is a side-effect-free function. Any error return indicates that the -// EpochSetup event is not compliant with protocol rules. -func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { - // STEP 1: general sanity checks - // the seed needs to be at least minimum length - if len(setup.RandomSource) != flow.EpochSetupRandomSourceLength { - return fmt.Errorf("seed has incorrect length (%d != %d)", len(setup.RandomSource), flow.EpochSetupRandomSourceLength) - } - - // STEP 2: sanity checks of all nodes listed as participants - // there should be no duplicate node IDs - identLookup := make(map[flow.Identifier]struct{}) - for _, participant := range setup.Participants { - _, ok := identLookup[participant.NodeID] - if ok { - return fmt.Errorf("duplicate node identifier (%x)", participant.NodeID) - } - identLookup[participant.NodeID] = struct{}{} - } - - if verifyNetworkAddress { - // there should be no duplicate node addresses - addrLookup := make(map[string]struct{}) - for _, participant := range setup.Participants { - _, ok := addrLookup[participant.Address] - if ok { - return fmt.Errorf("duplicate node address (%x)", participant.Address) - } - addrLookup[participant.Address] = struct{}{} - } - } - - // the participants must be listed in canonical order - if !setup.Participants.Sorted(order.Canonical) { - return fmt.Errorf("participants are not canonically ordered") - } - - // STEP 3: sanity checks for individual roles - // IMPORTANT: here we remove all nodes with zero weight, as they are allowed to partake - // in communication but not in respective node functions - activeParticipants := setup.Participants.Filter(filter.HasWeight(true)) - - // we need at least one node of each role - roles := make(map[flow.Role]uint) - for _, participant := range activeParticipants { - roles[participant.Role]++ - } - if roles[flow.RoleConsensus] < 1 { - return fmt.Errorf("need at least one consensus node") - } - if roles[flow.RoleCollection] < 1 { - return fmt.Errorf("need at least one collection node") - } - if roles[flow.RoleExecution] < 1 { - return fmt.Errorf("need at least one execution node") - } - if roles[flow.RoleVerification] < 1 { - return fmt.Errorf("need at least one verification node") - } - - // first view must be before final view - if setup.FirstView >= setup.FinalView { - return fmt.Errorf("first view (%d) must be before final view (%d)", setup.FirstView, setup.FinalView) - } - - // we need at least one collection cluster - if len(setup.Assignments) == 0 { - return fmt.Errorf("need at least one collection cluster") - } - - // the collection cluster assignments need to be valid - _, err := factory.NewClusterList(setup.Assignments, activeParticipants.Filter(filter.HasRole(flow.RoleCollection))) - if err != nil { - return fmt.Errorf("invalid cluster assignments: %w", err) - } - - return nil -} - -// isValidExtendingEpochCommit checks whether an epoch commit service being -// added to the state is valid. In addition to intrinsic validity, we also -// check that it is valid w.r.t. the previous epoch setup event, and the -// current epoch status. -// Assumes all inputs besides extendingCommit are already validated. -// Expected errors during normal operations: -// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status -func isValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { - - // We should only have a single epoch commit event per epoch. - if status.NextEpoch.CommitID != flow.ZeroID { - // true iff EpochCommit event for NEXT epoch was already included before - return protocol.NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) - } - - // The epoch setup event needs to happen before the commit. - if status.NextEpoch.SetupID == flow.ZeroID { - return protocol.NewInvalidServiceEventErrorf("missing epoch setup for epoch commit") - } - - // The commit event should have the counter increased by one. - if extendingCommit.Counter != activeSetup.Counter+1 { - return protocol.NewInvalidServiceEventErrorf("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) - } - - err := isValidEpochCommit(extendingCommit, extendingSetup) - if err != nil { - return protocol.NewInvalidServiceEventErrorf("invalid epoch commit: %s", err) - } - - return nil -} - -// isValidEpochCommit checks whether an epoch commit service event is intrinsically valid. -// Assumes the input flow.EpochSetup event has already been validated. -// Expected errors during normal operations: -// * protocol.InvalidServiceEventError if the EpochCommit is invalid -func isValidEpochCommit(commit *flow.EpochCommit, setup *flow.EpochSetup) error { - - if len(setup.Assignments) != len(commit.ClusterQCs) { - return protocol.NewInvalidServiceEventErrorf("number of clusters (%d) does not number of QCs (%d)", len(setup.Assignments), len(commit.ClusterQCs)) - } - - if commit.Counter != setup.Counter { - return protocol.NewInvalidServiceEventErrorf("inconsistent epoch counter between commit (%d) and setup (%d) events in same epoch", commit.Counter, setup.Counter) - } - - // make sure we have a valid DKG public key - if commit.DKGGroupKey == nil { - return protocol.NewInvalidServiceEventErrorf("missing DKG public group key") - } - - participants := setup.Participants.Filter(filter.IsValidDKGParticipant) - if len(participants) != len(commit.DKGParticipantKeys) { - return protocol.NewInvalidServiceEventErrorf("participant list (len=%d) does not match dkg key list (len=%d)", len(participants), len(commit.DKGParticipantKeys)) - } - - return nil -} - // IsValidRootSnapshot checks internal consistency of root state snapshot // if verifyResultID allows/disallows Result ID verification func IsValidRootSnapshot(snap protocol.Snapshot, verifyResultID bool) error { diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index 30ee94c40d6..efb187bbcfd 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -22,7 +22,7 @@ func TestEpochSetupValidity(t *testing.T) { // set an invalid final view for the first epoch setup.FinalView = setup.FirstView - err := verifyEpochSetup(setup, true) + err := protocol.VerifyEpochSetup(setup, true) require.Error(t, err) }) @@ -33,7 +33,7 @@ func TestEpochSetupValidity(t *testing.T) { var err error setup.Participants, err = setup.Participants.Shuffle() require.NoError(t, err) - err = verifyEpochSetup(setup, true) + err = protocol.VerifyEpochSetup(setup, true) require.Error(t, err) }) @@ -44,7 +44,7 @@ func TestEpochSetupValidity(t *testing.T) { collector := participants.Filter(filter.HasRole(flow.RoleCollection))[0] setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) - err := verifyEpochSetup(setup, true) + err := protocol.VerifyEpochSetup(setup, true) require.Error(t, err) }) @@ -53,7 +53,7 @@ func TestEpochSetupValidity(t *testing.T) { setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) setup.RandomSource = unittest.SeedFixture(crypto.SeedMinLenDKG - 1) - err := verifyEpochSetup(setup, true) + err := protocol.VerifyEpochSetup(setup, true) require.Error(t, err) }) } @@ -66,7 +66,7 @@ func TestBootstrapInvalidEpochCommit(t *testing.T) { // use a different counter for the commit commit.Counter = setup.Counter + 1 - err := isValidEpochCommit(commit, setup) + err := protocol.IsValidEpochCommit(commit, setup) require.Error(t, err) }) @@ -78,7 +78,7 @@ func TestBootstrapInvalidEpochCommit(t *testing.T) { extraQC := unittest.QuorumCertificateWithSignerIDsFixture() commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(extraQC)) - err := isValidEpochCommit(commit, setup) + err := protocol.IsValidEpochCommit(commit, setup) require.Error(t, err) }) @@ -88,7 +88,7 @@ func TestBootstrapInvalidEpochCommit(t *testing.T) { commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) commit.DKGGroupKey = nil - err := isValidEpochCommit(commit, setup) + err := protocol.IsValidEpochCommit(commit, setup) require.Error(t, err) }) @@ -99,7 +99,7 @@ func TestBootstrapInvalidEpochCommit(t *testing.T) { // add an extra DKG participant key commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) - err := isValidEpochCommit(commit, setup) + err := protocol.IsValidEpochCommit(commit, setup) require.Error(t, err) }) } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index f1d081b96a3..5550527e7bc 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -164,8 +164,7 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo switch ev := event.Event.(type) { case *flow.EpochSetup: - // TODO: validate the service event - //err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) + err := protocol.IsValidExtendingEpochSetup(ev, activeSetup, epochStatus) if err != nil { if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode @@ -190,10 +189,9 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo updater.SetInvalidStateTransitionAttempted() return dbUpdates, nil } - //extendingSetup := parentProtocolState.NextEpochSetup + extendingSetup := parentProtocolState.NextEpochSetup - // TODO: validate the service event - //err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) + err = protocol.IsValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) if err != nil { if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode diff --git a/state/protocol/validity.go b/state/protocol/validity.go new file mode 100644 index 00000000000..14b1eed041b --- /dev/null +++ b/state/protocol/validity.go @@ -0,0 +1,189 @@ +package protocol + +import ( + "fmt" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/factory" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/model/flow/order" +) + +// IsValidExtendingEpochSetup checks whether an epoch setup service being +// added to the state is valid. In addition to intrinsic validity, we also +// check that it is valid w.r.t. the previous epoch setup event, and the +// current epoch status. +// Assumes all inputs besides extendingSetup are already validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status +func IsValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { + // We should only have a single epoch setup event per epoch. + if status.NextEpoch.SetupID != flow.ZeroID { + // true iff EpochSetup event for NEXT epoch was already included before + return NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) + } + + // The setup event should have the counter increased by one. + if extendingSetup.Counter != activeSetup.Counter+1 { + return NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) + } + + // The first view needs to be exactly one greater than the current epoch final view + if extendingSetup.FirstView != activeSetup.FinalView+1 { + return NewInvalidServiceEventErrorf( + "next epoch first view must be exactly 1 more than current epoch final view (%d != %d+1)", + extendingSetup.FirstView, + activeSetup.FinalView, + ) + } + + // Finally, the epoch setup event must contain all necessary information. + err := VerifyEpochSetup(extendingSetup, true) + if err != nil { + return NewInvalidServiceEventErrorf("invalid epoch setup: %w", err) + } + + return nil +} + +// VerifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. +// The boolean parameter `verifyNetworkAddress` controls, whether we want to permit +// nodes to share a networking address. +// This is a side-effect-free function. Any error return indicates that the +// EpochSetup event is not compliant with protocol rules. +func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { + // STEP 1: general sanity checks + // the seed needs to be at least minimum length + if len(setup.RandomSource) != flow.EpochSetupRandomSourceLength { + return fmt.Errorf("seed has incorrect length (%d != %d)", len(setup.RandomSource), flow.EpochSetupRandomSourceLength) + } + + // STEP 2: sanity checks of all nodes listed as participants + // there should be no duplicate node IDs + identLookup := make(map[flow.Identifier]struct{}) + for _, participant := range setup.Participants { + _, ok := identLookup[participant.NodeID] + if ok { + return fmt.Errorf("duplicate node identifier (%x)", participant.NodeID) + } + identLookup[participant.NodeID] = struct{}{} + } + + if verifyNetworkAddress { + // there should be no duplicate node addresses + addrLookup := make(map[string]struct{}) + for _, participant := range setup.Participants { + _, ok := addrLookup[participant.Address] + if ok { + return fmt.Errorf("duplicate node address (%x)", participant.Address) + } + addrLookup[participant.Address] = struct{}{} + } + } + + // the participants must be listed in canonical order + if !setup.Participants.Sorted(order.Canonical) { + return fmt.Errorf("participants are not canonically ordered") + } + + // STEP 3: sanity checks for individual roles + // IMPORTANT: here we remove all nodes with zero weight, as they are allowed to partake + // in communication but not in respective node functions + activeParticipants := setup.Participants.Filter(filter.HasWeight(true)) + + // we need at least one node of each role + roles := make(map[flow.Role]uint) + for _, participant := range activeParticipants { + roles[participant.Role]++ + } + if roles[flow.RoleConsensus] < 1 { + return fmt.Errorf("need at least one consensus node") + } + if roles[flow.RoleCollection] < 1 { + return fmt.Errorf("need at least one collection node") + } + if roles[flow.RoleExecution] < 1 { + return fmt.Errorf("need at least one execution node") + } + if roles[flow.RoleVerification] < 1 { + return fmt.Errorf("need at least one verification node") + } + + // first view must be before final view + if setup.FirstView >= setup.FinalView { + return fmt.Errorf("first view (%d) must be before final view (%d)", setup.FirstView, setup.FinalView) + } + + // we need at least one collection cluster + if len(setup.Assignments) == 0 { + return fmt.Errorf("need at least one collection cluster") + } + + // the collection cluster assignments need to be valid + _, err := factory.NewClusterList(setup.Assignments, activeParticipants.Filter(filter.HasRole(flow.RoleCollection))) + if err != nil { + return fmt.Errorf("invalid cluster assignments: %w", err) + } + + return nil +} + +// IsValidExtendingEpochCommit checks whether an epoch commit service being +// added to the state is valid. In addition to intrinsic validity, we also +// check that it is valid w.r.t. the previous epoch setup event, and the +// current epoch status. +// Assumes all inputs besides extendingCommit are already validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status +func IsValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { + + // We should only have a single epoch commit event per epoch. + if status.NextEpoch.CommitID != flow.ZeroID { + // true iff EpochCommit event for NEXT epoch was already included before + return NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) + } + + // The epoch setup event needs to happen before the commit. + if status.NextEpoch.SetupID == flow.ZeroID { + return NewInvalidServiceEventErrorf("missing epoch setup for epoch commit") + } + + // The commit event should have the counter increased by one. + if extendingCommit.Counter != activeSetup.Counter+1 { + return NewInvalidServiceEventErrorf("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) + } + + err := IsValidEpochCommit(extendingCommit, extendingSetup) + if err != nil { + return NewInvalidServiceEventErrorf("invalid epoch commit: %s", err) + } + + return nil +} + +// IsValidEpochCommit checks whether an epoch commit service event is intrinsically valid. +// Assumes the input flow.EpochSetup event has already been validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the EpochCommit is invalid +func IsValidEpochCommit(commit *flow.EpochCommit, setup *flow.EpochSetup) error { + + if len(setup.Assignments) != len(commit.ClusterQCs) { + return NewInvalidServiceEventErrorf("number of clusters (%d) does not number of QCs (%d)", len(setup.Assignments), len(commit.ClusterQCs)) + } + + if commit.Counter != setup.Counter { + return NewInvalidServiceEventErrorf("inconsistent epoch counter between commit (%d) and setup (%d) events in same epoch", commit.Counter, setup.Counter) + } + + // make sure we have a valid DKG public key + if commit.DKGGroupKey == nil { + return NewInvalidServiceEventErrorf("missing DKG public group key") + } + + participants := setup.Participants.Filter(filter.IsValidDKGParticipant) + if len(participants) != len(commit.DKGParticipantKeys) { + return NewInvalidServiceEventErrorf("participant list (len=%d) does not match dkg key list (len=%d)", len(participants), len(commit.DKGParticipantKeys)) + } + + return nil +} diff --git a/state/protocol/validity_test.go b/state/protocol/validity_test.go new file mode 100644 index 00000000000..550fbebd457 --- /dev/null +++ b/state/protocol/validity_test.go @@ -0,0 +1,104 @@ +package protocol_test + +import ( + "github.com/onflow/flow-go/state/protocol" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/utils/unittest" +) + +var participants = unittest.IdentityListFixture(20, unittest.WithAllRoles()) + +func TestEpochSetupValidity(t *testing.T) { + t.Run("invalid first/final view", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + // set an invalid final view for the first epoch + setup.FinalView = setup.FirstView + + err := protocol.VerifyEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("non-canonically ordered identities", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + // randomly shuffle the identities so they are not canonically ordered + var err error + setup.Participants, err = setup.Participants.Shuffle() + require.NoError(t, err) + err = protocol.VerifyEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("invalid cluster assignments", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + // create an invalid cluster assignment (node appears in multiple clusters) + collector := participants.Filter(filter.HasRole(flow.RoleCollection))[0] + setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) + + err := protocol.VerifyEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("short seed", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + setup.RandomSource = unittest.SeedFixture(crypto.SeedMinLenDKG - 1) + + err := protocol.VerifyEpochSetup(setup, true) + require.Error(t, err) + }) +} + +func TestBootstrapInvalidEpochCommit(t *testing.T) { + t.Run("inconsistent counter", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + // use a different counter for the commit + commit.Counter = setup.Counter + 1 + + err := protocol.IsValidEpochCommit(commit, setup) + require.Error(t, err) + }) + + t.Run("inconsistent cluster QCs", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + // add an extra QC to commit + extraQC := unittest.QuorumCertificateWithSignerIDsFixture() + commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(extraQC)) + + err := protocol.IsValidEpochCommit(commit, setup) + require.Error(t, err) + }) + + t.Run("missing dkg group key", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + commit.DKGGroupKey = nil + + err := protocol.IsValidEpochCommit(commit, setup) + require.Error(t, err) + }) + + t.Run("inconsistent DKG participants", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) + // add an extra DKG participant key + commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) + + err := protocol.IsValidEpochCommit(commit, setup) + require.Error(t, err) + }) +} From 2f3ae8c1a1dcf3e3e541f75a43bb72b2dc43f6c8 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 12:46:17 +0300 Subject: [PATCH 13/87] Updated mutator tests regarding epoch service events validity --- state/protocol/badger/mutator_test.go | 78 +++++++++++++++------------ utils/unittest/protocol_state.go | 18 +++++-- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 0496d5e1adb..804ff3c9e34 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -1231,7 +1231,9 @@ func TestExtendConflictingEpochEvents(t *testing.T) { // \--B2<--B4(R2)<--B6(S2)<--B8 func TestExtendDuplicateEpochEvents(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + calculateExpectedStateId := calculateExpectedStateId(t, mutator) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -1240,12 +1242,12 @@ func TestExtendDuplicateEpochEvents(t *testing.T) { // add two conflicting blocks for each service event to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) block2 := unittest.BlockWithParentFixture(head) - block2.SetPayload(flow.EmptyPayload()) + block2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block2) require.NoError(t, err) @@ -1266,7 +1268,10 @@ func TestExtendDuplicateEpochEvents(t *testing.T) { // add block 1 receipt to block 3 payload block3 := unittest.BlockWithParentFixture(block1.Header) - block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block1Receipt))) + block3.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(block1Receipt), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -1276,20 +1281,24 @@ func TestExtendDuplicateEpochEvents(t *testing.T) { // add block 2 receipt to block 4 payload block4 := unittest.BlockWithParentFixture(block2.Header) - block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block2Receipt))) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(block2Receipt), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.Extend(context.Background(), block4) require.NoError(t, err) // seal for block 1 - seal1 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult)) + seals1 := []*flow.Seal{unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult))} // seal for block 2 - seal2 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult)) + seals2 := []*flow.Seal{unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult))} // block 5 builds on block 3, contains seal for block 1 block5 := unittest.BlockWithParentFixture(block3.Header) block5.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, + Seals: seals1, + ProtocolStateID: calculateExpectedStateId(block5.Header, seals1), }) err = state.Extend(context.Background(), block5) require.NoError(t, err) @@ -1297,19 +1306,20 @@ func TestExtendDuplicateEpochEvents(t *testing.T) { // block 6 builds on block 4, contains seal for block 2 block6 := unittest.BlockWithParentFixture(block4.Header) block6.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal2}, + Seals: seals2, + ProtocolStateID: calculateExpectedStateId(block6.Header, seals2), }) err = state.Extend(context.Background(), block6) require.NoError(t, err) // block 7 builds on block 5, contains QC for block 7 - block7 := unittest.BlockWithParentFixture(block5.Header) + block7 := unittest.BlockWithParentProtocolState(block5) err = state.Extend(context.Background(), block7) require.NoError(t, err) // block 8 builds on block 6, contains QC for block 6 // at this point we are inserting the duplicate EpochSetup, should not error - block8 := unittest.BlockWithParentFixture(block6.Header) + block8 := unittest.BlockWithParentProtocolState(block6) err = state.Extend(context.Background(), block8) require.NoError(t, err) @@ -1328,6 +1338,7 @@ func TestExtendDuplicateEpochEvents(t *testing.T) { // service event should trigger epoch fallback when the fork is finalized. func TestExtendEpochSetupInvalid(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) // setupState initializes the protocol state for a test case // * creates and finalizes a new block for the first seal to reference @@ -1344,7 +1355,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) unittest.InsertAndFinalize(t, state, block1) epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) @@ -1376,14 +1387,14 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // expect a setup event with wrong counter to trigger EECC without error t.Run("wrong counter (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.Counter = rand.Uint64() }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1397,14 +1408,14 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // expect a setup event with wrong final view to trigger EECC without error t.Run("invalid final view (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.FinalView = block1.Header.View }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1418,14 +1429,14 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // expect a setup event with empty seed to trigger EECC without error t.Run("empty seed (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.RandomSource = nil }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1442,6 +1453,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // service event should trigger epoch fallback when the fork is finalized. func TestExtendEpochCommitInvalid(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) // setupState initializes the protocol state for a test case // * creates and finalizes a new block for the first seal to reference @@ -1459,7 +1471,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) unittest.InsertAndFinalize(t, state, block1) epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) @@ -1505,12 +1517,12 @@ func TestExtendEpochCommitInvalid(t *testing.T) { } t.Run("without setup (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, _, createCommit := setupState(t, state) _, receipt, seal := createCommit(block1) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1524,26 +1536,26 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // expect a commit event with wrong counter to trigger EECC without error t.Run("inconsistent counter (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup, createCommit := setupState(t, state) // seal block 1, in which EpochSetup was emitted epoch2Setup, setupReceipt, setupSeal := createSetup(block1) - epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutator, block1, setupReceipt, setupSeal) err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) require.NoError(t, err) err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) require.NoError(t, err) // insert a block with a QC for block 2 - block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + block3 := unittest.BlockWithParentProtocolState(epochSetupSealingBlock) unittest.InsertAndFinalize(t, state, block3) _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { commit.Counter = epoch2Setup.Counter + 1 }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block3, receipt, seal) err = state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1557,26 +1569,26 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // expect a commit event with wrong cluster QCs to trigger EECC without error t.Run("inconsistent cluster QCs (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup, createCommit := setupState(t, state) // seal block 1, in which EpochSetup was emitted _, setupReceipt, setupSeal := createSetup(block1) - epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutator, block1, setupReceipt, setupSeal) err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) require.NoError(t, err) err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) require.NoError(t, err) // insert a block with a QC for block 2 - block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + block3 := unittest.BlockWithParentProtocolState(epochSetupSealingBlock) unittest.InsertAndFinalize(t, state, block3) _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(unittest.QuorumCertificateWithSignerIDsFixture())) }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block3, receipt, seal) err = state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1590,19 +1602,19 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // expect a commit event with wrong dkg participants to trigger EECC without error t.Run("inconsistent DKG participants (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup, createCommit := setupState(t, state) // seal block 1, in which EpochSetup was emitted _, setupReceipt, setupSeal := createSetup(block1) - epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutator, block1, setupReceipt, setupSeal) err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) require.NoError(t, err) err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) require.NoError(t, err) // insert a block with a QC for block 2 - block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + block3 := unittest.BlockWithParentProtocolState(epochSetupSealingBlock) unittest.InsertAndFinalize(t, state, block3) _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { @@ -1610,7 +1622,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block3, receipt, seal) err = state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization diff --git a/utils/unittest/protocol_state.go b/utils/unittest/protocol_state.go index f5dbcb88073..d571f6cb283 100644 --- a/utils/unittest/protocol_state.go +++ b/utils/unittest/protocol_state.go @@ -73,24 +73,32 @@ func FinalizedProtocolStateWithParticipants(participants flow.IdentityList) ( // a receipt for the block (BR), the second (BS) containing a seal for the block. // B <- BR(Result_B) <- BS(Seal_B) // Returns the two generated blocks. -func SealBlock(t *testing.T, st protocol.ParticipantState, block *flow.Block, receipt *flow.ExecutionReceipt, seal *flow.Seal) (br *flow.Header, bs *flow.Header) { +func SealBlock(t *testing.T, st protocol.ParticipantState, mutator protocol.StateMutator, block *flow.Block, receipt *flow.ExecutionReceipt, seal *flow.Seal) (br *flow.Block, bs *flow.Block) { block2 := BlockWithParentFixture(block.Header) block2.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{receipt.Meta()}, - Results: []*flow.ExecutionResult{&receipt.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{receipt.Meta()}, + Results: []*flow.ExecutionResult{&receipt.ExecutionResult}, + ProtocolStateID: block.Payload.ProtocolStateID, }) err := st.Extend(context.Background(), block2) require.NoError(t, err) block3 := BlockWithParentFixture(block2.Header) + updater, err := mutator.CreateUpdater(block3.Header.View, block3.Header.ParentID) + require.NoError(t, err) + seals := []*flow.Seal{seal} + _, err = mutator.ApplyServiceEvents(updater, seals) + require.NoError(t, err) + _, updatedStateId, _ := updater.Build() block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal}, + Seals: seals, + ProtocolStateID: updatedStateId, }) err = st.Extend(context.Background(), block3) require.NoError(t, err) - return block2.Header, block3.Header + return block2, block3 } // InsertAndFinalize inserts, then finalizes, the input block. From 06d6a1a426f2bfe26a3fe9cc397cd99d55faebe4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 13:19:34 +0300 Subject: [PATCH 14/87] Fixed tests that depend on epoch builder --- state/protocol/badger/snapshot_test.go | 18 +++++----- state/protocol/badger/state_test.go | 49 ++++++++++++++++---------- utils/unittest/epoch_builder.go | 26 ++++++++++---- 3 files changed, 58 insertions(+), 35 deletions(-) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index c4fe3614eb3..18569778ced 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -673,7 +673,7 @@ func TestSealingSegment_FailureCases(t *testing.T) { b3 := unittest.BlockWithParentFixture(b2.Header) // construct block b3 with seal for b1, append it to state and finalize b3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) - multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { for _, b := range []*flow.Block{b1, b2, b3} { buildFinalizedBlock(t, state, b) } @@ -995,11 +995,11 @@ func TestSnapshot_EpochQuery(t *testing.T) { result, _, err := rootSnapshot.SealedResult() require.NoError(t, err) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { epoch1Counter := result.ServiceEvents[0].Event.(*flow.EpochSetup).Counter epoch2Counter := epoch1Counter + 1 - epochBuilder := unittest.NewEpochBuilder(t, state) + epochBuilder := unittest.NewEpochBuilder(t, mutator, state) // build epoch 1 (prepare epoch 2) epochBuilder. BuildEpoch(). @@ -1086,9 +1086,9 @@ func TestSnapshot_EpochFirstView(t *testing.T) { result, _, err := rootSnapshot.SealedResult() require.NoError(t, err) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(t, state) + epochBuilder := unittest.NewEpochBuilder(t, mutator, state) // build epoch 1 (prepare epoch 2) epochBuilder. BuildEpoch(). @@ -1167,9 +1167,9 @@ func TestSnapshot_EpochHeightBoundaries(t *testing.T) { head, err := rootSnapshot.Head() require.NoError(t, err) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(t, state) + epochBuilder := unittest.NewEpochBuilder(t, mutator, state) epoch1FirstHeight := head.Height t.Run("first epoch - EpochStaking phase", func(t *testing.T) { @@ -1248,9 +1248,9 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { epoch3Identities := unittest.IdentityListFixture(10, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(epoch1Identities) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(t, state) + epochBuilder := unittest.NewEpochBuilder(t, mutator, state) // build epoch 1 (prepare epoch 2) epochBuilder. UsingSetupOpts(unittest.WithParticipants(epoch2Identities)). diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 085c8378f49..0d4f0cd4abe 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -101,8 +101,8 @@ func TestBootstrapAndOpen_EpochCommitted(t *testing.T) { require.NoError(t, err) // build an epoch on the root state and return a snapshot from the committed phase - committedPhaseSnapshot := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { - unittest.NewEpochBuilder(t, state).BuildEpoch().CompleteEpoch() + committedPhaseSnapshot := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutator, state).BuildEpoch().CompleteEpoch() // find the point where we transition to the epoch committed phase for height := rootBlock.Height + 1; ; height++ { @@ -190,8 +190,8 @@ func TestBootstrap_EpochHeightBoundaries(t *testing.T) { }) t.Run("with next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { - builder := unittest.NewEpochBuilder(t, state) + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + builder := unittest.NewEpochBuilder(t, mutator, state) builder.BuildEpoch().CompleteEpoch() heights, ok := builder.EpochHeights(1) require.True(t, ok) @@ -217,8 +217,8 @@ func TestBootstrap_EpochHeightBoundaries(t *testing.T) { t.Run("with previous epoch", func(t *testing.T) { var epoch1FinalHeight uint64 var epoch2FirstHeight uint64 - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { - builder := unittest.NewEpochBuilder(t, state) + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + builder := unittest.NewEpochBuilder(t, mutator, state) builder. BuildEpoch().CompleteEpoch(). // build epoch 2 BuildEpoch() // build epoch 3 @@ -260,26 +260,37 @@ func TestBootstrapNonRoot(t *testing.T) { // start with a regular post-spork root snapshot participants := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) rootBlock, err := rootSnapshot.Head() require.NoError(t, err) // should be able to bootstrap from snapshot after sealing a non-root block // ROOT <- B1 <- B2(R1) <- B3(S1) <- CHILD t.Run("with sealed block", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { block1 := unittest.BlockWithParentFixture(rootBlock) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block2) block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + updater, err := mutator.CreateUpdater(block3.Header.View, block3.Header.ParentID) + require.NoError(t, err) + _, err = mutator.ApplyServiceEvents(updater, []*flow.Seal{seal1}) + require.NoError(t, err) + _, updatedStateId, _ := updater.Build() + block3.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal1), + unittest.WithProtocolStateID(updatedStateId))) buildFinalizedBlock(t, state, block3) - child := unittest.BlockWithParentFixture(block3.Header) + child := unittest.BlockWithParentProtocolState(block3) buildBlock(t, state, child) return state.AtBlockID(block3.ID()) @@ -302,8 +313,8 @@ func TestBootstrapNonRoot(t *testing.T) { }) t.Run("with setup next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { - unittest.NewEpochBuilder(t, state).BuildEpoch() + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutator, state).BuildEpoch() // find the point where we transition to the epoch setup phase for height := rootBlock.Height + 1; ; height++ { @@ -322,8 +333,8 @@ func TestBootstrapNonRoot(t *testing.T) { }) t.Run("with committed next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { - unittest.NewEpochBuilder(t, state).BuildEpoch().CompleteEpoch() + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutator, state).BuildEpoch().CompleteEpoch() // find the point where we transition to the epoch committed phase for height := rootBlock.Height + 1; ; height++ { @@ -342,8 +353,8 @@ func TestBootstrapNonRoot(t *testing.T) { }) t.Run("with previous and next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { - unittest.NewEpochBuilder(t, state). + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutator, state). BuildEpoch().CompleteEpoch(). // build epoch 2 BuildEpoch() // build epoch 3 @@ -554,10 +565,10 @@ func bootstrap(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.S // // This is used for generating valid snapshots to use when testing bootstrapping // from non-root states. -func snapshotAfter(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.FollowerState) protocol.Snapshot) protocol.Snapshot { +func snapshotAfter(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.FollowerState, protocol.StateMutator) protocol.Snapshot) protocol.Snapshot { var after protocol.Snapshot - protoutil.RunWithFollowerProtocolState(t, rootSnapshot, func(_ *badger.DB, state *bprotocol.FollowerState) { - snap := f(state) + protoutil.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(_ *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + snap := f(state.FollowerState, mutator) var err error after, err = inmem.FromSnapshot(snap) require.NoError(t, err) diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index 321522f582a..beb1fe815bb 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -71,6 +71,7 @@ func (epoch EpochHeights) CommittedRange() []uint64 { // EpochBuilder is a testing utility for building epochs into chain state. type EpochBuilder struct { t *testing.T + mutator protocol.StateMutator states []protocol.FollowerState blocksByID map[flow.Identifier]*flow.Block blocks []*flow.Block @@ -82,11 +83,12 @@ type EpochBuilder struct { // NewEpochBuilder returns a new EpochBuilder which will build epochs using the // given states. At least one state must be provided. If more than one are // provided they must have the same initial state. -func NewEpochBuilder(t *testing.T, states ...protocol.FollowerState) *EpochBuilder { +func NewEpochBuilder(t *testing.T, mutator protocol.StateMutator, states ...protocol.FollowerState) *EpochBuilder { require.True(t, len(states) >= 1, "must provide at least one state") builder := &EpochBuilder{ t: t, + mutator: mutator, states: states, blocksByID: make(map[flow.Identifier]*flow.Block), blocks: make([]*flow.Block, 0), @@ -159,20 +161,20 @@ func (builder *EpochBuilder) BuildEpoch() *EpochBuilder { // prepare default values for the service events based on the current state identities, err := state.Final().Identities(filter.Any) - require.Nil(builder.t, err) + require.NoError(builder.t, err) epoch := state.Final().Epochs().Current() counter, err := epoch.Counter() - require.Nil(builder.t, err) + require.NoError(builder.t, err) finalView, err := epoch.FinalView() - require.Nil(builder.t, err) + require.NoError(builder.t, err) // retrieve block A A, err := state.Final().Head() - require.Nil(builder.t, err) + require.NoError(builder.t, err) // check that block A satisfies initial condition phase, err := state.Final().Phase() - require.Nil(builder.t, err) + require.NoError(builder.t, err) require.Equal(builder.t, flow.EpochPhaseStaking, phase) // Define receipts and seals for block B payload. They will be nil if A is @@ -366,9 +368,19 @@ func (builder *EpochBuilder) BuildBlocks(n uint) { // addBlock adds the given block to the state by: extending the state, // finalizing the block, marking the block as valid, and caching the block. func (builder *EpochBuilder) addBlock(block *flow.Block) { + updater, err := builder.mutator.CreateUpdater(block.Header.View, block.Header.ParentID) + require.NoError(builder.t, err) + + _, err = builder.mutator.ApplyServiceEvents(updater, block.Payload.Seals) + require.NoError(builder.t, err) + + _, updatedStateId, _ := updater.Build() + + block.Payload.ProtocolStateID = updatedStateId + block.Header.PayloadHash = block.Payload.Hash() blockID := block.ID() for _, state := range builder.states { - err := state.ExtendCertified(context.Background(), block, CertifyBlock(block.Header)) + err = state.ExtendCertified(context.Background(), block, CertifyBlock(block.Header)) require.NoError(builder.t, err) err = state.Finalize(context.Background(), blockID) From 8da7c3e018c1b1ed18f5f593e314930bebe5f1e1 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 14:13:08 +0300 Subject: [PATCH 15/87] Fixed next group of protocol tests --- state/protocol/badger/snapshot_test.go | 104 ++++++++++++++++++------- state/protocol/badger/state_test.go | 13 ++-- 2 files changed, 83 insertions(+), 34 deletions(-) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 18569778ced..06502ee6edf 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -37,18 +37,22 @@ func TestUnknownReferenceBlock(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants, func(block *flow.Block) { block.Header.Height = rootHeight }) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { // build some finalized non-root blocks (heights 101-110) - head := rootSnapshot.Encodable().Head + head := unittest.BlockWithParentFixture(rootSnapshot.Encodable().Head) + head.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) + buildFinalizedBlock(t, state, head) + const nBlocks = 10 - for i := 0; i < nBlocks; i++ { - next := unittest.BlockWithParentFixture(head) + for i := 1; i < nBlocks; i++ { + next := unittest.BlockWithParentProtocolState(head) buildFinalizedBlock(t, state, next) - head = next.Header + head = next } // build an unfinalized block (height 111) - buildBlock(t, state, unittest.BlockWithParentFixture(head)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(head)) finalizedHeader, err := state.Final().Head() require.NoError(t, err) @@ -156,6 +160,7 @@ func TestSnapshot_Params(t *testing.T) { func TestSnapshot_Descendants(t *testing.T) { participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(participants) + //rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) head, err := rootSnapshot.Head() require.NoError(t, err) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { @@ -255,6 +260,7 @@ func TestClusters(t *testing.T) { func TestSealingSegment(t *testing.T) { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -282,13 +288,14 @@ func TestSealingSegment(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build an extra block on top of root block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) segment, err := state.AtBlockID(block1.ID()).SealingSegment() require.NoError(t, err) // build a valid child B2 to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block1.Header)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(block1)) // sealing segment should contain B1 and B2 // B2 is reference of snapshot, B1 is latest sealed @@ -304,21 +311,29 @@ func TestSealingSegment(t *testing.T) { // ROOT <- B1 <- B2(R1) <- B3(S1) // Expected sealing segment: [B1, B2, B3], extra blocks: [ROOT] t.Run("non-root", func(t *testing.T) { - util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block2) // build a block sealing block1 - block3 := unittest.BlockWithParentFixture(block2.Header) + block3 := unittest.BlockWithParentProtocolState(block2) - block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + seals := []*flow.Seal{seal1} + block3.SetPayload(flow.Payload{ + Seals: seals, + ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), + }) buildFinalizedBlock(t, state, block3) segment, err := state.AtBlockID(block3.ID()).SealingSegment() @@ -328,7 +343,7 @@ func TestSealingSegment(t *testing.T) { assert.Equal(t, segment.ExtraBlocks[0].Header.Height, head.Height) // build a valid child B3 to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block3.Header)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(block3)) // sealing segment should contain B1, B2, B3 // B3 is reference of snapshot, B1 is latest sealed @@ -649,6 +664,7 @@ func TestSealingSegment(t *testing.T) { // (2b) An orphaned block is chosen as head; at this height a block other than the orphaned has been finalized. func TestSealingSegment_FailureCases(t *testing.T) { sporkRootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + rootProtocolStateID := getRootProtocolStateID(t, sporkRootSnapshot) sporkRoot, err := sporkRootSnapshot.Head() require.NoError(t, err) @@ -667,17 +683,24 @@ func TestSealingSegment_FailureCases(t *testing.T) { // └── head ──┘ // b1 := unittest.BlockWithParentFixture(sporkRoot) // construct block b1, append to state and finalize + b1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) receipt, seal := unittest.ReceiptAndSealForBlock(b1) b2 := unittest.BlockWithParentFixture(b1.Header) // construct block b2, append to state and finalize - b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt))) + b2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt), + unittest.WithProtocolStateID(rootProtocolStateID), + )) b3 := unittest.BlockWithParentFixture(b2.Header) // construct block b3 with seal for b1, append it to state and finalize - b3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) + b3.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal), + unittest.WithProtocolStateID(rootProtocolStateID), + )) multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { for _, b := range []*flow.Block{b1, b2, b3} { buildFinalizedBlock(t, state, b) } - b4 := unittest.BlockWithParentFixture(b3.Header) + b4 := unittest.BlockWithParentProtocolState(b3) require.NoError(t, state.ExtendCertified(context.Background(), b4, unittest.CertifyBlock(b4.Header))) // add child of b3 to ensure we have a QC for b3 return state.AtBlockID(b3.ID()) }) @@ -707,7 +730,9 @@ func TestSealingSegment_FailureCases(t *testing.T) { util.RunWithFollowerProtocolState(t, sporkRootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // add _unfinalized_ blocks b1 and b2 to state (block b5 is necessary, so b1 has a QC, which is a consistency requirement for subsequent finality) b1 := unittest.BlockWithParentFixture(sporkRoot) + b1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) b2 := unittest.BlockWithParentFixture(b1.Header) + b2.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) require.NoError(t, state.ExtendCertified(context.Background(), b1, b2.Header.QuorumCertificate())) require.NoError(t, state.ExtendCertified(context.Background(), b2, unittest.CertifyBlock(b2.Header))) // adding block b5 (providing required QC for b1) @@ -725,10 +750,13 @@ func TestSealingSegment_FailureCases(t *testing.T) { t.Run("sealing segment from orphaned block", func(t *testing.T) { util.RunWithFollowerProtocolState(t, sporkRootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { orphaned := unittest.BlockWithParentFixture(sporkRoot) - orphanedChild := unittest.BlockWithParentFixture(orphaned.Header) + orphaned.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) + orphanedChild := unittest.BlockWithParentProtocolState(orphaned) require.NoError(t, state.ExtendCertified(context.Background(), orphaned, orphanedChild.Header.QuorumCertificate())) require.NoError(t, state.ExtendCertified(context.Background(), orphanedChild, unittest.CertifyBlock(orphanedChild.Header))) - buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(sporkRoot)) + block := unittest.BlockWithParentFixture(sporkRoot) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) + buildFinalizedBlock(t, state, block) // consistency check: the finalized block at height `orphaned.Height` should be different than `orphaned` h, err := state.AtHeight(orphaned.Header.Height).Head() @@ -751,6 +779,7 @@ func TestSealingSegment_FailureCases(t *testing.T) { func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) rootEpoch := rootSnapshot.Epochs().Current() cluster, err := rootEpoch.Cluster(0) require.NoError(t, err) @@ -759,27 +788,38 @@ func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { require.NoError(t, err) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block2) receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) - block3 := unittest.BlockWithParentFixture(block2.Header) + block3 := unittest.BlockWithParentProtocolState(block2) buildFinalizedBlock(t, state, block3) block4 := unittest.BlockWithParentFixture(block3.Header) - block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), unittest.WithSeals(seal1))) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt2), + unittest.WithSeals(seal1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block4) - block5 := unittest.BlockWithParentFixture(block4.Header) + block5 := unittest.BlockWithParentProtocolState(block4) buildFinalizedBlock(t, state, block5) block6 := unittest.BlockWithParentFixture(block5.Header) - block6.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2))) + block6.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block6) snapshot := state.AtBlockID(block6.ID()) @@ -787,7 +827,7 @@ func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { require.NoError(t, err) // build a valid child to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block6.Header)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(block6)) // sealing segment should be [B2, B3, B4, B5, B6] require.Len(t, segment.Blocks, 5) @@ -808,7 +848,10 @@ func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { require.NoError(t, err) guarantee.SignerIndices = signerIndices - block7.SetPayload(unittest.PayloadFixture(unittest.WithGuarantees(guarantee))) + block7.SetPayload(unittest.PayloadFixture( + unittest.WithGuarantees(guarantee), + unittest.WithProtocolStateID(block6.Payload.ProtocolStateID), + )) buildBlock(t, state, block7) }) }) @@ -817,6 +860,7 @@ func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { func TestLatestSealedResult(t *testing.T) { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) t.Run("root snapshot", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { @@ -836,23 +880,31 @@ func TestLatestSealedResult(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) - block2 := unittest.BlockWithParentFixture(block1.Header) + block2 := unittest.BlockWithParentProtocolState(block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + block3.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal1), + unittest.WithProtocolStateID(rootProtocolStateID))) receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3) block4 := unittest.BlockWithParentFixture(block3.Header) block4.SetPayload(unittest.PayloadFixture( unittest.WithReceipts(receipt2, receipt3), + unittest.WithProtocolStateID(rootProtocolStateID), )) block5 := unittest.BlockWithParentFixture(block4.Header) block5.SetPayload(unittest.PayloadFixture( unittest.WithSeals(seal2, seal3), + unittest.WithProtocolStateID(rootProtocolStateID), )) err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate()) diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 0d4f0cd4abe..baec77ebf7c 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -279,15 +279,12 @@ func TestBootstrapNonRoot(t *testing.T) { unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block2) + seals := []*flow.Seal{seal1} block3 := unittest.BlockWithParentFixture(block2.Header) - updater, err := mutator.CreateUpdater(block3.Header.View, block3.Header.ParentID) - require.NoError(t, err) - _, err = mutator.ApplyServiceEvents(updater, []*flow.Seal{seal1}) - require.NoError(t, err) - _, updatedStateId, _ := updater.Build() - block3.SetPayload(unittest.PayloadFixture( - unittest.WithSeals(seal1), - unittest.WithProtocolStateID(updatedStateId))) + block3.SetPayload(flow.Payload{ + Seals: seals, + ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), + }) buildFinalizedBlock(t, state, block3) child := unittest.BlockWithParentProtocolState(block3) From 9fa616550f3712d0b0cb85759a286d151c7975b6 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 14:23:45 +0300 Subject: [PATCH 16/87] More updated to SealingSegment tests --- state/protocol/badger/snapshot_test.go | 62 +++++++++++++++++++------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 06502ee6edf..04ea06e56be 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -362,6 +362,7 @@ func TestSealingSegment(t *testing.T) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) @@ -369,11 +370,14 @@ func TestSealingSegment(t *testing.T) { parent := block1 // build a large chain of intermediary blocks for i := 0; i < 100; i++ { - next := unittest.BlockWithParentFixture(parent.Header) + next := unittest.BlockWithParentProtocolState(parent) if i == 0 { // Repetitions of the same receipt in one fork would be a protocol violation. // Hence, we include the result only once in the direct child of B1. - next.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + next.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(parent.Payload.ProtocolStateID), + )) } buildFinalizedBlock(t, state, next) parent = next @@ -381,8 +385,10 @@ func TestSealingSegment(t *testing.T) { // build the block sealing block 1 blockN := unittest.BlockWithParentFixture(parent.Header) - - blockN.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + blockN.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, blockN) segment, err := state.AtBlockID(blockN.ID()).SealingSegment() @@ -408,34 +414,45 @@ func TestSealingSegment(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block2) receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) - block3 := unittest.BlockWithParentFixture(block2.Header) + block3 := unittest.BlockWithParentProtocolState(block2) buildFinalizedBlock(t, state, block3) block4 := unittest.BlockWithParentFixture(block3.Header) - block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), unittest.WithSeals(seal1))) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt2), + unittest.WithSeals(seal1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block4) - block5 := unittest.BlockWithParentFixture(block4.Header) + block5 := unittest.BlockWithParentProtocolState(block4) buildFinalizedBlock(t, state, block5) block6 := unittest.BlockWithParentFixture(block5.Header) - block6.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2))) + block6.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block6) segment, err := state.AtBlockID(block6.ID()).SealingSegment() require.NoError(t, err) // build a valid child to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block6.Header)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(block6)) // sealing segment should be [B2, B3, B4, B5, B6] require.Len(t, segment.Blocks, 5) @@ -464,19 +481,32 @@ func TestSealingSegment(t *testing.T) { receiptB := unittest.ExecutionReceiptFixture() block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptA1))) + block1.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receiptA1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptB), unittest.WithReceiptsAndNoResults(receiptA2))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receiptB), + unittest.WithReceiptsAndNoResults(receiptA2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) receiptC, sealC := unittest.ReceiptAndSealForBlock(block2) block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptC))) + block3.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receiptC), + unittest.WithProtocolStateID(rootProtocolStateID), + )) - block4 := unittest.BlockWithParentFixture(block3.Header) + block4 := unittest.BlockWithParentProtocolState(block3) block5 := unittest.BlockWithParentFixture(block4.Header) - block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(sealC))) + block5.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(sealC), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block1) buildFinalizedBlock(t, state, block2) @@ -488,7 +518,7 @@ func TestSealingSegment(t *testing.T) { require.NoError(t, err) // build a valid child to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(block5)) require.Len(t, segment.Blocks, 4) unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5}, segment.Blocks) From 24e984eefcfb47aa2a3020544180d84a487b2b67 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 19:00:59 +0300 Subject: [PATCH 17/87] Fixed sealing segment tests --- state/protocol/badger/snapshot_test.go | 79 ++++++++++++++++++++------ 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 04ea06e56be..a74af1dd342 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -551,20 +551,34 @@ func TestSealingSegment(t *testing.T) { receiptB2 := unittest.ExecutionReceiptFixture(unittest.WithResult(&receiptB.ExecutionResult)) block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptA1))) + block1.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receiptA1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptB), unittest.WithReceiptsAndNoResults(receiptA2))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receiptB), + unittest.WithReceiptsAndNoResults(receiptA2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) receiptForSeal, seal := unittest.ReceiptAndSealForBlock(block2) block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receiptForSeal), unittest.WithReceiptsAndNoResults(receiptB2, receiptA3))) + block3.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receiptForSeal), + unittest.WithReceiptsAndNoResults(receiptB2, receiptA3), + unittest.WithProtocolStateID(rootProtocolStateID), + )) - block4 := unittest.BlockWithParentFixture(block3.Header) + block4 := unittest.BlockWithParentProtocolState(block3) block5 := unittest.BlockWithParentFixture(block4.Header) - block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) + block5.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block1) buildFinalizedBlock(t, state, block2) @@ -576,7 +590,7 @@ func TestSealingSegment(t *testing.T) { require.NoError(t, err) // build a valid child to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + buildBlock(t, state, unittest.BlockWithParentProtocolState(block5)) require.Len(t, segment.Blocks, 4) unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5}, segment.Blocks) @@ -596,28 +610,35 @@ func TestSealingSegment(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) // build a block sealing block1 block2 := unittest.BlockWithParentFixture(block1.Header) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block2) - block3 := unittest.BlockWithParentFixture(block2.Header) + block3 := unittest.BlockWithParentProtocolState(block2) buildFinalizedBlock(t, state, block3) block4 := unittest.BlockWithParentFixture(block3.Header) - block4.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block4) - block5 := unittest.BlockWithParentFixture(block4.Header) + block5 := unittest.BlockWithParentProtocolState(block4) buildFinalizedBlock(t, state, block5) snapshot := state.AtBlockID(block5.ID()) // build a valid child to ensure we have a QC - buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + buildFinalizedBlock(t, state, unittest.BlockWithParentProtocolState(block5)) segment, err := snapshot.SealingSegment() require.NoError(t, err) @@ -637,42 +658,64 @@ func TestSealingSegment(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) // build a block sealing block1 block2 := unittest.BlockWithParentFixture(block1.Header) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block2) receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1), unittest.WithReceipts(receipt2))) + block3.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal1), + unittest.WithReceipts(receipt2), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block3) receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3) block4 := unittest.BlockWithParentFixture(block3.Header) - block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt3))) + block4.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt3), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, block4) // build chain, so it's long enough to not target blocks as inside of flow.DefaultTransactionExpiry window. parent := block4 for i := 0; i < 1.5*flow.DefaultTransactionExpiry; i++ { - next := unittest.BlockWithParentFixture(parent.Header) + next := unittest.BlockWithParentProtocolState(parent) next.Header.View = next.Header.Height + 1 // set view so we are still in the same epoch buildFinalizedBlock(t, state, next) parent = next } receipt4, seal4 := unittest.ReceiptAndSealForBlock(block4) - lastBlock := unittest.BlockWithParentFixture(parent.Header) - lastBlock.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2, seal3, seal4), unittest.WithReceipts(receipt4))) + prevLastBlock := unittest.BlockWithParentFixture(parent.Header) + prevLastBlock.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt4), + unittest.WithProtocolStateID(rootProtocolStateID), + )) + buildFinalizedBlock(t, state, prevLastBlock) + + // since result and seal cannot be part of the same block, we need to build another block + lastBlock := unittest.BlockWithParentFixture(prevLastBlock.Header) + lastBlock.SetPayload(unittest.PayloadFixture( + unittest.WithSeals(seal2, seal3, seal4), + unittest.WithProtocolStateID(rootProtocolStateID), + )) buildFinalizedBlock(t, state, lastBlock) snapshot := state.AtBlockID(lastBlock.ID()) // build a valid child to ensure we have a QC - buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(lastBlock.Header)) + buildFinalizedBlock(t, state, unittest.BlockWithParentProtocolState(lastBlock)) segment, err := snapshot.SealingSegment() require.NoError(t, err) From 40fcdde35d883ceaecc7044ec9fa0b4392dbf8ba Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 19:17:21 +0300 Subject: [PATCH 18/87] Fixed remaining snapshot tests --- state/protocol/badger/snapshot_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index a74af1dd342..a86ba366fc8 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -160,16 +160,20 @@ func TestSnapshot_Params(t *testing.T) { func TestSnapshot_Descendants(t *testing.T) { participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(participants) - //rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) head, err := rootSnapshot.Head() require.NoError(t, err) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { var expectedBlocks []flow.Identifier for i := 5; i > 3; i-- { - for _, block := range unittest.ChainFixtureFrom(i, head) { + parent := head + for n := 0; n < i; n++ { + block := unittest.BlockWithParentFixture(parent) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err := state.Extend(context.Background(), block) require.NoError(t, err) expectedBlocks = append(expectedBlocks, block.ID()) + parent = block.Header } } @@ -231,6 +235,7 @@ func TestClusters(t *testing.T) { clusterQCs := unittest.QuorumCertificatesFromAssignments(setup.Assignments) commit.ClusterQCs = flow.ClusterQCVoteDatasFromQCs(clusterQCs) seal.ResultID = result.ID() + root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState(setup, commit).ID() rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) From b344f4355bf9e9843ecf2b3cb06fb19fdbabfd0f Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 10 Oct 2023 19:30:53 +0300 Subject: [PATCH 19/87] Fixed other broken tests in state/protocol --- state/protocol/inmem/convert_test.go | 6 ++-- state/protocol/protocol_state/mutator_test.go | 23 +++++++++++---- state/protocol/protocol_state/updater_test.go | 28 +++++++++---------- state/protocol/util_test.go | 10 +++---- utils/unittest/version_beacon.go | 13 ++++++--- 5 files changed, 48 insertions(+), 32 deletions(-) diff --git a/state/protocol/inmem/convert_test.go b/state/protocol/inmem/convert_test.go index 6da32088947..8c72240d535 100644 --- a/state/protocol/inmem/convert_test.go +++ b/state/protocol/inmem/convert_test.go @@ -23,9 +23,9 @@ func TestFromSnapshot(t *testing.T) { identities := unittest.IdentityListFixture(10, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { - - epochBuilder := unittest.NewEpochBuilder(t, state) + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, fullState *bprotocol.ParticipantState, mutator protocol.StateMutator) { + state := fullState.FollowerState + epochBuilder := unittest.NewEpochBuilder(t, mutator, state) // build epoch 1 (prepare epoch 2) epochBuilder. BuildEpoch(). diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 86c12a37905..51f9287bbbc 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -19,20 +19,29 @@ func TestProtocolStateMutator(t *testing.T) { type MutatorSuite struct { suite.Suite protocolStateDB *storagemock.ProtocolState + headersDB *storagemock.Headers + resultsDB *storagemock.ExecutionResults + setupsDB *storagemock.EpochSetups + commitsDB *storagemock.EpochCommits mutator *Mutator } func (s *MutatorSuite) SetupTest() { s.protocolStateDB = storagemock.NewProtocolState(s.T()) - s.mutator = NewMutator(s.protocolStateDB) + s.headersDB = storagemock.NewHeaders(s.T()) + s.resultsDB = storagemock.NewExecutionResults(s.T()) + s.setupsDB = storagemock.NewEpochSetups(s.T()) + s.commitsDB = storagemock.NewEpochCommits(s.T()) + + s.mutator = NewMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.protocolStateDB) } // TestCreateUpdaterForUnknownBlock tests that CreateUpdater returns an error if the parent protocol state is not found. func (s *MutatorSuite) TestCreateUpdaterForUnknownBlock() { candidate := unittest.BlockHeaderFixture() s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(nil, storerr.ErrNotFound) - updater, err := s.mutator.CreateUpdater(candidate) + updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) require.ErrorIs(s.T(), err, storerr.ErrNotFound) require.Nil(s.T(), updater) } @@ -42,12 +51,13 @@ func (s *MutatorSuite) TestMutatorHappyPathNoChanges() { parentState := unittest.ProtocolStateFixture() candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) - updater, err := s.mutator.CreateUpdater(candidate) + updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) require.NoError(s.T(), err) s.protocolStateDB.On("Index", candidate.ID(), parentState.ID()).Return(func(tx *transaction.Tx) error { return nil }) - err = s.mutator.CommitProtocolState(updater)(&transaction.Tx{}) + dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) + err = dbOps(&transaction.Tx{}) require.NoError(s.T(), err) } @@ -56,7 +66,7 @@ func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { parentState := unittest.ProtocolStateFixture() candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) - updater, err := s.mutator.CreateUpdater(candidate) + updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) require.NoError(s.T(), err) // update protocol state so it has some changes @@ -67,6 +77,7 @@ func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { s.protocolStateDB.On("StoreTx", updatedStateID, updatedState).Return(func(tx *transaction.Tx) error { return nil }) s.protocolStateDB.On("Index", candidate.ID(), updatedStateID).Return(func(tx *transaction.Tx) error { return nil }) - err = s.mutator.CommitProtocolState(updater)(&transaction.Tx{}) + dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) + err = dbOps(&transaction.Tx{}) require.NoError(s.T(), err) } diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index f1f2cefaa8f..1109962734a 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -32,7 +32,7 @@ func (s *UpdaterSuite) SetupTest() { s.parentBlock = unittest.BlockHeaderFixture(unittest.HeaderWithView(s.parentProtocolState.CurrentEpochSetup.FirstView + 1)) s.candidate = unittest.BlockHeaderWithParentFixture(s.parentBlock) - s.updater = NewUpdater(s.candidate, s.parentProtocolState) + s.updater = NewUpdater(s.candidate.View, s.parentProtocolState) } // TestNewUpdater tests if the constructor correctly setups invariants for updater. @@ -40,7 +40,7 @@ func (s *UpdaterSuite) TestNewUpdater() { require.NotSame(s.T(), s.updater.parentState, s.updater.state, "except to take deep copy of parent state") require.Nil(s.T(), s.updater.parentState.NextEpoch) require.Nil(s.T(), s.updater.state.NextEpoch) - require.Equal(s.T(), s.candidate, s.updater.Block()) + require.Equal(s.T(), s.candidate.View, s.updater.View()) require.Equal(s.T(), s.parentProtocolState, s.updater.ParentState()) } @@ -53,7 +53,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpoch() { candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(s.parentProtocolState.CurrentEpochSetup.FinalView + 1)) // since the candidate block is from next epoch, updater should transition to next epoch - s.updater = NewUpdater(candidate, s.parentProtocolState) + s.updater = NewUpdater(candidate.View, s.parentProtocolState) err := s.updater.TransitionToNextEpoch() require.NoError(s.T(), err) updatedState, _, _ := s.updater.Build() @@ -67,7 +67,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { protocolState := unittest.ProtocolStateFixture() candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewUpdater(candidate, protocolState) + updater := NewUpdater(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if there is no next epoch protocol state") }) @@ -78,7 +78,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { }) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewUpdater(candidate, protocolState) + updater := NewUpdater(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if it is not committed") }) @@ -88,7 +88,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { }) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewUpdater(candidate, protocolState) + updater := NewUpdater(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") }) @@ -96,7 +96,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView)) - updater := NewUpdater(candidate, protocolState) + updater := NewUpdater(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") }) @@ -123,7 +123,7 @@ func (s *UpdaterSuite) TestSetInvalidStateTransitionAttempted() { // update protocol state with next epoch information unittest.WithNextEpochProtocolState()(s.parentProtocolState) // create new updater with next epoch information - s.updater = NewUpdater(s.candidate, s.parentProtocolState) + s.updater = NewUpdater(s.candidate.View, s.parentProtocolState) s.updater.SetInvalidStateTransitionAttempted() updatedState, _, hasChanges := s.updater.Build() @@ -149,7 +149,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { require.Error(s.T(), err) }) s.Run("invalid state transition attempted", func() { - updater := NewUpdater(s.candidate, s.parentProtocolState) + updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 }) @@ -171,7 +171,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { require.Equal(s.T(), flow.ZeroID, newState.NextEpoch.CommitID, "operation must be no-op") }) s.Run("happy path processing", func() { - updater := NewUpdater(s.candidate, s.parentProtocolState) + updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 }) @@ -212,7 +212,7 @@ func (s *UpdaterSuite) TestUpdateIdentityUnknownIdentity() { func (s *UpdaterSuite) TestUpdateIdentityHappyPath() { // update protocol state to have next epoch protocol state unittest.WithNextEpochProtocolState()(s.parentProtocolState) - s.updater = NewUpdater(s.candidate, s.parentProtocolState) + s.updater = NewUpdater(s.candidate.View, s.parentProtocolState) currentEpochParticipants := s.parentProtocolState.CurrentEpochSetup.Participants.Copy() weightChanges, err := currentEpochParticipants.Sample(2) @@ -260,7 +260,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.Error(s.T(), err) }) s.Run("invalid state transition attempted", func() { - updater := NewUpdater(s.candidate, s.parentProtocolState) + updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 }) @@ -272,7 +272,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.Nil(s.T(), updatedState.NextEpoch, "should not process epoch setup if invalid state transition attempted") }) s.Run("processing second epoch setup", func() { - updater := NewUpdater(s.candidate, s.parentProtocolState) + updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 }) @@ -437,7 +437,7 @@ func (s *UpdaterSuite) TestEpochSetupAfterIdentityChange() { // now we can use it to construct updater for next block, which will process epoch setup event. nextBlock := unittest.BlockHeaderWithParentFixture(s.candidate) - s.updater = NewUpdater(nextBlock, updatedRichProtocolState) + s.updater = NewUpdater(nextBlock.View, updatedRichProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 diff --git a/state/protocol/util_test.go b/state/protocol/util_test.go index 7858f5767b7..81e9489815c 100644 --- a/state/protocol/util_test.go +++ b/state/protocol/util_test.go @@ -39,7 +39,7 @@ func TestOrderedSeals(t *testing.T) { payload := flow.EmptyPayload() headers := storagemock.NewHeaders(t) - ordered, err := protocol.OrderedSeals(&payload, headers) + ordered, err := protocol.OrderedSeals(payload.Seals, headers) require.NoError(t, err) require.Empty(t, ordered) }) @@ -49,7 +49,7 @@ func TestOrderedSeals(t *testing.T) { payload := unittest.PayloadFixture(unittest.WithSeals(seals...)) headers.On("ByBlockID", mock.Anything).Return(nil, storage.ErrNotFound) - ordered, err := protocol.OrderedSeals(&payload, headers) + ordered, err := protocol.OrderedSeals(payload.Seals, headers) require.ErrorIs(t, err, storage.ErrNotFound) require.Empty(t, ordered) }) @@ -60,7 +60,7 @@ func TestOrderedSeals(t *testing.T) { exception := errors.New("exception") headers.On("ByBlockID", mock.Anything).Return(nil, exception) - ordered, err := protocol.OrderedSeals(&payload, headers) + ordered, err := protocol.OrderedSeals(payload.Seals, headers) require.ErrorIs(t, err, exception) require.Empty(t, ordered) }) @@ -75,7 +75,7 @@ func TestOrderedSeals(t *testing.T) { } payload := unittest.PayloadFixture(unittest.WithSeals(seals...)) - ordered, err := protocol.OrderedSeals(&payload, headers) + ordered, err := protocol.OrderedSeals(payload.Seals, headers) require.NoError(t, err) require.Equal(t, seals, ordered) }) @@ -96,7 +96,7 @@ func TestOrderedSeals(t *testing.T) { }) payload := unittest.PayloadFixture(unittest.WithSeals(unorderedSeals...)) - ordered, err := protocol.OrderedSeals(&payload, headers) + ordered, err := protocol.OrderedSeals(payload.Seals, headers) require.NoError(t, err) require.Equal(t, orderedSeals, ordered) }) diff --git a/utils/unittest/version_beacon.go b/utils/unittest/version_beacon.go index 6518de747ef..0af4f627ab4 100644 --- a/utils/unittest/version_beacon.go +++ b/utils/unittest/version_beacon.go @@ -20,11 +20,14 @@ import ( func AddVersionBeacon(t *testing.T, beacon *flow.VersionBeacon, state protocol.FollowerState) { final, err := state.Final().Head() + require.NoError(t, err) + protocolState, err := state.Final().ProtocolState() require.NoError(t, err) + protocolStateID := protocolState.Entry().ID() A := BlockWithParentFixture(final) - A.SetPayload(flow.Payload{}) + A.SetPayload(PayloadFixture(WithProtocolStateID(protocolStateID))) addToState(t, state, A, true) receiptA := ReceiptForBlockFixture(A) @@ -32,8 +35,9 @@ func AddVersionBeacon(t *testing.T, beacon *flow.VersionBeacon, state protocol.F B := BlockWithParentFixture(A.Header) B.SetPayload(flow.Payload{ - Receipts: []*flow.ExecutionReceiptMeta{receiptA.Meta()}, - Results: []*flow.ExecutionResult{&receiptA.ExecutionResult}, + Receipts: []*flow.ExecutionReceiptMeta{receiptA.Meta()}, + Results: []*flow.ExecutionResult{&receiptA.ExecutionResult}, + ProtocolStateID: protocolStateID, }) addToState(t, state, B, true) @@ -43,7 +47,8 @@ func AddVersionBeacon(t *testing.T, beacon *flow.VersionBeacon, state protocol.F C := BlockWithParentFixture(B.Header) C.SetPayload(flow.Payload{ - Seals: sealsForB, + Seals: sealsForB, + ProtocolStateID: protocolStateID, }) addToState(t, state, C, true) } From a0b0ea311b0fbef5003a49cad9d5d6858522d57a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 11 Oct 2023 12:09:46 +0300 Subject: [PATCH 20/87] Updated how protocol state is opened and bootstrapped. Fixed tests for EECC --- state/protocol/badger/mutator_test.go | 47 +++++++++++++++++------- state/protocol/badger/state.go | 33 ++++++++++------- state/protocol/protocol_state/mutator.go | 13 ++++--- state/protocol/util/testing.go | 21 +++++++++-- 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 804ff3c9e34..8864c4f133b 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -861,7 +861,14 @@ func TestExtendEpochTransitionValid(t *testing.T) { ) require.NoError(t, err) - stateMutator := protocol_state.NewMutator(all.Headers, all.Results, all.Setups, all.EpochCommits, all.ProtocolState) + stateMutator := protocol_state.NewMutator( + all.Headers, + all.Results, + all.Setups, + all.EpochCommits, + all.ProtocolState, + state.Params(), + ) calculateExpectedStateId := calculateExpectedStateId(t, stateMutator) head, err := rootSnapshot.Head() @@ -1719,13 +1726,14 @@ func TestEmergencyEpochFallback(t *testing.T) { t.Run("passed epoch commitment deadline in EpochStaking phase - should trigger EECC", func(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) metricsMock := mockmodule.NewComplianceMetrics(t) mockMetricsForRootSnapshot(metricsMock, rootSnapshot) protoEventsMock := mockprotocol.NewConsumer(t) protoEventsMock.On("BlockFinalized", mock.Anything) protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) - util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, _ realprotocol.StateMutator) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1745,6 +1753,7 @@ func TestEmergencyEpochFallback(t *testing.T) { // block 1 will be the first block on or past the epoch commitment deadline block1 := unittest.BlockWithParentFixture(head) block1.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2 + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization @@ -1753,7 +1762,7 @@ func TestEmergencyEpochFallback(t *testing.T) { assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization // block 2 will be the first block past the first epoch boundary - block2 := unittest.BlockWithParentFixture(block1.Header) + block2 := unittest.BlockWithParentProtocolState(block1) block2.Header.View = epoch1FinalView + 1 err = state.Extend(context.Background(), block2) require.NoError(t, err) @@ -1777,13 +1786,14 @@ func TestEmergencyEpochFallback(t *testing.T) { t.Run("passed epoch commitment deadline in EpochSetup phase - should trigger EECC", func(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) metricsMock := mockmodule.NewComplianceMetrics(t) mockMetricsForRootSnapshot(metricsMock, rootSnapshot) protoEventsMock := mockprotocol.NewConsumer(t) protoEventsMock.On("BlockFinalized", mock.Anything) protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) - util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1793,7 +1803,7 @@ func TestEmergencyEpochFallback(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) err = state.Finalize(context.Background(), block1.ID()) @@ -1821,7 +1831,10 @@ func TestEmergencyEpochFallback(t *testing.T) { // add a block containing a receipt for block 1 block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.Extend(context.Background(), block2) require.NoError(t, err) err = state.Finalize(context.Background(), block2.ID()) @@ -1830,8 +1843,10 @@ func TestEmergencyEpochFallback(t *testing.T) { // block 3 seals block 1 and will be the first block on or past the epoch commitment deadline block3 := unittest.BlockWithParentFixture(block2.Header) block3.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2 + seals := []*flow.Seal{seal1} block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, + Seals: seals, + ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), }) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -1846,7 +1861,7 @@ func TestEmergencyEpochFallback(t *testing.T) { assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization // block 4 will be the first block past the first epoch boundary - block4 := unittest.BlockWithParentFixture(block3.Header) + block4 := unittest.BlockWithParentProtocolState(block3) block4.Header.View = epoch1FinalView + 1 err = state.Extend(context.Background(), block4) require.NoError(t, err) @@ -1870,13 +1885,14 @@ func TestEmergencyEpochFallback(t *testing.T) { t.Run("epoch transition with invalid service event - should trigger EECC", func(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) + rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) metricsMock := mockmodule.NewComplianceMetrics(t) mockMetricsForRootSnapshot(metricsMock, rootSnapshot) protoEventsMock := mockprotocol.NewConsumer(t) protoEventsMock.On("BlockFinalized", mock.Anything) protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) - util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1884,7 +1900,7 @@ func TestEmergencyEpochFallback(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) err = state.Extend(context.Background(), block1) require.NoError(t, err) err = state.Finalize(context.Background(), block1.ID()) @@ -1912,7 +1928,10 @@ func TestEmergencyEpochFallback(t *testing.T) { // add a block containing a receipt for block 1 block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipt1), + unittest.WithProtocolStateID(rootProtocolStateID), + )) err = state.Extend(context.Background(), block2) require.NoError(t, err) err = state.Finalize(context.Background(), block2.ID()) @@ -1920,8 +1939,10 @@ func TestEmergencyEpochFallback(t *testing.T) { // block 3 is where the service event state change comes into effect block3 := unittest.BlockWithParentFixture(block2.Header) + seals := []*flow.Seal{seal1} block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, + Seals: seals, + ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), }) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -1936,7 +1957,7 @@ func TestEmergencyEpochFallback(t *testing.T) { assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization // block 5 is the first block past the current epoch boundary - block4 := unittest.BlockWithParentFixture(block3.Header) + block4 := unittest.BlockWithParentProtocolState(block3) block4.Header.View = epoch1Setup.FinalView + 1 err = state.Extend(context.Background(), block4) require.NoError(t, err) diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index d24427bd2a7..f244b1496e6 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -110,8 +110,6 @@ func Bootstrap( return nil, fmt.Errorf("expected empty database") } - protocolStateMutator := protocol_state.NewMutator(headers, results, setups, commits, protocolStateSnapshotsDB) - protocolState := protocol_state.NewProtocolState(protocolStateSnapshotsDB, root.Params()) state := newState( metrics, db, @@ -122,9 +120,9 @@ func Bootstrap( qcs, setups, commits, - protocolStateMutator, - protocolState, + protocolStateSnapshotsDB, versionBeacons, + root.Params(), ) if err := IsValidRootSnapshot(root, !config.SkipNetworkAddressValidation); err != nil { @@ -649,8 +647,6 @@ func OpenState( if err != nil { return nil, fmt.Errorf("could not read global params") } - protocolStateMutator := protocol_state.NewMutator(headers, results, setups, commits, protocolState) - protocolStateReader := protocol_state.NewProtocolState(protocolState, globalParams) state := newState( metrics, db, @@ -661,9 +657,9 @@ func OpenState( qcs, setups, commits, - protocolStateMutator, - protocolStateReader, + protocolState, versionBeacons, + globalParams, ) // populate the protocol state cache err = state.populateCache() if err != nil { @@ -771,11 +767,11 @@ func newState( qcs storage.QuorumCertificates, setups storage.EpochSetups, commits storage.EpochCommits, - protocolStateMutator protocol.StateMutator, - protocolStateReader protocol.ProtocolState, + protocolStateSnapshots storage.ProtocolState, versionBeacons storage.VersionBeacons, + globalParams protocol.GlobalParams, ) *State { - return &State{ + state := &State{ metrics: metrics, db: db, headers: headers, @@ -790,12 +786,23 @@ func newState( setups: setups, commits: commits, }, - protocolState: protocolStateReader, - protocolStateMutator: protocolStateMutator, + protocolState: protocol_state.NewProtocolState(protocolStateSnapshots, globalParams), + protocolStateMutator: nil, versionBeacons: versionBeacons, cachedFinal: new(atomic.Pointer[cachedHeader]), cachedSealed: new(atomic.Pointer[cachedHeader]), } + + state.protocolStateMutator = protocol_state.NewMutator( + headers, + results, + setups, + commits, + protocolStateSnapshots, + state.Params(), + ) + + return state } // IsBootstrapped returns whether the database contains a bootstrapped state diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 5550527e7bc..fc1c3bb2ed5 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -21,6 +21,7 @@ type Mutator struct { setups storage.EpochSetups commits storage.EpochCommits protocolStateDB storage.ProtocolState + params protocol.InstanceParams } var _ protocol.StateMutator = (*Mutator)(nil) @@ -31,6 +32,7 @@ func NewMutator( setups storage.EpochSetups, commits storage.EpochCommits, protocolStateDB storage.ProtocolState, + params protocol.InstanceParams, ) *Mutator { return &Mutator{ headers: headers, @@ -38,6 +40,7 @@ func NewMutator( setups: setups, commits: commits, protocolStateDB: protocolStateDB, + params: params, } } @@ -106,12 +109,10 @@ func (m *Mutator) CommitProtocolState(blockID flow.Identifier, updater protocol. // // No errors are expected during normal operation. func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { - // TODO: hook up epoch fallback mode - epochFallbackTriggered := false - //epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() - //if err != nil { - // return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) - //} + epochFallbackTriggered, err := m.params.EpochFallbackTriggered() + if err != nil { + return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) + } parentProtocolState := updater.ParentState() epochStatus := parentProtocolState.EpochStatus() diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index b5d2e07d3e0..a742f12cd3f 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -285,7 +285,7 @@ func RunWithFullProtocolStateAndConsumer(t testing.TB, rootSnapshot protocol.Sna }) } -func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot protocol.Snapshot, metrics module.ComplianceMetrics, consumer protocol.Consumer, f func(*badger.DB, *pbadger.ParticipantState)) { +func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot protocol.Snapshot, metrics module.ComplianceMetrics, consumer protocol.Consumer, f func(*badger.DB, *pbadger.ParticipantState, protocol.StateMutator)) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { tracer := trace.NewNoopTracer() log := zerolog.Nop() @@ -320,7 +320,15 @@ func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot pr sealValidator, ) require.NoError(t, err) - f(db, fullState) + mutator := protocol_state.NewMutator( + all.Headers, + all.Results, + all.Setups, + all.EpochCommits, + all.ProtocolState, + state.Params(), + ) + f(db, fullState, mutator) }) } @@ -398,7 +406,14 @@ func RunWithFullProtocolStateAndMutator(t testing.TB, rootSnapshot protocol.Snap sealValidator, ) require.NoError(t, err) - mutator := protocol_state.NewMutator(all.Headers, all.Results, all.Setups, all.EpochCommits, all.ProtocolState) + mutator := protocol_state.NewMutator( + all.Headers, + all.Results, + all.Setups, + all.EpochCommits, + all.ProtocolState, + state.Params(), + ) f(db, fullState, mutator) }) } From f34610290dba318ac7012a073580c77dc4f54a68 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 11 Oct 2023 12:15:12 +0300 Subject: [PATCH 21/87] Fixed other tests for state module --- state/cluster/badger/mutator_test.go | 21 ++++++++++++++++--- state/protocol/protocol_state/mutator_test.go | 5 ++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 3c9974fd8b9..61cd40a27fc 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -3,6 +3,7 @@ package badger import ( "context" "fmt" + "github.com/onflow/flow-go/state/protocol/protocol_state" "math" "math/rand" "os" @@ -42,8 +43,9 @@ type MutatorSuite struct { epochCounter uint64 // protocol state for reference blocks for transactions - protoState protocol.FollowerState - protoGenesis *flow.Block + protoState protocol.FollowerState + protoStateMutator protocol.StateMutator + protoGenesis *flow.Block state cluster.MutableState } @@ -72,6 +74,10 @@ func (suite *MutatorSuite) SetupTest() { seal.ResultID = result.ID() qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) + genesis.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState( + result.ServiceEvents[0].Event.(*flow.EpochSetup), + result.ServiceEvents[1].Event.(*flow.EpochCommit), + ).ID() rootSnapshot, err := inmem.SnapshotFromBootstrapState(genesis, result, seal, qc) require.NoError(suite.T(), err) suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter @@ -95,6 +101,15 @@ func (suite *MutatorSuite) SetupTest() { suite.protoState, err = pbadger.NewFollowerState(log, tracer, events.NewNoop(), state, all.Index, all.Payloads, protocolutil.MockBlockTimer()) require.NoError(suite.T(), err) + suite.protoStateMutator = protocol_state.NewMutator( + all.Headers, + all.Results, + all.Setups, + all.EpochCommits, + all.ProtocolState, + state.Params(), + ) + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) suite.NoError(err) clusterState, err := Bootstrap(suite.db, clusterStateRoot) @@ -398,7 +413,7 @@ func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromClusterChain() { // using a reference block in a different epoch than the cluster's epoch. func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromDifferentEpoch() { // build and complete the current epoch, then use a reference block from next epoch - eb := unittest.NewEpochBuilder(suite.T(), suite.protoState) + eb := unittest.NewEpochBuilder(suite.T(), suite.protoStateMutator, suite.protoState) eb.BuildEpoch().CompleteEpoch() heights, ok := eb.EpochHeights(1) require.True(suite.T(), ok) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 51f9287bbbc..918d4581f02 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -1,6 +1,7 @@ package protocol_state import ( + "github.com/onflow/flow-go/state/protocol/mock" "testing" "github.com/stretchr/testify/require" @@ -23,6 +24,7 @@ type MutatorSuite struct { resultsDB *storagemock.ExecutionResults setupsDB *storagemock.EpochSetups commitsDB *storagemock.EpochCommits + params *mock.InstanceParams mutator *Mutator } @@ -33,8 +35,9 @@ func (s *MutatorSuite) SetupTest() { s.resultsDB = storagemock.NewExecutionResults(s.T()) s.setupsDB = storagemock.NewEpochSetups(s.T()) s.commitsDB = storagemock.NewEpochCommits(s.T()) + s.params = mock.NewInstanceParams(s.T()) - s.mutator = NewMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.protocolStateDB) + s.mutator = NewMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.protocolStateDB, s.params) } // TestCreateUpdaterForUnknownBlock tests that CreateUpdater returns an error if the parent protocol state is not found. From 9ab625339126fb7318890a8f9daca7490c6e1f07 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 12 Oct 2023 11:37:04 +0300 Subject: [PATCH 22/87] Fixed all tests from module package except consensus --- engine/verification/utils/unittest/fixture.go | 22 +++++++++----- engine/verification/utils/unittest/helper.go | 16 ++++++++-- module/builder/collection/builder_test.go | 29 +++++++++++++++---- .../jobqueue/finalized_block_reader_test.go | 8 +++-- 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index 57c9916e62d..03973924984 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -193,6 +193,7 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refBlkHeader *flow.Header, + protocolStateID flow.Identifier, clusterCommittee flow.IdentityList, source []byte, ) (*flow.ExecutionResult, *ExecutionReceiptData) { @@ -330,7 +331,8 @@ func ExecutionResultFixture(t *testing.T, } payload := flow.Payload{ - Guarantees: guarantees, + Guarantees: guarantees, + ProtocolStateID: protocolStateID, } referenceBlock = flow.Block{ Header: refBlkHeader, @@ -375,6 +377,7 @@ func ExecutionResultFixture(t *testing.T, // It returns a slice of complete execution receipt fixtures that contains a container block as well as all data to verify its contained receipts. func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, + rootProtocolStateID flow.Identifier, count int, sources [][]byte, opts ...CompleteExecutionReceiptBuilderOpt, @@ -404,9 +407,9 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, for i := 0; i < count; i++ { // Generates two blocks as parent <- R <- C where R is a reference block containing guarantees, // and C is a container block containing execution receipt for R. - receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder, sources[sourcesIndex:]) + receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, rootProtocolStateID, builder, sources[sourcesIndex:]) sourcesIndex += builder.resultsCount - containerBlock := ContainerBlockFixture(head, receipts, sources[sourcesIndex]) + containerBlock := ContainerBlockFixture(head, rootProtocolStateID, receipts, sources[sourcesIndex]) sourcesIndex++ completeERs = append(completeERs, &CompleteExecutionReceipt{ ContainerBlock: containerBlock, @@ -427,6 +430,7 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, // Each result may appear in more than one receipt depending on the builder parameters. func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, + protocolStateID flow.Identifier, builder *CompleteExecutionReceiptBuilder, sources [][]byte) ( []*flow.ExecutionReceipt, @@ -436,7 +440,7 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, allReceipts := make([]*flow.ExecutionReceipt, 0, builder.resultsCount*builder.executorCount) for i := 0; i < builder.resultsCount; i++ { - result, data := ExecutionResultFromParentBlockFixture(t, parent, builder, sources[i:]) + result, data := ExecutionResultFromParentBlockFixture(t, parent, protocolStateID, builder, sources[i:]) // makes several copies of the same result for cp := 0; cp < builder.executorCount; cp++ { @@ -456,21 +460,25 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, // ExecutionResultFromParentBlockFixture is a test helper that creates a child (reference) block from the parent, as well as an execution for it. func ExecutionResultFromParentBlockFixture(t *testing.T, parent *flow.Header, + protocolStateID flow.Identifier, builder *CompleteExecutionReceiptBuilder, sources [][]byte, ) (*flow.ExecutionResult, *ExecutionReceiptData) { // create the block header including a QC with source a index `i` refBlkHeader := unittest.BlockHeaderWithParentWithSoRFixture(parent, sources[0]) // execute the block with the source a index `i+1` (which will be included later in the child block) - return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee, sources[1]) + return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, protocolStateID, builder.clusterCommittee, sources[1]) } // ContainerBlockFixture builds and returns a block that contains input execution receipts. -func ContainerBlockFixture(parent *flow.Header, receipts []*flow.ExecutionReceipt, source []byte) *flow.Block { +func ContainerBlockFixture(parent *flow.Header, protocolStateID flow.Identifier, receipts []*flow.ExecutionReceipt, source []byte) *flow.Block { // container block is the block that contains the execution receipt of reference block containerBlock := unittest.BlockWithParentFixture(parent) containerBlock.Header.ParentVoterSigData = unittest.QCSigDataWithSoRFixture(source) - containerBlock.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipts...))) + containerBlock.SetPayload(unittest.PayloadFixture( + unittest.WithReceipts(receipts...), + unittest.WithProtocolStateID(protocolStateID), + )) return containerBlock } diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 62f26cd7f70..b26f1e9c498 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -3,6 +3,7 @@ package verificationtest import ( "context" "fmt" + "github.com/onflow/flow-go/state/protocol/protocol_state" "sync" "testing" "time" @@ -487,6 +488,10 @@ func withConsumers(t *testing.T, // hold any guarantees. root, err := s.State.Final().Head() require.NoError(t, err) + protocolState, err := s.State.Final().ProtocolState() + require.NoError(t, err) + protocolStateID := protocolState.Entry().ID() + chainID := root.ChainID ops = append(ops, WithExecutorIDs( participants.Filter(filter.HasRole(flow.RoleExecution)).NodeIDs()), func(builder *CompleteExecutionReceiptBuilder) { @@ -498,7 +503,7 @@ func withConsumers(t *testing.T, // - root block (block[0]) is executed with sources[0] (included in QC of child block[1]) // - block[i] is executed with sources[i] (included in QC of child block[i+1]) sources := unittest.RandomSourcesFixture(30) - completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, sources, ops...) + completeERs := CompleteExecutionReceiptChainFixture(t, root, protocolStateID, blockCount, sources, ops...) blocks := ExtendStateWithFinalizedBlocks(t, completeERs, s.State) // chunk assignment @@ -631,7 +636,14 @@ func bootstrapSystem( verID = unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) identities = identities.Union(flow.IdentityList{verID}) - epochBuilder := unittest.NewEpochBuilder(t, stateFixture.State) + mutator := protocol_state.NewMutator( + stateFixture.Storage.Headers, + stateFixture.Storage.Results, + stateFixture.Storage.Setups, + stateFixture.Storage.EpochCommits, + stateFixture.Storage.ProtocolState, + stateFixture.State.Params()) + epochBuilder := unittest.NewEpochBuilder(t, mutator, stateFixture.State) epochBuilder. UsingSetupOpts(unittest.WithParticipants(identities)). BuildEpoch() diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 6041ec2b367..fd2bf1826d9 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -85,11 +85,19 @@ func (suite *BuilderSuite) SetupTest() { // ensure we don't enter a new epoch for tests that build many blocks result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = root.Header.View + 100000 seal.ResultID = result.ID() + root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState( + result.ServiceEvents[0].Event.(*flow.EpochSetup), + result.ServiceEvents[1].Event.(*flow.EpochCommit), + ).ID() rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID()))) require.NoError(suite.T(), err) suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) + root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState( + result.ServiceEvents[0].Event.(*flow.EpochSetup), + result.ServiceEvents[1].Event.(*flow.EpochCommit), + ).ID() clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC, suite.epochCounter) suite.Require().NoError(err) clusterState, err := clusterkv.Bootstrap(suite.db, clusterStateRoot) @@ -280,8 +288,12 @@ func (suite *BuilderSuite) TestBuildOn_WithUnfinalizedReferenceBlock() { // add an unfinalized block to the protocol state genesis, err := suite.protoState.Final().Head() suite.Require().NoError(err) + protocolState, err := suite.protoState.Final().ProtocolState() + suite.Require().NoError(err) + protocolStateID := protocolState.Entry().ID() + unfinalizedReferenceBlock := unittest.BlockWithParentFixture(genesis) - unfinalizedReferenceBlock.SetPayload(flow.EmptyPayload()) + unfinalizedReferenceBlock.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(protocolStateID))) err = suite.protoState.ExtendCertified(context.Background(), unfinalizedReferenceBlock, unittest.CertifyBlock(unfinalizedReferenceBlock.Header)) suite.Require().NoError(err) @@ -316,14 +328,18 @@ func (suite *BuilderSuite) TestBuildOn_WithOrphanedReferenceBlock() { // add an orphaned block to the protocol state genesis, err := suite.protoState.Final().Head() suite.Require().NoError(err) + protocolState, err := suite.protoState.Final().ProtocolState() + suite.Require().NoError(err) + protocolStateID := protocolState.Entry().ID() + // create a block extending genesis which will be orphaned orphan := unittest.BlockWithParentFixture(genesis) - orphan.SetPayload(flow.EmptyPayload()) + orphan.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(protocolStateID))) err = suite.protoState.ExtendCertified(context.Background(), orphan, unittest.CertifyBlock(orphan.Header)) suite.Require().NoError(err) // create and finalize a block on top of genesis, orphaning `orphan` block1 := unittest.BlockWithParentFixture(genesis) - block1.SetPayload(flow.EmptyPayload()) + block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(protocolStateID))) err = suite.protoState.ExtendCertified(context.Background(), block1, unittest.CertifyBlock(block1.Header)) suite.Require().NoError(err) err = suite.protoState.Finalize(context.Background(), block1.ID()) @@ -626,13 +642,14 @@ func (suite *BuilderSuite) TestBuildOn_ExpiredTransaction() { // create enough main-chain blocks that an expired transaction is possible genesis, err := suite.protoState.Final().Head() suite.Require().NoError(err) + protocolState, err := suite.protoState.Final().ProtocolState() + suite.Require().NoError(err) + protocolStateID := protocolState.Entry().ID() head := genesis for i := 0; i < flow.DefaultTransactionExpiry+1; i++ { block := unittest.BlockWithParentFixture(head) - block.Payload.Guarantees = nil - block.Payload.Seals = nil - block.Header.PayloadHash = block.Payload.Hash() + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(protocolStateID))) err = suite.protoState.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header)) suite.Require().NoError(err) err = suite.protoState.Finalize(context.Background(), block.ID()) diff --git a/module/jobqueue/finalized_block_reader_test.go b/module/jobqueue/finalized_block_reader_test.go index 8349828d272..c816c290048 100644 --- a/module/jobqueue/finalized_block_reader_test.go +++ b/module/jobqueue/finalized_block_reader_test.go @@ -62,11 +62,15 @@ func withReader( // blocks (i.e., containing guarantees), and Cs are container blocks for their preceding reference block, // Container blocks only contain receipts of their preceding reference blocks. But they do not // hold any guarantees. - root, err := s.State.Params().FinalizedRoot() + root, err := s.State.Final().Head() require.NoError(t, err) + protocolState, err := s.State.Final().ProtocolState() + require.NoError(t, err) + protocolStateID := protocolState.Entry().ID() + clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) sources := unittest.RandomSourcesFixture(10) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) + results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, protocolStateID, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) withBlockReader(reader, blocks) From d0799536f40014c0ef04bd4f48db0ca48b286020 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 12 Oct 2023 11:46:25 +0300 Subject: [PATCH 23/87] Updated mocks. Updated builder constructor --- module/builder/consensus/builder.go | 64 +++++++++++----------- module/builder/consensus/builder_test.go | 19 ++++--- state/protocol/mock/state_mutator.go | 68 ++++++++++++++++++------ state/protocol/mock/state_updater.go | 30 +++++------ 4 files changed, 112 insertions(+), 69 deletions(-) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index d5c89fabbd5..bade14eadaa 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -25,21 +25,21 @@ import ( // Builder is the builder for consensus block payloads. Upon providing a payload // hash, it also memorizes which entities were included into the payload. type Builder struct { - metrics module.MempoolMetrics - tracer module.Tracer - db *badger.DB - state protocol.ParticipantState - seals storage.Seals - headers storage.Headers - index storage.Index - blocks storage.Blocks - resultsDB storage.ExecutionResults - receiptsDB storage.ExecutionReceipts - guarPool mempool.Guarantees - sealPool mempool.IncorporatedResultSeals - recPool mempool.ExecutionTree - protocolStateMutator protocol.StateMutator - cfg Config + metrics module.MempoolMetrics + tracer module.Tracer + db *badger.DB + state protocol.ParticipantState + seals storage.Seals + headers storage.Headers + index storage.Index + blocks storage.Blocks + resultsDB storage.ExecutionResults + receiptsDB storage.ExecutionReceipts + guarPool mempool.Guarantees + sealPool mempool.IncorporatedResultSeals + recPool mempool.ExecutionTree + protoStateMutator protocol.StateMutator + cfg Config } // NewBuilder creates a new block builder. @@ -53,6 +53,7 @@ func NewBuilder( blocks storage.Blocks, resultsDB storage.ExecutionResults, receiptsDB storage.ExecutionReceipts, + protoStateMutator protocol.StateMutator, guarPool mempool.Guarantees, sealPool mempool.IncorporatedResultSeals, recPool mempool.ExecutionTree, @@ -80,20 +81,21 @@ func NewBuilder( } b := &Builder{ - metrics: metrics, - db: db, - tracer: tracer, - state: state, - headers: headers, - seals: seals, - index: index, - blocks: blocks, - resultsDB: resultsDB, - receiptsDB: receiptsDB, - guarPool: guarPool, - sealPool: sealPool, - recPool: recPool, - cfg: cfg, + metrics: metrics, + db: db, + tracer: tracer, + state: state, + headers: headers, + seals: seals, + index: index, + blocks: blocks, + resultsDB: resultsDB, + receiptsDB: receiptsDB, + guarPool: guarPool, + sealPool: sealPool, + recPool: recPool, + protoStateMutator: protoStateMutator, + cfg: cfg, } err = b.repopulateExecutionTree() @@ -625,8 +627,8 @@ func (b *Builder) createProposal(parentID flow.Identifier, PayloadHash: flow.ZeroID, } - updater, err := b.protocolStateMutator.CreateUpdater(header.View, header.ParentID) - _, err = b.protocolStateMutator.ApplyServiceEvents(updater, seals) + updater, err := b.protoStateMutator.CreateUpdater(header.View, header.ParentID) + _, err = b.protoStateMutator.ApplyServiceEvents(updater, seals) _, protocolStateID, _ := updater.Build() diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index d8f82c8eda8..a77630f77ad 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -68,13 +68,14 @@ type BuilderSuite struct { setter func(*flow.Header) error // mocked dependencies - state *protocol.ParticipantState - headerDB *storage.Headers - sealDB *storage.Seals - indexDB *storage.Index - blockDB *storage.Blocks - resultDB *storage.ExecutionResults - receiptsDB *storage.ExecutionReceipts + state *protocol.ParticipantState + headerDB *storage.Headers + sealDB *storage.Seals + indexDB *storage.Index + blockDB *storage.Blocks + resultDB *storage.ExecutionResults + receiptsDB *storage.ExecutionReceipts + stateMutator *protocol.StateMutator guarPool *mempool.Guarantees sealPool *mempool.IncorporatedResultSeals @@ -409,6 +410,8 @@ func (bs *BuilderSuite) SetupTest() { nil, ) + bs.stateMutator = protocol.NewStateMutator(bs.T()) + // initialize the builder bs.build, err = NewBuilder( noopMetrics, @@ -420,6 +423,7 @@ func (bs *BuilderSuite) SetupTest() { bs.blockDB, bs.resultDB, bs.receiptsDB, + bs.stateMutator, bs.guarPool, bs.sealPool, bs.recPool, @@ -1431,6 +1435,7 @@ func (bs *BuilderSuite) TestIntegration_RepopulateExecutionTreeAtStartup() { bs.blockDB, bs.resultDB, bs.receiptsDB, + bs.stateMutator, bs.guarPool, bs.sealPool, recPool, diff --git a/state/protocol/mock/state_mutator.go b/state/protocol/mock/state_mutator.go index c5dbec4cbf8..ac3479e1eaf 100644 --- a/state/protocol/mock/state_mutator.go +++ b/state/protocol/mock/state_mutator.go @@ -16,41 +16,79 @@ type StateMutator struct { mock.Mock } -// CommitProtocolState provides a mock function with given fields: updater -func (_m *StateMutator) CommitProtocolState(updater protocol.StateUpdater) func(*transaction.Tx) error { - ret := _m.Called(updater) +// ApplyServiceEvents provides a mock function with given fields: updater, seals +func (_m *StateMutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) ([]func(*transaction.Tx) error, error) { + ret := _m.Called(updater, seals) + + var r0 []func(*transaction.Tx) error + var r1 error + if rf, ok := ret.Get(0).(func(protocol.StateUpdater, []*flow.Seal) ([]func(*transaction.Tx) error, error)); ok { + return rf(updater, seals) + } + if rf, ok := ret.Get(0).(func(protocol.StateUpdater, []*flow.Seal) []func(*transaction.Tx) error); ok { + r0 = rf(updater, seals) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]func(*transaction.Tx) error) + } + } + + if rf, ok := ret.Get(1).(func(protocol.StateUpdater, []*flow.Seal) error); ok { + r1 = rf(updater, seals) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CommitProtocolState provides a mock function with given fields: blockID, updater +func (_m *StateMutator) CommitProtocolState(blockID flow.Identifier, updater protocol.StateUpdater) (func(*transaction.Tx) error, flow.Identifier) { + ret := _m.Called(blockID, updater) var r0 func(*transaction.Tx) error - if rf, ok := ret.Get(0).(func(protocol.StateUpdater) func(*transaction.Tx) error); ok { - r0 = rf(updater) + var r1 flow.Identifier + if rf, ok := ret.Get(0).(func(flow.Identifier, protocol.StateUpdater) (func(*transaction.Tx) error, flow.Identifier)); ok { + return rf(blockID, updater) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, protocol.StateUpdater) func(*transaction.Tx) error); ok { + r0 = rf(blockID, updater) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(func(*transaction.Tx) error) } } - return r0 + if rf, ok := ret.Get(1).(func(flow.Identifier, protocol.StateUpdater) flow.Identifier); ok { + r1 = rf(blockID, updater) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(flow.Identifier) + } + } + + return r0, r1 } -// CreateUpdater provides a mock function with given fields: candidate -func (_m *StateMutator) CreateUpdater(candidate *flow.Header) (protocol.StateUpdater, error) { - ret := _m.Called(candidate) +// CreateUpdater provides a mock function with given fields: candidateView, parentID +func (_m *StateMutator) CreateUpdater(candidateView uint64, parentID flow.Identifier) (protocol.StateUpdater, error) { + ret := _m.Called(candidateView, parentID) var r0 protocol.StateUpdater var r1 error - if rf, ok := ret.Get(0).(func(*flow.Header) (protocol.StateUpdater, error)); ok { - return rf(candidate) + if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) (protocol.StateUpdater, error)); ok { + return rf(candidateView, parentID) } - if rf, ok := ret.Get(0).(func(*flow.Header) protocol.StateUpdater); ok { - r0 = rf(candidate) + if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) protocol.StateUpdater); ok { + r0 = rf(candidateView, parentID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(protocol.StateUpdater) } } - if rf, ok := ret.Get(1).(func(*flow.Header) error); ok { - r1 = rf(candidate) + if rf, ok := ret.Get(1).(func(uint64, flow.Identifier) error); ok { + r1 = rf(candidateView, parentID) } else { r1 = ret.Error(1) } diff --git a/state/protocol/mock/state_updater.go b/state/protocol/mock/state_updater.go index 87ad9c23f6c..66411981821 100644 --- a/state/protocol/mock/state_updater.go +++ b/state/protocol/mock/state_updater.go @@ -12,22 +12,6 @@ type StateUpdater struct { mock.Mock } -// Block provides a mock function with given fields: -func (_m *StateUpdater) Block() *flow.Header { - ret := _m.Called() - - var r0 *flow.Header - if rf, ok := ret.Get(0).(func() *flow.Header); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Header) - } - } - - return r0 -} - // Build provides a mock function with given fields: func (_m *StateUpdater) Build() (*flow.ProtocolStateEntry, flow.Identifier, bool) { ret := _m.Called() @@ -140,6 +124,20 @@ func (_m *StateUpdater) UpdateIdentity(updated *flow.DynamicIdentityEntry) error return r0 } +// View provides a mock function with given fields: +func (_m *StateUpdater) View() uint64 { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + return r0 +} + type mockConstructorTestingTNewStateUpdater interface { mock.TestingT Cleanup(func()) From 059bf82350e05f48e3cfa3487e22fda0475c9954 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 12 Oct 2023 12:11:08 +0300 Subject: [PATCH 24/87] Fixed consensus builder tests --- module/builder/consensus/builder_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index a77630f77ad..38b62053521 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -410,7 +410,18 @@ func (bs *BuilderSuite) SetupTest() { nil, ) + // setup mock state mutator, we don't need a real once since we are using mocked participant state. bs.stateMutator = protocol.NewStateMutator(bs.T()) + bs.stateMutator.On("CreateUpdater", mock.Anything, mock.Anything).Return( + func(_ uint64, _ flow.Identifier) realproto.StateUpdater { + updater := protocol.NewStateUpdater(bs.T()) + updater.On("Build").Return(nil, flow.Identifier{}, false) + return updater + }, func(_ uint64, _ flow.Identifier) error { + return nil + }, + ) + bs.stateMutator.On("ApplyServiceEvents", mock.Anything, mock.Anything).Return(nil, nil) // initialize the builder bs.build, err = NewBuilder( From 0ac1dd399b2a9ee0b2ab6c17790101d811492648 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 12 Oct 2023 13:11:26 +0300 Subject: [PATCH 25/87] Decoupled setting hotstuff fields and producing a signature in block building process. Updated mocks and tests --- .../hotstuff/blockproducer/block_producer.go | 5 +++- .../hotstuff/blockproducer/metrics_wrapper.go | 4 ++-- .../hotstuff/integration/instance_test.go | 7 +++--- consensus/integration/nodes_test.go | 21 ++++++++++++++-- module/builder.go | 2 +- module/builder/consensus/builder.go | 24 ++++++++++++------- module/mock/builder.go | 18 +++++++------- 7 files changed, 55 insertions(+), 26 deletions(-) diff --git a/consensus/hotstuff/blockproducer/block_producer.go b/consensus/hotstuff/blockproducer/block_producer.go index 74d01cc317d..64e89679b7d 100644 --- a/consensus/hotstuff/blockproducer/block_producer.go +++ b/consensus/hotstuff/blockproducer/block_producer.go @@ -43,7 +43,10 @@ func (bp *BlockProducer) MakeBlockProposal(view uint64, qc *flow.QuorumCertifica header.ParentVoterSigData = qc.SigData header.ProposerID = bp.committee.Self() header.LastViewTC = lastViewTC + return nil + } + signProposal := func(header *flow.Header) error { // turn the header into a block header proposal as known by hotstuff block := model.Block{ BlockID: header.ID(), @@ -65,7 +68,7 @@ func (bp *BlockProducer) MakeBlockProposal(view uint64, qc *flow.QuorumCertifica } // retrieve a fully built block header from the builder - header, err := bp.builder.BuildOn(qc.BlockID, setHotstuffFields) + header, err := bp.builder.BuildOn(qc.BlockID, setHotstuffFields, signProposal) if err != nil { return nil, fmt.Errorf("could not build block proposal on top of %v: %w", qc.BlockID, err) } diff --git a/consensus/hotstuff/blockproducer/metrics_wrapper.go b/consensus/hotstuff/blockproducer/metrics_wrapper.go index 004b238d3e1..80fcb563541 100644 --- a/consensus/hotstuff/blockproducer/metrics_wrapper.go +++ b/consensus/hotstuff/blockproducer/metrics_wrapper.go @@ -24,9 +24,9 @@ func NewMetricsWrapper(builder module.Builder, metrics module.HotstuffMetrics) * } } -func (w BlockBuilderMetricsWrapper) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { +func (w BlockBuilderMetricsWrapper) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) (*flow.Header, error) { processStart := time.Now() - header, err := w.builder.BuildOn(parentID, setter) + header, err := w.builder.BuildOn(parentID, setter, sign) w.metrics.PayloadProductionDuration(time.Since(processStart)) return header, err } diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index 8728b1b994e..58090d0e0fc 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -189,8 +189,8 @@ func NewInstance(t *testing.T, options ...Option) *Instance { in.committee.On("TimeoutThresholdForView", mock.Anything).Return(committees.WeightThresholdToTimeout(in.participants.ToSkeleton().TotalWeight()), nil) // program the builder module behaviour - in.builder.On("BuildOn", mock.Anything, mock.Anything).Return( - func(parentID flow.Identifier, setter func(*flow.Header) error) *flow.Header { + in.builder.On("BuildOn", mock.Anything, mock.Anything, mock.Anything).Return( + func(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) *flow.Header { in.updatingBlocks.Lock() defer in.updatingBlocks.Unlock() @@ -207,10 +207,11 @@ func NewInstance(t *testing.T, options ...Option) *Instance { Timestamp: time.Now().UTC(), } require.NoError(t, setter(header)) + require.NoError(t, sign(header)) in.headers[header.ID()] = header return header }, - func(parentID flow.Identifier, setter func(*flow.Header) error) error { + func(parentID flow.Identifier, _ func(*flow.Header) error, _ func(*flow.Header) error) error { in.updatingBlocks.RLock() _, ok := in.headers[parentID] in.updatingBlocks.RUnlock() diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 84a1611bfa3..e210d754327 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -3,6 +3,7 @@ package integration_test import ( "context" "fmt" + "github.com/onflow/flow-go/state/protocol/protocol_state" "os" "sort" "testing" @@ -459,9 +460,25 @@ func createNode( seals := stdmap.NewIncorporatedResultSeals(sealLimit) + protocolStateMutator := protocol_state.NewMutator(headersDB, resultsDB, setupsDB, commitsDB, protocolStateDB, state.Params()) + // initialize the block builder - build, err := builder.NewBuilder(metricsCollector, db, fullState, headersDB, sealsDB, indexDB, blocksDB, resultsDB, receiptsDB, - guarantees, consensusMempools.NewIncorporatedResultSeals(seals, receiptsDB), receipts, tracer) + build, err := builder.NewBuilder( + metricsCollector, + db, + fullState, + headersDB, + sealsDB, + indexDB, + blocksDB, + resultsDB, + receiptsDB, + protocolStateMutator, + guarantees, + consensusMempools.NewIncorporatedResultSeals(seals, receiptsDB), + receipts, + tracer, + ) require.NoError(t, err) // initialize the pending blocks cache diff --git a/module/builder.go b/module/builder.go index 2138134ec89..f67210183ab 100644 --- a/module/builder.go +++ b/module/builder.go @@ -19,5 +19,5 @@ type Builder interface { // // NOTE: Since the block is stored within Builder, HotStuff MUST propose the // block once BuildOn succcessfully returns. - BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) + BuildOn(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) (*flow.Header, error) } diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index bade14eadaa..f7220ffc50a 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -109,7 +109,7 @@ func NewBuilder( // BuildOn creates a new block header on top of the provided parent, using the // given view and applying the custom setter function to allow the caller to // make changes to the header before storing it. -func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { +func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) (*flow.Header, error) { // since we don't know the blockID when building the block we track the // time indirectly and insert the span directly at the end @@ -139,7 +139,8 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er insertableGuarantees, insertableSeals, insertableReceipts, - setter) + setter, + sign) if err != nil { return nil, fmt.Errorf("could not assemble proposal: %w", err) } @@ -609,7 +610,9 @@ func (b *Builder) createProposal(parentID flow.Identifier, guarantees []*flow.CollectionGuarantee, seals []*flow.Seal, insertableReceipts *InsertableReceipts, - setter func(*flow.Header) error) (*flow.Block, error) { + setter func(*flow.Header) error, + sign func(*flow.Header) error, +) (*flow.Block, error) { parent, err := b.headers.ByBlockID(parentID) if err != nil { @@ -627,17 +630,16 @@ func (b *Builder) createProposal(parentID flow.Identifier, PayloadHash: flow.ZeroID, } - updater, err := b.protoStateMutator.CreateUpdater(header.View, header.ParentID) - _, err = b.protoStateMutator.ApplyServiceEvents(updater, seals) - - _, protocolStateID, _ := updater.Build() - // apply the custom fields setter of the consensus algorithm err = setter(header) if err != nil { return nil, fmt.Errorf("could not apply setter: %w", err) } + updater, err := b.protoStateMutator.CreateUpdater(header.View, header.ParentID) + _, err = b.protoStateMutator.ApplyServiceEvents(updater, seals) + _, protocolStateID, _ := updater.Build() + proposal := &flow.Block{ Header: header, } @@ -649,6 +651,12 @@ func (b *Builder) createProposal(parentID flow.Identifier, ProtocolStateID: protocolStateID, }) + // sign the proposal + err = sign(header) + if err != nil { + return nil, fmt.Errorf("could not sign the proposal: %w", err) + } + return proposal, nil } diff --git a/module/mock/builder.go b/module/mock/builder.go index ad65271ddd7..ef109ae5689 100644 --- a/module/mock/builder.go +++ b/module/mock/builder.go @@ -12,25 +12,25 @@ type Builder struct { mock.Mock } -// BuildOn provides a mock function with given fields: parentID, setter -func (_m *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { - ret := _m.Called(parentID, setter) +// BuildOn provides a mock function with given fields: parentID, setter, sign +func (_m *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) (*flow.Header, error) { + ret := _m.Called(parentID, setter, sign) var r0 *flow.Header var r1 error - if rf, ok := ret.Get(0).(func(flow.Identifier, func(*flow.Header) error) (*flow.Header, error)); ok { - return rf(parentID, setter) + if rf, ok := ret.Get(0).(func(flow.Identifier, func(*flow.Header) error, func(*flow.Header) error) (*flow.Header, error)); ok { + return rf(parentID, setter, sign) } - if rf, ok := ret.Get(0).(func(flow.Identifier, func(*flow.Header) error) *flow.Header); ok { - r0 = rf(parentID, setter) + if rf, ok := ret.Get(0).(func(flow.Identifier, func(*flow.Header) error, func(*flow.Header) error) *flow.Header); ok { + r0 = rf(parentID, setter, sign) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*flow.Header) } } - if rf, ok := ret.Get(1).(func(flow.Identifier, func(*flow.Header) error) error); ok { - r1 = rf(parentID, setter) + if rf, ok := ret.Get(1).(func(flow.Identifier, func(*flow.Header) error, func(*flow.Header) error) error); ok { + r1 = rf(parentID, setter, sign) } else { r1 = ret.Error(1) } From d7a6dba09d9b4d4c48d065830c2f6f72acbe53c1 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 12 Oct 2023 13:51:19 +0300 Subject: [PATCH 26/87] Fixed broken builder tests --- cmd/consensus/main.go | 10 +++ engine/access/rpc/backend/backend_test.go | 21 ++++--- .../rpc/backend/backend_transactions_test.go | 4 +- .../test/cluster_switchover_test.go | 16 ++++- engine/testutil/mock/nodes.go | 40 ++++++------ engine/testutil/nodes.go | 45 ++++++++------ module/builder.go | 2 +- module/builder/collection/builder.go | 15 ++++- module/builder/collection/builder_test.go | 43 ++++++------- module/builder/consensus/builder.go | 3 +- module/builder/consensus/builder_test.go | 62 ++++++++++--------- 11 files changed, 155 insertions(+), 106 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 2090311b1ae..5678b791fe5 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/onflow/flow-go/state/protocol/protocol_state" "os" "path/filepath" "time" @@ -722,6 +723,14 @@ func main() { return ctl, nil }). Component("consensus participant", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + protocolStateMutator := protocol_state.NewMutator( + node.Storage.Headers, + node.Storage.Results, + node.Storage.Setups, + node.Storage.EpochCommits, + node.Storage.ProtocolState, + node.State.Params(), + ) // initialize the block builder var build module.Builder build, err = builder.NewBuilder( @@ -734,6 +743,7 @@ func main() { node.Storage.Blocks, node.Storage.Results, node.Storage.Receipts, + protocolStateMutator, guarantees, seals, receipts, diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 6bedde98329..3247aae8a81 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -3,6 +3,7 @@ package backend import ( "context" "fmt" + realprotocol "github.com/onflow/flow-go/state/protocol" "strconv" "testing" @@ -152,8 +153,8 @@ func (suite *Suite) TestGetLatestFinalizedBlockHeader() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_NoTransitionSpan() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolState(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) // build epoch 1 // Blocks in current State // P <- A(S_P-1) <- B(S_P) <- C(S_A) <- D(S_B) |setup| <- E(S_C) <- F(S_D) |commit| @@ -203,8 +204,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_NoTransitionSpan() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_TransitionSpans() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolState(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) // building 2 epochs allows us to take a snapshot at a point in time where // an epoch transition happens @@ -264,8 +265,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_TransitionSpans() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_PhaseTransitionSpan() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolState(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) // build epoch 1 // Blocks in current State // P <- A(S_P-1) <- B(S_P) <- C(S_A) <- D(S_B) |setup| <- E(S_C) <- F(S_D) |commit| @@ -316,8 +317,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_PhaseTransitionSpan() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_EpochTransitionSpan() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolState(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) // build epoch 1 // Blocks in current State // P <- A(S_P-1) <- B(S_P) <- C(S_A) <- D(S_B) |setup| <- E(S_C) <- F(S_D) |commit| @@ -380,8 +381,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_EpochTransitionSpan() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_HistoryLimit() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolState(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), state).BuildEpoch().CompleteEpoch() + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state).BuildEpoch().CompleteEpoch() // get heights of each phase in built epochs epoch1, ok := epochBuilder.EpochHeights(1) diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index 3b1464a3e13..7ce833cca2f 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -25,8 +25,8 @@ import ( func (suite *Suite) withPreConfiguredState(f func(snap protocol.Snapshot)) { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolState(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) epochBuilder. BuildEpoch(). diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index ec582fadfc4..c60f2420804 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -1,6 +1,7 @@ package test import ( + "github.com/onflow/flow-go/state/protocol/protocol_state" "sync" "testing" "time" @@ -96,6 +97,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) commit.ClusterQCs = rootClusterQCs seal.ResultID = result.ID() + root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState(setup, commit).ID() tc.root, err = inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) @@ -121,9 +123,21 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) for _, node := range tc.nodes { states = append(states, node.State) } + + // take first collection node and use its storage as data source for mutator + refNode := tc.nodes[0] + mutator := protocol_state.NewMutator( + refNode.Headers, + refNode.Results, + refNode.Setups, + refNode.EpochCommits, + refNode.ProtocolStateSnapshots, + refNode.State.Params(), + ) + // when building new epoch we would like to replace fixture cluster QCs with real ones, for that we need // to generate them using node infos - tc.builder = unittest.NewEpochBuilder(tc.T(), states...).UsingCommitOpts(func(commit *flow.EpochCommit) { + tc.builder = unittest.NewEpochBuilder(tc.T(), mutator, states...).UsingCommitOpts(func(commit *flow.EpochCommit) { // build a lookup table for node infos nodeInfoLookup := make(map[flow.Identifier]model.NodeInfo) for _, nodeInfo := range nodeInfos { diff --git a/engine/testutil/mock/nodes.go b/engine/testutil/mock/nodes.go index 191dde0e28b..2de074cf947 100644 --- a/engine/testutil/mock/nodes.go +++ b/engine/testutil/mock/nodes.go @@ -68,24 +68,28 @@ type GenericNode struct { Cancel context.CancelFunc Errs <-chan error - Log zerolog.Logger - Metrics *metrics.NoopCollector - Tracer module.Tracer - PublicDB *badger.DB - SecretsDB *badger.DB - Headers storage.Headers - Guarantees storage.Guarantees - Seals storage.Seals - Payloads storage.Payloads - Blocks storage.Blocks - QuorumCertificates storage.QuorumCertificates - State protocol.ParticipantState - Index storage.Index - Me module.Local - Net *stub.Network - DBDir string - ChainID flow.ChainID - ProtocolEvents *events.Distributor + Log zerolog.Logger + Metrics *metrics.NoopCollector + Tracer module.Tracer + PublicDB *badger.DB + SecretsDB *badger.DB + Headers storage.Headers + Guarantees storage.Guarantees + Seals storage.Seals + Payloads storage.Payloads + Blocks storage.Blocks + QuorumCertificates storage.QuorumCertificates + Results storage.ExecutionResults + Setups storage.EpochSetups + EpochCommits storage.EpochCommits + ProtocolStateSnapshots storage.ProtocolState + State protocol.ParticipantState + Index storage.Index + Me module.Local + Net *stub.Network + DBDir string + ChainID flow.ChainID + ProtocolEvents *events.Distributor } func (g *GenericNode) Done() { diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 420135ec8a0..c9b3af765a4 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -178,26 +178,31 @@ func GenericNodeWithStateFixture(t testing.TB, ctx, errs := irrecoverable.WithSignaler(parentCtx) return testmock.GenericNode{ - Ctx: ctx, - Cancel: cancel, - Errs: errs, - Log: log, - Metrics: metrics, - Tracer: tracer, - PublicDB: stateFixture.PublicDB, - SecretsDB: stateFixture.SecretsDB, - State: stateFixture.State, - Headers: stateFixture.Storage.Headers, - Guarantees: stateFixture.Storage.Guarantees, - Seals: stateFixture.Storage.Seals, - Payloads: stateFixture.Storage.Payloads, - Blocks: stateFixture.Storage.Blocks, - QuorumCertificates: stateFixture.Storage.QuorumCertificates, - Me: me, - Net: net, - DBDir: stateFixture.DBDir, - ChainID: chainID, - ProtocolEvents: stateFixture.ProtocolEvents, + Ctx: ctx, + Cancel: cancel, + Errs: errs, + Log: log, + Metrics: metrics, + Tracer: tracer, + PublicDB: stateFixture.PublicDB, + SecretsDB: stateFixture.SecretsDB, + Headers: stateFixture.Storage.Headers, + Guarantees: stateFixture.Storage.Guarantees, + Seals: stateFixture.Storage.Seals, + Payloads: stateFixture.Storage.Payloads, + Blocks: stateFixture.Storage.Blocks, + QuorumCertificates: stateFixture.Storage.QuorumCertificates, + Results: stateFixture.Storage.Results, + Setups: stateFixture.Storage.Setups, + EpochCommits: stateFixture.Storage.EpochCommits, + ProtocolStateSnapshots: stateFixture.Storage.ProtocolState, + State: stateFixture.State, + Index: stateFixture.Storage.Index, + Me: me, + Net: net, + DBDir: stateFixture.DBDir, + ChainID: chainID, + ProtocolEvents: stateFixture.ProtocolEvents, } } diff --git a/module/builder.go b/module/builder.go index f67210183ab..cc528524a8f 100644 --- a/module/builder.go +++ b/module/builder.go @@ -18,6 +18,6 @@ type Builder interface { // before returning it. // // NOTE: Since the block is stored within Builder, HotStuff MUST propose the - // block once BuildOn succcessfully returns. + // block once BuildOn successfully returns. BuildOn(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) (*flow.Header, error) } diff --git a/module/builder/collection/builder.go b/module/builder/collection/builder.go index 91f7fe93e37..fb37de87362 100644 --- a/module/builder/collection/builder.go +++ b/module/builder/collection/builder.go @@ -94,7 +94,7 @@ func NewBuilder( // BuildOn creates a new block built on the given parent. It produces a payload // that is valid with respect to the un-finalized chain it extends. -func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { +func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error, sign func(*flow.Header) error) (*flow.Header, error) { parentSpan, ctx := b.tracer.StartSpanFromContext(context.Background(), trace.COLBuildOn) defer parentSpan.End() @@ -180,7 +180,7 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er // STEP 3: we have a set of transactions that are valid to include on this fork. // Now we create the header for the cluster block. span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnCreateHeader) - header, err := b.buildHeader(buildCtx, payload, setter) + header, err := b.buildHeader(buildCtx, payload, setter, sign) span.End() if err != nil { return nil, fmt.Errorf("could not build header: %w", err) @@ -487,7 +487,12 @@ func (b *Builder) buildPayload(buildCtx *blockBuildContext) (*cluster.Payload, e // buildHeader constructs the header for the cluster block being built. // It invokes the HotStuff setter to set fields related to HotStuff (QC, etc.). // No errors are expected during normal operation. -func (b *Builder) buildHeader(ctx *blockBuildContext, payload *cluster.Payload, setter func(header *flow.Header) error) (*flow.Header, error) { +func (b *Builder) buildHeader( + ctx *blockBuildContext, + payload *cluster.Payload, + setter func(header *flow.Header) error, + sign func(*flow.Header) error, +) (*flow.Header, error) { header := &flow.Header{ ChainID: ctx.parent.ChainID, @@ -505,6 +510,10 @@ func (b *Builder) buildHeader(ctx *blockBuildContext, payload *cluster.Payload, if err != nil { return nil, fmt.Errorf("could not set fields to header: %w", err) } + err = sign(header) + if err != nil { + return nil, fmt.Errorf("could not sign proposal: %w", err) + } return header, nil } diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index fd2bf1826d9..2eaefdd1344 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -35,6 +35,7 @@ import ( ) var noopSetter = func(*flow.Header) error { return nil } +var noopSigner = func(*flow.Header) error { return nil } type BuilderSuite struct { suite.Suite @@ -216,7 +217,7 @@ func (suite *BuilderSuite) TestBuildOn_NonExistentParent() { // use a non-existent parent ID parentID := unittest.IdentifierFixture() - _, err := suite.builder.BuildOn(parentID, noopSetter) + _, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Assert().Error(err) } @@ -228,7 +229,7 @@ func (suite *BuilderSuite) TestBuildOn_Success() { return nil } - header, err := suite.builder.BuildOn(suite.genesis.ID(), setter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), setter, noopSigner) suite.Require().NoError(err) // setter should have been run @@ -263,7 +264,7 @@ func (suite *BuilderSuite) TestBuildOn_WithUnknownReferenceBlock() { unknownReferenceTx.ReferenceBlockID = unittest.IdentifierFixture() suite.pool.Add(&unknownReferenceTx) - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // should be able to retrieve built block from storage @@ -303,7 +304,7 @@ func (suite *BuilderSuite) TestBuildOn_WithUnfinalizedReferenceBlock() { unfinalizedReferenceTx.ReferenceBlockID = unfinalizedReferenceBlock.ID() suite.pool.Add(&unfinalizedReferenceTx) - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // should be able to retrieve built block from storage @@ -350,7 +351,7 @@ func (suite *BuilderSuite) TestBuildOn_WithOrphanedReferenceBlock() { orphanedReferenceTx.ReferenceBlockID = orphan.ID() suite.pool.Add(&orphanedReferenceTx) - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // should be able to retrieve built block from storage @@ -393,7 +394,7 @@ func (suite *BuilderSuite) TestBuildOn_WithForks() { suite.InsertBlock(block2) // build on top of fork 1 - header, err := suite.builder.BuildOn(block1.ID(), noopSetter) + header, err := suite.builder.BuildOn(block1.ID(), noopSetter, noopSigner) require.NoError(t, err) // should be able to retrieve built block from storage @@ -436,7 +437,7 @@ func (suite *BuilderSuite) TestBuildOn_ConflictingFinalizedBlock() { suite.FinalizeBlock(finalizedBlock) // build on the un-finalized block - header, err := suite.builder.BuildOn(unFinalizedBlock.ID(), noopSetter) + header, err := suite.builder.BuildOn(unFinalizedBlock.ID(), noopSetter, noopSigner) require.NoError(t, err) // retrieve the built block from storage @@ -485,7 +486,7 @@ func (suite *BuilderSuite) TestBuildOn_ConflictingInvalidatedForks() { suite.FinalizeBlock(finalizedBlock) // build on the finalized block - header, err := suite.builder.BuildOn(finalizedBlock.ID(), noopSetter) + header, err := suite.builder.BuildOn(finalizedBlock.ID(), noopSetter, noopSigner) require.NoError(t, err) // retrieve the built block from storage @@ -569,7 +570,7 @@ func (suite *BuilderSuite) TestBuildOn_LargeHistory() { t.Log("conflicting: ", len(invalidatedTxIds)) // build on the head block - header, err := suite.builder.BuildOn(head.ID(), noopSetter) + header, err := suite.builder.BuildOn(head.ID(), noopSetter, noopSigner) require.NoError(t, err) // retrieve the built block from storage @@ -588,7 +589,7 @@ func (suite *BuilderSuite) TestBuildOn_MaxCollectionSize() { suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(1)) // build a block - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // retrieve the built block from storage @@ -606,7 +607,7 @@ func (suite *BuilderSuite) TestBuildOn_MaxCollectionByteSize() { suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionByteSize(400)) // build a block - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // retrieve the built block from storage @@ -624,7 +625,7 @@ func (suite *BuilderSuite) TestBuildOn_MaxCollectionTotalGas() { suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionTotalGas(20000)) // build a block - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // retrieve the built block from storage @@ -681,7 +682,7 @@ func (suite *BuilderSuite) TestBuildOn_ExpiredTransaction() { suite.T().Log("tx2: ", tx2.ID()) // build a block - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) // retrieve the built block from storage @@ -703,7 +704,7 @@ func (suite *BuilderSuite) TestBuildOn_EmptyMempool() { suite.pool = herocache.NewTransactions(1000, unittest.Logger(), metrics.NewNoopCollector()) suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) - header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) + header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter, noopSigner) suite.Require().NoError(err) var built model.Block @@ -746,7 +747,7 @@ func (suite *BuilderSuite) TestBuildOn_NoRateLimiting() { // since we have no rate limiting we should fill all collections and in 10 blocks parentID := suite.genesis.ID() for i := 0; i < 10; i++ { - header, err := suite.builder.BuildOn(parentID, noopSetter) + header, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Require().NoError(err) parentID = header.ID() @@ -793,7 +794,7 @@ func (suite *BuilderSuite) TestBuildOn_RateLimitNonPayer() { // since rate limiting does not apply to non-payer keys, we should fill all collections in 10 blocks parentID := suite.genesis.ID() for i := 0; i < 10; i++ { - header, err := suite.builder.BuildOn(parentID, noopSetter) + header, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Require().NoError(err) parentID = header.ID() @@ -831,7 +832,7 @@ func (suite *BuilderSuite) TestBuildOn_HighRateLimit() { // rate-limiting should be applied, resulting in half-full collections (5/10) parentID := suite.genesis.ID() for i := 0; i < 10; i++ { - header, err := suite.builder.BuildOn(parentID, noopSetter) + header, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Require().NoError(err) parentID = header.ID() @@ -870,7 +871,7 @@ func (suite *BuilderSuite) TestBuildOn_LowRateLimit() { // having one transaction and empty collections otherwise parentID := suite.genesis.ID() for i := 0; i < 10; i++ { - header, err := suite.builder.BuildOn(parentID, noopSetter) + header, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Require().NoError(err) parentID = header.ID() @@ -911,7 +912,7 @@ func (suite *BuilderSuite) TestBuildOn_UnlimitedPayer() { // rate-limiting should not be applied, since the payer is marked as unlimited parentID := suite.genesis.ID() for i := 0; i < 10; i++ { - header, err := suite.builder.BuildOn(parentID, noopSetter) + header, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Require().NoError(err) parentID = header.ID() @@ -952,7 +953,7 @@ func (suite *BuilderSuite) TestBuildOn_RateLimitDryRun() { // rate-limiting should not be applied, since dry-run setting is enabled parentID := suite.genesis.ID() for i := 0; i < 10; i++ { - header, err := suite.builder.BuildOn(parentID, noopSetter) + header, err := suite.builder.BuildOn(parentID, noopSetter, noopSigner) suite.Require().NoError(err) parentID = header.ID() @@ -1059,7 +1060,7 @@ func benchmarkBuildOn(b *testing.B, size int) { b.StartTimer() for n := 0; n < b.N; n++ { - _, err := suite.builder.BuildOn(final.ID(), noopSetter) + _, err := suite.builder.BuildOn(final.ID(), noopSetter, noopSigner) assert.NoError(b, err) } } diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index f7220ffc50a..3bfb11b70da 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -630,7 +630,8 @@ func (b *Builder) createProposal(parentID flow.Identifier, PayloadHash: flow.ZeroID, } - // apply the custom fields setter of the consensus algorithm + // apply the custom fields setter of the consensus algorithm, we must do this before applying service events + // since we need to know the correct view of the block. err = setter(header) if err != nil { return nil, fmt.Errorf("could not apply setter: %w", err) diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index 38b62053521..32c24654889 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -66,6 +66,7 @@ type BuilderSuite struct { db *badger.DB sentinel uint64 setter func(*flow.Header) error + sign func(*flow.Header) error // mocked dependencies state *protocol.ParticipantState @@ -266,6 +267,9 @@ func (bs *BuilderSuite) SetupTest() { header.View = 1337 return nil } + bs.sign = func(_ *flow.Header) error { + return nil + } bs.state = &protocol.ParticipantState{} bs.state.On("Extend", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { @@ -455,7 +459,7 @@ func (bs *BuilderSuite) TearDownTest() { func (bs *BuilderSuite) TestPayloadEmptyValid() { // we should build an empty block with default setup - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") bs.Assert().Empty(bs.assembled.Seals, "should have no seals in payload with empty mempool") @@ -465,7 +469,7 @@ func (bs *BuilderSuite) TestPayloadGuaranteeValid() { // add sixteen guarantees to the pool bs.pendingGuarantees = unittest.CollectionGuaranteesFixture(16, unittest.WithCollRef(bs.finalID)) - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().ElementsMatch(bs.pendingGuarantees, bs.assembled.Guarantees, "should have guarantees from mempool in payload") } @@ -488,7 +492,7 @@ func (bs *BuilderSuite) TestPayloadGuaranteeDuplicate() { // add sixteen guarantees to the pool bs.pendingGuarantees = append(valid, duplicated...) - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().ElementsMatch(valid, bs.assembled.Guarantees, "should have valid guarantees from mempool in payload") } @@ -503,7 +507,7 @@ func (bs *BuilderSuite) TestPayloadGuaranteeReferenceUnknown() { // add all guarantees to the pool bs.pendingGuarantees = append(valid, unknown...) - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().ElementsMatch(valid, bs.assembled.Guarantees, "should have valid from mempool in payload") } @@ -521,7 +525,7 @@ func (bs *BuilderSuite) TestPayloadGuaranteeReferenceExpired() { // add all guarantees to the pool bs.pendingGuarantees = append(valid, expired...) - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().ElementsMatch(valid, bs.assembled.Guarantees, "should have valid from mempool in payload") } @@ -547,7 +551,7 @@ func (bs *BuilderSuite) TestPayloadSeals_AllValid() { // Populate seals mempool with valid chain of seals for blocks [F0], ..., [A2] bs.pendingSeals = bs.irsMap - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") bs.Assert().ElementsMatch(bs.chain, bs.assembled.Seals, "should have included valid chain of seals") @@ -562,7 +566,7 @@ func (bs *BuilderSuite) TestPayloadSeals_Limit() { limit := uint(2) bs.build.cfg.maxSealCount = limit - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") bs.Assert().Equal(bs.chain[:limit], bs.assembled.Seals, "should have excluded seals above maxSealCount") @@ -588,7 +592,7 @@ func (bs *BuilderSuite) TestPayloadSeals_OnlyFork() { } bs.pendingSeals = bs.irsMap - _, err := bs.build.BuildOn(forkHead.ID(), bs.setter) + _, err := bs.build.BuildOn(forkHead.ID(), bs.setter, bs.sign) bs.Require().NoError(err) // expected seals: [F0] <- ... <- [final] <- [B0] <- ... <- [B5] @@ -669,7 +673,7 @@ func (bs *BuilderSuite) TestPayloadSeals_EnforceGap() { bs.T().Run("Build on top of B4 and check that no seals are included", func(t *testing.T) { bs.sealDB.On("HighestInFork", b4.ID()).Return(b0seal, nil) - _, err := bs.build.BuildOn(b4.ID(), bs.setter) + _, err := bs.build.BuildOn(b4.ID(), bs.setter, bs.sign) require.NoError(t, err) bs.recPool.AssertExpectations(t) require.Empty(t, bs.assembled.Seals, "should not include any seals") @@ -680,7 +684,7 @@ func (bs *BuilderSuite) TestPayloadSeals_EnforceGap() { bs.storeBlock(b5) bs.sealDB.On("HighestInFork", b5.ID()).Return(b0seal, nil) - _, err := bs.build.BuildOn(b5.ID(), bs.setter) + _, err := bs.build.BuildOn(b5.ID(), bs.setter, bs.sign) require.NoError(t, err) bs.recPool.AssertExpectations(t) require.Equal(t, 1, len(bs.assembled.Seals), "only seal for B1 expected") @@ -709,7 +713,7 @@ func (bs *BuilderSuite) TestPayloadSeals_Duplicate() { // seals for all blocks [F0], ..., [A3] are still in the mempool: bs.pendingSeals = bs.irsMap - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().Equal(bs.chain[n:], bs.assembled.Seals, "should have rejected duplicate seals") } @@ -732,7 +736,7 @@ func (bs *BuilderSuite) TestPayloadSeals_MissingNextSeal() { delete(bs.irsMap, firstSeal.ID()) bs.pendingSeals = bs.irsMap - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") bs.Assert().Empty(bs.assembled.Seals, "should not have included any seals from cutoff chain") @@ -756,7 +760,7 @@ func (bs *BuilderSuite) TestPayloadSeals_MissingInterimSeal() { delete(bs.irsMap, seal.ID()) bs.pendingSeals = bs.irsMap - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().Empty(bs.assembled.Guarantees, "should have no guarantees in payload with empty mempool") bs.Assert().ElementsMatch(bs.chain[:3], bs.assembled.Seals, "should have included only beginning of broken chain") @@ -828,7 +832,7 @@ func (bs *BuilderSuite) TestValidatePayloadSeals_ExecutionForks() { bs.pendingSeals = make(map[flow.Identifier]*flow.IncorporatedResultSeal) storeSealForIncorporatedResult(&receiptChain2[1].ExecutionResult, blocks[2].ID(), bs.pendingSeals) - _, err := bs.build.BuildOn(blocks[4].ID(), bs.setter) + _, err := bs.build.BuildOn(blocks[4].ID(), bs.setter, bs.sign) require.NoError(t, err) require.Empty(t, bs.assembled.Seals, "should not have included seal for conflicting execution fork") }) @@ -840,7 +844,7 @@ func (bs *BuilderSuite) TestValidatePayloadSeals_ExecutionForks() { storeSealForIncorporatedResult(&receiptChain2[1].ExecutionResult, blocks[2].ID(), bs.pendingSeals) storeSealForIncorporatedResult(&receiptChain2[2].ExecutionResult, blocks[3].ID(), bs.pendingSeals) - _, err := bs.build.BuildOn(blocks[4].ID(), bs.setter) + _, err := bs.build.BuildOn(blocks[4].ID(), bs.setter, bs.sign) require.NoError(t, err) require.ElementsMatch(t, []*flow.Seal{sealResultA_1.Seal, sealResultB_1.Seal}, bs.assembled.Seals, "valid fork should have been sealed") }) @@ -879,21 +883,21 @@ func (bs *BuilderSuite) TestPayloadReceipts_TraverseExecutionTreeFromLastSealedR // building on top of X0: latest finalized block in fork is [lastSeal]; expect search to start with sealed result bs.sealDB.On("HighestInFork", x0.ID()).Return(bs.lastSeal, nil) bs.recPool.On("ReachableReceipts", bs.lastSeal.ResultID, mock.Anything, mock.Anything).Return([]*flow.ExecutionReceipt{}, nil).Once() - _, err := bs.build.BuildOn(x0.ID(), bs.setter) + _, err := bs.build.BuildOn(x0.ID(), bs.setter, bs.sign) bs.Require().NoError(err) bs.recPool.AssertExpectations(bs.T()) // building on top of X1: latest finalized block in fork is [F4]; expect search to start with sealed result bs.sealDB.On("HighestInFork", x1.ID()).Return(f4Seal, nil) bs.recPool.On("ReachableReceipts", f4Seal.ResultID, mock.Anything, mock.Anything).Return([]*flow.ExecutionReceipt{}, nil).Once() - _, err = bs.build.BuildOn(x1.ID(), bs.setter) + _, err = bs.build.BuildOn(x1.ID(), bs.setter, bs.sign) bs.Require().NoError(err) bs.recPool.AssertExpectations(bs.T()) // building on top of A3 (with ID bs.parentID): latest finalized block in fork is [F4]; expect search to start with sealed result bs.sealDB.On("HighestInFork", bs.parentID).Return(f2eal, nil) bs.recPool.On("ReachableReceipts", f2eal.ResultID, mock.Anything, mock.Anything).Return([]*flow.ExecutionReceipt{}, nil).Once() - _, err = bs.build.BuildOn(bs.parentID, bs.setter) + _, err = bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.recPool.AssertExpectations(bs.T()) } @@ -951,7 +955,7 @@ func (bs *BuilderSuite) TestPayloadReceipts_IncludeOnlyReceiptsForCurrentFork() }).Return([]*flow.ExecutionReceipt{}, nil).Once() bs.build.recPool = bs.recPool - _, err := bs.build.BuildOn(b5.ID(), bs.setter) + _, err := bs.build.BuildOn(b5.ID(), bs.setter, bs.sign) bs.Require().NoError(err) bs.recPool.AssertExpectations(bs.T()) } @@ -988,7 +992,7 @@ func (bs *BuilderSuite) TestPayloadReceipts_SkipDuplicatedReceipts() { }).Return([]*flow.ExecutionReceipt{}, nil).Once() bs.build.recPool = bs.recPool - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.recPool.AssertExpectations(bs.T()) } @@ -1018,7 +1022,7 @@ func (bs *BuilderSuite) TestPayloadReceipts_SkipReceiptsForSealedBlock() { }).Return([]*flow.ExecutionReceipt{}, nil).Once() bs.build.recPool = bs.recPool - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.recPool.AssertExpectations(bs.T()) } @@ -1046,7 +1050,7 @@ func (bs *BuilderSuite) TestPayloadReceipts_BlockLimit() { bs.build.cfg.maxReceiptCount = limit // ensure that only 3 of the 5 receipts were included - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().ElementsMatch(metas[:limit], bs.assembled.Receipts, "should have excluded receipts above maxReceiptCount") bs.Assert().ElementsMatch(expectedResults[:limit], bs.assembled.Results, "should have excluded results above maxReceiptCount") @@ -1069,7 +1073,7 @@ func (bs *BuilderSuite) TestPayloadReceipts_AsProvidedByReceiptForest() { bs.recPool.On("ReachableReceipts", mock.Anything, mock.Anything, mock.Anything).Return(expectedReceipts, nil).Once() bs.build.recPool = bs.recPool - _, err := bs.build.BuildOn(bs.parentID, bs.setter) + _, err := bs.build.BuildOn(bs.parentID, bs.setter, bs.sign) bs.Require().NoError(err) bs.Assert().ElementsMatch(expectedMetas, bs.assembled.Receipts, "should include receipts as returned by ExecutionTree") bs.Assert().ElementsMatch(expectedResults, bs.assembled.Results, "should include results as returned by ExecutionTree") @@ -1125,7 +1129,7 @@ func (bs *BuilderSuite) TestIntegration_PayloadReceiptNoParentResult() { _, _ = bs.build.recPool.AddReceipt(receiptSABC[1], blockSABC[1].Header) _, _ = bs.build.recPool.AddReceipt(receiptSABC[3], blockSABC[3].Header) - _, err := bs.build.BuildOn(blockSABC[3].ID(), bs.setter) + _, err := bs.build.BuildOn(blockSABC[3].ID(), bs.setter, bs.sign) bs.Require().NoError(err) expectedReceipts := flow.ExecutionReceiptMetaList{receiptSABC[1].Meta()} expectedResults := flow.ExecutionResultList{&receiptSABC[1].ExecutionResult} @@ -1188,7 +1192,7 @@ func (bs *BuilderSuite) TestIntegration_ExtendDifferentExecutionPathsOnSameFork( _, _ = bs.build.recPool.AddReceipt(recB1, B.Header) _, _ = bs.build.recPool.AddReceipt(recB2, B.Header) - _, err := bs.build.BuildOn(B.ID(), bs.setter) + _, err := bs.build.BuildOn(B.ID(), bs.setter, bs.sign) bs.Require().NoError(err) expectedReceipts := flow.ExecutionReceiptMetaList{recB1.Meta(), recB2.Meta()} expectedResults := flow.ExecutionResultList{&recB1.ExecutionResult, &recB2.ExecutionResult} @@ -1272,7 +1276,7 @@ func (bs *BuilderSuite) TestIntegration_ExtendDifferentExecutionPathsOnDifferent _, err = bs.build.recPool.AddReceipt(recB2, B.Header) bs.Require().NoError(err) - _, err = bs.build.BuildOn(B.ID(), bs.setter) + _, err = bs.build.BuildOn(B.ID(), bs.setter, bs.sign) bs.Require().NoError(err) expectedReceipts := []*flow.ExecutionReceiptMeta{recA2.Meta(), recB1.Meta(), recB2.Meta()} expectedResults := []*flow.ExecutionResult{&recA2.ExecutionResult, &recB1.ExecutionResult, &recB2.ExecutionResult} @@ -1317,7 +1321,7 @@ func (bs *BuilderSuite) TestIntegration_DuplicateReceipts() { } } - _, err := bs.build.BuildOn(B.ID(), bs.setter) + _, err := bs.build.BuildOn(B.ID(), bs.setter, bs.sign) bs.Require().NoError(err) expectedReceipts := []*flow.ExecutionReceiptMeta{} expectedResults := []*flow.ExecutionResult{} @@ -1357,7 +1361,7 @@ func (bs *BuilderSuite) TestIntegration_ResultAlreadyIncorporated() { _, err := bs.build.recPool.AddReceipt(recP_B, bs.blocks[recP_B.ExecutionResult.BlockID].Header) bs.NoError(err) - _, err = bs.build.BuildOn(A.ID(), bs.setter) + _, err = bs.build.BuildOn(A.ID(), bs.setter, bs.sign) bs.Require().NoError(err) expectedReceipts := []*flow.ExecutionReceiptMeta{recP_B.Meta()} expectedResults := []*flow.ExecutionResult{} @@ -1470,7 +1474,7 @@ func (bs *BuilderSuite) TestIntegration_RepopulateExecutionTreeAtStartup() { _, _ = bs.build.recPool.AddReceipt(recB2, B.Header) _, _ = bs.build.recPool.AddReceipt(recC, C.Header) - _, err = bs.build.BuildOn(C.ID(), bs.setter) + _, err = bs.build.BuildOn(C.ID(), bs.setter, bs.sign) bs.Require().NoError(err) expectedReceipts := flow.ExecutionReceiptMetaList{recB1.Meta(), recB2.Meta(), recC.Meta()} expectedResults := flow.ExecutionResultList{&recB1.ExecutionResult, &recB2.ExecutionResult, &recC.ExecutionResult} From f6e0e8b0d7b5cd7abe3da757bd35ffde705d73bf Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 13 Oct 2023 15:24:26 +0300 Subject: [PATCH 27/87] Fixed some broken tests --- engine/common/follower/integration_test.go | 4 ++++ .../assigner/blockconsumer/consumer_test.go | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/engine/common/follower/integration_test.go b/engine/common/follower/integration_test.go index f06f2a1002a..da81d81c854 100644 --- a/engine/common/follower/integration_test.go +++ b/engine/common/follower/integration_test.go @@ -86,6 +86,9 @@ func TestFollowerHappyPath(t *testing.T) { require.NoError(t, err) rootQC, err := rootSnapshot.QuorumCertificate() require.NoError(t, err) + rootProtocolState, err := rootSnapshot.ProtocolState() + require.NoError(t, err) + rootProtocolStateID := rootProtocolState.Entry().ID() // Hack EECC. // Since root snapshot is created with 1000 views for first epoch, we will forcefully enter EECC to avoid errors @@ -163,6 +166,7 @@ func TestFollowerHappyPath(t *testing.T) { // ensure sequential block views - that way we can easily know which block will be finalized after the test for i, block := range flowBlocks { block.Header.View = block.Header.Height + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) if i > 0 { block.Header.ParentView = flowBlocks[i-1].Header.View block.Header.ParentID = flowBlocks[i-1].Header.ID() diff --git a/engine/verification/assigner/blockconsumer/consumer_test.go b/engine/verification/assigner/blockconsumer/consumer_test.go index 67ea6773194..a320fbc0e14 100644 --- a/engine/verification/assigner/blockconsumer/consumer_test.go +++ b/engine/verification/assigner/blockconsumer/consumer_test.go @@ -146,11 +146,21 @@ func withConsumer( // blocks (i.e., containing guarantees), and Cs are container blocks for their preceding reference block, // Container blocks only contain receipts of their preceding reference blocks. But they do not // hold any guarantees. - root, err := s.State.Params().FinalizedRoot() + root, err := s.State.Final().Head() require.NoError(t, err) + rootProtocolState, err := s.State.Final().ProtocolState() + require.NoError(t, err) + rootProtocolStateID := rootProtocolState.Entry().ID() clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) sources := unittest.RandomSourcesFixture(110) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) + results := vertestutils.CompleteExecutionReceiptChainFixture( + t, + root, + rootProtocolStateID, + blockCount/2, + sources, + vertestutils.WithClusterCommittee(clusterCommittee), + ) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) // makes sure that we generated a block chain of requested length. require.Len(t, blocks, blockCount) From b93ae30024fbfd48619c55d54e55a3a4299bf1b5 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 13 Oct 2023 17:09:26 +0300 Subject: [PATCH 28/87] Updated protobuf files from onflow/flow. Fixed convert test --- engine/common/rpc/convert/blocks.go | 14 ++++++++------ go.mod | 2 +- go.sum | 2 ++ insecure/go.mod | 2 +- insecure/go.sum | 1 + integration/go.mod | 2 +- integration/go.sum | 1 + 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/engine/common/rpc/convert/blocks.go b/engine/common/rpc/convert/blocks.go index 2e7f5689515..62f1298911f 100644 --- a/engine/common/rpc/convert/blocks.go +++ b/engine/common/rpc/convert/blocks.go @@ -35,15 +35,16 @@ func BlockToMessage(h *flow.Block, signerIDs flow.IdentifierList) ( } bh := entities.Block{ - Id: id[:], + Id: IdentifierToMessage(id), Height: h.Header.Height, - ParentId: parentID[:], + ParentId: IdentifierToMessage(parentID), Timestamp: t, CollectionGuarantees: cg, BlockSeals: seals, Signatures: [][]byte{h.Header.ParentVoterSigData}, ExecutionReceiptMetaList: ExecutionResultMetaListToMessages(h.Payload.Receipts), ExecutionResultList: execResults, + ProtocolStateId: IdentifierToMessage(h.Payload.ProtocolStateID), BlockHeader: blockHeader, } @@ -147,9 +148,10 @@ func PayloadFromMessage(m *entities.Block) (*flow.Payload, error) { return nil, err } return &flow.Payload{ - Guarantees: cgs, - Seals: seals, - Receipts: receipts, - Results: results, + Guarantees: cgs, + Seals: seals, + Receipts: receipts, + Results: results, + ProtocolStateID: MessageToIdentifier(m.ProtocolStateId), }, nil } diff --git a/go.mod b/go.mod index 05ab2e0ea2e..ddb02917485 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-go-sdk v0.41.10 github.com/onflow/flow-go/crypto v0.24.9 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index a2abc8cef6b..ab1d003520a 100644 --- a/go.sum +++ b/go.sum @@ -1261,6 +1261,8 @@ github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9j github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b h1:bA+48+sA4CL2cM6g07HaudGKRP3OvUXja4AolXdq8ig= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/insecure/go.mod b/insecure/go.mod index c0f8048ecbe..f2dae87b1c5 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -186,7 +186,7 @@ require ( github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-go-sdk v0.41.10 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce // indirect + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 376c5d1543e..af641f0f9c3 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1235,6 +1235,7 @@ github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9j github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/integration/go.mod b/integration/go.mod index affeae7bc11..7da813ee8f8 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/onflow/flow-go-sdk v0.41.10 github.com/onflow/flow-go/crypto v0.24.9 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b github.com/plus3it/gorecurcopy v0.0.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.4.0 diff --git a/integration/go.sum b/integration/go.sum index 540ba104a09..0f9ca113b6d 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1375,6 +1375,7 @@ github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9j github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead h1:2j1Unqs76Z1b95Gu4C3Y28hzNUHBix7wL490e61SMSw= From 67c67997311729fe1dad81f4b50c0885ebe7513e Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Fri, 13 Oct 2023 17:00:50 +0200 Subject: [PATCH 29/87] change expected state comitment hashes --- utils/unittest/execution_state.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 9e843576195..329a40ed05c 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -24,7 +24,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "517138d362602fb11b17524a654b00d8eecdfbf56406b1636a2c58dad7c5d144" +const GenesisStateCommitmentHex = "8ac91e21ee8da3d4e421e67faf004b687cee2a10ffc79417b8b414218ebc901a" var GenesisStateCommitment flow.StateCommitment @@ -88,10 +88,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "dd8c079b196fced93e4c541a8f6c49a0ee5fda01b2653c5a03cc165ab1015423" + return "507e67071935b40c1226437480d530ee8b18a20198bcee9c616c94934aa32ad4" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "c6e7f204c774f4208e67451acfdf9783932df06e5ab29b7afc56d548a1573769" + return "74ba201c4cd99d0830369e783e9646763b64e57b8c7e88284762d609fce02192" } From 9810f16cc5c5ad721f6396fe69934cc122cdec14 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 13 Oct 2023 18:11:01 +0300 Subject: [PATCH 30/87] Fixed setup of integration tests. Updated go modules --- integration/go.sum | 1 + integration/testnet/network.go | 38 ++++++++++++++++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/integration/go.sum b/integration/go.sum index 0f9ca113b6d..53475ff7a24 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1375,6 +1375,7 @@ github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9j github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b h1:bA+48+sA4CL2cM6g07HaudGKRP3OvUXja4AolXdq8ig= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 993ee49b1dc..b7b31a9ad6d 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -1089,24 +1089,6 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl // generate root block root := run.GenerateRootBlock(chainID, parentID, height, timestamp) - // generate QC - signerData, err := run.GenerateQCParticipantData(consensusNodes, consensusNodes, dkg) - if err != nil { - return nil, err - } - votes, err := run.GenerateRootBlockVotes(root, signerData) - if err != nil { - return nil, err - } - qc, invalidVotesErr, err := run.GenerateRootQC(root, votes, signerData, signerData.Identities()) - if err != nil { - return nil, err - } - - if len(invalidVotesErr) > 0 { - return nil, fmt.Errorf("has invalid votes: %v", invalidVotesErr) - } - // generate root blocks for each collector cluster clusterRootBlocks, clusterAssignments, clusterQCs, err := setupClusterGenesisBlockQCs(networkConf.NClusters, epochCounter, stakedConfs) if err != nil { @@ -1156,6 +1138,8 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl DKGGroupKey: dkg.PubGroupKey, DKGParticipantKeys: dkg.PubKeyShares, } + root.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID( + inmem.ProtocolStateForBootstrapState(epochSetup, epochCommit).ID()))) cdcRandomSource, err := cadence.NewString(hex.EncodeToString(randomSource)) if err != nil { @@ -1200,6 +1184,24 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl return nil, fmt.Errorf("generating root seal failed: %w", err) } + // generate QC + signerData, err := run.GenerateQCParticipantData(consensusNodes, consensusNodes, dkg) + if err != nil { + return nil, err + } + votes, err := run.GenerateRootBlockVotes(root, signerData) + if err != nil { + return nil, err + } + qc, invalidVotesErr, err := run.GenerateRootQC(root, votes, signerData, signerData.Identities()) + if err != nil { + return nil, err + } + + if len(invalidVotesErr) > 0 { + return nil, fmt.Errorf("has invalid votes: %v", invalidVotesErr) + } + snapshot, err := inmem.SnapshotFromBootstrapStateWithParams(root, result, seal, qc, flow.DefaultProtocolVersion, networkConf.EpochCommitSafetyThreshold) if err != nil { return nil, fmt.Errorf("could not create bootstrap state snapshot: %w", err) From 652d6dc38a45f82ea3929faff6acff0a9be987de Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 13 Oct 2023 20:45:58 +0300 Subject: [PATCH 31/87] Fixed broken execution tests --- engine/execution/execution_test.go | 57 ++++++++++++++++++------------ 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/engine/execution/execution_test.go b/engine/execution/execution_test.go index f52796f85e5..675b7f79d89 100644 --- a/engine/execution/execution_test.go +++ b/engine/execution/execution_test.go @@ -71,7 +71,7 @@ func TestExecutionFlow(t *testing.T) { }, 1*time.Second, "could not start execution node on time") defer exeNode.Done(cancel) - genesis, err := exeNode.State.AtHeight(0).Head() + genesis, err := exeNode.Blocks.ByHeight(0) require.NoError(t, err) tx1 := flow.TransactionBody{ @@ -101,7 +101,7 @@ func TestExecutionFlow(t *testing.T) { clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs()) // signed by the only collector - block := unittest.BlockWithParentAndProposerFixture(t, genesis, conID.NodeID) + block := unittest.BlockWithParentAndProposerFixture(t, genesis.Header, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -124,12 +124,14 @@ func TestExecutionFlow(t *testing.T) { ReferenceBlockID: genesis.ID(), }, }, + ProtocolStateID: genesis.Payload.ProtocolStateID, }) child := unittest.BlockWithParentAndProposerFixture(t, block.Header, conID.NodeID) // the default signer indices is 2 bytes, but in this test cases // we need 1 byte child.Header.ParentVoterIndices = voterIndices + child.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(block.Payload.ProtocolStateID))) log.Info().Msgf("child block ID: %v, indices: %x", child.Header.ID(), child.Header.ParentVoterIndices) @@ -245,8 +247,16 @@ func TestExecutionFlow(t *testing.T) { consensusEngine.AssertExpectations(t) } -func deployContractBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Header, ref *flow.Header) ( - *flow.TransactionBody, *flow.Collection, flow.Block, *messages.BlockProposal, uint64) { +func deployContractBlock( + t *testing.T, + conID *flow.Identity, + colID *flow.Identity, + chain flow.Chain, + seq uint64, + parent *flow.Block, + ref *flow.Header, +) ( + *flow.TransactionBody, *flow.Collection, *flow.Block, *messages.BlockProposal, uint64) { // make tx tx := execTestutil.DeployCounterContractTransaction(chain.ServiceAddress(), chain) err := execTestutil.SignTransactionAsServiceAccount(tx, seq, chain) @@ -262,7 +272,7 @@ func deployContractBlock(t *testing.T, conID *flow.Identity, colID *flow.Identit clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs()) // make block - block := unittest.BlockWithParentAndProposerFixture(t, parent, conID.NodeID) + block := unittest.BlockWithParentAndProposerFixture(t, parent.Header, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -276,16 +286,17 @@ func deployContractBlock(t *testing.T, conID *flow.Identity, colID *flow.Identit ReferenceBlockID: ref.ID(), }, }, + ProtocolStateID: parent.Payload.ProtocolStateID, }) // make proposal proposal := unittest.ProposalFromBlock(&block) - return tx, col, block, proposal, seq + 1 + return tx, col, &block, proposal, seq + 1 } -func makePanicBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Header, ref *flow.Header) ( - *flow.TransactionBody, *flow.Collection, flow.Block, *messages.BlockProposal, uint64) { +func makePanicBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Block, ref *flow.Header) ( + *flow.TransactionBody, *flow.Collection, *flow.Block, *messages.BlockProposal, uint64) { // make tx tx := execTestutil.CreateCounterPanicTransaction(chain.ServiceAddress(), chain.ServiceAddress()) err := execTestutil.SignTransactionAsServiceAccount(tx, seq, chain) @@ -296,7 +307,7 @@ func makePanicBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, ch clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs()) // make block - block := unittest.BlockWithParentAndProposerFixture(t, parent, conID.NodeID) + block := unittest.BlockWithParentAndProposerFixture(t, parent.Header, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -310,15 +321,16 @@ func makePanicBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, ch Guarantees: []*flow.CollectionGuarantee{ {CollectionID: col.ID(), SignerIndices: signerIndices, ChainID: clusterChainID, ReferenceBlockID: ref.ID()}, }, + ProtocolStateID: parent.Payload.ProtocolStateID, }) proposal := unittest.ProposalFromBlock(&block) - return tx, col, block, proposal, seq + 1 + return tx, col, &block, proposal, seq + 1 } -func makeSuccessBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Header, ref *flow.Header) ( - *flow.TransactionBody, *flow.Collection, flow.Block, *messages.BlockProposal, uint64) { +func makeSuccessBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Block, ref *flow.Header) ( + *flow.TransactionBody, *flow.Collection, *flow.Block, *messages.BlockProposal, uint64) { tx := execTestutil.AddToCounterTransaction(chain.ServiceAddress(), chain.ServiceAddress()) err := execTestutil.SignTransactionAsServiceAccount(tx, seq, chain) require.NoError(t, err) @@ -329,7 +341,7 @@ func makeSuccessBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs()) col := &flow.Collection{Transactions: []*flow.TransactionBody{tx}} - block := unittest.BlockWithParentAndProposerFixture(t, parent, conID.NodeID) + block := unittest.BlockWithParentAndProposerFixture(t, parent.Header, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -338,11 +350,12 @@ func makeSuccessBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, Guarantees: []*flow.CollectionGuarantee{ {CollectionID: col.ID(), SignerIndices: signerIndices, ChainID: clusterChainID, ReferenceBlockID: ref.ID()}, }, + ProtocolStateID: parent.Payload.ProtocolStateID, }) proposal := unittest.ProposalFromBlock(&block) - return tx, col, block, proposal, seq + 1 + return tx, col, &block, proposal, seq + 1 } // Test a successful tx should change the statecommitment, @@ -381,7 +394,7 @@ func TestFailedTxWillNotChangeStateCommitment(t *testing.T) { }, 1*time.Second, "could not start execution node on time") defer exe1Node.Done(cancel) - genesis, err := exe1Node.State.AtHeight(0).Head() + genesis, err := exe1Node.Blocks.ByHeight(0) require.NoError(t, err) seq := uint64(0) @@ -390,14 +403,14 @@ func TestFailedTxWillNotChangeStateCommitment(t *testing.T) { // transaction that will change state and succeed, used to test that state commitment changes // genesis <- block1 [tx1] <- block2 [tx2] <- block3 [tx3] <- child - _, col1, block1, proposal1, seq := deployContractBlock(t, conID, colID, chain, seq, genesis, genesis) + _, col1, block1, proposal1, seq := deployContractBlock(t, conID, colID, chain, seq, genesis, genesis.Header) // we don't set the proper sequence number of this one - _, col2, block2, proposal2, _ := makePanicBlock(t, conID, colID, chain, uint64(0), block1.Header, genesis) + _, col2, block2, proposal2, _ := makePanicBlock(t, conID, colID, chain, uint64(0), block1, genesis.Header) - _, col3, block3, proposal3, seq := makeSuccessBlock(t, conID, colID, chain, seq, block2.Header, genesis) + _, col3, block3, proposal3, seq := makeSuccessBlock(t, conID, colID, chain, seq, block2, genesis.Header) - _, _, _, proposal4, _ := makeSuccessBlock(t, conID, colID, chain, seq, block3.Header, genesis) + _, _, _, proposal4, _ := makeSuccessBlock(t, conID, colID, chain, seq, block3, genesis.Header) // seq++ // setup mocks and assertions @@ -540,14 +553,14 @@ func TestBroadcastToMultipleVerificationNodes(t *testing.T) { verification2Node := testutil.GenericNodeFromParticipants(t, hub, ver2ID, identities, chainID) defer verification2Node.Done() - genesis, err := exeNode.State.AtHeight(0).Head() + genesis, err := exeNode.Blocks.ByHeight(0) require.NoError(t, err) - block := unittest.BlockWithParentAndProposerFixture(t, genesis, conID.NodeID) + block := unittest.BlockWithParentAndProposerFixture(t, genesis.Header, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices([]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) block.Header.ParentVoterIndices = voterIndices - block.SetPayload(flow.Payload{}) + block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(genesis.Payload.ProtocolStateID))) proposal := unittest.ProposalFromBlock(&block) child := unittest.BlockWithParentAndProposerFixture(t, block.Header, conID.NodeID) From 444459fb3a75fbef1b21d525c8d17f12ff15b56b Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 24 Oct 2023 11:06:28 +0300 Subject: [PATCH 32/87] Fixed broken tests --- state/protocol/badger/mutator_test.go | 4 ++-- state/protocol/protocol_state/updater_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 94371732156..b0fbb9368b4 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -1456,7 +1456,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { }) t.Run("participants not ordered (EECC)", func(t *testing.T) { - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { @@ -1465,7 +1465,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { require.NoError(t, err) }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index 5204fca5881..3a77ba40cad 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -298,7 +298,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("participants not sorted", func() { - updater := NewUpdater(s.candidate, s.parentProtocolState) + updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 var err error @@ -313,7 +313,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { conflictingIdentity := s.parentProtocolState.ProtocolStateEntry.CurrentEpoch.ActiveIdentities[0] conflictingIdentity.Dynamic.Ejected = true - updater := NewUpdater(s.candidate, s.parentProtocolState) + updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 // using same identities as in previous epoch should result in an error since From 084669facfeb9abea89a272d68fe005c8e9864a2 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 24 Oct 2023 11:09:24 +0300 Subject: [PATCH 33/87] Changed order of InvalidStateTransitionAttempted in updater --- state/protocol/protocol_state/updater.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index cd34c84b51e..a337c446425 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -61,15 +61,15 @@ func (u *Updater) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.I // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is not a valid state transition for the current protocol state func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { + if u.state.InvalidStateTransitionAttempted { + return nil // won't process new events if we are in epoch fallback mode. + } if epochSetup.Counter != u.parentState.CurrentEpochSetup.Counter+1 { return protocol.NewInvalidServiceEventErrorf("invalid epoch setup counter, expecting %d got %d", u.parentState.CurrentEpochSetup.Counter+1, epochSetup.Counter) } if u.state.NextEpoch != nil { return protocol.NewInvalidServiceEventErrorf("repeated setup for epoch %d", epochSetup.Counter) } - if u.state.InvalidStateTransitionAttempted { - return nil // won't process new events if we are in epoch fallback mode. - } // When observing setup event for subsequent epoch, construct the EpochStateContainer for `ProtocolStateEntry.NextEpoch`. // Context: @@ -144,6 +144,9 @@ func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` - an invalid service event with respect to the protocol state has been supplied. func (u *Updater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { + if u.state.InvalidStateTransitionAttempted { + return nil // won't process new events if we are going to enter epoch fallback mode + } if epochCommit.Counter != u.parentState.CurrentEpochSetup.Counter+1 { return protocol.NewInvalidServiceEventErrorf("invalid epoch commit counter, expecting %d got %d", u.parentState.CurrentEpochSetup.Counter+1, epochCommit.Counter) } @@ -153,9 +156,6 @@ func (u *Updater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { if u.state.NextEpoch.CommitID != flow.ZeroID { return protocol.NewInvalidServiceEventErrorf("protocol state has already a commit event") } - if u.state.InvalidStateTransitionAttempted { - return nil // won't process new events if we are going to enter epoch fallback mode - } u.state.NextEpoch.CommitID = epochCommit.ID() return nil @@ -197,19 +197,18 @@ func (u *Updater) SetInvalidStateTransitionAttempted() { // - candidate block is in the next epoch. // No errors are expected during normal operations. func (u *Updater) TransitionToNextEpoch() error { + if u.state.InvalidStateTransitionAttempted { + return fmt.Errorf("invalid state transition has been attempted, no transition is allowed") + } nextEpoch := u.state.NextEpoch // Check if there is next epoch protocol state if nextEpoch == nil { return fmt.Errorf("protocol state has not been setup yet") } - // Check if there is a commit event for next epoch if nextEpoch.CommitID == flow.ZeroID { return fmt.Errorf("protocol state has not been committed yet") } - if u.state.InvalidStateTransitionAttempted { - return fmt.Errorf("invalid state transition has been attempted, no transition is allowed") - } // Check if we are at the next epoch, only then a transition is allowed if u.view < u.parentState.NextEpochSetup.FirstView { return fmt.Errorf("protocol state transition is only allowed when enterring next epoch") From 7b5135a3426d7a7d4d8cd78961f3188aee9473b7 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 24 Oct 2023 11:12:29 +0300 Subject: [PATCH 34/87] Tidy --- go.sum | 2 -- insecure/go.sum | 3 +-- integration/go.sum | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 086940f98b7..052385d5973 100644 --- a/go.sum +++ b/go.sum @@ -1328,8 +1328,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b h1:bA+48+sA4CL2cM6g07HaudGKRP3OvUXja4AolXdq8ig= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= diff --git a/insecure/go.sum b/insecure/go.sum index eaf914c982f..f4b1467a20d 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1302,8 +1302,7 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b h1:bA+48+sA4CL2cM6g07HaudGKRP3OvUXja4AolXdq8ig= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= diff --git a/integration/go.sum b/integration/go.sum index 68629669f92..217287c2bdd 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1446,8 +1446,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b h1:bA+48+sA4CL2cM6g07HaudGKRP3OvUXja4AolXdq8ig= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231013125846-5e6814aca55b/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= From a46d6d8e266841e35f4c561205fa7c28d0fe0b41 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 24 Oct 2023 11:32:17 +0300 Subject: [PATCH 35/87] Linted and updated error handling --- cmd/consensus/main.go | 2 +- consensus/integration/nodes_test.go | 2 +- engine/access/rpc/backend/backend_test.go | 2 +- engine/collection/test/cluster_switchover_test.go | 2 +- engine/verification/utils/unittest/helper.go | 2 +- module/builder/consensus/builder.go | 6 ++++++ state/cluster/badger/mutator_test.go | 2 +- state/protocol/badger/mutator.go | 3 +++ state/protocol/badger/mutator_test.go | 2 +- state/protocol/protocol_state/mutator.go | 2 +- state/protocol/protocol_state/mutator_test.go | 2 +- state/protocol/util/testing.go | 2 +- state/protocol/validity_test.go | 2 +- 13 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 2077cabaabd..a6f9346aebb 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/onflow/flow-go/state/protocol/protocol_state" "os" "path/filepath" "time" @@ -66,6 +65,7 @@ import ( badgerState "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/blocktimer" "github.com/onflow/flow-go/state/protocol/events/gadgets" + "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/storage" bstorage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/utils/io" diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index b27cb4e3be7..ba395c7bc77 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -3,7 +3,6 @@ package integration_test import ( "context" "fmt" - "github.com/onflow/flow-go/state/protocol/protocol_state" "os" "sort" "testing" @@ -57,6 +56,7 @@ import ( "github.com/onflow/flow-go/state/protocol/blocktimer" "github.com/onflow/flow-go/state/protocol/events" "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/state/protocol/util" storage "github.com/onflow/flow-go/storage/badger" storagemock "github.com/onflow/flow-go/storage/mock" diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index ad2e3e78000..f6fbbe858bc 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -3,7 +3,6 @@ package backend import ( "context" "fmt" - realprotocol "github.com/onflow/flow-go/state/protocol" "strconv" "testing" @@ -28,6 +27,7 @@ import ( "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" + realprotocol "github.com/onflow/flow-go/state/protocol" bprotocol "github.com/onflow/flow-go/state/protocol/badger" protocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/state/protocol/util" diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index 05a4c0566ad..ab462ee83bf 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -1,7 +1,6 @@ package test import ( - "github.com/onflow/flow-go/state/protocol/protocol_state" "sync" "testing" "time" @@ -26,6 +25,7 @@ import ( bcluster "github.com/onflow/flow-go/state/cluster/badger" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 33745a08519..4bb645543f1 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -3,7 +3,6 @@ package verificationtest import ( "context" "fmt" - "github.com/onflow/flow-go/state/protocol/protocol_state" "sync" "testing" "time" @@ -36,6 +35,7 @@ import ( "github.com/onflow/flow-go/network/stub" "github.com/onflow/flow-go/state/protocol" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/utils/logging" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index 3bfb11b70da..bc88fda6c40 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -638,7 +638,13 @@ func (b *Builder) createProposal(parentID flow.Identifier, } updater, err := b.protoStateMutator.CreateUpdater(header.View, header.ParentID) + if err != nil { + return nil, fmt.Errorf("could not create protocol state updater for view %d: %w", header.View, err) + } _, err = b.protoStateMutator.ApplyServiceEvents(updater, seals) + if err != nil { + return nil, fmt.Errorf("could not apply service events as leader: %w", err) + } _, protocolStateID, _ := updater.Build() proposal := &flow.Block{ diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 61cd40a27fc..75ad3e5b60b 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -3,7 +3,6 @@ package badger import ( "context" "fmt" - "github.com/onflow/flow-go/state/protocol/protocol_state" "math" "math/rand" "os" @@ -25,6 +24,7 @@ import ( pbadger "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/events" "github.com/onflow/flow-go/state/protocol/inmem" + "github.com/onflow/flow-go/state/protocol/protocol_state" protocolutil "github.com/onflow/flow-go/state/protocol/util" storage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/storage/badger/operation" diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index bc013e811fd..8c03dec1b6d 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -509,6 +509,9 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi } protocolStateUpdater, err := m.protocolStateMutator.CreateUpdater(candidate.Header.View, parentID) + if err != nil { + return fmt.Errorf("could not create protocol state updater for view %d: %w", candidate.Header.View, err) + } // apply any state changes from service events sealed by this block dbUpdates, err := m.protocolStateMutator.ApplyServiceEvents(protocolStateUpdater, candidate.Payload.Seals) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index b0fbb9368b4..c811cbaa52c 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -5,7 +5,6 @@ package badger_test import ( "context" "errors" - "github.com/onflow/flow-go/state/protocol/protocol_state" "math/rand" "sync" "testing" @@ -32,6 +31,7 @@ import ( "github.com/onflow/flow-go/state/protocol/events" "github.com/onflow/flow-go/state/protocol/inmem" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/state/protocol/util" "github.com/onflow/flow-go/storage" stoerr "github.com/onflow/flow-go/storage" diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index fc1c3bb2ed5..7ddd872082d 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -3,9 +3,9 @@ package protocol_state import ( "errors" "fmt" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/transaction" diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 918d4581f02..5894801c3ab 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -1,12 +1,12 @@ package protocol_state import ( - "github.com/onflow/flow-go/state/protocol/mock" "testing" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/state/protocol/mock" storerr "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/transaction" storagemock "github.com/onflow/flow-go/storage/mock" diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index a742f12cd3f..73dbbd5773d 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -1,7 +1,6 @@ package util import ( - "github.com/onflow/flow-go/state/protocol/protocol_state" "testing" "github.com/dgraph-io/badger/v2" @@ -18,6 +17,7 @@ import ( pbadger "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/events" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/state/protocol/protocol_state" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/util" "github.com/onflow/flow-go/utils/unittest" diff --git a/state/protocol/validity_test.go b/state/protocol/validity_test.go index 2cc5fea8522..9eef6b29e4a 100644 --- a/state/protocol/validity_test.go +++ b/state/protocol/validity_test.go @@ -1,7 +1,6 @@ package protocol_test import ( - "github.com/onflow/flow-go/state/protocol" "testing" "github.com/stretchr/testify/require" @@ -9,6 +8,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/unittest" ) From d4d0f9a319c33e771209a0b054c7e032f538985d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 25 Oct 2023 11:48:27 +0300 Subject: [PATCH 36/87] Extracted epoch setup and commits events creation from finalize command, moved it to rootblock cmd --- cmd/bootstrap/cmd/block.go | 64 ++++++++++++++++++++++++++++++++-- cmd/bootstrap/cmd/finalize.go | 60 +++++++++++++++---------------- cmd/bootstrap/cmd/rootblock.go | 61 +++++++++++++++++++++++++++++--- cmd/bootstrap/cmd/seal.go | 44 ++--------------------- cmd/bootstrap/run/block.go | 15 ++------ cmd/bootstrap/utils/file.go | 23 +++--------- model/bootstrap/filenames.go | 1 + 7 files changed, 154 insertions(+), 114 deletions(-) diff --git a/cmd/bootstrap/cmd/block.go b/cmd/bootstrap/cmd/block.go index 0e9b3612559..6a20058c62a 100644 --- a/cmd/bootstrap/cmd/block.go +++ b/cmd/bootstrap/cmd/block.go @@ -2,24 +2,82 @@ package cmd import ( "encoding/hex" + "github.com/onflow/flow-go/model/dkg" + "github.com/onflow/flow-go/model/flow/order" + "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/state/protocol/inmem" "time" "github.com/onflow/flow-go/cmd/bootstrap/run" "github.com/onflow/flow-go/model/flow" ) -func constructRootBlock(rootChain string, rootParent string, rootHeight uint64, rootTimestamp string) *flow.Block { - +func constructRootHeader(rootChain string, rootParent string, rootHeight uint64, rootTimestamp string) *flow.Header { chainID := parseChainID(rootChain) parentID := parseParentID(rootParent) height := rootHeight timestamp := parseRootTimestamp(rootTimestamp) - block := run.GenerateRootBlock(chainID, parentID, height, timestamp) + return run.GenerateRootHeader(chainID, parentID, height, timestamp) +} +func constructRootBlock(rootHeader *flow.Header, setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.Block { + block := &flow.Block{ + Header: rootHeader, + Payload: nil, + } + block.SetPayload(flow.Payload{ + Guarantees: nil, + Seals: nil, + Receipts: nil, + Results: nil, + ProtocolStateID: inmem.ProtocolStateForBootstrapState(setup, commit).ID(), + }) return block } +func constructRootEpochEvents( + firstView uint64, + participants flow.IdentityList, + assignments flow.AssignmentList, + clusterQCs []*flow.QuorumCertificate, + dkgData dkg.DKGData) (*flow.EpochSetup, *flow.EpochCommit) { + epochSetup := &flow.EpochSetup{ + Counter: flagEpochCounter, + FirstView: firstView, + FinalView: firstView + flagNumViewsInEpoch - 1, + DKGPhase1FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase - 1, + DKGPhase2FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*2 - 1, + DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1, + Participants: participants.Sort(order.Canonical[flow.Identity]).ToSkeleton(), + Assignments: assignments, + RandomSource: GenerateRandomSeed(flow.EpochSetupRandomSourceLength), + } + + qcsWithSignerIDs := make([]*flow.QuorumCertificateWithSignerIDs, 0, len(clusterQCs)) + for i, clusterQC := range clusterQCs { + members := assignments[i] + signerIDs, err := signature.DecodeSignerIndicesToIdentifiers(members, clusterQC.SignerIndices) + if err != nil { + log.Fatal().Err(err).Msgf("could not decode signer IDs from clusterQC at index %v", i) + } + qcsWithSignerIDs = append(qcsWithSignerIDs, &flow.QuorumCertificateWithSignerIDs{ + View: clusterQC.View, + BlockID: clusterQC.BlockID, + SignerIDs: signerIDs, + SigData: clusterQC.SigData, + }) + } + + epochCommit := &flow.EpochCommit{ + Counter: flagEpochCounter, + ClusterQCs: flow.ClusterQCVoteDatasFromQCs(qcsWithSignerIDs), + DKGGroupKey: dkgData.PubGroupKey, + DKGParticipantKeys: dkgData.PubKeyShares, + } + return epochSetup, epochCommit +} + func parseChainID(chainID string) flow.ChainID { switch chainID { case "main": diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index a4680508569..ac48d9d8bab 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -38,15 +38,12 @@ var ( flagPartnerWeights string flagDKGDataPath string flagRootBlock string + flagRootEpoch string flagRootBlockVotesDir string flagRootCommit string flagProtocolVersion uint flagServiceAccountPublicKeyJSON string flagGenesisTokenSupply string - flagEpochCounter uint64 - flagNumViewsInEpoch uint64 - flagNumViewsInStakingAuction uint64 - flagNumViewsInDKGPhase uint64 flagEpochCommitSafetyThreshold uint64 ) @@ -93,21 +90,13 @@ func addFinalizeCmdFlags() { "path to a JSON file containing root block") finalizeCmd.Flags().StringVar(&flagRootBlockVotesDir, "root-block-votes-dir", "", "path to directory with votes for root block") finalizeCmd.Flags().StringVar(&flagRootCommit, "root-commit", "0000000000000000000000000000000000000000000000000000000000000000", "state commitment of root execution state") - finalizeCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "epoch counter for the epoch beginning with the root block") - finalizeCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 4000, "length of each epoch measured in views") - finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views") - finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views") finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline") finalizeCmd.Flags().UintVar(&flagProtocolVersion, "protocol-version", flow.DefaultProtocolVersion, "major software version used for the duration of this spork") cmd.MarkFlagRequired(finalizeCmd, "root-block") + cmd.MarkFlagRequired(finalizeCmd, "root-epoch") cmd.MarkFlagRequired(finalizeCmd, "root-block-votes-dir") cmd.MarkFlagRequired(finalizeCmd, "root-commit") - cmd.MarkFlagRequired(finalizeCmd, "epoch-counter") - cmd.MarkFlagRequired(finalizeCmd, "epoch-length") - cmd.MarkFlagRequired(finalizeCmd, "epoch-staking-phase-length") - cmd.MarkFlagRequired(finalizeCmd, "epoch-dkg-phase-length") - cmd.MarkFlagRequired(finalizeCmd, "epoch-commit-safety-threshold") cmd.MarkFlagRequired(finalizeCmd, "protocol-version") // optional parameters to influence various aspects of identity generation @@ -172,6 +161,11 @@ func finalize(cmd *cobra.Command, args []string) { dkgData := readDKGData() log.Info().Msg("") + log.Info().Msg("reading root epoch") + epochSetup, epochCommit := readRootEpoch() + clusterQCs := epochCommit.ClusterQCs + log.Info().Msg("") + log.Info().Msg("constructing root QC") rootQC := constructRootQC( block, @@ -182,26 +176,11 @@ func finalize(cmd *cobra.Command, args []string) { ) log.Info().Msg("") - log.Info().Msg("computing collection node clusters") - assignments, clusters, err := constructClusterAssignment(partnerNodes, internalNodes) - if err != nil { - log.Fatal().Err(err).Msg("unable to generate cluster assignment") - } - log.Info().Msg("") - - log.Info().Msg("constructing root blocks for collection node clusters") - clusterBlocks := run.GenerateRootClusterBlocks(flagEpochCounter, clusters) - log.Info().Msg("") - - log.Info().Msg("constructing root QCs for collection node clusters") - clusterQCs := constructRootQCsForClusters(clusters, internalNodes, clusterBlocks) - log.Info().Msg("") - // if no root commit is specified, bootstrap an empty execution state if flagRootCommit == "0000000000000000000000000000000000000000000000000000000000000000" { generateEmptyExecutionState( block.Header.ChainID, - assignments, + epochSetup.Assignments, clusterQCs, dkgData, participants, @@ -209,7 +188,7 @@ func finalize(cmd *cobra.Command, args []string) { } log.Info().Msg("constructing root execution result and block seal") - result, seal := constructRootResultAndSeal(flagRootCommit, block, participants, assignments, clusterQCs, dkgData) + result, seal := constructRootResultAndSeal(flagRootCommit, block, epochSetup, epochCommit) log.Info().Msg("") // construct serializable root protocol snapshot @@ -499,15 +478,32 @@ func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) []model.NodeIn // readRootBlock reads root block data from disc, this file needs to be prepared with // rootblock command func readRootBlock() *flow.Block { - rootBlock, err := utils.ReadRootBlock(flagRootBlock) + rootBlock, err := utils.ReadData[flow.Block](flagRootBlock) if err != nil { log.Fatal().Err(err).Msg("could not read root block data") } return rootBlock } +func readRootEpoch() (*flow.EpochSetup, *flow.EpochCommit) { + encodableEpoch, err := utils.ReadData[inmem.EncodableEpoch](flagRootEpoch) + if err != nil { + log.Fatal().Err(err).Msg("could not read root block data") + } + epoch := inmem.NewEpoch(*encodableEpoch) + setup, err := protocol.ToEpochSetup(epoch) + if err != nil { + log.Fatal().Err(err).Msg("could not extract setup event") + } + commit, err := protocol.ToEpochCommit(epoch) + if err != nil { + log.Fatal().Err(err).Msg("could not extract commit event") + } + return setup, commit +} + func readDKGData() dkg.DKGData { - encodableDKG, err := utils.ReadDKGData(flagDKGDataPath) + encodableDKG, err := utils.ReadData[inmem.EncodableFullDKG](flagDKGDataPath) if err != nil { log.Fatal().Err(err).Msg("could not read DKG data") } diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index 7060fdf1a4b..5d324006590 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -1,6 +1,9 @@ package cmd import ( + "github.com/onflow/flow-go/cmd/bootstrap/run" + "github.com/onflow/flow-go/model/flow/order" + "github.com/onflow/flow-go/state/protocol/inmem" "time" "github.com/spf13/cobra" @@ -11,10 +14,14 @@ import ( ) var ( - flagRootChain string - flagRootParent string - flagRootHeight uint64 - flagRootTimestamp string + flagRootChain string + flagRootParent string + flagRootHeight uint64 + flagRootTimestamp string + flagEpochCounter uint64 + flagNumViewsInEpoch uint64 + flagNumViewsInStakingAuction uint64 + flagNumViewsInDKGPhase uint64 ) // rootBlockCmd represents the rootBlock command @@ -48,6 +55,18 @@ func addRootBlockCmdFlags() { cmd.MarkFlagRequired(rootBlockCmd, "partner-dir") cmd.MarkFlagRequired(rootBlockCmd, "partner-weights") + // required parameters for generation of epoch setup and commit events + rootBlockCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "epoch counter for the epoch beginning with the root block") + rootBlockCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 4000, "length of each epoch measured in views") + rootBlockCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views") + rootBlockCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views") + + cmd.MarkFlagRequired(rootBlockCmd, "epoch-counter") + cmd.MarkFlagRequired(rootBlockCmd, "epoch-length") + cmd.MarkFlagRequired(rootBlockCmd, "epoch-staking-phase-length") + cmd.MarkFlagRequired(rootBlockCmd, "epoch-dkg-phase-length") + cmd.MarkFlagRequired(rootBlockCmd, "epoch-commit-safety-threshold") + // required parameters for generation of root block, root execution result and root block seal rootBlockCmd.Flags().StringVar(&flagRootChain, "root-chain", "local", "chain ID for the root block (can be 'main', 'test', 'sandbox', 'bench', or 'local'") rootBlockCmd.Flags().StringVar(&flagRootParent, "root-parent", "0000000000000000000000000000000000000000000000000000000000000000", "ID for the parent of the root block") @@ -92,8 +111,40 @@ func rootBlock(cmd *cobra.Command, args []string) { dkgData := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus)) log.Info().Msg("") + // create flow.IdentityList representation of participant set + participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical[flow.Identity]) + + log.Info().Msg("computing collection node clusters") + assignments, clusters, err := constructClusterAssignment(partnerNodes, internalNodes) + if err != nil { + log.Fatal().Err(err).Msg("unable to generate cluster assignment") + } + log.Info().Msg("") + + log.Info().Msg("constructing root blocks for collection node clusters") + clusterBlocks := run.GenerateRootClusterBlocks(flagEpochCounter, clusters) + log.Info().Msg("") + + log.Info().Msg("constructing root QCs for collection node clusters") + clusterQCs := constructRootQCsForClusters(clusters, internalNodes, clusterBlocks) + log.Info().Msg("") + + log.Info().Msg("constructing root header") + header := constructRootHeader(flagRootChain, flagRootParent, flagRootHeight, flagRootTimestamp) + log.Info().Msg("") + + log.Info().Msg("constructing epoch events") + epochSetup, epochCommit := constructRootEpochEvents(header.View, participants, assignments, clusterQCs, dkgData) + committedEpoch := inmem.NewCommittedEpoch(epochSetup, epochCommit) + encodableEpoch, err := inmem.FromEpoch(committedEpoch) + if err != nil { + log.Fatal().Msg("could not convert root epoch to encodable") + } + writeJSON(model.PathRootEpoch, encodableEpoch) + log.Info().Msg("") + log.Info().Msg("constructing root block") - block := constructRootBlock(flagRootChain, flagRootParent, flagRootHeight, flagRootTimestamp) + block := constructRootBlock(header, epochSetup, epochCommit) writeJSON(model.PathRootBlockData, block) log.Info().Msg("") diff --git a/cmd/bootstrap/cmd/seal.go b/cmd/bootstrap/cmd/seal.go index f5a7a5c72d6..fcec2397f71 100644 --- a/cmd/bootstrap/cmd/seal.go +++ b/cmd/bootstrap/cmd/seal.go @@ -4,19 +4,14 @@ import ( "encoding/hex" "github.com/onflow/flow-go/cmd/bootstrap/run" - "github.com/onflow/flow-go/model/dkg" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/order" - "github.com/onflow/flow-go/module/signature" ) func constructRootResultAndSeal( rootCommit string, block *flow.Block, - participants flow.IdentityList, - assignments flow.AssignmentList, - clusterQCs []*flow.QuorumCertificate, - dkgData dkg.DKGData, + epochSetup *flow.EpochSetup, + epochCommit *flow.EpochCommit, ) (*flow.ExecutionResult, *flow.Seal) { stateCommitBytes, err := hex.DecodeString(rootCommit) @@ -31,41 +26,6 @@ func constructRootResultAndSeal( Msg("root state commitment has incompatible length") } - firstView := block.Header.View - epochSetup := &flow.EpochSetup{ - Counter: flagEpochCounter, - FirstView: firstView, - FinalView: firstView + flagNumViewsInEpoch - 1, - DKGPhase1FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase - 1, - DKGPhase2FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*2 - 1, - DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1, - Participants: participants.Sort(order.Canonical[flow.Identity]).ToSkeleton(), - Assignments: assignments, - RandomSource: GenerateRandomSeed(flow.EpochSetupRandomSourceLength), - } - - qcsWithSignerIDs := make([]*flow.QuorumCertificateWithSignerIDs, 0, len(clusterQCs)) - for i, clusterQC := range clusterQCs { - members := assignments[i] - signerIDs, err := signature.DecodeSignerIndicesToIdentifiers(members, clusterQC.SignerIndices) - if err != nil { - log.Fatal().Err(err).Msgf("could not decode signer IDs from clusterQC at index %v", i) - } - qcsWithSignerIDs = append(qcsWithSignerIDs, &flow.QuorumCertificateWithSignerIDs{ - View: clusterQC.View, - BlockID: clusterQC.BlockID, - SignerIDs: signerIDs, - SigData: clusterQC.SigData, - }) - } - - epochCommit := &flow.EpochCommit{ - Counter: flagEpochCounter, - ClusterQCs: flow.ClusterQCVoteDatasFromQCs(qcsWithSignerIDs), - DKGGroupKey: dkgData.PubGroupKey, - DKGParticipantKeys: dkgData.PubKeyShares, - } - result := run.GenerateRootResult(block, stateCommit, epochSetup, epochCommit) seal, err := run.GenerateRootSeal(result) if err != nil { diff --git a/cmd/bootstrap/run/block.go b/cmd/bootstrap/run/block.go index d5a4a10a38d..cbd58b32d97 100644 --- a/cmd/bootstrap/run/block.go +++ b/cmd/bootstrap/run/block.go @@ -6,17 +6,11 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func GenerateRootBlock(chainID flow.ChainID, parentID flow.Identifier, height uint64, timestamp time.Time) *flow.Block { - - payload := &flow.Payload{ - Guarantees: nil, - Seals: nil, - } - header := &flow.Header{ +func GenerateRootHeader(chainID flow.ChainID, parentID flow.Identifier, height uint64, timestamp time.Time) *flow.Header { + return &flow.Header{ ChainID: chainID, ParentID: parentID, Height: height, - PayloadHash: payload.Hash(), Timestamp: timestamp, View: 0, ParentVoterIndices: nil, @@ -24,9 +18,4 @@ func GenerateRootBlock(chainID flow.ChainID, parentID flow.Identifier, height ui ProposerID: flow.ZeroID, ProposerSigData: nil, } - - return &flow.Block{ - Header: header, - Payload: payload, - } } diff --git a/cmd/bootstrap/utils/file.go b/cmd/bootstrap/utils/file.go index fc5f35c7122..b0d278b5249 100644 --- a/cmd/bootstrap/utils/file.go +++ b/cmd/bootstrap/utils/file.go @@ -7,7 +7,6 @@ import ( "github.com/onflow/flow-go/engine/common/rpc/convert" model "github.com/onflow/flow-go/model/bootstrap" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol/inmem" io "github.com/onflow/flow-go/utils/io" ) @@ -32,30 +31,16 @@ func ReadRootProtocolSnapshot(bootDir string) (*inmem.Snapshot, error) { return snapshot, nil } -func ReadRootBlock(rootBlockDataPath string) (*flow.Block, error) { - bytes, err := io.ReadFile(rootBlockDataPath) +func ReadData[T any](path string) (*T, error) { + bytes, err := io.ReadFile(path) if err != nil { - return nil, fmt.Errorf("could not read root block file: %w", err) + return nil, fmt.Errorf("could not read data from file: %w", err) } - var encodable flow.Block + var encodable T err = json.Unmarshal(bytes, &encodable) if err != nil { return nil, fmt.Errorf("could not unmarshal root block: %w", err) } return &encodable, nil } - -func ReadDKGData(dkgDataPath string) (*inmem.EncodableFullDKG, error) { - bytes, err := io.ReadFile(dkgDataPath) - if err != nil { - return nil, fmt.Errorf("could not read dkg data: %w", err) - } - - var encodable inmem.EncodableFullDKG - err = json.Unmarshal(bytes, &encodable) - if err != nil { - return nil, fmt.Errorf("could not unmarshal dkg data: %w", err) - } - return &encodable, nil -} diff --git a/model/bootstrap/filenames.go b/model/bootstrap/filenames.go index 8933aa31563..a57b31e2812 100644 --- a/model/bootstrap/filenames.go +++ b/model/bootstrap/filenames.go @@ -24,6 +24,7 @@ var ( FileNamePartnerWeights = "partner-weights.json" PathRootBlockData = filepath.Join(DirnamePublicBootstrap, "root-block.json") + PathRootEpoch = filepath.Join(DirnamePublicBootstrap, "root-epoch.json") PathRootProtocolStateSnapshot = filepath.Join(DirnamePublicBootstrap, "root-protocol-state-snapshot.json") FilenameWALRootCheckpoint = "root.checkpoint" From dc1ecf6477eb2bd8fb761f36ce4bcca55f961652 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 25 Oct 2023 18:05:38 +0300 Subject: [PATCH 37/87] Finished updating rootblock command to correctly generate first epoch. Updated finalize command to correctly read prepared data. --- cmd/bootstrap/cmd/block.go | 1 + cmd/bootstrap/cmd/clusters.go | 18 +++++++++ cmd/bootstrap/cmd/finalize.go | 17 +++++--- cmd/bootstrap/cmd/finalize_test.go | 5 +-- cmd/bootstrap/cmd/rootblock.go | 5 +-- cmd/bootstrap/cmd/rootblock_test.go | 63 ++++------------------------- state/protocol/inmem/epoch.go | 1 - 7 files changed, 42 insertions(+), 68 deletions(-) diff --git a/cmd/bootstrap/cmd/block.go b/cmd/bootstrap/cmd/block.go index 6a20058c62a..cf43c24ab63 100644 --- a/cmd/bootstrap/cmd/block.go +++ b/cmd/bootstrap/cmd/block.go @@ -36,6 +36,7 @@ func constructRootBlock(rootHeader *flow.Header, setup *flow.EpochSetup, commit return block } +// constructRootEpochEvents constructs the epoch setup and commit events for the first epoch after spork. func constructRootEpochEvents( firstView uint64, participants flow.IdentityList, diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 27ab1c52605..13cce34bef8 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -2,6 +2,7 @@ package cmd import ( "errors" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/cmd/bootstrap/run" model "github.com/onflow/flow-go/model/bootstrap" @@ -123,3 +124,20 @@ func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.N return filtered } + +// clusterRootQCsFromEpoch returns a list of cluster root QCs from the given epoch. +func clusterRootQCsFromEpoch(epoch protocol.Epoch) []*flow.QuorumCertificate { + clustering, err := epoch.Clustering() + if err != nil { + log.Fatal().Err(err).Msg("could not get epoch clustering") + } + qcs := make([]*flow.QuorumCertificate, 0, len(clustering)) + for i := range clustering { + cluster, err := epoch.Cluster(uint(i)) + if err != nil { + log.Fatal().Err(err).Msgf("could not get epoch cluster (index=%d)", i) + } + qcs = append(qcs, cluster.RootQC()) + } + return qcs +} diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index ac48d9d8bab..6841d06a790 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -88,6 +88,8 @@ func addFinalizeCmdFlags() { // required parameters for generation of root block, root execution result and root block seal finalizeCmd.Flags().StringVar(&flagRootBlock, "root-block", "", "path to a JSON file containing root block") + finalizeCmd.Flags().StringVar(&flagRootEpoch, "root-epoch", "", + "path to a JSON file containing root epoch") finalizeCmd.Flags().StringVar(&flagRootBlockVotesDir, "root-block-votes-dir", "", "path to directory with votes for root block") finalizeCmd.Flags().StringVar(&flagRootCommit, "root-commit", "0000000000000000000000000000000000000000000000000000000000000000", "state commitment of root execution state") finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline") @@ -98,6 +100,7 @@ func addFinalizeCmdFlags() { cmd.MarkFlagRequired(finalizeCmd, "root-block-votes-dir") cmd.MarkFlagRequired(finalizeCmd, "root-commit") cmd.MarkFlagRequired(finalizeCmd, "protocol-version") + cmd.MarkFlagRequired(finalizeCmd, "epoch-commit-safety-threshold") // optional parameters to influence various aspects of identity generation finalizeCmd.Flags().UintVar(&flagCollectionClusters, "collection-clusters", 2, "number of collection clusters") @@ -162,8 +165,8 @@ func finalize(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("reading root epoch") - epochSetup, epochCommit := readRootEpoch() - clusterQCs := epochCommit.ClusterQCs + epoch, epochSetup, epochCommit := readRootEpoch() + clusterQCs := clusterRootQCsFromEpoch(epoch) log.Info().Msg("") log.Info().Msg("constructing root QC") @@ -485,10 +488,12 @@ func readRootBlock() *flow.Block { return rootBlock } -func readRootEpoch() (*flow.EpochSetup, *flow.EpochCommit) { +// readRootEpoch reads root epoch data from disc, this file needs to be prepared with +// rootblock command +func readRootEpoch() (protocol.Epoch, *flow.EpochSetup, *flow.EpochCommit) { encodableEpoch, err := utils.ReadData[inmem.EncodableEpoch](flagRootEpoch) if err != nil { - log.Fatal().Err(err).Msg("could not read root block data") + log.Fatal().Err(err).Msg("could not read root epoch data") } epoch := inmem.NewEpoch(*encodableEpoch) setup, err := protocol.ToEpochSetup(epoch) @@ -499,9 +504,11 @@ func readRootEpoch() (*flow.EpochSetup, *flow.EpochCommit) { if err != nil { log.Fatal().Err(err).Msg("could not extract commit event") } - return setup, commit + return epoch, setup, commit } +// readDKGData reads DKG data from disc, this file needs to be prepared with +// rootblock command func readDKGData() dkg.DKGData { encodableDKG, err := utils.ReadData[inmem.EncodableFullDKG](flagDKGDataPath) if err != nil { diff --git a/cmd/bootstrap/cmd/finalize_test.go b/cmd/bootstrap/cmd/finalize_test.go index 58929d21e81..a63f79c1e15 100644 --- a/cmd/bootstrap/cmd/finalize_test.go +++ b/cmd/bootstrap/cmd/finalize_test.go @@ -29,10 +29,8 @@ const finalizeHappyPathLogs = "collecting partner network and staking keys" + `reading root block votes` + `read vote .*` + `reading dkg data` + + `reading root epoch` + `constructing root QC` + - `computing collection node clusters` + - `constructing root blocks for collection node clusters` + - `constructing root QCs for collection node clusters` + `constructing root execution result and block seal` + `constructing root protocol snapshot` + `wrote file \S+/root-protocol-state-snapshot.json` + @@ -79,6 +77,7 @@ func TestFinalize_HappyPath(t *testing.T) { flagNumViewsInDKGPhase = 2_000 flagEpochCommitSafetyThreshold = 1_000 flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData) + flagRootEpoch = filepath.Join(bootDir, model.PathRootEpoch) flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData) flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes) diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index 5d324006590..fe3116d71d6 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -65,7 +65,6 @@ func addRootBlockCmdFlags() { cmd.MarkFlagRequired(rootBlockCmd, "epoch-length") cmd.MarkFlagRequired(rootBlockCmd, "epoch-staking-phase-length") cmd.MarkFlagRequired(rootBlockCmd, "epoch-dkg-phase-length") - cmd.MarkFlagRequired(rootBlockCmd, "epoch-commit-safety-threshold") // required parameters for generation of root block, root execution result and root block seal rootBlockCmd.Flags().StringVar(&flagRootChain, "root-chain", "local", "chain ID for the root block (can be 'main', 'test', 'sandbox', 'bench', or 'local'") @@ -111,7 +110,7 @@ func rootBlock(cmd *cobra.Command, args []string) { dkgData := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus)) log.Info().Msg("") - // create flow.IdentityList representation of participant set + // create flow.IdentityList representation of the participant set participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical[flow.Identity]) log.Info().Msg("computing collection node clusters") @@ -140,7 +139,7 @@ func rootBlock(cmd *cobra.Command, args []string) { if err != nil { log.Fatal().Msg("could not convert root epoch to encodable") } - writeJSON(model.PathRootEpoch, encodableEpoch) + writeJSON(model.PathRootEpoch, encodableEpoch.Encodable()) log.Info().Msg("") log.Info().Msg("constructing root block") diff --git a/cmd/bootstrap/cmd/rootblock_test.go b/cmd/bootstrap/cmd/rootblock_test.go index a2ccb177e79..8e16600cd9b 100644 --- a/cmd/bootstrap/cmd/rootblock_test.go +++ b/cmd/bootstrap/cmd/rootblock_test.go @@ -2,18 +2,15 @@ package cmd import ( "encoding/hex" - "os" "path/filepath" "regexp" "strings" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/cmd/bootstrap/utils" model "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/utils/unittest" + "github.com/stretchr/testify/assert" ) const rootBlockHappyPathLogs = "collecting partner network and staking keys" + @@ -32,6 +29,12 @@ const rootBlockHappyPathLogs = "collecting partner network and staking keys" + `finished running DKG` + `.+/random-beacon.priv.json` + `wrote file \S+/root-dkg-data.priv.json` + + `computing collection node clusters` + + `constructing root blocks for collection node clusters` + + `constructing root QCs for collection node clusters` + + `constructing root header` + + `constructing epoch events` + + `wrote file \S+/root-epoch.json` + `constructing root block` + `wrote file \S+/root-block.json` + `constructing and writing votes` + @@ -69,55 +72,3 @@ func TestRootBlock_HappyPath(t *testing.T) { assert.FileExists(t, rootBlockDataPath) }) } - -func TestRootBlock_Deterministic(t *testing.T) { - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(1000) - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - rootBlock(nil, nil) - require.Regexp(t, rootBlockHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - rootBlockDataPath := filepath.Join(bootDir, model.PathRootBlockData) - assert.FileExists(t, rootBlockDataPath) - - // read snapshot - firstRootBlockData, err := utils.ReadRootBlock(rootBlockDataPath) - require.NoError(t, err) - - // delete snapshot file - err = os.Remove(rootBlockDataPath) - require.NoError(t, err) - - rootBlock(nil, nil) - require.Regexp(t, rootBlockHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - assert.FileExists(t, rootBlockDataPath) - - // read snapshot - secondRootBlockData, err := utils.ReadRootBlock(rootBlockDataPath) - require.NoError(t, err) - - assert.Equal(t, firstRootBlockData, secondRootBlockData) - }) -} diff --git a/state/protocol/inmem/epoch.go b/state/protocol/inmem/epoch.go index afc608ede20..ab14996b4cd 100644 --- a/state/protocol/inmem/epoch.go +++ b/state/protocol/inmem/epoch.go @@ -2,7 +2,6 @@ package inmem import ( "fmt" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" From 07167d02c8c222af41f00e1af1df8f5c1110ff97 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 25 Oct 2023 18:08:36 +0300 Subject: [PATCH 38/87] Linted --- cmd/bootstrap/cmd/block.go | 8 ++++---- cmd/bootstrap/cmd/clusters.go | 2 +- cmd/bootstrap/cmd/rootblock.go | 6 +++--- cmd/bootstrap/cmd/rootblock_test.go | 3 ++- state/protocol/inmem/epoch.go | 1 + 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cmd/bootstrap/cmd/block.go b/cmd/bootstrap/cmd/block.go index cf43c24ab63..e6a281a21f5 100644 --- a/cmd/bootstrap/cmd/block.go +++ b/cmd/bootstrap/cmd/block.go @@ -2,14 +2,14 @@ package cmd import ( "encoding/hex" - "github.com/onflow/flow-go/model/dkg" - "github.com/onflow/flow-go/model/flow/order" - "github.com/onflow/flow-go/module/signature" - "github.com/onflow/flow-go/state/protocol/inmem" "time" "github.com/onflow/flow-go/cmd/bootstrap/run" + "github.com/onflow/flow-go/model/dkg" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/order" + "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/state/protocol/inmem" ) func constructRootHeader(rootChain string, rootParent string, rootHeight uint64, rootTimestamp string) *flow.Header { diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 13cce34bef8..b649e668114 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -2,7 +2,6 @@ package cmd import ( "errors" - "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/cmd/bootstrap/run" model "github.com/onflow/flow-go/model/bootstrap" @@ -11,6 +10,7 @@ import ( "github.com/onflow/flow-go/model/flow/assignment" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/state/protocol" ) // Construct random cluster assignment with internal and partner nodes. diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index fe3116d71d6..97122e55a94 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -1,16 +1,16 @@ package cmd import ( - "github.com/onflow/flow-go/cmd/bootstrap/run" - "github.com/onflow/flow-go/model/flow/order" - "github.com/onflow/flow-go/state/protocol/inmem" "time" "github.com/spf13/cobra" "github.com/onflow/flow-go/cmd" + "github.com/onflow/flow-go/cmd/bootstrap/run" model "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/order" + "github.com/onflow/flow-go/state/protocol/inmem" ) var ( diff --git a/cmd/bootstrap/cmd/rootblock_test.go b/cmd/bootstrap/cmd/rootblock_test.go index 8e16600cd9b..e89152bdd3a 100644 --- a/cmd/bootstrap/cmd/rootblock_test.go +++ b/cmd/bootstrap/cmd/rootblock_test.go @@ -7,10 +7,11 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/onflow/flow-go/cmd/bootstrap/utils" model "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/utils/unittest" - "github.com/stretchr/testify/assert" ) const rootBlockHappyPathLogs = "collecting partner network and staking keys" + diff --git a/state/protocol/inmem/epoch.go b/state/protocol/inmem/epoch.go index ab14996b4cd..afc608ede20 100644 --- a/state/protocol/inmem/epoch.go +++ b/state/protocol/inmem/epoch.go @@ -2,6 +2,7 @@ package inmem import ( "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" From 76b8fa0fcc4d242bcdc3ce23d14708b8599fba13 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 25 Oct 2023 18:35:24 +0300 Subject: [PATCH 39/87] Fixed compilation of integration tests --- cmd/bootstrap/cmd/block.go | 2 ++ cmd/consensus/main.go | 2 -- integration/testnet/network.go | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/bootstrap/cmd/block.go b/cmd/bootstrap/cmd/block.go index e6a281a21f5..747344ffa32 100644 --- a/cmd/bootstrap/cmd/block.go +++ b/cmd/bootstrap/cmd/block.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/state/protocol/inmem" ) +// constructRootHeader constructs a header for the root block. func constructRootHeader(rootChain string, rootParent string, rootHeight uint64, rootTimestamp string) *flow.Header { chainID := parseChainID(rootChain) parentID := parseParentID(rootParent) @@ -21,6 +22,7 @@ func constructRootHeader(rootChain string, rootParent string, rootHeight uint64, return run.GenerateRootHeader(chainID, parentID, height, timestamp) } +// constructRootBlock constructs a valid root block based on the given header, setup, and commit. func constructRootBlock(rootHeader *flow.Header, setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.Block { block := &flow.Block{ Header: rootHeader, diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index a6f9346aebb..cf62933d5f4 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package main import ( diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 7960f73694f..96deaafd452 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -1087,7 +1087,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl participants := bootstrap.ToIdentityList(stakedNodeInfos) // generate root block - root := run.GenerateRootBlock(chainID, parentID, height, timestamp) + rootHeader := run.GenerateRootHeader(chainID, parentID, height, timestamp) // generate root blocks for each collector cluster clusterRootBlocks, clusterAssignments, clusterQCs, err := setupClusterGenesisBlockQCs(networkConf.NClusters, epochCounter, stakedConfs) @@ -1117,16 +1117,16 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl return nil, err } - dkgOffsetView := root.Header.View + networkConf.ViewsInStakingAuction - 1 + dkgOffsetView := rootHeader.View + networkConf.ViewsInStakingAuction - 1 // generate epoch service events epochSetup := &flow.EpochSetup{ Counter: epochCounter, - FirstView: root.Header.View, + FirstView: rootHeader.View, DKGPhase1FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase, DKGPhase2FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase*2, DKGPhase3FinalView: dkgOffsetView + networkConf.ViewsInDKGPhase*3, - FinalView: root.Header.View + networkConf.ViewsInEpoch - 1, + FinalView: rootHeader.View + networkConf.ViewsInEpoch - 1, Participants: participants.ToSkeleton(), Assignments: clusterAssignments, RandomSource: randomSource, @@ -1138,6 +1138,9 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl DKGGroupKey: dkg.PubGroupKey, DKGParticipantKeys: dkg.PubKeyShares, } + root := &flow.Block{ + Header: rootHeader, + } root.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID( inmem.ProtocolStateForBootstrapState(epochSetup, epochCommit).ID()))) From 3246926260e061a59e13d307a6fbe9832dae939d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 25 Oct 2023 18:56:06 +0300 Subject: [PATCH 40/87] godoc cleanup --- module/builder.go | 2 -- module/builder/consensus/builder.go | 2 -- state/protocol/badger/state.go | 3 --- state/protocol/inmem/convert.go | 9 +++++++++ 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/module/builder.go b/module/builder.go index cc528524a8f..59877c56b4c 100644 --- a/module/builder.go +++ b/module/builder.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package module import ( diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index bc88fda6c40..49b662e6e58 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package consensus import ( diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index f244b1496e6..65eb75b18be 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package badger import ( @@ -792,7 +790,6 @@ func newState( cachedFinal: new(atomic.Pointer[cachedHeader]), cachedSealed: new(atomic.Pointer[cachedHeader]), } - state.protocolStateMutator = protocol_state.NewMutator( headers, results, diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index a3ce3f15192..d2a19d94966 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -351,6 +351,15 @@ func SnapshotFromBootstrapStateWithParams( } // ProtocolStateForBootstrapState generates a protocol.ProtocolStateEntry for a root protocol state which is used for bootstrapping. +// +// CONTEXT: The EpochSetup event contains the IdentitySkeletons for each participant, thereby specifying active epoch members. +// While ejection status and dynamic weight are not part of the EpochSetup event, we can supplement this information as follows: +// - Per convention, service events are delivered (asynchronously) in an *order-preserving* manner. Furthermore, weight changes or +// node ejection is also mediated by system smart contracts and delivered via service events. +// - Therefore, the EpochSetup event contains the up-to-date snapshot of the epoch participants. Any weight changes or node ejection +// that happened before should be reflected in the EpochSetup event. Specifically, the initial weight should be reduced and ejected +// nodes should be no longer listed in the EpochSetup event. Hence, when the EpochSetup event is emitted / processed, the weight of +// all epoch participants equals their InitialWeight and the Ejected flag is false. func ProtocolStateForBootstrapState(setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.ProtocolStateEntry { identities := make(flow.DynamicIdentityEntryList, 0, len(setup.Participants)) for _, identity := range setup.Participants { From 64c9a5c85de89d75b17cb1d192258e215cb4fde4 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 25 Oct 2023 19:13:36 +0300 Subject: [PATCH 41/87] Godoc and state updater error handling updates --- state/protocol/protocol_state.go | 18 +++++++++++++++--- state/protocol/protocol_state/mutator.go | 21 +++++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 1823c89784a..813b5b6069b 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -114,9 +114,12 @@ type StateUpdater interface { } // StateMutator is an interface for creating protocol state updaters and committing protocol state to the database. -// It is used by the compliance layer to update protocol state when certain events that are stored in blocks are observed. +// It is used by the replica's compliance layer to update protocol state when certain events that are stored in blocks are observed. +// It is used by the primary in the block building process to obtain a correct protocol state for a proposal. +// Specifically, leader includes service events in the block payload; those service events when being processed by replica will mutate the protocol state, so +// leader needs to include correct protocol state ID in the block payload otherwise replicas won't be able to verify the validity of the block. +// He can do that by creating a state updater for the proposal, applying service events to it and get updated protocol state ID from updater. // It has to be used for each block that is added to the block tree to maintain a correct protocol state on a block-by-block basis. -// TODO: this should be a stand-alone interface to support evolving the protocol state in the compliance layer (already possible) as well as during block construction (complex with the current implementation). type StateMutator interface { // CreateUpdater creates a protocol state updater based on previous protocol state. // Has to be called for each block to correctly index the protocol state. @@ -127,6 +130,15 @@ type StateMutator interface { // Has to be called for each block to correctly index the protocol state. // No errors are expected during normal operations. CommitProtocolState(blockID flow.Identifier, updater StateUpdater) (func(tx *transaction.Tx) error, flow.Identifier) - + // ApplyServiceEvents handles applying state changes which occur as a result + // of service events being included in a block payload. + // All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. + // + // Return values: + // - dbUpdates - If the service events are valid, or there are no service events, + // this method returns a slice of Badger operations to apply while storing the block. + // This includes operations to insert service events for blocks that include them. + // + // No errors are expected during normal operation. ApplyServiceEvents(updater StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 7ddd872082d..cf175308d8f 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -78,10 +78,10 @@ func (m *Mutator) CommitProtocolState(blockID flow.Identifier, updater protocol. }, updatedStateID } -// handleEpochServiceEvents handles applying state changes which occur as a result +// ApplyServiceEvents handles applying state changes which occur as a result // of service events being included in a block payload: // - inserting incorporated service events -// - updating EpochStatus for the candidate block +// - updating protocol state for the candidate block // // Consider a chain where a service event is emitted during execution of block A. // Block B contains a receipt for A. Block C contains a seal for block A. @@ -101,11 +101,14 @@ func (m *Mutator) CommitProtocolState(blockID flow.Identifier, updater protocol. // input block has the form of block C (ie. contains a seal for a block in // which a service event was emitted). // +// All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. +// This method doesn't modify any data from self, all protocol state changes are applied to state updater. +// All other changes are returned as slice of deferred DB updates. +// // Return values: // - dbUpdates - If the service events are valid, or there are no service events, // this method returns a slice of Badger operations to apply while storing the block. -// This includes an operation to index the epoch status for every block, and -// operations to insert service events for blocks that include them. +// This includes operations to insert service events for blocks that include them. // // No errors are expected during normal operation. func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { @@ -177,6 +180,11 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo err = updater.ProcessEpochSetup(ev) if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + updater.SetInvalidStateTransitionAttempted() + return dbUpdates, nil + } return nil, irrecoverable.NewExceptionf("could not process epoch setup event: %w", err) } @@ -204,6 +212,11 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo err = updater.ProcessEpochCommit(ev) if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + updater.SetInvalidStateTransitionAttempted() + return dbUpdates, nil + } return nil, irrecoverable.NewExceptionf("could not process epoch commit event: %w", err) } From 7553ed9a548ad7c6907ee445b67bf331d1eb202a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 26 Oct 2023 15:08:31 +0300 Subject: [PATCH 42/87] Added tests for ApplyServiceEvents --- state/protocol/protocol_state/mutator_test.go | 61 +++++++++++++++++++ utils/unittest/fixtures.go | 2 +- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 5894801c3ab..3749d9a3edb 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -1,6 +1,7 @@ package protocol_state import ( + "github.com/onflow/flow-go/model/flow" "testing" "github.com/stretchr/testify/require" @@ -84,3 +85,63 @@ func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { err = dbOps(&transaction.Tx{}) require.NoError(s.T(), err) } + +func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + s.Run("invalid-counter", func() { + parentState := unittest.ProtocolStateFixture() + rootSetup := parentState.CurrentEpochSetup + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + + epochSetup := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+2), // invalid counter + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + updater.On("SetInvalidStateTransitionAttempted").Return().Once() + + updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) + require.NoError(s.T(), err) + require.Empty(s.T(), updates) + }) + +} + +// TestMutatorApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. +// In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. +// In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. +// In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. +func (s *MutatorSuite) TestMutatorApplyServiceEvents_EpochFallbackTriggered() { + s.Run("epoch-fallback-triggered", func() { + s.params.On("EpochFallbackTriggered").Return(true, nil) + parentState := unittest.ProtocolStateFixture() + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + seals := unittest.Seal.Fixtures(2) + updates, err := s.mutator.ApplyServiceEvents(updater, seals) + require.NoError(s.T(), err) + require.Empty(s.T(), updates) + }) + s.Run("invalid-service-event-incorporated", func() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture() + parentState.InvalidStateTransitionAttempted = true + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + seals := unittest.Seal.Fixtures(2) + updates, err := s.mutator.ApplyServiceEvents(updater, seals) + require.NoError(s.T(), err) + require.Empty(s.T(), updates) + }) +} diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 065769c8a3d..8eee5ac71d7 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2656,7 +2656,7 @@ func ProtocolStateFixture(options ...func(*flow.RichProtocolStateEntry)) *flow.R setup.Counter = prevEpochSetup.Counter + 1 // reuse same participant for current epoch sameParticipant := *prevEpochSetup.Participants[1] - setup.Participants[1] = &sameParticipant + setup.Participants = append(setup.Participants, &sameParticipant) setup.Participants = setup.Participants.Sort(order.Canonical[flow.IdentitySkeleton]) }) currentEpochCommit := EpochCommitFixture(func(commit *flow.EpochCommit) { From aa1169ca5a17a12db964f9fc1a912c45aa2cbaa8 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 27 Oct 2023 22:22:04 +0300 Subject: [PATCH 43/87] Updated tests for protocol state mutator --- state/protocol/protocol_state/mutator_test.go | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 3749d9a3edb..e70aa845800 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -1,7 +1,9 @@ package protocol_state import ( + "errors" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" "testing" "github.com/stretchr/testify/require" @@ -86,6 +88,8 @@ func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { require.NoError(s.T(), err) } +// TestMutatorApplyServiceEvents_InvalidEpochSetup tests that ApplyServiceEvents rejects invalid epoch setup event and sets +// InvalidStateTransitionAttempted flag in protocol.StateUpdater. func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { s.params.On("EpochFallbackTriggered").Return(false, nil) s.Run("invalid-counter", func() { @@ -115,7 +119,63 @@ func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { require.NoError(s.T(), err) require.Empty(s.T(), updates) }) + s.Run("conflicts-with-protocol-state", func() { + parentState := unittest.ProtocolStateFixture() + rootSetup := parentState.CurrentEpochSetup + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + + epochSetup := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + updater.On("ProcessEpochSetup", epochSetup).Return(protocol.NewInvalidServiceEventErrorf("")).Once() + updater.On("SetInvalidStateTransitionAttempted").Return().Once() + + updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) + require.NoError(s.T(), err) + require.Empty(s.T(), updates) + }) + s.Run("process-epoch-setup-exception", func() { + parentState := unittest.ProtocolStateFixture() + rootSetup := parentState.CurrentEpochSetup + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + + epochSetup := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + exception := errors.New("exception") + updater.On("ProcessEpochSetup", epochSetup).Return(exception).Once() + + updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) + require.Error(s.T(), err) + require.False(s.T(), protocol.IsInvalidServiceEventError(err)) + require.Empty(s.T(), updates) + }) } // TestMutatorApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. From 9e90ac0e40246a6379226d143563520a75c50a4e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 27 Oct 2023 23:17:58 +0300 Subject: [PATCH 44/87] Consolidated service events handling in protocol state updater. Updated tests and docs --- state/protocol/protocol_state.go | 7 +- state/protocol/protocol_state/mutator.go | 28 ------- state/protocol/protocol_state/mutator_test.go | 33 +------- state/protocol/protocol_state/updater.go | 20 +++-- state/protocol/protocol_state/updater_test.go | 83 ++++++++++++------- 5 files changed, 73 insertions(+), 98 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 813b5b6069b..b0e23227888 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -69,7 +69,6 @@ type ProtocolState interface { // StateUpdater is a dedicated interface for updating protocol state. // It is used by the compliance layer to update protocol state when certain events that are stored in blocks are observed. -// CAUTION: The compliance layer is responsible for validating events before passing them to StateUpdater. type StateUpdater interface { // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) @@ -78,17 +77,15 @@ type StateUpdater interface { // Observing an epoch setup event, transitions protocol state from staking to setup phase, we stop returning // identities from previous+current epochs and start returning identities from current+next epochs. // As a result of this operation protocol state for the next epoch will be created. - // CAUTION: Caller must validate input event. // Expected errors during normal operations: - // - `protocol.InvalidServiceEventError` if the service event is not a valid state transition for the current protocol state + // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state ProcessEpochSetup(epochSetup *flow.EpochSetup) error // ProcessEpochCommit updates current protocol state with data from epoch commit event. // Observing an epoch setup commit, transitions protocol state from setup to commit phase, at this point we have // finished construction of the next epoch. // As a result of this operation protocol state for next epoch will be committed. - // CAUTION: Caller must validate input event. // Expected errors during normal operations: - // - `protocol.InvalidServiceEventError` if the service event is not a valid state transition for the current protocol state + // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state ProcessEpochCommit(epochCommit *flow.EpochCommit) error // UpdateIdentity updates identity table with new identity entry. // Should pass identity which is already present in the table, otherwise an exception will be raised. diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index cf175308d8f..63dec784a61 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -168,16 +168,6 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo switch ev := event.Event.(type) { case *flow.EpochSetup: - err := protocol.IsValidExtendingEpochSetup(ev, activeSetup, epochStatus) - if err != nil { - if protocol.IsInvalidServiceEventError(err) { - // we have observed an invalid service event, which triggers epoch fallback mode - updater.SetInvalidStateTransitionAttempted() - return dbUpdates, nil - } - return nil, fmt.Errorf("unexpected error validating EpochSetup service event: %w", err) - } - err = updater.ProcessEpochSetup(ev) if err != nil { if protocol.IsInvalidServiceEventError(err) { @@ -192,24 +182,6 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo dbUpdates = append(dbUpdates, m.setups.StoreTx(ev)) case *flow.EpochCommit: - // if we receive an EpochCommit event, we must have already observed an EpochSetup event - // => otherwise, we have observed an EpochCommit without corresponding EpochSetup, which triggers epoch fallback mode - if epochStatus.NextEpoch.SetupID == flow.ZeroID { - updater.SetInvalidStateTransitionAttempted() - return dbUpdates, nil - } - extendingSetup := parentProtocolState.NextEpochSetup - - err = protocol.IsValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) - if err != nil { - if protocol.IsInvalidServiceEventError(err) { - // we have observed an invalid service event, which triggers epoch fallback mode - updater.SetInvalidStateTransitionAttempted() - return dbUpdates, nil - } - return nil, fmt.Errorf("unexpected error validating EpochCommit service event: %w", err) - } - err = updater.ProcessEpochCommit(ev) if err != nil { if protocol.IsInvalidServiceEventError(err) { diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index e70aa845800..e0dd04a2377 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -2,13 +2,13 @@ package protocol_state import ( "errors" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/state/protocol" "testing" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/mock" storerr "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/transaction" @@ -92,34 +92,7 @@ func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { // InvalidStateTransitionAttempted flag in protocol.StateUpdater. func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { s.params.On("EpochFallbackTriggered").Return(false, nil) - s.Run("invalid-counter", func() { - parentState := unittest.ProtocolStateFixture() - rootSetup := parentState.CurrentEpochSetup - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - - epochSetup := unittest.EpochSetupFixture( - unittest.WithParticipants(rootSetup.Participants), - unittest.SetupWithCounter(rootSetup.Counter+2), // invalid counter - unittest.WithFinalView(rootSetup.FinalView+1000), - unittest.WithFirstView(rootSetup.FinalView+1), - ) - result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { - result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} - }) - - block := unittest.BlockHeaderFixture() - seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) - s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) - s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) - - updater.On("SetInvalidStateTransitionAttempted").Return().Once() - - updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) - require.NoError(s.T(), err) - require.Empty(s.T(), updates) - }) - s.Run("conflicts-with-protocol-state", func() { + s.Run("invalid-epoch-setup", func() { parentState := unittest.ProtocolStateFixture() rootSetup := parentState.CurrentEpochSetup updater := mock.NewStateUpdater(s.T()) diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index a337c446425..8dea0f2c61e 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -59,13 +59,14 @@ func (u *Updater) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.I // // As a result of this operation protocol state for the next epoch will be created. // Expected errors during normal operations: -// - `protocol.InvalidServiceEventError` if the service event is not a valid state transition for the current protocol state +// - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are in epoch fallback mode. } - if epochSetup.Counter != u.parentState.CurrentEpochSetup.Counter+1 { - return protocol.NewInvalidServiceEventErrorf("invalid epoch setup counter, expecting %d got %d", u.parentState.CurrentEpochSetup.Counter+1, epochSetup.Counter) + err := protocol.IsValidExtendingEpochSetup(epochSetup, u.parentState.CurrentEpochSetup, u.parentState.EpochStatus()) + if err != nil { + return fmt.Errorf("invalid epoch setup event: %w", err) } if u.state.NextEpoch != nil { return protocol.NewInvalidServiceEventErrorf("repeated setup for epoch %d", epochSetup.Counter) @@ -142,20 +143,25 @@ func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { // finished construction of the next epoch. // As a result of this operation protocol state for next epoch will be committed. // Expected errors during normal operations: -// - `protocol.InvalidServiceEventError` - an invalid service event with respect to the protocol state has been supplied. +// - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state func (u *Updater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are going to enter epoch fallback mode } - if epochCommit.Counter != u.parentState.CurrentEpochSetup.Counter+1 { - return protocol.NewInvalidServiceEventErrorf("invalid epoch commit counter, expecting %d got %d", u.parentState.CurrentEpochSetup.Counter+1, epochCommit.Counter) - } if u.state.NextEpoch == nil { return protocol.NewInvalidServiceEventErrorf("protocol state has been setup yet") } if u.state.NextEpoch.CommitID != flow.ZeroID { return protocol.NewInvalidServiceEventErrorf("protocol state has already a commit event") } + err := protocol.IsValidExtendingEpochCommit( + epochCommit, + u.parentState.NextEpochSetup, + u.parentState.CurrentEpochSetup, + u.parentState.EpochStatus()) + if err != nil { + return fmt.Errorf("invalid epoch commit event: %w", err) + } u.state.NextEpoch.CommitID = epochCommit.ID() return nil diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index 3a77ba40cad..43a795c6a1f 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -155,6 +155,8 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { updater := NewUpdater(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 + setup.FirstView = s.parentProtocolState.CurrentEpochSetup.FinalView + 1 + setup.FinalView = s.parentProtocolState.CurrentEpochSetup.FinalView + 1000 }) // processing setup event results in creating next epoch protocol state err := updater.ProcessEpochSetup(setup) @@ -175,16 +177,31 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { }) s.Run("happy path processing", func() { updater := NewUpdater(s.candidate.View, s.parentProtocolState) - setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { - setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - }) + setup := unittest.EpochSetupFixture( + unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), + unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), + unittest.WithFinalView(s.parentProtocolState.CurrentEpochSetup.FinalView+1000), + ) // processing setup event results in creating next epoch protocol state err := updater.ProcessEpochSetup(setup) require.NoError(s.T(), err) - commit := unittest.EpochCommitFixture(func(commit *flow.EpochCommit) { - commit.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - }) + updatedState, _, _ := updater.Build() + + parentState, err := flow.NewRichProtocolStateEntry(updatedState, + s.parentProtocolState.PreviousEpochSetup, + s.parentProtocolState.PreviousEpochCommit, + s.parentProtocolState.CurrentEpochSetup, + s.parentProtocolState.CurrentEpochCommit, + setup, + nil, + ) + require.NoError(s.T(), err) + updater = NewUpdater(s.candidate.View+1, parentState) + commit := unittest.EpochCommitFixture( + unittest.CommitWithCounter(setup.Counter), + unittest.WithDKGFromParticipants(setup.Participants), + ) err = updater.ProcessEpochCommit(commit) require.NoError(s.T(), err) @@ -287,9 +304,11 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { }) s.Run("processing second epoch setup", func() { updater := NewUpdater(s.candidate.View, s.parentProtocolState) - setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { - setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - }) + setup := unittest.EpochSetupFixture( + unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), + unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), + unittest.WithFinalView(s.parentProtocolState.CurrentEpochSetup.FinalView+1000), + ) err := updater.ProcessEpochSetup(setup) require.NoError(s.T(), err) @@ -333,13 +352,15 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { // For current epoch we should have identity table with all the nodes from the current epoch + nodes from the next epoch with 0 weight. // For next epoch we should have identity table with all the nodes from the next epoch + nodes from the current epoch with 0 weight. func (s *UpdaterSuite) TestProcessEpochSetupHappyPath() { - setupParticipants := unittest.IdentityListFixture(5).Sort(order.Canonical[flow.Identity]) + setupParticipants := unittest.IdentityListFixture(5, unittest.WithAllRoles()).Sort(order.Canonical[flow.Identity]) setupParticipants[0].InitialWeight = 13 setupParticipants[0].Weight = setupParticipants[0].InitialWeight - setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { - setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - setup.Participants = setupParticipants.ToSkeleton() - }) + setup := unittest.EpochSetupFixture( + unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), + unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), + unittest.WithFinalView(s.parentProtocolState.CurrentEpochSetup.FinalView+1000), + unittest.WithParticipants(setupParticipants.ToSkeleton()), + ) // for next epoch we will have all the identities from setup event expectedNextEpochActiveIdentities := flow.DynamicIdentityEntryListFromIdentities(setupParticipants) @@ -377,12 +398,14 @@ func (s *UpdaterSuite) TestProcessEpochSetupWithSameParticipants() { overlappingNodes, err := participantsFromCurrentEpochSetup.Sample(2) require.NoError(s.T(), err) - setupParticipants := append(unittest.IdentityListFixture(len(s.parentProtocolState.CurrentEpochIdentityTable)), + setupParticipants := append(unittest.IdentityListFixture(len(s.parentProtocolState.CurrentEpochIdentityTable), unittest.WithAllRoles()), overlappingNodes...).Sort(order.Canonical[flow.Identity]) - setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { - setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - setup.Participants = setupParticipants.ToSkeleton() - }) + setup := unittest.EpochSetupFixture( + unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), + unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), + unittest.WithFinalView(s.parentProtocolState.CurrentEpochSetup.FinalView+1000), + unittest.WithParticipants(setupParticipants.ToSkeleton()), + ) err = s.updater.ProcessEpochSetup(setup) require.NoError(s.T(), err) updatedState, _, _ := s.updater.Build() @@ -448,15 +471,19 @@ func (s *UpdaterSuite) TestEpochSetupAfterIdentityChange() { nextBlock := unittest.BlockHeaderWithParentFixture(s.candidate) s.updater = NewUpdater(nextBlock.View, updatedRichProtocolState) - setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { - setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 - // add those nodes that were changed in the previous epoch, but not those that were ejected - // it's important to exclude ejected nodes, since we expect that service smart contract has emitted ejection operation - // and service events are delivered (asynchronously) in an *order-preserving* manner meaning if ejection has happened before - // epoch setup then there is no possible way that it will include ejected node unless there is a severe bug in the service contract. - setup.Participants = append(setup.Participants, weightChanges.ToSkeleton()...).Filter( - filter.Not(filter.In(ejectedChanges.ToSkeleton()))).Sort(order.Canonical[flow.IdentitySkeleton]) - }) + setup := unittest.EpochSetupFixture( + unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), + unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), + unittest.WithFinalView(s.parentProtocolState.CurrentEpochSetup.FinalView+1000), + func(setup *flow.EpochSetup) { + // add those nodes that were changed in the previous epoch, but not those that were ejected + // it's important to exclude ejected nodes, since we expect that service smart contract has emitted ejection operation + // and service events are delivered (asynchronously) in an *order-preserving* manner meaning if ejection has happened before + // epoch setup then there is no possible way that it will include ejected node unless there is a severe bug in the service contract. + setup.Participants = append(setup.Participants, weightChanges.ToSkeleton()...).Filter( + filter.Not(filter.In(ejectedChanges.ToSkeleton()))).Sort(order.Canonical[flow.IdentitySkeleton]) + }, + ) err = s.updater.ProcessEpochSetup(setup) require.NoError(s.T(), err) From a957702e63e620417ddb3337718c26e7ed3b27ba Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Fri, 27 Oct 2023 17:53:04 -0700 Subject: [PATCH 45/87] added minor todo comment for --- consensus/hotstuff/blockproducer/block_producer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/hotstuff/blockproducer/block_producer.go b/consensus/hotstuff/blockproducer/block_producer.go index 64e89679b7d..3238507393a 100644 --- a/consensus/hotstuff/blockproducer/block_producer.go +++ b/consensus/hotstuff/blockproducer/block_producer.go @@ -46,6 +46,7 @@ func (bp *BlockProducer) MakeBlockProposal(view uint64, qc *flow.QuorumCertifica return nil } + // TODO: We should utilize the `EventHandler`'s `SafetyRules` to generate the block signature instead of using an independent signing logic: https://github.com/dapperlabs/flow-go/issues/6892 signProposal := func(header *flow.Header) error { // turn the header into a block header proposal as known by hotstuff block := model.Block{ From e11d0c2b8683e4af931197ca8f6332f07db41582 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 30 Oct 2023 17:38:53 +0200 Subject: [PATCH 46/87] Updated mutator tests to cover all paths in ApplyServiceEvents --- state/protocol/protocol_state/mutator_test.go | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index e0dd04a2377..48983032dea 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -13,6 +13,7 @@ import ( storerr "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/transaction" storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/rand" "github.com/onflow/flow-go/utils/unittest" ) @@ -151,6 +152,125 @@ func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { }) } +// TestMutatorApplyServiceEvents_InvalidEpochCommit tests that ApplyServiceEvents rejects invalid epoch commit event and sets +// InvalidStateTransitionAttempted flag in protocol.StateUpdater. +func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochCommit() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + s.Run("invalid-epoch-commit", func() { + parentState := unittest.ProtocolStateFixture() + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + + epochCommit := unittest.EpochCommitFixture() + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + updater.On("ProcessEpochCommit", epochCommit).Return(protocol.NewInvalidServiceEventErrorf("")).Once() + updater.On("SetInvalidStateTransitionAttempted").Return().Once() + + updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) + require.NoError(s.T(), err) + require.Empty(s.T(), updates) + }) + s.Run("process-epoch-commit-exception", func() { + parentState := unittest.ProtocolStateFixture() + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + + epochCommit := unittest.EpochCommitFixture() + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + exception := errors.New("exception") + updater.On("ProcessEpochCommit", epochCommit).Return(exception).Once() + + updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) + require.Error(s.T(), err) + require.False(s.T(), protocol.IsInvalidServiceEventError(err)) + require.Empty(s.T(), updates) + }) +} + +// TestMutatorApplyServiceEventsSealsOrdered tests that ApplyServiceEvents processes seals in order of block height. +func (s *MutatorSuite) TestMutatorApplyServiceEventsSealsOrdered() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture() + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + + blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) + var seals []*flow.Seal + resultByHeight := make(map[flow.Identifier]uint64) + for _, block := range blocks { + receipt, seal := unittest.ReceiptAndSealForBlock(block) + resultByHeight[seal.ResultID] = block.Header.Height + s.headersDB.On("ByBlockID", seal.BlockID).Return(block.Header, nil).Once() + s.resultsDB.On("ByID", seal.ResultID).Return(&receipt.ExecutionResult, nil).Once() + seals = append(seals, seal) + } + + // shuffle seals to make sure we order them + require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { + seals[i], seals[j] = seals[j], seals[i] + })) + + updates, err := s.mutator.ApplyServiceEvents(updater, seals) + require.NoError(s.T(), err) + require.Empty(s.T(), updates) + // assert that results were queried in order of executed block height + // if seals were properly ordered before processing, then results should be ordered by block height + lastExecutedBlockHeight := uint64(0) + for _, call := range s.resultsDB.Calls { + resultID := call.Arguments.Get(0).(flow.Identifier) + executedBlockHeight, found := resultByHeight[resultID] + require.True(s.T(), found) + require.Less(s.T(), lastExecutedBlockHeight, executedBlockHeight, "seals must be ordered by block height") + } +} + +// TestMutatorApplyServiceEventsTransitionToNextEpoch tests that ApplyServiceEvents transitions to the next epoch +// when epoch has been committed, and we are at the first block of the next epoch. +func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + // we are at the first block of the next epoch + updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) + updater.On("TransitionToNextEpoch").Return(nil).Once() + dbUpdates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{}) + require.NoError(s.T(), err) + require.Empty(s.T(), dbUpdates) +} + +// TestMutatorApplyServiceEventsTransitionToNextEpoch_Error tests that error that has been +// observed in ApplyServiceEvents when transitioning to the next epoch is propagated to the caller. +func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch_Error() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) + updater := mock.NewStateUpdater(s.T()) + updater.On("ParentState").Return(parentState) + // we are at the first block of the next epoch + updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) + exception := errors.New("exception") + updater.On("TransitionToNextEpoch").Return(exception).Once() + _, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{}) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), protocol.IsInvalidServiceEventError(err)) +} + // TestMutatorApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. // In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. // In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. From 80f4bdfd003475c0685761a6acc640663af40201 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 31 Oct 2023 12:50:08 +0200 Subject: [PATCH 47/87] Updated dynamic protocol state main interfaces. Restructured interfaces and implementors to simplify design --- Makefile | 1 + module/builder/consensus/builder.go | 73 +-- module/builder/consensus/builder_test.go | 13 +- state/protocol/badger/mutator.go | 17 +- state/protocol/badger/state.go | 20 +- state/protocol/mock/cluster_events.go | 33 -- .../mock/cluster_id_update_consumer.go | 33 -- state/protocol/mock/mutable_protocol_state.go | 98 ++++ state/protocol/mock/state_mutator.go | 97 ++-- state/protocol/protocol_state.go | 90 +-- .../mock/protocol_state_machine.go} | 28 +- state/protocol/protocol_state/mutator.go | 98 ++-- state/protocol/protocol_state/mutator_test.go | 520 +++++++++--------- .../protocol/protocol_state/protocol_state.go | 44 +- state/protocol/protocol_state/statemachine.go | 46 ++ state/protocol/protocol_state/updater.go | 35 +- state/protocol/protocol_state/updater_test.go | 34 +- utils/unittest/epoch_builder.go | 37 +- utils/unittest/protocol_state.go | 10 +- 19 files changed, 695 insertions(+), 632 deletions(-) delete mode 100644 state/protocol/mock/cluster_events.go delete mode 100644 state/protocol/mock/cluster_id_update_consumer.go create mode 100644 state/protocol/mock/mutable_protocol_state.go rename state/protocol/{mock/state_updater.go => protocol_state/mock/protocol_state_machine.go} (69%) create mode 100644 state/protocol/protocol_state/statemachine.go diff --git a/Makefile b/Makefile index 5c62f86034f..6c0650fd61f 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,7 @@ generate-mocks: install-mock-generators mockery --name '.*' --dir=storage --case=underscore --output="./storage/mock" --outpkg="mock" mockery --name '.*' --dir="state/protocol" --case=underscore --output="state/protocol/mock" --outpkg="mock" mockery --name '.*' --dir="state/protocol/events" --case=underscore --output="./state/protocol/events/mock" --outpkg="mock" + mockery --name '.*' --dir="state/protocol/protocol_state" --case=underscore --output="state/protocol/protocol_state/mock" --outpkg="mock" mockery --name '.*' --dir=engine/execution/computation/computer --case=underscore --output="./engine/execution/computation/computer/mock" --outpkg="mock" mockery --name '.*' --dir=engine/execution/state --case=underscore --output="./engine/execution/state/mock" --outpkg="mock" mockery --name '.*' --dir=engine/collection --case=underscore --output="./engine/collection/mock" --outpkg="mock" diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index 49b662e6e58..455af4cd475 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -23,21 +23,21 @@ import ( // Builder is the builder for consensus block payloads. Upon providing a payload // hash, it also memorizes which entities were included into the payload. type Builder struct { - metrics module.MempoolMetrics - tracer module.Tracer - db *badger.DB - state protocol.ParticipantState - seals storage.Seals - headers storage.Headers - index storage.Index - blocks storage.Blocks - resultsDB storage.ExecutionResults - receiptsDB storage.ExecutionReceipts - guarPool mempool.Guarantees - sealPool mempool.IncorporatedResultSeals - recPool mempool.ExecutionTree - protoStateMutator protocol.StateMutator - cfg Config + metrics module.MempoolMetrics + tracer module.Tracer + db *badger.DB + state protocol.ParticipantState + seals storage.Seals + headers storage.Headers + index storage.Index + blocks storage.Blocks + resultsDB storage.ExecutionResults + receiptsDB storage.ExecutionReceipts + guarPool mempool.Guarantees + sealPool mempool.IncorporatedResultSeals + recPool mempool.ExecutionTree + mutableProtocolState protocol.MutableProtocolState + cfg Config } // NewBuilder creates a new block builder. @@ -51,7 +51,7 @@ func NewBuilder( blocks storage.Blocks, resultsDB storage.ExecutionResults, receiptsDB storage.ExecutionReceipts, - protoStateMutator protocol.StateMutator, + mutableProtocolState protocol.MutableProtocolState, guarPool mempool.Guarantees, sealPool mempool.IncorporatedResultSeals, recPool mempool.ExecutionTree, @@ -79,21 +79,21 @@ func NewBuilder( } b := &Builder{ - metrics: metrics, - db: db, - tracer: tracer, - state: state, - headers: headers, - seals: seals, - index: index, - blocks: blocks, - resultsDB: resultsDB, - receiptsDB: receiptsDB, - guarPool: guarPool, - sealPool: sealPool, - recPool: recPool, - protoStateMutator: protoStateMutator, - cfg: cfg, + metrics: metrics, + db: db, + tracer: tracer, + state: state, + headers: headers, + seals: seals, + index: index, + blocks: blocks, + resultsDB: resultsDB, + receiptsDB: receiptsDB, + guarPool: guarPool, + sealPool: sealPool, + recPool: recPool, + mutableProtocolState: mutableProtocolState, + cfg: cfg, } err = b.repopulateExecutionTree() @@ -635,15 +635,18 @@ func (b *Builder) createProposal(parentID flow.Identifier, return nil, fmt.Errorf("could not apply setter: %w", err) } - updater, err := b.protoStateMutator.CreateUpdater(header.View, header.ParentID) + stateMutator, err := b.mutableProtocolState.Mutator(header.View, header.ParentID) if err != nil { - return nil, fmt.Errorf("could not create protocol state updater for view %d: %w", header.View, err) + return nil, fmt.Errorf("could not create protocol state stateMutator for view %d: %w", header.View, err) } - _, err = b.protoStateMutator.ApplyServiceEvents(updater, seals) + err = stateMutator.ApplyServiceEvents(seals) if err != nil { return nil, fmt.Errorf("could not apply service events as leader: %w", err) } - _, protocolStateID, _ := updater.Build() + _, _, protocolStateID, _, err := stateMutator.Build() + if err != nil { + return nil, fmt.Errorf("could not build updated protocol state: %w", err) + } proposal := &flow.Block{ Header: header, diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index 32c24654889..16a80f8a926 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -76,7 +76,7 @@ type BuilderSuite struct { blockDB *storage.Blocks resultDB *storage.ExecutionResults receiptsDB *storage.ExecutionReceipts - stateMutator *protocol.StateMutator + stateMutator *protocol.MutableProtocolState guarPool *mempool.Guarantees sealPool *mempool.IncorporatedResultSeals @@ -415,17 +415,16 @@ func (bs *BuilderSuite) SetupTest() { ) // setup mock state mutator, we don't need a real once since we are using mocked participant state. - bs.stateMutator = protocol.NewStateMutator(bs.T()) - bs.stateMutator.On("CreateUpdater", mock.Anything, mock.Anything).Return( - func(_ uint64, _ flow.Identifier) realproto.StateUpdater { - updater := protocol.NewStateUpdater(bs.T()) - updater.On("Build").Return(nil, flow.Identifier{}, false) + bs.stateMutator = protocol.NewMutableProtocolState(bs.T()) + bs.stateMutator.On("Mutator", mock.Anything, mock.Anything).Return( + func(_ uint64, _ flow.Identifier) realproto.StateMutator { + updater := protocol.NewStateMutator(bs.T()) + updater.On("Build").Return(false, nil, flow.Identifier{}, nil) return updater }, func(_ uint64, _ flow.Identifier) error { return nil }, ) - bs.stateMutator.On("ApplyServiceEvents", mock.Anything, mock.Anything).Return(nil, nil) // initialize the builder bs.build, err = NewBuilder( diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index 8c03dec1b6d..1f835d7ae38 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -508,23 +508,30 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi return fmt.Errorf("could not retrieve block header for %x: %w", parentID, err) } - protocolStateUpdater, err := m.protocolStateMutator.CreateUpdater(candidate.Header.View, parentID) + stateMutator, err := m.protocolState.Mutator(candidate.Header.View, parentID) if err != nil { - return fmt.Errorf("could not create protocol state updater for view %d: %w", candidate.Header.View, err) + return fmt.Errorf("could not create protocol state mutator for view %d: %w", candidate.Header.View, err) } // apply any state changes from service events sealed by this block - dbUpdates, err := m.protocolStateMutator.ApplyServiceEvents(protocolStateUpdater, candidate.Payload.Seals) + err = stateMutator.ApplyServiceEvents(candidate.Payload.Seals) if err != nil { return fmt.Errorf("could not process service events: %w", err) } - commitStateDbUpdate, updatedStateID := m.protocolStateMutator.CommitProtocolState(blockID, protocolStateUpdater) + hasChanges, updatedState, updatedStateID, dbUpdates, err := stateMutator.Build() + if err != nil { + return fmt.Errorf("could not build updated protocol state: %w", err) + } + if updatedStateID != candidate.Payload.ProtocolStateID { return state.NewInvalidExtensionErrorf("invalid protocol state transition detected, "+ "payload contains (%x) but after applying changes got %x", candidate.Payload.ProtocolStateID, updatedStateID) } - dbUpdates = append(dbUpdates, commitStateDbUpdate) + if hasChanges { + dbUpdates = append(dbUpdates, m.protocolStateSnapshotsDB.StoreTx(updatedStateID, updatedState)) + dbUpdates = append(dbUpdates, m.protocolStateSnapshotsDB.Index(blockID, updatedStateID)) + } // events is a queue of node-internal events (aka notifications) that are emitted after the database write succeeded var events []func() diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index 65eb75b18be..f820a6dca8b 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -37,9 +37,9 @@ type State struct { setups storage.EpochSetups commits storage.EpochCommits } - protocolStateMutator protocol.StateMutator - protocolState protocol.ProtocolState - versionBeacons storage.VersionBeacons + protocolStateSnapshotsDB storage.ProtocolState + protocolState protocol.MutableProtocolState + versionBeacons storage.VersionBeacons // rootHeight marks the cutoff of the history this node knows about. We cache it in the state // because it cannot change over the lifecycle of a protocol state instance. It is frequently @@ -784,18 +784,18 @@ func newState( setups: setups, commits: commits, }, - protocolState: protocol_state.NewProtocolState(protocolStateSnapshots, globalParams), - protocolStateMutator: nil, - versionBeacons: versionBeacons, - cachedFinal: new(atomic.Pointer[cachedHeader]), - cachedSealed: new(atomic.Pointer[cachedHeader]), + protocolStateSnapshotsDB: protocolStateSnapshots, + versionBeacons: versionBeacons, + cachedFinal: new(atomic.Pointer[cachedHeader]), + cachedSealed: new(atomic.Pointer[cachedHeader]), } - state.protocolStateMutator = protocol_state.NewMutator( + state.protocolState = protocol_state.NewMutableProtocolState( + protocolStateSnapshots, + globalParams, headers, results, setups, commits, - protocolStateSnapshots, state.Params(), ) diff --git a/state/protocol/mock/cluster_events.go b/state/protocol/mock/cluster_events.go deleted file mode 100644 index a17e4db4a9a..00000000000 --- a/state/protocol/mock/cluster_events.go +++ /dev/null @@ -1,33 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mock - -import ( - flow "github.com/onflow/flow-go/model/flow" - mock "github.com/stretchr/testify/mock" -) - -// ClusterEvents is an autogenerated mock type for the ClusterEvents type -type ClusterEvents struct { - mock.Mock -} - -// ActiveClustersChanged provides a mock function with given fields: _a0 -func (_m *ClusterEvents) ActiveClustersChanged(_a0 flow.ChainIDList) { - _m.Called(_a0) -} - -type mockConstructorTestingTNewClusterEvents interface { - mock.TestingT - Cleanup(func()) -} - -// NewClusterEvents creates a new instance of ClusterEvents. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewClusterEvents(t mockConstructorTestingTNewClusterEvents) *ClusterEvents { - mock := &ClusterEvents{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/state/protocol/mock/cluster_id_update_consumer.go b/state/protocol/mock/cluster_id_update_consumer.go deleted file mode 100644 index 3594d504c0f..00000000000 --- a/state/protocol/mock/cluster_id_update_consumer.go +++ /dev/null @@ -1,33 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mock - -import ( - flow "github.com/onflow/flow-go/model/flow" - mock "github.com/stretchr/testify/mock" -) - -// ClusterIDUpdateConsumer is an autogenerated mock type for the ClusterIDUpdateConsumer type -type ClusterIDUpdateConsumer struct { - mock.Mock -} - -// ClusterIdsUpdated provides a mock function with given fields: _a0 -func (_m *ClusterIDUpdateConsumer) ActiveClustersChanged(_a0 flow.ChainIDList) { - _m.Called(_a0) -} - -type mockConstructorTestingTNewClusterIDUpdateConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewClusterIDUpdateConsumer creates a new instance of ClusterIDUpdateConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewClusterIDUpdateConsumer(t mockConstructorTestingTNewClusterIDUpdateConsumer) *ClusterIDUpdateConsumer { - mock := &ClusterIDUpdateConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/state/protocol/mock/mutable_protocol_state.go b/state/protocol/mock/mutable_protocol_state.go new file mode 100644 index 00000000000..a7d77f02d7f --- /dev/null +++ b/state/protocol/mock/mutable_protocol_state.go @@ -0,0 +1,98 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + protocol "github.com/onflow/flow-go/state/protocol" +) + +// MutableProtocolState is an autogenerated mock type for the MutableProtocolState type +type MutableProtocolState struct { + mock.Mock +} + +// AtBlockID provides a mock function with given fields: blockID +func (_m *MutableProtocolState) AtBlockID(blockID flow.Identifier) (protocol.DynamicProtocolState, error) { + ret := _m.Called(blockID) + + var r0 protocol.DynamicProtocolState + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier) (protocol.DynamicProtocolState, error)); ok { + return rf(blockID) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) protocol.DynamicProtocolState); ok { + r0 = rf(blockID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(protocol.DynamicProtocolState) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(blockID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GlobalParams provides a mock function with given fields: +func (_m *MutableProtocolState) GlobalParams() protocol.GlobalParams { + ret := _m.Called() + + var r0 protocol.GlobalParams + if rf, ok := ret.Get(0).(func() protocol.GlobalParams); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(protocol.GlobalParams) + } + } + + return r0 +} + +// Mutator provides a mock function with given fields: candidateView, parentID +func (_m *MutableProtocolState) Mutator(candidateView uint64, parentID flow.Identifier) (protocol.StateMutator, error) { + ret := _m.Called(candidateView, parentID) + + var r0 protocol.StateMutator + var r1 error + if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) (protocol.StateMutator, error)); ok { + return rf(candidateView, parentID) + } + if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) protocol.StateMutator); ok { + r0 = rf(candidateView, parentID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(protocol.StateMutator) + } + } + + if rf, ok := ret.Get(1).(func(uint64, flow.Identifier) error); ok { + r1 = rf(candidateView, parentID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewMutableProtocolState interface { + mock.TestingT + Cleanup(func()) +} + +// NewMutableProtocolState creates a new instance of MutableProtocolState. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMutableProtocolState(t mockConstructorTestingTNewMutableProtocolState) *MutableProtocolState { + mock := &MutableProtocolState{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/state/protocol/mock/state_mutator.go b/state/protocol/mock/state_mutator.go index ac3479e1eaf..9d545d30b9a 100644 --- a/state/protocol/mock/state_mutator.go +++ b/state/protocol/mock/state_mutator.go @@ -6,8 +6,6 @@ import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" - protocol "github.com/onflow/flow-go/state/protocol" - transaction "github.com/onflow/flow-go/storage/badger/transaction" ) @@ -16,84 +14,69 @@ type StateMutator struct { mock.Mock } -// ApplyServiceEvents provides a mock function with given fields: updater, seals -func (_m *StateMutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) ([]func(*transaction.Tx) error, error) { - ret := _m.Called(updater, seals) +// ApplyServiceEvents provides a mock function with given fields: seals +func (_m *StateMutator) ApplyServiceEvents(seals []*flow.Seal) error { + ret := _m.Called(seals) - var r0 []func(*transaction.Tx) error - var r1 error - if rf, ok := ret.Get(0).(func(protocol.StateUpdater, []*flow.Seal) ([]func(*transaction.Tx) error, error)); ok { - return rf(updater, seals) - } - if rf, ok := ret.Get(0).(func(protocol.StateUpdater, []*flow.Seal) []func(*transaction.Tx) error); ok { - r0 = rf(updater, seals) + var r0 error + if rf, ok := ret.Get(0).(func([]*flow.Seal) error); ok { + r0 = rf(seals) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]func(*transaction.Tx) error) - } + r0 = ret.Error(0) } - if rf, ok := ret.Get(1).(func(protocol.StateUpdater, []*flow.Seal) error); ok { - r1 = rf(updater, seals) - } else { - r1 = ret.Error(1) - } - - return r0, r1 + return r0 } -// CommitProtocolState provides a mock function with given fields: blockID, updater -func (_m *StateMutator) CommitProtocolState(blockID flow.Identifier, updater protocol.StateUpdater) (func(*transaction.Tx) error, flow.Identifier) { - ret := _m.Called(blockID, updater) - - var r0 func(*transaction.Tx) error - var r1 flow.Identifier - if rf, ok := ret.Get(0).(func(flow.Identifier, protocol.StateUpdater) (func(*transaction.Tx) error, flow.Identifier)); ok { - return rf(blockID, updater) +// Build provides a mock function with given fields: +func (_m *StateMutator) Build() (bool, *flow.ProtocolStateEntry, flow.Identifier, []func(*transaction.Tx) error, error) { + ret := _m.Called() + + var r0 bool + var r1 *flow.ProtocolStateEntry + var r2 flow.Identifier + var r3 []func(*transaction.Tx) error + var r4 error + if rf, ok := ret.Get(0).(func() (bool, *flow.ProtocolStateEntry, flow.Identifier, []func(*transaction.Tx) error, error)); ok { + return rf() } - if rf, ok := ret.Get(0).(func(flow.Identifier, protocol.StateUpdater) func(*transaction.Tx) error); ok { - r0 = rf(blockID, updater) + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(func(*transaction.Tx) error) - } + r0 = ret.Get(0).(bool) } - if rf, ok := ret.Get(1).(func(flow.Identifier, protocol.StateUpdater) flow.Identifier); ok { - r1 = rf(blockID, updater) + if rf, ok := ret.Get(1).(func() *flow.ProtocolStateEntry); ok { + r1 = rf() } else { if ret.Get(1) != nil { - r1 = ret.Get(1).(flow.Identifier) + r1 = ret.Get(1).(*flow.ProtocolStateEntry) } } - return r0, r1 -} - -// CreateUpdater provides a mock function with given fields: candidateView, parentID -func (_m *StateMutator) CreateUpdater(candidateView uint64, parentID flow.Identifier) (protocol.StateUpdater, error) { - ret := _m.Called(candidateView, parentID) - - var r0 protocol.StateUpdater - var r1 error - if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) (protocol.StateUpdater, error)); ok { - return rf(candidateView, parentID) + if rf, ok := ret.Get(2).(func() flow.Identifier); ok { + r2 = rf() + } else { + if ret.Get(2) != nil { + r2 = ret.Get(2).(flow.Identifier) + } } - if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) protocol.StateUpdater); ok { - r0 = rf(candidateView, parentID) + + if rf, ok := ret.Get(3).(func() []func(*transaction.Tx) error); ok { + r3 = rf() } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(protocol.StateUpdater) + if ret.Get(3) != nil { + r3 = ret.Get(3).([]func(*transaction.Tx) error) } } - if rf, ok := ret.Get(1).(func(uint64, flow.Identifier) error); ok { - r1 = rf(candidateView, parentID) + if rf, ok := ret.Get(4).(func() error); ok { + r4 = rf() } else { - r1 = ret.Error(1) + r4 = ret.Error(4) } - return r0, r1 + return r0, r1, r2, r3, r4 } type mockConstructorTestingTNewStateMutator interface { diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index b0e23227888..5cbc9bb1699 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -44,7 +44,6 @@ type DynamicProtocolState interface { // ProtocolState is the read-only interface for protocol state, it allows to query information // on a per-block and per-epoch basis. type ProtocolState interface { - // ByEpoch returns an object with static protocol state information by epoch number. // To be able to use this interface we need to observe both epoch setup and commit events. // Not available for next epoch unless we have observed an EpochCommit event. @@ -67,75 +66,38 @@ type ProtocolState interface { GlobalParams() GlobalParams } -// StateUpdater is a dedicated interface for updating protocol state. -// It is used by the compliance layer to update protocol state when certain events that are stored in blocks are observed. -type StateUpdater interface { - // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. - Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) - // ProcessEpochSetup updates current protocol state with data from epoch setup event. - // Processing epoch setup event also affects identity table for current epoch. - // Observing an epoch setup event, transitions protocol state from staking to setup phase, we stop returning - // identities from previous+current epochs and start returning identities from current+next epochs. - // As a result of this operation protocol state for the next epoch will be created. - // Expected errors during normal operations: - // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state - ProcessEpochSetup(epochSetup *flow.EpochSetup) error - // ProcessEpochCommit updates current protocol state with data from epoch commit event. - // Observing an epoch setup commit, transitions protocol state from setup to commit phase, at this point we have - // finished construction of the next epoch. - // As a result of this operation protocol state for next epoch will be committed. +type MutableProtocolState interface { + ProtocolState + + // Mutator instantiates a `StateMutator` based on the previous protocol state. + // Has to be called for each block to evolve the protocol state. // Expected errors during normal operations: - // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state - ProcessEpochCommit(epochCommit *flow.EpochCommit) error - // UpdateIdentity updates identity table with new identity entry. - // Should pass identity which is already present in the table, otherwise an exception will be raised. - // TODO: This function currently modifies both current+next identities based on input. - // This is incompatible with the design doc, and needs to be updated to modify current/next epoch separately - // No errors are expected during normal operations. - UpdateIdentity(updated *flow.DynamicIdentityEntry) error - // SetInvalidStateTransitionAttempted sets a flag indicating that invalid state transition was attempted. - // Such transition can be detected by compliance layer. - SetInvalidStateTransitionAttempted() - // TransitionToNextEpoch discards current protocol state and transitions to the next epoch. - // Epoch transition is only allowed when: - // - next epoch has been set up, - // - next epoch has been committed, - // - candidate block is in the next epoch. - // No errors are expected during normal operations. - TransitionToNextEpoch() error - // View returns the view that is associated with this state updater. - // StateUpdater is created for a view where protocol state changes will be applied. - View() uint64 - // ParentState returns parent protocol state that is associated with this state updater. - ParentState() *flow.RichProtocolStateEntry + // * `storage.ErrNotFound` if no protocol state for parent block is known. + Mutator(candidateView uint64, parentID flow.Identifier) (StateMutator, error) } -// StateMutator is an interface for creating protocol state updaters and committing protocol state to the database. -// It is used by the replica's compliance layer to update protocol state when certain events that are stored in blocks are observed. -// It is used by the primary in the block building process to obtain a correct protocol state for a proposal. -// Specifically, leader includes service events in the block payload; those service events when being processed by replica will mutate the protocol state, so -// leader needs to include correct protocol state ID in the block payload otherwise replicas won't be able to verify the validity of the block. -// He can do that by creating a state updater for the proposal, applying service events to it and get updated protocol state ID from updater. -// It has to be used for each block that is added to the block tree to maintain a correct protocol state on a block-by-block basis. +// StateMutator is a stateful object to evolve the protocol state. It is instantiated from the parent block's protocol state. +// State-changing operations can be iteratively applied and the StateMutator will internally evolve its in-memory state. +// While the StateMutator does not modify the database, it internally tracks all necessary updates. Upon calling `Build` +// the StateMutator returns the updated protocol state, its ID and all database updates necessary for persisting the updated +// protocol state. +// The StateMutator is used by a replica's compliance layer to update protocol state when observing state-changing service in +// blocks. It is used by the primary in the block building process to obtain the correct protocol state for a proposal. +// Specifically, the leader may include state-changing service events in the block payload. The flow protocol prescribes that +// the proposal needs to include the ID of the protocol state, _after_ processing the payload incl. all state-changing events. +// Therefore, the leader instantiates a StateMutator, applies the service events to it and builds the updated protocol state ID. type StateMutator interface { - // CreateUpdater creates a protocol state updater based on previous protocol state. - // Has to be called for each block to correctly index the protocol state. - // Expected errors during normal operations: - // * `storage.ErrNotFound` if no protocol state for parent block is known. - CreateUpdater(candidateView uint64, parentID flow.Identifier) (StateUpdater, error) - // CommitProtocolState commits the protocol state to the database. - // Has to be called for each block to correctly index the protocol state. - // No errors are expected during normal operations. - CommitProtocolState(blockID flow.Identifier, updater StateUpdater) (func(tx *transaction.Tx) error, flow.Identifier) + // Build returns: + // - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state + // - updatedState: the ProtocolState after applying all updates + // - stateID: the has commitment to the `updatedState` + // - dbUpdates: database updates necessary for persisting the updated protocol state + // updated protocol state entry, state ID and a flag indicating if there were any changes. + Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error, err error) + // ApplyServiceEvents handles applying state changes which occur as a result // of service events being included in a block payload. // All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. - // - // Return values: - // - dbUpdates - If the service events are valid, or there are no service events, - // this method returns a slice of Badger operations to apply while storing the block. - // This includes operations to insert service events for blocks that include them. - // // No errors are expected during normal operation. - ApplyServiceEvents(updater StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) + ApplyServiceEvents(seals []*flow.Seal) error } diff --git a/state/protocol/mock/state_updater.go b/state/protocol/protocol_state/mock/protocol_state_machine.go similarity index 69% rename from state/protocol/mock/state_updater.go rename to state/protocol/protocol_state/mock/protocol_state_machine.go index 66411981821..a197159f938 100644 --- a/state/protocol/mock/state_updater.go +++ b/state/protocol/protocol_state/mock/protocol_state_machine.go @@ -7,13 +7,13 @@ import ( mock "github.com/stretchr/testify/mock" ) -// StateUpdater is an autogenerated mock type for the StateUpdater type -type StateUpdater struct { +// ProtocolStateMachine is an autogenerated mock type for the ProtocolStateMachine type +type ProtocolStateMachine struct { mock.Mock } // Build provides a mock function with given fields: -func (_m *StateUpdater) Build() (*flow.ProtocolStateEntry, flow.Identifier, bool) { +func (_m *ProtocolStateMachine) Build() (*flow.ProtocolStateEntry, flow.Identifier, bool) { ret := _m.Called() var r0 *flow.ProtocolStateEntry @@ -48,7 +48,7 @@ func (_m *StateUpdater) Build() (*flow.ProtocolStateEntry, flow.Identifier, bool } // ParentState provides a mock function with given fields: -func (_m *StateUpdater) ParentState() *flow.RichProtocolStateEntry { +func (_m *ProtocolStateMachine) ParentState() *flow.RichProtocolStateEntry { ret := _m.Called() var r0 *flow.RichProtocolStateEntry @@ -64,7 +64,7 @@ func (_m *StateUpdater) ParentState() *flow.RichProtocolStateEntry { } // ProcessEpochCommit provides a mock function with given fields: epochCommit -func (_m *StateUpdater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { +func (_m *ProtocolStateMachine) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { ret := _m.Called(epochCommit) var r0 error @@ -78,7 +78,7 @@ func (_m *StateUpdater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error } // ProcessEpochSetup provides a mock function with given fields: epochSetup -func (_m *StateUpdater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { +func (_m *ProtocolStateMachine) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { ret := _m.Called(epochSetup) var r0 error @@ -92,12 +92,12 @@ func (_m *StateUpdater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { } // SetInvalidStateTransitionAttempted provides a mock function with given fields: -func (_m *StateUpdater) SetInvalidStateTransitionAttempted() { +func (_m *ProtocolStateMachine) SetInvalidStateTransitionAttempted() { _m.Called() } // TransitionToNextEpoch provides a mock function with given fields: -func (_m *StateUpdater) TransitionToNextEpoch() error { +func (_m *ProtocolStateMachine) TransitionToNextEpoch() error { ret := _m.Called() var r0 error @@ -111,7 +111,7 @@ func (_m *StateUpdater) TransitionToNextEpoch() error { } // UpdateIdentity provides a mock function with given fields: updated -func (_m *StateUpdater) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { +func (_m *ProtocolStateMachine) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { ret := _m.Called(updated) var r0 error @@ -125,7 +125,7 @@ func (_m *StateUpdater) UpdateIdentity(updated *flow.DynamicIdentityEntry) error } // View provides a mock function with given fields: -func (_m *StateUpdater) View() uint64 { +func (_m *ProtocolStateMachine) View() uint64 { ret := _m.Called() var r0 uint64 @@ -138,14 +138,14 @@ func (_m *StateUpdater) View() uint64 { return r0 } -type mockConstructorTestingTNewStateUpdater interface { +type mockConstructorTestingTNewProtocolStateMachine interface { mock.TestingT Cleanup(func()) } -// NewStateUpdater creates a new instance of StateUpdater. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewStateUpdater(t mockConstructorTestingTNewStateUpdater) *StateUpdater { - mock := &StateUpdater{} +// NewProtocolStateMachine creates a new instance of ProtocolStateMachine. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProtocolStateMachine(t mockConstructorTestingTNewProtocolStateMachine) *ProtocolStateMachine { + mock := &ProtocolStateMachine{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 63dec784a61..b72afe584a8 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -3,7 +3,6 @@ package protocol_state import ( "errors" "fmt" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" @@ -16,12 +15,13 @@ import ( // service events sealed in candidate block. This requirement is due to the fact that protocol state // is indexed by block ID, and we need to maintain such index. type Mutator struct { - headers storage.Headers - results storage.ExecutionResults - setups storage.EpochSetups - commits storage.EpochCommits - protocolStateDB storage.ProtocolState - params protocol.InstanceParams + headers storage.Headers + results storage.ExecutionResults + setups storage.EpochSetups + commits storage.EpochCommits + stateMachine ProtocolStateMachine + params protocol.InstanceParams + pendingDbUpdates []func(tx *transaction.Tx) error } var _ protocol.StateMutator = (*Mutator)(nil) @@ -31,54 +31,46 @@ func NewMutator( results storage.ExecutionResults, setups storage.EpochSetups, commits storage.EpochCommits, - protocolStateDB storage.ProtocolState, + stateMachine ProtocolStateMachine, params protocol.InstanceParams, ) *Mutator { return &Mutator{ - headers: headers, - results: results, - setups: setups, - commits: commits, - protocolStateDB: protocolStateDB, - params: params, + setups: setups, + params: params, + headers: headers, + results: results, + commits: commits, + stateMachine: stateMachine, } } -// CreateUpdater creates a new protocol state updater for the given candidate block. -// Has to be called for each block to correctly index the protocol state. -// Expected errors during normal operations: -// - `storage.ErrNotFound` if no protocol state for parent block is known. -func (m *Mutator) CreateUpdater(candidateView uint64, parentID flow.Identifier) (protocol.StateUpdater, error) { - parentState, err := m.protocolStateDB.ByBlockID(parentID) - if err != nil { - return nil, fmt.Errorf("could not retrieve protocol state for block (%v): %w", parentID, err) - } - return NewUpdater(candidateView, parentState), nil +// Build returns: +// - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state +// - updatedState: the ProtocolState after applying all updates +// - stateID: the has commitment to the `updatedState` +// - dbUpdates: database updates necessary for persisting the updated protocol state +// +// updated protocol state entry, state ID and a flag indicating if there were any changes. +func (m *Mutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error, err error) { + updatedState, stateID, hasChanges = m.stateMachine.Build() + dbUpdates = m.pendingDbUpdates + return } -// CommitProtocolState commits the protocol state updater as part of DB transaction. -// Has to be called for each block to correctly index the protocol state. -// No errors are expected during normal operations. -func (m *Mutator) CommitProtocolState(blockID flow.Identifier, updater protocol.StateUpdater) (func(tx *transaction.Tx) error, flow.Identifier) { - updatedState, updatedStateID, hasChanges := updater.Build() - return func(tx *transaction.Tx) error { - if hasChanges { - err := m.protocolStateDB.StoreTx(updatedStateID, updatedState)(tx) - if err != nil && !errors.Is(err, storage.ErrAlreadyExists) { - return fmt.Errorf("could not store protocol state (%v): %w", updatedStateID, err) - } - } - - err := m.protocolStateDB.Index(blockID, updatedStateID)(tx) - if err != nil { - return fmt.Errorf("could not index protocol state (%v) for block (%v): %w", - updatedStateID, blockID, err) - } - return nil - }, updatedStateID +// ApplyServiceEvents handles applying state changes which occur as a result +// of service events being included in a block payload. +// All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. +// No errors are expected during normal operation. +func (m *Mutator) ApplyServiceEvents(seals []*flow.Seal) error { + dbUpdates, err := m.handleServiceEvents(seals) + if err != nil { + return err + } + m.pendingDbUpdates = append(m.pendingDbUpdates, dbUpdates...) + return nil } -// ApplyServiceEvents handles applying state changes which occur as a result +// handleServiceEvents handles applying state changes which occur as a result // of service events being included in a block payload: // - inserting incorporated service events // - updating protocol state for the candidate block @@ -111,13 +103,13 @@ func (m *Mutator) CommitProtocolState(blockID flow.Identifier, updater protocol. // This includes operations to insert service events for blocks that include them. // // No errors are expected during normal operation. -func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { +func (m *Mutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { epochFallbackTriggered, err := m.params.EpochFallbackTriggered() if err != nil { return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) } - parentProtocolState := updater.ParentState() + parentProtocolState := m.stateMachine.ParentState() epochStatus := parentProtocolState.EpochStatus() activeSetup := parentProtocolState.CurrentEpochSetup @@ -132,10 +124,10 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo return nil, fmt.Errorf("could not determine epoch phase: %w", err) } if phase == flow.EpochPhaseCommitted { - if updater.View() > activeSetup.FinalView { + if m.stateMachine.View() > activeSetup.FinalView { // TODO: this is a temporary workaround to allow for the epoch transition to be triggered // most likely it will be not needed when we refactor protocol state entries and define strict safety rules. - err = updater.TransitionToNextEpoch() + err = m.stateMachine.TransitionToNextEpoch() if err != nil { return nil, fmt.Errorf("could not transition protocol state to next epoch: %w", err) } @@ -168,11 +160,11 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo switch ev := event.Event.(type) { case *flow.EpochSetup: - err = updater.ProcessEpochSetup(ev) + err = m.stateMachine.ProcessEpochSetup(ev) if err != nil { if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode - updater.SetInvalidStateTransitionAttempted() + m.stateMachine.SetInvalidStateTransitionAttempted() return dbUpdates, nil } return nil, irrecoverable.NewExceptionf("could not process epoch setup event: %w", err) @@ -182,11 +174,11 @@ func (m *Mutator) ApplyServiceEvents(updater protocol.StateUpdater, seals []*flo dbUpdates = append(dbUpdates, m.setups.StoreTx(ev)) case *flow.EpochCommit: - err = updater.ProcessEpochCommit(ev) + err = m.stateMachine.ProcessEpochCommit(ev) if err != nil { if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode - updater.SetInvalidStateTransitionAttempted() + m.stateMachine.SetInvalidStateTransitionAttempted() return dbUpdates, nil } return nil, irrecoverable.NewExceptionf("could not process epoch commit event: %w", err) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 48983032dea..a7e6ba11159 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -1,20 +1,13 @@ package protocol_state import ( - "errors" "testing" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/mock" - storerr "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/storage/badger/transaction" + protocolstatemock "github.com/onflow/flow-go/state/protocol/protocol_state/mock" storagemock "github.com/onflow/flow-go/storage/mock" - "github.com/onflow/flow-go/utils/rand" - "github.com/onflow/flow-go/utils/unittest" ) func TestProtocolStateMutator(t *testing.T) { @@ -29,6 +22,7 @@ type MutatorSuite struct { setupsDB *storagemock.EpochSetups commitsDB *storagemock.EpochCommits params *mock.InstanceParams + stateMachine *protocolstatemock.ProtocolStateMachine mutator *Mutator } @@ -41,260 +35,260 @@ func (s *MutatorSuite) SetupTest() { s.commitsDB = storagemock.NewEpochCommits(s.T()) s.params = mock.NewInstanceParams(s.T()) - s.mutator = NewMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.protocolStateDB, s.params) + s.mutator = NewMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.stateMachine, s.params) } -// TestCreateUpdaterForUnknownBlock tests that CreateUpdater returns an error if the parent protocol state is not found. -func (s *MutatorSuite) TestCreateUpdaterForUnknownBlock() { - candidate := unittest.BlockHeaderFixture() - s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(nil, storerr.ErrNotFound) - updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) - require.ErrorIs(s.T(), err, storerr.ErrNotFound) - require.Nil(s.T(), updater) -} - -// TestMutatorHappyPathNoChanges tests that Mutator correctly indexes the protocol state when there are no changes. -func (s *MutatorSuite) TestMutatorHappyPathNoChanges() { - parentState := unittest.ProtocolStateFixture() - candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) - s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) - updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) - require.NoError(s.T(), err) - - s.protocolStateDB.On("Index", candidate.ID(), parentState.ID()).Return(func(tx *transaction.Tx) error { return nil }) - - dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) - err = dbOps(&transaction.Tx{}) - require.NoError(s.T(), err) -} - -// TestMutatorHappyPathHasChanges tests that Mutator correctly persists and indexes the protocol state when there are changes. -func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { - parentState := unittest.ProtocolStateFixture() - candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) - s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) - updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) - require.NoError(s.T(), err) - - // update protocol state so it has some changes - updater.SetInvalidStateTransitionAttempted() - updatedState, updatedStateID, hasChanges := updater.Build() - require.True(s.T(), hasChanges) - - s.protocolStateDB.On("StoreTx", updatedStateID, updatedState).Return(func(tx *transaction.Tx) error { return nil }) - s.protocolStateDB.On("Index", candidate.ID(), updatedStateID).Return(func(tx *transaction.Tx) error { return nil }) - - dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) - err = dbOps(&transaction.Tx{}) - require.NoError(s.T(), err) -} - -// TestMutatorApplyServiceEvents_InvalidEpochSetup tests that ApplyServiceEvents rejects invalid epoch setup event and sets -// InvalidStateTransitionAttempted flag in protocol.StateUpdater. -func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - s.Run("invalid-epoch-setup", func() { - parentState := unittest.ProtocolStateFixture() - rootSetup := parentState.CurrentEpochSetup - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - - epochSetup := unittest.EpochSetupFixture( - unittest.WithParticipants(rootSetup.Participants), - unittest.SetupWithCounter(rootSetup.Counter+1), - unittest.WithFinalView(rootSetup.FinalView+1000), - unittest.WithFirstView(rootSetup.FinalView+1), - ) - result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { - result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} - }) - - block := unittest.BlockHeaderFixture() - seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) - s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) - s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) - - updater.On("ProcessEpochSetup", epochSetup).Return(protocol.NewInvalidServiceEventErrorf("")).Once() - updater.On("SetInvalidStateTransitionAttempted").Return().Once() - - updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) - require.NoError(s.T(), err) - require.Empty(s.T(), updates) - }) - s.Run("process-epoch-setup-exception", func() { - parentState := unittest.ProtocolStateFixture() - rootSetup := parentState.CurrentEpochSetup - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - - epochSetup := unittest.EpochSetupFixture( - unittest.WithParticipants(rootSetup.Participants), - unittest.SetupWithCounter(rootSetup.Counter+1), - unittest.WithFinalView(rootSetup.FinalView+1000), - unittest.WithFirstView(rootSetup.FinalView+1), - ) - result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { - result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} - }) - - block := unittest.BlockHeaderFixture() - seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) - s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) - s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) - - exception := errors.New("exception") - updater.On("ProcessEpochSetup", epochSetup).Return(exception).Once() - - updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) - require.Error(s.T(), err) - require.False(s.T(), protocol.IsInvalidServiceEventError(err)) - require.Empty(s.T(), updates) - }) -} - -// TestMutatorApplyServiceEvents_InvalidEpochCommit tests that ApplyServiceEvents rejects invalid epoch commit event and sets -// InvalidStateTransitionAttempted flag in protocol.StateUpdater. -func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochCommit() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - s.Run("invalid-epoch-commit", func() { - parentState := unittest.ProtocolStateFixture() - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - - epochCommit := unittest.EpochCommitFixture() - result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { - result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} - }) - - block := unittest.BlockHeaderFixture() - seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) - s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) - s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) - - updater.On("ProcessEpochCommit", epochCommit).Return(protocol.NewInvalidServiceEventErrorf("")).Once() - updater.On("SetInvalidStateTransitionAttempted").Return().Once() - - updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) - require.NoError(s.T(), err) - require.Empty(s.T(), updates) - }) - s.Run("process-epoch-commit-exception", func() { - parentState := unittest.ProtocolStateFixture() - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - - epochCommit := unittest.EpochCommitFixture() - result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { - result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} - }) - - block := unittest.BlockHeaderFixture() - seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) - s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) - s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) - - exception := errors.New("exception") - updater.On("ProcessEpochCommit", epochCommit).Return(exception).Once() - - updates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{seal}) - require.Error(s.T(), err) - require.False(s.T(), protocol.IsInvalidServiceEventError(err)) - require.Empty(s.T(), updates) - }) -} - -// TestMutatorApplyServiceEventsSealsOrdered tests that ApplyServiceEvents processes seals in order of block height. -func (s *MutatorSuite) TestMutatorApplyServiceEventsSealsOrdered() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - parentState := unittest.ProtocolStateFixture() - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - - blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) - var seals []*flow.Seal - resultByHeight := make(map[flow.Identifier]uint64) - for _, block := range blocks { - receipt, seal := unittest.ReceiptAndSealForBlock(block) - resultByHeight[seal.ResultID] = block.Header.Height - s.headersDB.On("ByBlockID", seal.BlockID).Return(block.Header, nil).Once() - s.resultsDB.On("ByID", seal.ResultID).Return(&receipt.ExecutionResult, nil).Once() - seals = append(seals, seal) - } - - // shuffle seals to make sure we order them - require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { - seals[i], seals[j] = seals[j], seals[i] - })) - - updates, err := s.mutator.ApplyServiceEvents(updater, seals) - require.NoError(s.T(), err) - require.Empty(s.T(), updates) - // assert that results were queried in order of executed block height - // if seals were properly ordered before processing, then results should be ordered by block height - lastExecutedBlockHeight := uint64(0) - for _, call := range s.resultsDB.Calls { - resultID := call.Arguments.Get(0).(flow.Identifier) - executedBlockHeight, found := resultByHeight[resultID] - require.True(s.T(), found) - require.Less(s.T(), lastExecutedBlockHeight, executedBlockHeight, "seals must be ordered by block height") - } -} - -// TestMutatorApplyServiceEventsTransitionToNextEpoch tests that ApplyServiceEvents transitions to the next epoch -// when epoch has been committed, and we are at the first block of the next epoch. -func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - // we are at the first block of the next epoch - updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) - updater.On("TransitionToNextEpoch").Return(nil).Once() - dbUpdates, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{}) - require.NoError(s.T(), err) - require.Empty(s.T(), dbUpdates) -} - -// TestMutatorApplyServiceEventsTransitionToNextEpoch_Error tests that error that has been -// observed in ApplyServiceEvents when transitioning to the next epoch is propagated to the caller. -func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch_Error() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - // we are at the first block of the next epoch - updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) - exception := errors.New("exception") - updater.On("TransitionToNextEpoch").Return(exception).Once() - _, err := s.mutator.ApplyServiceEvents(updater, []*flow.Seal{}) - require.ErrorIs(s.T(), err, exception) - require.False(s.T(), protocol.IsInvalidServiceEventError(err)) -} - -// TestMutatorApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. -// In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. -// In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. -// In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. -func (s *MutatorSuite) TestMutatorApplyServiceEvents_EpochFallbackTriggered() { - s.Run("epoch-fallback-triggered", func() { - s.params.On("EpochFallbackTriggered").Return(true, nil) - parentState := unittest.ProtocolStateFixture() - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - seals := unittest.Seal.Fixtures(2) - updates, err := s.mutator.ApplyServiceEvents(updater, seals) - require.NoError(s.T(), err) - require.Empty(s.T(), updates) - }) - s.Run("invalid-service-event-incorporated", func() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - parentState := unittest.ProtocolStateFixture() - parentState.InvalidStateTransitionAttempted = true - updater := mock.NewStateUpdater(s.T()) - updater.On("ParentState").Return(parentState) - seals := unittest.Seal.Fixtures(2) - updates, err := s.mutator.ApplyServiceEvents(updater, seals) - require.NoError(s.T(), err) - require.Empty(s.T(), updates) - }) -} +//// TestCreateUpdaterForUnknownBlock tests that CreateUpdater returns an error if the parent protocol state is not found. +//func (s *MutatorSuite) TestCreateUpdaterForUnknownBlock() { +// candidate := unittest.BlockHeaderFixture() +// s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(nil, storerr.ErrNotFound) +// updater, err := s.mutator.(candidate.View, candidate.ParentID) +// require.ErrorIs(s.T(), err, storerr.ErrNotFound) +// require.Nil(s.T(), updater) +//} + +//// TestMutatorHappyPathNoChanges tests that Mutator correctly indexes the protocol state when there are no changes. +//func (s *MutatorSuite) TestMutatorHappyPathNoChanges() { +// parentState := unittest.ProtocolStateFixture() +// candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) +// s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) +// updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) +// require.NoError(s.T(), err) +// +// s.protocolStateDB.On("Index", candidate.ID(), parentState.ID()).Return(func(tx *transaction.Tx) error { return nil }) +// +// dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) +// err = dbOps(&transaction.Tx{}) +// require.NoError(s.T(), err) +//} +// +//// TestMutatorHappyPathHasChanges tests that Mutator correctly persists and indexes the protocol state when there are changes. +//func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { +// parentState := unittest.ProtocolStateFixture() +// candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) +// s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) +// updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) +// require.NoError(s.T(), err) +// +// // update protocol state so it has some changes +// updater.SetInvalidStateTransitionAttempted() +// updatedState, updatedStateID, hasChanges := updater.Build() +// require.True(s.T(), hasChanges) +// +// s.protocolStateDB.On("StoreTx", updatedStateID, updatedState).Return(func(tx *transaction.Tx) error { return nil }) +// s.protocolStateDB.On("Index", candidate.ID(), updatedStateID).Return(func(tx *transaction.Tx) error { return nil }) +// +// dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) +// err = dbOps(&transaction.Tx{}) +// require.NoError(s.T(), err) +//} +// +//// TestMutatorApplyServiceEvents_InvalidEpochSetup tests that handleServiceEvents rejects invalid epoch setup event and sets +//// InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. +//func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { +// s.params.On("EpochFallbackTriggered").Return(false, nil) +// s.Run("invalid-epoch-setup", func() { +// parentState := unittest.ProtocolStateFixture() +// rootSetup := parentState.CurrentEpochSetup +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// +// epochSetup := unittest.EpochSetupFixture( +// unittest.WithParticipants(rootSetup.Participants), +// unittest.SetupWithCounter(rootSetup.Counter+1), +// unittest.WithFinalView(rootSetup.FinalView+1000), +// unittest.WithFirstView(rootSetup.FinalView+1), +// ) +// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { +// result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} +// }) +// +// block := unittest.BlockHeaderFixture() +// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) +// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) +// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) +// +// updater.On("ProcessEpochSetup", epochSetup).Return(protocol.NewInvalidServiceEventErrorf("")).Once() +// updater.On("SetInvalidStateTransitionAttempted").Return().Once() +// +// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) +// require.NoError(s.T(), err) +// require.Empty(s.T(), updates) +// }) +// s.Run("process-epoch-setup-exception", func() { +// parentState := unittest.ProtocolStateFixture() +// rootSetup := parentState.CurrentEpochSetup +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// +// epochSetup := unittest.EpochSetupFixture( +// unittest.WithParticipants(rootSetup.Participants), +// unittest.SetupWithCounter(rootSetup.Counter+1), +// unittest.WithFinalView(rootSetup.FinalView+1000), +// unittest.WithFirstView(rootSetup.FinalView+1), +// ) +// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { +// result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} +// }) +// +// block := unittest.BlockHeaderFixture() +// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) +// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) +// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) +// +// exception := errors.New("exception") +// updater.On("ProcessEpochSetup", epochSetup).Return(exception).Once() +// +// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) +// require.Error(s.T(), err) +// require.False(s.T(), protocol.IsInvalidServiceEventError(err)) +// require.Empty(s.T(), updates) +// }) +//} +// +//// TestMutatorApplyServiceEvents_InvalidEpochCommit tests that handleServiceEvents rejects invalid epoch commit event and sets +//// InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. +//func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochCommit() { +// s.params.On("EpochFallbackTriggered").Return(false, nil) +// s.Run("invalid-epoch-commit", func() { +// parentState := unittest.ProtocolStateFixture() +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// +// epochCommit := unittest.EpochCommitFixture() +// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { +// result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} +// }) +// +// block := unittest.BlockHeaderFixture() +// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) +// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) +// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) +// +// updater.On("ProcessEpochCommit", epochCommit).Return(protocol.NewInvalidServiceEventErrorf("")).Once() +// updater.On("SetInvalidStateTransitionAttempted").Return().Once() +// +// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) +// require.NoError(s.T(), err) +// require.Empty(s.T(), updates) +// }) +// s.Run("process-epoch-commit-exception", func() { +// parentState := unittest.ProtocolStateFixture() +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// +// epochCommit := unittest.EpochCommitFixture() +// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { +// result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} +// }) +// +// block := unittest.BlockHeaderFixture() +// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) +// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) +// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) +// +// exception := errors.New("exception") +// updater.On("ProcessEpochCommit", epochCommit).Return(exception).Once() +// +// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) +// require.Error(s.T(), err) +// require.False(s.T(), protocol.IsInvalidServiceEventError(err)) +// require.Empty(s.T(), updates) +// }) +//} +// +//// TestMutatorApplyServiceEventsSealsOrdered tests that handleServiceEvents processes seals in order of block height. +//func (s *MutatorSuite) TestMutatorApplyServiceEventsSealsOrdered() { +// s.params.On("EpochFallbackTriggered").Return(false, nil) +// parentState := unittest.ProtocolStateFixture() +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// +// blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) +// var seals []*flow.Seal +// resultByHeight := make(map[flow.Identifier]uint64) +// for _, block := range blocks { +// receipt, seal := unittest.ReceiptAndSealForBlock(block) +// resultByHeight[seal.ResultID] = block.Header.Height +// s.headersDB.On("ByBlockID", seal.BlockID).Return(block.Header, nil).Once() +// s.resultsDB.On("ByID", seal.ResultID).Return(&receipt.ExecutionResult, nil).Once() +// seals = append(seals, seal) +// } +// +// // shuffle seals to make sure we order them +// require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { +// seals[i], seals[j] = seals[j], seals[i] +// })) +// +// updates, err := s.mutator.handleServiceEvents(updater, seals) +// require.NoError(s.T(), err) +// require.Empty(s.T(), updates) +// // assert that results were queried in order of executed block height +// // if seals were properly ordered before processing, then results should be ordered by block height +// lastExecutedBlockHeight := uint64(0) +// for _, call := range s.resultsDB.Calls { +// resultID := call.Arguments.Get(0).(flow.Identifier) +// executedBlockHeight, found := resultByHeight[resultID] +// require.True(s.T(), found) +// require.Less(s.T(), lastExecutedBlockHeight, executedBlockHeight, "seals must be ordered by block height") +// } +//} +// +//// TestMutatorApplyServiceEventsTransitionToNextEpoch tests that handleServiceEvents transitions to the next epoch +//// when epoch has been committed, and we are at the first block of the next epoch. +//func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch() { +// s.params.On("EpochFallbackTriggered").Return(false, nil) +// parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// // we are at the first block of the next epoch +// updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) +// updater.On("TransitionToNextEpoch").Return(nil).Once() +// dbUpdates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{}) +// require.NoError(s.T(), err) +// require.Empty(s.T(), dbUpdates) +//} +// +//// TestMutatorApplyServiceEventsTransitionToNextEpoch_Error tests that error that has been +//// observed in handleServiceEvents when transitioning to the next epoch is propagated to the caller. +//func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch_Error() { +// s.params.On("EpochFallbackTriggered").Return(false, nil) +// parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// // we are at the first block of the next epoch +// updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) +// exception := errors.New("exception") +// updater.On("TransitionToNextEpoch").Return(exception).Once() +// _, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{}) +// require.ErrorIs(s.T(), err, exception) +// require.False(s.T(), protocol.IsInvalidServiceEventError(err)) +//} +// +//// TestMutatorApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. +//// In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. +//// In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. +//// In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. +//func (s *MutatorSuite) TestMutatorApplyServiceEvents_EpochFallbackTriggered() { +// s.Run("epoch-fallback-triggered", func() { +// s.params.On("EpochFallbackTriggered").Return(true, nil) +// parentState := unittest.ProtocolStateFixture() +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// seals := unittest.Seal.Fixtures(2) +// updates, err := s.mutator.handleServiceEvents(updater, seals) +// require.NoError(s.T(), err) +// require.Empty(s.T(), updates) +// }) +// s.Run("invalid-service-event-incorporated", func() { +// s.params.On("EpochFallbackTriggered").Return(false, nil) +// parentState := unittest.ProtocolStateFixture() +// parentState.InvalidStateTransitionAttempted = true +// updater := mock.NewStateUpdater(s.T()) +// updater.On("ParentState").Return(parentState) +// seals := unittest.Seal.Fixtures(2) +// updates, err := s.mutator.handleServiceEvents(updater, seals) +// require.NoError(s.T(), err) +// require.Empty(s.T(), updates) +// }) +//} diff --git a/state/protocol/protocol_state/protocol_state.go b/state/protocol/protocol_state/protocol_state.go index 05764e44494..7797197c507 100644 --- a/state/protocol/protocol_state/protocol_state.go +++ b/state/protocol/protocol_state/protocol_state.go @@ -2,7 +2,6 @@ package protocol_state import ( "fmt" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/inmem" @@ -44,3 +43,46 @@ func (s *ProtocolState) AtBlockID(blockID flow.Identifier) (protocol.DynamicProt func (s *ProtocolState) GlobalParams() protocol.GlobalParams { return s.globalParams } + +type MutableProtocolState struct { + ProtocolState + headers storage.Headers + results storage.ExecutionResults + setups storage.EpochSetups + commits storage.EpochCommits + params protocol.InstanceParams +} + +var _ protocol.MutableProtocolState = (*MutableProtocolState)(nil) + +func NewMutableProtocolState( + protocolStateDB storage.ProtocolState, + globalParams protocol.GlobalParams, + headers storage.Headers, + results storage.ExecutionResults, + setups storage.EpochSetups, + commits storage.EpochCommits, + params protocol.InstanceParams, +) *MutableProtocolState { + return &MutableProtocolState{ + ProtocolState: *NewProtocolState(protocolStateDB, globalParams), + headers: headers, + results: results, + setups: setups, + commits: commits, + params: params, + } +} + +// Mutator instantiates a `StateMutator` based on the previous protocol state. +// Has to be called for each block to evolve the protocol state. +// Expected errors during normal operations: +// - `storage.ErrNotFound` if no protocol state for parent block is known. +func (s *MutableProtocolState) Mutator(candidateView uint64, parentID flow.Identifier) (protocol.StateMutator, error) { + parentState, err := s.protocolStateDB.ByBlockID(parentID) + if err != nil { + return nil, fmt.Errorf("could not query parent protocol state at block (%x): %w", parentID, err) + } + stateMachine := NewStateMachine(candidateView, parentState) + return NewMutator(s.headers, s.results, s.setups, s.commits, stateMachine, s.params), nil +} diff --git a/state/protocol/protocol_state/statemachine.go b/state/protocol/protocol_state/statemachine.go new file mode 100644 index 00000000000..dcfded65209 --- /dev/null +++ b/state/protocol/protocol_state/statemachine.go @@ -0,0 +1,46 @@ +package protocol_state + +import "github.com/onflow/flow-go/model/flow" + +// ProtocolStateMachine is a dedicated interface for updating protocol state. +// It is used by the compliance layer to update protocol state when certain events that are stored in blocks are observed. +type ProtocolStateMachine interface { + // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. + Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) + // ProcessEpochSetup updates current protocol state with data from epoch setup event. + // Processing epoch setup event also affects identity table for current epoch. + // Observing an epoch setup event, transitions protocol state from staking to setup phase, we stop returning + // identities from previous+current epochs and start returning identities from current+next epochs. + // As a result of this operation protocol state for the next epoch will be created. + // Expected errors during normal operations: + // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state + ProcessEpochSetup(epochSetup *flow.EpochSetup) error + // ProcessEpochCommit updates current protocol state with data from epoch commit event. + // Observing an epoch setup commit, transitions protocol state from setup to commit phase, at this point we have + // finished construction of the next epoch. + // As a result of this operation protocol state for next epoch will be committed. + // Expected errors during normal operations: + // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state + ProcessEpochCommit(epochCommit *flow.EpochCommit) error + // UpdateIdentity updates identity table with new identity entry. + // Should pass identity which is already present in the table, otherwise an exception will be raised. + // TODO: This function currently modifies both current+next identities based on input. + // This is incompatible with the design doc, and needs to be updated to modify current/next epoch separately + // No errors are expected during normal operations. + UpdateIdentity(updated *flow.DynamicIdentityEntry) error + // SetInvalidStateTransitionAttempted sets a flag indicating that invalid state transition was attempted. + // Such transition can be detected by compliance layer. + SetInvalidStateTransitionAttempted() + // TransitionToNextEpoch discards current protocol state and transitions to the next epoch. + // Epoch transition is only allowed when: + // - next epoch has been set up, + // - next epoch has been committed, + // - candidate block is in the next epoch. + // No errors are expected during normal operations. + TransitionToNextEpoch() error + // View returns the view that is associated with this state updater. + // The view of the ProtocolStateMachine equals the view of the block carrying the respective updates. + View() uint64 + // ParentState returns parent protocol state that is associated with this state updater. + ParentState() *flow.RichProtocolStateEntry +} diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 8dea0f2c61e..441e3a2f150 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -2,13 +2,12 @@ package protocol_state import ( "fmt" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/state/protocol" ) -// Updater is a dedicated structure that encapsulates all logic for updating protocol state. +// StateMachine is a dedicated structure that encapsulates all logic for updating protocol state. // Only protocol state updater knows how to update protocol state in a way that is consistent with the protocol. // Protocol state updater implements the following state changes: // - epoch setup: transitions current epoch from staking to setup phase, creates next epoch protocol state when processed. @@ -17,7 +16,7 @@ import ( // - setting an invalid state transition flag: sets an invalid state transition flag for current epoch and next epoch(if available). // All updates are applied to a copy of parent protocol state, so parent protocol state is not modified. // It is NOT safe to use in concurrent environment. -type Updater struct { +type StateMachine struct { parentState *flow.RichProtocolStateEntry state *flow.ProtocolStateEntry view uint64 @@ -32,11 +31,11 @@ type Updater struct { nextEpochIdentitiesLookup map[flow.Identifier]*flow.DynamicIdentityEntry // lookup for nodes active in the next epoch, may be nil or empty } -var _ protocol.StateUpdater = (*Updater)(nil) +var _ ProtocolStateMachine = (*StateMachine)(nil) -// NewUpdater creates a new protocol state updater. -func NewUpdater(view uint64, parentState *flow.RichProtocolStateEntry) *Updater { - updater := &Updater{ +// NewStateMachine creates a new protocol state updater. +func NewStateMachine(view uint64, parentState *flow.RichProtocolStateEntry) *StateMachine { + updater := &StateMachine{ parentState: parentState, state: parentState.ProtocolStateEntry.Copy(), view: view, @@ -45,7 +44,7 @@ func NewUpdater(view uint64, parentState *flow.RichProtocolStateEntry) *Updater } // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. -func (u *Updater) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) { +func (u *StateMachine) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) { updatedState = u.state.Copy() stateID = updatedState.ID() hasChanges = stateID != u.parentState.ID() @@ -60,7 +59,7 @@ func (u *Updater) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.I // As a result of this operation protocol state for the next epoch will be created. // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state -func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { +func (u *StateMachine) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are in epoch fallback mode. } @@ -144,7 +143,7 @@ func (u *Updater) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { // As a result of this operation protocol state for next epoch will be committed. // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state -func (u *Updater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { +func (u *StateMachine) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are going to enter epoch fallback mode } @@ -170,7 +169,7 @@ func (u *Updater) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { // UpdateIdentity updates identity table with new identity entry. // Should pass identity which is already present in the table, otherwise an exception will be raised. // No errors are expected during normal operations. -func (u *Updater) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { +func (u *StateMachine) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { u.ensureLookupPopulated() prevEpochIdentity, foundInPrev := u.prevEpochIdentitiesLookup[updated.NodeID] if foundInPrev { @@ -191,7 +190,7 @@ func (u *Updater) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { } // SetInvalidStateTransitionAttempted sets a flag indicating that invalid state transition was attempted. -func (u *Updater) SetInvalidStateTransitionAttempted() { +func (u *StateMachine) SetInvalidStateTransitionAttempted() { u.state.InvalidStateTransitionAttempted = true } @@ -202,7 +201,7 @@ func (u *Updater) SetInvalidStateTransitionAttempted() { // - invalid state transition has not been attempted, // - candidate block is in the next epoch. // No errors are expected during normal operations. -func (u *Updater) TransitionToNextEpoch() error { +func (u *StateMachine) TransitionToNextEpoch() error { if u.state.InvalidStateTransitionAttempted { return fmt.Errorf("invalid state transition has been attempted, no transition is allowed") } @@ -229,19 +228,19 @@ func (u *Updater) TransitionToNextEpoch() error { } // View returns the view that is associated with this state updater. -// StateUpdater is created for a view where protocol state changes will be applied. -func (u *Updater) View() uint64 { +// The view of the StateUpdater equals the view of the block carrying the respective updates. +func (u *StateMachine) View() uint64 { return u.view } // ParentState returns parent protocol state that is associated with this state updater. -func (u *Updater) ParentState() *flow.RichProtocolStateEntry { +func (u *StateMachine) ParentState() *flow.RichProtocolStateEntry { return u.parentState } // ensureLookupPopulated ensures that current and next epoch identities lookups are populated. // We use this to avoid populating lookups on every UpdateIdentity call. -func (u *Updater) ensureLookupPopulated() { +func (u *StateMachine) ensureLookupPopulated() { if len(u.currentEpochIdentitiesLookup) > 0 { return } @@ -251,7 +250,7 @@ func (u *Updater) ensureLookupPopulated() { // rebuildIdentityLookup re-generates lookups of *active* participants for // previous (optional, if u.state.PreviousEpoch ≠ nil), current (required) and // next epoch (optional, if u.state.NextEpoch ≠ nil). -func (u *Updater) rebuildIdentityLookup() { +func (u *StateMachine) rebuildIdentityLookup() { if u.state.PreviousEpoch != nil { u.prevEpochIdentitiesLookup = u.state.PreviousEpoch.ActiveIdentities.Lookup() } else { diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index 43a795c6a1f..139b0d1cb81 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -25,7 +25,7 @@ type UpdaterSuite struct { parentBlock *flow.Header candidate *flow.Header - updater *Updater + updater *StateMachine } func (s *UpdaterSuite) SetupTest() { @@ -33,7 +33,7 @@ func (s *UpdaterSuite) SetupTest() { s.parentBlock = unittest.BlockHeaderFixture(unittest.HeaderWithView(s.parentProtocolState.CurrentEpochSetup.FirstView + 1)) s.candidate = unittest.BlockHeaderWithParentFixture(s.parentBlock) - s.updater = NewUpdater(s.candidate.View, s.parentProtocolState) + s.updater = NewStateMachine(s.candidate.View, s.parentProtocolState) } // TestNewUpdater tests if the constructor correctly setups invariants for updater. @@ -54,7 +54,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpoch() { candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(s.parentProtocolState.CurrentEpochSetup.FinalView + 1)) // since the candidate block is from next epoch, updater should transition to next epoch - s.updater = NewUpdater(candidate.View, s.parentProtocolState) + s.updater = NewStateMachine(candidate.View, s.parentProtocolState) err := s.updater.TransitionToNextEpoch() require.NoError(s.T(), err) updatedState, _, _ := s.updater.Build() @@ -68,7 +68,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { protocolState := unittest.ProtocolStateFixture() candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewUpdater(candidate.View, protocolState) + updater := NewStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if there is no next epoch protocol state") }) @@ -79,7 +79,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { }) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewUpdater(candidate.View, protocolState) + updater := NewStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if it is not committed") }) @@ -89,7 +89,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { }) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewUpdater(candidate.View, protocolState) + updater := NewStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") }) @@ -97,7 +97,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView)) - updater := NewUpdater(candidate.View, protocolState) + updater := NewStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") }) @@ -124,7 +124,7 @@ func (s *UpdaterSuite) TestSetInvalidStateTransitionAttempted() { // update protocol state with next epoch information unittest.WithNextEpochProtocolState()(s.parentProtocolState) // create new updater with next epoch information - s.updater = NewUpdater(s.candidate.View, s.parentProtocolState) + s.updater = NewStateMachine(s.candidate.View, s.parentProtocolState) s.updater.SetInvalidStateTransitionAttempted() updatedState, _, hasChanges := s.updater.Build() @@ -152,7 +152,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("invalid state transition attempted", func() { - updater := NewUpdater(s.candidate.View, s.parentProtocolState) + updater := NewStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 setup.FirstView = s.parentProtocolState.CurrentEpochSetup.FinalView + 1 @@ -176,7 +176,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { require.Equal(s.T(), flow.ZeroID, newState.NextEpoch.CommitID, "operation must be no-op") }) s.Run("happy path processing", func() { - updater := NewUpdater(s.candidate.View, s.parentProtocolState) + updater := NewStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture( unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), @@ -197,7 +197,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { nil, ) require.NoError(s.T(), err) - updater = NewUpdater(s.candidate.View+1, parentState) + updater = NewStateMachine(s.candidate.View+1, parentState) commit := unittest.EpochCommitFixture( unittest.CommitWithCounter(setup.Counter), unittest.WithDKGFromParticipants(setup.Participants), @@ -233,7 +233,7 @@ func (s *UpdaterSuite) TestUpdateIdentityUnknownIdentity() { func (s *UpdaterSuite) TestUpdateIdentityHappyPath() { // update protocol state to have next epoch protocol state unittest.WithNextEpochProtocolState()(s.parentProtocolState) - s.updater = NewUpdater(s.candidate.View, s.parentProtocolState) + s.updater = NewStateMachine(s.candidate.View, s.parentProtocolState) currentEpochParticipants := s.parentProtocolState.CurrentEpochIdentityTable.Copy() weightChanges, err := currentEpochParticipants.Sample(3) @@ -291,7 +291,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("invalid state transition attempted", func() { - updater := NewUpdater(s.candidate.View, s.parentProtocolState) + updater := NewStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 }) @@ -303,7 +303,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.Nil(s.T(), updatedState.NextEpoch, "should not process epoch setup if invalid state transition attempted") }) s.Run("processing second epoch setup", func() { - updater := NewUpdater(s.candidate.View, s.parentProtocolState) + updater := NewStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture( unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), @@ -317,7 +317,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("participants not sorted", func() { - updater := NewUpdater(s.candidate.View, s.parentProtocolState) + updater := NewStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 var err error @@ -332,7 +332,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { conflictingIdentity := s.parentProtocolState.ProtocolStateEntry.CurrentEpoch.ActiveIdentities[0] conflictingIdentity.Dynamic.Ejected = true - updater := NewUpdater(s.candidate.View, s.parentProtocolState) + updater := NewStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 // using same identities as in previous epoch should result in an error since @@ -469,7 +469,7 @@ func (s *UpdaterSuite) TestEpochSetupAfterIdentityChange() { // now we can use it to construct updater for next block, which will process epoch setup event. nextBlock := unittest.BlockHeaderWithParentFixture(s.candidate) - s.updater = NewUpdater(nextBlock.View, updatedRichProtocolState) + s.updater = NewStateMachine(nextBlock.View, updatedRichProtocolState) setup := unittest.EpochSetupFixture( unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index 04f3ff03711..49099400b50 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -70,29 +70,29 @@ func (epoch EpochHeights) CommittedRange() []uint64 { // EpochBuilder is a testing utility for building epochs into chain state. type EpochBuilder struct { - t *testing.T - mutator protocol.StateMutator - states []protocol.FollowerState - blocksByID map[flow.Identifier]*flow.Block - blocks []*flow.Block - built map[uint64]*EpochHeights - setupOpts []func(*flow.EpochSetup) // options to apply to the EpochSetup event - commitOpts []func(*flow.EpochCommit) // options to apply to the EpochCommit event + t *testing.T + mutableProtocolState protocol.MutableProtocolState + states []protocol.FollowerState + blocksByID map[flow.Identifier]*flow.Block + blocks []*flow.Block + built map[uint64]*EpochHeights + setupOpts []func(*flow.EpochSetup) // options to apply to the EpochSetup event + commitOpts []func(*flow.EpochCommit) // options to apply to the EpochCommit event } // NewEpochBuilder returns a new EpochBuilder which will build epochs using the // given states. At least one state must be provided. If more than one are // provided they must have the same initial state. -func NewEpochBuilder(t *testing.T, mutator protocol.StateMutator, states ...protocol.FollowerState) *EpochBuilder { +func NewEpochBuilder(t *testing.T, mutator protocol.MutableProtocolState, states ...protocol.FollowerState) *EpochBuilder { require.True(t, len(states) >= 1, "must provide at least one state") builder := &EpochBuilder{ - t: t, - mutator: mutator, - states: states, - blocksByID: make(map[flow.Identifier]*flow.Block), - blocks: make([]*flow.Block, 0), - built: make(map[uint64]*EpochHeights), + t: t, + mutableProtocolState: mutator, + states: states, + blocksByID: make(map[flow.Identifier]*flow.Block), + blocks: make([]*flow.Block, 0), + built: make(map[uint64]*EpochHeights), } return builder } @@ -368,13 +368,14 @@ func (builder *EpochBuilder) BuildBlocks(n uint) { // addBlock adds the given block to the state by: extending the state, // finalizing the block, marking the block as valid, and caching the block. func (builder *EpochBuilder) addBlock(block *flow.Block) { - updater, err := builder.mutator.CreateUpdater(block.Header.View, block.Header.ParentID) + stateMutator, err := builder.mutableProtocolState.Mutator(block.Header.View, block.Header.ParentID) require.NoError(builder.t, err) - _, err = builder.mutator.ApplyServiceEvents(updater, block.Payload.Seals) + err = stateMutator.ApplyServiceEvents(block.Payload.Seals) require.NoError(builder.t, err) - _, updatedStateId, _ := updater.Build() + _, _, updatedStateId, _, err := stateMutator.Build() + require.NoError(builder.t, err) block.Payload.ProtocolStateID = updatedStateId block.Header.PayloadHash = block.Payload.Hash() diff --git a/utils/unittest/protocol_state.go b/utils/unittest/protocol_state.go index dce5f2ba9ee..e6d8f835073 100644 --- a/utils/unittest/protocol_state.go +++ b/utils/unittest/protocol_state.go @@ -73,7 +73,7 @@ func FinalizedProtocolStateWithParticipants(participants flow.IdentityList) ( // a receipt for the block (BR), the second (BS) containing a seal for the block. // B <- BR(Result_B) <- BS(Seal_B) // Returns the two generated blocks. -func SealBlock(t *testing.T, st protocol.ParticipantState, mutator protocol.StateMutator, block *flow.Block, receipt *flow.ExecutionReceipt, seal *flow.Seal) (br *flow.Block, bs *flow.Block) { +func SealBlock(t *testing.T, st protocol.ParticipantState, mutableProtocolState protocol.MutableProtocolState, block *flow.Block, receipt *flow.ExecutionReceipt, seal *flow.Seal) (br *flow.Block, bs *flow.Block) { block2 := BlockWithParentFixture(block.Header) block2.SetPayload(flow.Payload{ @@ -85,12 +85,14 @@ func SealBlock(t *testing.T, st protocol.ParticipantState, mutator protocol.Stat require.NoError(t, err) block3 := BlockWithParentFixture(block2.Header) - updater, err := mutator.CreateUpdater(block3.Header.View, block3.Header.ParentID) + stateMutator, err := mutableProtocolState.Mutator(block3.Header.View, block3.Header.ParentID) require.NoError(t, err) seals := []*flow.Seal{seal} - _, err = mutator.ApplyServiceEvents(updater, seals) + err = stateMutator.ApplyServiceEvents(seals) require.NoError(t, err) - _, updatedStateId, _ := updater.Build() + _, _, updatedStateId, _, err := stateMutator.Build() + require.NoError(t, err) + block3.SetPayload(flow.Payload{ Seals: seals, ProtocolStateID: updatedStateId, From 8a733b30abcb027d12edc872860b6ba743475465 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 31 Oct 2023 13:45:44 +0200 Subject: [PATCH 48/87] Fixed tests for state/protocol/* --- state/protocol/badger/mutator.go | 4 +- state/protocol/badger/mutator_test.go | 74 +++++++++++++------------- state/protocol/badger/snapshot_test.go | 22 ++++---- state/protocol/badger/state.go | 4 +- state/protocol/badger/state_test.go | 34 ++++++------ state/protocol/inmem/convert_test.go | 4 +- state/protocol/util/testing.go | 18 ++++--- 7 files changed, 83 insertions(+), 77 deletions(-) diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index 1f835d7ae38..a753c7cb52b 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -529,9 +529,9 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi "payload contains (%x) but after applying changes got %x", candidate.Payload.ProtocolStateID, updatedStateID) } if hasChanges { - dbUpdates = append(dbUpdates, m.protocolStateSnapshotsDB.StoreTx(updatedStateID, updatedState)) - dbUpdates = append(dbUpdates, m.protocolStateSnapshotsDB.Index(blockID, updatedStateID)) + dbUpdates = append(dbUpdates, operation.SkipDuplicatesTx(m.protocolStateSnapshotsDB.StoreTx(updatedStateID, updatedState))) } + dbUpdates = append(dbUpdates, m.protocolStateSnapshotsDB.Index(blockID, updatedStateID)) // events is a queue of node-internal events (aka notifications) that are emitted after the database write succeeded var events []func() diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index c811cbaa52c..a2f4ff8231e 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -586,7 +586,7 @@ func TestExtendHeightTooLarge(t *testing.T) { }) } -// TestExtendInconsistentParentView tests if mutator rejects block with invalid ParentView. ParentView must be consistent +// TestExtendInconsistentParentView tests if mutableState rejects block with invalid ParentView. ParentView must be consistent // with view of block referred by ParentID. func TestExtendInconsistentParentView(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) @@ -861,15 +861,16 @@ func TestExtendEpochTransitionValid(t *testing.T) { ) require.NoError(t, err) - stateMutator := protocol_state.NewMutator( + mutableProtocolState := protocol_state.NewMutableProtocolState( + all.ProtocolState, + state.Params(), all.Headers, all.Results, all.Setups, all.EpochCommits, - all.ProtocolState, state.Params(), ) - calculateExpectedStateId := calculateExpectedStateId(t, stateMutator) + calculateExpectedStateId := calculateExpectedStateId(t, mutableProtocolState) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -1121,8 +1122,8 @@ func TestExtendEpochTransitionValid(t *testing.T) { func TestExtendConflictingEpochEvents(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, stateMutator realprotocol.StateMutator) { - calculateExpectedStateId := calculateExpectedStateId(t, stateMutator) + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + calculateExpectedStateId := calculateExpectedStateId(t, mutableState) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -1239,8 +1240,8 @@ func TestExtendConflictingEpochEvents(t *testing.T) { func TestExtendDuplicateEpochEvents(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) rootProtocolStateID := getRootProtocolStateID(t, rootSnapshot) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { - calculateExpectedStateId := calculateExpectedStateId(t, mutator) + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + calculateExpectedStateId := calculateExpectedStateId(t, mutableState) head, err := rootSnapshot.Head() require.NoError(t, err) @@ -1394,14 +1395,14 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // expect a setup event with wrong counter to trigger EECC without error t.Run("wrong counter (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.Counter = rand.Uint64() }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1415,14 +1416,14 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // expect a setup event with wrong final view to trigger EECC without error t.Run("invalid final view (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.FinalView = block1.Header.View }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1436,14 +1437,14 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // expect a setup event with empty seed to trigger EECC without error t.Run("empty seed (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.RandomSource = nil }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1456,7 +1457,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { }) t.Run("participants not ordered (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup := setupState(t, db, state) _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { @@ -1465,7 +1466,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { require.NoError(t, err) }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1546,12 +1547,12 @@ func TestExtendEpochCommitInvalid(t *testing.T) { } t.Run("without setup (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, _, createCommit := setupState(t, state) _, receipt, seal := createCommit(block1) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block1, receipt, seal) err := state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1565,12 +1566,12 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // expect a commit event with wrong counter to trigger EECC without error t.Run("inconsistent counter (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup, createCommit := setupState(t, state) // seal block 1, in which EpochSetup was emitted epoch2Setup, setupReceipt, setupSeal := createSetup(block1) - epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutator, block1, setupReceipt, setupSeal) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutableState, block1, setupReceipt, setupSeal) err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) require.NoError(t, err) err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) @@ -1584,7 +1585,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { commit.Counter = epoch2Setup.Counter + 1 }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block3, receipt, seal) err = state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1598,12 +1599,12 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // expect a commit event with wrong cluster QCs to trigger EECC without error t.Run("inconsistent cluster QCs (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup, createCommit := setupState(t, state) // seal block 1, in which EpochSetup was emitted _, setupReceipt, setupSeal := createSetup(block1) - epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutator, block1, setupReceipt, setupSeal) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutableState, block1, setupReceipt, setupSeal) err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) require.NoError(t, err) err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) @@ -1617,7 +1618,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(unittest.QuorumCertificateWithSignerIDsFixture())) }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block3, receipt, seal) err = state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1631,12 +1632,12 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // expect a commit event with wrong dkg participants to trigger EECC without error t.Run("inconsistent DKG participants (EECC)", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { block1, createSetup, createCommit := setupState(t, state) // seal block 1, in which EpochSetup was emitted _, setupReceipt, setupSeal := createSetup(block1) - epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutator, block1, setupReceipt, setupSeal) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, mutableState, block1, setupReceipt, setupSeal) err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) require.NoError(t, err) err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) @@ -1651,7 +1652,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) }) - receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutator, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, mutableState, block3, receipt, seal) err = state.Finalize(context.Background(), receiptBlock.ID()) require.NoError(t, err) // epoch fallback not triggered before finalization @@ -1755,7 +1756,7 @@ func TestEmergencyEpochFallback(t *testing.T) { protoEventsMock.On("BlockFinalized", mock.Anything) protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) - util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, _ realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, _ realprotocol.MutableProtocolState) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1815,7 +1816,7 @@ func TestEmergencyEpochFallback(t *testing.T) { protoEventsMock.On("BlockFinalized", mock.Anything) protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) - util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1868,7 +1869,7 @@ func TestEmergencyEpochFallback(t *testing.T) { seals := []*flow.Seal{seal1} block3.SetPayload(flow.Payload{ Seals: seals, - ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), + ProtocolStateID: calculateExpectedStateId(t, mutableState)(block3.Header, seals), }) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -1914,7 +1915,7 @@ func TestEmergencyEpochFallback(t *testing.T) { protoEventsMock.On("BlockFinalized", mock.Anything) protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything) - util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, mutator realprotocol.StateMutator) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1964,7 +1965,7 @@ func TestEmergencyEpochFallback(t *testing.T) { seals := []*flow.Seal{seal1} block3.SetPayload(flow.Payload{ Seals: seals, - ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), + ProtocolStateID: calculateExpectedStateId(t, mutableState)(block3.Header, seals), }) err = state.Extend(context.Background(), block3) require.NoError(t, err) @@ -2678,15 +2679,16 @@ func getRootProtocolStateID(t *testing.T, rootSnapshot *inmem.Snapshot) flow.Ide } // calculateExpectedStateId is a utility function which makes easier to get expected protocol state ID after applying service events contained in seals. -func calculateExpectedStateId(t *testing.T, stateMutator realprotocol.StateMutator) func(header *flow.Header, seals []*flow.Seal) flow.Identifier { +func calculateExpectedStateId(t *testing.T, mutableProtocolState realprotocol.MutableProtocolState) func(header *flow.Header, seals []*flow.Seal) flow.Identifier { return func(header *flow.Header, seals []*flow.Seal) flow.Identifier { - updater, err := stateMutator.CreateUpdater(header.View, header.ParentID) + stateMutator, err := mutableProtocolState.Mutator(header.View, header.ParentID) require.NoError(t, err) - _, err = stateMutator.ApplyServiceEvents(updater, seals) + err = stateMutator.ApplyServiceEvents(seals) require.NoError(t, err) - _, expectedStateID, _ := updater.Build() + _, _, expectedStateID, _, err := stateMutator.Build() + require.NoError(t, err) return expectedStateID } } diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index c104956a13d..b5fb77ca89f 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -316,7 +316,7 @@ func TestSealingSegment(t *testing.T) { // ROOT <- B1 <- B2(R1) <- B3(S1) // Expected sealing segment: [B1, B2, B3], extra blocks: [ROOT] t.Run("non-root", func(t *testing.T) { - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) @@ -337,7 +337,7 @@ func TestSealingSegment(t *testing.T) { seals := []*flow.Seal{seal1} block3.SetPayload(flow.Payload{ Seals: seals, - ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), + ProtocolStateID: calculateExpectedStateId(t, mutableState)(block3.Header, seals), }) buildFinalizedBlock(t, state, block3) @@ -774,7 +774,7 @@ func TestSealingSegment_FailureCases(t *testing.T) { unittest.WithProtocolStateID(rootProtocolStateID), )) - multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { for _, b := range []*flow.Block{b1, b2, b3} { buildFinalizedBlock(t, state, b) } @@ -1125,11 +1125,11 @@ func TestSnapshot_EpochQuery(t *testing.T) { result, _, err := rootSnapshot.SealedResult() require.NoError(t, err) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { epoch1Counter := result.ServiceEvents[0].Event.(*flow.EpochSetup).Counter epoch2Counter := epoch1Counter + 1 - epochBuilder := unittest.NewEpochBuilder(t, mutator, state) + epochBuilder := unittest.NewEpochBuilder(t, mutableState, state) // build epoch 1 (prepare epoch 2) epochBuilder. BuildEpoch(). @@ -1216,9 +1216,9 @@ func TestSnapshot_EpochFirstView(t *testing.T) { result, _, err := rootSnapshot.SealedResult() require.NoError(t, err) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { - epochBuilder := unittest.NewEpochBuilder(t, mutator, state) + epochBuilder := unittest.NewEpochBuilder(t, mutableState, state) // build epoch 1 (prepare epoch 2) epochBuilder. BuildEpoch(). @@ -1297,9 +1297,9 @@ func TestSnapshot_EpochHeightBoundaries(t *testing.T) { head, err := rootSnapshot.Head() require.NoError(t, err) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { - epochBuilder := unittest.NewEpochBuilder(t, mutator, state) + epochBuilder := unittest.NewEpochBuilder(t, mutableState, state) epoch1FirstHeight := head.Height t.Run("first epoch - EpochStaking phase", func(t *testing.T) { @@ -1378,9 +1378,9 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { epoch3Identities := unittest.IdentityListFixture(10, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(epoch1Identities) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { - epochBuilder := unittest.NewEpochBuilder(t, mutator, state) + epochBuilder := unittest.NewEpochBuilder(t, mutableState, state) // build epoch 1 (prepare epoch 2) epochBuilder. UsingSetupOpts(unittest.WithParticipants(epoch2Identities.ToSkeleton())). diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index f820a6dca8b..aabd6d65c6a 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -37,6 +37,7 @@ type State struct { setups storage.EpochSetups commits storage.EpochCommits } + globalParams protocol.GlobalParams protocolStateSnapshotsDB storage.ProtocolState protocolState protocol.MutableProtocolState versionBeacons storage.VersionBeacons @@ -689,7 +690,7 @@ func OpenState( func (state *State) Params() protocol.Params { return Params{ - GlobalParams: state.protocolState.GlobalParams(), + GlobalParams: state.globalParams, InstanceParams: &InstanceParams{state: state}, } } @@ -784,6 +785,7 @@ func newState( setups: setups, commits: commits, }, + globalParams: globalParams, protocolStateSnapshotsDB: protocolStateSnapshots, versionBeacons: versionBeacons, cachedFinal: new(atomic.Pointer[cachedHeader]), diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 8553023c620..07156aec385 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -101,8 +101,8 @@ func TestBootstrapAndOpen_EpochCommitted(t *testing.T) { require.NoError(t, err) // build an epoch on the root state and return a snapshot from the committed phase - committedPhaseSnapshot := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { - unittest.NewEpochBuilder(t, mutator, state).BuildEpoch().CompleteEpoch() + committedPhaseSnapshot := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutableState, state).BuildEpoch().CompleteEpoch() // find the point where we transition to the epoch committed phase for height := rootBlock.Height + 1; ; height++ { @@ -190,8 +190,8 @@ func TestBootstrap_EpochHeightBoundaries(t *testing.T) { }) t.Run("with next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { - builder := unittest.NewEpochBuilder(t, mutator, state) + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { + builder := unittest.NewEpochBuilder(t, mutableState, state) builder.BuildEpoch().CompleteEpoch() heights, ok := builder.EpochHeights(1) require.True(t, ok) @@ -217,8 +217,8 @@ func TestBootstrap_EpochHeightBoundaries(t *testing.T) { t.Run("with previous epoch", func(t *testing.T) { var epoch1FinalHeight uint64 var epoch2FirstHeight uint64 - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { - builder := unittest.NewEpochBuilder(t, mutator, state) + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { + builder := unittest.NewEpochBuilder(t, mutableState, state) builder. BuildEpoch().CompleteEpoch(). // build epoch 2 BuildEpoch() // build epoch 3 @@ -267,7 +267,7 @@ func TestBootstrapNonRoot(t *testing.T) { // should be able to bootstrap from snapshot after sealing a non-root block // ROOT <- B1 <- B2(R1) <- B3(S1) <- CHILD t.Run("with sealed block", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { block1 := unittest.BlockWithParentFixture(rootBlock) block1.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(rootProtocolStateID))) buildFinalizedBlock(t, state, block1) @@ -283,7 +283,7 @@ func TestBootstrapNonRoot(t *testing.T) { block3 := unittest.BlockWithParentFixture(block2.Header) block3.SetPayload(flow.Payload{ Seals: seals, - ProtocolStateID: calculateExpectedStateId(t, mutator)(block3.Header, seals), + ProtocolStateID: calculateExpectedStateId(t, mutableState)(block3.Header, seals), }) buildFinalizedBlock(t, state, block3) @@ -310,8 +310,8 @@ func TestBootstrapNonRoot(t *testing.T) { }) t.Run("with setup next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { - unittest.NewEpochBuilder(t, mutator, state).BuildEpoch() + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutableState, state).BuildEpoch() // find the point where we transition to the epoch setup phase for height := rootBlock.Height + 1; ; height++ { @@ -330,8 +330,8 @@ func TestBootstrapNonRoot(t *testing.T) { }) t.Run("with committed next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { - unittest.NewEpochBuilder(t, mutator, state).BuildEpoch().CompleteEpoch() + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutableState, state).BuildEpoch().CompleteEpoch() // find the point where we transition to the epoch committed phase for height := rootBlock.Height + 1; ; height++ { @@ -350,8 +350,8 @@ func TestBootstrapNonRoot(t *testing.T) { }) t.Run("with previous and next epoch", func(t *testing.T) { - after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutator protocol.StateMutator) protocol.Snapshot { - unittest.NewEpochBuilder(t, mutator, state). + after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState, mutableState protocol.MutableProtocolState) protocol.Snapshot { + unittest.NewEpochBuilder(t, mutableState, state). BuildEpoch().CompleteEpoch(). // build epoch 2 BuildEpoch() // build epoch 3 @@ -562,10 +562,10 @@ func bootstrap(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.S // // This is used for generating valid snapshots to use when testing bootstrapping // from non-root states. -func snapshotAfter(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.FollowerState, protocol.StateMutator) protocol.Snapshot) protocol.Snapshot { +func snapshotAfter(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.FollowerState, protocol.MutableProtocolState) protocol.Snapshot) protocol.Snapshot { var after protocol.Snapshot - protoutil.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(_ *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { - snap := f(state.FollowerState, mutator) + protoutil.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(_ *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { + snap := f(state.FollowerState, mutableState) var err error after, err = inmem.FromSnapshot(snap) require.NoError(t, err) diff --git a/state/protocol/inmem/convert_test.go b/state/protocol/inmem/convert_test.go index 8c72240d535..05cd513a26f 100644 --- a/state/protocol/inmem/convert_test.go +++ b/state/protocol/inmem/convert_test.go @@ -23,9 +23,9 @@ func TestFromSnapshot(t *testing.T) { identities := unittest.IdentityListFixture(10, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, fullState *bprotocol.ParticipantState, mutator protocol.StateMutator) { + util.RunWithFullProtocolStateAndMutator(t, rootSnapshot, func(db *badger.DB, fullState *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { state := fullState.FollowerState - epochBuilder := unittest.NewEpochBuilder(t, mutator, state) + epochBuilder := unittest.NewEpochBuilder(t, mutableState, state) // build epoch 1 (prepare epoch 2) epochBuilder. BuildEpoch(). diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index 73dbbd5773d..7aa5b0b12de 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -285,7 +285,7 @@ func RunWithFullProtocolStateAndConsumer(t testing.TB, rootSnapshot protocol.Sna }) } -func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot protocol.Snapshot, metrics module.ComplianceMetrics, consumer protocol.Consumer, f func(*badger.DB, *pbadger.ParticipantState, protocol.StateMutator)) { +func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot protocol.Snapshot, metrics module.ComplianceMetrics, consumer protocol.Consumer, f func(*badger.DB, *pbadger.ParticipantState, protocol.MutableProtocolState)) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { tracer := trace.NewNoopTracer() log := zerolog.Nop() @@ -320,15 +320,16 @@ func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot pr sealValidator, ) require.NoError(t, err) - mutator := protocol_state.NewMutator( + mutableProtocolState := protocol_state.NewMutableProtocolState( + all.ProtocolState, + state.Params(), all.Headers, all.Results, all.Setups, all.EpochCommits, - all.ProtocolState, state.Params(), ) - f(db, fullState, mutator) + f(db, fullState, mutableProtocolState) }) } @@ -369,7 +370,7 @@ func RunWithFollowerProtocolStateAndHeaders(t testing.TB, rootSnapshot protocol. }) } -func RunWithFullProtocolStateAndMutator(t testing.TB, rootSnapshot protocol.Snapshot, f func(*badger.DB, *pbadger.ParticipantState, protocol.StateMutator)) { +func RunWithFullProtocolStateAndMutator(t testing.TB, rootSnapshot protocol.Snapshot, f func(*badger.DB, *pbadger.ParticipantState, protocol.MutableProtocolState)) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() @@ -406,14 +407,15 @@ func RunWithFullProtocolStateAndMutator(t testing.TB, rootSnapshot protocol.Snap sealValidator, ) require.NoError(t, err) - mutator := protocol_state.NewMutator( + mutableProtocolState := protocol_state.NewMutableProtocolState( + all.ProtocolState, + state.Params(), all.Headers, all.Results, all.Setups, all.EpochCommits, - all.ProtocolState, state.Params(), ) - f(db, fullState, mutator) + f(db, fullState, mutableProtocolState) }) } From 74be7f8b14affe4ca944b67b4640fafa205f9809 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 31 Oct 2023 14:03:13 +0200 Subject: [PATCH 49/87] Fixed other broken tests. Linted --- cmd/consensus/main.go | 7 ++++--- consensus/integration/nodes_test.go | 12 +++++++++-- engine/access/rpc/backend/backend_test.go | 20 +++++++++---------- .../rpc/backend/backend_transactions_test.go | 4 ++-- .../test/cluster_switchover_test.go | 9 +++++---- engine/verification/utils/unittest/helper.go | 7 ++++--- module/builder/consensus/builder_test.go | 7 ++++--- state/cluster/badger/mutator_test.go | 13 ++++++------ state/protocol/protocol_state/mutator.go | 1 + .../protocol/protocol_state/protocol_state.go | 1 + state/protocol/protocol_state/updater.go | 1 + 11 files changed, 49 insertions(+), 33 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index cf62933d5f4..40cebcc0dc1 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -721,12 +721,13 @@ func main() { return ctl, nil }). Component("consensus participant", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - protocolStateMutator := protocol_state.NewMutator( + mutableProtocolState := protocol_state.NewMutableProtocolState( + node.Storage.ProtocolState, + node.State.Params(), node.Storage.Headers, node.Storage.Results, node.Storage.Setups, node.Storage.EpochCommits, - node.Storage.ProtocolState, node.State.Params(), ) // initialize the block builder @@ -741,7 +742,7 @@ func main() { node.Storage.Blocks, node.Storage.Results, node.Storage.Receipts, - protocolStateMutator, + mutableProtocolState, guarantees, seals, receipts, diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index ba395c7bc77..146be2450ec 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -460,7 +460,15 @@ func createNode( seals := stdmap.NewIncorporatedResultSeals(sealLimit) - protocolStateMutator := protocol_state.NewMutator(headersDB, resultsDB, setupsDB, commitsDB, protocolStateDB, state.Params()) + mutableProtocolState := protocol_state.NewMutableProtocolState( + protocolStateDB, + state.Params(), + headersDB, + resultsDB, + setupsDB, + commitsDB, + state.Params(), + ) // initialize the block builder build, err := builder.NewBuilder( @@ -473,7 +481,7 @@ func createNode( blocksDB, resultsDB, receiptsDB, - protocolStateMutator, + mutableProtocolState, guarantees, consensusMempools.NewIncorporatedResultSeals(seals, receiptsDB), receipts, diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index f6fbbe858bc..e391ff6611b 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -147,8 +147,8 @@ func (suite *Suite) TestGetLatestFinalizedBlockHeader() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_NoTransitionSpan() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutableState, state) // build epoch 1 // Blocks in current State // P <- A(S_P-1) <- B(S_P) <- C(S_A) <- D(S_B) |setup| <- E(S_C) <- F(S_D) |commit| @@ -194,8 +194,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_NoTransitionSpan() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_TransitionSpans() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutableState, state) // building 2 epochs allows us to take a snapshot at a point in time where // an epoch transition happens @@ -250,8 +250,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_TransitionSpans() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_PhaseTransitionSpan() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutableState, state) // build epoch 1 // Blocks in current State // P <- A(S_P-1) <- B(S_P) <- C(S_A) <- D(S_B) |setup| <- E(S_C) <- F(S_D) |commit| @@ -298,8 +298,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_PhaseTransitionSpan() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_EpochTransitionSpan() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutableState, state) // build epoch 1 // Blocks in current State // P <- A(S_P-1) <- B(S_P) <- C(S_A) <- D(S_B) |setup| <- E(S_C) <- F(S_D) |commit| @@ -358,8 +358,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_EpochTransitionSpan() { func (suite *Suite) TestGetLatestProtocolStateSnapshot_HistoryLimit() { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator realprotocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state).BuildEpoch().CompleteEpoch() + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState realprotocol.MutableProtocolState) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutableState, state).BuildEpoch().CompleteEpoch() // get heights of each phase in built epochs epoch1, ok := epochBuilder.EpochHeights(1) diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index 712e0c10536..cdde708720d 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -24,8 +24,8 @@ import ( func (suite *Suite) withPreConfiguredState(f func(snap protocol.Snapshot)) { identities := unittest.CompleteIdentitySet() rootSnapshot := unittest.RootSnapshotFixture(identities) - util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutator protocol.StateMutator) { - epochBuilder := unittest.NewEpochBuilder(suite.T(), mutator, state) + util.RunWithFullProtocolStateAndMutator(suite.T(), rootSnapshot, func(db *badger.DB, state *bprotocol.ParticipantState, mutableState protocol.MutableProtocolState) { + epochBuilder := unittest.NewEpochBuilder(suite.T(), mutableState, state) epochBuilder. BuildEpoch(). diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index ab462ee83bf..8798188a5fd 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -133,20 +133,21 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) states = append(states, node.State) } - // take first collection node and use its storage as data source for mutator + // take first collection node and use its storage as data source for stateMutator refNode := tc.nodes[0] - mutator := protocol_state.NewMutator( + stateMutator := protocol_state.NewMutableProtocolState( + refNode.ProtocolStateSnapshots, + refNode.State.Params(), refNode.Headers, refNode.Results, refNode.Setups, refNode.EpochCommits, - refNode.ProtocolStateSnapshots, refNode.State.Params(), ) // when building new epoch we would like to replace fixture cluster QCs with real ones, for that we need // to generate them using node infos - tc.builder = unittest.NewEpochBuilder(tc.T(), mutator, states...).UsingCommitOpts(func(commit *flow.EpochCommit) { + tc.builder = unittest.NewEpochBuilder(tc.T(), stateMutator, states...).UsingCommitOpts(func(commit *flow.EpochCommit) { // build a lookup table for node infos nodeInfoLookup := make(map[flow.Identifier]model.NodeInfo) for _, nodeInfo := range tc.nodeInfos { diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 4bb645543f1..3471f00d4d8 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -653,14 +653,15 @@ func bootstrapSystem( bootstrapNodesInfo = append(bootstrapNodesInfo, verID) identities = append(identities, verID.Identity()) - mutator := protocol_state.NewMutator( + mutableProtocolState := protocol_state.NewMutableProtocolState( + stateFixture.Storage.ProtocolState, + stateFixture.State.Params(), stateFixture.Storage.Headers, stateFixture.Storage.Results, stateFixture.Storage.Setups, stateFixture.Storage.EpochCommits, - stateFixture.Storage.ProtocolState, stateFixture.State.Params()) - epochBuilder := unittest.NewEpochBuilder(t, mutator, stateFixture.State) + epochBuilder := unittest.NewEpochBuilder(t, mutableProtocolState, stateFixture.State) epochBuilder. UsingSetupOpts(unittest.WithParticipants(identities.ToSkeleton())). BuildEpoch() diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index 16a80f8a926..5ebbe3c39c3 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -418,9 +418,10 @@ func (bs *BuilderSuite) SetupTest() { bs.stateMutator = protocol.NewMutableProtocolState(bs.T()) bs.stateMutator.On("Mutator", mock.Anything, mock.Anything).Return( func(_ uint64, _ flow.Identifier) realproto.StateMutator { - updater := protocol.NewStateMutator(bs.T()) - updater.On("Build").Return(false, nil, flow.Identifier{}, nil) - return updater + stateMutator := protocol.NewStateMutator(bs.T()) + stateMutator.On("ApplyServiceEvents", mock.Anything).Return(nil) + stateMutator.On("Build").Return(false, nil, flow.Identifier{}, nil, nil) + return stateMutator }, func(_ uint64, _ flow.Identifier) error { return nil }, diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 75ad3e5b60b..409810b0efc 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -43,9 +43,9 @@ type MutatorSuite struct { epochCounter uint64 // protocol state for reference blocks for transactions - protoState protocol.FollowerState - protoStateMutator protocol.StateMutator - protoGenesis *flow.Block + protoState protocol.FollowerState + mutableProtocolState protocol.MutableProtocolState + protoGenesis *flow.Block state cluster.MutableState } @@ -101,12 +101,13 @@ func (suite *MutatorSuite) SetupTest() { suite.protoState, err = pbadger.NewFollowerState(log, tracer, events.NewNoop(), state, all.Index, all.Payloads, protocolutil.MockBlockTimer()) require.NoError(suite.T(), err) - suite.protoStateMutator = protocol_state.NewMutator( + suite.mutableProtocolState = protocol_state.NewMutableProtocolState( + all.ProtocolState, + state.Params(), all.Headers, all.Results, all.Setups, all.EpochCommits, - all.ProtocolState, state.Params(), ) @@ -413,7 +414,7 @@ func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromClusterChain() { // using a reference block in a different epoch than the cluster's epoch. func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromDifferentEpoch() { // build and complete the current epoch, then use a reference block from next epoch - eb := unittest.NewEpochBuilder(suite.T(), suite.protoStateMutator, suite.protoState) + eb := unittest.NewEpochBuilder(suite.T(), suite.mutableProtocolState, suite.protoState) eb.BuildEpoch().CompleteEpoch() heights, ok := eb.EpochHeights(1) require.True(suite.T(), ok) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index b72afe584a8..ecc09a5088e 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -3,6 +3,7 @@ package protocol_state import ( "errors" "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" diff --git a/state/protocol/protocol_state/protocol_state.go b/state/protocol/protocol_state/protocol_state.go index 7797197c507..dcbe7835458 100644 --- a/state/protocol/protocol_state/protocol_state.go +++ b/state/protocol/protocol_state/protocol_state.go @@ -2,6 +2,7 @@ package protocol_state import ( "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/inmem" diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 441e3a2f150..272a955fe16 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -2,6 +2,7 @@ package protocol_state import ( "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/state/protocol" From 8f4e038ff3e64e7c3cf99a9354ae712eb3cf3482 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 31 Oct 2023 15:15:46 +0200 Subject: [PATCH 50/87] Updated tests and changed Build signature --- module/builder/consensus/builder.go | 2 +- state/protocol/badger/mutator.go | 2 +- state/protocol/badger/mutator_test.go | 2 +- state/protocol/mock/state_mutator.go | 13 +- state/protocol/protocol_state.go | 2 +- state/protocol/protocol_state/mutator.go | 18 +- state/protocol/protocol_state/mutator_test.go | 513 +++++++++--------- .../protocol/protocol_state/protocol_state.go | 6 +- .../protocol_state/protocol_state_test.go | 35 ++ state/protocol/protocol_state/updater.go | 32 +- state/protocol/protocol_state/updater_test.go | 34 +- utils/unittest/epoch_builder.go | 2 +- utils/unittest/protocol_state.go | 2 +- 13 files changed, 343 insertions(+), 320 deletions(-) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index 455af4cd475..b85b050ca6a 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -643,7 +643,7 @@ func (b *Builder) createProposal(parentID flow.Identifier, if err != nil { return nil, fmt.Errorf("could not apply service events as leader: %w", err) } - _, _, protocolStateID, _, err := stateMutator.Build() + _, _, protocolStateID, _ := stateMutator.Build() if err != nil { return nil, fmt.Errorf("could not build updated protocol state: %w", err) } diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index a753c7cb52b..8d5cb1d3646 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -519,7 +519,7 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi return fmt.Errorf("could not process service events: %w", err) } - hasChanges, updatedState, updatedStateID, dbUpdates, err := stateMutator.Build() + hasChanges, updatedState, updatedStateID, dbUpdates := stateMutator.Build() if err != nil { return fmt.Errorf("could not build updated protocol state: %w", err) } diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index a2f4ff8231e..ac85d8dba33 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -2687,7 +2687,7 @@ func calculateExpectedStateId(t *testing.T, mutableProtocolState realprotocol.Mu err = stateMutator.ApplyServiceEvents(seals) require.NoError(t, err) - _, _, expectedStateID, _, err := stateMutator.Build() + _, _, expectedStateID, _ := stateMutator.Build() require.NoError(t, err) return expectedStateID } diff --git a/state/protocol/mock/state_mutator.go b/state/protocol/mock/state_mutator.go index 9d545d30b9a..9ba2dbab0c1 100644 --- a/state/protocol/mock/state_mutator.go +++ b/state/protocol/mock/state_mutator.go @@ -29,15 +29,14 @@ func (_m *StateMutator) ApplyServiceEvents(seals []*flow.Seal) error { } // Build provides a mock function with given fields: -func (_m *StateMutator) Build() (bool, *flow.ProtocolStateEntry, flow.Identifier, []func(*transaction.Tx) error, error) { +func (_m *StateMutator) Build() (bool, *flow.ProtocolStateEntry, flow.Identifier, []func(*transaction.Tx) error) { ret := _m.Called() var r0 bool var r1 *flow.ProtocolStateEntry var r2 flow.Identifier var r3 []func(*transaction.Tx) error - var r4 error - if rf, ok := ret.Get(0).(func() (bool, *flow.ProtocolStateEntry, flow.Identifier, []func(*transaction.Tx) error, error)); ok { + if rf, ok := ret.Get(0).(func() (bool, *flow.ProtocolStateEntry, flow.Identifier, []func(*transaction.Tx) error)); ok { return rf() } if rf, ok := ret.Get(0).(func() bool); ok { @@ -70,13 +69,7 @@ func (_m *StateMutator) Build() (bool, *flow.ProtocolStateEntry, flow.Identifier } } - if rf, ok := ret.Get(4).(func() error); ok { - r4 = rf() - } else { - r4 = ret.Error(4) - } - - return r0, r1, r2, r3, r4 + return r0, r1, r2, r3 } type mockConstructorTestingTNewStateMutator interface { diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 5cbc9bb1699..d17ff1dc1cd 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -93,7 +93,7 @@ type StateMutator interface { // - stateID: the has commitment to the `updatedState` // - dbUpdates: database updates necessary for persisting the updated protocol state // updated protocol state entry, state ID and a flag indicating if there were any changes. - Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error, err error) + Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) // ApplyServiceEvents handles applying state changes which occur as a result // of service events being included in a block payload. diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index ecc09a5088e..abce735291f 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -11,11 +11,11 @@ import ( "github.com/onflow/flow-go/storage/badger/transaction" ) -// Mutator implements protocol.StateMutator interface. +// stateMutator implements protocol.StateMutator interface. // It has to be used for each block to update the protocol state, even if there are no state-changing // service events sealed in candidate block. This requirement is due to the fact that protocol state // is indexed by block ID, and we need to maintain such index. -type Mutator struct { +type stateMutator struct { headers storage.Headers results storage.ExecutionResults setups storage.EpochSetups @@ -25,17 +25,17 @@ type Mutator struct { pendingDbUpdates []func(tx *transaction.Tx) error } -var _ protocol.StateMutator = (*Mutator)(nil) +var _ protocol.StateMutator = (*stateMutator)(nil) -func NewMutator( +func newStateMutator( headers storage.Headers, results storage.ExecutionResults, setups storage.EpochSetups, commits storage.EpochCommits, stateMachine ProtocolStateMachine, params protocol.InstanceParams, -) *Mutator { - return &Mutator{ +) *stateMutator { + return &stateMutator{ setups: setups, params: params, headers: headers, @@ -52,7 +52,7 @@ func NewMutator( // - dbUpdates: database updates necessary for persisting the updated protocol state // // updated protocol state entry, state ID and a flag indicating if there were any changes. -func (m *Mutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error, err error) { +func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) { updatedState, stateID, hasChanges = m.stateMachine.Build() dbUpdates = m.pendingDbUpdates return @@ -62,7 +62,7 @@ func (m *Mutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntr // of service events being included in a block payload. // All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. // No errors are expected during normal operation. -func (m *Mutator) ApplyServiceEvents(seals []*flow.Seal) error { +func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { dbUpdates, err := m.handleServiceEvents(seals) if err != nil { return err @@ -104,7 +104,7 @@ func (m *Mutator) ApplyServiceEvents(seals []*flow.Seal) error { // This includes operations to insert service events for blocks that include them. // // No errors are expected during normal operation. -func (m *Mutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { +func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { epochFallbackTriggered, err := m.params.EpochFallbackTriggered() if err != nil { return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index a7e6ba11159..9b7a8a265c4 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -1,6 +1,13 @@ package protocol_state import ( + "errors" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage/badger/transaction" + "github.com/onflow/flow-go/utils/rand" + "github.com/onflow/flow-go/utils/unittest" + "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/suite" @@ -11,10 +18,10 @@ import ( ) func TestProtocolStateMutator(t *testing.T) { - suite.Run(t, new(MutatorSuite)) + suite.Run(t, new(StateMutatorSuite)) } -type MutatorSuite struct { +type StateMutatorSuite struct { suite.Suite protocolStateDB *storagemock.ProtocolState headersDB *storagemock.Headers @@ -24,271 +31,259 @@ type MutatorSuite struct { params *mock.InstanceParams stateMachine *protocolstatemock.ProtocolStateMachine - mutator *Mutator + mutator *stateMutator } -func (s *MutatorSuite) SetupTest() { +func (s *StateMutatorSuite) SetupTest() { s.protocolStateDB = storagemock.NewProtocolState(s.T()) s.headersDB = storagemock.NewHeaders(s.T()) s.resultsDB = storagemock.NewExecutionResults(s.T()) s.setupsDB = storagemock.NewEpochSetups(s.T()) s.commitsDB = storagemock.NewEpochCommits(s.T()) s.params = mock.NewInstanceParams(s.T()) + s.stateMachine = protocolstatemock.NewProtocolStateMachine(s.T()) + s.mutator = newStateMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.stateMachine, s.params) +} + +// TestHappyPathNoChanges tests that stateMutator doesn't cache any db updates when there are no changes. +func (s *StateMutatorSuite) TestHappyPathNoChanges() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture() + s.stateMachine.On("ParentState").Return(parentState) + s.stateMachine.On("Build").Return(parentState.ProtocolStateEntry, parentState.ID(), false) + err := s.mutator.ApplyServiceEvents([]*flow.Seal{}) + require.NoError(s.T(), err) + hasChanges, updatedState, updatedStateID, dbUpdates := s.mutator.Build() + require.False(s.T(), hasChanges) + require.Equal(s.T(), parentState.ProtocolStateEntry, updatedState) + require.Equal(s.T(), parentState.ID(), updatedStateID) + require.Empty(s.T(), dbUpdates) +} + +// TestHappyPathHasChanges tests that stateMutator returns cached db updates when building protocol state after applying service events. +func (s *StateMutatorSuite) TestHappyPathHasChanges() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture() + s.stateMachine.On("ParentState").Return(parentState) + s.stateMachine.On("Build").Return(unittest.ProtocolStateFixture().ProtocolStateEntry, + unittest.IdentifierFixture(), true) + + epochSetup := unittest.EpochSetupFixture() + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(nil).Once() + s.setupsDB.On("StoreTx", epochSetup).Return(func(*transaction.Tx) error { return nil }).Once() + + err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + require.NoError(s.T(), err) + + _, _, _, dbUpdates := s.mutator.Build() + require.NoError(s.T(), err) + require.Len(s.T(), dbUpdates, 1) +} + +// TestApplyServiceEvents_InvalidEpochSetup tests that handleServiceEvents rejects invalid epoch setup event and sets +// InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. +func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochSetup() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + s.Run("invalid-epoch-setup", func() { + parentState := unittest.ProtocolStateFixture() + rootSetup := parentState.CurrentEpochSetup + + s.stateMachine.On("ParentState").Return(parentState) + + epochSetup := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(protocol.NewInvalidServiceEventErrorf("")).Once() + s.stateMachine.On("SetInvalidStateTransitionAttempted").Return().Once() + + err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + require.NoError(s.T(), err) + }) + s.Run("process-epoch-setup-exception", func() { + parentState := unittest.ProtocolStateFixture() + rootSetup := parentState.CurrentEpochSetup + + s.stateMachine.On("ParentState").Return(parentState) + + epochSetup := unittest.EpochSetupFixture( + unittest.WithParticipants(rootSetup.Participants), + unittest.SetupWithCounter(rootSetup.Counter+1), + unittest.WithFinalView(rootSetup.FinalView+1000), + unittest.WithFirstView(rootSetup.FinalView+1), + ) + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + exception := errors.New("exception") + s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(exception).Once() + + err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + require.Error(s.T(), err) + require.False(s.T(), protocol.IsInvalidServiceEventError(err)) + }) +} - s.mutator = NewMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.stateMachine, s.params) +// TestApplyServiceEvents_InvalidEpochCommit tests that handleServiceEvents rejects invalid epoch commit event and sets +// InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. +func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochCommit() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + s.Run("invalid-epoch-commit", func() { + parentState := unittest.ProtocolStateFixture() + + s.stateMachine.On("ParentState").Return(parentState) + + epochCommit := unittest.EpochCommitFixture() + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + s.stateMachine.On("ProcessEpochCommit", epochCommit).Return(protocol.NewInvalidServiceEventErrorf("")).Once() + s.stateMachine.On("SetInvalidStateTransitionAttempted").Return().Once() + + err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + require.NoError(s.T(), err) + }) + s.Run("process-epoch-commit-exception", func() { + parentState := unittest.ProtocolStateFixture() + + s.stateMachine.On("ParentState").Return(parentState) + + epochCommit := unittest.EpochCommitFixture() + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} + }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + exception := errors.New("exception") + s.stateMachine.On("ProcessEpochCommit", epochCommit).Return(exception).Once() + + err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + require.Error(s.T(), err) + require.False(s.T(), protocol.IsInvalidServiceEventError(err)) + }) +} + +// TestApplyServiceEventsSealsOrdered tests that handleServiceEvents processes seals in order of block height. +func (s *StateMutatorSuite) TestApplyServiceEventsSealsOrdered() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture() + s.stateMachine.On("ParentState").Return(parentState) + + blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) + var seals []*flow.Seal + resultByHeight := make(map[flow.Identifier]uint64) + for _, block := range blocks { + receipt, seal := unittest.ReceiptAndSealForBlock(block) + resultByHeight[seal.ResultID] = block.Header.Height + s.headersDB.On("ByBlockID", seal.BlockID).Return(block.Header, nil).Once() + s.resultsDB.On("ByID", seal.ResultID).Return(&receipt.ExecutionResult, nil).Once() + seals = append(seals, seal) + } + + // shuffle seals to make sure we order them + require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { + seals[i], seals[j] = seals[j], seals[i] + })) + + err := s.mutator.ApplyServiceEvents(seals) + require.NoError(s.T(), err) + + // assert that results were queried in order of executed block height + // if seals were properly ordered before processing, then results should be ordered by block height + lastExecutedBlockHeight := uint64(0) + for _, call := range s.resultsDB.Calls { + resultID := call.Arguments.Get(0).(flow.Identifier) + executedBlockHeight, found := resultByHeight[resultID] + require.True(s.T(), found) + require.Less(s.T(), lastExecutedBlockHeight, executedBlockHeight, "seals must be ordered by block height") + } } -//// TestCreateUpdaterForUnknownBlock tests that CreateUpdater returns an error if the parent protocol state is not found. -//func (s *MutatorSuite) TestCreateUpdaterForUnknownBlock() { -// candidate := unittest.BlockHeaderFixture() -// s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(nil, storerr.ErrNotFound) -// updater, err := s.mutator.(candidate.View, candidate.ParentID) -// require.ErrorIs(s.T(), err, storerr.ErrNotFound) -// require.Nil(s.T(), updater) -//} - -//// TestMutatorHappyPathNoChanges tests that Mutator correctly indexes the protocol state when there are no changes. -//func (s *MutatorSuite) TestMutatorHappyPathNoChanges() { -// parentState := unittest.ProtocolStateFixture() -// candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) -// s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) -// updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) -// require.NoError(s.T(), err) -// -// s.protocolStateDB.On("Index", candidate.ID(), parentState.ID()).Return(func(tx *transaction.Tx) error { return nil }) -// -// dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) -// err = dbOps(&transaction.Tx{}) -// require.NoError(s.T(), err) -//} -// -//// TestMutatorHappyPathHasChanges tests that Mutator correctly persists and indexes the protocol state when there are changes. -//func (s *MutatorSuite) TestMutatorHappyPathHasChanges() { -// parentState := unittest.ProtocolStateFixture() -// candidate := unittest.BlockHeaderFixture(unittest.HeaderWithView(parentState.CurrentEpochSetup.FirstView)) -// s.protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) -// updater, err := s.mutator.CreateUpdater(candidate.View, candidate.ParentID) -// require.NoError(s.T(), err) -// -// // update protocol state so it has some changes -// updater.SetInvalidStateTransitionAttempted() -// updatedState, updatedStateID, hasChanges := updater.Build() -// require.True(s.T(), hasChanges) -// -// s.protocolStateDB.On("StoreTx", updatedStateID, updatedState).Return(func(tx *transaction.Tx) error { return nil }) -// s.protocolStateDB.On("Index", candidate.ID(), updatedStateID).Return(func(tx *transaction.Tx) error { return nil }) -// -// dbOps, _ := s.mutator.CommitProtocolState(candidate.ID(), updater) -// err = dbOps(&transaction.Tx{}) -// require.NoError(s.T(), err) -//} -// -//// TestMutatorApplyServiceEvents_InvalidEpochSetup tests that handleServiceEvents rejects invalid epoch setup event and sets -//// InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. -//func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochSetup() { -// s.params.On("EpochFallbackTriggered").Return(false, nil) -// s.Run("invalid-epoch-setup", func() { -// parentState := unittest.ProtocolStateFixture() -// rootSetup := parentState.CurrentEpochSetup -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// -// epochSetup := unittest.EpochSetupFixture( -// unittest.WithParticipants(rootSetup.Participants), -// unittest.SetupWithCounter(rootSetup.Counter+1), -// unittest.WithFinalView(rootSetup.FinalView+1000), -// unittest.WithFirstView(rootSetup.FinalView+1), -// ) -// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { -// result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} -// }) -// -// block := unittest.BlockHeaderFixture() -// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) -// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) -// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) -// -// updater.On("ProcessEpochSetup", epochSetup).Return(protocol.NewInvalidServiceEventErrorf("")).Once() -// updater.On("SetInvalidStateTransitionAttempted").Return().Once() -// -// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) -// require.NoError(s.T(), err) -// require.Empty(s.T(), updates) -// }) -// s.Run("process-epoch-setup-exception", func() { -// parentState := unittest.ProtocolStateFixture() -// rootSetup := parentState.CurrentEpochSetup -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// -// epochSetup := unittest.EpochSetupFixture( -// unittest.WithParticipants(rootSetup.Participants), -// unittest.SetupWithCounter(rootSetup.Counter+1), -// unittest.WithFinalView(rootSetup.FinalView+1000), -// unittest.WithFirstView(rootSetup.FinalView+1), -// ) -// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { -// result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} -// }) -// -// block := unittest.BlockHeaderFixture() -// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) -// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) -// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) -// -// exception := errors.New("exception") -// updater.On("ProcessEpochSetup", epochSetup).Return(exception).Once() -// -// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) -// require.Error(s.T(), err) -// require.False(s.T(), protocol.IsInvalidServiceEventError(err)) -// require.Empty(s.T(), updates) -// }) -//} -// -//// TestMutatorApplyServiceEvents_InvalidEpochCommit tests that handleServiceEvents rejects invalid epoch commit event and sets -//// InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. -//func (s *MutatorSuite) TestMutatorApplyServiceEvents_InvalidEpochCommit() { -// s.params.On("EpochFallbackTriggered").Return(false, nil) -// s.Run("invalid-epoch-commit", func() { -// parentState := unittest.ProtocolStateFixture() -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// -// epochCommit := unittest.EpochCommitFixture() -// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { -// result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} -// }) -// -// block := unittest.BlockHeaderFixture() -// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) -// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) -// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) -// -// updater.On("ProcessEpochCommit", epochCommit).Return(protocol.NewInvalidServiceEventErrorf("")).Once() -// updater.On("SetInvalidStateTransitionAttempted").Return().Once() -// -// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) -// require.NoError(s.T(), err) -// require.Empty(s.T(), updates) -// }) -// s.Run("process-epoch-commit-exception", func() { -// parentState := unittest.ProtocolStateFixture() -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// -// epochCommit := unittest.EpochCommitFixture() -// result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { -// result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} -// }) -// -// block := unittest.BlockHeaderFixture() -// seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) -// s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) -// s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) -// -// exception := errors.New("exception") -// updater.On("ProcessEpochCommit", epochCommit).Return(exception).Once() -// -// updates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{seal}) -// require.Error(s.T(), err) -// require.False(s.T(), protocol.IsInvalidServiceEventError(err)) -// require.Empty(s.T(), updates) -// }) -//} -// -//// TestMutatorApplyServiceEventsSealsOrdered tests that handleServiceEvents processes seals in order of block height. -//func (s *MutatorSuite) TestMutatorApplyServiceEventsSealsOrdered() { -// s.params.On("EpochFallbackTriggered").Return(false, nil) -// parentState := unittest.ProtocolStateFixture() -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// -// blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) -// var seals []*flow.Seal -// resultByHeight := make(map[flow.Identifier]uint64) -// for _, block := range blocks { -// receipt, seal := unittest.ReceiptAndSealForBlock(block) -// resultByHeight[seal.ResultID] = block.Header.Height -// s.headersDB.On("ByBlockID", seal.BlockID).Return(block.Header, nil).Once() -// s.resultsDB.On("ByID", seal.ResultID).Return(&receipt.ExecutionResult, nil).Once() -// seals = append(seals, seal) -// } -// -// // shuffle seals to make sure we order them -// require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { -// seals[i], seals[j] = seals[j], seals[i] -// })) -// -// updates, err := s.mutator.handleServiceEvents(updater, seals) -// require.NoError(s.T(), err) -// require.Empty(s.T(), updates) -// // assert that results were queried in order of executed block height -// // if seals were properly ordered before processing, then results should be ordered by block height -// lastExecutedBlockHeight := uint64(0) -// for _, call := range s.resultsDB.Calls { -// resultID := call.Arguments.Get(0).(flow.Identifier) -// executedBlockHeight, found := resultByHeight[resultID] -// require.True(s.T(), found) -// require.Less(s.T(), lastExecutedBlockHeight, executedBlockHeight, "seals must be ordered by block height") -// } -//} -// -//// TestMutatorApplyServiceEventsTransitionToNextEpoch tests that handleServiceEvents transitions to the next epoch -//// when epoch has been committed, and we are at the first block of the next epoch. -//func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch() { -// s.params.On("EpochFallbackTriggered").Return(false, nil) -// parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// // we are at the first block of the next epoch -// updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) -// updater.On("TransitionToNextEpoch").Return(nil).Once() -// dbUpdates, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{}) -// require.NoError(s.T(), err) -// require.Empty(s.T(), dbUpdates) -//} -// -//// TestMutatorApplyServiceEventsTransitionToNextEpoch_Error tests that error that has been -//// observed in handleServiceEvents when transitioning to the next epoch is propagated to the caller. -//func (s *MutatorSuite) TestMutatorApplyServiceEventsTransitionToNextEpoch_Error() { -// s.params.On("EpochFallbackTriggered").Return(false, nil) -// parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// // we are at the first block of the next epoch -// updater.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) -// exception := errors.New("exception") -// updater.On("TransitionToNextEpoch").Return(exception).Once() -// _, err := s.mutator.handleServiceEvents(updater, []*flow.Seal{}) -// require.ErrorIs(s.T(), err, exception) -// require.False(s.T(), protocol.IsInvalidServiceEventError(err)) -//} -// -//// TestMutatorApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. -//// In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. -//// In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. -//// In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. -//func (s *MutatorSuite) TestMutatorApplyServiceEvents_EpochFallbackTriggered() { -// s.Run("epoch-fallback-triggered", func() { -// s.params.On("EpochFallbackTriggered").Return(true, nil) -// parentState := unittest.ProtocolStateFixture() -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// seals := unittest.Seal.Fixtures(2) -// updates, err := s.mutator.handleServiceEvents(updater, seals) -// require.NoError(s.T(), err) -// require.Empty(s.T(), updates) -// }) -// s.Run("invalid-service-event-incorporated", func() { -// s.params.On("EpochFallbackTriggered").Return(false, nil) -// parentState := unittest.ProtocolStateFixture() -// parentState.InvalidStateTransitionAttempted = true -// updater := mock.NewStateUpdater(s.T()) -// updater.On("ParentState").Return(parentState) -// seals := unittest.Seal.Fixtures(2) -// updates, err := s.mutator.handleServiceEvents(updater, seals) -// require.NoError(s.T(), err) -// require.Empty(s.T(), updates) -// }) -//} +// TestApplyServiceEventsTransitionToNextEpoch tests that handleServiceEvents transitions to the next epoch +// when epoch has been committed, and we are at the first block of the next epoch. +func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) + s.stateMachine.On("ParentState").Return(parentState) + // we are at the first block of the next epoch + s.stateMachine.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) + s.stateMachine.On("TransitionToNextEpoch").Return(nil).Once() + err := s.mutator.ApplyServiceEvents([]*flow.Seal{}) + require.NoError(s.T(), err) +} + +// TestApplyServiceEventsTransitionToNextEpoch_Error tests that error that has been +// observed in handleServiceEvents when transitioning to the next epoch is propagated to the caller. +func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch_Error() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) + + s.stateMachine.On("ParentState").Return(parentState) + // we are at the first block of the next epoch + s.stateMachine.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) + exception := errors.New("exception") + s.stateMachine.On("TransitionToNextEpoch").Return(exception).Once() + err := s.mutator.ApplyServiceEvents([]*flow.Seal{}) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), protocol.IsInvalidServiceEventError(err)) +} + +// TestApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. +// In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. +// In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. +// In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. +func (s *StateMutatorSuite) TestApplyServiceEvents_EpochFallbackTriggered() { + s.Run("epoch-fallback-triggered", func() { + s.params.On("EpochFallbackTriggered").Return(true, nil) + parentState := unittest.ProtocolStateFixture() + s.stateMachine.On("ParentState").Return(parentState) + seals := unittest.Seal.Fixtures(2) + err := s.mutator.ApplyServiceEvents(seals) + require.NoError(s.T(), err) + }) + s.Run("invalid-service-event-incorporated", func() { + s.params.On("EpochFallbackTriggered").Return(false, nil) + parentState := unittest.ProtocolStateFixture() + parentState.InvalidStateTransitionAttempted = true + s.stateMachine.On("ParentState").Return(parentState) + seals := unittest.Seal.Fixtures(2) + err := s.mutator.ApplyServiceEvents(seals) + require.NoError(s.T(), err) + }) +} diff --git a/state/protocol/protocol_state/protocol_state.go b/state/protocol/protocol_state/protocol_state.go index dcbe7835458..9f6a1d70542 100644 --- a/state/protocol/protocol_state/protocol_state.go +++ b/state/protocol/protocol_state/protocol_state.go @@ -75,7 +75,7 @@ func NewMutableProtocolState( } } -// Mutator instantiates a `StateMutator` based on the previous protocol state. +// Mutator instantiates a `protocol.StateMutator` based on the previous protocol state. // Has to be called for each block to evolve the protocol state. // Expected errors during normal operations: // - `storage.ErrNotFound` if no protocol state for parent block is known. @@ -84,6 +84,6 @@ func (s *MutableProtocolState) Mutator(candidateView uint64, parentID flow.Ident if err != nil { return nil, fmt.Errorf("could not query parent protocol state at block (%x): %w", parentID, err) } - stateMachine := NewStateMachine(candidateView, parentState) - return NewMutator(s.headers, s.results, s.setups, s.commits, stateMachine, s.params), nil + return newStateMutator(s.headers, s.results, s.setups, s.commits, + newStateMachine(candidateView, parentState), s.params), nil } diff --git a/state/protocol/protocol_state/protocol_state_test.go b/state/protocol/protocol_state/protocol_state_test.go index 2023d4f5fb8..f7a36a20769 100644 --- a/state/protocol/protocol_state/protocol_state_test.go +++ b/state/protocol/protocol_state/protocol_state_test.go @@ -57,3 +57,38 @@ func TestProtocolState_AtBlockID(t *testing.T) { assert.Equal(t, expectedChainID, actualChainID) }) } + +// TestMutableProtocolState_Mutator tests happy path of creating a state mutator, and that `Mutator` returns an error +// if the parent protocol state has not been found. +func TestMutableProtocolState_Mutator(t *testing.T) { + protocolStateDB := storagemock.NewProtocolState(t) + globalParams := mock.NewGlobalParams(t) + headersDB := storagemock.NewHeaders(t) + resultsDB := storagemock.NewExecutionResults(t) + setupsDB := storagemock.NewEpochSetups(t) + commitsDB := storagemock.NewEpochCommits(t) + instanceParams := mock.NewInstanceParams(t) + + mutableState := NewMutableProtocolState(protocolStateDB, globalParams, + headersDB, + resultsDB, + setupsDB, + commitsDB, + instanceParams) + + t.Run("happy-path", func(t *testing.T) { + parentState := unittest.ProtocolStateFixture() + candidate := unittest.BlockHeaderFixture() + protocolStateDB.On("ByBlockID", candidate.ParentID).Return(parentState, nil) + mutator, err := mutableState.Mutator(candidate.View, candidate.ParentID) + require.NoError(t, err) + require.NotNil(t, mutator) + }) + t.Run("parent-not-found", func(t *testing.T) { + candidate := unittest.BlockHeaderFixture() + protocolStateDB.On("ByBlockID", candidate.ParentID).Return(nil, storage.ErrNotFound) + mutator, err := mutableState.Mutator(candidate.View, candidate.ParentID) + require.ErrorIs(t, err, storage.ErrNotFound) + require.Nil(t, mutator) + }) +} diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 272a955fe16..67085d6dfbb 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -8,7 +8,7 @@ import ( "github.com/onflow/flow-go/state/protocol" ) -// StateMachine is a dedicated structure that encapsulates all logic for updating protocol state. +// stateMachine is a dedicated structure that encapsulates all logic for updating protocol state. // Only protocol state updater knows how to update protocol state in a way that is consistent with the protocol. // Protocol state updater implements the following state changes: // - epoch setup: transitions current epoch from staking to setup phase, creates next epoch protocol state when processed. @@ -17,7 +17,7 @@ import ( // - setting an invalid state transition flag: sets an invalid state transition flag for current epoch and next epoch(if available). // All updates are applied to a copy of parent protocol state, so parent protocol state is not modified. // It is NOT safe to use in concurrent environment. -type StateMachine struct { +type stateMachine struct { parentState *flow.RichProtocolStateEntry state *flow.ProtocolStateEntry view uint64 @@ -32,11 +32,11 @@ type StateMachine struct { nextEpochIdentitiesLookup map[flow.Identifier]*flow.DynamicIdentityEntry // lookup for nodes active in the next epoch, may be nil or empty } -var _ ProtocolStateMachine = (*StateMachine)(nil) +var _ ProtocolStateMachine = (*stateMachine)(nil) -// NewStateMachine creates a new protocol state updater. -func NewStateMachine(view uint64, parentState *flow.RichProtocolStateEntry) *StateMachine { - updater := &StateMachine{ +// newStateMachine creates a new protocol state updater. +func newStateMachine(view uint64, parentState *flow.RichProtocolStateEntry) *stateMachine { + updater := &stateMachine{ parentState: parentState, state: parentState.ProtocolStateEntry.Copy(), view: view, @@ -45,7 +45,7 @@ func NewStateMachine(view uint64, parentState *flow.RichProtocolStateEntry) *Sta } // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. -func (u *StateMachine) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) { +func (u *stateMachine) Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) { updatedState = u.state.Copy() stateID = updatedState.ID() hasChanges = stateID != u.parentState.ID() @@ -60,7 +60,7 @@ func (u *StateMachine) Build() (updatedState *flow.ProtocolStateEntry, stateID f // As a result of this operation protocol state for the next epoch will be created. // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state -func (u *StateMachine) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { +func (u *stateMachine) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are in epoch fallback mode. } @@ -144,7 +144,7 @@ func (u *StateMachine) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { // As a result of this operation protocol state for next epoch will be committed. // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state -func (u *StateMachine) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { +func (u *stateMachine) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are going to enter epoch fallback mode } @@ -170,7 +170,7 @@ func (u *StateMachine) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { // UpdateIdentity updates identity table with new identity entry. // Should pass identity which is already present in the table, otherwise an exception will be raised. // No errors are expected during normal operations. -func (u *StateMachine) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { +func (u *stateMachine) UpdateIdentity(updated *flow.DynamicIdentityEntry) error { u.ensureLookupPopulated() prevEpochIdentity, foundInPrev := u.prevEpochIdentitiesLookup[updated.NodeID] if foundInPrev { @@ -191,7 +191,7 @@ func (u *StateMachine) UpdateIdentity(updated *flow.DynamicIdentityEntry) error } // SetInvalidStateTransitionAttempted sets a flag indicating that invalid state transition was attempted. -func (u *StateMachine) SetInvalidStateTransitionAttempted() { +func (u *stateMachine) SetInvalidStateTransitionAttempted() { u.state.InvalidStateTransitionAttempted = true } @@ -202,7 +202,7 @@ func (u *StateMachine) SetInvalidStateTransitionAttempted() { // - invalid state transition has not been attempted, // - candidate block is in the next epoch. // No errors are expected during normal operations. -func (u *StateMachine) TransitionToNextEpoch() error { +func (u *stateMachine) TransitionToNextEpoch() error { if u.state.InvalidStateTransitionAttempted { return fmt.Errorf("invalid state transition has been attempted, no transition is allowed") } @@ -230,18 +230,18 @@ func (u *StateMachine) TransitionToNextEpoch() error { // View returns the view that is associated with this state updater. // The view of the StateUpdater equals the view of the block carrying the respective updates. -func (u *StateMachine) View() uint64 { +func (u *stateMachine) View() uint64 { return u.view } // ParentState returns parent protocol state that is associated with this state updater. -func (u *StateMachine) ParentState() *flow.RichProtocolStateEntry { +func (u *stateMachine) ParentState() *flow.RichProtocolStateEntry { return u.parentState } // ensureLookupPopulated ensures that current and next epoch identities lookups are populated. // We use this to avoid populating lookups on every UpdateIdentity call. -func (u *StateMachine) ensureLookupPopulated() { +func (u *stateMachine) ensureLookupPopulated() { if len(u.currentEpochIdentitiesLookup) > 0 { return } @@ -251,7 +251,7 @@ func (u *StateMachine) ensureLookupPopulated() { // rebuildIdentityLookup re-generates lookups of *active* participants for // previous (optional, if u.state.PreviousEpoch ≠ nil), current (required) and // next epoch (optional, if u.state.NextEpoch ≠ nil). -func (u *StateMachine) rebuildIdentityLookup() { +func (u *stateMachine) rebuildIdentityLookup() { if u.state.PreviousEpoch != nil { u.prevEpochIdentitiesLookup = u.state.PreviousEpoch.ActiveIdentities.Lookup() } else { diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index 139b0d1cb81..d31ab41c6e2 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -25,7 +25,7 @@ type UpdaterSuite struct { parentBlock *flow.Header candidate *flow.Header - updater *StateMachine + updater *stateMachine } func (s *UpdaterSuite) SetupTest() { @@ -33,7 +33,7 @@ func (s *UpdaterSuite) SetupTest() { s.parentBlock = unittest.BlockHeaderFixture(unittest.HeaderWithView(s.parentProtocolState.CurrentEpochSetup.FirstView + 1)) s.candidate = unittest.BlockHeaderWithParentFixture(s.parentBlock) - s.updater = NewStateMachine(s.candidate.View, s.parentProtocolState) + s.updater = newStateMachine(s.candidate.View, s.parentProtocolState) } // TestNewUpdater tests if the constructor correctly setups invariants for updater. @@ -54,7 +54,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpoch() { candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(s.parentProtocolState.CurrentEpochSetup.FinalView + 1)) // since the candidate block is from next epoch, updater should transition to next epoch - s.updater = NewStateMachine(candidate.View, s.parentProtocolState) + s.updater = newStateMachine(candidate.View, s.parentProtocolState) err := s.updater.TransitionToNextEpoch() require.NoError(s.T(), err) updatedState, _, _ := s.updater.Build() @@ -68,7 +68,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { protocolState := unittest.ProtocolStateFixture() candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewStateMachine(candidate.View, protocolState) + updater := newStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if there is no next epoch protocol state") }) @@ -79,7 +79,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { }) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewStateMachine(candidate.View, protocolState) + updater := newStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if it is not committed") }) @@ -89,7 +89,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { }) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) - updater := NewStateMachine(candidate.View, protocolState) + updater := newStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") }) @@ -97,7 +97,7 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) candidate := unittest.BlockHeaderFixture( unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView)) - updater := NewStateMachine(candidate.View, protocolState) + updater := newStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") }) @@ -124,7 +124,7 @@ func (s *UpdaterSuite) TestSetInvalidStateTransitionAttempted() { // update protocol state with next epoch information unittest.WithNextEpochProtocolState()(s.parentProtocolState) // create new updater with next epoch information - s.updater = NewStateMachine(s.candidate.View, s.parentProtocolState) + s.updater = newStateMachine(s.candidate.View, s.parentProtocolState) s.updater.SetInvalidStateTransitionAttempted() updatedState, _, hasChanges := s.updater.Build() @@ -152,7 +152,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("invalid state transition attempted", func() { - updater := NewStateMachine(s.candidate.View, s.parentProtocolState) + updater := newStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 setup.FirstView = s.parentProtocolState.CurrentEpochSetup.FinalView + 1 @@ -176,7 +176,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { require.Equal(s.T(), flow.ZeroID, newState.NextEpoch.CommitID, "operation must be no-op") }) s.Run("happy path processing", func() { - updater := NewStateMachine(s.candidate.View, s.parentProtocolState) + updater := newStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture( unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), @@ -197,7 +197,7 @@ func (s *UpdaterSuite) TestProcessEpochCommit() { nil, ) require.NoError(s.T(), err) - updater = NewStateMachine(s.candidate.View+1, parentState) + updater = newStateMachine(s.candidate.View+1, parentState) commit := unittest.EpochCommitFixture( unittest.CommitWithCounter(setup.Counter), unittest.WithDKGFromParticipants(setup.Participants), @@ -233,7 +233,7 @@ func (s *UpdaterSuite) TestUpdateIdentityUnknownIdentity() { func (s *UpdaterSuite) TestUpdateIdentityHappyPath() { // update protocol state to have next epoch protocol state unittest.WithNextEpochProtocolState()(s.parentProtocolState) - s.updater = NewStateMachine(s.candidate.View, s.parentProtocolState) + s.updater = newStateMachine(s.candidate.View, s.parentProtocolState) currentEpochParticipants := s.parentProtocolState.CurrentEpochIdentityTable.Copy() weightChanges, err := currentEpochParticipants.Sample(3) @@ -291,7 +291,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("invalid state transition attempted", func() { - updater := NewStateMachine(s.candidate.View, s.parentProtocolState) + updater := newStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 }) @@ -303,7 +303,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.Nil(s.T(), updatedState.NextEpoch, "should not process epoch setup if invalid state transition attempted") }) s.Run("processing second epoch setup", func() { - updater := NewStateMachine(s.candidate.View, s.parentProtocolState) + updater := newStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture( unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), unittest.WithFirstView(s.parentProtocolState.CurrentEpochSetup.FinalView+1), @@ -317,7 +317,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { require.True(s.T(), protocol.IsInvalidServiceEventError(err)) }) s.Run("participants not sorted", func() { - updater := NewStateMachine(s.candidate.View, s.parentProtocolState) + updater := newStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 var err error @@ -332,7 +332,7 @@ func (s *UpdaterSuite) TestProcessEpochSetupInvariants() { conflictingIdentity := s.parentProtocolState.ProtocolStateEntry.CurrentEpoch.ActiveIdentities[0] conflictingIdentity.Dynamic.Ejected = true - updater := NewStateMachine(s.candidate.View, s.parentProtocolState) + updater := newStateMachine(s.candidate.View, s.parentProtocolState) setup := unittest.EpochSetupFixture(func(setup *flow.EpochSetup) { setup.Counter = s.parentProtocolState.CurrentEpochSetup.Counter + 1 // using same identities as in previous epoch should result in an error since @@ -469,7 +469,7 @@ func (s *UpdaterSuite) TestEpochSetupAfterIdentityChange() { // now we can use it to construct updater for next block, which will process epoch setup event. nextBlock := unittest.BlockHeaderWithParentFixture(s.candidate) - s.updater = NewStateMachine(nextBlock.View, updatedRichProtocolState) + s.updater = newStateMachine(nextBlock.View, updatedRichProtocolState) setup := unittest.EpochSetupFixture( unittest.SetupWithCounter(s.parentProtocolState.CurrentEpochSetup.Counter+1), diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index 49099400b50..f844b8bc3df 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -374,7 +374,7 @@ func (builder *EpochBuilder) addBlock(block *flow.Block) { err = stateMutator.ApplyServiceEvents(block.Payload.Seals) require.NoError(builder.t, err) - _, _, updatedStateId, _, err := stateMutator.Build() + _, _, updatedStateId, _ := stateMutator.Build() require.NoError(builder.t, err) block.Payload.ProtocolStateID = updatedStateId diff --git a/utils/unittest/protocol_state.go b/utils/unittest/protocol_state.go index e6d8f835073..ed998cc79ed 100644 --- a/utils/unittest/protocol_state.go +++ b/utils/unittest/protocol_state.go @@ -90,7 +90,7 @@ func SealBlock(t *testing.T, st protocol.ParticipantState, mutableProtocolState seals := []*flow.Seal{seal} err = stateMutator.ApplyServiceEvents(seals) require.NoError(t, err) - _, _, updatedStateId, _, err := stateMutator.Build() + _, _, updatedStateId, _ := stateMutator.Build() require.NoError(t, err) block3.SetPayload(flow.Payload{ From 4672345bfb78cc1e9941fea7c74ea459cbe9133b Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 31 Oct 2023 15:16:40 +0200 Subject: [PATCH 51/87] Linted --- state/protocol/protocol_state/mutator_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 9b7a8a265c4..53a5474f890 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -2,19 +2,19 @@ package protocol_state import ( "errors" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/storage/badger/transaction" - "github.com/onflow/flow-go/utils/rand" - "github.com/onflow/flow-go/utils/unittest" - "github.com/stretchr/testify/require" "testing" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/mock" protocolstatemock "github.com/onflow/flow-go/state/protocol/protocol_state/mock" + "github.com/onflow/flow-go/storage/badger/transaction" storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/rand" + "github.com/onflow/flow-go/utils/unittest" ) func TestProtocolStateMutator(t *testing.T) { From e7d75cd62eb535b8ff817ae2e036808b672864d2 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 31 Oct 2023 21:47:01 +0200 Subject: [PATCH 52/87] Updated godoc --- state/protocol/protocol_state/mutator.go | 11 +++++++---- state/protocol/protocol_state/protocol_state.go | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index abce735291f..75f8ca0f502 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -11,10 +11,12 @@ import ( "github.com/onflow/flow-go/storage/badger/transaction" ) -// stateMutator implements protocol.StateMutator interface. -// It has to be used for each block to update the protocol state, even if there are no state-changing -// service events sealed in candidate block. This requirement is due to the fact that protocol state -// is indexed by block ID, and we need to maintain such index. +// stateMutator is a stateful object to evolve the protocol state. It is instantiated from the parent block's protocol state. +// State-changing operations can be iteratively applied and the stateMutator will internally evolve its in-memory state. +// While the StateMutator does not modify the database, it internally tracks all necessary updates. Upon calling `Build` +// the stateMutator returns the updated protocol state, its ID and all database updates necessary for persisting the updated +// protocol state. +// stateMutator locally tracks pending DB updates, which are returned as a slice of deferred DB updates when calling `Build`. type stateMutator struct { headers storage.Headers results storage.ExecutionResults @@ -27,6 +29,7 @@ type stateMutator struct { var _ protocol.StateMutator = (*stateMutator)(nil) +// newStateMutator creates a new instance of stateMutator. func newStateMutator( headers storage.Headers, results storage.ExecutionResults, diff --git a/state/protocol/protocol_state/protocol_state.go b/state/protocol/protocol_state/protocol_state.go index 9f6a1d70542..99de050bd17 100644 --- a/state/protocol/protocol_state/protocol_state.go +++ b/state/protocol/protocol_state/protocol_state.go @@ -45,6 +45,8 @@ func (s *ProtocolState) GlobalParams() protocol.GlobalParams { return s.globalParams } +// MutableProtocolState is an implementation of the mutable interface for protocol state, it allows to evolve the protocol state +// by acting as factory for protocol.StateMutator which can be used to apply state-changing operations. type MutableProtocolState struct { ProtocolState headers storage.Headers @@ -56,6 +58,7 @@ type MutableProtocolState struct { var _ protocol.MutableProtocolState = (*MutableProtocolState)(nil) +// NewMutableProtocolState creates a new instance of MutableProtocolState. func NewMutableProtocolState( protocolStateDB storage.ProtocolState, globalParams protocol.GlobalParams, From d998fbfc961c81473b416551c2ef5928e4ef2a52 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 1 Nov 2023 14:09:56 +0200 Subject: [PATCH 53/87] Apply suggestions from code review Co-authored-by: Jordan Schalm --- state/protocol/badger/mutator.go | 3 --- state/protocol/inmem/convert.go | 7 +++---- state/protocol/protocol_state.go | 2 +- state/protocol/protocol_state/mutator.go | 5 ++--- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index 8d5cb1d3646..b1931fd5fc0 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -520,9 +520,6 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi } hasChanges, updatedState, updatedStateID, dbUpdates := stateMutator.Build() - if err != nil { - return fmt.Errorf("could not build updated protocol state: %w", err) - } if updatedStateID != candidate.Payload.ProtocolStateID { return state.NewInvalidExtensionErrorf("invalid protocol state transition detected, "+ diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index d2a19d94966..d8725f38462 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -354,12 +354,11 @@ func SnapshotFromBootstrapStateWithParams( // // CONTEXT: The EpochSetup event contains the IdentitySkeletons for each participant, thereby specifying active epoch members. // While ejection status and dynamic weight are not part of the EpochSetup event, we can supplement this information as follows: -// - Per convention, service events are delivered (asynchronously) in an *order-preserving* manner. Furthermore, weight changes or +// - Per convention, service events are delivered (asynchronously) in an *order-preserving* manner. Furthermore, // node ejection is also mediated by system smart contracts and delivered via service events. // - Therefore, the EpochSetup event contains the up-to-date snapshot of the epoch participants. Any weight changes or node ejection -// that happened before should be reflected in the EpochSetup event. Specifically, the initial weight should be reduced and ejected -// nodes should be no longer listed in the EpochSetup event. Hence, when the EpochSetup event is emitted / processed, the weight of -// all epoch participants equals their InitialWeight and the Ejected flag is false. +// that happened before should be reflected in the EpochSetup event. Specifically, ejected +// nodes should be no longer listed in the EpochSetup event. Hence, when the EpochSetup event is emitted / processed, the Ejected flag is false for all epoch participants. func ProtocolStateForBootstrapState(setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.ProtocolStateEntry { identities := make(flow.DynamicIdentityEntryList, 0, len(setup.Participants)) for _, identity := range setup.Participants { diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index d17ff1dc1cd..d85e2cec0ef 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -90,7 +90,7 @@ type StateMutator interface { // Build returns: // - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state // - updatedState: the ProtocolState after applying all updates - // - stateID: the has commitment to the `updatedState` + // - stateID: the hash commitment to the `updatedState` // - dbUpdates: database updates necessary for persisting the updated protocol state // updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 75f8ca0f502..a4ca448bf64 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -51,7 +51,7 @@ func newStateMutator( // Build returns: // - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state // - updatedState: the ProtocolState after applying all updates -// - stateID: the has commitment to the `updatedState` +// - stateID: the hash commitment to the `updatedState` // - dbUpdates: database updates necessary for persisting the updated protocol state // // updated protocol state entry, state ID and a flag indicating if there were any changes. @@ -143,8 +143,7 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func // epoch. In this case, we need to update the tentative protocol state. // We need to validate whether all information is available in the protocol // state to go to the next epoch when needed. In cases where there is a bug - // in the smart contract, it could be that this happens too late and the - // chain finalization should halt. + // in the smart contract, it could be that this happens too late and we should trigger epoch fallback mode. // block payload may not specify seals in order, so order them by block height before processing orderedSeals, err := protocol.OrderedSeals(seals, m.headers) From b3d92bcd13841e81981f89d54cbd211c51afb11d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 1 Nov 2023 14:13:18 +0200 Subject: [PATCH 54/87] Apply suggestions from code review Co-authored-by: Jordan Schalm --- state/protocol/protocol_state/mutator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 53a5474f890..284ff45706e 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -216,7 +216,7 @@ func (s *StateMutatorSuite) TestApplyServiceEventsSealsOrdered() { seals = append(seals, seal) } - // shuffle seals to make sure we order them + // shuffle seals to make sure they are not ordered in the payload, so `ApplyServiceEvents` needs to explicitly sort them. require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { seals[i], seals[j] = seals[j], seals[i] })) From 164fa95cf6f01f8e3019d9c2af2576f267cac792 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 1 Nov 2023 14:16:52 +0200 Subject: [PATCH 55/87] Renamed seals.go -> payload.go --- storage/badger/operation/{seals.go => payload.go} | 0 storage/badger/operation/{seals_test.go => payload_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename storage/badger/operation/{seals.go => payload.go} (100%) rename storage/badger/operation/{seals_test.go => payload_test.go} (100%) diff --git a/storage/badger/operation/seals.go b/storage/badger/operation/payload.go similarity index 100% rename from storage/badger/operation/seals.go rename to storage/badger/operation/payload.go diff --git a/storage/badger/operation/seals_test.go b/storage/badger/operation/payload_test.go similarity index 100% rename from storage/badger/operation/seals_test.go rename to storage/badger/operation/payload_test.go From f1965e1861dcc2f2e8b23bb60ab36abb758a2635 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Wed, 1 Nov 2023 20:49:11 +0200 Subject: [PATCH 56/87] Updated godocs --- state/protocol/protocol_state/statemachine.go | 6 ++++-- state/protocol/protocol_state/updater.go | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/state/protocol/protocol_state/statemachine.go b/state/protocol/protocol_state/statemachine.go index dcfded65209..2bdd168fc5b 100644 --- a/state/protocol/protocol_state/statemachine.go +++ b/state/protocol/protocol_state/statemachine.go @@ -2,8 +2,10 @@ package protocol_state import "github.com/onflow/flow-go/model/flow" -// ProtocolStateMachine is a dedicated interface for updating protocol state. -// It is used by the compliance layer to update protocol state when certain events that are stored in blocks are observed. +// ProtocolStateMachine implements a low-level interface for state-changing operations on the protocol state. +// It is used by higher level logic to evolve the protocol state when certain events that are stored in blocks are observed. +// The ProtocolStateMachine is stateful and internally tracks the current protocol state. A separate instance is created for +// each block that is being processed. type ProtocolStateMachine interface { // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 67085d6dfbb..4fb423b6a5a 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -8,9 +8,9 @@ import ( "github.com/onflow/flow-go/state/protocol" ) -// stateMachine is a dedicated structure that encapsulates all logic for updating protocol state. -// Only protocol state updater knows how to update protocol state in a way that is consistent with the protocol. -// Protocol state updater implements the following state changes: +// stateMachine is a dedicated structure that encapsulates all logic for evolving protocol state. +// Only protocol state machine knows how to evolve the dynamic state in a way that is consistent with the protocol. +// Protocol state machine implements the following state transitions: // - epoch setup: transitions current epoch from staking to setup phase, creates next epoch protocol state when processed. // - epoch commit: transitions current epoch from setup to commit phase, commits next epoch protocol state when processed. // - identity changes: updates identity table for current and next epoch(if available). From bdde39102ff25421d0a61efa79d429d482396e89 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Thu, 2 Nov 2023 00:39:34 -0700 Subject: [PATCH 57/87] detailed documentation of `StateMutator.ApplyServiceEvents` --- state/protocol/protocol_state.go | 59 ++++++++++++++++++++++-- state/protocol/protocol_state/mutator.go | 59 ++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index d85e2cec0ef..0da5731b008 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -95,9 +95,60 @@ type StateMutator interface { // updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) - // ApplyServiceEvents handles applying state changes which occur as a result - // of service events being included in a block payload. - // All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. - // No errors are expected during normal operation. + // ApplyServiceEvents applies the state changes that are delivered via + // sealed service events: + // - iterating over the sealed service events in order of increasing height + // - identifying state-changing service event and calling into the embedded + // ProtocolStateMachine to apply the respective state update + // - tracking deferred database updates necessary to persist the updated state + // and its data dependencies + // + // All updates only mutate the `StateMutator`'s internal in-memory copy of the + // protocol state, without changing the parent state (i.e. the state we started from). + // + // SAFETY REQUIREMENT: + // The StateMutator assumes that the proposal has passed the following correctness checks! + // - The seals in the payload continuously follow the ancestry of this fork. Specifically, + // there are no gaps in the seals. + // - The seals guarantee correctness of the sealed execution result, including the contained + // service events. This is actively checked by the verification node, whose aggregated + // approvals in the form of a seal attest to the correctness of the sealed execution result, + // including the contained. + // + // Consensus nodes actively verify protocol compliance for any block proposal they receive, + // including integrity of each seal individually as well as the seals continuously following the + // fork. Light clients only process certified blocks, which guarantees that consensus nodes already + // ran those checks and found the proposal to be valid. + // + // Details on SERVICE EVENTS: + // Consider a chain where a service event is emitted during execution of block A. + // Block B contains an execution receipt for A. Block C contains a seal for block + // A's execution result. + // + // A <- .. <- B(RA) <- .. <- C(SA) + // + // Service Events are included within execution results, which are stored + // opaquely as part of the block payload in block B. We only validate, process and persist + // the typed service event to storage once we process C, the block containing the + // seal for block A. This is because we rely on the sealing subsystem to validate + // correctness of the service event before processing it. + // Consequently, any change to the protocol state introduced by a service event + // emitted during execution of block A would only become visible when querying + // C or its descendants. + // + // Error returns: + // - Per convention, the input seals from the block payload have already confirmed to be protocol compliant. + // Hence, the service events in the sealed execution results represent the honest execution path. + // Therefore, the sealed service events should encode a valid evolution of the protocol state -- provided + // the system smart contracts are correct. + // - As we can rule out byzantine attacks as the source of failures, the only remaining sources of problems + // can be (a) bugs in the system smart contracts or (b) bugs in the node implementation. + // A service event not representing a valid state transition despite all consistency checks passing + // is interpreted as case (a) and handled internally within the StateMutator. In short, we go into Epoch + // Fallback Mode by copying the parent state (a valid state snapshot) and setting the + // `InvalidStateTransitionAttempted` flag. All subsequent Epoch-lifecycle events are ignored. + // - A consistency or sanity check failing within the StateMutator is likely the symptom of an internal bug + // in the node software or state corruption, i.e. case (b). This is the only scenario where the error return + // of this function is not nil. If such an exception is returned, continuing is not an option. ApplyServiceEvents(seals []*flow.Seal) error } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index a4ca448bf64..50ce51034fa 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -61,10 +61,61 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat return } -// ApplyServiceEvents handles applying state changes which occur as a result -// of service events being included in a block payload. -// All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. -// No errors are expected during normal operation. +// ApplyServiceEvents applies the state changes that are delivered via +// sealed service events: +// - iterating over the sealed service events in order of increasing height +// - identifying state-changing service event and calling into the embedded +// ProtocolStateMachine to apply the respective state update +// - tracking deferred database updates necessary to persist the updated state +// and its data dependencies +// +// All updates only mutate the `StateMutator`'s internal in-memory copy of the +// protocol state, without changing the parent state (i.e. the state we started from). +// +// SAFETY REQUIREMENT: +// The StateMutator assumes that the proposal has passed the following correctness checks! +// - The seals in the payload continuously follow the ancestry of this fork. Specifically, +// there are no gaps in the seals. +// - The seals guarantee correctness of the sealed execution result, including the contained +// service events. This is actively checked by the verification node, whose aggregated +// approvals in the form of a seal attest to the correctness of the sealed execution result, +// including the contained. +// +// Consensus nodes actively verify protocol compliance for any block proposal they receive, +// including integrity of each seal individually as well as the seals continuously following the +// fork. Light clients only process certified blocks, which guarantees that consensus nodes already +// ran those checks and found the proposal to be valid. +// +// Details on SERVICE EVENTS: +// Consider a chain where a service event is emitted during execution of block A. +// Block B contains an execution receipt for A. Block C contains a seal for block +// A's execution result. +// +// A <- .. <- B(RA) <- .. <- C(SA) +// +// Service Events are included within execution results, which are stored +// opaquely as part of the block payload in block B. We only validate, process and persist +// the typed service event to storage once we process C, the block containing the +// seal for block A. This is because we rely on the sealing subsystem to validate +// correctness of the service event before processing it. +// Consequently, any change to the protocol state introduced by a service event +// emitted during execution of block A would only become visible when querying +// C or its descendants. +// +// Error returns: +// - Per convention, the input seals from the block payload have already confirmed to be protocol compliant. +// Hence, the service events in the sealed execution results represent the honest execution path. +// Therefore, the sealed service events should encode a valid evolution of the protocol state -- provided +// the system smart contracts are correct. +// - As we can rule out byzantine attacks as the source of failures, the only remaining sources of problems +// can be (a) bugs in the system smart contracts or (b) bugs in the node implementation. +// A service event not representing a valid state transition despite all consistency checks passing +// is interpreted as case (a) and handled internally within the StateMutator. In short, we go into Epoch +// Fallback Mode by copying the parent state (a valid state snapshot) and setting the +// `InvalidStateTransitionAttempted` flag. All subsequent Epoch-lifecycle events are ignored. +// - A consistency or sanity check failing within the StateMutator is likely the symptom of an internal bug +// in the node software or state corruption, i.e. case (b). This is the only scenario where the error return +// of this function is not nil. If such an exception is returned, continuing is not an option. func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { dbUpdates, err := m.handleServiceEvents(seals) if err != nil { From d0bc661b63acd62fac5ed6d3f15c90a79b3ce699 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 2 Nov 2023 12:58:30 +0200 Subject: [PATCH 58/87] Apply suggestions from code review Co-authored-by: Alexander Hentschel --- state/protocol/protocol_state.go | 2 +- state/protocol/protocol_state/mutator.go | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 0da5731b008..cc88d568343 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -78,7 +78,7 @@ type MutableProtocolState interface { // StateMutator is a stateful object to evolve the protocol state. It is instantiated from the parent block's protocol state. // State-changing operations can be iteratively applied and the StateMutator will internally evolve its in-memory state. -// While the StateMutator does not modify the database, it internally tracks all necessary updates. Upon calling `Build` +// While the StateMutator does not modify the database, it internally tracks all necessary updates. Upon calling `Build`, // the StateMutator returns the updated protocol state, its ID and all database updates necessary for persisting the updated // protocol state. // The StateMutator is used by a replica's compliance layer to update protocol state when observing state-changing service in diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 50ce51034fa..78f9cc415ee 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -125,10 +125,16 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { return nil } -// handleServiceEvents handles applying state changes which occur as a result +// handleServiceEvents applies state changes which occur as a result // of service events being included in a block payload: -// - inserting incorporated service events -// - updating protocol state for the candidate block +// - iterating over the sealed service events in order of increasing height +// - identifying state-changing service event and calling into the embedded +// ProtocolStateMachine to apply the respective state update +// - tracking deferred database updates necessary to persist the updated state +// and its data dependencies +// +// All updates only mutate the internal ProtocolStateMachine's in-memory copy of the +// protocol state, without changing the parent state (i.e. the state we started from). // // Consider a chain where a service event is emitted during execution of block A. // Block B contains a receipt for A. Block C contains a seal for block A. @@ -136,7 +142,7 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { // A <- .. <- B(RA) <- .. <- C(SA) // // Service events are included within execution results, which are stored -// opaquely as part of the block payload in block B. We only validate and insert +// opaquely as part of the block payload in block B. We only validate, process and persist // the typed service event to storage once we process C, the block containing the // seal for block A. This is because we rely on the sealing subsystem to validate // correctness of the service event before processing it. @@ -144,13 +150,8 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { // emitted during execution of block A would only become visible when querying // C or its descendants. // -// This method will only apply service-event-induced state changes when the -// input block has the form of block C (ie. contains a seal for a block in -// which a service event was emitted). -// -// All updates that are incorporated in service events are applied to the protocol state by mutating protocol state updater. -// This method doesn't modify any data from self, all protocol state changes are applied to state updater. -// All other changes are returned as slice of deferred DB updates. +// All updates are only applied to the StateMutator's internal in-memory state, but not modify the Protocol State. +// All necessary updates to persist the updated state and its data dependencies are returned as slice of deferred DB updates. // // Return values: // - dbUpdates - If the service events are valid, or there are no service events, From 69842cd282c24c74972d412cb40f2fbff2ff3f62 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 2 Nov 2023 13:14:26 +0200 Subject: [PATCH 59/87] Apply suggestions from PR review --- state/protocol/protocol_state/mutator.go | 77 ++++++------------------ 1 file changed, 18 insertions(+), 59 deletions(-) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 78f9cc415ee..560df52449f 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -78,7 +78,7 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat // there are no gaps in the seals. // - The seals guarantee correctness of the sealed execution result, including the contained // service events. This is actively checked by the verification node, whose aggregated -// approvals in the form of a seal attest to the correctness of the sealed execution result, +// approvals in the form of a seal attestation to the correctness of the sealed execution result, // including the contained. // // Consensus nodes actively verify protocol compliance for any block proposal they receive, @@ -103,7 +103,7 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat // C or its descendants. // // Error returns: -// - Per convention, the input seals from the block payload have already confirmed to be protocol compliant. +// - Per convention, the input seals from the block payload have already been confirmed to be protocol compliant. // Hence, the service events in the sealed execution results represent the honest execution path. // Therefore, the sealed service events should encode a valid evolution of the protocol state -- provided // the system smart contracts are correct. @@ -117,52 +117,10 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat // in the node software or state corruption, i.e. case (b). This is the only scenario where the error return // of this function is not nil. If such an exception is returned, continuing is not an option. func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { - dbUpdates, err := m.handleServiceEvents(seals) - if err != nil { - return err - } - m.pendingDbUpdates = append(m.pendingDbUpdates, dbUpdates...) - return nil -} - -// handleServiceEvents applies state changes which occur as a result -// of service events being included in a block payload: -// - iterating over the sealed service events in order of increasing height -// - identifying state-changing service event and calling into the embedded -// ProtocolStateMachine to apply the respective state update -// - tracking deferred database updates necessary to persist the updated state -// and its data dependencies -// -// All updates only mutate the internal ProtocolStateMachine's in-memory copy of the -// protocol state, without changing the parent state (i.e. the state we started from). -// -// Consider a chain where a service event is emitted during execution of block A. -// Block B contains a receipt for A. Block C contains a seal for block A. -// -// A <- .. <- B(RA) <- .. <- C(SA) -// -// Service events are included within execution results, which are stored -// opaquely as part of the block payload in block B. We only validate, process and persist -// the typed service event to storage once we process C, the block containing the -// seal for block A. This is because we rely on the sealing subsystem to validate -// correctness of the service event before processing it. -// Consequently, any change to the protocol state introduced by a service event -// emitted during execution of block A would only become visible when querying -// C or its descendants. -// -// All updates are only applied to the StateMutator's internal in-memory state, but not modify the Protocol State. -// All necessary updates to persist the updated state and its data dependencies are returned as slice of deferred DB updates. -// -// Return values: -// - dbUpdates - If the service events are valid, or there are no service events, -// this method returns a slice of Badger operations to apply while storing the block. -// This includes operations to insert service events for blocks that include them. -// -// No errors are expected during normal operation. -func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func(*transaction.Tx) error, err error) { + var dbUpdates []func(tx *transaction.Tx) error epochFallbackTriggered, err := m.params.EpochFallbackTriggered() if err != nil { - return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) + return fmt.Errorf("could not retrieve epoch fallback status: %w", err) } parentProtocolState := m.stateMachine.ParentState() @@ -171,13 +129,13 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func // never process service events after epoch fallback is triggered if epochStatus.InvalidServiceEventIncorporated || epochFallbackTriggered { - return dbUpdates, nil + return nil } // perform protocol state transition to next epoch if next epoch is committed and we are at first block of epoch phase, err := epochStatus.Phase() if err != nil { - return nil, fmt.Errorf("could not determine epoch phase: %w", err) + return fmt.Errorf("could not determine epoch phase: %w", err) } if phase == flow.EpochPhaseCommitted { if m.stateMachine.View() > activeSetup.FinalView { @@ -185,7 +143,7 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func // most likely it will be not needed when we refactor protocol state entries and define strict safety rules. err = m.stateMachine.TransitionToNextEpoch() if err != nil { - return nil, fmt.Errorf("could not transition protocol state to next epoch: %w", err) + return fmt.Errorf("could not transition protocol state to next epoch: %w", err) } } } @@ -201,18 +159,17 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func orderedSeals, err := protocol.OrderedSeals(seals, m.headers) if err != nil { if errors.Is(err, storage.ErrNotFound) { - return nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) + return fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) } - return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + return fmt.Errorf("unexpected error ordering seals: %w", err) } for _, seal := range orderedSeals { result, err := m.results.ByID(seal.ResultID) if err != nil { - return nil, fmt.Errorf("could not get result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) + return fmt.Errorf("could not get result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) } for _, event := range result.ServiceEvents { - switch ev := event.Event.(type) { case *flow.EpochSetup: err = m.stateMachine.ProcessEpochSetup(ev) @@ -220,9 +177,9 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode m.stateMachine.SetInvalidStateTransitionAttempted() - return dbUpdates, nil + return nil } - return nil, irrecoverable.NewExceptionf("could not process epoch setup event: %w", err) + return irrecoverable.NewExceptionf("could not process epoch setup event: %w", err) } // we'll insert the setup event when we insert the block @@ -234,9 +191,9 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func if protocol.IsInvalidServiceEventError(err) { // we have observed an invalid service event, which triggers epoch fallback mode m.stateMachine.SetInvalidStateTransitionAttempted() - return dbUpdates, nil + return nil } - return nil, irrecoverable.NewExceptionf("could not process epoch commit event: %w", err) + return irrecoverable.NewExceptionf("could not process epoch commit event: %w", err) } // we'll insert the commit event when we insert the block @@ -244,9 +201,11 @@ func (m *stateMutator) handleServiceEvents(seals []*flow.Seal) (dbUpdates []func case *flow.VersionBeacon: // do nothing for now default: - return nil, fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) + return fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) } } } - return + + m.pendingDbUpdates = append(m.pendingDbUpdates, dbUpdates...) + return nil } From f57cb5e75f0025417a075de1e8626dbb0f8b7ed9 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 2 Nov 2023 13:31:59 +0200 Subject: [PATCH 60/87] Updated protocol state mutator to rely only on fork-based flag for epoch fallback. Added explicit filtering on what events we can process when in epoch fallback --- cmd/consensus/main.go | 1 - consensus/integration/nodes_test.go | 1 - .../test/cluster_switchover_test.go | 1 - engine/verification/utils/unittest/helper.go | 2 +- state/cluster/badger/mutator_test.go | 1 - state/protocol/badger/mutator_test.go | 1 - state/protocol/badger/state.go | 23 +++++----- state/protocol/protocol_state/mutator.go | 30 +++++-------- state/protocol/protocol_state/mutator_test.go | 43 +++++++------------ .../protocol/protocol_state/protocol_state.go | 5 +-- .../protocol_state/protocol_state_test.go | 8 ++-- state/protocol/util/testing.go | 2 - 12 files changed, 44 insertions(+), 74 deletions(-) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 40cebcc0dc1..04fd17cebc9 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -728,7 +728,6 @@ func main() { node.Storage.Results, node.Storage.Setups, node.Storage.EpochCommits, - node.State.Params(), ) // initialize the block builder var build module.Builder diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index 146be2450ec..6b92f218bf8 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -467,7 +467,6 @@ func createNode( resultsDB, setupsDB, commitsDB, - state.Params(), ) // initialize the block builder diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index 8798188a5fd..cb4940357fe 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -142,7 +142,6 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) refNode.Results, refNode.Setups, refNode.EpochCommits, - refNode.State.Params(), ) // when building new epoch we would like to replace fixture cluster QCs with real ones, for that we need diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 3471f00d4d8..cc08df60917 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -660,7 +660,7 @@ func bootstrapSystem( stateFixture.Storage.Results, stateFixture.Storage.Setups, stateFixture.Storage.EpochCommits, - stateFixture.State.Params()) + ) epochBuilder := unittest.NewEpochBuilder(t, mutableProtocolState, stateFixture.State) epochBuilder. UsingSetupOpts(unittest.WithParticipants(identities.ToSkeleton())). diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 409810b0efc..1d7f226a638 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -108,7 +108,6 @@ func (suite *MutatorSuite) SetupTest() { all.Results, all.Setups, all.EpochCommits, - state.Params(), ) clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index ac85d8dba33..747a8d47009 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -868,7 +868,6 @@ func TestExtendEpochTransitionValid(t *testing.T) { all.Results, all.Setups, all.EpochCommits, - state.Params(), ) calculateExpectedStateId := calculateExpectedStateId(t, mutableProtocolState) diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index aabd6d65c6a..cf36c4a417b 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -787,19 +787,18 @@ func newState( }, globalParams: globalParams, protocolStateSnapshotsDB: protocolStateSnapshots, - versionBeacons: versionBeacons, - cachedFinal: new(atomic.Pointer[cachedHeader]), - cachedSealed: new(atomic.Pointer[cachedHeader]), + protocolState: protocol_state.NewMutableProtocolState( + protocolStateSnapshots, + globalParams, + headers, + results, + setups, + commits, + ), + versionBeacons: versionBeacons, + cachedFinal: new(atomic.Pointer[cachedHeader]), + cachedSealed: new(atomic.Pointer[cachedHeader]), } - state.protocolState = protocol_state.NewMutableProtocolState( - protocolStateSnapshots, - globalParams, - headers, - results, - setups, - commits, - state.Params(), - ) return state } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 560df52449f..b824e12e73e 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -23,7 +23,6 @@ type stateMutator struct { setups storage.EpochSetups commits storage.EpochCommits stateMachine ProtocolStateMachine - params protocol.InstanceParams pendingDbUpdates []func(tx *transaction.Tx) error } @@ -36,11 +35,9 @@ func newStateMutator( setups storage.EpochSetups, commits storage.EpochCommits, stateMachine ProtocolStateMachine, - params protocol.InstanceParams, ) *stateMutator { return &stateMutator{ setups: setups, - params: params, headers: headers, results: results, commits: commits, @@ -117,27 +114,16 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat // in the node software or state corruption, i.e. case (b). This is the only scenario where the error return // of this function is not nil. If such an exception is returned, continuing is not an option. func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { - var dbUpdates []func(tx *transaction.Tx) error - epochFallbackTriggered, err := m.params.EpochFallbackTriggered() - if err != nil { - return fmt.Errorf("could not retrieve epoch fallback status: %w", err) - } - parentProtocolState := m.stateMachine.ParentState() - epochStatus := parentProtocolState.EpochStatus() - activeSetup := parentProtocolState.CurrentEpochSetup - - // never process service events after epoch fallback is triggered - if epochStatus.InvalidServiceEventIncorporated || epochFallbackTriggered { - return nil - } + invalidStateTransitionAttempted := parentProtocolState.InvalidStateTransitionAttempted // perform protocol state transition to next epoch if next epoch is committed and we are at first block of epoch - phase, err := epochStatus.Phase() + phase, err := parentProtocolState.EpochStatus().Phase() if err != nil { return fmt.Errorf("could not determine epoch phase: %w", err) } - if phase == flow.EpochPhaseCommitted { + if !invalidStateTransitionAttempted && phase == flow.EpochPhaseCommitted { + activeSetup := parentProtocolState.CurrentEpochSetup if m.stateMachine.View() > activeSetup.FinalView { // TODO: this is a temporary workaround to allow for the epoch transition to be triggered // most likely it will be not needed when we refactor protocol state entries and define strict safety rules. @@ -163,6 +149,7 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { } return fmt.Errorf("unexpected error ordering seals: %w", err) } + var dbUpdates []func(tx *transaction.Tx) error for _, seal := range orderedSeals { result, err := m.results.ByID(seal.ResultID) if err != nil { @@ -172,6 +159,9 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { for _, event := range result.ServiceEvents { switch ev := event.Event.(type) { case *flow.EpochSetup: + if invalidStateTransitionAttempted { + continue + } err = m.stateMachine.ProcessEpochSetup(ev) if err != nil { if protocol.IsInvalidServiceEventError(err) { @@ -186,6 +176,9 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { dbUpdates = append(dbUpdates, m.setups.StoreTx(ev)) case *flow.EpochCommit: + if invalidStateTransitionAttempted { + continue + } err = m.stateMachine.ProcessEpochCommit(ev) if err != nil { if protocol.IsInvalidServiceEventError(err) { @@ -205,7 +198,6 @@ func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { } } } - m.pendingDbUpdates = append(m.pendingDbUpdates, dbUpdates...) return nil } diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 284ff45706e..e5522b666c6 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -9,7 +9,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/mock" protocolstatemock "github.com/onflow/flow-go/state/protocol/protocol_state/mock" "github.com/onflow/flow-go/storage/badger/transaction" storagemock "github.com/onflow/flow-go/storage/mock" @@ -28,7 +27,6 @@ type StateMutatorSuite struct { resultsDB *storagemock.ExecutionResults setupsDB *storagemock.EpochSetups commitsDB *storagemock.EpochCommits - params *mock.InstanceParams stateMachine *protocolstatemock.ProtocolStateMachine mutator *stateMutator @@ -40,14 +38,12 @@ func (s *StateMutatorSuite) SetupTest() { s.resultsDB = storagemock.NewExecutionResults(s.T()) s.setupsDB = storagemock.NewEpochSetups(s.T()) s.commitsDB = storagemock.NewEpochCommits(s.T()) - s.params = mock.NewInstanceParams(s.T()) s.stateMachine = protocolstatemock.NewProtocolStateMachine(s.T()) - s.mutator = newStateMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.stateMachine, s.params) + s.mutator = newStateMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.stateMachine) } // TestHappyPathNoChanges tests that stateMutator doesn't cache any db updates when there are no changes. func (s *StateMutatorSuite) TestHappyPathNoChanges() { - s.params.On("EpochFallbackTriggered").Return(false, nil) parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) s.stateMachine.On("Build").Return(parentState.ProtocolStateEntry, parentState.ID(), false) @@ -62,7 +58,6 @@ func (s *StateMutatorSuite) TestHappyPathNoChanges() { // TestHappyPathHasChanges tests that stateMutator returns cached db updates when building protocol state after applying service events. func (s *StateMutatorSuite) TestHappyPathHasChanges() { - s.params.On("EpochFallbackTriggered").Return(false, nil) parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) s.stateMachine.On("Build").Return(unittest.ProtocolStateFixture().ProtocolStateEntry, @@ -92,7 +87,6 @@ func (s *StateMutatorSuite) TestHappyPathHasChanges() { // TestApplyServiceEvents_InvalidEpochSetup tests that handleServiceEvents rejects invalid epoch setup event and sets // InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochSetup() { - s.params.On("EpochFallbackTriggered").Return(false, nil) s.Run("invalid-epoch-setup", func() { parentState := unittest.ProtocolStateFixture() rootSetup := parentState.CurrentEpochSetup @@ -153,7 +147,6 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochSetup() { // TestApplyServiceEvents_InvalidEpochCommit tests that handleServiceEvents rejects invalid epoch commit event and sets // InvalidStateTransitionAttempted flag in protocol.ProtocolStateMachine. func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochCommit() { - s.params.On("EpochFallbackTriggered").Return(false, nil) s.Run("invalid-epoch-commit", func() { parentState := unittest.ProtocolStateFixture() @@ -201,7 +194,6 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochCommit() { // TestApplyServiceEventsSealsOrdered tests that handleServiceEvents processes seals in order of block height. func (s *StateMutatorSuite) TestApplyServiceEventsSealsOrdered() { - s.params.On("EpochFallbackTriggered").Return(false, nil) parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) @@ -238,7 +230,6 @@ func (s *StateMutatorSuite) TestApplyServiceEventsSealsOrdered() { // TestApplyServiceEventsTransitionToNextEpoch tests that handleServiceEvents transitions to the next epoch // when epoch has been committed, and we are at the first block of the next epoch. func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch() { - s.params.On("EpochFallbackTriggered").Return(false, nil) parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) s.stateMachine.On("ParentState").Return(parentState) // we are at the first block of the next epoch @@ -251,7 +242,6 @@ func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch() { // TestApplyServiceEventsTransitionToNextEpoch_Error tests that error that has been // observed in handleServiceEvents when transitioning to the next epoch is propagated to the caller. func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch_Error() { - s.params.On("EpochFallbackTriggered").Return(false, nil) parentState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) s.stateMachine.On("ParentState").Return(parentState) @@ -269,21 +259,20 @@ func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch_Error() // In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. // In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. func (s *StateMutatorSuite) TestApplyServiceEvents_EpochFallbackTriggered() { - s.Run("epoch-fallback-triggered", func() { - s.params.On("EpochFallbackTriggered").Return(true, nil) - parentState := unittest.ProtocolStateFixture() - s.stateMachine.On("ParentState").Return(parentState) - seals := unittest.Seal.Fixtures(2) - err := s.mutator.ApplyServiceEvents(seals) - require.NoError(s.T(), err) - }) - s.Run("invalid-service-event-incorporated", func() { - s.params.On("EpochFallbackTriggered").Return(false, nil) - parentState := unittest.ProtocolStateFixture() - parentState.InvalidStateTransitionAttempted = true - s.stateMachine.On("ParentState").Return(parentState) - seals := unittest.Seal.Fixtures(2) - err := s.mutator.ApplyServiceEvents(seals) - require.NoError(s.T(), err) + parentState := unittest.ProtocolStateFixture() + parentState.InvalidStateTransitionAttempted = true + s.stateMachine.On("ParentState").Return(parentState) + + epochCommit := unittest.EpochCommitFixture() + result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { + result.ServiceEvents = []flow.ServiceEvent{epochCommit.ServiceEvent()} }) + + block := unittest.BlockHeaderFixture() + seal := unittest.Seal.Fixture(unittest.Seal.WithBlockID(block.ID())) + s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) + s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + + err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + require.NoError(s.T(), err) } diff --git a/state/protocol/protocol_state/protocol_state.go b/state/protocol/protocol_state/protocol_state.go index 99de050bd17..357fa2b2529 100644 --- a/state/protocol/protocol_state/protocol_state.go +++ b/state/protocol/protocol_state/protocol_state.go @@ -53,7 +53,6 @@ type MutableProtocolState struct { results storage.ExecutionResults setups storage.EpochSetups commits storage.EpochCommits - params protocol.InstanceParams } var _ protocol.MutableProtocolState = (*MutableProtocolState)(nil) @@ -66,7 +65,6 @@ func NewMutableProtocolState( results storage.ExecutionResults, setups storage.EpochSetups, commits storage.EpochCommits, - params protocol.InstanceParams, ) *MutableProtocolState { return &MutableProtocolState{ ProtocolState: *NewProtocolState(protocolStateDB, globalParams), @@ -74,7 +72,6 @@ func NewMutableProtocolState( results: results, setups: setups, commits: commits, - params: params, } } @@ -88,5 +85,5 @@ func (s *MutableProtocolState) Mutator(candidateView uint64, parentID flow.Ident return nil, fmt.Errorf("could not query parent protocol state at block (%x): %w", parentID, err) } return newStateMutator(s.headers, s.results, s.setups, s.commits, - newStateMachine(candidateView, parentState), s.params), nil + newStateMachine(candidateView, parentState)), nil } diff --git a/state/protocol/protocol_state/protocol_state_test.go b/state/protocol/protocol_state/protocol_state_test.go index f7a36a20769..df070f818b2 100644 --- a/state/protocol/protocol_state/protocol_state_test.go +++ b/state/protocol/protocol_state/protocol_state_test.go @@ -67,14 +67,14 @@ func TestMutableProtocolState_Mutator(t *testing.T) { resultsDB := storagemock.NewExecutionResults(t) setupsDB := storagemock.NewEpochSetups(t) commitsDB := storagemock.NewEpochCommits(t) - instanceParams := mock.NewInstanceParams(t) - mutableState := NewMutableProtocolState(protocolStateDB, globalParams, + mutableState := NewMutableProtocolState( + protocolStateDB, + globalParams, headersDB, resultsDB, setupsDB, - commitsDB, - instanceParams) + commitsDB) t.Run("happy-path", func(t *testing.T) { parentState := unittest.ProtocolStateFixture() diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index 7aa5b0b12de..91fad8f2f6e 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -327,7 +327,6 @@ func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot pr all.Results, all.Setups, all.EpochCommits, - state.Params(), ) f(db, fullState, mutableProtocolState) }) @@ -414,7 +413,6 @@ func RunWithFullProtocolStateAndMutator(t testing.TB, rootSnapshot protocol.Snap all.Results, all.Setups, all.EpochCommits, - state.Params(), ) f(db, fullState, mutableProtocolState) }) From 6645d1e19abcf743c35bdae4de1d1c15fc3ca76e Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Thu, 2 Nov 2023 16:16:08 +0200 Subject: [PATCH 61/87] Apply suggestions from PR review --- module/builder/consensus/builder.go | 2 +- module/builder/consensus/builder_test.go | 2 +- state/protocol/badger/mutator.go | 2 +- state/protocol/badger/mutator_test.go | 2 +- state/protocol/mock/state_mutator.go | 4 ++-- state/protocol/protocol_state.go | 4 ++-- state/protocol/protocol_state/mutator.go | 2 +- state/protocol/protocol_state/mutator_test.go | 22 +++++++++---------- utils/unittest/epoch_builder.go | 2 +- utils/unittest/protocol_state.go | 2 +- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index b85b050ca6a..a65f1326625 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -639,7 +639,7 @@ func (b *Builder) createProposal(parentID flow.Identifier, if err != nil { return nil, fmt.Errorf("could not create protocol state stateMutator for view %d: %w", header.View, err) } - err = stateMutator.ApplyServiceEvents(seals) + err = stateMutator.ApplyServiceEventsFromValidatedSeals(seals) if err != nil { return nil, fmt.Errorf("could not apply service events as leader: %w", err) } diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index 5ebbe3c39c3..bec649d1e9f 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -419,7 +419,7 @@ func (bs *BuilderSuite) SetupTest() { bs.stateMutator.On("Mutator", mock.Anything, mock.Anything).Return( func(_ uint64, _ flow.Identifier) realproto.StateMutator { stateMutator := protocol.NewStateMutator(bs.T()) - stateMutator.On("ApplyServiceEvents", mock.Anything).Return(nil) + stateMutator.On("ApplyServiceEventsFromValidatedSeals", mock.Anything).Return(nil) stateMutator.On("Build").Return(false, nil, flow.Identifier{}, nil, nil) return stateMutator }, func(_ uint64, _ flow.Identifier) error { diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index b1931fd5fc0..6458ea6706b 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -514,7 +514,7 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi } // apply any state changes from service events sealed by this block - err = stateMutator.ApplyServiceEvents(candidate.Payload.Seals) + err = stateMutator.ApplyServiceEventsFromValidatedSeals(candidate.Payload.Seals) if err != nil { return fmt.Errorf("could not process service events: %w", err) } diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 747a8d47009..ed69805561d 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -2683,7 +2683,7 @@ func calculateExpectedStateId(t *testing.T, mutableProtocolState realprotocol.Mu stateMutator, err := mutableProtocolState.Mutator(header.View, header.ParentID) require.NoError(t, err) - err = stateMutator.ApplyServiceEvents(seals) + err = stateMutator.ApplyServiceEventsFromValidatedSeals(seals) require.NoError(t, err) _, _, expectedStateID, _ := stateMutator.Build() diff --git a/state/protocol/mock/state_mutator.go b/state/protocol/mock/state_mutator.go index 9ba2dbab0c1..5bb20d4561e 100644 --- a/state/protocol/mock/state_mutator.go +++ b/state/protocol/mock/state_mutator.go @@ -14,8 +14,8 @@ type StateMutator struct { mock.Mock } -// ApplyServiceEvents provides a mock function with given fields: seals -func (_m *StateMutator) ApplyServiceEvents(seals []*flow.Seal) error { +// ApplyServiceEventsFromValidatedSeals provides a mock function with given fields: seals +func (_m *StateMutator) ApplyServiceEventsFromValidatedSeals(seals []*flow.Seal) error { ret := _m.Called(seals) var r0 error diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index cc88d568343..2c738d6bfc6 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -95,7 +95,7 @@ type StateMutator interface { // updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) - // ApplyServiceEvents applies the state changes that are delivered via + // ApplyServiceEventsFromValidatedSeals applies the state changes that are delivered via // sealed service events: // - iterating over the sealed service events in order of increasing height // - identifying state-changing service event and calling into the embedded @@ -150,5 +150,5 @@ type StateMutator interface { // - A consistency or sanity check failing within the StateMutator is likely the symptom of an internal bug // in the node software or state corruption, i.e. case (b). This is the only scenario where the error return // of this function is not nil. If such an exception is returned, continuing is not an option. - ApplyServiceEvents(seals []*flow.Seal) error + ApplyServiceEventsFromValidatedSeals(seals []*flow.Seal) error } diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index b824e12e73e..4bde7ca1aa9 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -113,7 +113,7 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat // - A consistency or sanity check failing within the StateMutator is likely the symptom of an internal bug // in the node software or state corruption, i.e. case (b). This is the only scenario where the error return // of this function is not nil. If such an exception is returned, continuing is not an option. -func (m *stateMutator) ApplyServiceEvents(seals []*flow.Seal) error { +func (m *stateMutator) ApplyServiceEventsFromValidatedSeals(seals []*flow.Seal) error { parentProtocolState := m.stateMachine.ParentState() invalidStateTransitionAttempted := parentProtocolState.InvalidStateTransitionAttempted diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index e5522b666c6..1d8a9b9a9d1 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -47,7 +47,7 @@ func (s *StateMutatorSuite) TestHappyPathNoChanges() { parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) s.stateMachine.On("Build").Return(parentState.ProtocolStateEntry, parentState.ID(), false) - err := s.mutator.ApplyServiceEvents([]*flow.Seal{}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{}) require.NoError(s.T(), err) hasChanges, updatedState, updatedStateID, dbUpdates := s.mutator.Build() require.False(s.T(), hasChanges) @@ -76,7 +76,7 @@ func (s *StateMutatorSuite) TestHappyPathHasChanges() { s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(nil).Once() s.setupsDB.On("StoreTx", epochSetup).Return(func(*transaction.Tx) error { return nil }).Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.NoError(s.T(), err) _, _, _, dbUpdates := s.mutator.Build() @@ -111,7 +111,7 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochSetup() { s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(protocol.NewInvalidServiceEventErrorf("")).Once() s.stateMachine.On("SetInvalidStateTransitionAttempted").Return().Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.NoError(s.T(), err) }) s.Run("process-epoch-setup-exception", func() { @@ -138,7 +138,7 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochSetup() { exception := errors.New("exception") s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(exception).Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.Error(s.T(), err) require.False(s.T(), protocol.IsInvalidServiceEventError(err)) }) @@ -165,7 +165,7 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochCommit() { s.stateMachine.On("ProcessEpochCommit", epochCommit).Return(protocol.NewInvalidServiceEventErrorf("")).Once() s.stateMachine.On("SetInvalidStateTransitionAttempted").Return().Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.NoError(s.T(), err) }) s.Run("process-epoch-commit-exception", func() { @@ -186,7 +186,7 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_InvalidEpochCommit() { exception := errors.New("exception") s.stateMachine.On("ProcessEpochCommit", epochCommit).Return(exception).Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.Error(s.T(), err) require.False(s.T(), protocol.IsInvalidServiceEventError(err)) }) @@ -208,12 +208,12 @@ func (s *StateMutatorSuite) TestApplyServiceEventsSealsOrdered() { seals = append(seals, seal) } - // shuffle seals to make sure they are not ordered in the payload, so `ApplyServiceEvents` needs to explicitly sort them. + // shuffle seals to make sure they are not ordered in the payload, so `ApplyServiceEventsFromValidatedSeals` needs to explicitly sort them. require.NoError(s.T(), rand.Shuffle(uint(len(seals)), func(i, j uint) { seals[i], seals[j] = seals[j], seals[i] })) - err := s.mutator.ApplyServiceEvents(seals) + err := s.mutator.ApplyServiceEventsFromValidatedSeals(seals) require.NoError(s.T(), err) // assert that results were queried in order of executed block height @@ -235,7 +235,7 @@ func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch() { // we are at the first block of the next epoch s.stateMachine.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) s.stateMachine.On("TransitionToNextEpoch").Return(nil).Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{}) require.NoError(s.T(), err) } @@ -249,7 +249,7 @@ func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch_Error() s.stateMachine.On("View").Return(parentState.CurrentEpochSetup.FinalView + 1) exception := errors.New("exception") s.stateMachine.On("TransitionToNextEpoch").Return(exception).Once() - err := s.mutator.ApplyServiceEvents([]*flow.Seal{}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{}) require.ErrorIs(s.T(), err, exception) require.False(s.T(), protocol.IsInvalidServiceEventError(err)) } @@ -273,6 +273,6 @@ func (s *StateMutatorSuite) TestApplyServiceEvents_EpochFallbackTriggered() { s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) - err := s.mutator.ApplyServiceEvents([]*flow.Seal{seal}) + err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.NoError(s.T(), err) } diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index f844b8bc3df..fbd0131e835 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -371,7 +371,7 @@ func (builder *EpochBuilder) addBlock(block *flow.Block) { stateMutator, err := builder.mutableProtocolState.Mutator(block.Header.View, block.Header.ParentID) require.NoError(builder.t, err) - err = stateMutator.ApplyServiceEvents(block.Payload.Seals) + err = stateMutator.ApplyServiceEventsFromValidatedSeals(block.Payload.Seals) require.NoError(builder.t, err) _, _, updatedStateId, _ := stateMutator.Build() diff --git a/utils/unittest/protocol_state.go b/utils/unittest/protocol_state.go index ed998cc79ed..13aa0162eaa 100644 --- a/utils/unittest/protocol_state.go +++ b/utils/unittest/protocol_state.go @@ -88,7 +88,7 @@ func SealBlock(t *testing.T, st protocol.ParticipantState, mutableProtocolState stateMutator, err := mutableProtocolState.Mutator(block3.Header.View, block3.Header.ParentID) require.NoError(t, err) seals := []*flow.Seal{seal} - err = stateMutator.ApplyServiceEvents(seals) + err = stateMutator.ApplyServiceEventsFromValidatedSeals(seals) require.NoError(t, err) _, _, updatedStateId, _ := stateMutator.Build() require.NoError(t, err) From 7ad9e93894051d376ac75172ace4b596fca6b8ae Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Thu, 2 Nov 2023 21:54:53 -0700 Subject: [PATCH 62/87] minor documentation tweaks --- state/protocol/protocol_state/mutator.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index 4bde7ca1aa9..d306b25e710 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -58,7 +58,7 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat return } -// ApplyServiceEvents applies the state changes that are delivered via +// ApplyServiceEventsFromValidatedSeals applies the state changes that are delivered via // sealed service events: // - iterating over the sealed service events in order of increasing height // - identifying state-changing service event and calling into the embedded @@ -75,7 +75,7 @@ func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStat // there are no gaps in the seals. // - The seals guarantee correctness of the sealed execution result, including the contained // service events. This is actively checked by the verification node, whose aggregated -// approvals in the form of a seal attestation to the correctness of the sealed execution result, +// approvals in the form of a seal attest to the correctness of the sealed execution result, // including the contained. // // Consensus nodes actively verify protocol compliance for any block proposal they receive, @@ -144,6 +144,9 @@ func (m *stateMutator) ApplyServiceEventsFromValidatedSeals(seals []*flow.Seal) // block payload may not specify seals in order, so order them by block height before processing orderedSeals, err := protocol.OrderedSeals(seals, m.headers) if err != nil { + // Per API contract, the input seals must have already passed verification, which necessitates + // successful ordering. Hence, calling protocol.OrderedSeals with the same inputs that succeeded + // earlier now failed. In all cases, this is an exception. if errors.Is(err, storage.ErrNotFound) { return fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) } From 139d60ad645237ceee4d7986b8ae3a715dba8675 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 3 Nov 2023 10:08:28 +0200 Subject: [PATCH 63/87] Update state/protocol/protocol_state/mutator.go Co-authored-by: Alexander Hentschel --- state/protocol/protocol_state/mutator.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index d306b25e710..f26a69fa084 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -147,10 +147,7 @@ func (m *stateMutator) ApplyServiceEventsFromValidatedSeals(seals []*flow.Seal) // Per API contract, the input seals must have already passed verification, which necessitates // successful ordering. Hence, calling protocol.OrderedSeals with the same inputs that succeeded // earlier now failed. In all cases, this is an exception. - if errors.Is(err, storage.ErrNotFound) { - return fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) - } - return fmt.Errorf("unexpected error ordering seals: %w", err) + return irrecoverable.NewExceptionf("ordering already validated seals unexpectedly failed: %w", err) } var dbUpdates []func(tx *transaction.Tx) error for _, seal := range orderedSeals { From 054435d9684ef8ad6dea5eb87b0e4e5bedf1246c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Fri, 3 Nov 2023 14:22:35 +0200 Subject: [PATCH 64/87] Linted --- state/protocol/protocol_state/mutator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index f26a69fa084..e99d8a545d7 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -1,7 +1,6 @@ package protocol_state import ( - "errors" "fmt" "github.com/onflow/flow-go/model/flow" From 203f0490e6c983459d135584157a50db4819f21a Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 5 Nov 2023 13:27:27 -0800 Subject: [PATCH 65/87] =?UTF-8?q?extending=20and=20polishing=20documentati?= =?UTF-8?q?on=20of=20verification=20logic=20for=20EpochSetup,=20EpochCommi?= =?UTF-8?q?t,=20and=20Cluster=20unpacking=20logic=20(NodeIDs=20=E2=86=92?= =?UTF-8?q?=20IdentitySkeletons)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model/flow/factory/cluster_list.go | 45 +++++++---- state/protocol/validity.go | 123 +++++++++++++---------------- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/model/flow/factory/cluster_list.go b/model/flow/factory/cluster_list.go index 49ffae74447..05980d4dc17 100644 --- a/model/flow/factory/cluster_list.go +++ b/model/flow/factory/cluster_list.go @@ -7,43 +7,57 @@ import ( "github.com/onflow/flow-go/model/flow/order" ) -// NewClusterList creates a new cluster list based on the given cluster assignment -// and the provided list of identities. +// NewClusterList creates a new cluster list based on the given cluster assignment and the provided list of identities. +// the implementation enforces the following protocol rules and errors in case they are violated: +// +// (a) input `collectors` only contains collector nodes with positive weight +// (b) collectors have unique node IDs +// (c) each collector is assigned exactly to one cluster and is only listed once within that cluster +// +// Furthermore, for each cluster (i.e. element in `assignments`) we enforce: +// +// (d) cluster contains at least one collector (i.e. is not empty) +// (e) cluster is composed of known nodes, i.e. for each nodeID in `assignments` an IdentitySkeleton is given in `collectors` +// (f) cluster assignment lists the nodes in canonical ordering +// // The caller must ensure each assignment contains identities ordered in canonical order, so that // each cluster in the returned cluster list is ordered in canonical order as well. If not, // an error will be returned. +// This is a side-effect-free function. Any error return indicates that the input violate protocol rules. func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentitySkeletonList) (flow.ClusterList, error) { - // build a lookup for all the identities by node identifier lookup := collectors.Lookup() - for _, collector := range collectors { + for _, collector := range collectors { // enforce (a): `collectors` only contains collector nodes with positive weight + if collector.Role != flow.RoleCollection { + return nil, fmt.Errorf("node %v is not a collector", collector.NodeID) + } + if collector.InitialWeight == 0 { + return nil, fmt.Errorf("node %v has zero weight", collector.NodeID) + } lookup[collector.NodeID] = collector } - if len(lookup) != len(collectors) { + if len(lookup) != len(collectors) { // enforce (b): collectors have unique node IDs return nil, fmt.Errorf("duplicate collector in list") } - // assignments only contains the NodeIDs for each cluster. In the following, we - // substitute them with the respective IdentitySkeletons. + // assignments only contains the NodeIDs for each cluster. In the following, we substitute them with the respective IdentitySkeletons. clusters := make(flow.ClusterList, 0, len(assignments)) for i, participants := range assignments { cluster := make(flow.IdentitySkeletonList, 0, len(participants)) - if len(participants) == 0 { + if len(participants) == 0 { // enforce (d): each cluster contains at least one collector (i.e. is not empty) return nil, fmt.Errorf("particpants in assignment list is empty, cluster index %v", i) } - // Check assignments is sorted in canonical order - prev := participants[0] - + prev := participants[0] // for checking that cluster participants are listed in canonical order for i, participantID := range participants { - participant, found := lookup[participantID] + participant, found := lookup[participantID] // enforce (e): for each nodeID in assignments an IdentitySkeleton is given in `collectors` if !found { return nil, fmt.Errorf("could not find collector identity (%x)", participantID) } cluster = append(cluster, participant) - delete(lookup, participantID) + delete(lookup, participantID) // enforce (c) part 1: reject repeated assignment of the same node - if i > 0 { + if i > 0 { // enforce (f): canonical ordering if !order.IdentifierCanonical(prev, participantID) { return nil, fmt.Errorf("the assignments is not sorted in canonical order in cluster index %v, prev %v, next %v", i, prev, participantID) @@ -55,8 +69,7 @@ func NewClusterList(assignments flow.AssignmentList, collectors flow.IdentitySke clusters = append(clusters, cluster) } - // check that every collector was assigned - if len(lookup) != 0 { + if len(lookup) != 0 { // enforce (c) part 2: every collector was assigned return nil, fmt.Errorf("missing collector assignments (%s)", lookup) } diff --git a/state/protocol/validity.go b/state/protocol/validity.go index c28926d5e28..473be22822c 100644 --- a/state/protocol/validity.go +++ b/state/protocol/validity.go @@ -9,27 +9,22 @@ import ( "github.com/onflow/flow-go/model/flow/order" ) -// IsValidExtendingEpochSetup checks whether an epoch setup service being -// added to the state is valid. In addition to intrinsic validity, we also -// check that it is valid w.r.t. the previous epoch setup event, and the -// current epoch status. -// Assumes all inputs besides extendingSetup are already validated. +// IsValidExtendingEpochSetup checks whether an EpochSetup service event being added to the state is valid. +// In addition to intrinsic validity, we also check that it is valid w.r.t. the previous epoch setup event, +// and the current epoch status. +// CAUTION: This function assumes that all inputs besides extendingCommit are already validated. // Expected errors during normal operations: // * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status func IsValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { - // We should only have a single epoch setup event per epoch. - if status.NextEpoch.SetupID != flow.ZeroID { + // Enforce EpochSetup is valid w.r.t to current epoch state + if status.NextEpoch.SetupID != flow.ZeroID { // We should only have a single epoch setup event per epoch. // true iff EpochSetup event for NEXT epoch was already included before return NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) } - - // The setup event should have the counter increased by one. - if extendingSetup.Counter != activeSetup.Counter+1 { + if extendingSetup.Counter != activeSetup.Counter+1 { // The setup event should have the counter increased by one. return NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) } - - // The first view needs to be exactly one greater than the current epoch final view - if extendingSetup.FirstView != activeSetup.FinalView+1 { + if extendingSetup.FirstView != activeSetup.FinalView+1 { // The first view needs to be exactly one greater than the current epoch final view return NewInvalidServiceEventErrorf( "next epoch first view must be exactly 1 more than current epoch final view (%d != %d+1)", extendingSetup.FirstView, @@ -37,31 +32,37 @@ func IsValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *fl ) } - // Finally, the epoch setup event must contain all necessary information. + // Enforce the EpochSetup event is syntactically correct err := VerifyEpochSetup(extendingSetup, true) if err != nil { return NewInvalidServiceEventErrorf("invalid epoch setup: %w", err) } - return nil } -// VerifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. -// The boolean parameter `verifyNetworkAddress` controls, whether we want to permit -// nodes to share a networking address. -// This is a side-effect-free function. Any error return indicates that the -// EpochSetup event is not compliant with protocol rules. +// VerifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. The boolean parameter `verifyNetworkAddress` +// controls, whether we want to permit nodes to share a networking address. +// This is a side-effect-free function. Any error return indicates that the EpochSetup event is not compliant with protocol rules. func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { - // STEP 1: general sanity checks - // the seed needs to be at least minimum length + // 1. CHECK: Enforce protocol compliance of Epoch parameters: + // - RandomSource of entropy in Epoch Setup event should the protocol-prescribed length + // - first view must be before final view if len(setup.RandomSource) != flow.EpochSetupRandomSourceLength { return fmt.Errorf("seed has incorrect length (%d != %d)", len(setup.RandomSource), flow.EpochSetupRandomSourceLength) } + if setup.FirstView >= setup.FinalView { + return fmt.Errorf("first view (%d) must be before final view (%d)", setup.FirstView, setup.FinalView) + } - // STEP 2: sanity checks of all nodes listed as participants - // there should be no duplicate node IDs + // 2. CHECK: Enforce protocol compliance active participants: + // (a) each has a unique node ID, + // (b) each has a unique network address (if `verifyNetworkAddress` is true), + // (c) participants are sorted in canonical order. + // Note that the system smart contracts manage the identity table as an unordered set! For the protocol state, we desire a fixed + // ordering to simplify various implementation details, like the DKG. Therefore, we order identities in `flow.EpochSetup` during + // conversion from cadence to Go in the function `convert.ServiceEvent(flow.ChainID, flow.Event)` in package `model/convert` identLookup := make(map[flow.Identifier]struct{}) - for _, participant := range setup.Participants { + for _, participant := range setup.Participants { // (a) enforce uniqueness of NodeIDs _, ok := identLookup[participant.NodeID] if ok { return fmt.Errorf("duplicate node identifier (%x)", participant.NodeID) @@ -69,8 +70,7 @@ func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { identLookup[participant.NodeID] = struct{}{} } - if verifyNetworkAddress { - // there should be no duplicate node addresses + if verifyNetworkAddress { // (b) enforce uniqueness of networking address addrLookup := make(map[string]struct{}) for _, participant := range setup.Participants { _, ok := addrLookup[participant.Address] @@ -81,83 +81,72 @@ func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { } } - // the participants must be listed in canonical order - if !setup.Participants.Sorted(order.Canonical[flow.IdentitySkeleton]) { + if !setup.Participants.Sorted(order.Canonical[flow.IdentitySkeleton]) { // (c) enforce canonical ordering return fmt.Errorf("participants are not canonically ordered") } - // STEP 3: sanity checks for individual roles - // IMPORTANT: here we remove all nodes with zero weight, as they are allowed to partake - // in communication but not in respective node functions + // 3. CHECK: Enforce sufficient number of nodes for each role + // IMPORTANT: here we remove all nodes with zero weight, as they are allowed to partake in communication but not in respective node functions activeParticipants := setup.Participants.Filter(filter.HasInitialWeight[flow.IdentitySkeleton](true)) - - // we need at least one node of each role - roles := make(map[flow.Role]uint) + activeNodeCountByRole := make(map[flow.Role]uint) for _, participant := range activeParticipants { - roles[participant.Role]++ + activeNodeCountByRole[participant.Role]++ } - if roles[flow.RoleConsensus] < 1 { + if activeNodeCountByRole[flow.RoleConsensus] < 1 { return fmt.Errorf("need at least one consensus node") } - if roles[flow.RoleCollection] < 1 { + if activeNodeCountByRole[flow.RoleCollection] < 1 { return fmt.Errorf("need at least one collection node") } - if roles[flow.RoleExecution] < 1 { + if activeNodeCountByRole[flow.RoleExecution] < 1 { return fmt.Errorf("need at least one execution node") } - if roles[flow.RoleVerification] < 1 { + if activeNodeCountByRole[flow.RoleVerification] < 1 { return fmt.Errorf("need at least one verification node") } - // first view must be before final view - if setup.FirstView >= setup.FinalView { - return fmt.Errorf("first view (%d) must be before final view (%d)", setup.FirstView, setup.FinalView) - } - - // we need at least one collection cluster - if len(setup.Assignments) == 0 { + // 4. CHECK: Enforce protocol compliance of collector cluster assignment + // (0) there is at least one collector cluster + // (a) assignment only contains nodes with collector role and positive weight + // (b) collectors have unique node IDs + // (c) each collector is assigned exactly to one cluster and is only listed once within that cluster + // (d) cluster contains at least one collector (i.e. is not empty) + // (e) cluster is composed of known nodes + // (f) cluster assignment lists the nodes in canonical ordering + if len(setup.Assignments) == 0 { // enforce (0): at least one cluster return fmt.Errorf("need at least one collection cluster") } - - // the collection cluster assignments need to be valid + // Unpacking the cluster assignments (NodeIDs → IdentitySkeletons) enforces (a) - (f) _, err := factory.NewClusterList(setup.Assignments, activeParticipants.Filter(filter.HasRole[flow.IdentitySkeleton](flow.RoleCollection))) if err != nil { return fmt.Errorf("invalid cluster assignments: %w", err) } - return nil } -// IsValidExtendingEpochCommit checks whether an epoch commit service being -// added to the state is valid. In addition to intrinsic validity, we also -// check that it is valid w.r.t. the previous epoch setup event, and the -// current epoch status. -// Assumes all inputs besides extendingCommit are already validated. +// IsValidExtendingEpochCommit checks whether an EpochCommit service event being added to the state is valid. +// In addition to intrinsic validity, we also check that it is valid w.r.t. the previous epoch setup event, and +// the current epoch status. +// CAUTION: This function assumes that all inputs besides extendingCommit are already validated. // Expected errors during normal operations: // * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status func IsValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { - - // We should only have a single epoch commit event per epoch. - if status.NextEpoch.CommitID != flow.ZeroID { - // true iff EpochCommit event for NEXT epoch was already included before + // Enforce EpochSetup is valid w.r.t to current epoch state + if status.NextEpoch.CommitID != flow.ZeroID { // We should only have a single epoch commit event per epoch. return NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) } - - // The epoch setup event needs to happen before the commit. - if status.NextEpoch.SetupID == flow.ZeroID { + if status.NextEpoch.SetupID == flow.ZeroID { // The epoch setup event needs to happen before the commit. return NewInvalidServiceEventErrorf("missing epoch setup for epoch commit") } - - // The commit event should have the counter increased by one. - if extendingCommit.Counter != activeSetup.Counter+1 { + if extendingCommit.Counter != activeSetup.Counter+1 { // The commit event should have the counter increased by one. return NewInvalidServiceEventErrorf("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) } + // Enforce the EpochSetup event is syntactically correct and compatible with the respective EpochSetup err := IsValidEpochCommit(extendingCommit, extendingSetup) if err != nil { return NewInvalidServiceEventErrorf("invalid epoch commit: %s", err) } - return nil } @@ -166,7 +155,6 @@ func IsValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSet // Expected errors during normal operations: // * protocol.InvalidServiceEventError if the EpochCommit is invalid func IsValidEpochCommit(commit *flow.EpochCommit, setup *flow.EpochSetup) error { - if len(setup.Assignments) != len(commit.ClusterQCs) { return NewInvalidServiceEventErrorf("number of clusters (%d) does not number of QCs (%d)", len(setup.Assignments), len(commit.ClusterQCs)) } @@ -184,6 +172,5 @@ func IsValidEpochCommit(commit *flow.EpochCommit, setup *flow.EpochSetup) error if len(participants) != len(commit.DKGParticipantKeys) { return NewInvalidServiceEventErrorf("participant list (len=%d) does not match dkg key list (len=%d)", len(participants), len(commit.DKGParticipantKeys)) } - return nil } From 17f69ed71b32abe714620d7a960960daa0589010 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 5 Nov 2023 13:40:54 -0800 Subject: [PATCH 66/87] fixed formatting problem of goDoc --- state/protocol/validity.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/state/protocol/validity.go b/state/protocol/validity.go index 473be22822c..d75f1e3a4d6 100644 --- a/state/protocol/validity.go +++ b/state/protocol/validity.go @@ -106,13 +106,13 @@ func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { } // 4. CHECK: Enforce protocol compliance of collector cluster assignment - // (0) there is at least one collector cluster - // (a) assignment only contains nodes with collector role and positive weight - // (b) collectors have unique node IDs - // (c) each collector is assigned exactly to one cluster and is only listed once within that cluster - // (d) cluster contains at least one collector (i.e. is not empty) - // (e) cluster is composed of known nodes - // (f) cluster assignment lists the nodes in canonical ordering + // (0) there is at least one collector cluster + // (a) assignment only contains nodes with collector role and positive weight + // (b) collectors have unique node IDs + // (c) each collector is assigned exactly to one cluster and is only listed once within that cluster + // (d) cluster contains at least one collector (i.e. is not empty) + // (e) cluster is composed of known nodes + // (f) cluster assignment lists the nodes in canonical ordering if len(setup.Assignments) == 0 { // enforce (0): at least one cluster return fmt.Errorf("need at least one collection cluster") } From 6da91f5692ddc7af34fd1663db872cf20cce3867 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 5 Nov 2023 13:42:11 -0800 Subject: [PATCH 67/87] fixed formatting problem of goDoc --- state/protocol/validity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/state/protocol/validity.go b/state/protocol/validity.go index d75f1e3a4d6..5c1a1cbe88b 100644 --- a/state/protocol/validity.go +++ b/state/protocol/validity.go @@ -59,8 +59,8 @@ func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { // (b) each has a unique network address (if `verifyNetworkAddress` is true), // (c) participants are sorted in canonical order. // Note that the system smart contracts manage the identity table as an unordered set! For the protocol state, we desire a fixed - // ordering to simplify various implementation details, like the DKG. Therefore, we order identities in `flow.EpochSetup` during - // conversion from cadence to Go in the function `convert.ServiceEvent(flow.ChainID, flow.Event)` in package `model/convert` + // ordering to simplify various implementation details, like the DKG. Therefore, we order identities in `flow.EpochSetup` during + // conversion from cadence to Go in the function `convert.ServiceEvent(flow.ChainID, flow.Event)` in package `model/convert` identLookup := make(map[flow.Identifier]struct{}) for _, participant := range setup.Participants { // (a) enforce uniqueness of NodeIDs _, ok := identLookup[participant.NodeID] From 2c863f3d37941aa2bb7de1c13d5f52d6d47ef9be Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 5 Nov 2023 13:50:48 -0800 Subject: [PATCH 68/87] compactified code --- state/protocol/protocol_state/protocol_state.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/state/protocol/protocol_state/protocol_state.go b/state/protocol/protocol_state/protocol_state.go index 357fa2b2529..b6d56835d02 100644 --- a/state/protocol/protocol_state/protocol_state.go +++ b/state/protocol/protocol_state/protocol_state.go @@ -84,6 +84,5 @@ func (s *MutableProtocolState) Mutator(candidateView uint64, parentID flow.Ident if err != nil { return nil, fmt.Errorf("could not query parent protocol state at block (%x): %w", parentID, err) } - return newStateMutator(s.headers, s.results, s.setups, s.commits, - newStateMachine(candidateView, parentState)), nil + return newStateMutator(s.headers, s.results, s.setups, s.commits, newStateMachine(candidateView, parentState)), nil } From 1e03f9fd471c3e3190ed0801927f4028f8625ef2 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sun, 5 Nov 2023 13:55:07 -0800 Subject: [PATCH 69/87] minor extension of test description --- state/protocol/protocol_state/protocol_state_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/state/protocol/protocol_state/protocol_state_test.go b/state/protocol/protocol_state/protocol_state_test.go index df070f818b2..45469cb6651 100644 --- a/state/protocol/protocol_state/protocol_state_test.go +++ b/state/protocol/protocol_state/protocol_state_test.go @@ -27,7 +27,7 @@ func TestProtocolState_AtBlockID(t *testing.T) { globalParams := mock.NewGlobalParams(t) protocolState := NewProtocolState(protocolStateDB, globalParams) - t.Run("exists", func(t *testing.T) { + t.Run("retrieve state for existing blocks", func(t *testing.T) { dynamicProtocolState, err := protocolState.AtBlockID(blockID) require.NoError(t, err) @@ -37,20 +37,20 @@ func TestProtocolState_AtBlockID(t *testing.T) { require.NoError(t, err) require.NotEqual(t, dynamicProtocolState.Identities(), other.Identities()) }) - t.Run("not-exists", func(t *testing.T) { + t.Run("retrieve state for non-existing block yields storage.ErrNotFound error", func(t *testing.T) { blockID := unittest.IdentifierFixture() protocolStateDB.On("ByBlockID", blockID).Return(nil, storage.ErrNotFound).Once() _, err := protocolState.AtBlockID(blockID) require.ErrorIs(t, err, storage.ErrNotFound) }) - t.Run("exception", func(t *testing.T) { + t.Run("exception during retrieve is propagated", func(t *testing.T) { blockID := unittest.IdentifierFixture() exception := errors.New("exception") protocolStateDB.On("ByBlockID", blockID).Return(nil, exception).Once() _, err := protocolState.AtBlockID(blockID) require.ErrorIs(t, err, exception) }) - t.Run("global-params", func(t *testing.T) { + t.Run("retrieve global-params", func(t *testing.T) { expectedChainID := flow.Testnet globalParams.On("ChainID").Return(expectedChainID, nil).Once() actualChainID := protocolState.GlobalParams().ChainID() From 98aec8852eb43520cf2a42fca012fe586c112f27 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Tue, 7 Nov 2023 15:23:38 -0800 Subject: [PATCH 70/87] extended and polished docs for StateMutator --- state/protocol/protocol_state.go | 21 ++++++++++++++------- state/protocol/protocol_state/mutator.go | 22 ++++++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 2c738d6bfc6..37e23569758 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -78,20 +78,27 @@ type MutableProtocolState interface { // StateMutator is a stateful object to evolve the protocol state. It is instantiated from the parent block's protocol state. // State-changing operations can be iteratively applied and the StateMutator will internally evolve its in-memory state. -// While the StateMutator does not modify the database, it internally tracks all necessary updates. Upon calling `Build`, -// the StateMutator returns the updated protocol state, its ID and all database updates necessary for persisting the updated -// protocol state. +// While the StateMutator does not modify the database, it internally tracks the necessary database updates to persist its +// dependencies (specifically EpochSetup and EpochCommit events). Upon calling `Build` the StateMutator returns the updated +// protocol state, its ID and all database updates necessary for persisting the updated protocol state. +// // The StateMutator is used by a replica's compliance layer to update protocol state when observing state-changing service in // blocks. It is used by the primary in the block building process to obtain the correct protocol state for a proposal. // Specifically, the leader may include state-changing service events in the block payload. The flow protocol prescribes that // the proposal needs to include the ID of the protocol state, _after_ processing the payload incl. all state-changing events. // Therefore, the leader instantiates a StateMutator, applies the service events to it and builds the updated protocol state ID. +// +// Not safe for concurrent use. type StateMutator interface { // Build returns: - // - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state - // - updatedState: the ProtocolState after applying all updates - // - stateID: the hash commitment to the `updatedState` - // - dbUpdates: database updates necessary for persisting the updated protocol state + // - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state + // - updatedState: the ProtocolState after applying all updates. + // - stateID: the hash commitment to the `updatedState` + // - dbUpdates: database updates necessary for persisting the updated protocol state's *dependencies*. + // If hasChanges is false, updatedState is empty. Caution: persisting the `updatedState` itself and adding + // it to the relevant indices is _not_ in `dbUpdates`. Persisting and indexing `updatedState` is the responsibility + // of the calling code (specifically `FollowerState`). + // // updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index e99d8a545d7..f225252b4b2 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -12,10 +12,17 @@ import ( // stateMutator is a stateful object to evolve the protocol state. It is instantiated from the parent block's protocol state. // State-changing operations can be iteratively applied and the stateMutator will internally evolve its in-memory state. -// While the StateMutator does not modify the database, it internally tracks all necessary updates. Upon calling `Build` -// the stateMutator returns the updated protocol state, its ID and all database updates necessary for persisting the updated -// protocol state. -// stateMutator locally tracks pending DB updates, which are returned as a slice of deferred DB updates when calling `Build`. +// While the StateMutator does not modify the database, it internally tracks the necessary database updates to persist its +// dependencies (specifically EpochSetup and EpochCommit events). Upon calling `Build` the stateMutator returns the updated +// protocol state, its ID and all database updates necessary for persisting the updated protocol state. +// +// The StateMutator is used by a replica's compliance layer to update protocol state when observing state-changing service in +// blocks. It is used by the primary in the block building process to obtain the correct protocol state for a proposal. +// Specifically, the leader may include state-changing service events in the block payload. The flow protocol prescribes that +// the proposal needs to include the ID of the protocol state, _after_ processing the payload incl. all state-changing events. +// Therefore, the leader instantiates a StateMutator, applies the service events to it and builds the updated protocol state ID. +// +// Not safe for concurrent use. type stateMutator struct { headers storage.Headers results storage.ExecutionResults @@ -46,9 +53,12 @@ func newStateMutator( // Build returns: // - hasChanges: flag whether there were any changes; otherwise, `updatedState` and `stateID` equal the parent state -// - updatedState: the ProtocolState after applying all updates +// - updatedState: the ProtocolState after applying all updates. // - stateID: the hash commitment to the `updatedState` -// - dbUpdates: database updates necessary for persisting the updated protocol state +// - dbUpdates: database updates necessary for persisting the updated protocol state's *dependencies*. +// If hasChanges is false, updatedState is empty. Caution: persisting the `updatedState` itself and adding +// it to the relevant indices is _not_ in `dbUpdates`. Persisting and indexing `updatedState` is the responsibility +// of the calling code (specifically `FollowerState`). // // updated protocol state entry, state ID and a flag indicating if there were any changes. func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) { From df1fd29e05138460c96e2e0d298dd44b58406639 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Tue, 7 Nov 2023 15:33:03 -0800 Subject: [PATCH 71/87] goDoc polishing of state machine --- state/protocol/protocol_state.go | 1 + state/protocol/protocol_state/statemachine.go | 7 ++++++ state/protocol/protocol_state/updater.go | 22 +++++++++++-------- storage/badger/operation/prefix.go | 2 +- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 37e23569758..5eb324b3f3c 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -62,6 +62,7 @@ type ProtocolState interface { // - (nil, storage.ErrNotFound) - if there is no protocol state associated with given block ID. // - (nil, exception) - any other error should be treated as exception. AtBlockID(blockID flow.Identifier) (DynamicProtocolState, error) + // GlobalParams returns params that are the same for all nodes in the network. GlobalParams() GlobalParams } diff --git a/state/protocol/protocol_state/statemachine.go b/state/protocol/protocol_state/statemachine.go index 2bdd168fc5b..61db6b4ad13 100644 --- a/state/protocol/protocol_state/statemachine.go +++ b/state/protocol/protocol_state/statemachine.go @@ -9,6 +9,7 @@ import "github.com/onflow/flow-go/model/flow" type ProtocolStateMachine interface { // Build returns updated protocol state entry, state ID and a flag indicating if there were any changes. Build() (updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, hasChanges bool) + // ProcessEpochSetup updates current protocol state with data from epoch setup event. // Processing epoch setup event also affects identity table for current epoch. // Observing an epoch setup event, transitions protocol state from staking to setup phase, we stop returning @@ -17,6 +18,7 @@ type ProtocolStateMachine interface { // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state ProcessEpochSetup(epochSetup *flow.EpochSetup) error + // ProcessEpochCommit updates current protocol state with data from epoch commit event. // Observing an epoch setup commit, transitions protocol state from setup to commit phase, at this point we have // finished construction of the next epoch. @@ -24,15 +26,18 @@ type ProtocolStateMachine interface { // Expected errors during normal operations: // - `protocol.InvalidServiceEventError` if the service event is invalid or is not a valid state transition for the current protocol state ProcessEpochCommit(epochCommit *flow.EpochCommit) error + // UpdateIdentity updates identity table with new identity entry. // Should pass identity which is already present in the table, otherwise an exception will be raised. // TODO: This function currently modifies both current+next identities based on input. // This is incompatible with the design doc, and needs to be updated to modify current/next epoch separately // No errors are expected during normal operations. UpdateIdentity(updated *flow.DynamicIdentityEntry) error + // SetInvalidStateTransitionAttempted sets a flag indicating that invalid state transition was attempted. // Such transition can be detected by compliance layer. SetInvalidStateTransitionAttempted() + // TransitionToNextEpoch discards current protocol state and transitions to the next epoch. // Epoch transition is only allowed when: // - next epoch has been set up, @@ -40,9 +45,11 @@ type ProtocolStateMachine interface { // - candidate block is in the next epoch. // No errors are expected during normal operations. TransitionToNextEpoch() error + // View returns the view that is associated with this state updater. // The view of the ProtocolStateMachine equals the view of the block carrying the respective updates. View() uint64 + // ParentState returns parent protocol state that is associated with this state updater. ParentState() *flow.RichProtocolStateEntry } diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 4fb423b6a5a..4e72beedcea 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -8,15 +8,19 @@ import ( "github.com/onflow/flow-go/state/protocol" ) -// stateMachine is a dedicated structure that encapsulates all logic for evolving protocol state. -// Only protocol state machine knows how to evolve the dynamic state in a way that is consistent with the protocol. -// Protocol state machine implements the following state transitions: -// - epoch setup: transitions current epoch from staking to setup phase, creates next epoch protocol state when processed. -// - epoch commit: transitions current epoch from setup to commit phase, commits next epoch protocol state when processed. -// - identity changes: updates identity table for current and next epoch(if available). -// - setting an invalid state transition flag: sets an invalid state transition flag for current epoch and next epoch(if available). -// All updates are applied to a copy of parent protocol state, so parent protocol state is not modified. -// It is NOT safe to use in concurrent environment. +// stateMachine is a dedicated structure that encapsulates all logic for evolving protocol state, based on the content +// of a new block. It guarantees protocol-compliant evolution of the protocol state by implementing the +// following state transitions: +// - epoch setup: transitions current epoch from staking to setup phase, creates next epoch protocol state when processed. +// - epoch commit: transitions current epoch from setup to commit phase, commits next epoch protocol state when processed. +// - epoch transition: on the first block of the new epoch (Formally, the block's parent is still in the last epoch, +// while the new block has a view in the next epoch. Caution: the block's view is not necessarily the first view +// in the epoch, as there might be leader failures) +// - identity changes: updates identity table for previous (if available), current, and next epoch (if available). +// - setting an invalid state transition flag: sets an invalid state transition flag for current epoch and next epoch(if available). +// +// All updates are applied to a copy of parent protocol state, so parent protocol state is not modified. The stateMachine internally +// tracks the current protocol state. A separate instance should be created for each block to processing the updates therein. type stateMachine struct { parentState *flow.RichProtocolStateEntry state *flow.ProtocolStateEntry diff --git a/storage/badger/operation/prefix.go b/storage/badger/operation/prefix.go index e91de2d4144..3a69f0011c1 100644 --- a/storage/badger/operation/prefix.go +++ b/storage/badger/operation/prefix.go @@ -60,8 +60,8 @@ const ( // codes for indexing multiple identifiers by identifier // NOTE: 51 was used for identity indexes before epochs codeBlockChildren = 50 // index mapping block ID to children blocks - codePayloadGuarantees = 52 // index mapping block ID to payload guarantees codePayloadProtocolStateID = 51 // index mapping block ID to payload protocol state ID + codePayloadGuarantees = 52 // index mapping block ID to payload guarantees codePayloadSeals = 53 // index mapping block ID to payload seals codeCollectionBlock = 54 // index mapping collection ID to block ID codeOwnBlockReceipt = 55 // index mapping block ID to execution receipt ID for execution nodes From 2e59e756d9af51ae03e36a3492b7e411d27a9aa3 Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Tue, 7 Nov 2023 15:39:44 -0800 Subject: [PATCH 72/87] typo --- model/flow/factory/cluster_list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/flow/factory/cluster_list.go b/model/flow/factory/cluster_list.go index 05980d4dc17..ed2763db0d3 100644 --- a/model/flow/factory/cluster_list.go +++ b/model/flow/factory/cluster_list.go @@ -8,7 +8,7 @@ import ( ) // NewClusterList creates a new cluster list based on the given cluster assignment and the provided list of identities. -// the implementation enforces the following protocol rules and errors in case they are violated: +// The implementation enforces the following protocol rules and errors in case they are violated: // // (a) input `collectors` only contains collector nodes with positive weight // (b) collectors have unique node IDs From 79e079eacf2ca6bc4f04bc455693e9b0ddf5fb7a Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Tue, 7 Nov 2023 16:56:55 -0800 Subject: [PATCH 73/87] minor revision of test names for better readability. e.g. TestHappyPathNoChanges -> TestOnHappyPathNoDbCchanges --- state/protocol/protocol_state/mutator_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 1d8a9b9a9d1..99688aaa54c 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -42,8 +42,8 @@ func (s *StateMutatorSuite) SetupTest() { s.mutator = newStateMutator(s.headersDB, s.resultsDB, s.setupsDB, s.commitsDB, s.stateMachine) } -// TestHappyPathNoChanges tests that stateMutator doesn't cache any db updates when there are no changes. -func (s *StateMutatorSuite) TestHappyPathNoChanges() { +// TestOnHappyPathNoDbChanges tests that stateMutator doesn't cache any db updates when there are no changes. +func (s *StateMutatorSuite) TestOnHappyPathNoDbChanges() { parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) s.stateMachine.On("Build").Return(parentState.ProtocolStateEntry, parentState.ID(), false) @@ -56,8 +56,8 @@ func (s *StateMutatorSuite) TestHappyPathNoChanges() { require.Empty(s.T(), dbUpdates) } -// TestHappyPathHasChanges tests that stateMutator returns cached db updates when building protocol state after applying service events. -func (s *StateMutatorSuite) TestHappyPathHasChanges() { +// TestHappyPathWithDbChanges tests that stateMutator returns cached db updates when building protocol state after applying service events. +func (s *StateMutatorSuite) TestHappyPathWithDbChanges() { parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) s.stateMachine.On("Build").Return(unittest.ProtocolStateFixture().ProtocolStateEntry, From 6a3eb1ca2e8e0eacae8436eef139d7f20586b53a Mon Sep 17 00:00:00 2001 From: Alexander Hentschel Date: Sat, 11 Nov 2023 00:05:47 -0800 Subject: [PATCH 74/87] comment formating --- model/flow/payload.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model/flow/payload.go b/model/flow/payload.go index fd3d6fc5798..1d16afe445d 100644 --- a/model/flow/payload.go +++ b/model/flow/payload.go @@ -16,10 +16,10 @@ type Payload struct { Seals []*Seal Receipts ExecutionReceiptMetaList Results ExecutionResultList - // ProtocolStateID is the root hash of protocol state. - // Per convention, this is the resulting state after applying all identity-changing operations potentially contained in the block. - // The block payload itself is validated wrt to the protocol state committed to by its parent. Thereby, we are only - // accepting protocol states that have been certified by a valid QC. + // ProtocolStateID is the root hash of protocol state. Per convention, this is the resulting + // state after applying all identity-changing operations potentially contained in the block. + // The block payload itself is validated wrt to the protocol state committed to by its parent. + // Thereby, we are only accepting protocol states that have been certified by a valid QC. ProtocolStateID Identifier } From cb08aa2c6cc3a78960d32ade89432375d3ec83c1 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 14:09:45 +0200 Subject: [PATCH 75/87] Updated modules for insecure and integration --- go.sum | 2 -- insecure/go.sum | 2 -- integration/go.sum | 2 -- 3 files changed, 6 deletions(-) diff --git a/go.sum b/go.sum index 44ed724d7a8..22bd56ca024 100644 --- a/go.sum +++ b/go.sum @@ -1344,8 +1344,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 h1:SX8OhYbyKBExhy4qEDR/Hw6MVTBTzlDb8LfCHfFyte4= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231110212518-071176bb06b8 h1:AsIyEDiwxpRAifgBK/0lsjEdNfqFtHqNHedpMeHoA4w= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231110212518-071176bb06b8/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= diff --git a/insecure/go.sum b/insecure/go.sum index 0f5ab3b32ed..c876aed50eb 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1299,8 +1299,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 h1:SX8OhYbyKBExhy4qEDR/Hw6MVTBTzlDb8LfCHfFyte4= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231110212518-071176bb06b8 h1:AsIyEDiwxpRAifgBK/0lsjEdNfqFtHqNHedpMeHoA4w= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231110212518-071176bb06b8/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= diff --git a/integration/go.sum b/integration/go.sum index 898f772fae9..2ada1506358 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1393,8 +1393,6 @@ github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63 h1:SX8OhYbyKBExhy4qEDR/Hw6MVTBTzlDb8LfCHfFyte4= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231018182244-e72527c55c63/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231110212518-071176bb06b8 h1:AsIyEDiwxpRAifgBK/0lsjEdNfqFtHqNHedpMeHoA4w= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231110212518-071176bb06b8/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= From 602358fb0dc930d3615302ec1a64193890d6b440 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 14:22:04 +0200 Subject: [PATCH 76/87] Apply suggestions from PR review --- state/protocol/badger/state.go | 6 +++--- state/protocol/badger/validity_test.go | 8 ++++---- state/protocol/protocol_state/statemachine.go | 1 - state/protocol/protocol_state/updater_test.go | 6 ++++-- state/protocol/validity.go | 6 +++--- state/protocol/validity_test.go | 8 ++++---- utils/unittest/epoch_builder.go | 2 +- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index cf36c4a417b..36ed3874b44 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -465,7 +465,7 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get previous epoch commit event: %w", err) } - if err := protocol.VerifyEpochSetup(setup, verifyNetworkAddress); err != nil { + if err := protocol.IsValidEpochSetup(setup, verifyNetworkAddress); err != nil { return fmt.Errorf("invalid setup: %w", err) } if err := protocol.IsValidEpochCommit(commit, setup); err != nil { @@ -495,7 +495,7 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get current epoch commit event: %w", err) } - if err := protocol.VerifyEpochSetup(setup, verifyNetworkAddress); err != nil { + if err := protocol.IsValidEpochSetup(setup, verifyNetworkAddress); err != nil { return fmt.Errorf("invalid setup: %w", err) } if err := protocol.IsValidEpochCommit(commit, setup); err != nil { @@ -521,7 +521,7 @@ func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, verifyNetworkAddr return fmt.Errorf("could not get next epoch setup event: %w", err) } - if err := protocol.VerifyEpochSetup(setup, verifyNetworkAddress); err != nil { + if err := protocol.IsValidEpochSetup(setup, verifyNetworkAddress); err != nil { return fmt.Errorf("invalid setup: %w", err) } diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index fbc76946ec1..e051cac8481 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -22,7 +22,7 @@ func TestEpochSetupValidity(t *testing.T) { // set an invalid final view for the first epoch setup.FinalView = setup.FirstView - err := protocol.VerifyEpochSetup(setup, true) + err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) @@ -33,7 +33,7 @@ func TestEpochSetupValidity(t *testing.T) { var err error setup.Participants, err = setup.Participants.Shuffle() require.NoError(t, err) - err = protocol.VerifyEpochSetup(setup, true) + err = protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) @@ -44,7 +44,7 @@ func TestEpochSetupValidity(t *testing.T) { collector := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))[0] setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) - err := protocol.VerifyEpochSetup(setup, true) + err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) @@ -53,7 +53,7 @@ func TestEpochSetupValidity(t *testing.T) { setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) setup.RandomSource = unittest.SeedFixture(crypto.SeedMinLenDKG - 1) - err := protocol.VerifyEpochSetup(setup, true) + err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) } diff --git a/state/protocol/protocol_state/statemachine.go b/state/protocol/protocol_state/statemachine.go index 61db6b4ad13..5cc42ae6847 100644 --- a/state/protocol/protocol_state/statemachine.go +++ b/state/protocol/protocol_state/statemachine.go @@ -35,7 +35,6 @@ type ProtocolStateMachine interface { UpdateIdentity(updated *flow.DynamicIdentityEntry) error // SetInvalidStateTransitionAttempted sets a flag indicating that invalid state transition was attempted. - // Such transition can be detected by compliance layer. SetInvalidStateTransitionAttempted() // TransitionToNextEpoch discards current protocol state and transitions to the next epoch. diff --git a/state/protocol/protocol_state/updater_test.go b/state/protocol/protocol_state/updater_test.go index d31ab41c6e2..c369d731276 100644 --- a/state/protocol/protocol_state/updater_test.go +++ b/state/protocol/protocol_state/updater_test.go @@ -91,7 +91,8 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView + 1)) updater := newStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() - require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") + require.Error(s.T(), err, "should not allow transition to next epoch if we are in Epoch Fallback Mode, "+ + "i.e. an invalid state transition was attempted") }) s.Run("candidate block is not from next epoch", func() { protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) @@ -99,7 +100,8 @@ func (s *UpdaterSuite) TestTransitionToNextEpochNotAllowed() { unittest.HeaderWithView(protocolState.CurrentEpochSetup.FinalView)) updater := newStateMachine(candidate.View, protocolState) err := updater.TransitionToNextEpoch() - require.Error(s.T(), err, "should not allow transition to next epoch if next block is not first block from next epoch") + require.Error(s.T(), err, "should not allow transition to next epoch if we are in Epoch Fallback Mode, "+ + "i.e. an invalid state transition was attempted") }) } diff --git a/state/protocol/validity.go b/state/protocol/validity.go index 5c1a1cbe88b..603ca480fec 100644 --- a/state/protocol/validity.go +++ b/state/protocol/validity.go @@ -33,17 +33,17 @@ func IsValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *fl } // Enforce the EpochSetup event is syntactically correct - err := VerifyEpochSetup(extendingSetup, true) + err := IsValidEpochSetup(extendingSetup, true) if err != nil { return NewInvalidServiceEventErrorf("invalid epoch setup: %w", err) } return nil } -// VerifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. The boolean parameter `verifyNetworkAddress` +// IsValidEpochSetup checks whether an `EpochSetup` event is syntactically correct. The boolean parameter `verifyNetworkAddress` // controls, whether we want to permit nodes to share a networking address. // This is a side-effect-free function. Any error return indicates that the EpochSetup event is not compliant with protocol rules. -func VerifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { +func IsValidEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { // 1. CHECK: Enforce protocol compliance of Epoch parameters: // - RandomSource of entropy in Epoch Setup event should the protocol-prescribed length // - first view must be before final view diff --git a/state/protocol/validity_test.go b/state/protocol/validity_test.go index 9eef6b29e4a..87ec08c21b6 100644 --- a/state/protocol/validity_test.go +++ b/state/protocol/validity_test.go @@ -21,7 +21,7 @@ func TestEpochSetupValidity(t *testing.T) { // set an invalid final view for the first epoch setup.FinalView = setup.FirstView - err := protocol.VerifyEpochSetup(setup, true) + err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) @@ -32,7 +32,7 @@ func TestEpochSetupValidity(t *testing.T) { var err error setup.Participants, err = setup.Participants.Shuffle() require.NoError(t, err) - err = protocol.VerifyEpochSetup(setup, true) + err = protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) @@ -43,7 +43,7 @@ func TestEpochSetupValidity(t *testing.T) { collector := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))[0] setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) - err := protocol.VerifyEpochSetup(setup, true) + err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) @@ -52,7 +52,7 @@ func TestEpochSetupValidity(t *testing.T) { setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) setup.RandomSource = unittest.SeedFixture(crypto.SeedMinLenDKG - 1) - err := protocol.VerifyEpochSetup(setup, true) + err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) } diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index fbd0131e835..b1cb5193933 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -366,7 +366,7 @@ func (builder *EpochBuilder) BuildBlocks(n uint) { } // addBlock adds the given block to the state by: extending the state, -// finalizing the block, marking the block as valid, and caching the block. +// finalizing the block, and caching the block. func (builder *EpochBuilder) addBlock(block *flow.Block) { stateMutator, err := builder.mutableProtocolState.Mutator(block.Header.View, block.Header.ParentID) require.NoError(builder.t, err) From cbf414996b9a489a5a1d5d2ed32bd84a8ec8e3ff Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 14:37:38 +0200 Subject: [PATCH 77/87] Apply suggestions from PR review --- state/protocol/protocol_state/updater.go | 8 ++---- state/protocol/validity.go | 34 +++++++++++------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/state/protocol/protocol_state/updater.go b/state/protocol/protocol_state/updater.go index 4e72beedcea..8a48d2b9da6 100644 --- a/state/protocol/protocol_state/updater.go +++ b/state/protocol/protocol_state/updater.go @@ -68,7 +68,7 @@ func (u *stateMachine) ProcessEpochSetup(epochSetup *flow.EpochSetup) error { if u.state.InvalidStateTransitionAttempted { return nil // won't process new events if we are in epoch fallback mode. } - err := protocol.IsValidExtendingEpochSetup(epochSetup, u.parentState.CurrentEpochSetup, u.parentState.EpochStatus()) + err := protocol.IsValidExtendingEpochSetup(epochSetup, u.parentState.ProtocolStateEntry, u.parentState.CurrentEpochSetup) if err != nil { return fmt.Errorf("invalid epoch setup event: %w", err) } @@ -158,11 +158,7 @@ func (u *stateMachine) ProcessEpochCommit(epochCommit *flow.EpochCommit) error { if u.state.NextEpoch.CommitID != flow.ZeroID { return protocol.NewInvalidServiceEventErrorf("protocol state has already a commit event") } - err := protocol.IsValidExtendingEpochCommit( - epochCommit, - u.parentState.NextEpochSetup, - u.parentState.CurrentEpochSetup, - u.parentState.EpochStatus()) + err := protocol.IsValidExtendingEpochCommit(epochCommit, u.parentState.ProtocolStateEntry, u.parentState.NextEpochSetup) if err != nil { return fmt.Errorf("invalid epoch commit event: %w", err) } diff --git a/state/protocol/validity.go b/state/protocol/validity.go index 603ca480fec..48c3c6830b3 100644 --- a/state/protocol/validity.go +++ b/state/protocol/validity.go @@ -2,7 +2,6 @@ package protocol import ( "fmt" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" @@ -15,20 +14,20 @@ import ( // CAUTION: This function assumes that all inputs besides extendingCommit are already validated. // Expected errors during normal operations: // * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status -func IsValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { +func IsValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, protocolStateEntry *flow.ProtocolStateEntry, currentEpochSetupEvent *flow.EpochSetup) error { // Enforce EpochSetup is valid w.r.t to current epoch state - if status.NextEpoch.SetupID != flow.ZeroID { // We should only have a single epoch setup event per epoch. + if protocolStateEntry.NextEpoch != nil { // We should only have a single epoch setup event per epoch. // true iff EpochSetup event for NEXT epoch was already included before - return NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) + return NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", protocolStateEntry.NextEpoch.SetupID) } - if extendingSetup.Counter != activeSetup.Counter+1 { // The setup event should have the counter increased by one. - return NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) + if extendingSetup.Counter != currentEpochSetupEvent.Counter+1 { // The setup event should have the counter increased by one. + return NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", currentEpochSetupEvent.Counter, extendingSetup.Counter) } - if extendingSetup.FirstView != activeSetup.FinalView+1 { // The first view needs to be exactly one greater than the current epoch final view + if extendingSetup.FirstView != currentEpochSetupEvent.FinalView+1 { // The first view needs to be exactly one greater than the current epoch final view return NewInvalidServiceEventErrorf( "next epoch first view must be exactly 1 more than current epoch final view (%d != %d+1)", extendingSetup.FirstView, - activeSetup.FinalView, + currentEpochSetupEvent.FinalView, ) } @@ -129,21 +128,18 @@ func IsValidEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error // the current epoch status. // CAUTION: This function assumes that all inputs besides extendingCommit are already validated. // Expected errors during normal operations: -// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status -func IsValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { - // Enforce EpochSetup is valid w.r.t to current epoch state - if status.NextEpoch.CommitID != flow.ZeroID { // We should only have a single epoch commit event per epoch. - return NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) - } - if status.NextEpoch.SetupID == flow.ZeroID { // The epoch setup event needs to happen before the commit. +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch +func IsValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, protocolStateEntry *flow.ProtocolStateEntry, nextEpochSetupEvent *flow.EpochSetup) error { + // The epoch setup event needs to happen before the commit. + if protocolStateEntry.NextEpoch == nil { return NewInvalidServiceEventErrorf("missing epoch setup for epoch commit") } - if extendingCommit.Counter != activeSetup.Counter+1 { // The commit event should have the counter increased by one. - return NewInvalidServiceEventErrorf("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) + // Enforce EpochSetup is valid w.r.t to current epoch state + if protocolStateEntry.NextEpoch.CommitID != flow.ZeroID { // We should only have a single epoch commit event per epoch. + return NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", protocolStateEntry.NextEpoch.CommitID) } - // Enforce the EpochSetup event is syntactically correct and compatible with the respective EpochSetup - err := IsValidEpochCommit(extendingCommit, extendingSetup) + err := IsValidEpochCommit(extendingCommit, nextEpochSetupEvent) if err != nil { return NewInvalidServiceEventErrorf("invalid epoch commit: %s", err) } From 8264c2acfb635eb6ce000369490567280212930a Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 22:09:29 +0200 Subject: [PATCH 78/87] Introduced DeferredDBUpdated --- state/protocol/protocol_state.go | 2 +- state/protocol/protocol_state/mutator.go | 6 +++--- storage/badger/transaction/tx.go | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/state/protocol/protocol_state.go b/state/protocol/protocol_state.go index 5eb324b3f3c..42b3ebab391 100644 --- a/state/protocol/protocol_state.go +++ b/state/protocol/protocol_state.go @@ -101,7 +101,7 @@ type StateMutator interface { // of the calling code (specifically `FollowerState`). // // updated protocol state entry, state ID and a flag indicating if there were any changes. - Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) + Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates transaction.DeferredDBUpdate) // ApplyServiceEventsFromValidatedSeals applies the state changes that are delivered via // sealed service events: diff --git a/state/protocol/protocol_state/mutator.go b/state/protocol/protocol_state/mutator.go index f225252b4b2..59ca3edfd11 100644 --- a/state/protocol/protocol_state/mutator.go +++ b/state/protocol/protocol_state/mutator.go @@ -29,7 +29,7 @@ type stateMutator struct { setups storage.EpochSetups commits storage.EpochCommits stateMachine ProtocolStateMachine - pendingDbUpdates []func(tx *transaction.Tx) error + pendingDbUpdates transaction.DeferredDBUpdate } var _ protocol.StateMutator = (*stateMutator)(nil) @@ -61,7 +61,7 @@ func newStateMutator( // of the calling code (specifically `FollowerState`). // // updated protocol state entry, state ID and a flag indicating if there were any changes. -func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates []func(tx *transaction.Tx) error) { +func (m *stateMutator) Build() (hasChanges bool, updatedState *flow.ProtocolStateEntry, stateID flow.Identifier, dbUpdates transaction.DeferredDBUpdate) { updatedState, stateID, hasChanges = m.stateMachine.Build() dbUpdates = m.pendingDbUpdates return @@ -158,7 +158,7 @@ func (m *stateMutator) ApplyServiceEventsFromValidatedSeals(seals []*flow.Seal) // earlier now failed. In all cases, this is an exception. return irrecoverable.NewExceptionf("ordering already validated seals unexpectedly failed: %w", err) } - var dbUpdates []func(tx *transaction.Tx) error + var dbUpdates transaction.DeferredDBUpdate for _, seal := range orderedSeals { result, err := m.results.ByID(seal.ResultID) if err != nil { diff --git a/storage/badger/transaction/tx.go b/storage/badger/transaction/tx.go index b6a001f7c16..e79461ed1e7 100644 --- a/storage/badger/transaction/tx.go +++ b/storage/badger/transaction/tx.go @@ -30,6 +30,10 @@ type Tx struct { callbacks []func() } +// DeferredDBUpdate is a shorthand notation for an anonymous function. The function takes +// a badger transaction as input to runs some database updates as part of that transaction. +type DeferredDBUpdate = []func(tx *Tx) error + // OnSucceed adds a callback to execute after the batch has been successfully flushed. // Useful for implementing the cache where we will only cache after the batch of database // operations has been successfully applied. From 52bacec50d6ce1cab1196bb28c532e31bc1a657c Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 22:49:01 +0200 Subject: [PATCH 79/87] Renamed ProtocolStateForBoostrapState -> ProtocolStateFromEpochServiceEvents --- cmd/bootstrap/cmd/block.go | 2 +- consensus/integration/nodes_test.go | 2 +- engine/collection/test/cluster_switchover_test.go | 2 +- integration/testnet/network.go | 2 +- module/builder/collection/builder_test.go | 4 ++-- state/cluster/badger/mutator_test.go | 2 +- state/protocol/badger/snapshot_test.go | 2 +- state/protocol/inmem/convert.go | 6 +++--- utils/unittest/fixtures.go | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/bootstrap/cmd/block.go b/cmd/bootstrap/cmd/block.go index 747344ffa32..46d905f0227 100644 --- a/cmd/bootstrap/cmd/block.go +++ b/cmd/bootstrap/cmd/block.go @@ -33,7 +33,7 @@ func constructRootBlock(rootHeader *flow.Header, setup *flow.EpochSetup, commit Seals: nil, Receipts: nil, Results: nil, - ProtocolStateID: inmem.ProtocolStateForBootstrapState(setup, commit).ID(), + ProtocolStateID: inmem.ProtocolStateFromEpochServiceEvents(setup, commit).ID(), }) return block } diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index a2ac0ef4dd2..6aacd343e69 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -281,7 +281,7 @@ func createRootBlockData(participantData *run.ParticipantData) (*flow.Block, *fl }, ) - root.SetPayload(flow.Payload{ProtocolStateID: inmem.ProtocolStateForBootstrapState(setup, commit).ID()}) + root.SetPayload(flow.Payload{ProtocolStateID: inmem.ProtocolStateFromEpochServiceEvents(setup, commit).ID()}) result := unittest.BootstrapExecutionResultFixture(root, unittest.GenesisStateCommitment) result.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent(), commit.ServiceEvent()} diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index cb4940357fe..618fc327bc9 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -99,7 +99,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) commit.ClusterQCs = rootClusterQCs seal.ResultID = result.ID() - root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState(setup, commit).ID() + root.Payload.ProtocolStateID = inmem.ProtocolStateFromEpochServiceEvents(setup, commit).ID() tc.root, err = inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 11c9dec7438..e5af9220bd9 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -1145,7 +1145,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl Header: rootHeader, } root.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID( - inmem.ProtocolStateForBootstrapState(epochSetup, epochCommit).ID()))) + inmem.ProtocolStateFromEpochServiceEvents(epochSetup, epochCommit).ID()))) cdcRandomSource, err := cadence.NewString(hex.EncodeToString(randomSource)) if err != nil { diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 2eaefdd1344..1d83b3d4553 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -86,7 +86,7 @@ func (suite *BuilderSuite) SetupTest() { // ensure we don't enter a new epoch for tests that build many blocks result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = root.Header.View + 100000 seal.ResultID = result.ID() - root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState( + root.Payload.ProtocolStateID = inmem.ProtocolStateFromEpochServiceEvents( result.ServiceEvents[0].Event.(*flow.EpochSetup), result.ServiceEvents[1].Event.(*flow.EpochCommit), ).ID() @@ -95,7 +95,7 @@ func (suite *BuilderSuite) SetupTest() { suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) - root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState( + root.Payload.ProtocolStateID = inmem.ProtocolStateFromEpochServiceEvents( result.ServiceEvents[0].Event.(*flow.EpochSetup), result.ServiceEvents[1].Event.(*flow.EpochCommit), ).ID() diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 1d7f226a638..f6650d09791 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -74,7 +74,7 @@ func (suite *MutatorSuite) SetupTest() { seal.ResultID = result.ID() qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) - genesis.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState( + genesis.Payload.ProtocolStateID = inmem.ProtocolStateFromEpochServiceEvents( result.ServiceEvents[0].Event.(*flow.EpochSetup), result.ServiceEvents[1].Event.(*flow.EpochCommit), ).ID() diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index b5fb77ca89f..a1210707233 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -235,7 +235,7 @@ func TestClusters(t *testing.T) { clusterQCs := unittest.QuorumCertificatesFromAssignments(setup.Assignments) commit.ClusterQCs = flow.ClusterQCVoteDatasFromQCs(clusterQCs) seal.ResultID = result.ID() - root.Payload.ProtocolStateID = inmem.ProtocolStateForBootstrapState(setup, commit).ID() + root.Payload.ProtocolStateID = inmem.ProtocolStateFromEpochServiceEvents(setup, commit).ID() rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index d8725f38462..b4e2dcb2ea9 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -323,7 +323,7 @@ func SnapshotFromBootstrapStateWithParams( EpochCommitSafetyThreshold: epochCommitSafetyThreshold, // see protocol.Params for details } - rootProtocolState := ProtocolStateForBootstrapState(setup, commit) + rootProtocolState := ProtocolStateFromEpochServiceEvents(setup, commit) if rootProtocolState.ID() != root.Payload.ProtocolStateID { return nil, fmt.Errorf("incorrect protocol state ID in root block, expected (%x) but got (%x)", root.Payload.ProtocolStateID, rootProtocolState.ID()) @@ -350,7 +350,7 @@ func SnapshotFromBootstrapStateWithParams( return snap, nil } -// ProtocolStateForBootstrapState generates a protocol.ProtocolStateEntry for a root protocol state which is used for bootstrapping. +// ProtocolStateFromEpochServiceEvents generates a protocol.ProtocolStateEntry for a root protocol state which is used for bootstrapping. // // CONTEXT: The EpochSetup event contains the IdentitySkeletons for each participant, thereby specifying active epoch members. // While ejection status and dynamic weight are not part of the EpochSetup event, we can supplement this information as follows: @@ -359,7 +359,7 @@ func SnapshotFromBootstrapStateWithParams( // - Therefore, the EpochSetup event contains the up-to-date snapshot of the epoch participants. Any weight changes or node ejection // that happened before should be reflected in the EpochSetup event. Specifically, ejected // nodes should be no longer listed in the EpochSetup event. Hence, when the EpochSetup event is emitted / processed, the Ejected flag is false for all epoch participants. -func ProtocolStateForBootstrapState(setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.ProtocolStateEntry { +func ProtocolStateFromEpochServiceEvents(setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.ProtocolStateEntry { identities := make(flow.DynamicIdentityEntryList, 0, len(setup.Participants)) for _, identity := range setup.Participants { identities = append(identities, &flow.DynamicIdentityEntry{ diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index c6faa1dc4e0..01f8aa52599 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2184,7 +2184,7 @@ func BootstrapFixtureWithChainID( WithDKGFromParticipants(participants.ToSkeleton()), ) - root.SetPayload(flow.Payload{ProtocolStateID: inmem.ProtocolStateForBootstrapState(setup, commit).ID()}) + root.SetPayload(flow.Payload{ProtocolStateID: inmem.ProtocolStateFromEpochServiceEvents(setup, commit).ID()}) stateCommit := GenesisStateCommitmentByChainID(chainID) result := BootstrapExecutionResultFixture(root, stateCommit) From 27d32d1f84dc50d723621ce171048da048453965 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 23:01:52 +0200 Subject: [PATCH 80/87] Linted --- state/protocol/validity.go | 1 + 1 file changed, 1 insertion(+) diff --git a/state/protocol/validity.go b/state/protocol/validity.go index 48c3c6830b3..8c8455dfb46 100644 --- a/state/protocol/validity.go +++ b/state/protocol/validity.go @@ -2,6 +2,7 @@ package protocol import ( "fmt" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" From 38b32108500ab36b606a10a6acabcb0f61f830d5 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Mon, 13 Nov 2023 23:53:16 +0200 Subject: [PATCH 81/87] Updated docs and implementation for TestHappyPathWithDbChanges --- state/protocol/protocol_state/mutator_test.go | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 99688aaa54c..fa93dcdeb9f 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -14,6 +14,7 @@ import ( storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/rand" "github.com/onflow/flow-go/utils/unittest" + "github.com/stretchr/testify/mock" ) func TestProtocolStateMutator(t *testing.T) { @@ -56,7 +57,9 @@ func (s *StateMutatorSuite) TestOnHappyPathNoDbChanges() { require.Empty(s.T(), dbUpdates) } -// TestHappyPathWithDbChanges tests that stateMutator returns cached db updates when building protocol state after applying service events. +// TestHappyPathWithDbChanges tests that `stateMutator` returns cached db updates when building protocol state after applying service events. +// Whenever `stateMutator` successfully processes an epoch setup or epoch commit event, it has to create a deferred db update to store the event. +// Deferred db updates are cached in `stateMutator` and returned when building protocol state when calling `Build`. func (s *StateMutatorSuite) TestHappyPathWithDbChanges() { parentState := unittest.ProtocolStateFixture() s.stateMachine.On("ParentState").Return(parentState) @@ -64,8 +67,9 @@ func (s *StateMutatorSuite) TestHappyPathWithDbChanges() { unittest.IdentifierFixture(), true) epochSetup := unittest.EpochSetupFixture() + epochCommit := unittest.EpochCommitFixture() result := unittest.ExecutionResultFixture(func(result *flow.ExecutionResult) { - result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent()} + result.ServiceEvents = []flow.ServiceEvent{epochSetup.ServiceEvent(), epochCommit.ServiceEvent()} }) block := unittest.BlockHeaderFixture() @@ -73,15 +77,36 @@ func (s *StateMutatorSuite) TestHappyPathWithDbChanges() { s.headersDB.On("ByBlockID", seal.BlockID).Return(block, nil) s.resultsDB.On("ByID", seal.ResultID).Return(result, nil) + epochSetupStored := mock.Mock{} + epochSetupStored.On("EpochSetupStored").Return() s.stateMachine.On("ProcessEpochSetup", epochSetup).Return(nil).Once() - s.setupsDB.On("StoreTx", epochSetup).Return(func(*transaction.Tx) error { return nil }).Once() + s.setupsDB.On("StoreTx", epochSetup).Return(func(*transaction.Tx) error { + epochSetupStored.MethodCalled("EpochSetupStored") + return nil + }).Once() + + epochCommitStored := mock.Mock{} + epochCommitStored.On("EpochCommitStored").Return() + s.stateMachine.On("ProcessEpochCommit", epochCommit).Return(nil).Once() + s.commitsDB.On("StoreTx", epochCommit).Return(func(*transaction.Tx) error { + epochCommitStored.MethodCalled("EpochCommitStored") + return nil + }).Once() err := s.mutator.ApplyServiceEventsFromValidatedSeals([]*flow.Seal{seal}) require.NoError(s.T(), err) _, _, _, dbUpdates := s.mutator.Build() - require.NoError(s.T(), err) - require.Len(s.T(), dbUpdates, 1) + // in next loop we assert that we have received expected deferred db updates by executing them + // and expecting that corresponding mock methods will be called + tx := &transaction.Tx{} + for _, dbUpdate := range dbUpdates { + err := dbUpdate(tx) + require.NoError(s.T(), err) + } + // make sure that mock methods were indeed called + epochSetupStored.AssertExpectations(s.T()) + epochCommitStored.AssertExpectations(s.T()) } // TestApplyServiceEvents_InvalidEpochSetup tests that handleServiceEvents rejects invalid epoch setup event and sets From 87f36aa80d0674190b6e78f0dc662dc28878201d Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 14 Nov 2023 11:59:06 +0200 Subject: [PATCH 82/87] Removed duplicated tests --- state/protocol/badger/validity_test.go | 91 -------------------------- 1 file changed, 91 deletions(-) diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index e051cac8481..e1bfa95bb62 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -5,9 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" @@ -15,95 +13,6 @@ import ( var participants = unittest.IdentityListFixture(20, unittest.WithAllRoles()) -func TestEpochSetupValidity(t *testing.T) { - t.Run("invalid first/final view", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - // set an invalid final view for the first epoch - setup.FinalView = setup.FirstView - - err := protocol.IsValidEpochSetup(setup, true) - require.Error(t, err) - }) - - t.Run("non-canonically ordered identities", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - // randomly shuffle the identities so they are not canonically ordered - var err error - setup.Participants, err = setup.Participants.Shuffle() - require.NoError(t, err) - err = protocol.IsValidEpochSetup(setup, true) - require.Error(t, err) - }) - - t.Run("invalid cluster assignments", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - // create an invalid cluster assignment (node appears in multiple clusters) - collector := participants.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))[0] - setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) - - err := protocol.IsValidEpochSetup(setup, true) - require.Error(t, err) - }) - - t.Run("short seed", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - setup.RandomSource = unittest.SeedFixture(crypto.SeedMinLenDKG - 1) - - err := protocol.IsValidEpochSetup(setup, true) - require.Error(t, err) - }) -} - -func TestBootstrapInvalidEpochCommit(t *testing.T) { - t.Run("inconsistent counter", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - // use a different counter for the commit - commit.Counter = setup.Counter + 1 - - err := protocol.IsValidEpochCommit(commit, setup) - require.Error(t, err) - }) - - t.Run("inconsistent cluster QCs", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - // add an extra QC to commit - extraQC := unittest.QuorumCertificateWithSignerIDsFixture() - commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(extraQC)) - - err := protocol.IsValidEpochCommit(commit, setup) - require.Error(t, err) - }) - - t.Run("missing dkg group key", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - commit.DKGGroupKey = nil - - err := protocol.IsValidEpochCommit(commit, setup) - require.Error(t, err) - }) - - t.Run("inconsistent DKG participants", func(t *testing.T) { - _, result, _ := unittest.BootstrapFixture(participants) - setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) - commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) - // add an extra DKG participant key - commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) - - err := protocol.IsValidEpochCommit(commit, setup) - require.Error(t, err) - }) -} - // TestEntityExpirySnapshotValidation tests that we perform correct sanity checks when // bootstrapping consensus nodes and access nodes we expect that we only bootstrap snapshots // with sufficient history. From 761d74a7a1a557b48c637fbe35a6aa8542bacbb6 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 14 Nov 2023 12:31:37 +0200 Subject: [PATCH 83/87] Added test cases for epoch setup validity --- state/protocol/validity_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/state/protocol/validity_test.go b/state/protocol/validity_test.go index 87ec08c21b6..7a168a50bc2 100644 --- a/state/protocol/validity_test.go +++ b/state/protocol/validity_test.go @@ -55,6 +55,36 @@ func TestEpochSetupValidity(t *testing.T) { err := protocol.IsValidEpochSetup(setup, true) require.Error(t, err) }) + + t.Run("node role missing", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + allWithoutExecutionNodes := setup.Participants.Filter(func(identitySkeleton *flow.IdentitySkeleton) bool { + return identitySkeleton.Role != flow.RoleExecution + }) + setup.Participants = allWithoutExecutionNodes + + err := protocol.IsValidEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("network addresses are not unique", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + setup.Participants[0].Address = setup.Participants[1].Address + + err := protocol.IsValidEpochSetup(setup, true) + require.Error(t, err) + }) + + t.Run("no cluster assignment", func(t *testing.T) { + _, result, _ := unittest.BootstrapFixture(participants) + setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + setup.Assignments = flow.AssignmentList{} + + err := protocol.IsValidEpochSetup(setup, true) + require.Error(t, err) + }) } func TestBootstrapInvalidEpochCommit(t *testing.T) { From 4ba03bc8182a089298fd0858d0960250ec86b770 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 14 Nov 2023 13:04:34 +0200 Subject: [PATCH 84/87] Updated tests for NewClusterList --- model/flow/factory/cluster_list_test.go | 72 +++++++++++++++++++++---- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/model/flow/factory/cluster_list_test.go b/model/flow/factory/cluster_list_test.go index 6a0f11e3f73..3833886c123 100644 --- a/model/flow/factory/cluster_list_test.go +++ b/model/flow/factory/cluster_list_test.go @@ -1,6 +1,7 @@ package factory_test import ( + "github.com/onflow/flow-go/model/flow/order" "testing" "github.com/stretchr/testify/require" @@ -10,16 +11,67 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -// NewClusterList assumes the input assignments are sorted, and fail if not. -// This tests verifies that NewClusterList has implemented the check on the assumption. -func TestNewClusterListFail(t *testing.T) { +// TestNewClusterList ensures that implementation enforces the following protocol rules in case they are violated: +// +// (a) input `collectors` only contains collector nodes with positive weight +// (b) collectors have unique node IDs +// (c) each collector is assigned exactly to one cluster and is only listed once within that cluster +// (d) cluster contains at least one collector (i.e. is not empty) +// (e) cluster is composed of known nodes, i.e. for each nodeID in `assignments` an IdentitySkeleton is given in `collectors` +// (f) cluster assignment lists the nodes in canonical ordering +func TestNewClusterList(t *testing.T) { identities := unittest.IdentityListFixture(100, unittest.WithRole(flow.RoleCollection)) - assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) - tmp := assignments[1][0] - assignments[1][0] = assignments[1][1] - assignments[1][1] = tmp - - _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) - require.Error(t, err) + t.Run("valid inputs", func(t *testing.T) { + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.NoError(t, err) + }) + t.Run("(a) input `collectors` only contains collector nodes with positive weight", func(t *testing.T) { + identities := identities.Copy() + identities[0].InitialWeight = 0 + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) + t.Run("(b) collectors have unique node IDs", func(t *testing.T) { + identities := identities.Copy() + identities[0].NodeID = identities[1].NodeID + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) + t.Run("(c) each collector is assigned exactly to one cluster", func(t *testing.T) { + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + assignments[1][0] = assignments[0][0] + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) + t.Run("(c) each collector is only listed once within that cluster", func(t *testing.T) { + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + assignments[0][0] = assignments[0][1] + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) + t.Run("(d) cluster contains at least one collector (i.e. is not empty)", func(t *testing.T) { + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + assignments[0] = flow.IdentifierList{} + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) + t.Run("(e) cluster is composed of known nodes, i.e. for each nodeID in `assignments` an IdentitySkeleton is given in `collectors` ", func(t *testing.T) { + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + assignments[0][0] = unittest.IdentifierFixture() + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) + t.Run("(f) cluster assignment lists the nodes in canonical ordering", func(t *testing.T) { + assignments := unittest.ClusterAssignment(10, identities.ToSkeleton()) + // sort in non-canonical order + assignments[0] = assignments[0].Sort(func(lhs flow.Identifier, rhs flow.Identifier) bool { + return !order.IdentifierCanonical(lhs, rhs) + }) + _, err := factory.NewClusterList(assignments, identities.ToSkeleton()) + require.Error(t, err) + }) } From fad73b36742277a033bad6533f2441f174b34639 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 14 Nov 2023 14:18:31 +0200 Subject: [PATCH 85/87] Added test for validating extending setup and commit events --- state/protocol/validity_test.go | 104 ++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/state/protocol/validity_test.go b/state/protocol/validity_test.go index 7a168a50bc2..79935aebcce 100644 --- a/state/protocol/validity_test.go +++ b/state/protocol/validity_test.go @@ -132,3 +132,107 @@ func TestBootstrapInvalidEpochCommit(t *testing.T) { require.Error(t, err) }) } + +// TestIsValidExtendingEpochSetup tests that implementation enforces the following protocol rules in case they are violated: +// (a) We should only have a single epoch setup event per epoch. +// (b) The setup event should have the counter increased by one +// (c) The first view needs to be exactly one greater than the current epoch final view +// additionally we require other conditions, but they are tested by separate test `TestEpochSetupValidity`. +func TestIsValidExtendingEpochSetup(t *testing.T) { + t.Run("happy path", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture() + currentEpochSetup := protocolState.CurrentEpochSetup + extendingSetup := unittest.EpochSetupFixture( + unittest.WithFirstView(currentEpochSetup.FinalView+1), + unittest.WithFinalView(currentEpochSetup.FinalView+1000), + unittest.SetupWithCounter(currentEpochSetup.Counter+1), + unittest.WithParticipants(participants.ToSkeleton()), + ) + err := protocol.IsValidExtendingEpochSetup(extendingSetup, protocolState.ProtocolStateEntry, currentEpochSetup) + require.NoError(t, err) + }) + t.Run("(a) We should only have a single epoch setup event per epoch.", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) + currentEpochSetup := protocolState.CurrentEpochSetup + extendingSetup := unittest.EpochSetupFixture( + unittest.WithFirstView(currentEpochSetup.FinalView+1), + unittest.WithFinalView(currentEpochSetup.FinalView+1000), + unittest.SetupWithCounter(currentEpochSetup.Counter+1), + unittest.WithParticipants(participants.ToSkeleton()), + ) + err := protocol.IsValidExtendingEpochSetup(extendingSetup, protocolState.ProtocolStateEntry, currentEpochSetup) + require.Error(t, err) + }) + t.Run("(b) The setup event should have the counter increased by one", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture() + currentEpochSetup := protocolState.CurrentEpochSetup + extendingSetup := unittest.EpochSetupFixture( + unittest.WithFirstView(currentEpochSetup.FinalView+1), + unittest.WithFinalView(currentEpochSetup.FinalView+1000), + unittest.SetupWithCounter(currentEpochSetup.Counter+2), + unittest.WithParticipants(participants.ToSkeleton()), + ) + err := protocol.IsValidExtendingEpochSetup(extendingSetup, protocolState.ProtocolStateEntry, currentEpochSetup) + require.Error(t, err) + }) + t.Run("(c) The first view needs to be exactly one greater than the current epoch final view", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture() + currentEpochSetup := protocolState.CurrentEpochSetup + extendingSetup := unittest.EpochSetupFixture( + unittest.WithFirstView(currentEpochSetup.FinalView+2), + unittest.WithFinalView(currentEpochSetup.FinalView+1000), + unittest.SetupWithCounter(currentEpochSetup.Counter+1), + unittest.WithParticipants(participants.ToSkeleton()), + ) + err := protocol.IsValidExtendingEpochSetup(extendingSetup, protocolState.ProtocolStateEntry, currentEpochSetup) + require.Error(t, err) + }) +} + +// TestIsValidExtendingEpochCommit tests that implementation enforces the following protocol rules in case they are violated: +// (a) The epoch setup event needs to happen before the commit. +// (b) We should only have a single epoch commit event per epoch. +// additionally we require other conditions, but they are tested by separate test `TestEpochCommitValidity`. +func TestIsValidExtendingEpochCommit(t *testing.T) { + t.Run("happy path", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState(), func(entry *flow.RichProtocolStateEntry) { + entry.NextEpochCommit = nil + entry.NextEpoch.CommitID = flow.ZeroID + }) + + nextEpochSetup := protocolState.NextEpochSetup + extendingSetup := unittest.EpochCommitFixture( + unittest.CommitWithCounter(nextEpochSetup.Counter), + unittest.WithDKGFromParticipants(nextEpochSetup.Participants), + ) + err := protocol.IsValidExtendingEpochCommit(extendingSetup, protocolState.ProtocolStateEntry, nextEpochSetup) + require.NoError(t, err) + }) + t.Run("(a) The epoch setup event needs to happen before the commit", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture() + currentEpochSetup := protocolState.CurrentEpochSetup + nextEpochSetup := unittest.EpochSetupFixture( + unittest.WithFirstView(currentEpochSetup.FinalView+1), + unittest.WithFinalView(currentEpochSetup.FinalView+1000), + unittest.SetupWithCounter(currentEpochSetup.Counter+1), + unittest.WithParticipants(participants.ToSkeleton()), + ) + extendingSetup := unittest.EpochCommitFixture( + unittest.CommitWithCounter(nextEpochSetup.Counter), + unittest.WithDKGFromParticipants(nextEpochSetup.Participants), + ) + err := protocol.IsValidExtendingEpochCommit(extendingSetup, protocolState.ProtocolStateEntry, nextEpochSetup) + require.Error(t, err) + }) + t.Run("We should only have a single epoch commit event per epoch", func(t *testing.T) { + protocolState := unittest.ProtocolStateFixture(unittest.WithNextEpochProtocolState()) + + nextEpochSetup := protocolState.NextEpochSetup + extendingSetup := unittest.EpochCommitFixture( + unittest.CommitWithCounter(nextEpochSetup.Counter), + unittest.WithDKGFromParticipants(nextEpochSetup.Participants), + ) + err := protocol.IsValidExtendingEpochCommit(extendingSetup, protocolState.ProtocolStateEntry, nextEpochSetup) + require.Error(t, err) + }) +} From 45c90815ae880db5ceadfbf07ac121a486789d82 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 14 Nov 2023 14:21:52 +0200 Subject: [PATCH 86/87] Linted --- model/flow/factory/cluster_list_test.go | 2 +- state/protocol/protocol_state/mutator_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/model/flow/factory/cluster_list_test.go b/model/flow/factory/cluster_list_test.go index 3833886c123..fc4560818c5 100644 --- a/model/flow/factory/cluster_list_test.go +++ b/model/flow/factory/cluster_list_test.go @@ -1,13 +1,13 @@ package factory_test import ( - "github.com/onflow/flow-go/model/flow/order" "testing" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" + "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index fa93dcdeb9f..64af007e925 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -14,7 +15,6 @@ import ( storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/rand" "github.com/onflow/flow-go/utils/unittest" - "github.com/stretchr/testify/mock" ) func TestProtocolStateMutator(t *testing.T) { From a389919ff8b523f2a1c4424d6a43893d65163d80 Mon Sep 17 00:00:00 2001 From: Yurii Oleksyshyn Date: Tue, 14 Nov 2023 19:49:39 +0200 Subject: [PATCH 87/87] Update state/protocol/protocol_state/mutator_test.go Co-authored-by: Alexander Hentschel --- state/protocol/protocol_state/mutator_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/state/protocol/protocol_state/mutator_test.go b/state/protocol/protocol_state/mutator_test.go index 64af007e925..468f3f20ee1 100644 --- a/state/protocol/protocol_state/mutator_test.go +++ b/state/protocol/protocol_state/mutator_test.go @@ -279,10 +279,10 @@ func (s *StateMutatorSuite) TestApplyServiceEventsTransitionToNextEpoch_Error() require.False(s.T(), protocol.IsInvalidServiceEventError(err)) } -// TestApplyServiceEvents_EpochFallbackTriggered tests two scenarios when network is in epoch fallback mode. -// In first case we have observed a global flag that we have finalized fork with invalid event and network is in epoch fallback mode. -// In second case we have observed an InvalidStateTransitionAttempted flag which is part of some fork but has not been finalized yet. -// In both cases we shouldn't process any service events. This is asserted by using mocked state updater without any expected methods. +// TestApplyServiceEvents_EpochFallbackTriggered tests the scenario, where the system has entered epoch fallback mode. This +// happens when the protocol state observes an invalid state transition, in which case it sets the `InvalidStateTransitionAttempted` +// flag to true. After this happened, we should not process any Epoch lifecycle service events. This is asserted by using mocked +// state updater without any expected methods. func (s *StateMutatorSuite) TestApplyServiceEvents_EpochFallbackTriggered() { parentState := unittest.ProtocolStateFixture() parentState.InvalidStateTransitionAttempted = true