[Dynamic Protocol State] Protocol state used in FollowerState and ParticipantState

cmd/bootstrap/run/cluster_qc.go

@@ -20,6 +20,9 @@ import (
 
 // GenerateClusterRootQC creates votes and generates a QC based on participant data
 func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flow.IdentityList, clusterBlock *cluster.Block) (*flow.QuorumCertificate, error) {
+	if !allCommitteeMembers.Sorted(order.Canonical) {
+		return nil, fmt.Errorf("can't create root cluster QC: committee members are not sorted in canonical order")
+	}
 	clusterRootBlock := model.GenesisBlockFromFlow(clusterBlock.Header)
 
 	// STEP 1: create votes for cluster root block
@@ -29,8 +32,7 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo
 	}
 
 	// STEP 2: create VoteProcessor
-	ordered := allCommitteeMembers.Sort(order.Canonical)
-	committee, err := committees.NewStaticCommittee(ordered, flow.Identifier{}, nil, nil)
+	committee, err := committees.NewStaticCommittee(allCommitteeMembers, flow.Identifier{}, nil, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/bootstrap/run/cluster_qc_test.go

@@ -8,6 +8,7 @@ import (
 	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/cluster"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/order"
 	"github.com/onflow/flow-go/utils/unittest"
 )
 
@@ -32,7 +33,8 @@ func TestGenerateClusterRootQC(t *testing.T) {
 	payload := cluster.EmptyPayload(flow.ZeroID)
 	clusterBlock.SetPayload(payload)
 
-	_, err := GenerateClusterRootQC(participants, model.ToIdentityList(participants), &clusterBlock)
+	orderedParticipants := model.ToIdentityList(participants).Sort(order.Canonical)
+	_, err := GenerateClusterRootQC(participants, orderedParticipants, &clusterBlock)
 	require.NoError(t, err)
 }
 

cmd/scaffold.go

@@ -979,6 +979,7 @@ func (fnb *FlowNodeBuilder) initStorage() error {
 	epochCommits := bstorage.NewEpochCommits(fnb.Metrics.Cache, fnb.DB)
 	statuses := bstorage.NewEpochStatuses(fnb.Metrics.Cache, fnb.DB)
 	commits := bstorage.NewCommits(fnb.Metrics.Cache, fnb.DB)
+	protocolState := bstorage.NewProtocolState(fnb.Metrics.Cache, setups, epochCommits, fnb.DB, bstorage.DefaultCacheSize)
 	versionBeacons := bstorage.NewVersionBeacons(fnb.DB)
 
 	fnb.Storage = Storage{
@@ -996,6 +997,7 @@ func (fnb *FlowNodeBuilder) initStorage() error {
 		Setups:             setups,
 		EpochCommits:       epochCommits,
 		VersionBeacons:     versionBeacons,
+		ProtocolState:      protocolState,
 		Statuses:           statuses,
 		Commits:            commits,
 	}
@@ -1060,7 +1062,7 @@ func (fnb *FlowNodeBuilder) initState() error {
 			fnb.Storage.QuorumCertificates,
 			fnb.Storage.Setups,
 			fnb.Storage.EpochCommits,
-			fnb.Storage.Statuses,
+			fnb.Storage.ProtocolState,
 			fnb.Storage.VersionBeacons,
 		)
 		if err != nil {
@@ -1112,7 +1114,7 @@ func (fnb *FlowNodeBuilder) initState() error {
 			fnb.Storage.QuorumCertificates,
 			fnb.Storage.Setups,
 			fnb.Storage.EpochCommits,
-			fnb.Storage.Statuses,
+			fnb.Storage.ProtocolState,
 			fnb.Storage.VersionBeacons,
 			fnb.RootSnapshot,
 			options...,

cmd/util/cmd/common/state.go

@@ -24,7 +24,7 @@ func InitProtocolState(db *badger.DB, storages *storage.All) (protocol.State, er
 		storages.QuorumCertificates,
 		storages.Setups,
 		storages.EpochCommits,
-		storages.Statuses,
+		storages.ProtocolState,
 		storages.VersionBeacons,
 	)
 

consensus/integration/epoch_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/onflow/flow-go/model/flow/filter"
 	"github.com/onflow/flow-go/model/flow/mapfunc"
 	"github.com/onflow/flow-go/model/flow/order"
+	"github.com/onflow/flow-go/state/protocol"
 	"github.com/onflow/flow-go/state/protocol/inmem"
 	"github.com/onflow/flow-go/utils/unittest"
 )
@@ -255,5 +256,26 @@ func withNextEpoch(
 			Map(mapfunc.WithWeight(0))...,
 	).Sort(order.Canonical)
 
+	// update protocol state
+	protocolState := encodableSnapshot.ProtocolState
+	// update protocol state identities since we are in committed phase
+	protocolState.Identities = flow.DynamicIdentityEntryListFromIdentities(encodableSnapshot.Identities)
+	// setup ID has changed, need to update it
+	convertedEpochSetup, _ := protocol.ToEpochSetup(inmem.NewEpoch(*currEpoch))
+	protocolState.CurrentEpochEventIDs.SetupID = convertedEpochSetup.ID()
+	// create next epoch protocol state
+	convertedEpochSetup, _ = protocol.ToEpochSetup(inmem.NewEpoch(*encodableSnapshot.Epochs.Next))
+	convertedEpochCommit, _ := protocol.ToEpochCommit(inmem.NewEpoch(*encodableSnapshot.Epochs.Next))
+	protocolState.NextEpochProtocolState = &flow.ProtocolStateEntry{
+		CurrentEpochEventIDs: flow.EventIDs{
+			SetupID:  convertedEpochSetup.ID(),
+			CommitID: convertedEpochCommit.ID(),
+		},
+		PreviousEpochEventIDs:           protocolState.CurrentEpochEventIDs,
+		Identities:                      flow.DynamicIdentityEntryListFromIdentities(encodableSnapshot.Identities),
+		InvalidStateTransitionAttempted: false,
+		NextEpochProtocolState:          nil,
+	}
+
 	return inmem.SnapshotFromEncodable(encodableSnapshot)
 }

consensus/integration/nodes_test.go

@@ -375,7 +375,7 @@ func createNode(
 	qcsDB := storage.NewQuorumCertificates(metricsCollector, db, storage.DefaultCacheSize)
 	setupsDB := storage.NewEpochSetups(metricsCollector, db)
 	commitsDB := storage.NewEpochCommits(metricsCollector, db)
-	statusesDB := storage.NewEpochStatuses(metricsCollector, db)
+	protocolStateDB := storage.NewProtocolState(metricsCollector, setupsDB, commitsDB, db, storage.DefaultCacheSize)
 	versionBeaconDB := storage.NewVersionBeacons(db)
 	protocolStateEvents := events.NewDistributor()
 
@@ -396,7 +396,7 @@ func createNode(
 		qcsDB,
 		setupsDB,
 		commitsDB,
-		statusesDB,
+		protocolStateDB,
 		versionBeaconDB,
 		rootSnapshot,
 	)

engine/collection/test/cluster_switchover_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/factory"
 	"github.com/onflow/flow-go/model/flow/filter"
+	"github.com/onflow/flow-go/model/flow/order"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/module/util"
 	"github.com/onflow/flow-go/network/channels"
@@ -66,19 +67,18 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf)
 	rootClusterQCs := make([]flow.ClusterQCVoteData, len(rootClusterBlocks))
 	for i, cluster := range clusters {
 		signers := make([]model.NodeInfo, 0)
-		signerIDs := make([]flow.Identifier, 0)
 		for _, identity := range nodeInfos {
 			if _, inCluster := cluster.ByNodeID(identity.NodeID); inCluster {
 				signers = append(signers, identity)
-				signerIDs = append(signerIDs, identity.NodeID)
 			}
 		}
-		qc, err := run.GenerateClusterRootQC(signers, model.ToIdentityList(signers), rootClusterBlocks[i])
+		signerIdentities := model.ToIdentityList(signers).Sort(order.Canonical)
+		qc, err := run.GenerateClusterRootQC(signers, signerIdentities, rootClusterBlocks[i])
 		require.NoError(t, err)
 		rootClusterQCs[i] = flow.ClusterQCVoteDataFromQC(&flow.QuorumCertificateWithSignerIDs{
 			View:      qc.View,
 			BlockID:   qc.BlockID,
-			SignerIDs: signerIDs,
+			SignerIDs: signerIdentities.NodeIDs(),
 			SigData:   qc.SigData,
 		})
 	}

engine/common/follower/integration_test.go

@@ -63,7 +63,7 @@ func TestFollowerHappyPath(t *testing.T) {
 			all.QuorumCertificates,
 			all.Setups,
 			all.EpochCommits,
-			all.Statuses,
+			all.ProtocolState,
 			all.VersionBeacons,
 			rootSnapshot,
 		)

engine/testutil/nodes.go

@@ -249,7 +249,7 @@ func CompleteStateFixture(
 		s.QuorumCertificates,
 		s.Setups,
 		s.EpochCommits,
-		s.Statuses,
+		s.ProtocolState,
 		s.VersionBeacons,
 		rootSnapshot,
 	)

integration/testnet/container.go

@@ -390,7 +390,7 @@ func (c *Container) OpenState() (*state.State, error) {
 	qcs := storage.NewQuorumCertificates(metrics, db, storage.DefaultCacheSize)
 	setups := storage.NewEpochSetups(metrics, db)
 	commits := storage.NewEpochCommits(metrics, db)
-	statuses := storage.NewEpochStatuses(metrics, db)
+	protocolState := storage.NewProtocolState(metrics, setups, commits, db, storage.DefaultCacheSize)
 	versionBeacons := storage.NewVersionBeacons(db)
 
 	return state.OpenState(
@@ -403,7 +403,7 @@ func (c *Container) OpenState() (*state.State, error) {
 		qcs,
 		setups,
 		commits,
-		statuses,
+		protocolState,
 		versionBeacons,
 	)
 }

model/flow/protocol_state.go

@@ -82,32 +82,56 @@ func NewRichProtocolStateEntry(
 		NextEpochProtocolState: nil,
 	}
 
+	// ensure data is consistent
+	if protocolState.PreviousEpochEventIDs.SetupID != ZeroID {
+		if protocolState.PreviousEpochEventIDs.SetupID != previousEpochSetup.ID() {
+			return nil, fmt.Errorf("supplied previous epoch setup (%x) does not match protocol state (%x)",
+				previousEpochSetup.ID(),
+				protocolState.PreviousEpochEventIDs.SetupID)
+		}
+		if protocolState.PreviousEpochEventIDs.CommitID != previousEpochCommit.ID() {
+			return nil, fmt.Errorf("supplied previous epoch commit (%x) does not match protocol state (%x)",
+				previousEpochCommit.ID(),
+				protocolState.PreviousEpochEventIDs.CommitID)
+		}
+	}
+	if protocolState.CurrentEpochEventIDs.SetupID != currentEpochSetup.ID() {
+		return nil, fmt.Errorf("supplied current epoch setup (%x) does not match protocol state (%x)",
+			currentEpochSetup.ID(),
+			protocolState.CurrentEpochEventIDs.SetupID)
+	}
+	if protocolState.CurrentEpochEventIDs.CommitID != currentEpochCommit.ID() {
+		return nil, fmt.Errorf("supplied current epoch commit (%x) does not match protocol state (%x)",
+			currentEpochCommit.ID(),
+			protocolState.CurrentEpochEventIDs.CommitID)
+	}
+
 	var err error
+	nextEpochProtocolState := protocolState.NextEpochProtocolState
 	// if next epoch has been already committed, fill in data for it as well.
-	if protocolState.NextEpochProtocolState != nil {
+	if nextEpochProtocolState != nil {
 		// sanity check consistency of input data
-		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID != nextEpochSetup.ID() {
+		if nextEpochProtocolState.CurrentEpochEventIDs.SetupID != nextEpochSetup.ID() {
 			return nil, fmt.Errorf("inconsistent EpochSetup for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
 				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID, nextEpochSetup.ID())
 		}
-		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID != nextEpochCommit.ID() {
+		if nextEpochProtocolState.CurrentEpochEventIDs.CommitID != nextEpochCommit.ID() {
 			return nil, fmt.Errorf("inconsistent EpochCommit for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
 				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID, nextEpochCommit.ID())
 		}
 
 		// sanity check consistency of input data
-		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID != nextEpochSetup.ID() {
+		if nextEpochProtocolState.CurrentEpochEventIDs.SetupID != nextEpochSetup.ID() {
 			return nil, fmt.Errorf("inconsistent EpochSetup for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
 				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.SetupID, nextEpochSetup.ID())
 		}
-		if protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID != nextEpochCommit.ID() {
+		if nextEpochProtocolState.CurrentEpochEventIDs.CommitID != nextEpochCommit.ID() {
 			return nil, fmt.Errorf("inconsistent EpochCommit for constucting RichProtocolStateEntry, next protocol state states ID %v while input event has ID %v",
 				protocolState.NextEpochProtocolState.CurrentEpochEventIDs.CommitID, nextEpochCommit.ID())
 		}
 
 		// if next epoch is available, it means that we have observed epoch setup event and we are not anymore in staking phase,
 		// so we need to build the identity table using current and next epoch setup events.
-		// so we need to build the identity table using current and next epoch setup events.
 		result.Identities, err = buildIdentityTable(
 			protocolState.Identities,
 			currentEpochSetup.Participants,
@@ -117,7 +141,6 @@ func NewRichProtocolStateEntry(
 			return nil, fmt.Errorf("could not build identity table for setup/commit phase: %w", err)
 		}
 
-		nextEpochProtocolState := protocolState.NextEpochProtocolState
 		nextEpochIdentityTable, err := buildIdentityTable(
 			nextEpochProtocolState.Identities,
 			nextEpochSetup.Participants,

module/builder/collection/builder_test.go

@@ -108,7 +108,7 @@ func (suite *BuilderSuite) SetupTest() {
 		all.QuorumCertificates,
 		all.Setups,
 		all.EpochCommits,
-		all.Statuses,
+		all.ProtocolState,
 		all.VersionBeacons,
 		rootSnapshot,
 	)

module/state_synchronization/requester/execution_data_requester_test.go

@@ -787,11 +787,12 @@ func (m *mockSnapshot) Identity(nodeID flow.Identifier) (*flow.Identity, error)
 func (m *mockSnapshot) SealedResult() (*flow.ExecutionResult, *flow.Seal, error) {
 	return nil, nil, nil
 }
-func (m *mockSnapshot) Commit() (flow.StateCommitment, error)             { return flow.DummyStateCommitment, nil }
-func (m *mockSnapshot) SealingSegment() (*flow.SealingSegment, error)     { return nil, nil }
-func (m *mockSnapshot) Descendants() ([]flow.Identifier, error)           { return nil, nil }
-func (m *mockSnapshot) RandomSource() ([]byte, error)                     { return nil, nil }
-func (m *mockSnapshot) Phase() (flow.EpochPhase, error)                   { return flow.EpochPhaseUndefined, nil }
-func (m *mockSnapshot) Epochs() protocol.EpochQuery                       { return nil }
-func (m *mockSnapshot) Params() protocol.GlobalParams                     { return nil }
-func (m *mockSnapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { return nil, nil }
+func (m *mockSnapshot) Commit() (flow.StateCommitment, error)                 { return flow.DummyStateCommitment, nil }
+func (m *mockSnapshot) SealingSegment() (*flow.SealingSegment, error)         { return nil, nil }
+func (m *mockSnapshot) Descendants() ([]flow.Identifier, error)               { return nil, nil }
+func (m *mockSnapshot) RandomSource() ([]byte, error)                         { return nil, nil }
+func (m *mockSnapshot) Phase() (flow.EpochPhase, error)                       { return flow.EpochPhaseUndefined, nil }
+func (m *mockSnapshot) Epochs() protocol.EpochQuery                           { return nil }
+func (m *mockSnapshot) Params() protocol.GlobalParams                         { return nil }
+func (m *mockSnapshot) ProtocolState() (protocol.DynamicProtocolState, error) { return nil, nil }
+func (m *mockSnapshot) VersionBeacon() (*flow.SealedVersionBeacon, error)     { return nil, nil }

state/cluster/badger/mutator_test.go

@@ -85,7 +85,7 @@ func (suite *MutatorSuite) SetupTest() {
 		all.QuorumCertificates,
 		all.Setups,
 		all.EpochCommits,
-		all.Statuses,
+		all.ProtocolState,
 		all.VersionBeacons,
 		rootSnapshot,
 	)

state/cluster/badger/snapshot_test.go

@@ -66,7 +66,7 @@ func (suite *SnapshotSuite) SetupTest() {
 		all.QuorumCertificates,
 		all.Setups,
 		all.EpochCommits,
-		all.Statuses,
+		all.ProtocolState,
 		all.VersionBeacons,
 		root,
 	)