[Dynamic Protocol State] Block payload contains protocol state ID

Makefile

@@ -184,6 +184,7 @@ generate-mocks: install-mock-generators
 	mockery --name '.*' --dir=storage --case=underscore --output="./storage/mock" --outpkg="mock"
 	mockery --name '.*' --dir="state/protocol" --case=underscore --output="state/protocol/mock" --outpkg="mock"
 	mockery --name '.*' --dir="state/protocol/events" --case=underscore --output="./state/protocol/events/mock" --outpkg="mock"
+	mockery --name '.*' --dir="state/protocol/protocol_state" --case=underscore --output="state/protocol/protocol_state/mock" --outpkg="mock"
 	mockery --name '.*' --dir=engine/execution/computation/computer --case=underscore --output="./engine/execution/computation/computer/mock" --outpkg="mock"
 	mockery --name '.*' --dir=engine/execution/state --case=underscore --output="./engine/execution/state/mock" --outpkg="mock"
 	mockery --name '.*' --dir=engine/collection --case=underscore --output="./engine/collection/mock" --outpkg="mock"

cmd/bootstrap/cmd/block.go

@@ -5,21 +5,82 @@ import (
 	"time"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
+	"github.com/onflow/flow-go/model/dkg"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/order"
+	"github.com/onflow/flow-go/module/signature"
+	"github.com/onflow/flow-go/state/protocol/inmem"
 )
 
-func constructRootBlock(rootChain string, rootParent string, rootHeight uint64, rootTimestamp string) *flow.Block {
-
+// constructRootHeader constructs a header for the root block.
+func constructRootHeader(rootChain string, rootParent string, rootHeight uint64, rootTimestamp string) *flow.Header {
 	chainID := parseChainID(rootChain)
 	parentID := parseParentID(rootParent)
 	height := rootHeight
 	timestamp := parseRootTimestamp(rootTimestamp)
 
-	block := run.GenerateRootBlock(chainID, parentID, height, timestamp)
+	return run.GenerateRootHeader(chainID, parentID, height, timestamp)
+}
 
+// constructRootBlock constructs a valid root block based on the given header, setup, and commit.
+func constructRootBlock(rootHeader *flow.Header, setup *flow.EpochSetup, commit *flow.EpochCommit) *flow.Block {
+	block := &flow.Block{
+		Header:  rootHeader,
+		Payload: nil,
+	}
+	block.SetPayload(flow.Payload{
+		Guarantees:      nil,
+		Seals:           nil,
+		Receipts:        nil,
+		Results:         nil,
+		ProtocolStateID: inmem.ProtocolStateFromEpochServiceEvents(setup, commit).ID(),
+	})
 	return block
 }
 
+// constructRootEpochEvents constructs the epoch setup and commit events for the first epoch after spork.
+func constructRootEpochEvents(
+	firstView uint64,
+	participants flow.IdentityList,
+	assignments flow.AssignmentList,
+	clusterQCs []*flow.QuorumCertificate,
+	dkgData dkg.DKGData) (*flow.EpochSetup, *flow.EpochCommit) {
+	epochSetup := &flow.EpochSetup{
+		Counter:            flagEpochCounter,
+		FirstView:          firstView,
+		FinalView:          firstView + flagNumViewsInEpoch - 1,
+		DKGPhase1FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase - 1,
+		DKGPhase2FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*2 - 1,
+		DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1,
+		Participants:       participants.Sort(order.Canonical[flow.Identity]).ToSkeleton(),
+		Assignments:        assignments,
+		RandomSource:       GenerateRandomSeed(flow.EpochSetupRandomSourceLength),
+	}
+
+	qcsWithSignerIDs := make([]*flow.QuorumCertificateWithSignerIDs, 0, len(clusterQCs))
+	for i, clusterQC := range clusterQCs {
+		members := assignments[i]
+		signerIDs, err := signature.DecodeSignerIndicesToIdentifiers(members, clusterQC.SignerIndices)
+		if err != nil {
+			log.Fatal().Err(err).Msgf("could not decode signer IDs from clusterQC at index %v", i)
+		}
+		qcsWithSignerIDs = append(qcsWithSignerIDs, &flow.QuorumCertificateWithSignerIDs{
+			View:      clusterQC.View,
+			BlockID:   clusterQC.BlockID,
+			SignerIDs: signerIDs,
+			SigData:   clusterQC.SigData,
+		})
+	}
+
+	epochCommit := &flow.EpochCommit{
+		Counter:            flagEpochCounter,
+		ClusterQCs:         flow.ClusterQCVoteDatasFromQCs(qcsWithSignerIDs),
+		DKGGroupKey:        dkgData.PubGroupKey,
+		DKGParticipantKeys: dkgData.PubKeyShares,
+	}
+	return epochSetup, epochCommit
+}
+
 func parseChainID(chainID string) flow.ChainID {
 	switch chainID {
 	case "main":

cmd/bootstrap/cmd/clusters.go

@@ -10,6 +10,7 @@ import (
 	"github.com/onflow/flow-go/model/flow/assignment"
 	"github.com/onflow/flow-go/model/flow/factory"
 	"github.com/onflow/flow-go/model/flow/filter"
+	"github.com/onflow/flow-go/state/protocol"
 )
 
 // Construct random cluster assignment with internal and partner nodes.
@@ -123,3 +124,20 @@ func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.N
 
 	return filtered
 }
+
+// clusterRootQCsFromEpoch returns a list of cluster root QCs from the given epoch.
+func clusterRootQCsFromEpoch(epoch protocol.Epoch) []*flow.QuorumCertificate {
+	clustering, err := epoch.Clustering()
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not get epoch clustering")
+	}
+	qcs := make([]*flow.QuorumCertificate, 0, len(clustering))
+	for i := range clustering {
+		cluster, err := epoch.Cluster(uint(i))
+		if err != nil {
+			log.Fatal().Err(err).Msgf("could not get epoch cluster (index=%d)", i)
+		}
+		qcs = append(qcs, cluster.RootQC())
+	}
+	return qcs
+}

cmd/bootstrap/cmd/finalize.go

@@ -38,15 +38,12 @@ var (
 	flagPartnerWeights              string
 	flagDKGDataPath                 string
 	flagRootBlock                   string
+	flagRootEpoch                   string
 	flagRootBlockVotesDir           string
 	flagRootCommit                  string
 	flagProtocolVersion             uint
 	flagServiceAccountPublicKeyJSON string
 	flagGenesisTokenSupply          string
-	flagEpochCounter                uint64
-	flagNumViewsInEpoch             uint64
-	flagNumViewsInStakingAuction    uint64
-	flagNumViewsInDKGPhase          uint64
 	flagEpochCommitSafetyThreshold  uint64
 )
 
@@ -91,24 +88,19 @@ func addFinalizeCmdFlags() {
 	// required parameters for generation of root block, root execution result and root block seal
 	finalizeCmd.Flags().StringVar(&flagRootBlock, "root-block", "",
 		"path to a JSON file containing root block")
+	finalizeCmd.Flags().StringVar(&flagRootEpoch, "root-epoch", "",
+		"path to a JSON file containing root epoch")
 	finalizeCmd.Flags().StringVar(&flagRootBlockVotesDir, "root-block-votes-dir", "", "path to directory with votes for root block")
 	finalizeCmd.Flags().StringVar(&flagRootCommit, "root-commit", "0000000000000000000000000000000000000000000000000000000000000000", "state commitment of root execution state")
-	finalizeCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "epoch counter for the epoch beginning with the root block")
-	finalizeCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 4000, "length of each epoch measured in views")
-	finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views")
-	finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views")
 	finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline")
 	finalizeCmd.Flags().UintVar(&flagProtocolVersion, "protocol-version", flow.DefaultProtocolVersion, "major software version used for the duration of this spork")
 
 	cmd.MarkFlagRequired(finalizeCmd, "root-block")
+	cmd.MarkFlagRequired(finalizeCmd, "root-epoch")
 	cmd.MarkFlagRequired(finalizeCmd, "root-block-votes-dir")
 	cmd.MarkFlagRequired(finalizeCmd, "root-commit")
-	cmd.MarkFlagRequired(finalizeCmd, "epoch-counter")
-	cmd.MarkFlagRequired(finalizeCmd, "epoch-length")
-	cmd.MarkFlagRequired(finalizeCmd, "epoch-staking-phase-length")
-	cmd.MarkFlagRequired(finalizeCmd, "epoch-dkg-phase-length")
-	cmd.MarkFlagRequired(finalizeCmd, "epoch-commit-safety-threshold")
 	cmd.MarkFlagRequired(finalizeCmd, "protocol-version")
+	cmd.MarkFlagRequired(finalizeCmd, "epoch-commit-safety-threshold")
 
 	// optional parameters to influence various aspects of identity generation
 	finalizeCmd.Flags().UintVar(&flagCollectionClusters, "collection-clusters", 2, "number of collection clusters")
@@ -172,6 +164,11 @@ func finalize(cmd *cobra.Command, args []string) {
 	dkgData := readDKGData()
 	log.Info().Msg("")
 
+	log.Info().Msg("reading root epoch")
+	epoch, epochSetup, epochCommit := readRootEpoch()
+	clusterQCs := clusterRootQCsFromEpoch(epoch)
+	log.Info().Msg("")
+
 	log.Info().Msg("constructing root QC")
 	rootQC := constructRootQC(
 		block,
@@ -182,34 +179,19 @@ func finalize(cmd *cobra.Command, args []string) {
 	)
 	log.Info().Msg("")
 
-	log.Info().Msg("computing collection node clusters")
-	assignments, clusters, err := constructClusterAssignment(partnerNodes, internalNodes)
-	if err != nil {
-		log.Fatal().Err(err).Msg("unable to generate cluster assignment")
-	}
-	log.Info().Msg("")
-
-	log.Info().Msg("constructing root blocks for collection node clusters")
-	clusterBlocks := run.GenerateRootClusterBlocks(flagEpochCounter, clusters)
-	log.Info().Msg("")
-
-	log.Info().Msg("constructing root QCs for collection node clusters")
-	clusterQCs := constructRootQCsForClusters(clusters, internalNodes, clusterBlocks)
-	log.Info().Msg("")
-
 	// if no root commit is specified, bootstrap an empty execution state
 	if flagRootCommit == "0000000000000000000000000000000000000000000000000000000000000000" {
 		generateEmptyExecutionState(
 			block.Header.ChainID,
-			assignments,
+			epochSetup.Assignments,
 			clusterQCs,
 			dkgData,
 			participants,
 		)
 	}
 
 	log.Info().Msg("constructing root execution result and block seal")
-	result, seal := constructRootResultAndSeal(flagRootCommit, block, participants, assignments, clusterQCs, dkgData)
+	result, seal := constructRootResultAndSeal(flagRootCommit, block, epochSetup, epochCommit)
 	log.Info().Msg("")
 
 	// construct serializable root protocol snapshot
@@ -499,15 +481,36 @@ func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) []model.NodeIn
 // readRootBlock reads root block data from disc, this file needs to be prepared with
 // rootblock command
 func readRootBlock() *flow.Block {
-	rootBlock, err := utils.ReadRootBlock(flagRootBlock)
+	rootBlock, err := utils.ReadData[flow.Block](flagRootBlock)
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not read root block data")
 	}
 	return rootBlock
 }
 
+// readRootEpoch reads root epoch data from disc, this file needs to be prepared with
+// rootblock command
+func readRootEpoch() (protocol.Epoch, *flow.EpochSetup, *flow.EpochCommit) {
+	encodableEpoch, err := utils.ReadData[inmem.EncodableEpoch](flagRootEpoch)
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not read root epoch data")
+	}
+	epoch := inmem.NewEpoch(*encodableEpoch)
+	setup, err := protocol.ToEpochSetup(epoch)
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not extract setup event")
+	}
+	commit, err := protocol.ToEpochCommit(epoch)
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not extract commit event")
+	}
+	return epoch, setup, commit
+}
+
+// readDKGData reads DKG data from disc, this file needs to be prepared with
+// rootblock command
 func readDKGData() dkg.DKGData {
-	encodableDKG, err := utils.ReadDKGData(flagDKGDataPath)
+	encodableDKG, err := utils.ReadData[inmem.EncodableFullDKG](flagDKGDataPath)
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not read DKG data")
 	}

cmd/bootstrap/cmd/finalize_test.go

@@ -29,10 +29,8 @@ const finalizeHappyPathLogs = "collecting partner network and staking keys" +
 	`reading root block votes` +
 	`read vote .*` +
 	`reading dkg data` +
+	`reading root epoch` +
 	`constructing root QC` +
-	`computing collection node clusters` +
-	`constructing root blocks for collection node clusters` +
-	`constructing root QCs for collection node clusters` +
 	`constructing root execution result and block seal` +
 	`constructing root protocol snapshot` +
 	`wrote file \S+/root-protocol-state-snapshot.json` +
@@ -79,6 +77,7 @@ func TestFinalize_HappyPath(t *testing.T) {
 		flagNumViewsInDKGPhase = 2_000
 		flagEpochCommitSafetyThreshold = 1_000
 		flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData)
+		flagRootEpoch = filepath.Join(bootDir, model.PathRootEpoch)
 		flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData)
 		flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes)
 

cmd/bootstrap/cmd/rootblock.go

@@ -6,15 +6,22 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/onflow/flow-go/cmd"
+	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/order"
+	"github.com/onflow/flow-go/state/protocol/inmem"
 )
 
 var (
-	flagRootChain     string
-	flagRootParent    string
-	flagRootHeight    uint64
-	flagRootTimestamp string
+	flagRootChain                string
+	flagRootParent               string
+	flagRootHeight               uint64
+	flagRootTimestamp            string
+	flagEpochCounter             uint64
+	flagNumViewsInEpoch          uint64
+	flagNumViewsInStakingAuction uint64
+	flagNumViewsInDKGPhase       uint64
 )
 
 // rootBlockCmd represents the rootBlock command
@@ -48,6 +55,17 @@ func addRootBlockCmdFlags() {
 	cmd.MarkFlagRequired(rootBlockCmd, "partner-dir")
 	cmd.MarkFlagRequired(rootBlockCmd, "partner-weights")
 
+	// required parameters for generation of epoch setup and commit events
+	rootBlockCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "epoch counter for the epoch beginning with the root block")
+	rootBlockCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 4000, "length of each epoch measured in views")
+	rootBlockCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views")
+	rootBlockCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views")
+
+	cmd.MarkFlagRequired(rootBlockCmd, "epoch-counter")
+	cmd.MarkFlagRequired(rootBlockCmd, "epoch-length")
+	cmd.MarkFlagRequired(rootBlockCmd, "epoch-staking-phase-length")
+	cmd.MarkFlagRequired(rootBlockCmd, "epoch-dkg-phase-length")
+
 	// required parameters for generation of root block, root execution result and root block seal
 	rootBlockCmd.Flags().StringVar(&flagRootChain, "root-chain", "local", "chain ID for the root block (can be 'main', 'test', 'sandbox', 'bench', or 'local'")
 	rootBlockCmd.Flags().StringVar(&flagRootParent, "root-parent", "0000000000000000000000000000000000000000000000000000000000000000", "ID for the parent of the root block")
@@ -92,8 +110,40 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	dkgData := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus))
 	log.Info().Msg("")
 
+	// create flow.IdentityList representation of the participant set
+	participants := model.ToIdentityList(stakingNodes).Sort(order.Canonical[flow.Identity])
+
+	log.Info().Msg("computing collection node clusters")
+	assignments, clusters, err := constructClusterAssignment(partnerNodes, internalNodes)
+	if err != nil {
+		log.Fatal().Err(err).Msg("unable to generate cluster assignment")
+	}
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root blocks for collection node clusters")
+	clusterBlocks := run.GenerateRootClusterBlocks(flagEpochCounter, clusters)
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root QCs for collection node clusters")
+	clusterQCs := constructRootQCsForClusters(clusters, internalNodes, clusterBlocks)
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root header")
+	header := constructRootHeader(flagRootChain, flagRootParent, flagRootHeight, flagRootTimestamp)
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing epoch events")
+	epochSetup, epochCommit := constructRootEpochEvents(header.View, participants, assignments, clusterQCs, dkgData)
+	committedEpoch := inmem.NewCommittedEpoch(epochSetup, epochCommit)
+	encodableEpoch, err := inmem.FromEpoch(committedEpoch)
+	if err != nil {
+		log.Fatal().Msg("could not convert root epoch to encodable")
+	}
+	writeJSON(model.PathRootEpoch, encodableEpoch.Encodable())
+	log.Info().Msg("")
+
 	log.Info().Msg("constructing root block")
-	block := constructRootBlock(flagRootChain, flagRootParent, flagRootHeight, flagRootTimestamp)
+	block := constructRootBlock(header, epochSetup, epochCommit)
 	writeJSON(model.PathRootBlockData, block)
 	log.Info().Msg("")