Khalil/6959 Recover Epoch transaction args script

cmd/bootstrap/cmd/db_encryption_key.go

@@ -56,5 +56,5 @@ func dbEncryptionKeyRun(_ *cobra.Command, _ []string) {
 		log.Fatal().Err(err).Msg("failed to write file")
 	}
 
-	log.Info().Msgf("wrote file %v", dbEncryptionKeyPath)
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, dbEncryptionKeyPath)
 }

cmd/bootstrap/cmd/dkg.go

@@ -43,6 +43,7 @@ func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData {
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed to write json")
 		}
+		log.Info().Msgf("wrote file %s/%s", flagOutdir, fmt.Sprintf(model.PathRandomBeaconPriv, nodeID))
 	}
 
 	// write full DKG info that will be used to construct QC
@@ -56,6 +57,7 @@ func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathRootDKGData)
 
 	return dkgData
 }

cmd/bootstrap/cmd/final_list.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/spf13/cobra"
 
 	"github.com/onflow/flow-go/cmd"
@@ -71,6 +72,7 @@ func finalList(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathFinallist)
 }
 
 func validateNodes(localNodes []model.NodeInfo, registeredNodes []model.NodeInfo) {

cmd/bootstrap/cmd/finalize.go

@@ -203,6 +203,7 @@ func finalize(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathRootProtocolStateSnapshot)
 	log.Info().Msg("")
 
 	// read snapshot and verify consistency

cmd/bootstrap/cmd/finalize_test.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"encoding/hex"
-	"fmt"
 	"math/rand"
 	"path/filepath"
 	"regexp"
@@ -93,7 +92,6 @@ func TestFinalize_HappyPath(t *testing.T) {
 		log = log.Hook(hook)
 
 		finalize(nil, nil)
-		fmt.Println(hook.logs.String())
 		assert.Regexp(t, finalizeHappyPathRegex, hook.logs.String())
 		hook.logs.Reset()
 

cmd/bootstrap/cmd/genconfig.go

@@ -61,6 +61,7 @@ func genconfigCmdRun(_ *cobra.Command, _ []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, flagConfig)
 }
 
 // genconfigCmd represents the genconfig command

cmd/bootstrap/cmd/key.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/onflow/crypto"
 	"github.com/spf13/cobra"
 
@@ -97,23 +98,26 @@ func keyCmdRun(_ *cobra.Command, _ []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write file")
 	}
-	log.Info().Msgf("wrote file %v", model.PathNodeID)
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeID)
 
 	err = common.WriteJSON(fmt.Sprintf(model.PathNodeInfoPriv, nodeInfo.NodeID), flagOutdir, private)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeInfoPriv)
 
 	err = common.WriteText(fmt.Sprintf(model.PathSecretsEncryptionKey, nodeInfo.NodeID), flagOutdir, secretsDBKey)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write file")
 	}
-	log.Info().Msgf("wrote file %v", model.PathSecretsEncryptionKey)
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathSecretsEncryptionKey)
 
 	err = common.WriteJSON(fmt.Sprintf(model.PathNodeInfoPub, nodeInfo.NodeID), flagOutdir, nodeInfo.Public())
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeInfoPub)
+
 	// write machine account info
 	if role == flow.RoleCollection || role == flow.RoleConsensus {
 
@@ -130,6 +134,7 @@ func keyCmdRun(_ *cobra.Command, _ []string) {
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed to write json")
 		}
+		log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeMachineAccountPrivateKey)
 	}
 }
 

cmd/bootstrap/cmd/keygen.go

@@ -129,4 +129,5 @@ func genNodePubInfo(nodes []model.NodeInfo) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathInternalNodeInfosPub)
 }

cmd/bootstrap/cmd/machine_account.go

@@ -85,6 +85,7 @@ func machineAccountRun(_ *cobra.Command, _ []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, fmt.Sprintf(model.PathNodeMachineAccountInfoPriv, nodeID))
 }
 
 // readMachineAccountPriv reads the machine account private key files in the bootstrap dir

cmd/bootstrap/cmd/machine_account_key.go

@@ -61,4 +61,5 @@ func machineAccountKeyRun(_ *cobra.Command, _ []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msg(fmt.Sprintf("wrote file %s/%s", flagOutdir, machineAccountKeyPath))
 }

cmd/bootstrap/cmd/partner_infos.go

@@ -207,6 +207,7 @@ func writeNodePubInfoFile(info *bootstrap.NodeInfoPub) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, fileOutputPath)
 }
 
 // writePartnerWeightsFile writes the partner weights file
@@ -215,6 +216,7 @@ func writePartnerWeightsFile(partnerWeights common.PartnerWeights) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, bootstrap.FileNamePartnerWeights)
 }
 
 func printNodeCounts(numOfNodesByType map[flow.Role]int, totalNumOfPartnerNodes, skippedNodes int) {

cmd/bootstrap/cmd/qc.go

@@ -53,5 +53,6 @@ func constructRootVotes(block *flow.Block, allNodes, internalNodes []bootstrap.N
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed to write json")
 		}
+		log.Info().Msgf("wrote file %s/%s", flagOutdir, path)
 	}
 }

cmd/bootstrap/cmd/rootblock.go

@@ -169,6 +169,7 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeInfosPub)
 	log.Info().Msg("")
 
 	log.Info().Msg("running DKG for consensus nodes")
@@ -221,6 +222,7 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathIntermediaryBootstrappingData)
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing root block")
@@ -229,6 +231,7 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathRootBlockData)
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing and writing votes")

cmd/dynamic_startup.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/onflow/crypto"
+
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	"github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"

cmd/util/cmd/common/clusters.go

@@ -3,9 +3,11 @@ package common
 import (
 	"errors"
 	"fmt"
+
 	"github.com/rs/zerolog"
 
 	"github.com/onflow/cadence"
+
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/model/bootstrap"
 	model "github.com/onflow/flow-go/model/bootstrap"
@@ -32,7 +34,7 @@ import (
 // - log: the logger instance.
 // - partnerNodes: identity list of partner nodes.
 // - internalNodes: identity list of internal nodes.
-// - numCollectionClusters: the number of collectors in each generated cluster.
+// - numCollectionClusters: the number of clusters to generate
 // Returns:
 // - flow.AssignmentList: the generated assignment list.
 // - flow.ClusterList: the generate collection cluster list.
@@ -98,13 +100,12 @@ func ConstructClusterAssignment(log zerolog.Logger, partnerNodes, internalNodes
 // ConstructRootQCsForClusters constructs a root QC for each cluster in the list.
 // Args:
 // - log: the logger instance.
-// - clusterList: identity list of partner nodes.
-// - nodeInfos: identity list of internal nodes.
-// - clusterBlocks: the number of collectors in each generated cluster.
+// - clusterList: list of clusters
+// - nodeInfos: list of NodeInfos (must contain all internal nodes)
+// - clusterBlocks: list of root blocks for each cluster
 // Returns:
 // - flow.AssignmentList: the generated assignment list.
 // - flow.ClusterList: the generate collection cluster list.
-// - error: if any error occurs. Any error returned from this function is irrecoverable.
 func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterList, nodeInfos []bootstrap.NodeInfo, clusterBlocks []*cluster.Block) []*flow.QuorumCertificate {
 
 	if len(clusterBlocks) != len(clusterList) {
@@ -126,21 +127,21 @@ func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterLis
 	return qcs
 }
 
-// ConvertClusterAssignmentsCdc converts golang cluster assignments type to cadence array of arrays.
+// ConvertClusterAssignmentsCdc converts golang cluster assignments type to Cadence type `[[String]]`.
 func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array {
 	assignmentsCdc := make([]cadence.Value, len(assignments))
 	for i, asmt := range assignments {
 		vals := make([]cadence.Value, asmt.Len())
-		for j, k := range asmt {
-			vals[j] = cadence.String(k.String())
+		for j, nodeID := range asmt {
+			vals[j] = cadence.String(nodeID.String())
 		}
 		assignmentsCdc[i] = cadence.NewArray(vals).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{}))
 	}
 
 	return cadence.NewArray(assignmentsCdc).WithType(cadence.NewVariableSizedArrayType(cadence.NewVariableSizedArrayType(cadence.StringType{})))
 }
 
-// ConvertClusterQcsCdc converts golang cluster qcs type to cadence struct.
+// ConvertClusterQcsCdc converts cluster QCs from `QuorumCertificate` type to `ClusterQCVoteData` type.
 func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList) ([]*flow.ClusterQCVoteData, error) {
 	voteData := make([]*flow.ClusterQCVoteData, len(qcs))
 	for i, qc := range qcs {

cmd/util/cmd/common/utils.go

@@ -8,10 +8,11 @@ import (
 	"path/filepath"
 	"strconv"
 
+	"github.com/multiformats/go-multiaddr"
 	"github.com/rs/zerolog"
 
-	"github.com/multiformats/go-multiaddr"
 	"github.com/onflow/crypto"
+
 	"github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/encodable"
 	"github.com/onflow/flow-go/model/flow"

cmd/util/cmd/epochs/cmd/recover.go

@@ -3,9 +3,11 @@ package cmd
 import (
 	"context"
 	"fmt"
+
 	"github.com/spf13/cobra"
 
 	"github.com/onflow/cadence"
+
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	epochcmdutil "github.com/onflow/flow-go/cmd/util/cmd/epochs/utils"
@@ -19,12 +21,21 @@ import (
 // The full epoch data must be generated manually and submitted with this transaction in order for an
 // EpochRecover event to be emitted. This command retrieves the current protocol state identities, computes the cluster assignment using those
 // identities, generates the cluster QC's and retrieves the DKG key vector of the last successful epoch.
+// This recovery process has some constraints:
+//   - The RecoveryEpoch must have exactly the same consensus committee as participated in the most recent successful DKG.
+//   - The RecoveryEpoch must contain enough "internal" collection nodes so that all clusters contain a supermajority of "internal" collection nodes (same constraint as sporks)
 var (
 	generateRecoverEpochTxArgsCmd = &cobra.Command{
 		Use:   "efm-recover-tx-args",
 		Short: "Generates recover epoch transaction arguments",
-		Long:  "Generates transaction arguments for the epoch recovery transaction.",
-		Run:   generateRecoverEpochTxArgs(getSnapshot),
+		Long: `
+Generates transaction arguments for the epoch recovery transaction.
+The epoch recovery transaction is used to recover from any failure in the epoch transition process without requiring a spork.
+This recovery process has some constraints:
+- The RecoveryEpoch must have exactly the same consensus committee as participated in the most recent successful DKG.
+- The RecoveryEpoch must contain enough "internal" collection nodes so that all clusters contain a supermajority of "internal" collection nodes (same constraint as sporks)
+`,
+		Run: generateRecoverEpochTxArgs(getSnapshot),
 	}
 
 	flagAnAddress                string
@@ -39,20 +50,41 @@ var (
 
 func init() {
 	rootCmd.AddCommand(generateRecoverEpochTxArgsCmd)
-	addGenerateRecoverEpochTxArgsCmdFlags()
+	err := addGenerateRecoverEpochTxArgsCmdFlags()
+	if err != nil {
+		panic(err)
+	}
 }
 
-func addGenerateRecoverEpochTxArgsCmdFlags() {
+func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().IntVar(&flagCollectionClusters, "collection-clusters", 3,
 		"number of collection clusters")
 	// required parameters for network configuration and generation of root node identities
-	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagNodeConfigJson, "node-config", "",
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagNodeConfigJson, "config", "",
 		"path to a JSON file containing multiple node configurations (fields Role, Address, Weight)")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagInternalNodePrivInfoDir, "internal-priv-dir", "", "path to directory "+
 		"containing the output from the `keygen` command for internal nodes")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 4000, "length of each epoch measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
+
+	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-length")
+	if err != nil {
+		return fmt.Errorf("failed to mark epoch-length flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-staking-phase-length")
+	if err != nil {
+		return fmt.Errorf("failed to mark epoch-staking-phase-length flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-counter")
+	if err != nil {
+		return fmt.Errorf("failed to mark epoch-counter flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("collection-clusters")
+	if err != nil {
+		return fmt.Errorf("failed to mark collection-clusters flag as required")
+	}
+	return nil
 }
 
 func getSnapshot() *inmem.Snapshot {
@@ -97,17 +129,17 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 	}
 }
 
-// extractResetEpochArgs extracts the required transaction arguments for the `resetEpoch` transaction
+// extractRecoverEpochArgs extracts the required transaction arguments for the `recoverEpoch` transaction.
 func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 	epoch := snapshot.Epochs().Current()
 
-	ids, err := snapshot.Identities(filter.IsValidProtocolParticipant)
+	currentEpochIdentities, err := snapshot.Identities(filter.IsValidProtocolParticipant)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to get  valid protocol participants from snapshot")
 	}
 
 	// separate collector nodes by internal and partner nodes
-	collectors := ids.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
+	collectors := currentEpochIdentities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
 	internalCollectors := make(flow.IdentityList, 0)
 	partnerCollectors := make(flow.IdentityList, 0)
 
@@ -119,7 +151,7 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 
 	internalNodesMap := make(map[flow.Identifier]struct{})
 	for _, node := range internalNodes {
-		if !ids.Exists(node.Identity()) {
+		if !currentEpochIdentities.Exists(node.Identity()) {
 			log.Fatal().Msg(fmt.Sprintf("node ID found in internal node infos missing from protocol snapshot identities: %s", node.NodeID))
 		}
 		internalNodesMap[node.NodeID] = struct{}{}
@@ -158,12 +190,14 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 	dkgPubKeys := make([]cadence.Value, 0)
 	nodeIds := make([]cadence.Value, 0)
 
+	// NOTE: The RecoveryEpoch will re-use the last successful DKG output. This means that the consensus
+	// committee in the RecoveryEpoch must be identical to the committee which participated in that DKG.
 	dkgGroupKeyCdc, cdcErr := cadence.NewString(currentEpochDKG.GroupKey().String())
 	if cdcErr != nil {
 		log.Fatal().Err(cdcErr).Msg("failed to get dkg group key cadence string")
 	}
 	dkgPubKeys = append(dkgPubKeys, dkgGroupKeyCdc)
-	for _, id := range ids {
+	for _, id := range currentEpochIdentities {
 		if id.GetRole() == flow.RoleConsensus {
 			dkgPubKey, keyShareErr := currentEpochDKG.KeyShare(id.GetNodeID())
 			if keyShareErr != nil {