Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[randomness part 8] update math/rand usage in /cmd/bootstrap #4362

Merged
merged 8 commits into from
May 31, 2023
Merged

[randomness part 8] update math/rand usage in /cmd/bootstrap #4362

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b7d2185
remove deterministic bootstrap seed
May 17, 2023
bdb5330
use utils/rand to shuffle identity list
May 17, 2023
84fb4a7
make build clusters guarantee that internal nodes are 2/3 in each clu…
May 20, 2023
362cbca
add tests for valid and invalid cluster assignment
May 20, 2023
79a8d9a
fix a test bug
May 20, 2023
b416840
ensureUniformNodeWeightsPerRole panics if one role has no roles
May 31, 2023
192070f
move checkClusterConstraint as a cluster logic unit test
May 31, 2023
86bc599
Merge branch 'master' into tarak/8-bootstrap-seed
tarakby May 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 40 additions & 12 deletions cmd/bootstrap/cmd/clusters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,35 @@
package cmd

import (
"errors"

"github.com/onflow/flow-go/cmd/bootstrap/run"
model "github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/model/cluster"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/model/flow/assignment"
"github.com/onflow/flow-go/model/flow/factory"
"github.com/onflow/flow-go/model/flow/filter"
"github.com/onflow/flow-go/utils/rand"
)

// Construct cluster assignment with internal and partner nodes uniformly
// distributed across clusters. This function will produce the same cluster
// assignments for the same partner and internal lists, and the same seed.
func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, seed int64) (flow.AssignmentList, flow.ClusterList) {
// Construct random cluster assignment with internal and partner nodes.
// The number of clusters is read from the `flagCollectionClusters` flag.
// The number of nodes in each cluster is deterministic and only depends on the number of clusters
// and the number of nodes. The repartition of internal and partner nodes is also deterministic
// and only depends on the number of clusters and nodes.
// The identity of internal and partner nodes in each cluster is the non-deterministic and is randomized
// using the system entropy.
// The function guarantees a specific constraint when partitioning the nodes into clusters:
// Each cluster must contain strictly more than 2/3 of internal nodes. If the constraint can't be
// satisfied, an exception is returned.
// Note that if an exception is returned with a certain number of internal/partner nodes, there is no chance
// of succeeding the assignment by re-running the function without increasing the internal nodes ratio.
func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (flow.AssignmentList, flow.ClusterList, error) {

partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole(flow.RoleCollection))
internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole(flow.RoleCollection))
nClusters := flagCollectionClusters
nClusters := int(flagCollectionClusters)
nCollectors := len(partners) + len(internals)

// ensure we have at least as many collection nodes as clusters
Expand All @@ -26,22 +38,38 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, se
nCollectors, flagCollectionClusters)
}

// deterministically shuffle both collector lists based on the input seed
// by using a different seed each spork, we will have different clusters
// even with the same collectors
partners = partners.DeterministicShuffle(seed)
internals = internals.DeterministicShuffle(seed)
// shuffle both collector lists based on a non-deterministic algorithm
var err error
err = rand.Shuffle(uint(len(partners)), func(i, j uint) { partners[i], partners[j] = partners[j], partners[i] })
if err != nil {
log.Fatal().Err(err).Msg("could not shuffle partners")
}
err = rand.Shuffle(uint(len(internals)), func(i, j uint) { internals[i], internals[j] = internals[j], internals[i] })
if err != nil {
log.Fatal().Err(err).Msg("could not shuffle internals")
}

identifierLists := make([]flow.IdentifierList, nClusters)
// array to track the 2/3 internal-nodes constraint (internal_nodes > 2 * partner_nodes)
constraint := make([]int, nClusters)

// first, round-robin internal nodes into each cluster
for i, node := range internals {
identifierLists[i%len(identifierLists)] = append(identifierLists[i%len(identifierLists)], node.NodeID)
identifierLists[i%nClusters] = append(identifierLists[i%nClusters], node.NodeID)
constraint[i%nClusters] += 1
}

// next, round-robin partner nodes into each cluster
for i, node := range partners {
identifierLists[i%len(identifierLists)] = append(identifierLists[i%len(identifierLists)], node.NodeID)
constraint[i%nClusters] -= 2
}

// check the 2/3 constraint: for every cluster `i`, constraint[i] must be strictly positive
for i := 0; i < nClusters; i++ {
if constraint[i] <= 0 {
return nil, nil, errors.New("there isn't enough internal nodes to have at least 2/3 internal nodes in each cluster")
}
}

assignments := assignment.FromIdentifierLists(identifierLists)
Expand All @@ -52,7 +80,7 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, se
log.Fatal().Err(err).Msg("could not create cluster list")
}

return assignments, clusters
return assignments, clusters, nil
}

func constructRootQCsForClusters(
Expand Down
34 changes: 22 additions & 12 deletions cmd/bootstrap/cmd/constraints.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ func ensureUniformNodeWeightsPerRole(allNodes flow.IdentityList) {
// ensure all nodes of the same role have equal weight
for _, role := range flow.Roles() {
withRole := allNodes.Filter(filter.HasRole(role))
if len(withRole) == 0 {
continue
}
tarakby marked this conversation as resolved.
Show resolved Hide resolved
expectedWeight := withRole[0].Weight
for _, node := range withRole {
if node.Weight != expectedWeight {
Expand All @@ -35,14 +38,23 @@ func checkConstraints(partnerNodes, internalNodes []model.NodeInfo) {

ensureUniformNodeWeightsPerRole(all)

// check collection committee Byzantine threshold for each cluster
// for checking Byzantine constraints, the seed doesn't matter
_, clusters := constructClusterAssignment(partnerNodes, internalNodes, 0)
partnerCOLCount := uint(0)
internalCOLCount := uint(0)
for _, cluster := range clusters {
clusterPartnerCount := uint(0)
clusterInternalCount := uint(0)
// check collection committee threshold of internal nodes in each cluster
// although the assignmment is non-deterministic, the number of internal/partner
// nodes in each cluster is deterministic. The following check is only a sanity
// check about the number of internal/partner nodes in each cluster. The identites
// in each cluster do not matter for this sanity check.
_, clusters, err := constructClusterAssignment(partnerNodes, internalNodes)
if err != nil {
log.Fatal().Msgf("can't bootstrap because the cluster assignment failed: %s", err)
tarakby marked this conversation as resolved.
Show resolved Hide resolved
}
checkClusterConstraint(clusters, partners, internals)
}

// Sanity check about the number of internal/partner nodes in each cluster. The identites
// in each cluster do not matter for this sanity check.
func checkClusterConstraint(clusters flow.ClusterList, partners flow.IdentityList, internals flow.IdentityList) {
for i, cluster := range clusters {
tarakby marked this conversation as resolved.
Show resolved Hide resolved
var clusterPartnerCount, clusterInternalCount int
for _, node := range cluster {
if _, exists := partners.ByNodeID(node.NodeID); exists {
clusterPartnerCount++
Expand All @@ -53,11 +65,9 @@ func checkConstraints(partnerNodes, internalNodes []model.NodeInfo) {
}
if clusterInternalCount <= clusterPartnerCount*2 {
log.Fatal().Msgf(
"will not bootstrap configuration without Byzantine majority within cluster: "+
"can't bootstrap because cluster %d doesn't have enough internal nodes: "+
"(partners=%d, internals=%d, min_internals=%d)",
clusterPartnerCount, clusterInternalCount, clusterPartnerCount*2+1)
i, clusterPartnerCount, clusterInternalCount, clusterPartnerCount*2+1)
}
partnerCOLCount += clusterPartnerCount
internalCOLCount += clusterInternalCount
}
}
26 changes: 9 additions & 17 deletions cmd/bootstrap/cmd/finalize.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package cmd

import (
"encoding/binary"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -48,9 +48,6 @@ var (
flagNumViewsInStakingAuction uint64
flagNumViewsInDKGPhase uint64
flagEpochCommitSafetyThreshold uint64

// this flag is used to seed the DKG, clustering and cluster QC generation
flagBootstrapRandomSeed []byte
)

// PartnerWeights is the format of the JSON file specifying partner node weights.
Expand Down Expand Up @@ -101,7 +98,6 @@ func addFinalizeCmdFlags() {
finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views")
finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views")
finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline")
finalizeCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(flow.EpochSetupRandomSourceLength), "The seed used to for DKG, Clustering and Cluster QC generation")
finalizeCmd.Flags().UintVar(&flagProtocolVersion, "protocol-version", flow.DefaultProtocolVersion, "major software version used for the duration of this spork")

cmd.MarkFlagRequired(finalizeCmd, "root-block")
Expand Down Expand Up @@ -143,14 +139,6 @@ func finalize(cmd *cobra.Command, args []string) {
log.Fatal().Err(err).Msg("invalid or unsafe epoch commit threshold config")
}

if len(flagBootstrapRandomSeed) != flow.EpochSetupRandomSourceLength {
log.Error().Int("expected", flow.EpochSetupRandomSourceLength).Int("actual", len(flagBootstrapRandomSeed)).Msg("random seed provided length is not valid")
return
}

log.Info().Str("seed", hex.EncodeToString(flagBootstrapRandomSeed)).Msg("deterministic bootstrapping random seed")
log.Info().Msg("")

log.Info().Msg("collecting partner network and staking keys")
partnerNodes := readPartnerNodeInfos()
log.Info().Msg("")
Expand Down Expand Up @@ -195,8 +183,10 @@ func finalize(cmd *cobra.Command, args []string) {
log.Info().Msg("")

log.Info().Msg("computing collection node clusters")
clusterAssignmentSeed := binary.BigEndian.Uint64(flagBootstrapRandomSeed)
assignments, clusters := constructClusterAssignment(partnerNodes, internalNodes, int64(clusterAssignmentSeed))
assignments, clusters, err := constructClusterAssignment(partnerNodes, internalNodes)
if err != nil {
log.Fatal().Err(err).Msg("unable to generate cluster assignment")
}
log.Info().Msg("")

log.Info().Msg("constructing root blocks for collection node clusters")
Expand All @@ -211,7 +201,6 @@ func finalize(cmd *cobra.Command, args []string) {
if flagRootCommit == "0000000000000000000000000000000000000000000000000000000000000000" {
generateEmptyExecutionState(
block.Header.ChainID,
flagBootstrapRandomSeed,
assignments,
clusterQCs,
dkgData,
Expand Down Expand Up @@ -587,7 +576,6 @@ func loadRootProtocolSnapshot(path string) (*inmem.Snapshot, error) {
// given configuration. Sets the flagRootCommit variable for future reads.
func generateEmptyExecutionState(
chainID flow.ChainID,
randomSource []byte,
assignments flow.AssignmentList,
clusterQCs []*flow.QuorumCertificate,
dkgData dkg.DKGData,
Expand All @@ -606,6 +594,10 @@ func generateEmptyExecutionState(
log.Fatal().Err(err).Msg("invalid genesis token supply")
}

randomSource := make([]byte, flow.EpochSetupRandomSourceLength)
if _, err = rand.Read(randomSource); err != nil {
log.Fatal().Err(err).Msg("failed to generate a random source")
}
cdcRandomSource, err := cadence.NewString(hex.EncodeToString(randomSource))
if err != nil {
log.Fatal().Err(err).Msg("invalid random source")
Expand Down
Loading