[Checkpointing] ExitEvent storage and proof generation (#838)

* Exit event storage and query functions

* ExitProof generation

* Reorganize tests

* Optimize key composition

* Extract common function for decoding state sync and exit events

* Comments fix

* Comments fix

* [Checkpointing] JSON RPC point for generating exit proofs (#844)

* JSON RPC point for generating exit proofs

* Comments fix

Co-authored-by: Stefan Negovanović <stefan@ethernal.tech>

consensus/consensus.go

@@ -34,6 +34,9 @@ type Consensus interface {
 	// GetSyncProgression retrieves the current sync progression, if any
 	GetSyncProgression() *progress.Progression
 
+	// GetBridgeProvider returns an instance of BridgeDataProvider
+	GetBridgeProvider() BridgeDataProvider
+
 	// Initialize initializes the consensus (e.g. setup data)
 	Initialize() error
 
@@ -75,3 +78,9 @@ type Params struct {
 
 // Factory is the factory function to create a discovery consensus
 type Factory func(*Params) (Consensus, error)
+
+// BridgeDataProvider is an interface providing bridge related functions
+type BridgeDataProvider interface {
+	// GenerateExit proof generates proof of exit for given exit event
+	GenerateExitProof(exitID, epoch, checkpointBlock uint64) ([]types.Hash, error)
+}

consensus/dev/dev.go

@@ -242,3 +242,7 @@ func (d *Dev) Close() error {
 
 	return nil
 }
+
+func (d *Dev) GetBridgeProvider() consensus.BridgeDataProvider {
+	return nil
+}

consensus/dummy/dummy.go

@@ -75,6 +75,10 @@ func (d *Dummy) Close() error {
 	return nil
 }
 
+func (d *Dummy) GetBridgeProvider() consensus.BridgeDataProvider {
+	return nil
+}
+
 func (d *Dummy) run() {
 	d.logger.Info("started")
 	// do nothing

consensus/ibft/ibft.go

@@ -532,6 +532,11 @@ func (i *backendIBFT) SetHeaderHash() {
 	}
 }
 
+// GetBridgeProvider returns an instance of BridgeDataProvider
+func (i *backendIBFT) GetBridgeProvider() consensus.BridgeDataProvider {
+	return nil
+}
+
 // updateCurrentModules updates Signer, Hooks, and Validators
 // that are used at specified height
 // by fetching from ForkManager

consensus/polybft/consensus_runtime.go

@@ -150,7 +150,7 @@ func (c *consensusRuntime) AddLog(eventLog *ethgo.Log) {
 		"index", eventLog.LogIndex,
 	)
 
-	event, err := decodeEvent(eventLog)
+	event, err := decodeStateSyncEvent(eventLog)
 	if err != nil {
 		c.logger.Error("failed to decode state sync event", "hash", eventLog.TransactionHash, "err", err)
 
@@ -281,15 +281,16 @@ func (c *consensusRuntime) FSM() (*fsm, error) {
 	isEndOfEpoch := c.isEndOfEpoch(pendingBlockNumber)
 
 	ff := &fsm{
-		config:         c.config.PolyBFTConfig,
-		parent:         parent,
-		backend:        c.config.blockchain,
-		polybftBackend: c.config.polybftBackend,
-		blockBuilder:   blockBuilder,
-		validators:     newValidatorSet(types.BytesToAddress(parent.Miner), epoch.Validators),
-		isEndOfEpoch:   isEndOfEpoch,
-		isEndOfSprint:  isEndOfSprint,
-		logger:         c.logger.Named("fsm"),
+		config:            c.config.PolyBFTConfig,
+		parent:            parent,
+		backend:           c.config.blockchain,
+		polybftBackend:    c.config.polybftBackend,
+		blockBuilder:      blockBuilder,
+		validators:        newValidatorSet(types.BytesToAddress(parent.Miner), epoch.Validators),
+		isEndOfEpoch:      isEndOfEpoch,
+		isEndOfSprint:     isEndOfSprint,
+		generateEventRoot: c.getExitEventRootHash,
+		logger:            c.logger.Named("fsm"),
 	}
 
 	if c.IsBridgeEnabled() {
@@ -760,6 +761,50 @@ func (c *consensusRuntime) calculateUptime(currentBlock *types.Header) (*CommitE
 	return commitEpoch, nil
 }
 
+// getExitEventRootHash returns the exit event root hash from gathered exit events in given epoch
+func (c *consensusRuntime) getExitEventRootHash(epoch uint64) (types.Hash, error) {
+	exitEvents, err := c.state.getExitEventsByEpoch(epoch)
+	if err != nil {
+		return types.ZeroHash, err
+	}
+
+	if len(exitEvents) == 0 {
+		return types.ZeroHash, nil
+	}
+
+	tree, err := createExitTree(exitEvents)
+	if err != nil {
+		return types.ZeroHash, err
+	}
+
+	return tree.Hash(), nil
+}
+
+// GenerateExitProof generates proof of exit
+func (c *consensusRuntime) GenerateExitProof(exitID, epoch, checkpointBlock uint64) ([]types.Hash, error) {
+	exitEvent, err := c.state.getExitEvent(exitID, epoch, checkpointBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	e, err := exitEventABIType.Encode(exitEvent)
+	if err != nil {
+		return nil, err
+	}
+
+	exitEvents, err := c.state.getExitEventsForProof(epoch, checkpointBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	tree, err := createExitTree(exitEvents)
+	if err != nil {
+		return nil, err
+	}
+
+	return tree.GenerateProofForLeaf(e, 0)
+}
+
 // setIsActiveValidator updates the activeValidatorFlag field
 func (c *consensusRuntime) setIsActiveValidator(isActiveValidator bool) {
 	if isActiveValidator {
@@ -889,3 +934,20 @@ func validateVote(vote *MessageSignature, epoch *epochMetadata) error {
 
 	return nil
 }
+
+// createExitTree creates an exit event merkle tree from provided exit events
+func createExitTree(exitEvents []*ExitEvent) (*MerkleTree, error) {
+	numOfEvents := len(exitEvents)
+	data := make([][]byte, numOfEvents)
+
+	for i := 0; i < numOfEvents; i++ {
+		b, err := exitEventABIType.Encode(exitEvents[i])
+		if err != nil {
+			return nil, err
+		}
+
+		data[i] = b
+	}
+
+	return NewMerkleTree(data)
+}

consensus/polybft/consensus_runtime_test.go

@@ -150,7 +150,7 @@ func TestConsensusRuntime_AddLog(t *testing.T) {
 		config: &runtimeConfig{Key: createTestKey(t)},
 	}
 	topics := make([]ethgo.Hash, 4)
-	topics[0] = stateTransferEvent.ID()
+	topics[0] = stateTransferEventABI.ID()
 	topics[1] = ethgo.BytesToHash([]byte{0x1})
 	topics[2] = ethgo.BytesToHash(runtime.config.Key.Address().Bytes())
 	topics[3] = ethgo.BytesToHash(contracts.NativeTokenContract[:])
@@ -167,7 +167,7 @@ func TestConsensusRuntime_AddLog(t *testing.T) {
 		Topics:          topics,
 		Data:            encodedData,
 	}
-	event, err := decodeEvent(log)
+	event, err := decodeStateSyncEvent(log)
 	require.NoError(t, err)
 	runtime.AddLog(log)
 
@@ -534,6 +534,8 @@ func TestConsensusRuntime_FSM_NotInValidatorSet(t *testing.T) {
 func TestConsensusRuntime_FSM_NotEndOfEpoch_NotEndOfSprint(t *testing.T) {
 	t.Parallel()
 
+	state := newTestState(t)
+
 	lastBlock := &types.Header{Number: 1}
 	validators := newTestValidators(3)
 	blockchainMock := new(blockchainMock)
@@ -555,6 +557,7 @@ func TestConsensusRuntime_FSM_NotEndOfEpoch_NotEndOfSprint(t *testing.T) {
 			Validators: validators.getPublicIdentities(),
 		},
 		lastBuiltBlock: lastBlock,
+		state:          state,
 	}
 
 	fsm, err := runtime.FSM()
@@ -1639,6 +1642,100 @@ func TestConsensusRuntime_FSM_EndOfEpoch_PostHook(t *testing.T) {
 	blockchainMock.AssertExpectations(t)
 }
 
+func TestConsensusRuntime_getExitEventRootHash(t *testing.T) {
+	const (
+		numOfBlocks         = 10
+		numOfEventsPerBlock = 2
+	)
+
+	state := newTestState(t)
+	runtime := &consensusRuntime{
+		state: state,
+	}
+
+	encodedEvents := setupExitEventsForProofVerification(t, state, numOfBlocks, numOfEventsPerBlock)
+
+	t.Run("Get exit event root hash", func(t *testing.T) {
+		tree, err := NewMerkleTree(encodedEvents)
+		require.NoError(t, err)
+
+		hash, err := runtime.getExitEventRootHash(1)
+		require.NoError(t, err)
+		require.Equal(t, tree.Hash(), hash)
+	})
+
+	t.Run("Get exit event root hash - no events", func(t *testing.T) {
+		hash, err := runtime.getExitEventRootHash(2)
+		require.NoError(t, err)
+		require.Equal(t, types.Hash{}, hash)
+	})
+}
+
+func TestConsensusRuntime_GenerateExitProof(t *testing.T) {
+	const (
+		numOfBlocks         = 10
+		numOfEventsPerBlock = 2
+	)
+
+	state := newTestState(t)
+	runtime := &consensusRuntime{
+		state: state,
+	}
+
+	encodedEvents := setupExitEventsForProofVerification(t, state, numOfBlocks, numOfEventsPerBlock)
+	checkpointEvents := encodedEvents[:numOfEventsPerBlock]
+
+	// manually create merkle tree for a desired checkpoint to verify the generated proof
+	tree, err := NewMerkleTree(checkpointEvents)
+	require.NoError(t, err)
+
+	proof, err := runtime.GenerateExitProof(1, 1, 1)
+	require.NoError(t, err)
+	require.NotNil(t, proof)
+
+	t.Run("Generate and validate exit proof", func(t *testing.T) {
+		// verify generated proof on desired tree
+		require.NoError(t, VerifyProof(1, encodedEvents[1], proof, tree.Hash()))
+	})
+
+	t.Run("Generate and validate exit proof - invalid proof", func(t *testing.T) {
+		invalidProof := proof
+		invalidProof[0][0]++
+
+		// verify generated proof on desired tree
+		require.ErrorContains(t, VerifyProof(1, encodedEvents[1], invalidProof, tree.Hash()), "not a member of merkle tree")
+	})
+
+	t.Run("Generate exit proof - no event", func(t *testing.T) {
+		_, err := runtime.GenerateExitProof(21, 1, 1)
+		require.ErrorContains(t, err, "could not find any exit event that has an id")
+	})
+}
+
+func setupExitEventsForProofVerification(t *testing.T, state *State,
+	numOfBlocks, numOfEventsPerBlock uint64) [][]byte {
+	t.Helper()
+
+	encodedEvents := make([][]byte, numOfBlocks*numOfEventsPerBlock)
+	index := uint64(0)
+
+	for i := uint64(1); i <= numOfBlocks; i++ {
+		for j := uint64(1); j <= numOfEventsPerBlock; j++ {
+			e := &ExitEvent{index, ethgo.ZeroAddress, ethgo.ZeroAddress, []byte{0, 1}, 1, i}
+			require.NoError(t, state.insertExitEvent(e))
+
+			b, err := exitEventABIType.Encode(e)
+
+			require.NoError(t, err)
+
+			encodedEvents[index] = b
+			index++
+		}
+	}
+
+	return encodedEvents
+}
+
 func createTestTransportMessage(t *testing.T, hash []byte, epochNumber uint64, key *wallet.Key) *TransportMessage {
 	t.Helper()
 

consensus/polybft/fsm.go

@@ -85,6 +85,9 @@ type fsm struct {
 	// stateSyncExecutionIndex is the next state sync execution index in smart contract
 	stateSyncExecutionIndex uint64
 
+	// generateEventRoot returns the root hash of exit event tree
+	generateEventRoot func(epoch uint64) (types.Hash, error)
+
 	logger hcf.Logger // The logger object
 }
 
@@ -158,6 +161,8 @@ func (f *fsm) BuildProposal() (*pbft.Proposal, error) {
 		headerTime = time.Now()
 	}
 
+	// TODO - Here we will call f.generateEventRootFunc(epoch) to get the exit root after adding transactions
+
 	stateBlock, err := f.blockBuilder.Build(func(h *types.Header) {
 		h.Timestamp = uint64(headerTime.Unix())
 		h.ExtraData = append(make([]byte, signer.IstanbulExtraVanity), extra.MarshalRLPTo(nil)...)

consensus/polybft/fsm_test.go

@@ -1559,7 +1559,7 @@ func createTestCommitment(t *testing.T, accounts []*wallet.Account) *CommitmentM
 			uint64(i),
 			accounts[i].Ecdsa.Address(),
 			accounts[0].Ecdsa.Address(),
-			[]byte{}, nil,
+			[]byte{},
 		)
 
 		bitmap.Set(uint64(i))

consensus/polybft/merkle_tree.go

@@ -12,6 +12,8 @@ import (
 	"github.com/0xPolygon/polygon-edge/types"
 )
 
+var errLeafNotFound = errors.New("leaf not found")
+
 // MerkleTree is the structure for the Merkle tree.
 type MerkleTree struct {
 	// hasher is a pointer to the hashing struct (e.g., Keccak256)
@@ -65,6 +67,17 @@ func NewMerkleTreeWithHashing(data [][]byte, hash hash.Hash) (*MerkleTree, error
 	return tree, nil
 }
 
+// LeafIndex returns the index of given leaf if found in tree
+func (t *MerkleTree) LeafIndex(leaf []byte) (uint64, error) {
+	for i, d := range t.data {
+		if bytes.Equal(d, leaf) {
+			return uint64(i), nil
+		}
+	}
+
+	return 0, errLeafNotFound
+}
+
 // Hash is the Merkle Tree root hash
 func (t *MerkleTree) Hash() types.Hash {
 	return types.BytesToHash(t.nodes[1])
@@ -76,7 +89,6 @@ func (t *MerkleTree) String() string {
 }
 
 // GenerateProof generates the proof of membership for a piece of data in the Merkle tree.
-// If the data is not present in the tree this will return an error.
 func (t *MerkleTree) GenerateProof(index uint64, height int) []types.Hash {
 	proofLen := int(math.Ceil(math.Log2(float64(len(t.data))))) - height
 	proofHashes := make([]types.Hash, proofLen)
@@ -92,6 +104,17 @@ func (t *MerkleTree) GenerateProof(index uint64, height int) []types.Hash {
 	return proofHashes
 }
 
+// GenerateProofForLeaf generates the proof of membership for a piece of data in the Merkle tree.
+// If the data is not present in the tree this will return an error
+func (t *MerkleTree) GenerateProofForLeaf(leaf []byte, height int) ([]types.Hash, error) {
+	leafIndex, err := t.LeafIndex(leaf)
+	if err != nil {
+		return nil, err
+	}
+
+	return t.GenerateProof(leafIndex, height), nil
+}
+
 // VerifyProof verifies a Merkle tree proof of membership for provided data using the default hash type (Keccak256)
 func VerifyProof(index uint64, leaf []byte, proof []types.Hash, root types.Hash) error {
 	return VerifyProofUsing(index, leaf, proof, root, crypto.NewKeccakState())

consensus/polybft/polybft.go

@@ -618,6 +618,11 @@ func (p *Polybft) PreCommitState(_ *types.Header, _ *state.Transition) error {
 	return nil
 }
 
+// GetBridgeProvider returns an instance of BridgeDataProvider
+func (p *Polybft) GetBridgeProvider() consensus.BridgeDataProvider {
+	return p.runtime
+}
+
 type pbftTransportWrapper struct {
 	topic *network.Topic
 }