Sync execution update on demand (#1154)

* Update for Dencun

* build geth&lodestar from source

* Fix makeTrie

* Update go ethereum to v1.13.5

* Update packages & Speedup e2e setup

* Update lodestar to v1.12.0

* Remove unused

* Remove unrelated changes

* Ignore build geth when binary exist

* Cleanup for deprecated ethashproof

* Update with VersionedExecutionPayloadHeader

* Fix breaking tests

* Update node package

* Update git modules

* Sync packages with nix

* Update sdk

* Update contract binding

* More cleanup

* For deneb(1)

* For deneb(2)

* Update sdk

* For deneb(3)

* Update sdk

* For deneb(4)

* Enable deneb for local setup

* Fix tests

* Fix BeaconStateDeneb & more tests

* Generate test fixture for Deneb and more tests accordingly

* Fix parse empty field

* Setup for switchover test

* Test workflow specific for deneb

* Ignore change detect building polkadot binary

* Encode with error handling

* Fix format

* Update lodestar

* Improve scripts setting up nodes for production

* Sync execution update on demand

* Fix import

* Reorganize deneb primitives

* Update go mod

* Fix format

* Remove obsolete

* Fix import

* Fix the deneb ForkVersion

* Upgrade geth and lodestar

* Download geth release to replace the nix default

* Naming consistently

* Update lodestar with the mock hack

* Fix generate mainnet fixture

* Remove fixture unused

* Backward compatible

* Check execution_branch

* Remove build geth

* Remove cross check

* sync all lagging periods

* Clean up

* More refactoring

* Cleanup scripts

* Update sdk

* Ignore go.work

* Rename function

* Fix test

* Fix populate checkpoint

* Fix smoke test

* Update lodestar & polkadot-sdk

* Check execution already exist before submit

* Update relayer/cmd/generate_beacon_data.go

Co-authored-by: Clara van Staden <claravanstaden64@gmail.com>

* Update relayer/relays/beacon/header/header.go

Co-authored-by: Clara van Staden <claravanstaden64@gmail.com>

* Rename to OptionalExecutionHeader

* Rename import alias

* Improve logs

* Fetch checkpoint on chain from history

* Remove LastExecutionHeaderState & Remove execution header from update

---------

Co-authored-by: Clara van Staden <claravanstaden64@gmail.com>

.gitignore

@@ -30,3 +30,5 @@ contracts/beefy-state.json
 
 go/
 gocache/
+go.work*
+control/target/

relayer/chain/ethereum/header.go

@@ -5,11 +5,12 @@ package ethereum
 
 import (
 	"fmt"
+	"math/big"
+
 	etypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/snowfork/go-substrate-rpc-client/v4/scale"
 	types "github.com/snowfork/go-substrate-rpc-client/v4/types"
-	"math/big"
 )
 
 type HeaderID struct {

relayer/chain/parachain/writer.go

@@ -266,31 +266,6 @@ func (wr *ParachainWriter) GetLastBasicChannelNonceByAddress(address common.Addr
 	return uint64(nonce), nil
 }
 
-func (wr *ParachainWriter) GetLastExecutionHeaderState() (state.ExecutionHeader, error) {
-	key, err := types.CreateStorageKey(wr.conn.Metadata(), "EthereumBeaconClient", "LatestExecutionState", nil, nil)
-	if err != nil {
-		return state.ExecutionHeader{}, fmt.Errorf("create storage key for LatestExecutionHeaderState: %w", err)
-	}
-
-	var storageState struct {
-		BeaconBlockRoot types.H256
-		BeaconSlot      types.U64
-		BlockHash       types.H256
-		BlockNumber     types.U64
-	}
-	_, err = wr.conn.API().RPC.State.GetStorageLatest(key, &storageState)
-	if err != nil {
-		return state.ExecutionHeader{}, fmt.Errorf("get storage for LatestExecutionHeaderState (err): %w", err)
-	}
-
-	return state.ExecutionHeader{
-		BeaconBlockRoot: common.Hash(storageState.BeaconBlockRoot),
-		BeaconSlot:      uint64(storageState.BeaconSlot),
-		BlockHash:       common.Hash(storageState.BlockHash),
-		BlockNumber:     uint64(storageState.BlockNumber),
-	}, nil
-}
-
 func (wr *ParachainWriter) GetLastFinalizedHeaderState() (state.FinalizedHeader, error) {
 	finalizedState, err := wr.GetFinalizedStateByStorageKey("LatestFinalizedBlockRoot")
 	if err != nil {
@@ -385,3 +360,91 @@ func (wr *ParachainWriter) getNumberFromParachain(pallet, storage string) (uint6
 
 	return uint64(number), nil
 }
+
+func (wr *ParachainWriter) GetCompactExecutionHeaderStateByBlockHash(blockHash types.H256) (state.CompactExecutionHeaderState, error) {
+	var headerState state.CompactExecutionHeaderState
+	key, err := types.CreateStorageKey(wr.conn.Metadata(), "EthereumBeaconClient", "ExecutionHeaders", blockHash[:], nil)
+	if err != nil {
+		return headerState, fmt.Errorf("create storage key for ExecutionHeaders: %w", err)
+	}
+
+	var compactExecutionHeader scale.CompactExecutionHeader
+	_, err = wr.conn.API().RPC.State.GetStorageLatest(key, &compactExecutionHeader)
+	if err != nil {
+		return headerState, fmt.Errorf("get storage for ExecutionHeaders (err): %w", err)
+	}
+	headerState = state.CompactExecutionHeaderState{
+		ParentHash:   common.Hash(compactExecutionHeader.ParentHash),
+		BlockNumber:  uint64(compactExecutionHeader.BlockNumber.Int64()),
+		StateRoot:    common.Hash(compactExecutionHeader.StateRoot),
+		ReceiptsRoot: common.Hash(compactExecutionHeader.ReceiptsRoot),
+	}
+	return headerState, nil
+}
+
+func (wr *ParachainWriter) GetLastFinalizedStateIndex() (types.U32, error) {
+	var index types.U32
+	key, err := types.CreateStorageKey(wr.conn.Metadata(), "EthereumBeaconClient", "FinalizedBeaconStateIndex", nil, nil)
+	if err != nil {
+		return index, fmt.Errorf("create storage key for FinalizedBeaconStateIndex: %w", err)
+	}
+
+	_, err = wr.conn.API().RPC.State.GetStorageLatest(key, &index)
+	if err != nil {
+		return index, fmt.Errorf("get storage for FinalizedBeaconStateIndex (err): %w", err)
+	}
+
+	return index, nil
+}
+
+func (wr *ParachainWriter) GetFinalizedBeaconRootByIndex(index uint32) (types.H256, error) {
+	var beaconRoot types.H256
+	encodedIndex, err := types.EncodeToBytes(types.NewU32(index))
+	if err != nil {
+		return beaconRoot, fmt.Errorf("get finalized beacon root encode index error: %w", err)
+	}
+	key, err := types.CreateStorageKey(wr.conn.Metadata(), "EthereumBeaconClient", "FinalizedBeaconStateMapping", encodedIndex, nil)
+	if err != nil {
+		return beaconRoot, fmt.Errorf("create storage key for FinalizedBeaconStateMapping: %w", err)
+	}
+
+	_, err = wr.conn.API().RPC.State.GetStorageLatest(key, &beaconRoot)
+	if err != nil {
+		return beaconRoot, fmt.Errorf("get storage for FinalizedBeaconStateMapping (err): %w", err)
+	}
+
+	return beaconRoot, nil
+}
+
+func (wr *ParachainWriter) FindCheckPointBackward(slot uint64) (state.FinalizedHeader, error) {
+	var beaconState state.FinalizedHeader
+	lastIndex, err := wr.GetLastFinalizedStateIndex()
+	if err != nil {
+		return beaconState, fmt.Errorf("GetLastFinalizedStateIndex error: %w", err)
+	}
+	startIndex := uint32(lastIndex)
+	endIndex := uint32(0)
+	if lastIndex > 256 {
+		endIndex = endIndex - 256
+	}
+	for index := startIndex; index >= endIndex; index-- {
+		beaconRoot, err := wr.GetFinalizedBeaconRootByIndex(index)
+		if err != nil {
+			return beaconState, fmt.Errorf("GetFinalizedBeaconRootByIndex %d, error: %w", index, err)
+		}
+		beaconState, err = wr.GetFinalizedHeaderStateByBlockRoot(beaconRoot)
+		if err != nil {
+			return beaconState, fmt.Errorf("GetFinalizedHeaderStateByBlockRoot %s, error: %w", beaconRoot.Hex(), err)
+		}
+		if beaconState.BeaconSlot < slot {
+			break
+		}
+		if beaconState.BeaconSlot > slot && beaconState.BeaconSlot < slot+8192 {
+			break
+		}
+	}
+	if beaconState.BeaconSlot > slot && beaconState.BeaconSlot < slot+8192 {
+		return beaconState, nil
+	}
+	return beaconState, fmt.Errorf("Can't find checkpoint on chain for slot %d", slot)
+}

relayer/cmd/generate_beacon_data.go

@@ -53,7 +53,7 @@ func generateBeaconCheckpointCmd() *cobra.Command {
 	}
 
 	cmd.Flags().String("url", "http://127.0.0.1:9596", "Beacon URL")
-	cmd.Flags().Bool("export_json", false, "Export Json")
+	cmd.Flags().Bool("export-json", false, "Export Json")
 
 	return cmd
 }
@@ -115,9 +115,9 @@ func generateBeaconCheckpoint(cmd *cobra.Command, _ []string) error {
 		if err != nil {
 			return fmt.Errorf("get initial sync: %w", err)
 		}
-		exportJson, err := cmd.Flags().GetBool("export_json")
+		exportJson, err := cmd.Flags().GetBool("export-json")
 		if err != nil {
-			return err
+			return fmt.Errorf("get export-json flag: %w", err)
 		}
 		if exportJson {
 			initialSync := checkPointScale.ToJSON()
@@ -282,7 +282,7 @@ func generateBeaconTestFixture(cmd *cobra.Command, _ []string) error {
 			BlockRootsTree:     finalizedUpdateAfterMessage.BlockRootsTree,
 			Slot:               uint64(finalizedUpdateAfterMessage.Payload.FinalizedHeader.Slot),
 		}
-		headerUpdateScale, err := s.GetNextHeaderUpdateBySlotWithCheckpoint(beaconBlockSlot, &checkPoint)
+		headerUpdateScale, err := s.GetHeaderUpdateBySlotWithCheckpoint(beaconBlockSlot, &checkPoint)
 		if err != nil {
 			return fmt.Errorf("get header update: %w", err)
 		}

relayer/relays/beacon/cache/cache.go

@@ -67,14 +67,6 @@ func (b *BeaconCache) SetLastSyncedFinalizedState(finalizedHeaderRoot common.Has
 	}
 }
 
-func (b *BeaconCache) SetLastSyncedExecutionSlot(slot uint64) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if slot > b.LastSyncedExecutionSlot {
-		b.LastSyncedExecutionSlot = slot
-	}
-}
-
 func (b *BeaconCache) SetInitialCheckpointSlot(slot uint64) {
 	b.mu.Lock()
 	defer b.mu.Unlock()