update Deneb for blob sidecar inclusion proofs (#5565)

`BlobSidecar` is no longer signed, instead use Merkle proof to link
blobs with block.

- ethereum/consensus-specs#3531

Associated beacon-API / builder-specs still TBD; minimal changes done
to compile in similar style to previous spec, but not standardized yet.

- ethereum/beacon-APIs#369
- ethereum/builder-specs#90

AllTests-mainnet.md

@@ -358,15 +358,14 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 ```diff
 + Aggregate and proof signatures                                                             OK
 + Attestation signatures                                                                     OK
-+ Blob sidecar signatures                                                                    OK
 + Deposit signatures                                                                         OK
 + Slot signatures                                                                            OK
 + Sync committee message signatures                                                          OK
 + Sync committee selection proof signatures                                                  OK
 + Sync committee signed contribution and proof signatures                                    OK
 + Voluntary exit signatures                                                                  OK
 ```
-OK: 9/9 Fail: 0/9 Skip: 0/9
+OK: 8/8 Fail: 0/8 Skip: 0/8
 ## Network metadata
 ```diff
 + goerli                                                                                     OK
@@ -715,4 +714,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 404/409 Fail: 0/409 Skip: 5/409
+OK: 403/408 Fail: 0/408 Skip: 5/408

ConsensusSpecPreset-mainnet.md

@@ -2258,7 +2258,6 @@ OK: 34/34 Fail: 0/34 Skip: 0/34
 +   Testing    SignedBLSToExecutionChange                                                    OK
 +   Testing    SignedBeaconBlock                                                             OK
 +   Testing    SignedBeaconBlockHeader                                                       OK
-+   Testing    SignedBlobSidecar                                                             OK
 +   Testing    SignedContributionAndProof                                                    OK
 +   Testing    SignedVoluntaryExit                                                           OK
 +   Testing    SigningData                                                                   OK
@@ -2271,7 +2270,7 @@ OK: 34/34 Fail: 0/34 Skip: 0/34
 +   Testing    VoluntaryExit                                                                 OK
 +   Testing    Withdrawal                                                                    OK
 ```
-OK: 49/49 Fail: 0/49 Skip: 0/49
+OK: 48/48 Fail: 0/48 Skip: 0/48
 ## EF - Deneb - Sanity - Blocks  [Preset: mainnet]
 ```diff
 + [Invalid] EF - Deneb - Sanity - Blocks - invalid_all_zeroed_sig [Preset: mainnet]          OK
@@ -2411,6 +2410,11 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + Light client - Single merkle proof - mainnet/deneb/light_client/single_merkle_proof/Beacon OK
 ```
 OK: 14/14 Fail: 0/14 Skip: 0/14
+## EF - Merkle proof [Preset: mainnet]
+```diff
++ Merkle proof - Single merkle proof - mainnet/deneb/merkle_proof/single_merkle_proof/Beacon OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: mainnet]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: mainnet]                 OK
@@ -2825,6 +2829,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40
 + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head  OK
 + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_we OK
 + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK
+  ForkChoice - mainnet/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root  Skip
+  ForkChoice - mainnet/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_roo Skip
 + ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/basic                        OK
 + ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root     OK
   ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/on_block_future_block        Skip
@@ -2842,6 +2848,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40
 + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/proposer_boost_correct_he OK
 + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier OK
 + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_atte OK
+  ForkChoice - mainnet/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_ro Skip
+  ForkChoice - mainnet/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ Skip
 + ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/basic                     OK
 + ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root  OK
   ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/on_block_future_block     Skip
@@ -2852,6 +2860,7 @@ OK: 40/40 Fail: 0/40 Skip: 0/40
   ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/block_lookup_failed Skip
   ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/too_early_for_merge Skip
   ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/too_late_for_merge  Skip
+  ForkChoice - mainnet/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ Skip
 + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_attestations_is_grea OK
 + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_boost_ OK
 + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest OK
@@ -2863,12 +2872,15 @@ OK: 40/40 Fail: 0/40 Skip: 0/40
 + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK
 + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_w OK
 + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attest OK
+  ForkChoice - mainnet/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip
+  ForkChoice - mainnet/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ro Skip
 + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/basic                       OK
 + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root    OK
   ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/on_block_future_block       Skip
 + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost              OK
 + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost_is_first_blo OK
 + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_sl OK
+  ForkChoice - mainnet/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh Skip
 + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_attestations_is_greate OK
 + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_boost_no OK
 + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest_a OK
@@ -2880,6 +2892,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40
 + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head   OK
 + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_wei OK
 + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attestat OK
+  ForkChoice - mainnet/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root   Skip
+  ForkChoice - mainnet/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_root Skip
 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/basic                         OK
 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/invalid_data_unavailable      OK
 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/invalid_incorrect_proof       OK
@@ -2891,8 +2905,9 @@ OK: 40/40 Fail: 0/40 Skip: 0/40
 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/proposer_boost_is_first_block OK
 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_slot OK
 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/simple_blob_data              OK
+  ForkChoice - mainnet/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou Skip
 ```
-OK: 69/77 Fail: 0/77 Skip: 8/77
+OK: 69/88 Fail: 0/88 Skip: 19/88
 ## Sync
 ```diff
 + Sync - mainnet/bellatrix/sync/optimistic/pyspec_tests/from_syncing_to_invalid              OK
@@ -2902,4 +2917,4 @@ OK: 69/77 Fail: 0/77 Skip: 8/77
 OK: 3/3 Fail: 0/3 Skip: 0/3
 
 ---TOTAL---
-OK: 2336/2344 Fail: 0/2344 Skip: 8/2344
+OK: 2336/2355 Fail: 0/2355 Skip: 19/2355

ConsensusSpecPreset-minimal.md

@@ -2355,7 +2355,6 @@ OK: 34/34 Fail: 0/34 Skip: 0/34
 +   Testing    SignedBLSToExecutionChange                                                    OK
 +   Testing    SignedBeaconBlock                                                             OK
 +   Testing    SignedBeaconBlockHeader                                                       OK
-+   Testing    SignedBlobSidecar                                                             OK
 +   Testing    SignedContributionAndProof                                                    OK
 +   Testing    SignedVoluntaryExit                                                           OK
 +   Testing    SigningData                                                                   OK
@@ -2368,7 +2367,7 @@ OK: 34/34 Fail: 0/34 Skip: 0/34
 +   Testing    VoluntaryExit                                                                 OK
 +   Testing    Withdrawal                                                                    OK
 ```
-OK: 49/49 Fail: 0/49 Skip: 0/49
+OK: 48/48 Fail: 0/48 Skip: 0/48
 ## EF - Deneb - Sanity - Blocks  [Preset: minimal]
 ```diff
 + [Invalid] EF - Deneb - Sanity - Blocks - invalid_all_zeroed_sig [Preset: minimal]          OK
@@ -2551,6 +2550,11 @@ OK: 20/20 Fail: 0/20 Skip: 0/20
 + Light client - Update ranking - minimal/deneb/light_client/update_ranking/pyspec_tests/upd OK
 ```
 OK: 4/4 Fail: 0/4 Skip: 0/4
+## EF - Merkle proof [Preset: minimal]
+```diff
++ Merkle proof - Single merkle proof - minimal/deneb/merkle_proof/single_merkle_proof/Beacon OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: minimal]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: minimal]                 OK
@@ -2977,6 +2981,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK
 + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_epo OK
 + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/voting_source_within_two_epo OK
+  ForkChoice - minimal/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root  Skip
+  ForkChoice - minimal/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_roo Skip
 + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/basic                        OK
 + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/incompatible_justification_u OK
 + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/incompatible_justification_u OK
@@ -3023,6 +3029,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_atte OK
 + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_ OK
 + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/voting_source_within_two_ OK
+  ForkChoice - minimal/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_ro Skip
+  ForkChoice - minimal/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ Skip
 + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/basic                     OK
 + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/incompatible_justificatio OK
 + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/incompatible_justificatio OK
@@ -3058,6 +3066,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delay OK
 + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delay OK
 + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_witho OK
+  ForkChoice - minimal/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ Skip
+  ForkChoice - minimal/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ Skip
 + ForkChoice - minimal/bellatrix/fork_choice/withholding/pyspec_tests/withholding_attack     OK
 + ForkChoice - minimal/bellatrix/fork_choice/withholding/pyspec_tests/withholding_attack_unv OK
 + ForkChoice - minimal/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest OK
@@ -3073,6 +3083,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attest OK
 + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_ep OK
 + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/voting_source_within_two_ep OK
+  ForkChoice - minimal/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root Skip
+  ForkChoice - minimal/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ro Skip
 + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/basic                       OK
 + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/incompatible_justification_ OK
 + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/incompatible_justification_ OK
@@ -3104,6 +3116,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed OK
 + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed OK
 + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_without OK
+  ForkChoice - minimal/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh Skip
+  ForkChoice - minimal/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh Skip
 + ForkChoice - minimal/capella/fork_choice/withholding/pyspec_tests/withholding_attack       OK
 + ForkChoice - minimal/capella/fork_choice/withholding/pyspec_tests/withholding_attack_unvia OK
 + ForkChoice - minimal/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest_a OK
@@ -3119,6 +3133,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attestat OK
 + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_epoc OK
 + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/voting_source_within_two_epoc OK
+  ForkChoice - minimal/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root   Skip
+  ForkChoice - minimal/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_root Skip
 + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/basic                         OK
 + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/incompatible_justification_up OK
 + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/incompatible_justification_up OK
@@ -3155,10 +3171,12 @@ OK: 45/45 Fail: 0/45 Skip: 0/45
 + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed_j OK
 + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed_j OK
 + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_without_e OK
+  ForkChoice - minimal/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou Skip
+  ForkChoice - minimal/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou Skip
 + ForkChoice - minimal/deneb/fork_choice/withholding/pyspec_tests/withholding_attack         OK
 + ForkChoice - minimal/deneb/fork_choice/withholding/pyspec_tests/withholding_attack_unviabl OK
 ```
-OK: 185/193 Fail: 0/193 Skip: 8/193
+OK: 185/207 Fail: 0/207 Skip: 22/207
 ## Sync
 ```diff
 + Sync - minimal/bellatrix/sync/optimistic/pyspec_tests/from_syncing_to_invalid              OK
@@ -3168,4 +3186,4 @@ OK: 185/193 Fail: 0/193 Skip: 8/193
 OK: 3/3 Fail: 0/3 Skip: 0/3
 
 ---TOTAL---
-OK: 2578/2586 Fail: 0/2586 Skip: 8/2586
+OK: 2578/2600 Fail: 0/2600 Skip: 22/2600

beacon_chain/beacon_chain_db.nim

@@ -793,7 +793,8 @@ proc putBlock*(
 proc putBlobSidecar*(
     db: BeaconChainDB,
     value: BlobSidecar) =
-  db.blobs.putSZSSZ(blobkey(value.block_root, value.index), value)
+  let block_root = hash_tree_root(value.signed_block_header.message)
+  db.blobs.putSZSSZ(blobkey(block_root, value.index), value)
 
 proc delBlobSidecar*(
     db: BeaconChainDB,

beacon_chain/consensus_object_pools/blob_quarantine.nim

@@ -7,7 +7,7 @@
 
 {.push raises: [].}
 
-import ../spec/datatypes/deneb
+import ../spec/helpers
 from std/sequtils import mapIt
 from std/strutils import join
 
@@ -37,8 +37,9 @@ func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) =
       oldest_blob_key = k
       break
     quarantine.blobs.del oldest_blob_key
+  let block_root = hash_tree_root(blobSidecar.signed_block_header.message)
   discard quarantine.blobs.hasKeyOrPut(
-    (blobSidecar.block_root, blobSidecar.index), blobSidecar)
+    (block_root, blobSidecar.index), blobSidecar)
 
 func blobIndices*(quarantine: BlobQuarantine, digest: Eth2Digest):
      seq[BlobIndex] =
@@ -54,8 +55,9 @@ func hasBlob*(
     proposer_index: uint64,
     index: BlobIndex): bool =
   for blob_sidecar in quarantine.blobs.values:
-    if blob_sidecar.slot == slot and
-        blob_sidecar.proposer_index == proposer_index and
+    template block_header: untyped = blob_sidecar.signed_block_header.message
+    if block_header.slot == slot and
+        block_header.proposer_index == proposer_index and
         blob_sidecar.index == index:
       return true
   false