Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add blob sidecar inclusion proof #3531

Merged
merged 31 commits into from
Nov 2, 2023
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
530efa8
Add blob sidecar inclusion proof
dapplion Oct 27, 2023
e8bccec
remove verify_blob_sidecar_signature
dapplion Oct 27, 2023
29bbdf4
compute KZG_COMMITMENT_INCLUSION_PROOF_DEPTH
dapplion Oct 27, 2023
a4a29a1
List typo
dapplion Oct 27, 2023
3dbe54e
doctoc
dapplion Oct 27, 2023
c2a64a1
pass lint
dapplion Oct 27, 2023
caa79a5
build tree
dapplion Oct 27, 2023
8712451
Update specs/deneb/p2p-interface.md
dapplion Oct 27, 2023
f2649f6
fix unit tests
dapplion Oct 27, 2023
83e5930
doctoc
dapplion Oct 27, 2023
0bf9e75
review PR
dapplion Oct 27, 2023
26516ec
Move `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` to preset and cast `int()`
hwwhww Oct 27, 2023
1657d16
Add `BLOB_KZG_COMMITMENTS_GINDEX` to "Constant". Use pyspec parser tr…
hwwhww Oct 28, 2023
de3b6a2
Fix toc
hwwhww Oct 28, 2023
b7e0b88
Fix test
hwwhww Oct 28, 2023
b018fbc
Remove `BLOB_KZG_COMMITMENTS_GINDEX` from the preset files
hwwhww Oct 28, 2023
ae6a9eb
Fix lint
hwwhww Oct 28, 2023
0e4737e
Add a general `compute_merkle_proof` helper to replace container-spec…
hwwhww Oct 30, 2023
c680212
drop is_valid_merkle_path
dapplion Oct 30, 2023
126e807
Update specs/deneb/p2p-interface.md
dapplion Oct 30, 2023
b803f1c
Update specs/deneb/p2p-interface.md
dapplion Oct 30, 2023
d323f05
drop sidecar alias
dapplion Oct 30, 2023
a124414
Enhance `blob_sidecar_inclusion_proof` tests
hwwhww Oct 30, 2023
51343f5
Fix typing and delete the `signed_sidecar`
hwwhww Oct 30, 2023
1bac25a
Add Merkle proof test
hwwhww Oct 30, 2023
19883ec
Add verify_blob_kzg_proof condition
dapplion Oct 31, 2023
7f63f00
Merge branch 'dev' into blob-p2p-proof
dapplion Oct 31, 2023
4a609ce
rename to kzg_commitment_inclusion_proof
dapplion Nov 1, 2023
71106f1
Remove `BLOB_KZG_COMMITMENTS_GINDEX`
hwwhww Nov 2, 2023
3492c0a
minor refactoring
hwwhww Nov 2, 2023
7118c30
a few cleanups to sidecar gossip conditions
djrtwo Nov 2, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions presets/mainnet/deneb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
# `uint64(6)`
MAX_BLOBS_PER_BLOCK: 6
# `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17
2 changes: 2 additions & 0 deletions presets/minimal/deneb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
# `uint64(6)`
MAX_BLOBS_PER_BLOCK: 6
# [customized] `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9
10 changes: 7 additions & 3 deletions pysetup/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
if k in [
"ceillog2",
"floorlog2",
"compute_merkle_proof_for_block_body",
"compute_merkle_proof_for_state",
"compute_merkle_proof",
]:
del spec_object.functions[k]

Expand Down Expand Up @@ -111,8 +110,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
return out

# Merge all constant objects
hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
# Concatenate all strings
imports = reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name) ).strip("\n"), builders, "")
preparations = reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations() ).strip("\n"), builders, "")
Expand All @@ -126,6 +126,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
spec_strs = [
imports,
preparations,
Expand All @@ -147,6 +148,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
# Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
# as same as the spec definition.
ssz_dep_constants_verification,
func_dep_presets_verification,
]
return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) + "\n"

Expand Down Expand Up @@ -223,6 +225,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars)
config_vars = combine_dicts(spec0.config_vars, spec1.config_vars)
ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants)
func_dep_presets = combine_dicts(spec0.func_dep_presets, spec1.func_dep_presets)
ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types)
dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses)
return SpecObject(
Expand All @@ -233,6 +236,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
preset_vars=preset_vars,
config_vars=config_vars,
ssz_dep_constants=ssz_dep_constants,
func_dep_presets=func_dep_presets,
ssz_objects=ssz_objects,
dataclasses=dataclasses,
)
Expand Down
8 changes: 4 additions & 4 deletions pysetup/spec_builders/altair.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,16 @@ def preparations(cls):
@classmethod
def sundry_functions(cls) -> str:
return '''
def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex:
def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex:
ssz_path = Path(ssz_class)
for item in path:
ssz_path = ssz_path / item
return GeneralizedIndex(ssz_path.gindex())


def compute_merkle_proof_for_state(state: BeaconState,
index: GeneralizedIndex) -> Sequence[Bytes32]:
return build_proof(state.get_backing(), index)'''
def compute_merkle_proof(object: SSZObject,
index: GeneralizedIndex) -> Sequence[Bytes32]:
return build_proof(object.get_backing(), index)'''


@classmethod
Expand Down
4 changes: 4 additions & 0 deletions pysetup/spec_builders/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: #
"""
return {}

@classmethod
def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
return {}

@classmethod
def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
return functions
9 changes: 0 additions & 9 deletions pysetup/spec_builders/capella.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,6 @@ def imports(cls, preset_name: str):
from eth2spec.bellatrix import {preset_name} as bellatrix
'''


@classmethod
def sundry_functions(cls) -> str:
return '''
def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
index: GeneralizedIndex) -> Sequence[Bytes32]:
return build_proof(body.get_backing(), index)'''


@classmethod
def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
return {
Expand Down
19 changes: 17 additions & 2 deletions pysetup/spec_builders/deneb.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import Dict
from .base import BaseSpecBuilder
from ..constants import DENEB

Expand All @@ -23,7 +24,8 @@ def sundry_functions(cls) -> str:
return '''
def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]:
# pylint: disable=unused-argument
return [], []'''
return [], []
'''

@classmethod
def execution_engine_cls(cls) -> str:
Expand Down Expand Up @@ -63,9 +65,22 @@ def verify_and_notify_new_payload(self: ExecutionEngine,


@classmethod
def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
return {
'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
}

@classmethod
def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
return {
'BLOB_KZG_COMMITMENTS_GINDEX': 'GeneralizedIndex(27)',
}

@classmethod
def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
return {
'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value,
mkalinin marked this conversation as resolved.
Show resolved Hide resolved
}
1 change: 1 addition & 0 deletions pysetup/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class SpecObject(NamedTuple):
preset_vars: Dict[str, VariableDefinition]
config_vars: Dict[str, VariableDefinition]
ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects
func_dep_presets: Dict[str, str] # the constants that depend on functions
ssz_objects: Dict[str, str]
dataclasses: Dict[str, str]

Expand Down
15 changes: 15 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
preset_vars: Dict[str, VariableDefinition] = {}
config_vars: Dict[str, VariableDefinition] = {}
ssz_dep_constants: Dict[str, str] = {}
func_dep_presets: Dict[str, str] = {}
ssz_objects: Dict[str, str] = {}
dataclasses: Dict[str, str] = {}
custom_types: Dict[str, str] = {}
Expand Down Expand Up @@ -214,6 +215,16 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr

value_cell = cells[1]
value = value_cell.children[0].children

description = None
if len(cells) >= 3:
description_cell = cells[2]
if len(description_cell.children) > 0:
description = description_cell.children[0].children
if isinstance(description, list):
# marko parses `**X**` as a list containing a X
description = description[0].children

if isinstance(value, list):
# marko parses `**X**` as a list containing a X
value = value[0].children
Expand All @@ -228,6 +239,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
ssz_dep_constants[name] = value
continue

if description is not None and description.startswith("<!-- predefined -->"):
func_dep_presets[name] = value

value_def = _parse_value(name, value)
if name in preset:
preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None)
Expand Down Expand Up @@ -256,6 +270,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
preset_vars=preset_vars,
config_vars=config_vars,
ssz_dep_constants=ssz_dep_constants,
func_dep_presets=func_dep_presets,
ssz_objects=ssz_objects,
dataclasses=dataclasses,
)
Expand Down
18 changes: 9 additions & 9 deletions specs/altair/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

- [Introduction](#introduction)
- [Helper functions](#helper-functions)
- [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state)
- [`compute_merkle_proof`](#compute_merkle_proof)
- [`block_to_light_client_header`](#block_to_light_client_header)
- [Deriving light client data](#deriving-light-client-data)
- [`create_light_client_bootstrap`](#create_light_client_bootstrap)
Expand All @@ -27,11 +27,13 @@ This document provides helper functions to enable full nodes to serve light clie

## Helper functions

### `compute_merkle_proof_for_state`
### `compute_merkle_proof`

This function return the Merkle proof of the given SSZ object `object` at generalized index `index`.

```python
def compute_merkle_proof_for_state(state: BeaconState,
index: GeneralizedIndex) -> Sequence[Bytes32]:
def compute_merkle_proof(object: SSZObject,
index: GeneralizedIndex) -> Sequence[Bytes32]:
...
```

Expand Down Expand Up @@ -73,7 +75,7 @@ def create_light_client_bootstrap(state: BeaconState,
return LightClientBootstrap(
header=block_to_light_client_header(block),
current_sync_committee=state.current_sync_committee,
current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX),
current_sync_committee_branch=compute_merkle_proof(state, CURRENT_SYNC_COMMITTEE_INDEX),
)
```

Expand Down Expand Up @@ -120,8 +122,7 @@ def create_light_client_update(state: BeaconState,
# `next_sync_committee` is only useful if the message is signed by the current sync committee
if update_attested_period == update_signature_period:
update.next_sync_committee = attested_state.next_sync_committee
update.next_sync_committee_branch = compute_merkle_proof_for_state(
attested_state, NEXT_SYNC_COMMITTEE_INDEX)
update.next_sync_committee_branch = compute_merkle_proof(attested_state, NEXT_SYNC_COMMITTEE_INDEX)

# Indicate finality whenever possible
if finalized_block is not None:
Expand All @@ -130,8 +131,7 @@ def create_light_client_update(state: BeaconState,
assert hash_tree_root(update.finalized_header.beacon) == attested_state.finalized_checkpoint.root
else:
assert attested_state.finalized_checkpoint.root == Bytes32()
update.finality_branch = compute_merkle_proof_for_state(
attested_state, FINALIZED_ROOT_INDEX)
update.finality_branch = compute_merkle_proof(attested_state, FINALIZED_ROOT_INDEX)

update.sync_aggregate = block.message.body.sync_aggregate
update.signature_slot = block.message.slot
Expand Down
11 changes: 1 addition & 10 deletions specs/capella/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

- [Introduction](#introduction)
- [Helper functions](#helper-functions)
- [`compute_merkle_proof_for_block_body`](#compute_merkle_proof_for_block_body)
- [Modified `block_to_light_client_header`](#modified-block_to_light_client_header)

<!-- END doctoc generated TOC please keep comment here to allow auto update -->
Expand All @@ -22,14 +21,6 @@ This upgrade adds information about the execution payload to light client data a

## Helper functions

### `compute_merkle_proof_for_block_body`

```python
def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
index: GeneralizedIndex) -> Sequence[Bytes32]:
...
```

### Modified `block_to_light_client_header`

```python
Expand All @@ -55,7 +46,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
transactions_root=hash_tree_root(payload.transactions),
withdrawals_root=hash_tree_root(payload.withdrawals),
)
execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
else:
# Note that during fork transitions, `finalized_header` may still point to earlier forks.
# While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),
Expand Down
2 changes: 1 addition & 1 deletion specs/deneb/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
execution_header.blob_gas_used = payload.blob_gas_used
execution_header.excess_blob_gas = payload.excess_blob_gas

execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
else:
# Note that during fork transitions, `finalized_header` may still point to earlier forks.
# While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),
Expand Down
Loading