diff --git a/setup.py b/setup.py index f25a3eea78..42ddc69021 100644 --- a/setup.py +++ b/setup.py @@ -482,6 +482,7 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariab def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: constants = { 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'CURRENT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', } return {**super().hardcoded_ssz_dep_constants(), **constants} @@ -1094,7 +1095,7 @@ def run(self): extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], "lint": ["flake8==3.7.7", "mypy==0.812", "pylint==2.12.2"], - "generator": ["python-snappy==0.5.4"], + "generator": ["python-snappy==0.5.4", "filelock"], }, install_requires=[ "eth-utils>=1.3.0,<2", diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index 4234143573..8d420eda56 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -13,20 +13,24 @@ - [Preset](#preset) - [Misc](#misc) - [Containers](#containers) + - [`LightClientBootstrap`](#lightclientbootstrap) - [`LightClientUpdate`](#lightclientupdate) - [`LightClientStore`](#lightclientstore) - [Helper functions](#helper-functions) - [`is_sync_committee_update`](#is_sync_committee_update) - [`is_finality_update`](#is_finality_update) - [`is_better_update`](#is_better_update) + - [`is_next_sync_committee_known`](#is_next_sync_committee_known) - [`get_safety_threshold`](#get_safety_threshold) - [`get_subtree_index`](#get_subtree_index) - [`compute_sync_committee_period_at_slot`](#compute_sync_committee_period_at_slot) +- [Light client initialization](#light-client-initialization) + - [`initialize_light_client_store`](#initialize_light_client_store) - [Light client state updates](#light-client-state-updates) - - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) - - [`validate_light_client_update`](#validate_light_client_update) - - [`apply_light_client_update`](#apply_light_client_update) - - [`process_light_client_update`](#process_light_client_update) + - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store) + - [`validate_light_client_update`](#validate_light_client_update) + - [`apply_light_client_update`](#apply_light_client_update) + - [`process_light_client_update`](#process_light_client_update) @@ -35,7 +39,7 @@ The beacon chain is designed to be light client friendly for constrained environments to access Ethereum with reasonable safety and liveness. -Such environments include resource-constrained devices (e.g. phones for trust-minimised wallets) +Such environments include resource-constrained devices (e.g. phones for trust-minimized wallets) and metered VMs (e.g. blockchain VMs for cross-chain bridges). This document suggests a minimal light client design for the beacon chain that @@ -46,6 +50,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | | `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` (= 105) | +| `CURRENT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'current_sync_committee')` (= 54) | | `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` (= 55) | ## Preset @@ -59,6 +64,17 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. ## Containers +### `LightClientBootstrap` + +```python +class LightClientBootstrap(Container): + # The requested beacon block header + header: BeaconBlockHeader + # Current sync committee corresponding to `header` + current_sync_committee: SyncCommittee + current_sync_committee_branch: Vector[Bytes32, floorlog2(CURRENT_SYNC_COMMITTEE_INDEX)] +``` + ### `LightClientUpdate` ```python @@ -127,6 +143,18 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat if not new_has_supermajority and new_num_active_participants != old_num_active_participants: return new_num_active_participants > old_num_active_participants + # Compare presence of relevant sync committee + new_has_relevant_sync_committee = is_sync_committee_update(new_update) and ( + compute_sync_committee_period_at_slot(new_update.attested_header.slot) + == compute_sync_committee_period_at_slot(new_update.signature_slot) + ) + old_has_relevant_sync_committee = is_sync_committee_update(old_update) and ( + compute_sync_committee_period_at_slot(old_update.attested_header.slot) + == compute_sync_committee_period_at_slot(old_update.signature_slot) + ) + if new_has_relevant_sync_committee != old_has_relevant_sync_committee: + return new_has_relevant_sync_committee + # Compare indication of any finality new_has_finality = is_finality_update(new_update) old_has_finality = is_finality_update(old_update) @@ -156,6 +184,13 @@ def is_better_update(new_update: LightClientUpdate, old_update: LightClientUpdat return new_update.signature_slot < old_update.signature_slot ``` +### `is_next_sync_committee_known` + +```python +def is_next_sync_committee_known(store: LightClientStore) -> bool: + return store.next_sync_committee != SyncCommittee() +``` + ### `get_safety_threshold` ```python @@ -180,11 +215,41 @@ def compute_sync_committee_period_at_slot(slot: Slot) -> uint64: return compute_sync_committee_period(compute_epoch_at_slot(slot)) ``` +## Light client initialization + +A light client maintains its state in a `store` object of type `LightClientStore`. `initialize_light_client_store` initializes a new `store` with a received `LightClientBootstrap` derived from a given `trusted_block_root`. + +### `initialize_light_client_store` + +```python +def initialize_light_client_store(trusted_block_root: Root, + bootstrap: LightClientBootstrap) -> LightClientStore: + assert hash_tree_root(bootstrap.header) == trusted_block_root + + assert is_valid_merkle_branch( + leaf=hash_tree_root(bootstrap.current_sync_committee), + branch=bootstrap.current_sync_committee_branch, + depth=floorlog2(CURRENT_SYNC_COMMITTEE_INDEX), + index=get_subtree_index(CURRENT_SYNC_COMMITTEE_INDEX), + root=bootstrap.header.state_root, + ) + + return LightClientStore( + finalized_header=bootstrap.header, + current_sync_committee=bootstrap.current_sync_committee, + next_sync_committee=SyncCommittee(), + best_valid_update=None, + optimistic_header=bootstrap.header, + previous_max_active_participants=0, + current_max_active_participants=0, + ) +``` + ## Light client state updates -A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. +A light client receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. -#### `process_slot_for_light_client_store` +### `process_slot_for_light_client_store` ```python def process_slot_for_light_client_store(store: LightClientStore, current_slot: Slot) -> None: @@ -205,7 +270,7 @@ def process_slot_for_light_client_store(store: LightClientStore, current_slot: S store.best_valid_update = None ``` -#### `validate_light_client_update` +### `validate_light_client_update` ```python def validate_light_client_update(store: LightClientStore, @@ -220,11 +285,20 @@ def validate_light_client_update(store: LightClientStore, assert current_slot >= update.signature_slot > update.attested_header.slot >= update.finalized_header.slot store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) update_signature_period = compute_sync_committee_period_at_slot(update.signature_slot) - assert update_signature_period in (store_period, store_period + 1) + if is_next_sync_committee_known(store): + assert update_signature_period in (store_period, store_period + 1) + else: + assert update_signature_period == store_period # Verify update is relevant update_attested_period = compute_sync_committee_period_at_slot(update.attested_header.slot) - assert update.attested_header.slot > store.finalized_header.slot + update_has_next_sync_committee = not is_next_sync_committee_known(store) and ( + is_sync_committee_update(update) and update_attested_period == store_period + ) + assert ( + update.attested_header.slot > store.finalized_header.slot + or update_has_next_sync_committee + ) # Verify that the `finality_branch`, if present, confirms `finalized_header` # to match the finalized checkpoint root saved in the state of `attested_header`. @@ -248,10 +322,9 @@ def validate_light_client_update(store: LightClientStore, # Verify that the `next_sync_committee`, if present, actually is the next sync committee saved in the # state of the `attested_header` if not is_sync_committee_update(update): - assert update_attested_period == store_period assert update.next_sync_committee == SyncCommittee() else: - if update_attested_period == store_period: + if update_attested_period == store_period and is_next_sync_committee_known(store): assert update.next_sync_committee == store.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), @@ -276,21 +349,25 @@ def validate_light_client_update(store: LightClientStore, assert bls.FastAggregateVerify(participant_pubkeys, signing_root, sync_aggregate.sync_committee_signature) ``` -#### `apply_light_client_update` +### `apply_light_client_update` ```python def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None: store_period = compute_sync_committee_period_at_slot(store.finalized_header.slot) update_finalized_period = compute_sync_committee_period_at_slot(update.finalized_header.slot) - if update_finalized_period == store_period + 1: + if not is_next_sync_committee_known(store): + assert update_finalized_period == store_period + store.next_sync_committee = update.next_sync_committee + elif update_finalized_period == store_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee - store.finalized_header = update.finalized_header - if store.finalized_header.slot > store.optimistic_header.slot: - store.optimistic_header = store.finalized_header + if update.finalized_header.slot > store.finalized_header.slot: + store.finalized_header = update.finalized_header + if store.finalized_header.slot > store.optimistic_header.slot: + store.optimistic_header = store.finalized_header ``` -#### `process_light_client_update` +### `process_light_client_update` ```python def process_light_client_update(store: LightClientStore, @@ -322,9 +399,19 @@ def process_light_client_update(store: LightClientStore, store.optimistic_header = update.attested_header # Update finalized header + update_has_finalized_next_sync_committee = ( + not is_next_sync_committee_known(store) + and is_sync_committee_update(update) and is_finality_update(update) and ( + compute_sync_committee_period_at_slot(update.finalized_header.slot) + == compute_sync_committee_period_at_slot(update.attested_header.slot) + ) + ) if ( sum(sync_committee_bits) * 3 >= len(sync_committee_bits) * 2 - and update.finalized_header.slot > store.finalized_header.slot + and ( + update.finalized_header.slot > store.finalized_header.slot + or update_has_finalized_next_sync_committee + ) ): # Normal update through 2/3 threshold apply_light_client_update(store, update) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 6850af188d..3e5d71d148 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -3,7 +3,9 @@ import shutil import argparse from pathlib import Path +from filelock import FileLock import sys +import json from typing import Iterable, AnyStr, Any, Callable import traceback @@ -111,6 +113,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): collected_test_count = 0 generated_test_count = 0 skipped_test_count = 0 + test_identifiers = [] + provider_start = time.time() for tprov in test_providers: if not collect_only: @@ -123,12 +127,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): / Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.suite_name) / Path(test_case.case_name) ) - incomplete_tag_file = case_dir / "INCOMPLETE" - collected_test_count += 1 - if collect_only: - print(f"Collected test at: {case_dir}") - continue + print(f"Collected test at: {case_dir}") + + incomplete_tag_file = case_dir / "INCOMPLETE" if case_dir.exists(): if not args.force and not incomplete_tag_file.exists(): @@ -198,6 +200,15 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): shutil.rmtree(case_dir) else: generated_test_count += 1 + test_identifier = "::".join([ + test_case.preset_name, + test_case.fork_name, + test_case.runner_name, + test_case.handler_name, + test_case.suite_name, + test_case.case_name + ]) + test_identifiers.append(test_identifier) # Only remove `INCOMPLETE` tag file os.remove(incomplete_tag_file) test_end = time.time() @@ -216,6 +227,28 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): if span > TIME_THRESHOLD_TO_PRINT: summary_message += f" in {span} seconds" print(summary_message) + diagnostics = { + "collected_test_count": collected_test_count, + "generated_test_count": generated_test_count, + "skipped_test_count": skipped_test_count, + "test_identifiers": test_identifiers, + "durations": [f"{span} seconds"], + } + diagnostics_path = Path(os.path.join(output_dir, "diagnostics.json")) + diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics.json.lock")) + with diagnostics_lock: + diagnostics_path.touch(exist_ok=True) + if os.path.getsize(diagnostics_path) == 0: + with open(diagnostics_path, "w+") as f: + json.dump(diagnostics, f) + else: + with open(diagnostics_path, "r+") as f: + existing_diagnostics = json.load(f) + for k, v in diagnostics.items(): + existing_diagnostics[k] += v + with open(diagnostics_path, "w+") as f: + json.dump(existing_diagnostics, f) + print(f"wrote diagnostics to {diagnostics_path}") def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): diff --git a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py index 31cdd13bb1..d90feb1466 100644 --- a/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py @@ -5,6 +5,25 @@ from eth2spec.test.helpers.merkle import build_proof +@with_altair_and_later +@spec_state_test +def test_current_sync_committee_merkle_proof(spec, state): + yield "state", state + current_sync_committee_branch = build_proof(state.get_backing(), spec.CURRENT_SYNC_COMMITTEE_INDEX) + yield "proof", { + "leaf": "0x" + state.current_sync_committee.hash_tree_root().hex(), + "leaf_index": spec.CURRENT_SYNC_COMMITTEE_INDEX, + "branch": ['0x' + root.hex() for root in current_sync_committee_branch] + } + assert spec.is_valid_merkle_branch( + leaf=state.current_sync_committee.hash_tree_root(), + branch=current_sync_committee_branch, + depth=spec.floorlog2(spec.CURRENT_SYNC_COMMITTEE_INDEX), + index=spec.get_subtree_index(spec.CURRENT_SYNC_COMMITTEE_INDEX), + root=state.hash_tree_root(), + ) + + @with_altair_and_later @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py index 453d7235ba..4b73a15ba8 100644 --- a/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py +++ b/tests/core/pyspec/eth2spec/test/altair/sync_protocol/test_update_ranking.py @@ -85,10 +85,8 @@ def test_update_ranking(spec, state): # Create updates (in descending order of quality) updates = [ # Updates with sync committee finality - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=1.0), - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), @@ -97,34 +95,66 @@ def test_update_ranking(spec, state): create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.8), # Updates without indication of any finality - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=1.0), - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.8), + # Updates with sync committee finality but no `next_sync_committee` + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + + # Updates without sync committee finality and also no `next_sync_committee` + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=1.0), + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=0.8), + + # Updates without indication of any finality nor `next_sync_committee` + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=1.0), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=0.8), + # Updates with low sync committee participation - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.4), - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.4), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=0.4), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=0.4), # Updates with very low sync committee participation - create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), create_update(spec, fin, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), create_update(spec, lat, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), create_update(spec, att, with_next_sync_committee=1, with_finality=1, participation_rate=0.2), - create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), create_update(spec, att, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), create_update(spec, fin, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), create_update(spec, lat, with_next_sync_committee=1, with_finality=0, participation_rate=0.2), + create_update(spec, sig, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, fin, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, lat, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, att, with_next_sync_committee=0, with_finality=1, participation_rate=0.2), + create_update(spec, sig, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, att, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, fin, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), + create_update(spec, lat, with_next_sync_committee=0, with_finality=0, participation_rate=0.2), ] yield "updates", updates