From d61f95e138bca0069b69c10111d4ade30c4d29b1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 28 Aug 2024 12:46:53 +0200 Subject: [PATCH 1/2] Fix blob transaction serialization to use RLP EIP-4844 uses RLP not SSZ for blob transactions. --- .../test_process_execution_payload.py | 28 ++--- .../test/deneb/fork_choice/test_on_block.py | 6 +- .../merkle_proof/test_single_merkle_proof.py | 6 +- .../eth2spec/test/deneb/sanity/test_blocks.py | 6 +- .../test_polynomial_commitments.py | 2 +- .../unittests/validator/test_validator.py | 8 +- .../merkle_proof/test_single_merkle_proof.py | 6 +- .../test/eip7594/unittests/das/test_das.py | 2 +- .../test_polynomial_commitments.py | 2 +- .../test/helpers/{sharding.py => blob.py} | 101 +++++++----------- .../test/utils/randomized_block_tests.py | 6 +- 11 files changed, 77 insertions(+), 96 deletions(-) rename tests/core/pyspec/eth2spec/test/helpers/{sharding.py => blob.py} (57%) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index 85175a3b9c..d0c4a6f22a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -10,8 +10,8 @@ expect_assertion_error, with_deneb_and_later ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -74,7 +74,7 @@ def test_incorrect_blob_tx_type(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type execution_payload.transactions = [opaque_tx] @@ -91,7 +91,7 @@ def test_incorrect_transaction_length_1_extra_byte(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx + b'\x12' # incorrect tx length, longer execution_payload.transactions = [opaque_tx] @@ -108,7 +108,7 @@ def test_incorrect_transaction_length_1_byte_short(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx[:-1] # incorrect tx length, shorter execution_payload.transactions = [opaque_tx] @@ -125,7 +125,7 @@ def test_incorrect_transaction_length_empty(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx[0:0] # incorrect tx length, empty execution_payload.transactions = [opaque_tx] @@ -142,7 +142,7 @@ def test_incorrect_transaction_length_32_extra_bytes(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length execution_payload.transactions = [opaque_tx] @@ -159,7 +159,7 @@ def test_no_transactions_with_commitments(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - _, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + _, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) @@ -175,7 +175,7 @@ def test_incorrect_commitment(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment execution_payload.transactions = [opaque_tx] @@ -192,7 +192,7 @@ def test_incorrect_commitments_order(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=Random(1111)) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=2, rng=Random(1111)) blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order execution_payload.transactions = [opaque_tx] @@ -206,7 +206,7 @@ def test_incorrect_commitments_order(spec, state): def test_incorrect_block_hash(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = b'\x12' * 32 # incorrect block hash @@ -223,7 +223,7 @@ def test_zeroed_commitment(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=1, is_valid_blob=False) assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) execution_payload.transactions = [opaque_tx] @@ -240,7 +240,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) @@ -254,7 +254,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): def test_invalid_exceed_max_blobs_per_block(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index a7e7f784e4..905cb390eb 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -25,14 +25,14 @@ from eth2spec.test.helpers.state import ( state_transition_and_sign_block, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) def get_block_with_blob(spec, state, rng=None): block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs = get_sample_opaque_tx(spec, blob_count=1, rng=rng) + opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs = get_sample_blob_tx(spec, blob_count=1, rng=rng) block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) block.body.blob_kzg_commitments = blob_kzg_commitments diff --git a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py index 1dda310123..6deccd31e2 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py @@ -12,8 +12,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.debug.random_value import ( RandomizationMode, @@ -22,7 +22,7 @@ def _run_blob_kzg_commitment_merkle_proof_test(spec, state, rng=None): - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=1) + opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_blob_tx(spec, blob_count=1) if rng is None: block = build_empty_block_for_next_slot(spec, state) else: diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 94db2c34fc..b019175369 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -14,8 +14,8 @@ compute_el_block_hash, get_random_tx, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -27,7 +27,7 @@ def run_block_with_blobs(spec, state, blob_count, tx_count=1, blob_gas_used=1, e txs = [] blob_kzg_commitments = [] for _ in range(tx_count): - opaque_tx, _, commits, _ = get_sample_opaque_tx(spec, blob_count=blob_count) + opaque_tx, _, commits, _ = get_sample_blob_tx(spec, blob_count=blob_count) txs.append(opaque_tx) blob_kzg_commitments += commits diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py index 1d43d07caf..964fe16608 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -7,7 +7,7 @@ expect_assertion_error, always_bls ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, get_poly_in_both_forms, eval_poly_in_coeff_form, diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 6724e8304a..0e02ab8c3c 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -6,8 +6,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -20,8 +20,8 @@ def _get_sample_sidecars(spec, state, rng): # 2 txs, each has 2 blobs blob_count = 2 - opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) - opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_blob_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_blob_tx(spec, blob_count=blob_count, rng=rng) assert opaque_tx_1 != opaque_tx_2 block.body.blob_kzg_commitments = blob_kzg_commitments_1 + blob_kzg_commitments_2 diff --git a/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py index 98c751508d..4bd5fb4912 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py @@ -12,8 +12,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.debug.random_value import ( RandomizationMode, @@ -22,7 +22,7 @@ def _run_blob_kzg_commitments_merkle_proof_test(spec, state, rng=None): - opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1) + opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=1) if rng is None: block = build_empty_block_for_next_slot(spec, state) else: diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py index 625136b73e..dd0a80fda4 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py @@ -6,7 +6,7 @@ with_config_overrides, with_eip7594_and_later, ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, ) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py index c68c606765..4baf9fc83f 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -5,7 +5,7 @@ expect_assertion_error, with_eip7594_and_later, ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, ) from eth2spec.utils.bls import BLS_MODULUS diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/blob.py similarity index 57% rename from tests/core/pyspec/eth2spec/test/helpers/sharding.py rename to tests/core/pyspec/eth2spec/test/helpers/blob.py index c06b1aa9cb..e8117ece6c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/blob.py @@ -1,57 +1,28 @@ import random -from eth2spec.utils.ssz.ssz_typing import ( - Container, - Bytes20, Bytes32, - ByteList, - List, - Union, - boolean, - uint256, uint64, - uint8, -) -from eth2spec.utils.ssz.ssz_impl import serialize - - -# -# Containers from EIP-4844 -# -MAX_CALLDATA_SIZE = 2**24 -MAX_VERSIONED_HASHES_LIST_SIZE = 2**24 -MAX_ACCESS_LIST_STORAGE_KEYS = 2**24 -MAX_ACCESS_LIST_SIZE = 2**24 - - -BLOB_TX_TYPE = uint8(0x03) - - -class AccessTuple(Container): - address: Bytes20 # Address = Bytes20 - storage_keys: List[Bytes32, MAX_ACCESS_LIST_STORAGE_KEYS] - - -class ECDSASignature(Container): - y_parity: boolean - r: uint256 - s: uint256 - - -class BlobTransaction(Container): - chain_id: uint256 - nonce: uint64 - max_priority_fee_per_gas: uint256 - max_fee_per_gas: uint256 - gas: uint64 - to: Union[None, Bytes20] # Address = Bytes20 - value: uint256 - data: ByteList[MAX_CALLDATA_SIZE] - access_list: List[AccessTuple, MAX_ACCESS_LIST_SIZE] - max_fee_per_blob_gas: uint256 - blob_versioned_hashes: List[Bytes32, MAX_VERSIONED_HASHES_LIST_SIZE] - - -class SignedBlobTransaction(Container): - message: BlobTransaction - signature: ECDSASignature +from rlp import encode, Serializable +from rlp.sedes import Binary, CountableList, List as RLPList, big_endian_int, binary + + +class Eip4844RlpTransaction(Serializable): + fields = ( + ('chain_id', big_endian_int), + ('nonce', big_endian_int), + ('max_priority_fee_per_gas', big_endian_int), + ('max_fee_per_gas', big_endian_int), + ('gas_limit', big_endian_int), + ('to', Binary(20, 20)), + ('value', big_endian_int), + ('data', binary), + ('access_list', CountableList(RLPList([ + Binary(20, 20), + CountableList(Binary(32, 32)), + ]))), + ('max_fee_per_blob_gas', big_endian_int), + ('blob_versioned_hashes', CountableList(Binary(32, 32))), + ('signature_y_parity', big_endian_int), + ('signature_r', big_endian_int), + ('signature_s', big_endian_int), + ) def get_sample_blob(spec, rng=random.Random(5566), is_valid_blob=True): @@ -99,7 +70,7 @@ def get_poly_in_both_forms(spec, rng=None): return coeffs, evals -def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): +def get_sample_blob_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): blobs = [] blob_kzg_commitments = [] blob_kzg_proofs = [] @@ -118,11 +89,21 @@ def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_b blob_kzg_proofs.append(blob_kzg_proof) blob_versioned_hashes.append(blob_versioned_hash) - signed_blob_tx = SignedBlobTransaction( - message=BlobTransaction( - blob_versioned_hashes=blob_versioned_hashes, - ) + signed_blob_tx = Eip4844RlpTransaction( + chain_id=0, + nonce=0, + max_priority_fee_per_gas=0, + max_fee_per_gas=0, + gas_limit=0, + to=bytes.fromhex("0000000000000000000000000000000000000000"), + value=0, + data=bytes.fromhex(""), + access_list=[], + max_fee_per_blob_gas=0, + blob_versioned_hashes=[bytes(h) for h in blob_versioned_hashes], + signature_y_parity=0, + signature_r=0, + signature_s=0, ) - serialized_tx = serialize(signed_blob_tx) - opaque_tx = spec.uint_to_bytes(BLOB_TX_TYPE) + serialized_tx + opaque_tx = bytes([0x03]) + encode(signed_blob_tx) return opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 78802a6ddd..6e6cdd7683 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -24,8 +24,8 @@ randomize_state as randomize_state_helper, patch_state_to_non_leaking, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.test.helpers.state import ( next_slot, @@ -250,7 +250,7 @@ def random_block_capella(spec, state, signed_blocks, scenario_state, rng=Random( def random_block_deneb(spec, state, signed_blocks, scenario_state, rng=Random(3456)): block = random_block_capella(spec, state, signed_blocks, scenario_state, rng=rng) # TODO: more commitments. blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx( + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx( spec, blob_count=rng.randint(0, spec.config.MAX_BLOBS_PER_BLOCK), rng=rng) block.body.execution_payload.transactions.append(opaque_tx) block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) From 72f51900a79f417683634f8f3c7d9b78c2336f9c Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 11:55:49 -0500 Subject: [PATCH 2/2] Fix lint --- .../eth2spec/test/eip7594/unittests/test_networking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py index 633508f9a2..931cc9c1d1 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py @@ -15,8 +15,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -25,7 +25,7 @@ def compute_data_column_sidecar(spec, state): rng = random.Random(5566) - opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2) + opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=2) block = get_random_ssz_object( rng, spec.BeaconBlock,