Merge pull request #2937 from asn-d6/eip4844-exe-squashed-and-rebased

 Executable EIP-4844 specs (PR #2901 squashed and rebased)

.circleci/config.yml

@@ -142,7 +142,19 @@ jobs:
           command: make citest fork=capella
       - store_test_results:
           path: tests/core/pyspec/test-reports
-
+  test-eip4844:
+    docker:
+      - image: circleci/python:3.8
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_pyspec_cached_venv
+      - run:
+          name: Run py-tests
+          command: make citest fork=eip4844
+      - store_test_results:
+          path: tests/core/pyspec/test-reports
   table_of_contents:
     docker:
       - image: circleci/node:10.16.3
@@ -260,6 +272,9 @@ workflows:
       - test-capella:
           requires:
             - install_pyspec_test
+      - test-eip4844:
+          requires:
+            - install_pyspec_test
       - table_of_contents
       - codespell
       - lint:

.gitignore

@@ -19,6 +19,7 @@ tests/core/pyspec/eth2spec/phase0/
 tests/core/pyspec/eth2spec/altair/
 tests/core/pyspec/eth2spec/bellatrix/
 tests/core/pyspec/eth2spec/capella/
+tests/core/pyspec/eth2spec/eip4844/
 
 # coverage reports
 .htmlcov

Makefile

@@ -136,7 +136,7 @@ codespell:
 lint: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
-	&& pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix \
+	&& pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella \
 	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella
 
 lint_generators: pyspec

configs/mainnet.yaml

@@ -47,9 +47,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615
 # Capella
 CAPELLA_FORK_VERSION: 0x03000000
 CAPELLA_FORK_EPOCH: 18446744073709551615
-# Sharding
-SHARDING_FORK_VERSION: 0x04000000
-SHARDING_FORK_EPOCH: 18446744073709551615
+# EIP4844
+EIP4844_FORK_VERSION: 0x04000000
+EIP4844_FORK_EPOCH: 18446744073709551615
 
 
 

configs/minimal.yaml

@@ -46,9 +46,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615
 # Capella
 CAPELLA_FORK_VERSION: 0x03000001
 CAPELLA_FORK_EPOCH: 18446744073709551615
-# Sharding
-SHARDING_FORK_VERSION: 0x04000001
-SHARDING_FORK_EPOCH: 18446744073709551615
+# EIP4844
+EIP4844_FORK_VERSION: 0x04000001
+EIP4844_FORK_EPOCH: 18446744073709551615
 
 
 # Time parameters

presets/mainnet/eip4844.yaml

@@ -0,0 +1,8 @@
+# Mainnet preset - Phase0
+
+# Misc
+# ---------------------------------------------------------------
+# `uint64(4096)`
+FIELD_ELEMENTS_PER_BLOB: 4096
+# `uint64(2**4)` (= 16)
+MAX_BLOBS_PER_BLOCK: 16

presets/minimal/eip4844.yaml

@@ -0,0 +1,8 @@
+# Minimal preset - Phase0
+
+# Misc
+# ---------------------------------------------------------------
+# [customized]
+FIELD_ELEMENTS_PER_BLOB: 4
+# `uint64(2**4)` (= 16)
+MAX_BLOBS_PER_BLOCK: 16

setup.py

@@ -45,6 +45,7 @@ def installPackage(package: str):
 ALTAIR = 'altair'
 BELLATRIX = 'bellatrix'
 CAPELLA = 'capella'
+EIP4844 = 'eip4844'
 
 
 # The helper functions that are used when defining constants
@@ -230,7 +231,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
 
                     if not _is_constant_id(name):
                         # Check for short type declarations
-                        if value.startswith(("uint", "Bytes", "ByteList", "Union")):
+                        if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List")):
                             custom_types[name] = value
                         continue
 
@@ -304,7 +305,7 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
 
     @classmethod
     @abstractmethod
-    def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]:  # TODO
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:  # TODO
         """
         The constants that are required for custom types.
         """
@@ -432,7 +433,7 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
         return {}
 
     @classmethod
-    def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]:
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
         return {}
 
     @classmethod
@@ -548,11 +549,11 @@ def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayloa
 
 
     @classmethod
-    def hardcoded_custom_type_dep_constants(cls) -> str:
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
         constants = {
-            'MAX_BYTES_PER_TRANSACTION': 'uint64(2**30)',
+            'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value,
         }
-        return {**super().hardcoded_custom_type_dep_constants(), **constants}
+        return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants}
 
 
 #
@@ -568,14 +569,57 @@ def imports(cls, preset_name: str):
 '''
 
 
+#
+# EIP4844SpecBuilder
+#
+class EIP4844SpecBuilder(BellatrixSpecBuilder):
+    fork: str = EIP4844
+
+    @classmethod
+    def imports(cls, preset_name: str):
+        return super().imports(preset_name) + f'''
+from eth2spec.utils import kzg
+from eth2spec.bellatrix import {preset_name} as bellatrix
+'''
+
+    @classmethod
+    def sundry_functions(cls) -> str:
+        return super().sundry_functions() + '''
+# TODO: for mainnet, load pre-generated trusted setup file to reduce building time.
+# TESTING_FIELD_ELEMENTS_PER_BLOB is hardcoded copy from minimal presets
+TESTING_FIELD_ELEMENTS_PER_BLOB = 4
+TESTING_SECRET = 1337
+TESTING_KZG_SETUP_G1 = kzg.generate_setup(bls.G1, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB)
+TESTING_KZG_SETUP_G2 = kzg.generate_setup(bls.G2, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB)
+TESTING_KZG_SETUP_LAGRANGE = kzg.get_lagrange(TESTING_KZG_SETUP_G1)
+
+KZG_SETUP_G1 = [bls.G1_to_bytes48(p) for p in TESTING_KZG_SETUP_G1]
+KZG_SETUP_G2 = [bls.G2_to_bytes96(p) for p in TESTING_KZG_SETUP_G2]
+KZG_SETUP_LAGRANGE = TESTING_KZG_SETUP_LAGRANGE
+ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB)
+
+
+def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> BlobsSidecar:
+    pass'''
+
+    @classmethod
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
+        constants = {
+            'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
+            'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+        }
+        return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants}
+
+
+
 spec_builders = {
     builder.fork: builder
-    for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder)
+    for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, EIP4844SpecBuilder)
 }
 
 
 def is_spec_defined_type(value: str) -> bool:
-    return value.startswith('ByteList') or value.startswith('Union')
+    return value.startswith(('ByteList', 'Union', 'Vector', 'List'))
 
 
 def objects_to_spec(preset_name: str,
@@ -653,7 +697,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
     ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants()))
     ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants()))
-    custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants()[x]), builder.hardcoded_custom_type_dep_constants()))
+    custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants(spec_object)[x]), builder.hardcoded_custom_type_dep_constants(spec_object)))
     spec = (
             builder.imports(preset_name)
             + builder.preparations()
@@ -870,14 +914,14 @@ def finalize_options(self):
         if len(self.md_doc_paths) == 0:
             print("no paths were specified, using default markdown file paths for pyspec"
                   " build (spec fork: %s)" % self.spec_fork)
-            if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA):
+            if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844):
                 self.md_doc_paths = """
                     specs/phase0/beacon-chain.md
                     specs/phase0/fork-choice.md
                     specs/phase0/validator.md
                     specs/phase0/weak-subjectivity.md
                 """
-            if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA):
+            if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, EIP4844):
                 self.md_doc_paths += """
                     specs/altair/beacon-chain.md
                     specs/altair/bls.md
@@ -886,7 +930,7 @@ def finalize_options(self):
                     specs/altair/p2p-interface.md
                     specs/altair/sync-protocol.md
                 """
-            if self.spec_fork in (BELLATRIX, CAPELLA):
+            if self.spec_fork in (BELLATRIX, CAPELLA, EIP4844):
                 self.md_doc_paths += """
                     specs/bellatrix/beacon-chain.md
                     specs/bellatrix/fork.md
@@ -903,6 +947,14 @@ def finalize_options(self):
                     specs/capella/validator.md
                     specs/capella/p2p-interface.md
                 """
+            if self.spec_fork == EIP4844:
+                self.md_doc_paths += """
+                    specs/eip4844/beacon-chain.md
+                    specs/eip4844/fork.md
+                    specs/eip4844/polynomial-commitments.md
+                    specs/eip4844/p2p-interface.md
+                    specs/eip4844/validator.md
+                """
             if len(self.md_doc_paths) == 0:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)