bug(pytest, forks): fix yul_test marker and improve solc --evm-version param handling

docs/CHANGELOG.md

@@ -4,6 +4,19 @@ Test fixtures for use by clients are available for each release on the [Github r
 
 **Key:** ✨ = New, 🐞 = Fixed, 🔀 = Changed, 💥 = Breaking change.
 
+## 🔜 [Unreleased] https://github.com/ethereum/execution-spec-tests/releases/tag/v-Unreleased) - 2024-xx-xx
+
+### 🧪 Test Cases
+
+### 🛠️ Framework
+
+- ✨ Improve handling of the argument passed to `solc --evm-version` when compiling Yul code ([#418](https://github.com/ethereum/execution-spec-tests/pull/418)).
+- 🐞 Fix `fill -m yul_test` which failed to filter tests that are (dynamically) marked as a yul test ([#418](https://github.com/ethereum/execution-spec-tests/pull/418)).
+
+### 🔧 EVM Tools
+
+### 📋 Misc
+
 ## [v2.1.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v2.1.0) - 2024-01-29: 🐍🏖️ Cancun
 
 Release [v2.1.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v2.1.0) primarily fixes a small bug introduced within the previous release where transition forks are used within the new `StateTest` format. This was highlighted by @chfast within #405 (https://github.com/ethereum/execution-spec-tests/issues/405), where the fork name `ShanghaiToCancunAtTime15k` was found within state tests.

src/ethereum_test_forks/__init__.py

@@ -2,7 +2,7 @@
 Ethereum test fork definitions.
 """
 
-from .base_fork import Fork, ForkAttribute
+from .base_fork import SOLC_VERSION_TO_SUPPORTED_FORKS, Fork, ForkAttribute
 from .forks.forks import (
     ArrowGlacier,
     Berlin,
@@ -28,15 +28,19 @@
     InvalidForkError,
     forks_from,
     forks_from_until,
+    get_closest_fork_with_solc_support,
     get_deployed_forks,
     get_development_forks,
     get_forks,
+    get_forks_with_solc_support,
+    get_forks_without_solc_support,
     get_transition_forks,
     transition_fork_from_to,
     transition_fork_to,
 )
 
 __all__ = [
+    "SOLC_VERSION_TO_SUPPORTED_FORKS",
     "Fork",
     "ForkAttribute",
     "ArrowGlacier",
@@ -63,6 +67,9 @@
     "get_deployed_forks",
     "get_development_forks",
     "get_forks",
+    "get_forks_with_solc_support",
+    "get_forks_without_solc_support",
+    "get_closest_fork_with_solc_support",
     "transition_fork_from_to",
     "transition_fork_to",
 ]

src/ethereum_test_forks/base_fork.py

@@ -4,8 +4,32 @@
 from abc import ABC, ABCMeta, abstractmethod
 from typing import Any, ClassVar, List, Mapping, Optional, Protocol, Type
 
+from semver import Version
+
 from .base_decorators import prefer_transition_to_method
 
+SOLC_FORKS_0_8_20 = [
+    "homestead",
+    "tangerineWhistle",
+    "spuriousDragon",
+    "byzantium",
+    "constantinople",
+    "petersburg",
+    "istanbul",
+    "berlin",
+    "london",
+    "paris",
+    "shanghai",
+]
+SOLC_VERSION_TO_SUPPORTED_FORKS = {
+    Version.parse("0.8.20"): SOLC_FORKS_0_8_20,
+    Version.parse("0.8.21"): SOLC_FORKS_0_8_20,
+    Version.parse("0.8.22"): SOLC_FORKS_0_8_20,
+    Version.parse("0.8.23"): SOLC_FORKS_0_8_20,
+    # TBA: 0.8.24 is the current dev version.
+    Version.parse("0.8.24"): SOLC_FORKS_0_8_20,
+}
+
 
 class ForkAttribute(Protocol):
     """
@@ -258,6 +282,16 @@ def solc_name(cls, block_number: int = 0, timestamp: int = 0) -> str:
         """
         pass
 
+    @classmethod
+    def is_supported_by_solc(cls, version: Version) -> bool:
+        """
+        Returns True if the fork is supported by solc (--evm-version).
+        """
+        assert (
+            version in SOLC_VERSION_TO_SUPPORTED_FORKS.keys()
+        ), f"Unsupported solc version: {version}"
+        return cls.solc_name() in SOLC_VERSION_TO_SUPPORTED_FORKS[version]
+
     @classmethod
     def blockchain_test_network_name(cls) -> str:
         """

src/ethereum_test_forks/forks/forks.py

@@ -7,7 +7,7 @@
 
 
 # All forks must be listed here !!! in the order they were introduced !!!
-class Frontier(BaseFork):
+class Frontier(BaseFork, solc_name=None):
     """
     Frontier fork
     """
@@ -150,7 +150,7 @@ def pre_allocation(cls, block_number: int = 0, timestamp: int = 0) -> Mapping:
         return {}
 
 
-class Homestead(Frontier):
+class Homestead(Frontier, solc_name="homestead"):
     """
     Homestead fork
     """
@@ -163,7 +163,7 @@ def precompiles(cls, block_number: int = 0, timestamp: int = 0) -> List[int]:
         return [1, 2, 3, 4] + super(Homestead, cls).precompiles(block_number, timestamp)
 
 
-class Byzantium(Homestead):
+class Byzantium(Homestead, solc_name="byzantium"):
     """
     Byzantium fork
     """
@@ -186,7 +186,7 @@ def precompiles(cls, block_number: int = 0, timestamp: int = 0) -> List[int]:
         return [5, 6, 7, 8] + super(Byzantium, cls).precompiles(block_number, timestamp)
 
 
-class Constantinople(Byzantium):
+class Constantinople(Byzantium, solc_name="constantinople"):
     """
     Constantinople fork
     """
@@ -200,15 +200,15 @@ def get_reward(cls, block_number: int = 0, timestamp: int = 0) -> int:
         return 2_000_000_000_000_000_000
 
 
-class ConstantinopleFix(Constantinople, solc_name="Constantinople"):
+class ConstantinopleFix(Constantinople, solc_name="constantinople"):
     """
     Constantinople Fix fork
     """
 
     pass
 
 
-class Istanbul(ConstantinopleFix):
+class Istanbul(ConstantinopleFix, solc_name="istanbul"):
     """
     Istanbul fork
     """
@@ -230,7 +230,7 @@ class MuirGlacier(Istanbul):
     pass
 
 
-class Berlin(Istanbul):
+class Berlin(Istanbul, solc_name="berlin"):
     """
     Berlin fork
     """
@@ -243,7 +243,7 @@ def tx_types(cls, block_number: int = 0, timestamp: int = 0) -> List[int]:
         return [1] + super(Berlin, cls).tx_types(block_number, timestamp)
 
 
-class London(Berlin):
+class London(Berlin, solc_name="london"):
     """
     London fork
     """
@@ -282,6 +282,7 @@ class GrayGlacier(ArrowGlacier):
 
 class Paris(
     London,
+    solc_name="paris",
     transition_tool_name="Merge",
     blockchain_test_network_name="Merge",
 ):
@@ -320,7 +321,7 @@ def engine_new_payload_version(
         return 1
 
 
-class Shanghai(Paris):
+class Shanghai(Paris, solc_name="shanghai"):
     """
     Shanghai fork
     """
@@ -342,7 +343,7 @@ def engine_new_payload_version(
         return 2
 
 
-class Cancun(Shanghai):
+class Cancun(Shanghai, solc_name="cancun"):
     """
     Cancun fork
     """

src/ethereum_test_forks/helpers.py

@@ -3,8 +3,11 @@
 """
 from typing import List
 
+from semver import Version
+
 from .base_fork import BaseFork, Fork
 from .forks import forks, transition
+from .forks.forks import Frontier, Homestead
 from .transition_base_fork import TransitionBaseClass
 
 
@@ -33,15 +36,15 @@ def get_forks() -> List[Fork]:
     return all_forks
 
 
-def get_deployed_forks():
+def get_deployed_forks() -> List[Fork]:
     """
     Returns a list of all the fork classes implemented by `ethereum_test_forks`
     that have been deployed to mainnet, chronologically ordered by deployment.
     """
     return [fork for fork in get_forks() if fork.is_deployed()]
 
 
-def get_development_forks():
+def get_development_forks() -> List[Fork]:
     """
     Returns a list of all the fork classes implemented by `ethereum_test_forks`
     that have been not yet deployed to mainnet and are currently under
@@ -57,6 +60,35 @@ def get_parent_fork(fork: Fork) -> Fork:
     return fork.__base__
 
 
+def get_forks_with_solc_support(solc_version: Version) -> List[Fork]:
+    """
+    Returns a list of all fork classes that are supported by solc.
+    """
+    return [fork for fork in get_forks() if fork.is_supported_by_solc(solc_version)]
+
+
+def get_forks_without_solc_support(solc_version: Version) -> List[Fork]:
+    """
+    Returns a list of all fork classes that aren't supported by solc.
+    """
+    return [fork for fork in get_forks() if not fork.is_supported_by_solc(solc_version)]
+
+
+def get_closest_fork_with_solc_support(fork: Fork, solc_version: Version) -> Fork:
+    """
+    Returns the closest fork, potentially the provided fork itself, that has
+    solc support.
+    """
+    if fork == Frontier:
+        return Homestead
+    if fork in get_forks_with_solc_support(solc_version):
+        return fork
+    parent_fork = get_parent_fork(fork)
+    if parent_fork.is_supported_by_solc(solc_version):
+        return parent_fork
+    return get_closest_fork_with_solc_support(parent_fork, solc_version)
+
+
 def get_transition_forks() -> List[Fork]:
     """
     Returns all the transition forks

src/ethereum_test_tools/code/__init__.py

@@ -3,9 +3,10 @@
 """
 from .code import Code
 from .generators import CalldataCase, Case, CodeGasMeasure, Conditional, Initcode, Switch
-from .yul import Yul, YulCompiler
+from .yul import SOLC_SUPPORTED_VERSIONS, Yul, YulCompiler
 
 __all__ = (
+    "SOLC_SUPPORTED_VERSIONS",
     "Case",
     "CalldataCase",
     "Code",

src/ethereum_test_tools/code/yul.py

@@ -11,13 +11,15 @@
 
 from semver import Version
 
-from ethereum_test_forks import Fork
+from ethereum_test_forks import SOLC_VERSION_TO_SUPPORTED_FORKS, Fork
 
 from ..common.conversions import to_bytes
 from .code import Code
 
 DEFAULT_SOLC_ARGS = ("--assemble", "-")
 
+SOLC_SUPPORTED_VERSIONS = list(SOLC_VERSION_TO_SUPPORTED_FORKS.keys())
+
 
 def get_evm_version_from_fork(fork: Fork | None):
     """
@@ -33,7 +35,7 @@ def get_evm_version_from_fork(fork: Fork | None):
     """
     if not fork:
         return None
-    return fork.solc_name().lower()
+    return fork.solc_name()
 
 
 class Yul(SupportsBytes, Sized):

src/ethereum_test_tools/tests/conftest.py

@@ -5,11 +5,7 @@
 import pytest
 from semver import Version
 
-from ..code import Yul
-
-SUPPORTED_SOLC_VERSIONS = [
-    Version.parse(v) for v in ["0.8.20", "0.8.21", "0.8.22", "0.8.23", "0.8.24"]
-]
+from ..code import SOLC_SUPPORTED_VERSIONS, Yul
 
 SOLC_PADDING_VERSION = Version.parse("0.8.21")
 
@@ -18,6 +14,6 @@
 def solc_version() -> Version:
     """Return the version of solc being used for tests."""
     solc_version = Yul("").version().finalize_version()
-    if solc_version not in SUPPORTED_SOLC_VERSIONS:
+    if solc_version not in SOLC_SUPPORTED_VERSIONS:
         raise Exception("Unsupported solc version: {}".format(solc_version))
     return solc_version

src/ethereum_test_tools/tests/test_code.py

@@ -8,10 +8,26 @@
 import pytest
 from semver import Version
 
-from ethereum_test_forks import Fork, Homestead, Shanghai, forks_from_until, get_deployed_forks
+from ethereum_test_forks import (
+    Fork,
+    Homestead,
+    Shanghai,
+    forks_from_until,
+    get_deployed_forks,
+    get_forks_with_solc_support,
+)
 from evm_transition_tool import FixtureFormats, GethTransitionTool
 
-from ..code import CalldataCase, Case, Code, Conditional, Initcode, Switch, Yul
+from ..code import (
+    SOLC_SUPPORTED_VERSIONS,
+    CalldataCase,
+    Case,
+    Code,
+    Conditional,
+    Initcode,
+    Switch,
+    Yul,
+)
 from ..common import Account, Environment, TestAddress, Transaction, to_hash_bytes
 from ..spec import StateTest
 from ..vm.opcode import Opcodes as Op
@@ -49,14 +65,14 @@ def test_code_operations(code: Code, expected_bytes: bytes):
     assert bytes(code) == expected_bytes
 
 
-@pytest.fixture(params=forks_from_until(get_deployed_forks()[1], get_deployed_forks()[-1]))
+@pytest.fixture(params=get_forks_with_solc_support(SOLC_SUPPORTED_VERSIONS[-1]))
 def fork(request: pytest.FixtureRequest):
     """
     Return the target evm-version (fork) for solc compilation.
 
     Note:
-    - get_deployed_forks()[1] (Homestead) is the first fork that solc supports.
-    - forks_from_util: Used to remove the Glacier forks
+    - Homestead.
+    - forks_from_until: Used to remove the Glacier forks
     """
     return request.param