Merge pull request #3111 from ethereum/testgen-fix

Fix --preset-list argument and enhance error output

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -4,7 +4,6 @@
 import shutil
 import argparse
 from pathlib import Path
-from filelock import FileLock
 import sys
 import json
 from typing import Iterable, AnyStr, Any, Callable
@@ -13,6 +12,7 @@
     YAML,
 )
 
+from filelock import FileLock
 from snappy import compress
 
 from eth2spec.test import context
@@ -141,6 +141,10 @@ def cfg_represent_quoted_str(self, data):
             tprov.prepare()
 
         for test_case in tprov.make_cases():
+            # If preset list is assigned, filter by presets.
+            if len(presets) != 0 and test_case.preset_name not in presets:
+                continue
+
             case_dir = (
                 Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name)
                 / Path(test_case.runner_name) / Path(test_case.handler_name)
@@ -179,7 +183,16 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
                     try:
                         fn(case_dir)
                     except IOError as e:
-                        sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}')
+                        error_message = (
+                            f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}'
+                        )
+                        # Write to error log file
+                        with log_file.open("a+") as f:
+                            f.write(error_message)
+                            traceback.print_exc(file=f)
+                            f.write('\n')
+
+                        sys.exit(error_message)
 
                 meta = dict()
 
@@ -210,13 +223,13 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]):
                 if not written_part:
                     print(f"test case {case_dir} did not produce any test case parts")
             except Exception as e:
-                print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
-                traceback.print_exc()
-                # Write to log file
+                error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}"
+                # Write to error log file
                 with log_file.open("a+") as f:
-                    f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}")
+                    f.write(error_message)
                     traceback.print_exc(file=f)
                     f.write('\n')
+                traceback.print_exc()
             else:
                 # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder.
                 if not written_part:

tests/generators/README.md

@@ -186,7 +186,7 @@ if __name__ == "__main__":
         ALTAIR: altair_mods,
     }
 
-    run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods)
+    run_state_test_generators(runner_name="sanity", all_mods=all_mods)
 ```
 
 Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner.
@@ -210,7 +210,7 @@ To add a new test generator that builds `New Tests`:
  with any dependencies it may need. Leave it empty if your generator has none.
 3. Your generator is assumed to have a `main.py` file in its root.
  By adding the base generator to your requirements, you can make a generator really easily. See docs below.
-4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`.
+4. Your generator is called with `-o some/file/path/for_testing/can/be_anything --preset-list mainnet minimal`.
  The base generator helps you handle this; you only have to define test case providers.
 5. Finally, add any linting or testing commands to the
  [circleci config file](../../.circleci/config.yml) if desired to increase code quality.