From f05bcd8eca9a4993a3f68f5f30a689d5b740e9bc Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Tue, 14 May 2024 16:44:33 +0200 Subject: [PATCH 1/4] Acceptence tests --- pyproject.toml | 3 ++ src/cloudai/schema/core/test_template.py | 2 + tests/test_acceptance.py | 47 ++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 tests/test_acceptance.py diff --git a/pyproject.toml b/pyproject.toml index 2dedebf4..2f5c9281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,3 +21,6 @@ select = ["I", "B", "E", "F", "SIM", "W", "C90", "EXE"] [tool.ruff.format] indent-style = "space" + +[tool.pytest.ini_options] +test_dir = "tests" diff --git a/src/cloudai/schema/core/test_template.py b/src/cloudai/schema/core/test_template.py index 4e647c4c..a761b214 100644 --- a/src/cloudai/schema/core/test_template.py +++ b/src/cloudai/schema/core/test_template.py @@ -49,6 +49,8 @@ class TestTemplate: based on test outcomes. """ + __test__ = False + def __init__( self, system: System, diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py new file mode 100644 index 00000000..dc407c8f --- /dev/null +++ b/tests/test_acceptance.py @@ -0,0 +1,47 @@ +import argparse +from pathlib import Path + +import pytest +from cloudai.__main__ import handle_dry_run_and_run + +TEST_SCENARIOS = list(Path().glob("conf/v0.6/general/test_scenario/**/*.toml")) + + +@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS) +def test_slurm(tmp_path: Path, test_scenario_path: Path): + args = argparse.Namespace( + log_file=None, + log_level=None, + mode="dry-run", + output_path=str(tmp_path), + system_config_path="conf/v0.6/general/system/example_slurm_cluster.toml", + test_scenario_path=str(test_scenario_path), + test_path="conf/v0.6/general/test", + test_template_path="conf/v0.6/general/test_template", + ) + handle_dry_run_and_run(args) + + test_dir = list(tmp_path.glob("*"))[0] + for td in test_dir.iterdir(): + assert td.is_dir(), "Invalid test directory" + assert "Tests." in td.name, "Invalid test directory name" + + +@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS) +def test_standalone(tmp_path: Path, test_scenario_path: Path): + args = argparse.Namespace( + log_file=None, + log_level=None, + mode="dry-run", + output_path=str(tmp_path), + system_config_path="conf/v0.6/general/system/standalone_system.toml", + test_scenario_path=str(test_scenario_path), + test_path="conf/v0.6/general/test", + test_template_path="conf/v0.6/general/test_template", + ) + handle_dry_run_and_run(args) + + test_dir = list(tmp_path.glob("*"))[0] + for td in test_dir.iterdir(): + assert td.is_dir(), "Invalid test directory" + assert "Tests." in td.name, "Invalid test directory name" From d26b2467120d60e2093ee3aab944a680c0ecd874 Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Wed, 15 May 2024 12:24:55 +0200 Subject: [PATCH 2/4] Fix one case --- conf/v0.6/general/system/example_slurm_cluster.toml | 2 +- tests/test_acceptance.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/v0.6/general/system/example_slurm_cluster.toml b/conf/v0.6/general/system/example_slurm_cluster.toml index 6eedd829..633d028c 100644 --- a/conf/v0.6/general/system/example_slurm_cluster.toml +++ b/conf/v0.6/general/system/example_slurm_cluster.toml @@ -3,7 +3,7 @@ scheduler = "slurm" install_path = "./install" output_path = "./results" -default_partition = "DEFAULT_PARTITION" +default_partition = "partition_1" gpus_per_node = 8 ntasks_per_node = 8 diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index dc407c8f..a7721092 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -7,7 +7,7 @@ TEST_SCENARIOS = list(Path().glob("conf/v0.6/general/test_scenario/**/*.toml")) -@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS) +@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS, ids=lambda x: str(x)) def test_slurm(tmp_path: Path, test_scenario_path: Path): args = argparse.Namespace( log_file=None, @@ -27,7 +27,7 @@ def test_slurm(tmp_path: Path, test_scenario_path: Path): assert "Tests." in td.name, "Invalid test directory name" -@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS) +@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS, ids=lambda x: str(x)) def test_standalone(tmp_path: Path, test_scenario_path: Path): args = argparse.Namespace( log_file=None, From 930f40d1b6c5763f168e8edf4eca32967c8e3432 Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Thu, 16 May 2024 11:56:58 +0200 Subject: [PATCH 3/4] Limit the number of tests configs and add coverage --- .github/workflows/ci.yml | 2 +- requirements-dev.txt | 1 + tests/test_acceptance.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06eda41f..2a6279ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,4 +52,4 @@ jobs: run: pip install pytest -r requirements-dev.txt - name: Run pytest - run: pytest -vv + run: pytest -vv --cov diff --git a/requirements-dev.txt b/requirements-dev.txt index fe537ca2..341dde74 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ ruff==0.3.7 pandas-stubs==2.2.* pyright==1.1.359 build==1.2.* +pytest-cov==5.0.* diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index a7721092..128ba8c4 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -4,10 +4,17 @@ import pytest from cloudai.__main__ import handle_dry_run_and_run -TEST_SCENARIOS = list(Path().glob("conf/v0.6/general/test_scenario/**/*.toml")) +SLURM_TEST_SCENARIOS = [ + Path("conf/v0.6/general/test_scenario/sleep/test_scenario.toml"), + Path("conf/v0.6/general/test_scenario/ucc_test/test_scenario.toml"), +] +STANDALONE_TEST_SCENARIOS = [ + Path("conf/v0.6/general/test_scenario/sleep/test_scenario.toml"), +] -@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS, ids=lambda x: str(x)) + +@pytest.mark.parametrize("test_scenario_path", SLURM_TEST_SCENARIOS, ids=lambda x: str(x)) def test_slurm(tmp_path: Path, test_scenario_path: Path): args = argparse.Namespace( log_file=None, @@ -27,7 +34,7 @@ def test_slurm(tmp_path: Path, test_scenario_path: Path): assert "Tests." in td.name, "Invalid test directory name" -@pytest.mark.parametrize("test_scenario_path", TEST_SCENARIOS, ids=lambda x: str(x)) +@pytest.mark.parametrize("test_scenario_path", STANDALONE_TEST_SCENARIOS, ids=lambda x: str(x)) def test_standalone(tmp_path: Path, test_scenario_path: Path): args = argparse.Namespace( log_file=None, From b35f84a8ccb02388f8413a3940b9afe6efb84a7f Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Thu, 16 May 2024 12:09:30 +0200 Subject: [PATCH 4/4] Remove test that hangs in GHA --- tests/test_acceptance.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index 128ba8c4..2ae1fb84 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -9,10 +9,6 @@ Path("conf/v0.6/general/test_scenario/ucc_test/test_scenario.toml"), ] -STANDALONE_TEST_SCENARIOS = [ - Path("conf/v0.6/general/test_scenario/sleep/test_scenario.toml"), -] - @pytest.mark.parametrize("test_scenario_path", SLURM_TEST_SCENARIOS, ids=lambda x: str(x)) def test_slurm(tmp_path: Path, test_scenario_path: Path): @@ -32,23 +28,3 @@ def test_slurm(tmp_path: Path, test_scenario_path: Path): for td in test_dir.iterdir(): assert td.is_dir(), "Invalid test directory" assert "Tests." in td.name, "Invalid test directory name" - - -@pytest.mark.parametrize("test_scenario_path", STANDALONE_TEST_SCENARIOS, ids=lambda x: str(x)) -def test_standalone(tmp_path: Path, test_scenario_path: Path): - args = argparse.Namespace( - log_file=None, - log_level=None, - mode="dry-run", - output_path=str(tmp_path), - system_config_path="conf/v0.6/general/system/standalone_system.toml", - test_scenario_path=str(test_scenario_path), - test_path="conf/v0.6/general/test", - test_template_path="conf/v0.6/general/test_template", - ) - handle_dry_run_and_run(args) - - test_dir = list(tmp_path.glob("*"))[0] - for td in test_dir.iterdir(): - assert td.is_dir(), "Invalid test directory" - assert "Tests." in td.name, "Invalid test directory name"