From f1767bb7e5200f303b4a38b8604b5ce0ce2a4b26 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 7 Feb 2024 13:51:34 -0800 Subject: [PATCH 1/5] Turning examples into integration tests run through subprocess. --- .../integration/test_cluster_number_counts.py | 55 ++++++++++++++ tests/integration/test_cosmic_shear.py | 48 +++++++++++++ tests/integration/test_des_y1_3x2pt.py | 72 +++++++++++++++++++ tests/integration/test_srd_sn.py | 48 +++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 tests/integration/test_cluster_number_counts.py create mode 100644 tests/integration/test_cosmic_shear.py create mode 100644 tests/integration/test_des_y1_3x2pt.py create mode 100644 tests/integration/test_srd_sn.py diff --git a/tests/integration/test_cluster_number_counts.py b/tests/integration/test_cluster_number_counts.py new file mode 100644 index 00000000..b6ad1da3 --- /dev/null +++ b/tests/integration/test_cluster_number_counts.py @@ -0,0 +1,55 @@ +import subprocess +import pytest + + +@pytest.mark.integration +def test_cluster_number_counts_cosmosis(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/cluster_number_counts + python generate_rich_mean_mass_sacc_data.py + cosmosis cluster_counts_redshift_richness.ini + cosmosis cluster_mean_mass_redshift_richness.ini + cosmosis cluster_counts_mean_mass_redshift_richness.ini + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) + + +@pytest.mark.integration +def test_cluster_number_counts_numcosmo(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/cluster_number_counts + numcosmo from-cosmosis cluster_counts_redshift_richness.ini\\ + --matter-ps eisenstein_hu + numcosmo run test cluster_counts_redshift_richness.yaml + numcosmo from-cosmosis cluster_mean_mass_redshift_richness.ini\\ + --matter-ps eisenstein_hu + numcosmo run test cluster_mean_mass_redshift_richness.yaml + numcosmo from-cosmosis cluster_counts_mean_mass_redshift_richness.ini\\ + --matter-ps eisenstein_hu + numcosmo run test cluster_counts_mean_mass_redshift_richness.yaml + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) diff --git a/tests/integration/test_cosmic_shear.py b/tests/integration/test_cosmic_shear.py new file mode 100644 index 00000000..a579941f --- /dev/null +++ b/tests/integration/test_cosmic_shear.py @@ -0,0 +1,48 @@ +import subprocess +import pytest + + +@pytest.mark.integration +def test_cosmic_shear_cosmosis(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/cosmicshear + python generate_cosmicshear_data.py + cosmosis cosmicshear.ini + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) + + +@pytest.mark.integration +def test_cosmic_shear_numcosmo(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/cosmicshear + numcosmo from-cosmosis cosmicshear.ini \\ + --matter-ps eisenstein_hu \\ + --nonlin-matter-ps halofit + numcosmo run test cosmicshear.yaml + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) diff --git a/tests/integration/test_des_y1_3x2pt.py b/tests/integration/test_des_y1_3x2pt.py new file mode 100644 index 00000000..0f01f90e --- /dev/null +++ b/tests/integration/test_des_y1_3x2pt.py @@ -0,0 +1,72 @@ +import subprocess +import pytest + + +@pytest.mark.integration +def test_des_y1_3x2pt_cosmosis(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/des_y1_3x2pt + cosmosis des_y1_3x2pt.ini + cosmosis des_y1_3x2pt_PT.ini + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) + + +@pytest.mark.integration +def test_des_y1_3x2pt_numcosmo(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/des_y1_3x2pt + numcosmo from-cosmosis des_y1_3x2pt.ini --matter-ps eisenstein_hu\\ + --nonlin-matter-ps halofit + numcosmo run test des_y1_3x2pt.yaml + numcosmo from-cosmosis des_y1_3x2pt_PT.ini --matter-ps eisenstein_hu\\ + --nonlin-matter-ps halofit + numcosmo run test des_y1_3x2pt_PT.yaml + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) + + +@pytest.mark.integration +def test_des_y1_3x2pt_cobaya(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/des_y1_3x2pt + cobaya-run cobaya_evaluate.yaml + cobaya-run cobaya_evaluate_PT.yaml + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) diff --git a/tests/integration/test_srd_sn.py b/tests/integration/test_srd_sn.py new file mode 100644 index 00000000..36e3d55e --- /dev/null +++ b/tests/integration/test_srd_sn.py @@ -0,0 +1,48 @@ +import subprocess +import pytest + + +@pytest.mark.integration +def test_srd_sn_cosmosis(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/srd_sn + cosmosis sn_srd.ini + cosmosis sn_only.ini + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) + + +@pytest.mark.integration +def test_srd_sn_numcosmo(): + result = subprocess.run( + [ + "bash", + "-c", + """ + set -e + cd examples/srd_sn + numcosmo from-cosmosis sn_srd.ini + numcosmo run test sn_srd.yaml + numcosmo from-cosmosis sn_only.ini + numcosmo run test sn_only.yaml + """, + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.returncode == 0 + print(result.stdout) From a93de9a464af166bd087e64149d92953a61ba75f Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 7 Feb 2024 13:52:20 -0800 Subject: [PATCH 2/5] Updating pytest to skip integration tests unless asked for explicitly. --- tests/conftest.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6944986e..fe71627f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,10 +21,17 @@ def pytest_addoption(parser): """Add handling of firecrown-specific options for the pytest test runner. --runslow: used to run tests marked as slow, which are otherwise not run. + --integration: used to run only integration tests, which are otherwise not run. """ parser.addoption( "--runslow", action="store_true", default=False, help="run slow tests" ) + parser.addoption( + "--integration", + action="store_true", + default=False, + help="run integration tests", + ) def pytest_configure(config): @@ -39,13 +46,19 @@ def pytest_configure(config): def pytest_collection_modifyitems(config, items): """Apply our special markers and option handling for pytest.""" - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests - return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") + + if not config.getoption("--integration"): + _skip_tests(items, "integration", "need --integration option to run") + + if not config.getoption("--runslow"): + _skip_tests(items, "slow", "need --runslow option to run") + + +def _skip_tests(items, keyword, reason): + tests_to_skip = pytest.mark.skip(reason=reason) for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) + if keyword in item.keywords: + item.add_marker(tests_to_skip) # Fixtures From 58e63ddcaaf9185931d89c4311de27824ded9049 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 7 Feb 2024 13:54:36 -0800 Subject: [PATCH 3/5] Renaming another test to avoid confusion. --- ...ster_number_counts.py => test_binned_cluster_number_counts.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/likelihood/gauss_family/statistic/{test_cluster_number_counts.py => test_binned_cluster_number_counts.py} (100%) diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_binned_cluster_number_counts.py similarity index 100% rename from tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py rename to tests/likelihood/gauss_family/statistic/test_binned_cluster_number_counts.py From f2f16cadf9a599ae12dc1c76e9780e12b62d6ed8 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 7 Feb 2024 14:02:48 -0800 Subject: [PATCH 4/5] Changing CI to run integration tests through pytest. --- .github/workflows/ci.yml | 68 ++-------------------------------------- 1 file changed, 3 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56e08776..a9409e51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,74 +95,12 @@ jobs: pylint firecrown pylint --rcfile tests/pylintrc tests pylint --rcfile firecrown/models/pylintrc firecrown/models - - name: Running pytest + - name: Running unit tests shell: bash -l {0} run: python -m pytest -vv --runslow --cov firecrown --cov-report xml - - name: Running example - cosmic-shear - cosmosis + - name: Running integration tests shell: bash -l {0} - run: | - cd examples/cosmicshear - python generate_cosmicshear_data.py - cosmosis cosmicshear.ini - - name: Running example - cosmic-shear - NumCosmo - shell: bash -l {0} - run: | - cd examples/cosmicshear - numcosmo from-cosmosis cosmicshear.ini --matter-ps eisenstein_hu --nonlin-matter-ps halofit - numcosmo run test cosmicshear.yaml - - name: Running example - des-y1-3x2pt - cosmosis - shell: bash -l {0} - run: | - cd examples/des_y1_3x2pt - cosmosis des_y1_3x2pt.ini - cosmosis des_y1_3x2pt_PT.ini - - name: Running example - des-y1-3x2pt - cobaya - shell: bash -l {0} - run: | - cd examples/des_y1_3x2pt - cobaya-run cobaya_evaluate.yaml - cobaya-run cobaya_evaluate_PT.yaml - - name: Running example - des-y1-3x2pt - NumCosmo - shell: bash -l {0} - run: | - cd examples/des_y1_3x2pt - numcosmo from-cosmosis des_y1_3x2pt.ini --matter-ps eisenstein_hu --nonlin-matter-ps halofit - numcosmo run test des_y1_3x2pt.yaml - numcosmo from-cosmosis des_y1_3x2pt_PT.ini --matter-ps eisenstein_hu --nonlin-matter-ps halofit - numcosmo run test des_y1_3x2pt_PT.yaml - - name: Running example - srd_sn - cosmosis - shell: bash -l {0} - run: | - cd examples/srd_sn - cosmosis sn_srd.ini - cosmosis sn_only.ini - - name: Running example - srd_sn - NumCosmo - shell: bash -l {0} - run: | - cd examples/srd_sn - numcosmo from-cosmosis sn_srd.ini - numcosmo run test sn_srd.yaml - numcosmo from-cosmosis sn_only.ini - numcosmo run test sn_only.yaml - - name: Running example - cluster_number_counts - cosmosis - shell: bash -l {0} - run: | - cd examples/cluster_number_counts - python generate_rich_mean_mass_sacc_data.py - cosmosis cluster_counts_redshift_richness.ini - cosmosis cluster_mean_mass_redshift_richness.ini - cosmosis cluster_counts_mean_mass_redshift_richness.ini - - name: Running example - cluster_number_counts - NumCosmo - shell: bash -l {0} - run: | - cd examples/cluster_number_counts - numcosmo from-cosmosis cluster_counts_redshift_richness.ini --matter-ps eisenstein_hu - numcosmo run test cluster_counts_redshift_richness.yaml - numcosmo from-cosmosis cluster_mean_mass_redshift_richness.ini --matter-ps eisenstein_hu - numcosmo run test cluster_mean_mass_redshift_richness.yaml - numcosmo from-cosmosis cluster_counts_mean_mass_redshift_richness.ini --matter-ps eisenstein_hu - numcosmo run test cluster_counts_mean_mass_redshift_richness.yaml - + run: python -m pytest -vv -s --integration tests/integration --cov firecrown --cov-report xml --cov-append - name: Upload coverage reports to Codecov if: ${{ (matrix.os == 'ubuntu') && (matrix.python-version == '3.11') }} uses: codecov/codecov-action@v4 From 92ddc4f4a6e1be81321a630e0c594f697a0a3303 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 7 Feb 2024 14:56:16 -0800 Subject: [PATCH 5/5] Code quality changes --- tests/conftest.py | 2 ++ tests/integration/test_cluster_number_counts.py | 2 ++ tests/integration/test_cosmic_shear.py | 2 ++ tests/integration/test_des_y1_3x2pt.py | 2 ++ tests/integration/test_srd_sn.py | 2 ++ 5 files changed, 10 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index fe71627f..10b81c73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,6 +55,8 @@ def pytest_collection_modifyitems(config, items): def _skip_tests(items, keyword, reason): + """Helper method to skip tests based on a marker name.""" + tests_to_skip = pytest.mark.skip(reason=reason) for item in items: if keyword in item.keywords: diff --git a/tests/integration/test_cluster_number_counts.py b/tests/integration/test_cluster_number_counts.py index b6ad1da3..0e9259b0 100644 --- a/tests/integration/test_cluster_number_counts.py +++ b/tests/integration/test_cluster_number_counts.py @@ -1,3 +1,5 @@ +"""Integration tests for cluster number counts""" + import subprocess import pytest diff --git a/tests/integration/test_cosmic_shear.py b/tests/integration/test_cosmic_shear.py index a579941f..153ce5ff 100644 --- a/tests/integration/test_cosmic_shear.py +++ b/tests/integration/test_cosmic_shear.py @@ -1,3 +1,5 @@ +"""Integration tests for cosmic shear""" + import subprocess import pytest diff --git a/tests/integration/test_des_y1_3x2pt.py b/tests/integration/test_des_y1_3x2pt.py index 0f01f90e..75c3a7a8 100644 --- a/tests/integration/test_des_y1_3x2pt.py +++ b/tests/integration/test_des_y1_3x2pt.py @@ -1,3 +1,5 @@ +"""Integration tests for the DES Y1 3x2pt analysis""" + import subprocess import pytest diff --git a/tests/integration/test_srd_sn.py b/tests/integration/test_srd_sn.py index 36e3d55e..b5304eb1 100644 --- a/tests/integration/test_srd_sn.py +++ b/tests/integration/test_srd_sn.py @@ -1,3 +1,5 @@ +"""Integration tests for SRD SN""" + import subprocess import pytest