Migrating examples to integration tests to be run by pytest

.github/workflows/ci.yml

@@ -95,74 +95,12 @@ jobs:
           pylint firecrown
           pylint --rcfile tests/pylintrc  tests
           pylint --rcfile firecrown/models/pylintrc firecrown/models
-      - name: Running pytest
+      - name: Running unit tests
         shell: bash -l {0}
         run: python -m pytest -vv --runslow --cov firecrown --cov-report xml
-      - name: Running example - cosmic-shear - cosmosis
+      - name: Running integration tests
         shell: bash -l {0}
-        run: |
-          cd examples/cosmicshear
-          python generate_cosmicshear_data.py
-          cosmosis cosmicshear.ini
-      - name: Running example - cosmic-shear - NumCosmo
-        shell: bash -l {0}
-        run: |
-          cd examples/cosmicshear
-          numcosmo from-cosmosis cosmicshear.ini --matter-ps eisenstein_hu --nonlin-matter-ps halofit 
-          numcosmo run test cosmicshear.yaml
-      - name: Running example - des-y1-3x2pt - cosmosis
-        shell: bash -l {0}
-        run: |
-          cd examples/des_y1_3x2pt
-          cosmosis des_y1_3x2pt.ini
-          cosmosis des_y1_3x2pt_PT.ini
-      - name: Running example - des-y1-3x2pt - cobaya
-        shell: bash -l {0}
-        run: |
-          cd examples/des_y1_3x2pt
-          cobaya-run cobaya_evaluate.yaml
-          cobaya-run cobaya_evaluate_PT.yaml
-      - name: Running example - des-y1-3x2pt - NumCosmo
-        shell: bash -l {0}
-        run: |
-          cd examples/des_y1_3x2pt
-          numcosmo from-cosmosis des_y1_3x2pt.ini --matter-ps eisenstein_hu --nonlin-matter-ps halofit 
-          numcosmo run test des_y1_3x2pt.yaml
-          numcosmo from-cosmosis des_y1_3x2pt_PT.ini --matter-ps eisenstein_hu --nonlin-matter-ps halofit 
-          numcosmo run test des_y1_3x2pt_PT.yaml
-      - name: Running example - srd_sn - cosmosis
-        shell: bash -l {0}
-        run: |
-          cd examples/srd_sn
-          cosmosis sn_srd.ini
-          cosmosis sn_only.ini
-      - name: Running example - srd_sn - NumCosmo
-        shell: bash -l {0}
-        run: |
-          cd examples/srd_sn
-          numcosmo from-cosmosis sn_srd.ini
-          numcosmo run test sn_srd.yaml
-          numcosmo from-cosmosis sn_only.ini
-          numcosmo run test sn_only.yaml
-      - name: Running example - cluster_number_counts - cosmosis
-        shell: bash -l {0}
-        run: |  
-          cd examples/cluster_number_counts
-          python generate_rich_mean_mass_sacc_data.py
-          cosmosis cluster_counts_redshift_richness.ini
-          cosmosis cluster_mean_mass_redshift_richness.ini
-          cosmosis cluster_counts_mean_mass_redshift_richness.ini
-      - name: Running example - cluster_number_counts - NumCosmo
-        shell: bash -l {0}
-        run: |
-          cd examples/cluster_number_counts
-          numcosmo from-cosmosis cluster_counts_redshift_richness.ini --matter-ps eisenstein_hu
-          numcosmo run test cluster_counts_redshift_richness.yaml
-          numcosmo from-cosmosis cluster_mean_mass_redshift_richness.ini --matter-ps eisenstein_hu
-          numcosmo run test cluster_mean_mass_redshift_richness.yaml
-          numcosmo from-cosmosis cluster_counts_mean_mass_redshift_richness.ini --matter-ps eisenstein_hu
-          numcosmo run test cluster_counts_mean_mass_redshift_richness.yaml
-
+        run: python -m pytest -vv -s --integration tests/integration --cov firecrown --cov-report xml --cov-append
       - name: Upload coverage reports to Codecov
         if: ${{ (matrix.os == 'ubuntu') && (matrix.python-version == '3.11') }}
         uses: codecov/codecov-action@v4

tests/conftest.py

@@ -21,10 +21,17 @@ def pytest_addoption(parser):
     """Add handling of firecrown-specific options for the pytest test runner.
 
     --runslow: used to run tests marked as slow, which are otherwise not run.
+    --integration: used to run only integration tests, which are otherwise not run.
     """
     parser.addoption(
         "--runslow", action="store_true", default=False, help="run slow tests"
     )
+    parser.addoption(
+        "--integration",
+        action="store_true",
+        default=False,
+        help="run integration tests",
+    )
 
 
 def pytest_configure(config):
@@ -39,13 +46,21 @@ def pytest_configure(config):
 
 def pytest_collection_modifyitems(config, items):
     """Apply our special markers and option handling for pytest."""
-    if config.getoption("--runslow"):
-        # --runslow given in cli: do not skip slow tests
-        return
-    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+
+    if not config.getoption("--integration"):
+        _skip_tests(items, "integration", "need --integration option to run")
+
+    if not config.getoption("--runslow"):
+        _skip_tests(items, "slow", "need --runslow option to run")
+
+
+def _skip_tests(items, keyword, reason):
+    """Helper method to skip tests based on a marker name."""
+
+    tests_to_skip = pytest.mark.skip(reason=reason)
     for item in items:
-        if "slow" in item.keywords:
-            item.add_marker(skip_slow)
+        if keyword in item.keywords:
+            item.add_marker(tests_to_skip)
 
 
 # Fixtures

tests/integration/test_cluster_number_counts.py

@@ -0,0 +1,57 @@
+"""Integration tests for cluster number counts"""
+
+import subprocess
+import pytest
+
+
+@pytest.mark.integration
+def test_cluster_number_counts_cosmosis():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/cluster_number_counts
+                python generate_rich_mean_mass_sacc_data.py
+                cosmosis cluster_counts_redshift_richness.ini
+                cosmosis cluster_mean_mass_redshift_richness.ini
+                cosmosis cluster_counts_mean_mass_redshift_richness.ini
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)
+
+
+@pytest.mark.integration
+def test_cluster_number_counts_numcosmo():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/cluster_number_counts
+                numcosmo from-cosmosis cluster_counts_redshift_richness.ini\\
+                    --matter-ps eisenstein_hu
+                numcosmo run test cluster_counts_redshift_richness.yaml
+                numcosmo from-cosmosis cluster_mean_mass_redshift_richness.ini\\
+                    --matter-ps eisenstein_hu
+                numcosmo run test cluster_mean_mass_redshift_richness.yaml
+                numcosmo from-cosmosis cluster_counts_mean_mass_redshift_richness.ini\\
+                    --matter-ps eisenstein_hu
+                numcosmo run test cluster_counts_mean_mass_redshift_richness.yaml
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)

tests/integration/test_cosmic_shear.py

@@ -0,0 +1,50 @@
+"""Integration tests for cosmic shear"""
+
+import subprocess
+import pytest
+
+
+@pytest.mark.integration
+def test_cosmic_shear_cosmosis():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/cosmicshear
+                python generate_cosmicshear_data.py
+                cosmosis cosmicshear.ini
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)
+
+
+@pytest.mark.integration
+def test_cosmic_shear_numcosmo():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/cosmicshear
+                numcosmo from-cosmosis cosmicshear.ini \\
+                    --matter-ps eisenstein_hu \\
+                    --nonlin-matter-ps halofit
+                numcosmo run test cosmicshear.yaml
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)

tests/integration/test_des_y1_3x2pt.py

@@ -0,0 +1,74 @@
+"""Integration tests for the DES Y1 3x2pt analysis"""
+
+import subprocess
+import pytest
+
+
+@pytest.mark.integration
+def test_des_y1_3x2pt_cosmosis():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/des_y1_3x2pt
+                cosmosis des_y1_3x2pt.ini
+                cosmosis des_y1_3x2pt_PT.ini
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)
+
+
+@pytest.mark.integration
+def test_des_y1_3x2pt_numcosmo():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/des_y1_3x2pt
+                numcosmo from-cosmosis des_y1_3x2pt.ini --matter-ps eisenstein_hu\\
+                    --nonlin-matter-ps halofit
+                numcosmo run test des_y1_3x2pt.yaml
+                numcosmo from-cosmosis des_y1_3x2pt_PT.ini --matter-ps eisenstein_hu\\
+                    --nonlin-matter-ps halofit
+                numcosmo run test des_y1_3x2pt_PT.yaml
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)
+
+
+@pytest.mark.integration
+def test_des_y1_3x2pt_cobaya():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/des_y1_3x2pt
+                cobaya-run cobaya_evaluate.yaml
+                cobaya-run cobaya_evaluate_PT.yaml
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)

tests/integration/test_srd_sn.py

@@ -0,0 +1,50 @@
+"""Integration tests for SRD SN"""
+
+import subprocess
+import pytest
+
+
+@pytest.mark.integration
+def test_srd_sn_cosmosis():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/srd_sn
+                cosmosis sn_srd.ini
+                cosmosis sn_only.ini
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)
+
+
+@pytest.mark.integration
+def test_srd_sn_numcosmo():
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            """
+                set -e
+                cd examples/srd_sn
+                numcosmo from-cosmosis sn_srd.ini
+                numcosmo run test sn_srd.yaml
+                numcosmo from-cosmosis sn_only.ini
+                numcosmo run test sn_only.yaml
+            """,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.returncode == 0
+    print(result.stdout)