Merge pull request #3811 from yt-project/cphyc/issue3785

BUG: bring back support to group folder structure in RAMSES

nose_unit.cfg

@@ -6,5 +6,5 @@ nologcapture=1
 verbosity=2
 where=yt
 with-timer=1
-ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py|test_eps_writer.py|test_registration.py|test_invalid_origin.py|test_outputs_pytest\.py|test_normal_plot_api\.py|test_load_archive\.py|test_stream_particles\.py)
+ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py|test_eps_writer.py|test_registration.py|test_invalid_origin.py|test_outputs_pytest\.py|test_normal_plot_api\.py|test_load_archive\.py|test_stream_particles\.py|test_file_sanitizer\.py)
 exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF

tests/tests.yaml

@@ -199,6 +199,7 @@ other_tests:
      - "--ignore-files=test_load_archive\\.py"
      - "--ignore-files=test_outputs_pytest\\.py"
      - "--ignore-files=test_normal_plot_api\\.py"
+     - "--ignore-file=test_file_sanitizer\\.py"
      - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF"
      - "--exclude-test=yt.frontends.adaptahop.tests.test_outputs"
      - "--exclude-test=yt.frontends.stream.tests.test_stream_particles.test_stream_non_cartesian_particles"

yt/frontends/ramses/data_structures.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 from itertools import product
 from pathlib import Path
+from typing import Optional, Tuple
 
 import numpy as np
 
@@ -57,21 +58,16 @@ def __init__(self, filename):
 
         # Loop on both the functions and the tested paths
         for path, check_fun in product(paths_to_try, check_functions):
-            ok, output_dir, info_fname = check_fun(path)
+            ok, output_dir, group_dir, info_fname = check_fun(path)
             if ok:
                 break
 
         # Early exit if the ok flag is False
         if not ok:
             return
 
-        # Special case when organized in groups
-        if output_dir.name == "group_00001":
-            self.root_folder = output_dir.parent
-            self.group_name = output_dir.name
-        else:
-            self.root_folder = output_dir
-            self.group_name = None
+        self.root_folder = output_dir
+        self.group_name = group_dir.name if group_dir else None
         self.info_fname = info_fname
 
     def validate(self) -> None:
@@ -80,18 +76,19 @@ def validate(self) -> None:
         filename: str = os.path.expanduser(self.original_filename)
 
         if not os.path.exists(filename):
-            raise FileNotFoundError(f"No such file or directory {filename!r}")
+            raise FileNotFoundError(rf"No such file or directory '{filename!s}'")
         if self.root_folder is None:
             raise ValueError(
-                f"Could not determine output directory from {filename!r}\n"
-                f"Expected a directory name of form {OUTPUT_DIR_EXP!r}"
+                f"Could not determine output directory from '{filename!s}'\n"
+                f"Expected a directory name of form {OUTPUT_DIR_EXP!r} "
+                "containing an info_*.txt file and amr_* files."
             )
 
         # This last case is (erroneously ?) marked as unreachable by mypy
         # If/when this bug is fixed upstream, mypy will warn that the unused
         # 'type: ignore' comment can be removed
         if self.info_fname is None:  # type: ignore [unreachable]
-            raise ValueError(f"Failed to detect info file from {filename!r}")
+            raise ValueError(f"Failed to detect info file from '{filename!s}'")
 
     @property
     def is_valid(self) -> bool:
@@ -110,37 +107,70 @@ def check_standard_files(folder, iout):
         ok &= (folder / f"info_{iout}.txt").is_file()
         return ok
 
+    @staticmethod
+    def _match_output_and_group(
+        path: Path,
+    ) -> Tuple[Path, Optional[Path], Optional[str]]:
+
+        # Make sure we work with a directory of the form `output_XXXXX`
+        for p in (path, path.parent):
+            match = OUTPUT_DIR_RE.match(p.name)
+
+            if match:
+                path = p
+                break
+
+        if match is None:
+            return path, None, None
+
+        iout = match.group(1)
+
+        # See whether a folder named `group_YYYYY` exists
+        group_dir = path / "group_00001"
+        if group_dir.is_dir():
+            return path, group_dir, iout
+        else:
+            return path, None, iout
+
     @classmethod
-    def test_with_folder_name(cls, output_dir):
-        iout_match = OUTPUT_DIR_RE.match(output_dir.name)
-        ok = output_dir.is_dir() and iout_match is not None
+    def test_with_folder_name(
+        cls, output_dir: Path
+    ) -> Tuple[bool, Optional[Path], Optional[Path], Optional[Path]]:
+        output_dir, group_dir, iout = cls._match_output_and_group(output_dir)
+        ok = output_dir.is_dir() and iout is not None
+
+        info_fname: Optional[Path]
+
         if ok:
-            iout = iout_match.group(2)
-            ok &= cls.check_standard_files(output_dir, iout)
-            info_fname = output_dir / f"info_{iout}.txt"
+            parent_dir = group_dir or output_dir
+            ok &= cls.check_standard_files(parent_dir, iout)
+            info_fname = parent_dir / f"info_{iout}.txt"
         else:
-            output_dir, info_fname = None, None
+            info_fname = None
 
-        return ok, output_dir, info_fname
+        return ok, output_dir, group_dir, info_fname
 
     @classmethod
-    def test_with_standard_file(cls, filename):
-        iout_match = OUTPUT_DIR_RE.match(filename.parent.name)
+    def test_with_standard_file(
+        cls, filename: Path
+    ) -> Tuple[bool, Optional[Path], Optional[Path], Optional[Path]]:
+        output_dir, group_dir, iout = cls._match_output_and_group(filename.parent)
         ok = (
             filename.is_file()
             and STANDARD_FILE_RE.match(filename.name) is not None
-            and iout_match is not None
+            and iout is not None
         )
 
+        info_fname: Optional[Path]
+
         if ok:
-            iout = iout_match.group(2)
-            ok &= cls.check_standard_files(filename.parent, iout)
-            output_dir = filename.parent
-            info_fname = output_dir / f"info_{iout}.txt"
+            parent_dir = group_dir or output_dir
+            ok &= cls.check_standard_files(parent_dir, iout)
+            info_fname = parent_dir / f"info_{iout}.txt"
         else:
-            output_dir, info_fname = None, None
+            info_fname = None
 
-        return ok, output_dir, info_fname
+        return ok, output_dir, group_dir, info_fname
 
 
 class RAMSESDomainFile:

yt/frontends/ramses/definitions.py

@@ -53,7 +53,7 @@ def ramses_header(hvals):
 # on the left hand side
 VAR_DESC_RE = re.compile(r"\s*([^\s]+),\s*([^\s]+),\s*([^\s]+)")
 
-OUTPUT_DIR_EXP = r"(output|group)_(\d{5})"
+OUTPUT_DIR_EXP = r"output_(\d{5})"
 OUTPUT_DIR_RE = re.compile(OUTPUT_DIR_EXP)
 STANDARD_FILE_RE = re.compile(r"((amr|hydro|part|grav)_\d{5}\.out\d{5}|info_\d{5}.txt)")
 

yt/frontends/ramses/tests/test_file_sanitizer.py

@@ -0,0 +1,108 @@
+import re
+import tempfile
+from collections import namedtuple
+from itertools import chain
+from pathlib import Path
+
+import pytest
+
+from yt.frontends.ramses.data_structures import RAMSESFileSanitizer
+
+PathTuple = namedtuple(
+    "PathTuple", ("output_dir", "group_dir_name", "info_file", "paths_to_try")
+)
+
+
+def generate_paths(create):
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_dir = Path(tmpdir) / "output_00123"
+        output_dir.mkdir()
+        amr_file = output_dir / "amr_00123.out00001"
+        if create:
+            amr_file.touch()
+        info_file = output_dir / "info_00123.txt"
+        if create:
+            info_file.touch()
+
+        # Test with regular structure
+        output_dir2 = Path(tmpdir) / "output_00124"
+        output_dir2.mkdir()
+
+        group_dir2 = output_dir2 / "group_00001"
+        group_dir2.mkdir()
+        info_file2 = group_dir2 / "info_00124.txt"
+        if create:
+            info_file2.touch()
+        amr_file2 = group_dir2 / "amr_00124.out00001"
+        if create:
+            amr_file2.touch()
+
+        yield (
+            PathTuple(
+                output_dir=output_dir,
+                group_dir_name=None,
+                info_file=info_file,
+                paths_to_try=(output_dir, info_file, amr_file),
+            ),
+            PathTuple(
+                output_dir=output_dir2,
+                group_dir_name=group_dir2.name,
+                info_file=info_file2,
+                paths_to_try=(output_dir2, info_file2, group_dir2, amr_file2),
+            ),
+        )
+
+
+@pytest.fixture
+def valid_path_tuples():
+    yield from generate_paths(create=True)
+
+
+@pytest.fixture
+def invalid_path_tuples():
+    yield from generate_paths(create=False)
+
+
+def test_valid_sanitizing(valid_path_tuples):
+    for answer in valid_path_tuples:
+        for path in answer.paths_to_try:
+            sanitizer = RAMSESFileSanitizer(path)
+            sanitizer.validate()
+
+            assert sanitizer.root_folder == answer.output_dir
+            assert sanitizer.group_name == answer.group_dir_name
+            assert sanitizer.info_fname == answer.info_file
+
+
+def test_invalid_sanitizing(valid_path_tuples, invalid_path_tuples):
+    for path in chain(*(pt.paths_to_try for pt in invalid_path_tuples)):
+        sanitizer = RAMSESFileSanitizer(path)
+
+        if path.exists():
+            expected_error = ValueError
+            expected_error_message = (
+                "Could not determine output directory from '.*'\n"
+                "Expected a directory name of form .* "
+                "containing an info_\\*.txt file and amr_\\* files."
+            )
+        else:
+            expected_error = FileNotFoundError
+            expected_error_message = re.escape(
+                f"No such file or directory '{str(path)}'"
+            )
+
+        with pytest.raises(expected_error, match=expected_error_message):
+            sanitizer.validate()
+
+    for path in chain(*(pt.paths_to_try for pt in valid_path_tuples)):
+        expected_error_message = re.escape(
+            f"No such file or directory '{str(path/'does_not_exist.txt')}'"
+        )
+        sanitizer = RAMSESFileSanitizer(path / "does_not_exist.txt")
+        with pytest.raises(FileNotFoundError, match=expected_error_message):
+            sanitizer.validate()
+
+    expected_error_message = "No such file or directory '.*'"
+    sanitizer = RAMSESFileSanitizer(Path("this") / "does" / "not" / "exist")
+    with pytest.raises(FileNotFoundError, match=expected_error_message):
+        sanitizer.validate()