feat: Support editable installs dynamically exposing modules from oth…

…er directories

Issue #229: #229

src/griffe/finder.py

@@ -17,6 +17,7 @@
 import os
 import re
 import sys
+from collections import defaultdict
 from contextlib import suppress
 from dataclasses import dataclass
 from pathlib import Path
@@ -95,6 +96,8 @@ def __init__(self, search_paths: Sequence[str | Path] | None = None) -> None:
         # Optimization: pre-compute Paths to relieve CPU when joining paths.
         self.search_paths = [path if isinstance(path, Path) else Path(path) for path in search_paths or sys.path]
         """The finder search paths."""
+
+        self._always_scan_for: dict[str, list[Path]] = defaultdict(list)
         self._extend_from_pth_files()
 
     def find_spec(
@@ -232,6 +235,8 @@ def iter_submodules(
         self,
         path: Path | list[Path],
         seen: set | None = None,
+        *,
+        additional: bool = True,
     ) -> Iterator[NamePartsAndPathType]:
         """Iterate on a module's submodules, if any.
 
@@ -250,9 +255,11 @@ def iter_submodules(
             filepath (Path): A submodule filepath.
         """
         if isinstance(path, list):
-            seen = set()
+            seen = seen if seen is not None else set()
             for path_elem in path:
-                yield from self.iter_submodules(path_elem, seen)
+                if path_elem not in seen:
+                    seen.add(path_elem)
+                    yield from self.iter_submodules(path_elem, seen, additional=additional)
             return
 
         if path.stem == "__init__":
@@ -287,6 +294,9 @@ def iter_submodules(
             else:
                 yield rel_subpath.with_name(stem).parts, subpath
 
+        if additional:
+            yield from self.iter_submodules(self._always_scan_for[path.stem], seen=seen, additional=False)
+
     def submodules(self, module: Module) -> list[NamePartsAndPathType]:
         """Return the list of a module's submodules.
 
@@ -330,7 +340,9 @@ def _extend_from_pth_files(self) -> None:
             for item in self._contents(path):
                 if item.suffix == ".pth":
                     for directory in _handle_pth_file(item):
-                        self._append_search_path(directory)
+                        if scan := directory.always_scan_for:
+                            self._always_scan_for[scan].append(directory.path.joinpath(scan))
+                        self._append_search_path(directory.path)
 
     def _filter_py_modules(self, path: Path) -> Iterator[Path]:
         for root, dirs, files in os.walk(path, topdown=True):
@@ -373,7 +385,13 @@ def _module_depth(name_parts_and_path: NamePartsAndPathType) -> int:
     return len(name_parts_and_path[0])
 
 
-def _handle_pth_file(path: Path) -> list[Path]:
+@dataclass
+class _SP:
+    path: Path
+    always_scan_for: str = ""
+
+
+def _handle_pth_file(path: Path) -> list[_SP]:
     # Support for .pth files pointing to directories.
     # From https://docs.python.org/3/library/site.html:
     # A path configuration file is a file whose name has the form name.pth
@@ -392,11 +410,11 @@ def _handle_pth_file(path: Path) -> list[Path]:
             with suppress(UnhandledEditableModuleError):
                 return _handle_editable_module(editable_module)
         if line and not line.startswith("#") and os.path.exists(line):
-            directories.append(Path(line))
+            directories.append(_SP(Path(line)))
     return directories
 
 
-def _handle_editable_module(path: Path) -> list[Path]:
+def _handle_editable_module(path: Path) -> list[_SP]:
     if _match_pattern(path.name, (*_editable_editables_patterns, *_editable_scikit_build_core_patterns)):
         # Support for how 'editables' write these files:
         # example line: `F.map_module('griffe', '/media/data/dev/griffe/src/griffe/__init__.py')`.
@@ -408,8 +426,8 @@ def _handle_editable_module(path: Path) -> list[Path]:
             raise UnhandledEditableModuleError(path) from error
         new_path = Path(editable_lines[-1].split("'")[3])
         if new_path.name.startswith("__init__"):
-            return [new_path.parent.parent]
-        return [new_path]
+            return [_SP(new_path.parent.parent)]
+        return [_SP(new_path)]
     if _match_pattern(path.name, _editable_setuptools_patterns):
         # Support for how 'setuptools' writes these files:
         # example line: `MAPPING = {'griffe': '/media/data/dev/griffe/src/griffe', 'briffe': '/media/data/dev/griffe/src/briffe'}`.
@@ -420,15 +438,23 @@ def _handle_editable_module(path: Path) -> list[Path]:
                 and isinstance(node.targets[0], ast.Name)
                 and node.targets[0].id == "MAPPING"
             ) and isinstance(node.value, ast.Dict):
-                return [Path(cst.value).parent for cst in node.value.values if isinstance(cst, ast.Constant)]
+                return [_SP(Path(cst.value).parent) for cst in node.value.values if isinstance(cst, ast.Constant)]
     if _match_pattern(path.name, _editable_meson_python_patterns):
         # Support for how 'meson-python' writes these files:
         # example line: `install({'package', 'module1'}, '/media/data/dev/griffe/build/cp311', ["path"], False)`.
         # Compiled modules then found in the cp311 folder, under src/package.
         parsed_module = ast.parse(path.read_text())
         for node in parsed_module.body:
-            if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) and node.value.func.id == "install":
-                return [Path(node.value.args[1].value, "src")]
+            if (
+                isinstance(node, ast.Expr)
+                and isinstance(node.value, ast.Call)
+                and isinstance(node.value.func, ast.Name)
+                and node.value.func.id == "install"
+                and isinstance(node.value.args[1], ast.Constant)
+            ):
+                build_path = Path(node.value.args[1].value, "src")
+                pkg_name = next(build_path.iterdir()).name
+                return [_SP(build_path, always_scan_for=pkg_name)]
     raise UnhandledEditableModuleError(path)
 
 

tests/test_finder.py

@@ -90,8 +90,8 @@ def test_pth_file_handling(tmp_path: Path) -> None:
             """,
         ),
     )
-    directories = _handle_pth_file(pth_file)
-    assert directories == [Path("tests")]
+    paths = [sp.path for sp in _handle_pth_file(pth_file)]
+    assert paths == [Path("tests")]
 
 
 def test_pth_file_handling_with_semi_colon(tmp_path: Path) -> None:
@@ -109,8 +109,8 @@ def test_pth_file_handling_with_semi_colon(tmp_path: Path) -> None:
             """,
         ),
     )
-    directories = _handle_pth_file(pth_file)
-    assert directories == [Path("tests")]
+    paths = [sp.path for sp in _handle_pth_file(pth_file)]
+    assert paths == [Path("tests")]
 
 
 @pytest.mark.parametrize("editable_file_name", ["__editables_whatever.py", "_editable_impl_whatever.py"])
@@ -122,7 +122,8 @@ def test_editables_file_handling(tmp_path: Path, editable_file_name: str) -> Non
     """
     pth_file = tmp_path / editable_file_name
     pth_file.write_text("hello\nF.map_module('griffe', 'src/griffe/__init__.py')")
-    assert _handle_editable_module(pth_file) == [Path("src")]
+    paths = [sp.path for sp in _handle_editable_module(pth_file)]
+    assert paths == [Path("src")]
 
 
 def test_setuptools_file_handling(tmp_path: Path) -> None:
@@ -133,7 +134,8 @@ def test_setuptools_file_handling(tmp_path: Path) -> None:
     """
     pth_file = tmp_path / "__editable__whatever.py"
     pth_file.write_text("hello\nMAPPING = {'griffe': 'src/griffe'}")
-    assert _handle_editable_module(pth_file) == [Path("src")]
+    paths = [sp.path for sp in _handle_editable_module(pth_file)]
+    assert paths == [Path("src")]
 
 
 def test_setuptools_file_handling_multiple_paths(tmp_path: Path) -> None:
@@ -146,7 +148,8 @@ def test_setuptools_file_handling_multiple_paths(tmp_path: Path) -> None:
     pth_file.write_text(
         "hello=1\nMAPPING = {\n'griffe':\n 'src1/griffe', 'briffe':'src2/briffe'}\ndef printer():\n  print(hello)",
     )
-    assert _handle_editable_module(pth_file) == [Path("src1"), Path("src2")]
+    paths = [sp.path for sp in _handle_editable_module(pth_file)]
+    assert paths == [Path("src1"), Path("src2")]
 
 
 def test_scikit_build_core_file_handling(tmp_path: Path) -> None:
@@ -163,7 +166,8 @@ def test_scikit_build_core_file_handling(tmp_path: Path) -> None:
     # in a location that Griffe won't be able to discover anyway
     # (they don't respect standard package or namespace package layouts,
     # and rely on dynamic meta path finder stuff)
-    assert _handle_editable_module(pth_file) == [Path("/path/to/whatever")]
+    paths = [sp.path for sp in _handle_editable_module(pth_file)]
+    assert paths == [Path("/path/to/whatever")]
 
 
 def test_meson_python_file_handling(tmp_path: Path) -> None:
@@ -174,9 +178,13 @@ def test_meson_python_file_handling(tmp_path: Path) -> None:
     """
     pth_file = tmp_path / "_whatever_editable_loader.py"
     pth_file.write_text(
-        "hello=1\ninstall({'whatever', 'hello'}, '/path/to/build', ['/tmp/ninja'], False)",
+        # the path in argument 2 suffixed with src must exist, so we pass '.'
+        "hello=1\ninstall({'griffe', 'hello'}, '.', ['/tmp/ninja'], False)",
     )
-    assert _handle_editable_module(pth_file) == [Path("/path/to/build/src")]
+    search_paths = _handle_editable_module(pth_file)
+    assert all(sp.always_scan_for == "griffe" for sp in search_paths)
+    paths = [sp.path for sp in search_paths]
+    assert paths == [Path("src")]
 
 
 @pytest.mark.parametrize(