fix: Support nested namespace packages

src/griffe/finder.py

@@ -11,9 +11,11 @@
 
 from griffe.dataclasses import Module
 from griffe.exceptions import UnhandledEditablesModuleError
+from griffe.logger import get_logger
 
 NamePartsType = Tuple[str, ...]
 NamePartsAndPathType = Tuple[NamePartsType, Path]
+logger = get_logger(__name__)
 
 
 class Package:
@@ -156,19 +158,31 @@ def find_package(self, module_name: str) -> Package | NamespacePackage:  # noqa:
 
         raise ModuleNotFoundError(module_name)
 
-    def iter_submodules(self, path: Path | list[Path]) -> Iterator[NamePartsAndPathType]:  # noqa: WPS231,WPS234
+    def iter_submodules(  # noqa: WPS231,WPS234
+        self,
+        path: Path | list[Path],
+        seen: set | None = None,
+    ) -> Iterator[NamePartsAndPathType]:
         """Iterate on a module's submodules, if any.
 
         Parameters:
             path: The module path.
+            seen: If not none, this set is used to skip some files.
+                The goal is to replicate the behavior of Python by
+                only using the first packages (with `__init__` modules)
+                of the same name found in different namespace packages.
+                As soon as we find an `__init__` module, we add its parent
+                path to the `seen` set, which will be reused when scanning
+                the next namespace packages.
 
         Yields:
             name_parts (tuple[str, ...]): The parts of a submodule name.
             filepath (Path): A submodule filepath.
         """
         if isinstance(path, list):
+            seen = set()
             for path_elem in path:
-                yield from self.iter_submodules(path_elem)
+                yield from self.iter_submodules(path_elem, seen)
             return
 
         if path.stem == "__init__":
@@ -179,8 +193,12 @@ def iter_submodules(self, path: Path | list[Path]) -> Iterator[NamePartsAndPathT
         elif path.suffix in self.extensions_set:
             return
 
+        skip = set(seen) if seen else set()
         for subpath in self._filter_py_modules(path):
             rel_subpath = subpath.relative_to(path)
+            if rel_subpath.parent in skip:
+                logger.debug(f"Skip {subpath}, another module took precedence")
+                continue
             py_file = rel_subpath.suffix == ".py"
             stem = rel_subpath.stem
             if not py_file:
@@ -194,6 +212,8 @@ def iter_submodules(self, path: Path | list[Path]) -> Iterator[NamePartsAndPathT
                 if len(rel_subpath.parts) == 1:
                     continue
                 yield rel_subpath.parts[:-1], subpath
+                if seen is not None:
+                    seen.add(rel_subpath.parent)
             elif py_file:
                 yield rel_subpath.with_suffix("").parts, subpath
             else:

src/griffe/loader.py

@@ -428,21 +428,22 @@ def _load_submodules(self, module: Module) -> None:
 
     def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Path) -> None:
         try:
-            member_parent = self._member_parent(module, subparts, subpath)
+            parent_module = self._get_or_create_parent_module(module, subparts, subpath)
         except UnimportableModuleError as error:
             # TODO: maybe add option to still load them
             # TODO: maybe increase level to WARNING
             logger.debug(f"{error}. Missing __init__ module?")
             return
+        submodule_name = subparts[-1]
+        if "." in submodule_name:
+            logger.debug(f"Skip {subpath}, dots in filenames are not supported")
+            return
         try:
-            member_parent[subparts[-1]] = self._load_module(
-                subparts[-1], subpath, submodules=False, parent=member_parent
+            parent_module[submodule_name] = self._load_module(
+                submodule_name, subpath, submodules=False, parent=parent_module
             )
         except LoadingError as error:  # noqa: WPS440
             logger.debug(str(error))
-        except KeyError:
-            if "." in subparts[-1]:
-                logger.debug(f"Skip {subpath}, dots in filenames are not supported")
 
     def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Module:
         return Module(
@@ -492,23 +493,35 @@ def _inspect_module(self, module_name: str, filepath: Path | None = None, parent
         self._time_stats["time_spent_inspecting"] += elapsed.microseconds
         return module
 
-    def _member_parent(self, module: Module, subparts: tuple[str, ...], subpath: Path) -> Module:
+    def _get_or_create_parent_module(  # noqa: WPS231
+        self,
+        module: Module,
+        subparts: tuple[str, ...],
+        subpath: Path,
+    ) -> Module:
         parent_parts = subparts[:-1]
-        try:
-            return module[parent_parts]
-        except ValueError:
+        if not parent_parts:
             return module
-        except KeyError:
-            if module.is_namespace_package or module.is_namespace_subpackage:
-                member_parent = Module(
-                    subparts[0],
-                    filepath=subpath.parent,
-                    lines_collection=self.lines_collection,
-                    modules_collection=self.modules_collection,
-                )
-                module[parent_parts] = member_parent
-                return member_parent
-        raise UnimportableModuleError(f"{subpath} is not importable")
+        parent_module = module
+        parents = list(subpath.parents)
+        if subpath.stem == "__init__":
+            parents.pop(0)
+        for parent_offset, parent_part in enumerate(parent_parts, 2):
+            module_filepath = parents[len(subparts) - parent_offset]
+            try:
+                parent_module = parent_module[parent_part]
+            except KeyError:
+                if parent_module.is_namespace_package or parent_module.is_namespace_subpackage:
+                    next_parent_module = self._create_module(parent_part, [module_filepath])
+                    parent_module[parent_part] = next_parent_module
+                    parent_module = next_parent_module
+                else:
+                    raise UnimportableModuleError(f"Skip {subpath}, it is not importable")
+            else:
+                if parent_module.is_namespace_package or parent_module.is_namespace_subpackage:
+                    if module_filepath not in parent_module.filepath:  # type: ignore[operator]
+                        parent_module.filepath.append(module_filepath)  # type: ignore[union-attr]
+        return parent_module
 
     def _expand_wildcard(self, wildcard_obj: Alias) -> list[tuple[Object | Alias, int | None, int | None]]:
         module = self.modules_collection[wildcard_obj.wildcard]  # type: ignore[index]  # we know it's a wildcard

tests/helpers.py

@@ -39,15 +39,17 @@ def temporary_pyfile(code: str) -> Iterator[tuple[str, Path]]:
 
 
 @contextmanager
-def temporary_pypackage(package: str, modules: list[str] | None = None) -> Iterator[TmpPackage]:
+def temporary_pypackage(package: str, modules: list[str] | None = None, init: bool = True) -> Iterator[TmpPackage]:
     """Create a module.py file containing the given code in a temporary directory.
 
     Parameters:
         package: The package name. Example: `"a"` gives
             a package named `a`, while `"a/b"` gives a namespace package
             named `a` with a package inside named `b`.
+            If `init` is false, then `b` is also a namespace package.
         modules: Additional modules to create in the package,
             like '["b.py", "c/d.py", "e/f"]`.
+        init: Whether to create an `__init__` module in the leaf package.
 
     Yields:
         tmp_dir: The temporary directory containing the package.
@@ -61,7 +63,8 @@ def temporary_pypackage(package: str, modules: list[str] | None = None) -> Itera
         package_name = ".".join(Path(package).parts)
         package_path = tmpdirpath / package
         package_path.mkdir(**mkdir_kwargs)
-        package_path.joinpath("__init__.py").touch()
+        if init:
+            package_path.joinpath("__init__.py").touch()
         for module in modules:
             current_path = package_path
             for part in Path(module).parts:

tests/test_loader.py

@@ -264,3 +264,71 @@ def test_skip_modules_with_dots_in_filename(caplog):
         loader = GriffeLoader(search_paths=[tmp_folder.tmpdir])
         loader.load_module("package")
     assert any("gunicorn.conf.py" in record.message and record.levelname == "DEBUG" for record in caplog.records)
+
+
+def test_nested_namespace_packages():
+    """Load a deeply nested namespace package."""
+    with temporary_pypackage("a/b/c/d", ["mod.py"]) as tmp_folder:
+        loader = GriffeLoader(search_paths=[tmp_folder.tmpdir])
+        a_package = loader.load_module("a")
+        assert "b" in a_package.members
+        b_package = a_package.members["b"]
+        assert "c" in b_package.members
+        c_package = b_package.members["c"]
+        assert "d" in c_package.members
+        d_package = c_package.members["d"]
+        assert "mod" in d_package.members
+
+
+def test_multiple_nested_namespace_packages():
+    """Load a deeply nested namespace package appearing in several places."""
+    with temporary_pypackage("a/b/c/d", ["mod1.py"], init=False) as tmp_ns1:
+        with temporary_pypackage("a/b/c/d", ["mod2.py"], init=False) as tmp_ns2:
+            with temporary_pypackage("a/b/c/d", ["mod3.py"], init=False) as tmp_ns3:
+                tmp_namespace_pkgs = [tmp_ns.tmpdir for tmp_ns in (tmp_ns1, tmp_ns2, tmp_ns3)]
+                loader = GriffeLoader(search_paths=tmp_namespace_pkgs)
+
+                a_package = loader.load_module("a")
+                for tmp_ns in tmp_namespace_pkgs:
+                    assert tmp_ns.joinpath("a") in a_package.filepath
+                assert "b" in a_package.members
+
+                b_package = a_package.members["b"]
+                for tmp_ns in tmp_namespace_pkgs:  # noqa: WPS440
+                    assert tmp_ns.joinpath("a/b") in b_package.filepath
+                assert "c" in b_package.members
+
+                c_package = b_package.members["c"]
+                for tmp_ns in tmp_namespace_pkgs:  # noqa: WPS440
+                    assert tmp_ns.joinpath("a/b/c") in c_package.filepath
+                assert "d" in c_package.members
+
+                d_package = c_package.members["d"]
+                for tmp_ns in tmp_namespace_pkgs:  # noqa: WPS440
+                    assert tmp_ns.joinpath("a/b/c/d") in d_package.filepath
+                assert "mod1" in d_package.members
+                assert "mod2" in d_package.members
+                assert "mod3" in d_package.members
+
+
+def test_stop_at_first_package_inside_namespace_package():
+    """Stop loading similar paths once we found a non-namespace package."""
+    with temporary_pypackage("a/b/c/d", ["mod1.py"], init=True) as tmp_ns1:
+        with temporary_pypackage("a/b/c/d", ["mod2.py"], init=True) as tmp_ns2:
+            tmp_namespace_pkgs = [tmp_ns.tmpdir for tmp_ns in (tmp_ns1, tmp_ns2)]
+            loader = GriffeLoader(search_paths=tmp_namespace_pkgs)
+
+            a_package = loader.load_module("a")
+            assert "b" in a_package.members
+
+            b_package = a_package.members["b"]
+            assert "c" in b_package.members
+
+            c_package = b_package.members["c"]
+            assert "d" in c_package.members
+
+            d_package = c_package.members["d"]
+            assert d_package.is_subpackage
+            assert d_package.filepath == tmp_ns1.tmpdir.joinpath("a/b/c/d/__init__.py")
+            assert "mod1" in d_package.members
+            assert "mod2" not in d_package.members