fix: The file structure of stubs resembles the "package" path. (#106)

Closes #81 

### Summary of Changes

Changed how the file structure for stubs is build. Now we follow the
"package" path for the structure.

---------

Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com>
Co-authored-by: Lars Reimann <mail@larsreimann.com>

src/safeds_stubgen/__init__.py

@@ -1,2 +1,9 @@
 """Safe-DS stubs generator."""
+
 from __future__ import annotations
+
+from ._helpers import is_internal
+
+__all__ = [
+    "is_internal",
+]

src/safeds_stubgen/_helpers.py

@@ -0,0 +1,2 @@
+def is_internal(name: str) -> bool:
+    return name.startswith("_")

src/safeds_stubgen/api_analyzer/_ast_visitor.py

@@ -8,6 +8,7 @@
 import mypy.types as mp_types
 
 import safeds_stubgen.api_analyzer._types as sds_types
+from safeds_stubgen import is_internal
 from safeds_stubgen.docstring_parsing import ResultDocstring
 
 from ._api import (
@@ -1187,10 +1188,6 @@ def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module |
         return None
 
 
-def is_internal(name: str) -> bool:
-    return name.startswith("_")
-
-
 def result_name_generator() -> Generator:
     """Generate a name for callable type parameters starting from 'a' until 'zz'."""
     while True:

src/safeds_stubgen/api_analyzer/cli/_cli.py

@@ -6,7 +6,7 @@
 from typing import TYPE_CHECKING
 
 from safeds_stubgen.api_analyzer import get_api
-from safeds_stubgen.stubs_generator import generate_stubs
+from safeds_stubgen.stubs_generator import StubsStringGenerator, create_stub_files, generate_stub_data
 
 if TYPE_CHECKING:
     from safeds_stubgen.docstring_parsing import DocstringStyle
@@ -17,7 +17,7 @@ def cli() -> None:
     if args.verbose:
         logging.basicConfig(level=logging.INFO)
 
-    _run_api_command(args.src, args.out, args.docstyle, args.testrun, args.naming_convert)
+    _run_stub_generator(args.src, args.out, args.docstyle, args.testrun, args.naming_convert)
 
 
 def _get_args() -> argparse.Namespace:
@@ -64,15 +64,15 @@ def _get_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def _run_api_command(
+def _run_stub_generator(
     src_dir_path: Path,
     out_dir_path: Path,
     docstring_style: DocstringStyle,
     is_test_run: bool,
     convert_identifiers: bool,
 ) -> None:
     """
-    List the API of a package.
+    Create API data of a package and Safe-DS stub files.
 
     Parameters
     ----------
@@ -83,8 +83,14 @@ def _run_api_command(
     is_test_run : bool
         Set True if files in test directories should be parsed too.
     """
-    api = get_api(src_dir_path, docstring_style, is_test_run)
+    # Generate the API data
+    api = get_api(root=src_dir_path, docstring_style=docstring_style, is_test_run=is_test_run)
+    # Create an API file
     out_file_api = out_dir_path.joinpath(f"{src_dir_path.stem}__api.json")
     api.to_json_file(out_file_api)
 
-    generate_stubs(api, out_dir_path, convert_identifiers)
+    # Generate the stub data
+    stubs_generator = StubsStringGenerator(api=api, convert_identifiers=convert_identifiers)
+    stub_data = generate_stub_data(stubs_generator=stubs_generator, out_path=out_dir_path)
+    # Create the stub files
+    create_stub_files(stubs_generator=stubs_generator, stubs_data=stub_data, out_path=out_dir_path)

src/safeds_stubgen/stubs_generator/__init__.py

@@ -2,8 +2,11 @@
 
 from __future__ import annotations
 
-from ._generate_stubs import generate_stubs
+from ._generate_stubs import NamingConvention, StubsStringGenerator, create_stub_files, generate_stub_data
 
 __all__ = [
-    "generate_stubs",
+    "create_stub_files",
+    "generate_stub_data",
+    "NamingConvention",
+    "StubsStringGenerator",
 ]

src/safeds_stubgen/stubs_generator/_generate_stubs.py

@@ -6,6 +6,7 @@
 from types import NoneType
 from typing import TYPE_CHECKING
 
+from safeds_stubgen import is_internal
 from safeds_stubgen.api_analyzer import (
     API,
     Attribute,
@@ -34,33 +35,27 @@ class NamingConvention(IntEnum):
     SAFE_DS = 2
 
 
-def generate_stubs(api: API, out_path: Path, convert_identifiers: bool) -> None:
+def generate_stub_data(
+    stubs_generator: StubsStringGenerator,
+    out_path: Path,
+) -> list[tuple[Path, str, str]]:
     """Generate Safe-DS stubs.
 
-    Generates stub files from an API object and writes them to the out_path path.
+    Generates stub data from an API object.
 
     Parameters
     ----------
-    api
-        The API object from which the stubs
+    stubs_generator
+        The class for generating the stubs.
     out_path
         The path in which the stub files should be created. If no such path exists this function creates the directory
         files.
-    convert_identifiers
-        Set this True if the identifiers should be converted to Safe-DS standard (UpperCamelCase for classes and
-        camelCase for everything else).
-    """
-    naming_convention = NamingConvention.SAFE_DS if convert_identifiers else NamingConvention.PYTHON
-    stubs_generator = StubsStringGenerator(api, naming_convention)
-    stubs_data = _generate_stubs_data(api, out_path, stubs_generator)
-    _generate_stubs_files(stubs_data, out_path, stubs_generator, naming_convention)
 
-
-def _generate_stubs_data(
-    api: API,
-    out_path: Path,
-    stubs_generator: StubsStringGenerator,
-) -> list[tuple[Path, str, str]]:
+    Returns
+    -------
+    A list of tuples, which are 1. the path of the stub file, 2. the name of the stub file and 3. its content.
+    """
+    api = stubs_generator.api
     stubs_data: list[tuple[Path, str, str]] = []
     for module in api.modules.values():
         if module.name == "__init__":
@@ -69,7 +64,7 @@ def _generate_stubs_data(
         log_msg = f"Creating stub data for {module.id}"
         logging.info(log_msg)
 
-        module_text = stubs_generator(module)
+        module_text, package_info = stubs_generator(module)
 
         # Each text block we create ends with "\n", therefore, if there is only the package information
         # the file would look like this: "package path.to.myPackage\n" or this:
@@ -83,17 +78,17 @@ def _generate_stubs_data(
         if len(splitted_text) <= 2 or (len(splitted_text) == 3 and splitted_text[1].startswith("package ")):
             continue
 
-        module_dir = Path(out_path / module.id)
+        module_dir = Path(out_path / package_info.replace(".", "/"))
         stubs_data.append((module_dir, module.name, module_text))
     return stubs_data
 
 
-def _generate_stubs_files(
+def create_stub_files(
+    stubs_generator: StubsStringGenerator,
     stubs_data: list[tuple[Path, str, str]],
     out_path: Path,
-    stubs_generator: StubsStringGenerator,
-    naming_convention: NamingConvention,
 ) -> None:
+    naming_convention = stubs_generator.naming_convention
     for module_dir, module_name, module_text in stubs_data:
         log_msg = f"Creating stub file for {module_dir}"
         logging.info(log_msg)
@@ -102,7 +97,8 @@ def _generate_stubs_files(
         module_dir.mkdir(parents=True, exist_ok=True)
 
         # Create and open module file
-        file_path = Path(module_dir / f"{module_name}.sdsstub")
+        public_module_name = module_name.lstrip("_")
+        file_path = Path(module_dir / f"{public_module_name}.sdsstub")
         Path(file_path).touch()
 
         with file_path.open("w") as f:
@@ -179,19 +175,19 @@ class StubsStringGenerator:
     method.
     """
 
-    def __init__(self, api: API, naming_convention: NamingConvention) -> None:
+    def __init__(self, api: API, convert_identifiers: bool) -> None:
         self.api = api
-        self.naming_convention = naming_convention
+        self.naming_convention = NamingConvention.SAFE_DS if convert_identifiers else NamingConvention.PYTHON
         self.classes_outside_package: set[str] = set()
 
-    def __call__(self, module: Module) -> str:
+    def __call__(self, module: Module) -> tuple[str, str]:
         self.module_imports: set[str] = set()
         self._current_todo_msgs: set[str] = set()
         self.module = module
         self.class_generics: list = []
         return self._create_module_string()
 
-    def _create_module_string(self) -> str:
+    def _create_module_string(self) -> tuple[str, str]:
         # Create package info
         package_info = self._get_shortest_public_reexport()
         package_info_camel_case = _convert_name_to_convention(package_info, self.naming_convention)
@@ -223,7 +219,7 @@ def _create_module_string(self) -> str:
         # Create imports - We have to create them last, since we have to check all used types in this module first
         module_header += self._create_imports_string()
 
-        return docstring + module_header + module_text
+        return f"{docstring}{module_header}{module_text}", package_info
 
     def _create_imports_string(self) -> str:
         if not self.module_imports:
@@ -985,7 +981,6 @@ def _add_to_imports(self, qname: str) -> None:
             for class_ in self.api.classes:
                 if class_.endswith(qname_path):
                     qname = class_.replace("/", ".")
-                    qname = _convert_name_to_convention(qname, self.naming_convention)
                     in_package = True
                     break
 
@@ -1137,15 +1132,12 @@ def _replace_if_safeds_keyword(keyword: str) -> str:
         "sub",
         "super",
         "_",
+        "unknown",
     }:
         return f"`{keyword}`"
     return keyword
 
 
-def is_internal(name: str) -> bool:
-    return name.startswith("_")
-
-
 def _convert_name_to_convention(
     name: str,
     naming_convention: NamingConvention,

tests/safeds_stubgen/stubs_generator/test_generate_stubs.py

@@ -6,22 +6,17 @@
 import pytest
 from safeds_stubgen.api_analyzer import get_api
 from safeds_stubgen.docstring_parsing import DocstringStyle
-from safeds_stubgen.stubs_generator import generate_stubs
+from safeds_stubgen.stubs_generator import NamingConvention, StubsStringGenerator, create_stub_files, generate_stub_data
 
 # noinspection PyProtectedMember
-from safeds_stubgen.stubs_generator._generate_stubs import (
-    NamingConvention,
-    StubsStringGenerator,
-    _convert_name_to_convention,
-    _generate_stubs_data,
-    _generate_stubs_files,
-)
+from safeds_stubgen.stubs_generator._generate_stubs import _convert_name_to_convention
 
 if TYPE_CHECKING:
     from collections.abc import Generator
 
     from syrupy import SnapshotAssertion
 
+
 # Setup - Run API to create stub files
 _lib_dir = Path(__file__).parent.parent.parent
 _test_package_name = "various_modules_package"
@@ -33,50 +28,43 @@
 _docstring_package_dir = Path(_lib_dir / "data" / _docstring_package_name)
 
 api = get_api(_test_package_dir, is_test_run=True)
-stubs_generator = StubsStringGenerator(api, naming_convention=NamingConvention.SAFE_DS)
-stubs_data = _generate_stubs_data(api, _out_dir, stubs_generator)
+stubs_generator = StubsStringGenerator(api=api, convert_identifiers=True)
+stubs_data = generate_stub_data(stubs_generator=stubs_generator, out_path=_out_dir)
 
 
 def test_file_creation() -> None:
-    _generate_stubs_files(stubs_data, _out_dir, stubs_generator, naming_convention=NamingConvention.SAFE_DS)
-    _assert_file_creation_recursive(
-        python_path=Path(_test_package_dir / "file_creation"),
-        stub_path=Path(_out_dir_stubs / "file_creation"),
-    )
-
-
-def _assert_file_creation_recursive(python_path: Path, stub_path: Path) -> None:
-    assert python_path.is_dir()
-    assert stub_path.is_dir()
-
-    python_files: list[Path] = list(python_path.iterdir())
-    stub_files: list[Path] = list(stub_path.iterdir())
-
-    # Remove __init__ files and private files without public reexported content.
-    # We reexport public content from _module_3 and _module_6, not from empty_module, _module_2 and _module_4.
-    actual_python_files = []
-    for item in python_files:
-        if not (item.is_file() and item.stem in {"__init__", "_module_2", "_module_4"}):
-            actual_python_files.append(item)
-
-    assert len(actual_python_files) == len(stub_files)
-
-    actual_python_files.sort(key=lambda x: x.stem)
-    stub_files.sort(key=lambda x: x.stem)
-
-    for py_item, stub_item in zip(actual_python_files, stub_files, strict=True):
-        if py_item.is_file():
-            assert stub_item.is_dir()
-            stub_files = list(stub_item.iterdir())
-            assert len(stub_files) == 1
-            assert stub_files[0].stem == py_item.stem
-        else:
-            _assert_file_creation_recursive(py_item, stub_item)
+    data_to_test: list[tuple[str, str]] = [
+        ("/".join(stub_data[0].parts), stub_data[1]) for stub_data in stubs_data if "file_creation" in str(stub_data[0])
+    ]
+    data_to_test.sort(key=lambda x: x[1])
+
+    expected_files: list[tuple[str, str]] = [
+        # We reexport these three modules from another package into the file_creation package.
+        ("tests/data/various_modules_package/file_creation", "_reexported_from_another_package"),
+        ("tests/data/various_modules_package/file_creation", "_reexported_from_another_package_2"),
+        ("tests/data/various_modules_package/file_creation", "_reexported_from_another_package_3"),
+        # module_1 is public
+        ("tests/data/various_modules_package/file_creation/module_1", "module_1"),
+        # _module_6 has a public reexport in the file_creation package
+        ("tests/data/various_modules_package/file_creation", "_module_6"),
+        # module_5 is publich
+        ("tests/data/various_modules_package/file_creation/package_1/module_5", "module_5"),
+        # _module_3 is not created, even though it is reexported, since it's also reexported in the parent
+        # package.
+        # _module_2 is not created, since the reexport is still private
+        # _module_4 is not created, since it has no (public) reexport
+    ]
+    expected_files.sort(key=lambda x: x[1])
+
+    assert len(data_to_test) == len(expected_files)
+    for data_tuple, expected_tuple in zip(data_to_test, expected_files, strict=True):
+        assert data_tuple[0].endswith(expected_tuple[0])
+        assert data_tuple[1] == expected_tuple[1]
 
 
 def test_file_creation_limited_stubs_outside_package(snapshot_sds_stub: SnapshotAssertion) -> None:
-    # Somehow the stubs get overwritten by other tests, therefore we have to call the function before asserting
-    generate_stubs(api, _out_dir, convert_identifiers=True)
+    create_stub_files(stubs_generator=stubs_generator, stubs_data=stubs_data, out_path=_out_dir)
+
     path = Path(_out_dir / "tests/data/main_package/another_path/another_module/another_module.sdsstub")
     assert path.is_file()
 
@@ -161,8 +149,8 @@ def test_stub_docstring_creation(
         docstring_style=docstring_style,
         is_test_run=True,
     )
-    docstring_stubs_generator = StubsStringGenerator(docstring_api, naming_convention=NamingConvention.SAFE_DS)
-    docstring_stubs_data = _generate_stubs_data(docstring_api, _out_dir, docstring_stubs_generator)
+    docstring_stubs_generator = StubsStringGenerator(api=docstring_api, convert_identifiers=True)
+    docstring_stubs_data = generate_stub_data(stubs_generator=docstring_stubs_generator, out_path=_out_dir)
 
     for stub_text in docstring_stubs_data:
         if stub_text[1] == filename: