Unit tests

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

docs/conf.py

@@ -193,6 +193,7 @@
     ("py:class", "substra.sdk.schemas.FunctionOutputSpec"),
     ("py:class", "substra.sdk.schemas.FunctionInputSpec"),
     ("py:class", "ComputePlan"),
+    ("py:class", "Path"),
 ]
 
 html_css_files = [

substrafl/exceptions.py

@@ -18,8 +18,8 @@ class InvalidUserModuleError(Exception):
     """The local folder passed by the user as a dependency is not a valid Python module."""
 
 
-class IncompatibleDependenciesError(Exception):
-    """The set of constraints given on dependencies cannot be solved."""
+class InvalidDependenciesError(Exception):
+    """The set of constraints given on dependencies cannot be solved or is otherwise invalid (wrong package name)."""
 
 
 class CriterionReductionError(Exception):

substrafl/remote/register/manage_dependencies.py

@@ -11,7 +11,7 @@
 from typing import List
 
 from substrafl.dependency import Dependency
-from substrafl.exceptions import IncompatibleDependenciesError
+from substrafl.exceptions import InvalidDependenciesError
 from substrafl.exceptions import InvalidUserModuleError
 
 logger = logging.getLogger(__name__)
@@ -23,8 +23,8 @@ def build_user_dependency_wheel(lib_path: Path, operation_dir: Path) -> str:
     """Build the wheel for user dependencies passed as a local module.
 
     Args:
-        lib_path (Path): where the module is located
-        operation_dir (Path): where the wheel needs to be copied
+        lib_path (Path): where the module is located.
+        operation_dir (Path): where the wheel needs to be copied.
 
     Returns:
         str: the filename of the wheel
@@ -46,7 +46,7 @@ def build_user_dependency_wheel(lib_path: Path, operation_dir: Path) -> str:
             text=True,
         )
     except subprocess.CalledProcessError as e:
-        raise InvalidUserModuleError(e.stdout + e.stderr)
+        raise InvalidUserModuleError from e
     wheel_name = re.findall(r"filename=(\S*)", ret.stdout)[0]
 
     return wheel_name
@@ -69,7 +69,7 @@ def copy_local_wheels(path: Path, dependencies: Dependency) -> List[str]:
         wheel_paths = []
         for dependency in dependencies_paths:
             if dependency.__str__().endswith(".whl"):
-                wheel_paths.append(dependency)
+                wheel_paths.append(str(dependency))
                 shutil.copy(tmp_dir / dependency, path)
             else:
                 wheel_name = build_user_dependency_wheel(
@@ -177,7 +177,7 @@ def compile_requirements(dependency_list: List[str], *, operation_dir: Path, sub
         sub_dir: sub directory of the root dir where the `requirements.in` and `requirements.txt` files will be created
 
     Raises:
-        IncompatibleDependenciesError if pip-compile does not find a set of compatible dependencies
+        InvalidDependenciesError: if pip-compile does not find a set of compatible dependencies
 
     """
     requirements_in = operation_dir / sub_dir / "requirements.in"
@@ -203,4 +203,4 @@ def compile_requirements(dependency_list: List[str], *, operation_dir: Path, sub
             cwd=operation_dir,
         )
     except subprocess.CalledProcessError as e:
-        raise IncompatibleDependenciesError from e
+        raise InvalidDependenciesError from e

substrafl/remote/register/register.py

@@ -139,7 +139,7 @@ def _get_base_docker_image(python_major_minor: str, editable_mode: bool) -> str:
     return substratools_image
 
 
-def _generate_copy_local_files(local_files):
+def _generate_copy_local_files(local_files: typing.List[str]) -> str:
     return "\n".join([f"COPY {file} {file}" for file in local_files])
 
 
@@ -198,8 +198,8 @@ def _create_dockerfile(install_libraries: bool, dependencies: Dependency, operat
         copy_wheels_cmd = ""
 
     # user-defined local dependencies
-    dependencies.copy_dependencies_local_code(dest_dir=operation_dir)
-    copy_local_code_cmd = _generate_copy_local_files(dependencies.local_code)
+    local_paths = dependencies.copy_dependencies_local_code(dest_dir=operation_dir)
+    copy_local_code_cmd = _generate_copy_local_files(local_paths)
 
     # pip-compile the requirements.in into a requirements.txt
     compile_requirements(dependency_list, operation_dir=operation_dir, sub_dir=SUBSTRAFL_FOLDER)

tests/remote/register/conftest.py

@@ -0,0 +1,24 @@
+import pytest
+
+SETUP_CONTENT = """from setuptools import setup, find_packages
+
+setup(
+    name='mymodule',
+    version='1.0.2',
+    author='Author Name',
+    description='Description of my package',
+    packages=find_packages(),
+    install_requires=['numpy >= 1.11.1', 'matplotlib >= 3.5.1'],
+)"""
+
+
+@pytest.fixture
+def local_installable_module():
+    def _local_installable_module(root_dir):
+        module_root = root_dir / "my_module"
+        module_root.mkdir()
+        setup_file = module_root / "setup.py"
+        setup_file.write_text(SETUP_CONTENT)
+        return module_root
+
+    return _local_installable_module

tests/remote/register/test_manage_dependencies.py

@@ -1,21 +1,20 @@
 import os
-import sys
 import tempfile
 from pathlib import Path
 
+import numpy
 import pytest
 import substratools
 
-from substrafl.exceptions import IncompatibleDependenciesError
+from substrafl.dependency import Dependency
+from substrafl.exceptions import InvalidDependenciesError
 from substrafl.exceptions import InvalidUserModuleError
 from substrafl.remote.register.manage_dependencies import build_user_dependency_wheel
 from substrafl.remote.register.manage_dependencies import compile_requirements
+from substrafl.remote.register.manage_dependencies import copy_local_wheels
 from substrafl.remote.register.manage_dependencies import get_pypi_dependencies_versions
 from substrafl.remote.register.manage_dependencies import local_lib_wheels
 
-PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
-
-
 SETUP_CONTENT = """from setuptools import setup, find_packages
 
 setup(
@@ -28,13 +27,10 @@
 )"""
 
 
-def test_build_user_dependency_wheel(tmp_path):
+def test_build_user_dependency_wheel(tmp_path, local_installable_module):
     operation_dir = tmp_path / "local_dir"
-    os.mkdir(operation_dir)
-    module_root = operation_dir / "my_module"
-    os.mkdir(module_root)
-    setup_file = module_root / "setup.py"
-    setup_file.write_text(SETUP_CONTENT)
+    operation_dir.mkdir()
+    module_root = local_installable_module(operation_dir)
     wheel_name = build_user_dependency_wheel(module_root, operation_dir)
     assert wheel_name == "mymodule-1.0.2-py3-none-any.whl"
     assert (operation_dir / wheel_name).exists()
@@ -51,6 +47,22 @@ def test_build_user_dependency_wheel_invalid_wheel(tmp_path):
         build_user_dependency_wheel(module_root, operation_dir)
 
 
+def test_copy_local_wheels(tmp_path, local_installable_module):
+    dest_dir = tmp_path / "dest"
+    src_dir = tmp_path / "src"
+    dest_dir.mkdir()
+    src_dir.mkdir()
+    precompiled_path = src_dir / "precompiled-wheel-0.0.1-py3-none-any.whl"
+    precompiled_path.touch()
+    to_be_built_path = local_installable_module(src_dir)
+
+    dependency_object = Dependency(local_installable_dependencies=[precompiled_path, to_be_built_path])
+    wheel_paths = copy_local_wheels(dest_dir, dependency_object)
+    assert wheel_paths == ["precompiled-wheel-0.0.1-py3-none-any.whl", "mymodule-1.0.2-py3-none-any.whl"]
+    assert (dest_dir / "precompiled-wheel-0.0.1-py3-none-any.whl").is_file()
+    assert (dest_dir / "mymodule-1.0.2-py3-none-any.whl").is_file()
+
+
 def test_generate_local_lib_wheel(session_dir):
     # Test that editable wheel are generated
     libs = [substratools]
@@ -75,17 +87,14 @@ def test_generate_local_lib_wheel(session_dir):
 
 def test_get_pypi_dependencies_versions():
     # Test that pypi wheel can be downloaded
-    libs = [substratools]
+    libs = [substratools, numpy]
 
     # We check that we have access to the pypi repo not the specific packages version otherwise this test will fail
     # when trying to create a new version of substrafl as the running dev version on the ci and on a local computer
     # (0.x.0) won't have been released yet.
 
-    # save the current versions the libs to set them back later
-    substratools_version = substratools.__version__
-
     dependencies = get_pypi_dependencies_versions(lib_modules=libs)
-    assert dependencies == [f"substratools=={substratools_version}"]
+    assert dependencies == [f"substratools=={substratools.__version__}", f"numpy=={numpy.__version__}"]
 
 
 def test_compile_requirements(tmp_path):
@@ -99,8 +108,8 @@ def test_compile_requirements(tmp_path):
     assert "numpy" in requirements_path.read_text()
 
 
-def test_compile_requirements_incompatible_versions(tmp_path):
+@pytest.mark.parametrize("dependency_list", [["numpy == 1.11", "numpy == 1.12"], ["numpy", "pndas"]])
+def test_compile_requirements_invalid_dependencies(tmp_path, dependency_list):
     os.mkdir(tmp_path / "substrafl_internals")
-    dependency_list = ["numpy == 1.11", "numpy == 1.12"]
-    with pytest.raises(IncompatibleDependenciesError):
+    with pytest.raises(InvalidDependenciesError):
         compile_requirements(dependency_list, operation_dir=tmp_path, sub_dir="substrafl_internals")

tests/remote/register/test_register.py

@@ -1,16 +1,20 @@
+import sys
 import tarfile
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
 import pytest
 import substra
+import substratools
 
+import substrafl
 from substrafl.dependency import Dependency
 from substrafl.exceptions import UnsupportedPythonVersionError
 from substrafl.nodes import TestDataNode
 from substrafl.remote.decorators import remote_data
 from substrafl.remote.register import register
 from substrafl.remote.register import register_metrics
+from substrafl.remote.register.register import _create_dockerfile
 
 
 class RemoteClass:
@@ -77,6 +81,56 @@ def test_latest_substratools_image_selection(use_latest, monkeypatch, default_pe
         assert "latest" not in str(lines[1])
 
 
+def test_create_dockerfile(tmp_path, mocker, local_installable_module):
+    mocker.patch("substrafl.remote.register.register._get_base_docker_image", return_value="substratools-mocked")
+    python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+    substrafl_wheel = f"substrafl_internal/dist/substrafl-{substrafl.__version__}-py3-none-any.whl"
+    substra_wheel = f"substrafl_internal/dist/substra-{substra.__version__}-py3-none-any.whl"
+    substratools_wheel = f"substrafl_internal/dist/substratools-{substratools.__version__}-py3-none-any.whl"
+    local_installable_dependencies = local_installable_module(tmp_path)
+    local_installable_wheel = "substrafl_internal/local_dependencies/mymodule-1.0.2-py3-none-any.whl"
+    local_code_folder = tmp_path / "local"
+    local_code_folder.mkdir()
+    local_code = local_code_folder / "foo.py"
+    local_code.touch()
+
+    dependencies = Dependency(
+        editable_mode=True,
+        pypi_dependencies=[],
+        local_installable_dependencies=[local_installable_dependencies],
+        local_code=[local_code_folder],
+    )
+
+    expected_dockerfile = f"""
+FROM substratools-mocked
+
+# install dependencies
+RUN python{python_version} -m pip install -U pip
+
+# Copy local wheels
+COPY {substrafl_wheel} {substrafl_wheel}
+COPY {substra_wheel} {substra_wheel}
+COPY {substratools_wheel} {substratools_wheel}
+COPY {local_installable_wheel} {local_installable_wheel}
+
+# Copy requirements.txt
+COPY substrafl_internal/requirements.txt requirements.txt
+
+# Install requirements
+RUN python{python_version} -m pip install --no-cache-dir -r requirements.txt
+
+# Copy all other files
+COPY function.py .
+COPY substrafl_internal/cls_cloudpickle substrafl_internal/
+COPY substrafl_internal/description.md substrafl_internal/
+COPY local local
+
+ENTRYPOINT ["python{python_version}", "function.py", "--function-name", "foo_bar"]
+"""
+    dockerfile = _create_dockerfile(True, dependencies, tmp_path, "foo_bar")
+    assert dockerfile == expected_dockerfile
+
+
 @pytest.mark.parametrize("algo_name, result", [("Dummy Algo Name", "Dummy Algo Name"), (None, "foo_RemoteClass")])
 def test_algo_name(algo_name, result):
     my_class = RemoteClass()