diff --git a/README.md b/README.md index 9a4c5e986..835eab2a2 100644 --- a/README.md +++ b/README.md @@ -388,8 +388,19 @@ A dependency will also be treated as a `pip` dependency if explicitly marked wit [tool.conda-lock.dependencies] ampel-ztf = {source = "pypi"} ``` +##### Defaulting non-conda dependency sources to PyPI -In both these cases, the dependencies of `pip`-installable packages will also be +Alternatively, the above behavior is defaulted for all dependencies defined outside of `[tool.conda-lock.dependencies]`, i.e.: +- Default to `pip` dependencies for `[tool.poetry.dependencies]`, `[project.dependencies]`, etc. +- Default to `conda` dependencies for `[tool.conda-lock.dependencies]` + +by explicitly providing `default-non-conda-source = "pip"` in `[tool.conda-lock]` section, e.g.: +```toml +[tool.conda-lock] +default-non-conda-source = "pip" +``` + +In all cases, the dependencies of `pip`-installable packages will also be installed with `pip`, unless they were already requested by a `conda` dependency. diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 9dc81227a..25aff2423 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -125,9 +125,10 @@ def parse_poetry_pyproject_toml( * dependencies in each `key` of [tool.poetry.extras] have category `key` * dependencies in [tool.poetry.{group}.dependencies] have category `group` - * By default, dependency names are translated to the conda equivalent, with two exceptions: + * By default, dependency names are translated to the conda equivalent, with three exceptions: - If a dependency has `source = "pypi"`, it is treated as a pip dependency (by name) - If a dependency has a url, it is treated as a direct pip dependency (by url) + - If all dependencies are default-sourced to pip, `default-non-conda-source = "pip"` * markers are not supported @@ -150,12 +151,17 @@ def parse_poetry_pyproject_toml( group_key = tuple(["group", group_name, "dependencies"]) categories[group_key] = group_name + default_non_conda_source = get_in( + ["tool", "conda-lock", "default-non-conda-source"], + contents, + "conda", + ) for section, default_category in categories.items(): for depname, depattrs in get_in( ["tool", "poetry", *section], contents, {} ).items(): category: str = dep_to_extra.get(depname) or default_category - manager: Literal["conda", "pip"] = "conda" + manager: Literal["conda", "pip"] = default_non_conda_source url = None extras = [] in_extra: bool = False @@ -377,10 +383,17 @@ def parse_requirements_pyproject_toml( ): sections[(*prefix, optional_tag, extra)] = extra + default_non_conda_source = get_in( + ["tool", "conda-lock", "default-non-conda-source"], + contents, + "conda", + ) for path, category in sections.items(): for dep in get_in(list(path), contents, []): dependencies.append( - parse_python_requirement(dep, manager="conda", category=category) + parse_python_requirement( + dep, manager=default_non_conda_source, category=category + ) ) return specification_with_dependencies( @@ -407,11 +420,17 @@ def parse_pdm_pyproject_toml( ) dev_reqs = [] - + default_non_conda_source = get_in( + ["tool", "conda-lock", "default-non-conda-source"], + contents, + "conda", + ) for section, deps in get_in(["tool", "pdm", "dev-dependencies"], contents).items(): dev_reqs.extend( [ - parse_python_requirement(dep, manager="conda", category="dev") + parse_python_requirement( + dep, manager=default_non_conda_source, category="dev" + ) for dep in deps ] ) diff --git a/docs/pip.md b/docs/pip.md index e1e402f7b..6ae6a4cb8 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -45,6 +45,14 @@ python = "3.9" ampel-ztf = {version = "^0.8.0-alpha.2", source = "pypi"} ``` -In both these cases, the dependencies of `pip`-installable packages will also be +Alternatively, explicitly providing `default-non-conda-source = "pip"` in the `[tool.conda-lock]` section will treat all non-conda dependencies -- all dependencies defined outside of `[tool.conda-lock.dependencies]` -- as `pip` dependencies, i.e.: +- Default to `pip` dependencies for `[tool.poetry.dependencies]`, `[project.dependencies]`, etc. +- Default to `conda` dependencies for `[tool.conda-lock.dependencies]` +```toml +[tool.conda-lock] +default-non-conda-source = "pip" +``` + +In all cases, the dependencies of `pip`-installable packages will also be installed with `pip`, unless they were already requested by a `conda` dependency. diff --git a/docs/src_pyproject.md b/docs/src_pyproject.md index 1577690f4..0854aba98 100644 --- a/docs/src_pyproject.md +++ b/docs/src_pyproject.md @@ -77,7 +77,15 @@ python = "3.9" ampel-ztf = {version = "^0.8.0-alpha.2", source = "pypi"} ``` -In both these cases, the dependencies of `pip`-installable packages will also be +Alternatively, explicitly providing `default-non-conda-source = "pip"` in the `[tool.conda-lock]` section will treat all non-conda dependencies -- all dependencies defined outside of `[tool.conda-lock.dependencies]` -- as `pip` dependencies, i.e.: +- Default to `pip` dependencies for `[tool.poetry.dependencies]`, `[project.dependencies]`, etc. +- Default to `conda` dependencies for `[tool.conda-lock.dependencies]` +```toml +[tool.conda-lock] +default-non-conda-source = "pip" +``` + +In all cases, the dependencies of `pip`-installable packages will also be installed with `pip`, unless they were already requested by a `conda` dependency. diff --git a/tests/test-flit-default-pip/pyproject.toml b/tests/test-flit-default-pip/pyproject.toml new file mode 100644 index 000000000..9de757aef --- /dev/null +++ b/tests/test-flit-default-pip/pyproject.toml @@ -0,0 +1,29 @@ +[tool.flit.metadata] +name = "conda-lock-test-poetry" +version = "0.0.1" +description = "" +authors = ["conda-lock"] +requires = [ + "requests >=2.13.0", + "toml >=0.10", + "tomlkit >=0.7", + ] + +[tool.flit.metadata.requires-extra] +test = [ + "pytest >=5.1.0" +] + +[build-system] +requires = ["flit_core >=2,<4"] +build-backend = "flit_core.buildapi" + +[tool.conda-lock] +channels = [ + 'defaults' +] +default-non-conda-source = "pip" + +[tool.conda-lock.dependencies] +sqlite = "<3.34" +certifi = ">=2019.11.28" diff --git a/tests/test-pdm-default-pip/pyproject.toml b/tests/test-pdm-default-pip/pyproject.toml new file mode 100644 index 000000000..5dd33681c --- /dev/null +++ b/tests/test-pdm-default-pip/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "conda-lock-test-pdm" +authors = ["conda-lock"] +description = "" +requires-python = ">=3.7" +dependencies = [ + "requests >=2.13.0", + "toml >=0.10", + "tomlkit >=0.7", +] + +[project.optional-dependencies] +cli = ["click >=7.0"] + +[tool.pdm.dev-dependencies] +test = [ + "pytest >=5.1.0", +] + +[tool.conda-lock] +channels = [ + "defaults", +] +default-non-conda-source = "pip" + +[tool.conda-lock.dependencies] +certifi = ">=2019.11.28" +sqlite = "<3.34" diff --git a/tests/test-poetry-default-pip/pyproject.toml b/tests/test-poetry-default-pip/pyproject.toml new file mode 100644 index 000000000..2509979d1 --- /dev/null +++ b/tests/test-poetry-default-pip/pyproject.toml @@ -0,0 +1,30 @@ +[tool.poetry] +name = "conda-lock-test-poetry" +version = "0.0.1" +description = "" +authors = ["conda-lock"] + +[tool.poetry.dependencies] +requests = "^2.13.0" +toml = ">=0.10" +tomlkit = { version = ">=0.7.0,<1.0.0", optional = true } + +[tool.poetry.dev-dependencies] +pytest = "~5.1.0" + +[tool.poetry.extras] +tomlkit = ["tomlkit"] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + +[tool.conda-lock] +channels = [ + 'defaults' +] +default-non-conda-source = "pip" + +[tool.conda-lock.dependencies] +sqlite = "<3.34" +certifi = ">=2019.11.28" diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 6e325f575..352c27097 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -189,6 +189,13 @@ def poetry_pyproject_toml(tmp_path: Path): return clone_test_dir("test-poetry", tmp_path).joinpath("pyproject.toml") +@pytest.fixture +def poetry_pyproject_toml_default_pip(tmp_path: Path): + return clone_test_dir("test-poetry-default-pip", tmp_path).joinpath( + "pyproject.toml" + ) + + @pytest.fixture def poetry_pyproject_toml_no_pypi(tmp_path: Path): return clone_test_dir("test-poetry-no-pypi", tmp_path).joinpath("pyproject.toml") @@ -218,6 +225,16 @@ def pdm_pyproject_toml(tmp_path: Path): return clone_test_dir("test-pdm", tmp_path).joinpath("pyproject.toml") +@pytest.fixture +def flit_pyproject_toml_default_pip(tmp_path: Path): + return clone_test_dir("test-flit-default-pip", tmp_path).joinpath("pyproject.toml") + + +@pytest.fixture +def pdm_pyproject_toml_default_pip(tmp_path: Path): + return clone_test_dir("test-pdm-default-pip", tmp_path).joinpath("pyproject.toml") + + @pytest.fixture def channel_inversion(tmp_path: Path): """Path to an environment.yaml that has a hardcoded channel in one of the dependencies""" @@ -632,6 +649,22 @@ def test_parse_poetry(poetry_pyproject_toml: Path): assert res.channels == [Channel.from_string("defaults")] +def test_parse_poetry_default_pip(poetry_pyproject_toml_default_pip: Path): + res = parse_pyproject_toml(poetry_pyproject_toml_default_pip, ["linux-64"]) + + specs = { + dep.name: typing.cast(VersionedDependency, dep) + for dep in res.dependencies["linux-64"] + } + + assert specs["sqlite"].manager == "conda" + assert specs["certifi"].manager == "conda" + assert specs["requests"].manager == "pip" + assert specs["toml"].manager == "pip" + assert specs["pytest"].manager == "pip" + assert specs["tomlkit"].manager == "pip" + + def test_parse_poetry_no_pypi(poetry_pyproject_toml_no_pypi: Path): platforms = parse_platforms_from_pyproject_toml(poetry_pyproject_toml_no_pypi) res = parse_pyproject_toml(poetry_pyproject_toml_no_pypi, platforms) @@ -730,6 +763,22 @@ def test_parse_flit(flit_pyproject_toml: Path): assert res.channels == [Channel.from_string("defaults")] +def test_parse_flit_default_pip(flit_pyproject_toml_default_pip: Path): + res = parse_pyproject_toml(flit_pyproject_toml_default_pip, ["linux-64"]) + + specs = { + dep.name: typing.cast(VersionedDependency, dep) + for dep in res.dependencies["linux-64"] + } + + assert specs["sqlite"].manager == "conda" + assert specs["certifi"].manager == "conda" + assert specs["requests"].manager == "pip" + assert specs["toml"].manager == "pip" + assert specs["pytest"].manager == "pip" + assert specs["tomlkit"].manager == "pip" + + def test_parse_pdm(pdm_pyproject_toml: Path): res = parse_pyproject_toml(pdm_pyproject_toml, ["linux-64"]) @@ -754,6 +803,23 @@ def test_parse_pdm(pdm_pyproject_toml: Path): assert res.channels == [Channel.from_string("defaults")] +def test_parse_pdm_default_pip(pdm_pyproject_toml_default_pip: Path): + res = parse_pyproject_toml(pdm_pyproject_toml_default_pip, ["linux-64"]) + + specs = { + dep.name: typing.cast(VersionedDependency, dep) + for dep in res.dependencies["linux-64"] + } + + assert specs["sqlite"].manager == "conda" + assert specs["certifi"].manager == "conda" + assert specs["requests"].manager == "pip" + assert specs["toml"].manager == "pip" + assert specs["pytest"].manager == "pip" + assert specs["tomlkit"].manager == "pip" + assert specs["click"].manager == "pip" + + def test_parse_poetry_invalid_optionals(pyproject_optional_toml: Path): filename = pyproject_optional_toml.name