From d555b1deb8949ea1613daedd0abbd49614205eee Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Wed, 22 Mar 2023 21:19:11 +0000 Subject: [PATCH 1/8] implement support for Poetry version syntax and optional dependencies --- grayskull/__main__.py | 14 +- grayskull/strategy/py_base.py | 2 + grayskull/strategy/py_toml.py | 203 ++++++++++++++++--- grayskull/strategy/pypi.py | 10 + pyproject.toml | 1 + tests/data/poetry/langchain-expected.yaml | 86 ++++++++ tests/data/{pyproject => poetry}/poetry.toml | 0 tests/data/{pyproject => tox}/tox.toml | 0 tests/test_poetry.py | 122 +++++++++++ tests/{test_pyproject.py => test_tox.py} | 46 +---- 10 files changed, 411 insertions(+), 73 deletions(-) create mode 100644 tests/data/poetry/langchain-expected.yaml rename tests/data/{pyproject => poetry}/poetry.toml (100%) rename tests/data/{pyproject => tox}/tox.toml (100%) create mode 100644 tests/test_poetry.py rename tests/{test_pyproject.py => test_tox.py} (51%) diff --git a/grayskull/__main__.py b/grayskull/__main__.py index e3e225ac9..25364d6c3 100644 --- a/grayskull/__main__.py +++ b/grayskull/__main__.py @@ -20,11 +20,7 @@ init(autoreset=True) logging.basicConfig(format="%(levelname)s:%(message)s") - -def main(args=None): - if not args: - args = sys.argv[1:] or ["--help"] - +def init_parser(): # create the top-level parser parser = argparse.ArgumentParser(description="Grayskull - Conda recipe generator") subparsers = parser.add_subparsers(help="sub-command help") @@ -246,6 +242,14 @@ def main(args=None): help="Exclude folders when searching for licence.", ) + return parser + + +def main(args=None): + if not args: + args = sys.argv[1:] or ["--help"] + + parser = init_parser() args = parser.parse_args(args) if args.version: diff --git a/grayskull/strategy/py_base.py b/grayskull/strategy/py_base.py index 836997a29..ddd3c0cc7 100644 --- a/grayskull/strategy/py_base.py +++ b/grayskull/strategy/py_base.py @@ -732,6 +732,8 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) -> setup_metadata.get("install_requires", []), pyproject_metadata["requirements"]["run"], ) + if pyproject_metadata["requirements"]["run_constrained"]: + setup_metadata["requirements_run_constrained"] = pyproject_metadata["requirements"]["run_constrained"] return setup_metadata diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 02f66248b..b06e618f3 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -1,47 +1,198 @@ from collections import defaultdict from pathlib import Path -from typing import Union +from typing import Union, Dict, Optional, Tuple +import re import tomli +import semver from grayskull.utils import nested_dict +VERSION_REGEX = re.compile( + r"""[vV]? + (?P0|[1-9]\d*) + (\. + (?P0|[1-9]\d*) + (\. + (?P0|[1-9]\d*) + )? + )? + """, + re.VERBOSE, +) + +class InvalidVersion(BaseException): + pass + +class InvalidPoetryDependency(BaseException): + pass + +def parse_version(version: str) -> Dict[str, Optional[str]]: + """ + Parses a version string (not necessarily semver) to a dictionary with keys + "major", "minor", and "patch". "minor" and "patch" are possibly None. + """ + match = VERSION_REGEX.search(version) + if not match: + raise InvalidVersion(f"Could not parse version {version}.") + + return { + key: None if value is None else int(value) for key, value in match.groupdict().items() + } + +def vdict_to_vinfo(version_dict: Dict[str, Optional[str]]) -> semver.VersionInfo: + """ + Coerces version dictionary to a semver.VersionInfo object. If minor or patch + numbers are missing, 0 is substituted in their place. + """ + ver = { + key: 0 if value is None else value for key, value in version_dict.items() + } + return semver.VersionInfo(**ver) + +def coerce_to_semver(version: str) -> str: + """ + Coerces a version string to a semantic version. + """ + if semver.VersionInfo.isvalid(version): + return version + + return str(vdict_to_vinfo(parse_version(version))) + +def get_caret_ceiling(target: str) -> str: + """ + Accepts a Poetry caret target and returns the exclusive version ceiling. + + Targets that are invalid semver strings (e.g. "1.2", "0") are handled + according to the Poetry caret requirements specification, which is based on + whether the major version is 0: + + - If the major version is 0, the ceiling is determined by bumping the + rightmost specified digit and then coercing it to semver. + Example: 0 => 1.0.0, 0.1 => 0.2.0, 0.1.2 => 0.1.3 + + - If the major version is not 0, the ceiling is determined by + coercing it to semver and then bumping the major version. + Example: 1 => 2.0.0, 1.2 => 2.0.0, 1.2.3 => 2.0.0 + """ + if not semver.VersionInfo.isvalid(target): + target_dict = parse_version(target) + + if target_dict["major"] == 0: + if target_dict["minor"] == None: + target_dict["major"] += 1 + elif target_dict["patch"] == None: + target_dict["minor"] += 1 + else: + target_dict["patch"] += 1 + return str(vdict_to_vinfo(target_dict)) + + vdict_to_vinfo(target_dict) + return str(vdict_to_vinfo(target_dict).bump_major()) + + target_vinfo = semver.VersionInfo.parse(target) + + if target_vinfo.major == 0: + if target_vinfo.minor == 0: + return str(target_vinfo.bump_patch()) + else: + return str(target_vinfo.bump_minor()) + else: + return str(target_vinfo.bump_major()) + +def get_tilde_ceiling(target: str) -> str: + """ + Accepts a Poetry tilde target and returns the exclusive version ceiling. + """ + target_dict = parse_version(target) + if target_dict["minor"]: + return str(vdict_to_vinfo(target_dict).bump_minor()) + + return str(vdict_to_vinfo(target_dict).bump_major()) + +def encode_poetry_version(poetry_specifier: str) -> str: + """ + Encodes Poetry version specifier as a Conda version specifier. + + Example: ^1 => >=1.0.0,<2.0.0 + """ + poetry_clauses = poetry_specifier.split(',') + + conda_clauses = [] + for poetry_clause in poetry_clauses: + poetry_clause = poetry_clause.replace(" ", "") + if poetry_clause.startswith('^'): + # handle ^ operator + target = poetry_clause[1:] + floor = coerce_to_semver(target) + ceiling = get_caret_ceiling(target) + conda_clauses.append(">=" + floor) + conda_clauses.append("<" + ceiling) + continue + + if poetry_clause.startswith('~'): + # handle ~ operator + target = poetry_clause[1:] + floor = coerce_to_semver(target) + ceiling = get_tilde_ceiling(target) + conda_clauses.append(">=" + floor) + conda_clauses.append("<" + ceiling) + continue + + # other poetry clauses should be conda-compatible + conda_clauses.append(poetry_clause) + + return ",".join(conda_clauses) + +def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: + run = [] + run_constrained = [] + for dep_name, dep_spec in poetry_deps.items(): + dep_name = dep_name.strip() + + if isinstance(dep_spec, dict): + conda_version = encode_poetry_version(dep_spec["version"]) + if dep_spec.get("optional", False) == True: + run_constrained.append(f"{dep_name} {conda_version}") + else: + run.append(f"{dep_name} {conda_version}") + continue + + if isinstance(dep_spec, str): + conda_version = encode_poetry_version(dep_spec) + run.append(f"{dep_name} {conda_version}") + continue + + raise InvalidPoetryDependency(f"Expected Poetry dependency specification to be of type str or dict, received {type(dep_spec).__name__}") + + return run, run_constrained def add_poetry_metadata(metadata: dict, toml_metadata: dict) -> dict: if not is_poetry_present(toml_metadata): return metadata - def flat_deps(dict_deps: dict) -> list: - result = [] - for pkg_name, version in dict_deps.items(): - if isinstance(version, dict): - version_spec = version["version"].strip() - del version["version"] - version = ( - f"{version_spec}{' ; '.join(f'{k} {v}' for k,v in version.items())}" - ) - version = f"=={version}" if version and version[0].isdigit() else version - result.append(f"{pkg_name} {version}".strip()) - return result - poetry_metadata = toml_metadata["tool"]["poetry"] - if poetry_run := flat_deps(poetry_metadata.get("dependencies", {})): - if not metadata["requirements"]["run"]: - metadata["requirements"]["run"] = [] - metadata["requirements"]["run"].extend(poetry_run) + poetry_deps = poetry_metadata.get("dependencies", {}) + req_run, req_run_constrained = encode_poetry_deps(poetry_deps) + + # add dependencies + metadata["requirements"].setdefault("run", []) + metadata["requirements"]["run"].extend(req_run) + + # add optional dependencies + if len(req_run_constrained): + metadata["requirements"].setdefault("run_constrained", []) + metadata["requirements"]["run_constrained"].extend(req_run_constrained) host_metadata = metadata["requirements"].get("host", []) if "poetry" not in host_metadata and "poetry-core" not in host_metadata: metadata["requirements"]["host"] = host_metadata + ["poetry-core"] - test_metadata = metadata["test"].get("requires", []) or [] - if ( - test_deps := poetry_metadata.get("group", {}) - .get("test", {}) - .get("dependencies", {}) - ): - test_deps = flat_deps(test_deps) - metadata["test"]["requires"] = test_metadata + test_deps + poetry_test_deps = poetry_metadata.get("group", {}).get("test", {}).get("dependencies", {}) + # add required test dependencies and ignore optional test dependencies, as + # there doesn't appear to be a way to specify them in Conda recipe metadata. + test_reqs, _ = encode_poetry_deps(poetry_test_deps) + metadata["test"].get("requires", []).extend(test_reqs) return metadata diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 0fd3d82ee..3d389b22b 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -107,6 +107,7 @@ def get_val(key): "extras_require": get_val("extras_require"), "requires_dist": requires_dist, "sdist_path": get_val("sdist_path"), + "requirements_run_constrained": get_val("requirements_run_constrained") } @@ -382,10 +383,14 @@ def get_metadata(recipe, config) -> dict: requirements_section = extract_requirements(metadata, config, recipe) optional_requirements = extract_optional_requirements(metadata, config) for key in requirements_section: + log.error(key) + log.error(requirements_section[key]) requirements_section[key] = normalize_requirements_list( requirements_section[key], config ) for key in optional_requirements: + log.error(key) + log.error(optional_requirements[key]) optional_requirements[key] = normalize_requirements_list( optional_requirements[key], config ) @@ -571,6 +576,11 @@ def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]] "run": rm_duplicated_deps(sort_reqs(map(lambda x: x.lower(), run_req))), } ) + + if "requirements_run_constrained" in metadata and metadata["requirements_run_constrained"]: + result.update({ + "run_constrained": metadata["requirements_run_constrained"] + }) update_requirements_with_pin(result) return result diff --git a/pyproject.toml b/pyproject.toml index 9742c0115..d1abeb0c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "ruamel.yaml >=0.16.10", "ruamel.yaml.jinja2", "setuptools >=30.3.0", + "semver~=2.13.0", "stdlib-list", "tomli", "tomli-w", diff --git a/tests/data/poetry/langchain-expected.yaml b/tests/data/poetry/langchain-expected.yaml new file mode 100644 index 000000000..ad76a634a --- /dev/null +++ b/tests/data/poetry/langchain-expected.yaml @@ -0,0 +1,86 @@ +{% set name = "langchain" %} +{% set version = "0.0.119" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/langchain-{{ version }}.tar.gz + sha256: 95a93c966b1a2ff056c43870747aba1c39924c145179f0b8ffa27fef6a525610 + +build: + entry_points: + - langchain-server = langchain.server:main + noarch: python + script: {{ PYTHON }} -m pip install . -vv + number: 0 + +requirements: + host: + - python >=3.8,<4.0 + - poetry-core + - pip + run: + - python >=3.8.1,<4.0 + - pydantic >=1.0.0,<2.0.0 + - sqlalchemy >=1.0.0,<2.0.0 + - requests >=2.0.0,<3.0.0 + - pyyaml >=5.4.1 + - numpy >=1.0.0,<2.0.0 + - dataclasses-json >=0.5.7,<0.6.0 + - tenacity >=8.1.0,<9.0.0 + - aiohttp >=3.8.3,<4.0.0 + run_constrained: + - faiss-cpu >=1.0.0,<2.0.0 + - wikipedia >=1.0.0,<2.0.0 + - elasticsearch >=8.0.0,<9.0.0 + - opensearch-py >=2.0.0,<3.0.0 + - redis-py >=4.0.0,<5.0.0 + - manifest-ml >=0.0.1,<0.0.2 + - spacy >=3.0.0,<4.0.0 + - nltk >=3.0.0,<4.0.0 + - transformers >=4.0.0,<5.0.0 + - beautifulsoup4 >=4.0.0,<5.0.0 + - pytorch >=1.0.0,<2.0.0 + - jinja2 >=3.0.0,<4.0.0 + - tiktoken >=0.0.0,<1.0.0 + - pinecone-client >=2.0.0,<3.0.0 + - weaviate-client >=3.0.0,<4.0.0 + - google-api-python-client 2.70.0 + - wolframalpha 5.0.0 + - anthropic >=0.2.2,<0.3.0 + - qdrant-client >=1.0.4,<2.0.0 + - tensorflow-text >=2.11.0,<3.0.0 + - cohere >=3.0.0,<4.0.0 + - openai >=0.0.0,<1.0.0 + - nlpcloud >=1.0.0,<2.0.0 + - nomic >=1.0.43,<2.0.0 + - huggingface_hub >=0.0.0,<1.0.0 + - google-search-results >=2.0.0,<3.0.0 + - sentence-transformers >=2.0.0,<3.0.0 + - pypdf >=3.4.0,<4.0.0 + - networkx >=2.6.3,<3.0.0 + - aleph-alpha-client >=2.15.0,<3.0.0 + - deeplake >=3.2.9,<4.0.0 + - pgvector >=0.1.6,<0.2.0 + - psycopg2-binary >=2.9.5,<3.0.0 + +test: + imports: + - langchain + commands: + - pip check + - langchain-server --help + requires: + - pip + +about: + home: https://www.github.com/hwchase17/langchain + summary: Building applications with LLMs through composability + license: MIT + license_file: LICENSE + +extra: + recipe-maintainers: + - AddYourGitHubIdHere diff --git a/tests/data/pyproject/poetry.toml b/tests/data/poetry/poetry.toml similarity index 100% rename from tests/data/pyproject/poetry.toml rename to tests/data/poetry/poetry.toml diff --git a/tests/data/pyproject/tox.toml b/tests/data/tox/tox.toml similarity index 100% rename from tests/data/pyproject/tox.toml rename to tests/data/tox/tox.toml diff --git a/tests/test_poetry.py b/tests/test_poetry.py new file mode 100644 index 000000000..c41006ba3 --- /dev/null +++ b/tests/test_poetry.py @@ -0,0 +1,122 @@ +"""Unit and integration tests for recipifying Poetry projects.""" + +import pytest +from pathlib import Path +import filecmp + +from grayskull.__main__ import init_parser, generate_recipes_from_list +from grayskull.utils import generate_recipe +from grayskull.strategy.py_toml import add_poetry_metadata, get_all_toml_info, InvalidVersion, get_caret_ceiling, get_tilde_ceiling, parse_version, encode_poetry_version +from grayskull.config import Configuration + +def test_parse_version(): + assert parse_version("0") == { "major": 0, "minor": None, "patch": None } + assert parse_version("1") == { "major": 1, "minor": None, "patch": None } + assert parse_version("1.2") == { "major": 1, "minor": 2, "patch": None } + assert parse_version("1.2.3") == { "major": 1, "minor": 2, "patch": 3 } + + with pytest.raises(InvalidVersion): + parse_version("asdf") + with pytest.raises(InvalidVersion): + parse_version("") + with pytest.raises(InvalidVersion): + parse_version(".") + + +def test_get_caret_ceiling(): + # examples from Poetry docs + assert get_caret_ceiling("0") == "1.0.0" + assert get_caret_ceiling("0.0") == "0.1.0" + assert get_caret_ceiling("0.0.3") == "0.0.4" + assert get_caret_ceiling("0.2.3") == "0.3.0" + assert get_caret_ceiling("1") == "2.0.0" + assert get_caret_ceiling("1.2") == "2.0.0" + assert get_caret_ceiling("1.2.3") == "2.0.0" + + +def test_get_tilde_ceiling(): + # examples from Poetry docs + assert get_tilde_ceiling("1") == "2.0.0" + assert get_tilde_ceiling("1.2") == "1.3.0" + assert get_tilde_ceiling("1.2.3") == "1.3.0" + + +def test_encode_poetry_version(): + # should be unchanged + assert encode_poetry_version("1.*") == "1.*" + assert encode_poetry_version(">=1,<2") == ">=1,<2" + assert encode_poetry_version("==1.2.3") == "==1.2.3" + assert encode_poetry_version("!=1.2.3") == "!=1.2.3" + + # strip spaces + assert encode_poetry_version(">= 1, < 2") == ">=1,<2" + + # handle exact version specifiers correctly + assert encode_poetry_version("1.2.3") == "1.2.3" + assert encode_poetry_version("==1.2.3") == "==1.2.3" + + # handle caret operator correctly + # examples from Poetry docs + assert encode_poetry_version("^0") == ">=0.0.0,<1.0.0" + assert encode_poetry_version("^0.0") == ">=0.0.0,<0.1.0" + assert encode_poetry_version("^0.0.3") == ">=0.0.3,<0.0.4" + assert encode_poetry_version("^0.2.3") == ">=0.2.3,<0.3.0" + assert encode_poetry_version("^1") == ">=1.0.0,<2.0.0" + assert encode_poetry_version("^1.2") == ">=1.2.0,<2.0.0" + assert encode_poetry_version("^1.2.3") == ">=1.2.3,<2.0.0" + + # handle tilde operator correctly + # examples from Poetry docs + assert encode_poetry_version("~1") == ">=1.0.0,<2.0.0" + assert encode_poetry_version("~1.2") == ">=1.2.0,<1.3.0" + assert encode_poetry_version("~1.2.3") == ">=1.2.3,<1.3.0" + + +def test_add_poetry_metadata(): + toml_metadata = { + "tool": { + "poetry": { + "dependencies": {"tomli": ">=1.0.0", "requests": ">=1.0.0"}, + "group": {"test": {"dependencies": {"tox": ">=1.0.0", "pytest": ">=1.0.0"}}}, + } + } + } + metadata = { + "requirements": { + "host": ["pkg_host1 >=1.0.0", "pkg_host2"], + "run": ["pkg_run1", "pkg_run2 >=2.0.0"], + }, + "test": {"requires": ["mock", "pkg_test >=1.0.0"]}, + } + assert add_poetry_metadata(metadata, toml_metadata) == { + "requirements": { + "host": ["pkg_host1 >=1.0.0", "pkg_host2", "poetry-core"], + "run": ["pkg_run1", "pkg_run2 >=2.0.0", "tomli >=1.0.0", "requests >=1.0.0"], + }, + "test": {"requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest >=1.0.0"]}, + } + +def test_poetry_dependencies(): + toml_path = Path(__file__).parent / "data" / "poetry" / "poetry.toml" + result = get_all_toml_info(toml_path) + + assert result["test"]["requires"] == ["cachy 0.3.0", "deepdiff >=6.2.0,<7.0.0"] + assert result["requirements"]["host"] == ["setuptools>=1.1.0", "poetry-core"] + assert result["requirements"]["run"] == [ + "python >=3.7.0,<4.0.0", + "cleo >=2.0.0,<3.0.0", + "html5lib >=1.0.0,<2.0.0", + "urllib3 >=1.26.0,<2.0.0", + ] + +def test_poetry_langchain_snapshot(tmpdir): + """Snapshot test that asserts correct recipifying of an example Poetry project.""" + snapshot_path = Path(__file__).parent / "data" / "poetry" / "langchain-expected.yaml" + output_path = tmpdir / "langchain" / "meta.yaml" + + parser = init_parser() + args = parser.parse_args(["pypi", "langchain==0.0.119", "-o", str(tmpdir)]) + + generate_recipes_from_list(args.pypi_packages, args) + assert filecmp.cmp(snapshot_path, output_path, shallow=False) + diff --git a/tests/test_pyproject.py b/tests/test_tox.py similarity index 51% rename from tests/test_pyproject.py rename to tests/test_tox.py index 3b0977ed3..5cd34a7e0 100644 --- a/tests/test_pyproject.py +++ b/tests/test_tox.py @@ -1,24 +1,11 @@ -from pathlib import Path - -from grayskull.strategy.py_toml import add_poetry_metadata, get_all_toml_info +"""Unit and integration tests for recipifying Tox projects.""" +from pathlib import Path -def test_get_all_toml_info_poetry(): - toml_path = Path(__file__).parent / "data" / "pyproject" / "poetry.toml" - result = get_all_toml_info(toml_path) - - assert result["test"]["requires"] == ["cachy ==0.3.0", "deepdiff ^6.2"] - assert result["requirements"]["host"] == ["setuptools>=1.1.0", "poetry-core"] - assert result["requirements"]["run"] == [ - "python ^3.7", - "cleo ^2.0.0", - "html5lib ^1.0", - "urllib3 ^1.26.0", - ] - +from grayskull.strategy.py_toml import get_all_toml_info def test_get_all_toml_info(): - toml_path = Path(__file__).parent / "data" / "pyproject" / "tox.toml" + toml_path = Path(__file__).parent / "data" / "tox" / "tox.toml" result = get_all_toml_info(toml_path) assert result["build"]["entry_points"] == ["tox = tox.run:run"] @@ -66,28 +53,3 @@ def test_get_all_toml_info(): 'typing-extensions>=4.4; python_version < "3.8"', "python >=3.7", ] - - -def test_add_poetry_metadata(): - toml_metadata = { - "tool": { - "poetry": { - "dependencies": {"tomli": ">=1.0.0", "requests": ""}, - "group": {"test": {"dependencies": {"tox": ">=1.0.0", "pytest": ""}}}, - } - } - } - metadata = { - "requirements": { - "host": ["pkg_host1 >=1.0.0", "pkg_host2"], - "run": ["pkg_run1", "pkg_run2 >=2.0.0"], - }, - "test": {"requires": ["mock", "pkg_test >=1.0.0"]}, - } - assert add_poetry_metadata(metadata, toml_metadata) == { - "requirements": { - "host": ["pkg_host1 >=1.0.0", "pkg_host2", "poetry-core"], - "run": ["pkg_run1", "pkg_run2 >=2.0.0", "tomli >=1.0.0", "requests"], - }, - "test": {"requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest"]}, - } From fd28b1d2915c53c1f7409d8360d97ac06babd395 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Mar 2023 22:11:31 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- grayskull/__main__.py | 1 + grayskull/strategy/py_base.py | 4 ++- grayskull/strategy/py_toml.py | 43 +++++++++++++++++++----------- grayskull/strategy/pypi.py | 11 ++++---- tests/test_poetry.py | 50 +++++++++++++++++++++++++---------- tests/test_tox.py | 1 + 6 files changed, 75 insertions(+), 35 deletions(-) diff --git a/grayskull/__main__.py b/grayskull/__main__.py index 25364d6c3..aee19bdf0 100644 --- a/grayskull/__main__.py +++ b/grayskull/__main__.py @@ -20,6 +20,7 @@ init(autoreset=True) logging.basicConfig(format="%(levelname)s:%(message)s") + def init_parser(): # create the top-level parser parser = argparse.ArgumentParser(description="Grayskull - Conda recipe generator") diff --git a/grayskull/strategy/py_base.py b/grayskull/strategy/py_base.py index ddd3c0cc7..76a185535 100644 --- a/grayskull/strategy/py_base.py +++ b/grayskull/strategy/py_base.py @@ -733,7 +733,9 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) -> pyproject_metadata["requirements"]["run"], ) if pyproject_metadata["requirements"]["run_constrained"]: - setup_metadata["requirements_run_constrained"] = pyproject_metadata["requirements"]["run_constrained"] + setup_metadata["requirements_run_constrained"] = pyproject_metadata[ + "requirements" + ]["run_constrained"] return setup_metadata diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index b06e618f3..3ec2d09c4 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -1,10 +1,10 @@ +import re from collections import defaultdict from pathlib import Path -from typing import Union, Dict, Optional, Tuple -import re +from typing import Dict, Optional, Tuple, Union -import tomli import semver +import tomli from grayskull.utils import nested_dict @@ -21,12 +21,15 @@ re.VERBOSE, ) + class InvalidVersion(BaseException): pass + class InvalidPoetryDependency(BaseException): pass + def parse_version(version: str) -> Dict[str, Optional[str]]: """ Parses a version string (not necessarily semver) to a dictionary with keys @@ -35,21 +38,22 @@ def parse_version(version: str) -> Dict[str, Optional[str]]: match = VERSION_REGEX.search(version) if not match: raise InvalidVersion(f"Could not parse version {version}.") - + return { - key: None if value is None else int(value) for key, value in match.groupdict().items() + key: None if value is None else int(value) + for key, value in match.groupdict().items() } + def vdict_to_vinfo(version_dict: Dict[str, Optional[str]]) -> semver.VersionInfo: """ Coerces version dictionary to a semver.VersionInfo object. If minor or patch numbers are missing, 0 is substituted in their place. """ - ver = { - key: 0 if value is None else value for key, value in version_dict.items() - } + ver = {key: 0 if value is None else value for key, value in version_dict.items()} return semver.VersionInfo(**ver) + def coerce_to_semver(version: str) -> str: """ Coerces a version string to a semantic version. @@ -59,6 +63,7 @@ def coerce_to_semver(version: str) -> str: return str(vdict_to_vinfo(parse_version(version))) + def get_caret_ceiling(target: str) -> str: """ Accepts a Poetry caret target and returns the exclusive version ceiling. @@ -100,6 +105,7 @@ def get_caret_ceiling(target: str) -> str: else: return str(target_vinfo.bump_major()) + def get_tilde_ceiling(target: str) -> str: """ Accepts a Poetry tilde target and returns the exclusive version ceiling. @@ -107,21 +113,22 @@ def get_tilde_ceiling(target: str) -> str: target_dict = parse_version(target) if target_dict["minor"]: return str(vdict_to_vinfo(target_dict).bump_minor()) - + return str(vdict_to_vinfo(target_dict).bump_major()) + def encode_poetry_version(poetry_specifier: str) -> str: """ Encodes Poetry version specifier as a Conda version specifier. Example: ^1 => >=1.0.0,<2.0.0 """ - poetry_clauses = poetry_specifier.split(',') + poetry_clauses = poetry_specifier.split(",") conda_clauses = [] for poetry_clause in poetry_clauses: poetry_clause = poetry_clause.replace(" ", "") - if poetry_clause.startswith('^'): + if poetry_clause.startswith("^"): # handle ^ operator target = poetry_clause[1:] floor = coerce_to_semver(target) @@ -130,7 +137,7 @@ def encode_poetry_version(poetry_specifier: str) -> str: conda_clauses.append("<" + ceiling) continue - if poetry_clause.startswith('~'): + if poetry_clause.startswith("~"): # handle ~ operator target = poetry_clause[1:] floor = coerce_to_semver(target) @@ -144,6 +151,7 @@ def encode_poetry_version(poetry_specifier: str) -> str: return ",".join(conda_clauses) + def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: run = [] run_constrained = [] @@ -163,10 +171,13 @@ def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: run.append(f"{dep_name} {conda_version}") continue - raise InvalidPoetryDependency(f"Expected Poetry dependency specification to be of type str or dict, received {type(dep_spec).__name__}") - + raise InvalidPoetryDependency( + f"Expected Poetry dependency specification to be of type str or dict, received {type(dep_spec).__name__}" + ) + return run, run_constrained + def add_poetry_metadata(metadata: dict, toml_metadata: dict) -> dict: if not is_poetry_present(toml_metadata): return metadata @@ -188,7 +199,9 @@ def add_poetry_metadata(metadata: dict, toml_metadata: dict) -> dict: if "poetry" not in host_metadata and "poetry-core" not in host_metadata: metadata["requirements"]["host"] = host_metadata + ["poetry-core"] - poetry_test_deps = poetry_metadata.get("group", {}).get("test", {}).get("dependencies", {}) + poetry_test_deps = ( + poetry_metadata.get("group", {}).get("test", {}).get("dependencies", {}) + ) # add required test dependencies and ignore optional test dependencies, as # there doesn't appear to be a way to specify them in Conda recipe metadata. test_reqs, _ = encode_poetry_deps(poetry_test_deps) diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 3d389b22b..c407ce0a5 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -107,7 +107,7 @@ def get_val(key): "extras_require": get_val("extras_require"), "requires_dist": requires_dist, "sdist_path": get_val("sdist_path"), - "requirements_run_constrained": get_val("requirements_run_constrained") + "requirements_run_constrained": get_val("requirements_run_constrained"), } @@ -577,10 +577,11 @@ def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]] } ) - if "requirements_run_constrained" in metadata and metadata["requirements_run_constrained"]: - result.update({ - "run_constrained": metadata["requirements_run_constrained"] - }) + if ( + "requirements_run_constrained" in metadata + and metadata["requirements_run_constrained"] + ): + result.update({"run_constrained": metadata["requirements_run_constrained"]}) update_requirements_with_pin(result) return result diff --git a/tests/test_poetry.py b/tests/test_poetry.py index c41006ba3..7201e0ccc 100644 --- a/tests/test_poetry.py +++ b/tests/test_poetry.py @@ -1,19 +1,29 @@ """Unit and integration tests for recipifying Poetry projects.""" -import pytest -from pathlib import Path import filecmp +from pathlib import Path -from grayskull.__main__ import init_parser, generate_recipes_from_list -from grayskull.utils import generate_recipe -from grayskull.strategy.py_toml import add_poetry_metadata, get_all_toml_info, InvalidVersion, get_caret_ceiling, get_tilde_ceiling, parse_version, encode_poetry_version +import pytest + +from grayskull.__main__ import generate_recipes_from_list, init_parser from grayskull.config import Configuration +from grayskull.strategy.py_toml import ( + InvalidVersion, + add_poetry_metadata, + encode_poetry_version, + get_all_toml_info, + get_caret_ceiling, + get_tilde_ceiling, + parse_version, +) +from grayskull.utils import generate_recipe + def test_parse_version(): - assert parse_version("0") == { "major": 0, "minor": None, "patch": None } - assert parse_version("1") == { "major": 1, "minor": None, "patch": None } - assert parse_version("1.2") == { "major": 1, "minor": 2, "patch": None } - assert parse_version("1.2.3") == { "major": 1, "minor": 2, "patch": 3 } + assert parse_version("0") == {"major": 0, "minor": None, "patch": None} + assert parse_version("1") == {"major": 1, "minor": None, "patch": None} + assert parse_version("1.2") == {"major": 1, "minor": 2, "patch": None} + assert parse_version("1.2.3") == {"major": 1, "minor": 2, "patch": 3} with pytest.raises(InvalidVersion): parse_version("asdf") @@ -77,7 +87,9 @@ def test_add_poetry_metadata(): "tool": { "poetry": { "dependencies": {"tomli": ">=1.0.0", "requests": ">=1.0.0"}, - "group": {"test": {"dependencies": {"tox": ">=1.0.0", "pytest": ">=1.0.0"}}}, + "group": { + "test": {"dependencies": {"tox": ">=1.0.0", "pytest": ">=1.0.0"}} + }, } } } @@ -91,11 +103,19 @@ def test_add_poetry_metadata(): assert add_poetry_metadata(metadata, toml_metadata) == { "requirements": { "host": ["pkg_host1 >=1.0.0", "pkg_host2", "poetry-core"], - "run": ["pkg_run1", "pkg_run2 >=2.0.0", "tomli >=1.0.0", "requests >=1.0.0"], + "run": [ + "pkg_run1", + "pkg_run2 >=2.0.0", + "tomli >=1.0.0", + "requests >=1.0.0", + ], + }, + "test": { + "requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest >=1.0.0"] }, - "test": {"requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest >=1.0.0"]}, } + def test_poetry_dependencies(): toml_path = Path(__file__).parent / "data" / "poetry" / "poetry.toml" result = get_all_toml_info(toml_path) @@ -109,9 +129,12 @@ def test_poetry_dependencies(): "urllib3 >=1.26.0,<2.0.0", ] + def test_poetry_langchain_snapshot(tmpdir): """Snapshot test that asserts correct recipifying of an example Poetry project.""" - snapshot_path = Path(__file__).parent / "data" / "poetry" / "langchain-expected.yaml" + snapshot_path = ( + Path(__file__).parent / "data" / "poetry" / "langchain-expected.yaml" + ) output_path = tmpdir / "langchain" / "meta.yaml" parser = init_parser() @@ -119,4 +142,3 @@ def test_poetry_langchain_snapshot(tmpdir): generate_recipes_from_list(args.pypi_packages, args) assert filecmp.cmp(snapshot_path, output_path, shallow=False) - diff --git a/tests/test_tox.py b/tests/test_tox.py index 5cd34a7e0..e3b68d138 100644 --- a/tests/test_tox.py +++ b/tests/test_tox.py @@ -4,6 +4,7 @@ from grayskull.strategy.py_toml import get_all_toml_info + def test_get_all_toml_info(): toml_path = Path(__file__).parent / "data" / "tox" / "tox.toml" From 0f76637ca82226f475a36b23263270421f84faef Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Wed, 22 Mar 2023 22:15:49 +0000 Subject: [PATCH 3/8] flake8 fixes --- grayskull/strategy/py_toml.py | 9 +++++---- tests/test_poetry.py | 2 -- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 3ec2d09c4..9656db300 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -84,9 +84,9 @@ def get_caret_ceiling(target: str) -> str: target_dict = parse_version(target) if target_dict["major"] == 0: - if target_dict["minor"] == None: + if target_dict["minor"] is None: target_dict["major"] += 1 - elif target_dict["patch"] == None: + elif target_dict["patch"] is None: target_dict["minor"] += 1 else: target_dict["patch"] += 1 @@ -160,7 +160,7 @@ def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: if isinstance(dep_spec, dict): conda_version = encode_poetry_version(dep_spec["version"]) - if dep_spec.get("optional", False) == True: + if dep_spec.get("optional", False) is True: run_constrained.append(f"{dep_name} {conda_version}") else: run.append(f"{dep_name} {conda_version}") @@ -172,7 +172,8 @@ def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: continue raise InvalidPoetryDependency( - f"Expected Poetry dependency specification to be of type str or dict, received {type(dep_spec).__name__}" + "Expected Poetry dependency specification to be of type str or dict, " + f"received {type(dep_spec).__name__}" ) return run, run_constrained diff --git a/tests/test_poetry.py b/tests/test_poetry.py index 7201e0ccc..8eadabf33 100644 --- a/tests/test_poetry.py +++ b/tests/test_poetry.py @@ -6,7 +6,6 @@ import pytest from grayskull.__main__ import generate_recipes_from_list, init_parser -from grayskull.config import Configuration from grayskull.strategy.py_toml import ( InvalidVersion, add_poetry_metadata, @@ -16,7 +15,6 @@ get_tilde_ceiling, parse_version, ) -from grayskull.utils import generate_recipe def test_parse_version(): From 310f95980cfa85f5426b2695a3ed7a97101dcf6e Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Thu, 23 Mar 2023 17:38:01 +0000 Subject: [PATCH 4/8] add semver to environment.yaml --- environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yaml b/environment.yaml index fb1d5e639..1c801084f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -28,3 +28,4 @@ dependencies: - tomli-w - libcblas - beautifulsoup4 + - semver From 6a8005fca9f379b91ad932c1fca601067732347c Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Sun, 26 Mar 2023 11:23:09 -0700 Subject: [PATCH 5/8] parametrize tests --- tests/test_poetry.py | 125 ++++++++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/tests/test_poetry.py b/tests/test_poetry.py index 8eadabf33..429128d10 100644 --- a/tests/test_poetry.py +++ b/tests/test_poetry.py @@ -17,67 +17,82 @@ ) -def test_parse_version(): - assert parse_version("0") == {"major": 0, "minor": None, "patch": None} - assert parse_version("1") == {"major": 1, "minor": None, "patch": None} - assert parse_version("1.2") == {"major": 1, "minor": 2, "patch": None} - assert parse_version("1.2.3") == {"major": 1, "minor": 2, "patch": 3} - - with pytest.raises(InvalidVersion): - parse_version("asdf") - with pytest.raises(InvalidVersion): - parse_version("") - with pytest.raises(InvalidVersion): - parse_version(".") - - -def test_get_caret_ceiling(): - # examples from Poetry docs - assert get_caret_ceiling("0") == "1.0.0" - assert get_caret_ceiling("0.0") == "0.1.0" - assert get_caret_ceiling("0.0.3") == "0.0.4" - assert get_caret_ceiling("0.2.3") == "0.3.0" - assert get_caret_ceiling("1") == "2.0.0" - assert get_caret_ceiling("1.2") == "2.0.0" - assert get_caret_ceiling("1.2.3") == "2.0.0" +@pytest.mark.parametrize( + "version, major, minor, patch", + [ + ("0", 0, None, None), + ("1", 1, None, None), + ("1.2", 1, 2, None), + ("1.2.3", 1, 2, 3), + ], +) +def test_parse_version_success(version, major, minor, patch): + assert parse_version(version) == {"major": major, "minor": minor, "patch": patch} -def test_get_tilde_ceiling(): +@pytest.mark.parametrize("invalid_version", ["asdf", "", "."]) +def test_parse_version_failure(invalid_version): + with pytest.raises(InvalidVersion): + parse_version(invalid_version) + + +@pytest.mark.parametrize( + "version, ceiling_version", + [ + ("0", "1.0.0"), + ("0.0", "0.1.0"), + ("0.0.3", "0.0.4"), + ("0.2.3", "0.3.0"), + ("1", "2.0.0"), + ("1.2", "2.0.0"), + ("1.2.3", "2.0.0"), + ], +) +def test_get_caret_ceiling(version, ceiling_version): # examples from Poetry docs - assert get_tilde_ceiling("1") == "2.0.0" - assert get_tilde_ceiling("1.2") == "1.3.0" - assert get_tilde_ceiling("1.2.3") == "1.3.0" - - -def test_encode_poetry_version(): - # should be unchanged - assert encode_poetry_version("1.*") == "1.*" - assert encode_poetry_version(">=1,<2") == ">=1,<2" - assert encode_poetry_version("==1.2.3") == "==1.2.3" - assert encode_poetry_version("!=1.2.3") == "!=1.2.3" + assert get_caret_ceiling(version) == ceiling_version - # strip spaces - assert encode_poetry_version(">= 1, < 2") == ">=1,<2" - # handle exact version specifiers correctly - assert encode_poetry_version("1.2.3") == "1.2.3" - assert encode_poetry_version("==1.2.3") == "==1.2.3" - - # handle caret operator correctly - # examples from Poetry docs - assert encode_poetry_version("^0") == ">=0.0.0,<1.0.0" - assert encode_poetry_version("^0.0") == ">=0.0.0,<0.1.0" - assert encode_poetry_version("^0.0.3") == ">=0.0.3,<0.0.4" - assert encode_poetry_version("^0.2.3") == ">=0.2.3,<0.3.0" - assert encode_poetry_version("^1") == ">=1.0.0,<2.0.0" - assert encode_poetry_version("^1.2") == ">=1.2.0,<2.0.0" - assert encode_poetry_version("^1.2.3") == ">=1.2.3,<2.0.0" - - # handle tilde operator correctly +@pytest.mark.parametrize( + "version, ceiling_version", + [("1", "2.0.0"), ("1.2", "1.3.0"), ("1.2.3", "1.3.0")], +) +def test_get_tilde_ceiling(version, ceiling_version): # examples from Poetry docs - assert encode_poetry_version("~1") == ">=1.0.0,<2.0.0" - assert encode_poetry_version("~1.2") == ">=1.2.0,<1.3.0" - assert encode_poetry_version("~1.2.3") == ">=1.2.3,<1.3.0" + assert get_tilde_ceiling(version) == ceiling_version + + +@pytest.mark.parametrize( + "version, encoded_version", + [ + # should be unchanged + ("1.*", "1.*"), + (">=1,<2", ">=1,<2"), + ("==1.2.3", "==1.2.3"), + ("!=1.2.3", "!=1.2.3"), + # strip spaces + (">= 1, < 2", ">=1,<2"), + # handle exact version specifiers correctly + ("1.2.3", "1.2.3"), + ("==1.2.3", "==1.2.3"), + # handle caret operator correctly + # examples from Poetry docs + ("^0", ">=0.0.0,<1.0.0"), + ("^0.0", ">=0.0.0,<0.1.0"), + ("^0.0.3", ">=0.0.3,<0.0.4"), + ("^0.2.3", ">=0.2.3,<0.3.0"), + ("^1", ">=1.0.0,<2.0.0"), + ("^1.2", ">=1.2.0,<2.0.0"), + ("^1.2.3", ">=1.2.3,<2.0.0"), + # handle tilde operator correctly + # examples from Poetry docs + ("~1", ">=1.0.0,<2.0.0"), + ("~1.2", ">=1.2.0,<1.3.0"), + ("~1.2.3", ">=1.2.3,<1.3.0"), + ], +) +def test_encode_poetry_version(version, encoded_version): + assert encode_poetry_version(version) == encoded_version def test_add_poetry_metadata(): From d19ed3a26b4688a96b191eb54019c335d07adb32 Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Sun, 26 Mar 2023 11:24:45 -0700 Subject: [PATCH 6/8] remove dev logs --- grayskull/strategy/pypi.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index c407ce0a5..6ad80da6e 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -383,14 +383,10 @@ def get_metadata(recipe, config) -> dict: requirements_section = extract_requirements(metadata, config, recipe) optional_requirements = extract_optional_requirements(metadata, config) for key in requirements_section: - log.error(key) - log.error(requirements_section[key]) requirements_section[key] = normalize_requirements_list( requirements_section[key], config ) for key in optional_requirements: - log.error(key) - log.error(optional_requirements[key]) optional_requirements[key] = normalize_requirements_list( optional_requirements[key], config ) From 39758935b92dcae9a4b61d03d0b669366c6f350f Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Sun, 26 Mar 2023 11:25:31 -0700 Subject: [PATCH 7/8] simplify if statement --- grayskull/strategy/pypi.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 6ad80da6e..a38dc9012 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -573,10 +573,7 @@ def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]] } ) - if ( - "requirements_run_constrained" in metadata - and metadata["requirements_run_constrained"] - ): + if metadata.get("requirements_run_constrained", None): result.update({"run_constrained": metadata["requirements_run_constrained"]}) update_requirements_with_pin(result) return result From 0f9200d5afe7b791984c0db3abb10aa434a59f7c Mon Sep 17 00:00:00 2001 From: "David L. Qiu" Date: Mon, 27 Mar 2023 08:34:00 -0700 Subject: [PATCH 8/8] use functools.singledispatch --- grayskull/strategy/py_toml.py | 47 ++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 9656db300..147fd8adb 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -1,5 +1,6 @@ import re from collections import defaultdict +from functools import singledispatch from pathlib import Path from typing import Dict, Optional, Tuple, Union @@ -152,30 +153,36 @@ def encode_poetry_version(poetry_specifier: str) -> str: return ",".join(conda_clauses) -def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: - run = [] - run_constrained = [] - for dep_name, dep_spec in poetry_deps.items(): - dep_name = dep_name.strip() +@singledispatch +def get_constrained_dep(dep_spec, dep_name): + raise InvalidPoetryDependency( + "Expected Poetry dependency specification to be of type str or dict, " + f"received {type(dep_spec).__name__}" + ) - if isinstance(dep_spec, dict): - conda_version = encode_poetry_version(dep_spec["version"]) - if dep_spec.get("optional", False) is True: - run_constrained.append(f"{dep_name} {conda_version}") - else: - run.append(f"{dep_name} {conda_version}") - continue - if isinstance(dep_spec, str): - conda_version = encode_poetry_version(dep_spec) - run.append(f"{dep_name} {conda_version}") - continue +@get_constrained_dep.register +def __get_constrained_dep_dict(dep_spec: dict, dep_name: str): + conda_version = encode_poetry_version(dep_spec["version"]) + return f"{dep_name} {conda_version}" + - raise InvalidPoetryDependency( - "Expected Poetry dependency specification to be of type str or dict, " - f"received {type(dep_spec).__name__}" - ) +@get_constrained_dep.register +def __get_constrained_dep_str(dep_spec: str, dep_name: str): + conda_version = encode_poetry_version(dep_spec) + return f"{dep_name} {conda_version}" + +def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: + run = [] + run_constrained = [] + for dep_name, dep_spec in poetry_deps.items(): + constrained_dep = get_constrained_dep(dep_spec, dep_name) + try: + assert dep_spec.get("optional", False) + run_constrained.append(constrained_dep) + except (AttributeError, AssertionError): + run.append(constrained_dep) return run, run_constrained