Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix tests #552

Merged
merged 9 commits into from
Aug 18, 2024
Merged

Fix tests #552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
dc20702
add simpler license matcher
dholth Jul 12, 2024
ad767e8
Fix --pypi-url
beenje Aug 17, 2024
b4df151
Fix meta.yaml output directory
beenje Aug 16, 2024
b51f964
Replace pypi.io with pypi.org
beenje Aug 17, 2024
26b6d6e
Explicitely return None
beenje Aug 17, 2024
059ba34
Fix test_ciso_recipe
beenje Aug 18, 2024
4fe560d
Fix test_injection_distutils_compiler_gsw
beenje Aug 18, 2024
21960de
Fix test_merge_pypi_sdist_metadata
beenje Aug 18, 2024
284a501
Skip test_console_script_toml_format for python 3.12
beenje Aug 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 23 additions & 5 deletions grayskull/license/discovery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,19 @@ def get_all_licenses_from_spdx() -> List:
]


def _match_scrambled_exact(candidate, licenses) -> str | None:
Fixed Show fixed Hide fixed
"""
Return license with rearranged word order only.

Fancy scorer confuses BSD-3-Clause with DEC-3-Clause.
"""
bag = set(re.findall(r"\w+", candidate.lower()))
for license in licenses:
if bag == set(re.findall(r"\w+", license.lower())):
return license
return None


def match_license(name: str) -> dict:
"""Match if the given license name matches any license present on
spdx.org
Expand All @@ -75,11 +88,16 @@ def match_license(name: str) -> dict:
name = re.sub(r"\s+license\s*", "", name.strip(), flags=re.IGNORECASE)
name = name.strip()

best_matches = process.extract(
name, _get_all_license_choice(all_licenses), scorer=partial_ratio
)
best_matches = process.extract(name, [lc for lc, *_ in best_matches])
spdx_license = best_matches[0]
exact_match = _match_scrambled_exact(name, _get_all_license_choice(all_licenses))
if exact_match:
best_matches = [(exact_match, 100, 0)]
spdx_license = best_matches[0]
else:
best_matches = process.extract(
name, _get_all_license_choice(all_licenses), scorer=partial_ratio
)
best_matches = process.extract(name, [lc for lc, *_ in best_matches])
spdx_license = best_matches[0]

if spdx_license[1] < 100:
# Prefer "-or-later" licenses over the "-only"
Expand Down
5 changes: 4 additions & 1 deletion grayskull/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,10 @@ def generate_recipes_from_list(list_pkgs, args):
args.output = pkg_name
try:
# TODO: Remove before 3.0 release
if args.url_pypi_metadata_deprecated and args.url_pypi_metadata:
if (
args.url_pypi_metadata_deprecated
and args.url_pypi_metadata != DEFAULT_PYPI_META_URL
):
raise RuntimeError(
"--pypi-url is deprecated in favor of --pypi-url-metadata "
+ "and may not be passed in conjunction with --pypi-url-metadata"
Expand Down
4 changes: 3 additions & 1 deletion grayskull/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,9 @@ def generate_recipe(
logging.debug(f"Generating recipe on: {recipe_dir}")
if not recipe_dir.is_dir():
recipe_dir.mkdir()
recipe_path = recipe_dir / "recipe.yaml" if use_v1_format else "meta.yaml"
recipe_path = (
recipe_dir / "recipe.yaml" if use_v1_format else recipe_dir / "meta.yaml"
)
recipe_folder = recipe_dir
add_new_lines_after_section(recipe.yaml)

Expand Down
2 changes: 1 addition & 1 deletion tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def pkg_pytest(tmpdir_factory) -> str:
# Correct info should be extracted from the metadata and not filename
dest_pkg = str(folder / "PYTEST-PKG-1.0.0.tar.gz")
download_sdist_pkg(
"https://pypi.io/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg
"https://pypi.org/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg
)
shutil.unpack_archive(dest_pkg, str(folder))
return dest_pkg
Expand Down
2 changes: 1 addition & 1 deletion tests/data/poetry/langchain-expected.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package:
version: {{ version }}

source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/langchain-{{ version }}.tar.gz
url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/langchain-{{ version }}.tar.gz
sha256: 95a93c966b1a2ff056c43870747aba1c39924c145179f0b8ffa27fef6a525610

build:
Expand Down
53 changes: 39 additions & 14 deletions tests/test_pypi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ def test_get_extra_from_requires_dist():
def dask_sdist_metadata():
config = Configuration(name="dask")
return get_sdist_metadata(
"https://pypi.io/packages/source/d/dask/dask-2022.6.1.tar.gz",
"https://pypi.org/packages/source/d/dask/dask-2022.6.1.tar.gz",
config,
)

Expand Down Expand Up @@ -283,7 +283,7 @@ def test_compose_test_section_with_console_scripts():
config = Configuration(name="pytest", version="7.1.2")
metadata1 = get_pypi_metadata(config)
metadata2 = get_sdist_metadata(
"https://pypi.io/packages/source/p/pytest/pytest-7.1.2.tar.gz", config
"https://pypi.org/packages/source/p/pytest/pytest-7.1.2.tar.gz", config
)
metadata = merge_pypi_sdist_metadata(metadata1, metadata2, config)
test_requirements = []
Expand Down Expand Up @@ -604,7 +604,7 @@ def test_get_sha256_from_pypi_metadata():
def test_injection_distutils(name):
config = Configuration(name="hypothesis")
data = get_sdist_metadata(
"https://pypi.io/packages/source/h/hypothesis/hypothesis-5.5.1.tar.gz",
"https://pypi.org/packages/source/h/hypothesis/hypothesis-5.5.1.tar.gz",
config,
)
assert sorted(data["install_requires"]) == sorted(
Expand All @@ -621,7 +621,7 @@ def test_injection_distutils(name):
def test_injection_distutils_pytest():
config = Configuration(name="pytest", version="5.3.2")
data = get_sdist_metadata(
"https://pypi.io/packages/source/p/pytest/pytest-5.3.2.tar.gz", config
"https://pypi.org/packages/source/p/pytest/pytest-5.3.2.tar.gz", config
)
assert sorted(data["install_requires"]) == sorted(
[
Expand All @@ -644,31 +644,42 @@ def test_injection_distutils_pytest():


def test_injection_distutils_compiler_gsw():
config = Configuration(name="gsw", version="3.3.1")
config = Configuration(name="gsw", version="3.6.19")
data = get_sdist_metadata(
"https://pypi.io/packages/source/g/gsw/gsw-3.3.1.tar.gz", config
"https://pypi.org/packages/source/g/gsw/gsw-3.6.19.tar.gz", config
)
assert data.get("compilers") == ["c"]
assert data["packages"] == ["gsw"]
assert data["name"] == "gsw"


def test_injection_distutils_setup_reqs_ensure_list():
pkg_name, pkg_ver = "pyinstaller-hooks-contrib", "2020.7"
config = Configuration(name=pkg_name, version=pkg_ver)
data = get_sdist_metadata(
f"https://pypi.io/packages/source/p/{pkg_name}/{pkg_name}-{pkg_ver}.tar.gz",
f"https://pypi.org/packages/source/p/{pkg_name}/{pkg_name}-{pkg_ver}.tar.gz",
config,
)
assert data.get("setup_requires") == ["setuptools >= 30.3.0"]


def test_merge_pypi_sdist_metadata():
config = Configuration(name="gsw", version="3.3.1")
config = Configuration(name="gsw", version="3.6.19")
pypi_metadata = get_pypi_metadata(config)
sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config)
merged_data = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata, config)
assert merged_data["compilers"] == ["c"]
assert sorted(merged_data["setup_requires"]) == sorted(["numpy"])
assert sorted(merged_data["setup_requires"]) == sorted(
[
"build",
'numpy<3,>=2.0.0rc1; python_version >= "3.9"',
'oldest-supported-numpy; python_version < "3.9"',
"pip>9.0.1",
"setuptools>=42",
"setuptools_scm[toml]>=3.4",
"wheel",
"python >=3.8",
]
)


def test_update_requirements_with_pin():
Expand Down Expand Up @@ -809,13 +820,22 @@ def test_download_pkg_sdist(pkg_pytest):

def test_ciso_recipe():
recipe = GrayskullFactory.create_recipe(
"pypi", Configuration(name="ciso", version="0.1.0")
"pypi", Configuration(name="ciso", version="0.2.2")
)
assert sorted(recipe["requirements"]["host"]) == sorted(
["cython", "numpy", "pip", "python"]
[
"cython >=3",
"numpy >=2.0.0rc1",
"oldest-supported-numpy",
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We shouldn't have both numpy >=2.0.0rc1 and oldest-supported-numpy but it looks like grayskull doesn't support selectors?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

which selectors are you talking about? It does support a few of them but in this case they might not be being generated for some reason

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this is requirements/host. Don't think we want oldest-supported-numpy here

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see my comment here ;)
https://github.com/conda/grayskull/pull/552/files#r1721631804

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that equal over there is just a syntax sugar, but if you properly compare the numpy and oldest-supported-numpy you will see the selectors :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah! Indeed, looking at the generated meta.yaml, I see:

    - oldest-supported-numpy  # [py<39]
    - numpy >=2.0.0rc1  # [py>=39]

"pip",
"python >=3.9",
"setuptools >=41.2",
"setuptools-scm",
"wheel",
]
)
assert sorted(recipe["requirements"]["run"]) == sorted(
["cython", "python", "<{ pin_compatible('numpy') }}"]
["<{ pin_compatible('numpy') }}", "oldest-supported-numpy", "python >=3.9"]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also not sure why oldest-supported-numpy is added to requirements/run

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @jakirkham , at first sight it seems odd, indeed, but it comes directly from the requirements in that package gsw, it is not something that grayskull adds,
https://github.com/TEOS-10/GSW-Python/blob/c3407368ef01d0b3d8e85f8cf7b94abae52bcd25/pyproject.toml#L6

that is a separated package on pypi
https://pypi.org/project/oldest-supported-numpy/

Copy link
Contributor Author

@beenje beenje Aug 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed this is actually for the ciso project that has oldest-supported-numpy not only in build-system requirements , but in dependencies as well:

dependencies = [
    "oldest-supported-numpy ; python_version < '3.9'",
    "numpy>=2.0.0rc1 ; python_version >= '3.9'",
]

gsw has only numpy>=1.21 in dependencies (oldest-supported-numpy is only in the build-system).
oldest-supported-numpy doesn't appear the run requirements in that case.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's ok to have in pyproject.toml

However what this appears to be testing is whether this is added to requirements/host and requirements/run. The latter doesn't make sense

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it is going to appear in the requirements run because oldest-supported-numpy is specified in the dependencies section of the pyproject.toml. If it is was only in build section of pyproject.toml that would appear only in host then.

It is possible to do a workaround and always remove it from the requirements/run and always let it in host, but I am just not so sure if it covers all scenarios, what do you think?

Copy link
Member

@jakirkham jakirkham Aug 20, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should remove oldest-supported-numpy whenever it is seen. That's used by wheels specifically to configure builds with specific numpy versions (not sure why a wheel would use oldest-supported-numpy at runtime unless they are some kind of build tool)

In any event it doesn't make sense in the Conda case where we configure NumPy versions used in some conda_build_config.yaml or other variants

On the Conda side, we need only add numpy (unconstrained) to requirements/host and copy over any dependencies value of numpy to requirements/run

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, gotcha
That is pretty much a new PR to do this workaround as it was not a requirement before.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha thanks for the context! 🙏

The issue is oldest-supported-numpy doesn't exist in Conda. So if Grayskull adds it to recipes, they will be broken. Maintainers will need to know they can clean this out to fix the issue

In any event, we can discuss more in the new issue (thanks for raising! 🙏): #553

)
assert recipe["test"]["commands"] == ["pip check"]
assert recipe["test"]["requires"] == ["pip"]
Expand Down Expand Up @@ -1112,7 +1132,7 @@ def test_recipe_extension():
recipe = create_python_recipe("azure-identity=1.3.1")[0]
assert (
recipe["source"]["url"]
== "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/"
== "https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/"
"azure-identity-{{ version }}.zip"
)

Expand Down Expand Up @@ -1288,6 +1308,11 @@ def test_notice_file_different_licence():
assert recipe["about"]["license"] in ["MIT AND Apache-2.0", "Apache-2.0 AND MIT"]


# Need to find another package for this test
@pytest.mark.skipif(
sys.version_info >= (3, 12),
reason="consolemd setup.py requires lower than python 3.12",
)
def test_console_script_toml_format():
recipe, _ = create_python_recipe("consolemd", version="0.5.1")
assert recipe["build"]["entry_points"] == ["consolemd = consolemd.cli:cli"]
Expand Down
Loading