Fix tests (#552)

* add simpler license matcher

* Fix --pypi-url

args.url_pypi_metadata has a default value and will never be empty.
To check if both options aren't given, we have to check if it's different
fro the default value.

* Fix meta.yaml output directory

A small bug was introduced when adding v1 format.
The "meta.yaml" file was always created in the current directory instead of recipe_dir.

* Replace pypi.io with pypi.org

pypi.org is the official domain.

See pypi/warehouse#2184


Fix: Mixing implicit and explicit returns may indicate an error as implicit returns always return None

* Fix test_ciso_recipe

Bump version to 0.2.2.
Version 0.1.0 includes versioneer.py which isn't compatible with Python 3.12
(using SafeConfigParser).

* Fix test_injection_distutils_compiler_gsw

Bump gsw version to 3.6.19.
Version 3.3.1 includes versioneer.py which tries to import SafeConfigParser
that was removed in Python 3.12.

"packages" key isn't detected anymore as most metadata were moved
to pyproject.toml. Only C extension definition is left in setup.py.

* Fix test_merge_pypi_sdist_metadata

Bump gsw version to 3.6.19.
Version 3.3.1 includes versioneer.py which tries to import SafeConfigParser
that was removed in Python 3.12.

* Skip test_console_script_toml_format for python 3.12

Package setup.py not compatible with 3.12 (try to import imp).
New package to find for this test. Skip for now.

---------

Co-authored-by: Daniel Holth <dholth@anaconda.com>

grayskull/license/discovery.py

@@ -62,6 +62,19 @@ def get_all_licenses_from_spdx() -> List:
     ]
 
 
+def _match_scrambled_exact(candidate, licenses) -> str | None:
+    """
+    Return license with rearranged word order only.
+
+    Fancy scorer confuses BSD-3-Clause with DEC-3-Clause.
+    """
+    bag = set(re.findall(r"\w+", candidate.lower()))
+    for license in licenses:
+        if bag == set(re.findall(r"\w+", license.lower())):
+            return license
+    return None
+
+
 def match_license(name: str) -> dict:
     """Match if the given license name matches any license present on
     spdx.org
@@ -75,11 +88,16 @@ def match_license(name: str) -> dict:
     name = re.sub(r"\s+license\s*", "", name.strip(), flags=re.IGNORECASE)
     name = name.strip()
 
-    best_matches = process.extract(
-        name, _get_all_license_choice(all_licenses), scorer=partial_ratio
-    )
-    best_matches = process.extract(name, [lc for lc, *_ in best_matches])
-    spdx_license = best_matches[0]
+    exact_match = _match_scrambled_exact(name, _get_all_license_choice(all_licenses))
+    if exact_match:
+        best_matches = [(exact_match, 100, 0)]
+        spdx_license = best_matches[0]
+    else:
+        best_matches = process.extract(
+            name, _get_all_license_choice(all_licenses), scorer=partial_ratio
+        )
+        best_matches = process.extract(name, [lc for lc, *_ in best_matches])
+        spdx_license = best_matches[0]
 
     if spdx_license[1] < 100:
         # Prefer "-or-later" licenses over the "-only"

grayskull/main.py

@@ -332,7 +332,10 @@ def generate_recipes_from_list(list_pkgs, args):
             args.output = pkg_name
         try:
             # TODO: Remove before 3.0 release
-            if args.url_pypi_metadata_deprecated and args.url_pypi_metadata:
+            if (
+                args.url_pypi_metadata_deprecated
+                and args.url_pypi_metadata != DEFAULT_PYPI_META_URL
+            ):
                 raise RuntimeError(
                     "--pypi-url is deprecated in favor of --pypi-url-metadata "
                     + "and may not be passed in conjunction with --pypi-url-metadata"

grayskull/utils.py

@@ -218,7 +218,9 @@ def generate_recipe(
         logging.debug(f"Generating recipe on: {recipe_dir}")
         if not recipe_dir.is_dir():
             recipe_dir.mkdir()
-        recipe_path = recipe_dir / "recipe.yaml" if use_v1_format else "meta.yaml"
+        recipe_path = (
+            recipe_dir / "recipe.yaml" if use_v1_format else recipe_dir / "meta.yaml"
+        )
         recipe_folder = recipe_dir
         add_new_lines_after_section(recipe.yaml)
 

tests/conftest.py

@@ -19,7 +19,7 @@ def pkg_pytest(tmpdir_factory) -> str:
     # Correct info should be extracted from the metadata and not filename
     dest_pkg = str(folder / "PYTEST-PKG-1.0.0.tar.gz")
     download_sdist_pkg(
-        "https://pypi.io/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg
+        "https://pypi.org/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg
     )
     shutil.unpack_archive(dest_pkg, str(folder))
     return dest_pkg

tests/data/poetry/langchain-expected.yaml

@@ -6,7 +6,7 @@ package:
   version: {{ version }}
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/langchain-{{ version }}.tar.gz
+  url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/langchain-{{ version }}.tar.gz
   sha256: 95a93c966b1a2ff056c43870747aba1c39924c145179f0b8ffa27fef6a525610
 
 build:

tests/test_pypi.py

@@ -179,7 +179,7 @@ def test_get_extra_from_requires_dist():
 def dask_sdist_metadata():
     config = Configuration(name="dask")
     return get_sdist_metadata(
-        "https://pypi.io/packages/source/d/dask/dask-2022.6.1.tar.gz",
+        "https://pypi.org/packages/source/d/dask/dask-2022.6.1.tar.gz",
         config,
     )
 
@@ -283,7 +283,7 @@ def test_compose_test_section_with_console_scripts():
     config = Configuration(name="pytest", version="7.1.2")
     metadata1 = get_pypi_metadata(config)
     metadata2 = get_sdist_metadata(
-        "https://pypi.io/packages/source/p/pytest/pytest-7.1.2.tar.gz", config
+        "https://pypi.org/packages/source/p/pytest/pytest-7.1.2.tar.gz", config
     )
     metadata = merge_pypi_sdist_metadata(metadata1, metadata2, config)
     test_requirements = []
@@ -604,7 +604,7 @@ def test_get_sha256_from_pypi_metadata():
 def test_injection_distutils(name):
     config = Configuration(name="hypothesis")
     data = get_sdist_metadata(
-        "https://pypi.io/packages/source/h/hypothesis/hypothesis-5.5.1.tar.gz",
+        "https://pypi.org/packages/source/h/hypothesis/hypothesis-5.5.1.tar.gz",
         config,
     )
     assert sorted(data["install_requires"]) == sorted(
@@ -621,7 +621,7 @@ def test_injection_distutils(name):
 def test_injection_distutils_pytest():
     config = Configuration(name="pytest", version="5.3.2")
     data = get_sdist_metadata(
-        "https://pypi.io/packages/source/p/pytest/pytest-5.3.2.tar.gz", config
+        "https://pypi.org/packages/source/p/pytest/pytest-5.3.2.tar.gz", config
     )
     assert sorted(data["install_requires"]) == sorted(
         [
@@ -644,31 +644,42 @@ def test_injection_distutils_pytest():
 
 
 def test_injection_distutils_compiler_gsw():
-    config = Configuration(name="gsw", version="3.3.1")
+    config = Configuration(name="gsw", version="3.6.19")
     data = get_sdist_metadata(
-        "https://pypi.io/packages/source/g/gsw/gsw-3.3.1.tar.gz", config
+        "https://pypi.org/packages/source/g/gsw/gsw-3.6.19.tar.gz", config
     )
     assert data.get("compilers") == ["c"]
-    assert data["packages"] == ["gsw"]
+    assert data["name"] == "gsw"
 
 
 def test_injection_distutils_setup_reqs_ensure_list():
     pkg_name, pkg_ver = "pyinstaller-hooks-contrib", "2020.7"
     config = Configuration(name=pkg_name, version=pkg_ver)
     data = get_sdist_metadata(
-        f"https://pypi.io/packages/source/p/{pkg_name}/{pkg_name}-{pkg_ver}.tar.gz",
+        f"https://pypi.org/packages/source/p/{pkg_name}/{pkg_name}-{pkg_ver}.tar.gz",
         config,
     )
     assert data.get("setup_requires") == ["setuptools >= 30.3.0"]
 
 
 def test_merge_pypi_sdist_metadata():
-    config = Configuration(name="gsw", version="3.3.1")
+    config = Configuration(name="gsw", version="3.6.19")
     pypi_metadata = get_pypi_metadata(config)
     sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config)
     merged_data = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata, config)
     assert merged_data["compilers"] == ["c"]
-    assert sorted(merged_data["setup_requires"]) == sorted(["numpy"])
+    assert sorted(merged_data["setup_requires"]) == sorted(
+        [
+            "build",
+            'numpy<3,>=2.0.0rc1; python_version >= "3.9"',
+            'oldest-supported-numpy; python_version < "3.9"',
+            "pip>9.0.1",
+            "setuptools>=42",
+            "setuptools_scm[toml]>=3.4",
+            "wheel",
+            "python >=3.8",
+        ]
+    )
 
 
 def test_update_requirements_with_pin():
@@ -809,13 +820,22 @@ def test_download_pkg_sdist(pkg_pytest):
 
 def test_ciso_recipe():
     recipe = GrayskullFactory.create_recipe(
-        "pypi", Configuration(name="ciso", version="0.1.0")
+        "pypi", Configuration(name="ciso", version="0.2.2")
     )
     assert sorted(recipe["requirements"]["host"]) == sorted(
-        ["cython", "numpy", "pip", "python"]
+        [
+            "cython >=3",
+            "numpy >=2.0.0rc1",
+            "oldest-supported-numpy",
+            "pip",
+            "python >=3.9",
+            "setuptools >=41.2",
+            "setuptools-scm",
+            "wheel",
+        ]
     )
     assert sorted(recipe["requirements"]["run"]) == sorted(
-        ["cython", "python", "<{ pin_compatible('numpy') }}"]
+        ["<{ pin_compatible('numpy') }}", "oldest-supported-numpy", "python >=3.9"]
     )
     assert recipe["test"]["commands"] == ["pip check"]
     assert recipe["test"]["requires"] == ["pip"]
@@ -1112,7 +1132,7 @@ def test_recipe_extension():
     recipe = create_python_recipe("azure-identity=1.3.1")[0]
     assert (
         recipe["source"]["url"]
-        == "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/"
+        == "https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/"
         "azure-identity-{{ version }}.zip"
     )
 
@@ -1288,6 +1308,11 @@ def test_notice_file_different_licence():
     assert recipe["about"]["license"] in ["MIT AND Apache-2.0", "Apache-2.0 AND MIT"]
 
 
+# Need to find another package for this test
+@pytest.mark.skipif(
+    sys.version_info >= (3, 12),
+    reason="consolemd setup.py requires lower than python 3.12",
+)
 def test_console_script_toml_format():
     recipe, _ = create_python_recipe("consolemd", version="0.5.1")
     assert recipe["build"]["entry_points"] == ["consolemd = consolemd.cli:cli"]