Merge pull request #148 from marcelotrevisani/fb-improvements-package…

…-availability

Fb improvements package availability

grayskull/__main__.py

@@ -73,6 +73,13 @@ def main(args=None):
         help="Disable or enable stdout, if it is False, Grayskull"
         " will disable the prints. Default is True",
     )
+    pypi_cmds.add_argument(
+        "--list-missing-deps",
+        default=False,
+        action="store_true",
+        dest="list_missing_deps",
+        help="After the execution Grayskull will print all the missing dependencies.",
+    )
 
     args = parser.parse_args(args)
 
@@ -90,6 +97,7 @@ def main(args=None):
         return
 
     CLIConfig().stdout = args.stdout
+    CLIConfig().list_missing_deps = args.list_missing_deps
 
     print_msg(Style.RESET_ALL)
     print_msg(clear_screen())

grayskull/base/pkg_info.py

@@ -0,0 +1,51 @@
+import re
+from functools import lru_cache
+from typing import List, Optional, Tuple
+
+import requests
+
+
+@lru_cache(maxsize=35)
+def is_pkg_available(pkg_name: str, channel: str = "conda-forge") -> bool:
+    """Verify if the package is available on Anaconda for a specific channel.
+
+    :param pkg_name: Package name
+    :param channel: Anaconda channel
+    :return: Return True if the package is present on the given channel
+    """
+    response = requests.get(
+        url=f"https://anaconda.org/{channel}/{pkg_name}/files", allow_redirects=False
+    )
+    return response.status_code == 200
+
+
+def check_pkgs_availability(
+    list_pkgs: List[str], channel: Optional[str] = None
+) -> List[Tuple[str, bool]]:
+    """Check if the list is
+
+    :param list_pkgs: List with packages name
+    :return:
+    """
+    list_pkgs.sort()
+    re_search = re.compile(r"^\s*[a-z0-9\.\-\_]+", re.IGNORECASE)
+
+    result_list = []
+    all_pkg = set()
+    for pkg in list_pkgs:
+        if not pkg:
+            continue
+        search_result = re_search.search(pkg)
+        if not search_result:
+            continue
+
+        pkg_name = search_result.group()
+        if pkg_name in all_pkg:
+            continue
+
+        all_pkg.add(pkg_name)
+        if channel:
+            result_list.append((pkg, is_pkg_available(pkg_name, channel)))
+        else:
+            result_list.append((pkg, is_pkg_available(pkg_name)))
+    return result_list

grayskull/cli/__init__.py

@@ -15,8 +15,9 @@
 class CLIConfig:
     __instance: Optional["CLIConfig"] = None
 
-    def __new__(cls, stdout: bool = False):
+    def __new__(cls, stdout: bool = False, list_missing_deps: bool = False):
         if CLIConfig.__instance is None:
             CLIConfig.__instance = object.__new__(cls)
             CLIConfig.__instance.stdout = stdout
+            CLIConfig.__instance.list_missing_deps = list_missing_deps
         return CLIConfig.__instance

grayskull/cli/stdout.py

@@ -1,8 +1,13 @@
+import re
 from contextlib import contextmanager
 from copy import deepcopy
+from typing import Dict
 
+import progressbar
+from colorama import Fore, Style
 from progressbar import ProgressBar
 
+from grayskull.base.pkg_info import is_pkg_available
 from grayskull.cli import WIDGET_BAR_DOWNLOAD, CLIConfig
 
 
@@ -25,3 +30,76 @@ def update(self, *args, **kargs):
                 pass
 
         yield DisabledBar()
+
+
+@contextmanager
+def progressbar_with_status(max_value: int):
+    if CLIConfig().stdout:
+        with ProgressBar(
+            widgets=[
+                " ",
+                progressbar.Percentage(),
+                " ",
+                progressbar.Bar(),
+                "[",
+                progressbar.Timer(),
+                "]",
+            ],
+            prefix="Checking >> {variables.pkg_name}",
+            variables={"pkg_name": "--"},
+            max_value=max_value,
+        ) as bar:
+            yield bar
+    else:
+
+        class DisabledBar:
+            def update(self, *args, **kargs):
+                pass
+
+        yield DisabledBar()
+
+
+def print_requirements(all_requirements: Dict):
+    if not CLIConfig().stdout:
+        return
+
+    re_search = re.compile(r"^\s*([a-z0-9\.\-\_]+)(.*)", re.IGNORECASE | re.DOTALL)
+    all_missing_deps = set()
+
+    def print_req(list_pkg):
+        if isinstance(list_pkg, str):
+            list_pkg = [list_pkg]
+        for pkg in list_pkg:
+            if not pkg:
+                continue
+
+            search_result = re_search.search(pkg)
+            if pkg.strip().startswith("{{") or pkg.strip().startswith("<{"):
+                pkg_name = pkg.replace("<{", "{{")
+                options = ""
+                colour = Fore.GREEN
+            elif search_result:
+                pkg_name, options = search_result.groups()
+                if is_pkg_available(pkg_name):
+                    colour = Fore.GREEN
+                else:
+                    all_missing_deps.add(pkg_name)
+                    colour = Fore.RED
+            else:
+                continue
+            print_msg(f"  - {colour}{Style.BRIGHT}{pkg_name}{Style.RESET_ALL}{options}")
+
+    if all_requirements.get("build"):
+        print_msg("Build requirements:")
+        print_req(sorted(all_requirements.get("build", [])))
+    print_msg("Host requirements:")
+    print_req(sorted(all_requirements.get("host", [])))
+    print_msg("\nRun requirements:")
+    print_req(sorted(all_requirements.get("run", [])))
+    print_msg(f"\n{Fore.RED}RED{Style.RESET_ALL}: Missing packages")
+    print_msg(f"{Fore.GREEN}GREEN{Style.RESET_ALL}: Packages available on conda-forge")
+    if CLIConfig().list_missing_deps:
+        if all_missing_deps:
+            print_msg(f"Missing dependencies: {', '.join(all_missing_deps)}")
+        else:
+            print_msg("All dependencies are already on conda-forge.")

grayskull/pypi/pypi.py

@@ -20,8 +20,14 @@
 from requests import HTTPError
 
 from grayskull.base.base_recipe import AbstractRecipeModel
+from grayskull.base.pkg_info import is_pkg_available
 from grayskull.base.track_packages import solve_list_pkg_name
-from grayskull.cli.stdout import manage_progressbar, print_msg
+from grayskull.cli.stdout import (
+    manage_progressbar,
+    print_msg,
+    print_requirements,
+    progressbar_with_status,
+)
 from grayskull.license.discovery import ShortLicense, search_license_file
 from grayskull.utils import get_vendored_dependencies
 
@@ -255,8 +261,9 @@ def __fake_distutils_setup(*args, **kwargs):
                 f" {traceback.format_exc()}"
             )
             yield data_dist
-        core.setup = setup_core_original
-        os.chdir(old_dir)
+        finally:
+            core.setup = setup_core_original
+            os.chdir(old_dir)
 
     @staticmethod
     def __run_setup_py(
@@ -456,19 +463,22 @@ def _merge_requires_dist(pypi_metadata: dict, sdist_metadata: dict) -> List:
 
         requires_dist = []
         pypi_deps_name = set()
-        for sdist_pkg in all_deps:
-            match_deps = PyPi.RE_DEPS_NAME.match(sdist_pkg)
-            if not match_deps:
-                continue
-            match_deps = match_deps.group(0).strip()
-            pkg_name = PyPi._normalize_pkg_name(match_deps)
-            if current_pkg and current_pkg == pkg_name:
-                continue
-            if pkg_name in pypi_deps_name:
-                continue
+        with progressbar_with_status(len(all_deps)) as bar:
+            for pos, sdist_pkg in enumerate(all_deps, 1):
+                match_deps = PyPi.RE_DEPS_NAME.match(sdist_pkg)
+                if not match_deps:
+                    bar.update(pos)
+                    continue
+                match_deps = match_deps.group(0).strip()
+                pkg_name = PyPi._normalize_pkg_name(match_deps)
+                bar.update(pos, pkg_name=pkg_name)
+                if current_pkg and current_pkg == pkg_name:
+                    continue
+                if pkg_name in pypi_deps_name:
+                    continue
 
-            pypi_deps_name.add(pkg_name)
-            requires_dist.append(sdist_pkg.replace(match_deps, pkg_name))
+                pypi_deps_name.add(pkg_name)
+                requires_dist.append(sdist_pkg.replace(match_deps, pkg_name))
         return requires_dist
 
     @staticmethod
@@ -534,17 +544,7 @@ def _get_metadata(self) -> dict:
         all_requirements["run"] = solve_list_pkg_name(
             all_requirements["run"], self.PYPI_CONFIG
         )
-
-        def print_req(name, list_req: List):
-            prefix_req = f"\n   - {Fore.LIGHTCYAN_EX}"
-            print_msg(
-                f"{name} requirements:" f"{prefix_req}{prefix_req.join(list_req)}"
-            )
-
-        if all_requirements.get("build"):
-            print_req("Build", all_requirements["build"])
-        print_req("Host", all_requirements["host"])
-        print_req("run", all_requirements["run"])
+        print_requirements(all_requirements)
 
         test_entry_points = PyPi._get_test_entry_points(metadata.get("entry_points"))
         test_imports = PyPi._get_test_imports(metadata, pypi_metadata["name"])
@@ -1080,24 +1080,6 @@ def get_small_py3_version(list_py_ver: List[PyVer]) -> PyVer:
             return py_ver
 
 
-@lru_cache(maxsize=20)
-def is_pkg_available(pkg_name: str, channel: str = "conda-forge") -> bool:
-    response = requests.get(
-        url=f"https://anaconda.org/{channel}/{pkg_name}/files", allow_redirects=False
-    )
-    status_response = response.status_code == 200
-    if status_response:
-        msg_pkg = f"{Style.BRIGHT}{Fore.GREEN}Available"
-    else:
-        msg_pkg = f"{Style.BRIGHT}{Fore.RED}NOT Available"
-    print_msg(
-        f" - Package {Style.BRIGHT}{Fore.LIGHTCYAN_EX}{pkg_name}{Fore.RESET}:"
-        f" {msg_pkg}"
-        f" {Fore.RESET}on {channel}"
-    )
-    return status_response
-
-
 def search_setup_root(path_folder: Union[Path, str]) -> Path:
     setup_py = list(Path(path_folder).rglob("setup.py"))
     if setup_py:

tests/base/test_pkg_info.py

@@ -0,0 +1,29 @@
+from grayskull.base.pkg_info import check_pkgs_availability, is_pkg_available
+
+
+def test_pkg_available():
+    assert is_pkg_available("pytest")
+
+
+def test_pkg_not_available():
+    assert not is_pkg_available("NOT_PACKAGE_654987321")
+
+
+def test_check_pkgs_availability(capsys):
+    all_pkgs = check_pkgs_availability(
+        [
+            "pytest >=4.0.0,<5.0.0  # [win]",
+            "requests",
+            "pandas[test]",
+            "NOT_PACKAGE_13248743",
+        ]
+    )
+
+    assert sorted(all_pkgs) == sorted(
+        [
+            ("pytest >=4.0.0,<5.0.0  # [win]", True),
+            ("requests", True),
+            ("pandas[test]", True),
+            ("NOT_PACKAGE_13248743", False),
+        ]
+    )

tests/test_track_packages.py → tests/base/test_track_packages.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 import pytest
@@ -17,7 +16,7 @@
 
 @pytest.fixture
 def path_example() -> Path:
-    return Path(os.path.dirname(__file__)) / "data" / "track_package" / "example.yaml"
+    return Path(__file__).parent.parent / "data" / "track_package" / "example.yaml"
 
 
 def test_config_pkg():

tests/test_pypi.py

@@ -392,10 +392,7 @@ def test_requests_recipe_extra_deps(capsys):
     assert "win-inet-pton" not in recipe["requirements"]["run"]
     assert recipe["build"]["noarch"]
     assert not recipe["build"]["skip"]
-    assert (
-        f"Package {Style.BRIGHT}{Fore.LIGHTCYAN_EX}urllib3{Fore.RESET}:"
-        f" {Style.BRIGHT}{Fore.GREEN}Available" in captured_stdout.out
-    )
+    assert f"{Fore.GREEN}{Style.BRIGHT}python" in captured_stdout.out
 
 
 def test_zipp_recipe_tags_on_deps():