Fix virtualenv path derivations

- Fix inadvertent occasional global installation of files
- Fix inadvertent occcasional global removal of files
- Fix empty output from `pipenv update --outdated`
- Fixes #2828
- Fixes #3113
- Fixes #3047
- Fixes #3055

Signed-off-by: Dan Ryan <dan@danryan.co>

news/2828.feature.rst

@@ -0,0 +1 @@
+Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date.

news/3047.bugfix.rst

@@ -0,0 +1 @@
+Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages.

news/3055.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash.

news/3113.bugfix.rst

@@ -1 +1 @@
-Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems.
+Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely.

pipenv/__init__.py

@@ -22,6 +22,7 @@
 from pipenv.vendor.vistir.compat import ResourceWarning, fs_str
 warnings.filterwarnings("ignore", category=DependencyWarning)
 warnings.filterwarnings("ignore", category=ResourceWarning)
+warnings.filterwarnings("ignore", category=UserWarning)
 
 if sys.version_info >= (3, 1) and sys.version_info <= (3, 6):
     if sys.stdout.isatty() and sys.stderr.isatty():

pipenv/core.py

@@ -41,7 +41,8 @@
     rmtree,
     clean_resolved_dep,
     parse_indexes,
-    escape_cmd
+    escape_cmd,
+    fix_venv_site
 )
 from . import environments, pep508checker, progress
 from .environments import (
@@ -1705,6 +1706,7 @@ def do_py(system=False):
 
 
 def do_outdated(pypi_mirror=None):
+    # TODO: Allow --skip-lock here?
     from .vendor.requirementslib.models.requirements import Requirement
 
     packages = {}
@@ -1729,6 +1731,9 @@ def do_outdated(pypi_mirror=None):
                 outdated.append(
                     (package, updated_packages[norm_name], packages[package])
                 )
+    if not outdated:
+        click.echo(crayons.green("All packages are up to date!", bold=True))
+        sys.exit(0)
     for package, new_version, old_version in outdated:
         click.echo(
             "Package {0!r} out-of-date: {1!r} installed, {2!r} available.".format(
@@ -2062,6 +2067,7 @@ def do_uninstall(
 ):
     from .environments import PIPENV_USE_SYSTEM
     from .vendor.requirementslib.models.requirements import Requirement
+    from .vendor.packaging.utils import canonicalize_name
 
     # Automatically use an activated virtualenv.
     if PIPENV_USE_SYSTEM:
@@ -2074,6 +2080,24 @@ def do_uninstall(
         Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p
     ]
     package_names = [p for p in packages if p] + editable_pkgs
+    installed_package_names = set([
+        canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages()
+    ])
+    # Intelligently detect if --dev should be used or not.
+    if project.lockfile_exists:
+        develop = set(
+            [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()]
+        )
+        default = set(
+            [canonicalize_name(k) for k in project.lockfile_content["default"].keys()]
+        )
+    else:
+        develop = set(
+            [canonicalize_name(k) for k in project.dev_packages.keys()]
+        )
+        default = set(
+            [canonicalize_name(k) for k in project.packages.keys()]
+        )
     pipfile_remove = True
     # Un-install all dependencies, if --all was provided.
     if all is True:
@@ -2084,7 +2108,7 @@ def do_uninstall(
         return
     # Uninstall [dev-packages], if --dev was provided.
     if all_dev:
-        if "dev-packages" not in project.parsed_pipfile:
+        if "dev-packages" not in project.parsed_pipfile and not develop:
             click.echo(
                 crayons.normal(
                     "No {0} to uninstall.".format(crayons.red("[dev-packages]")),
@@ -2097,40 +2121,64 @@ def do_uninstall(
                 fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True
             )
         )
-        package_names = project.dev_packages.keys()
     if packages is False and editable_packages is False and not all_dev:
         click.echo(crayons.red("No package provided!"), err=True)
         return 1
-    for package_name in package_names:
-        click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name))))
-        cmd = "{0} uninstall {1} -y".format(
-            escape_grouped_arguments(which_pip(allow_global=system)), package_name
+    fix_venv_site(project.env_paths["lib"])
+    # Remove known "bad packages" from the list.
+    for bad_package in BAD_PACKAGES:
+        if canonicalize_name(bad_package) in package_names:
+            if environments.is_verbose():
+                click.echo("Ignoring {0}.".format(repr(bad_package)), err=True)
+            del package_names[package_names.index(
+                canonicalize_name(bad_package)
+            )]
+    used_packages = (develop | default) & installed_package_names
+    failure = False
+    packages_to_remove = set()
+    if all_dev:
+        packages_to_remove |= develop & installed_package_names
+    package_names = set([canonicalize_name(pkg_name) for pkg_name in package_names])
+    packages_to_remove = package_names & used_packages
+    for package_name in packages_to_remove:
+        click.echo(
+            crayons.white(
+                fix_utf8("Uninstalling {0}…".format(repr(package_name))), bold=True
+            )
         )
+        # Uninstall the package.
+        cmd = "{0} uninstall {1} -y".format(
+                    escape_grouped_arguments(which_pip()), package_name
+                )
         if environments.is_verbose():
             click.echo("$ {0}".format(cmd))
         c = delegator.run(cmd)
         click.echo(crayons.blue(c.out))
-        if pipfile_remove:
-            in_packages = project.get_package_name_in_pipfile(package_name, dev=False)
-            in_dev_packages = project.get_package_name_in_pipfile(
-                package_name, dev=True
-            )
-            if not in_dev_packages and not in_packages:
-                click.echo(
-                    "No package {0} to remove from Pipfile.".format(
-                        crayons.green(package_name)
-                    )
+        if c.return_code != 0:
+            failure = True
+        else:
+            if pipfile_remove:
+                in_packages = project.get_package_name_in_pipfile(package_name, dev=False)
+                in_dev_packages = project.get_package_name_in_pipfile(
+                    package_name, dev=True
                 )
-                continue
+                if not in_dev_packages and not in_packages:
+                    click.echo(
+                        "No package {0} to remove from Pipfile.".format(
+                            crayons.green(package_name)
+                        )
+                    )
+                    continue
 
-            click.echo(
-                fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name)))
-            )
-            # Remove package from both packages and dev-packages.
-            project.remove_package_from_pipfile(package_name, dev=True)
-            project.remove_package_from_pipfile(package_name, dev=False)
+                click.echo(
+                    fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name)))
+                )
+                # Remove package from both packages and dev-packages.
+                project.remove_package_from_pipfile(package_name, dev=True)
+                project.remove_package_from_pipfile(package_name, dev=False)
     if lock:
         do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror)
+    sys.exit(int(failure))
 
 
 def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None):
@@ -2593,6 +2641,9 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro
     from packaging.utils import canonicalize_name
     ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror)
     ensure_lockfile(pypi_mirror=pypi_mirror)
+    # Make sure that the virtualenv's site packages are configured correctly
+    # otherwise we may end up removing from the global site packages directory
+    fix_venv_site(project.env_paths["lib"])
     installed_package_names = [
         canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages()
     ]

pipenv/project.py

@@ -5,6 +5,7 @@
 import re
 import sys
 import base64
+import itertools
 import fnmatch
 import hashlib
 import contoml
@@ -33,6 +34,7 @@
     get_workon_home,
     is_virtual_environment,
     looks_like_dir,
+    sys_version
 )
 from .environments import (
     PIPENV_MAX_DEPTH,
@@ -42,6 +44,7 @@
     PIPENV_TEST_INDEX,
     PIPENV_PYTHON,
     PIPENV_DEFAULT_PYTHON_VERSION,
+    PIPENV_CACHE_DIR
 )
 from requirementslib.utils import is_vcs
 
@@ -301,9 +304,9 @@ def find_egg(self, egg_dist):
             user_site = site.USER_SITE
         search_locations = [site_packages, user_site]
         for site_directory in search_locations:
-                egg = os.path.join(site_directory, search_filename)
-                if os.path.isfile(egg):
-                    return egg
+            egg = os.path.join(site_directory, search_filename)
+            if os.path.isfile(egg):
+                return egg
 
     def locate_dist(self, dist):
         location = self.find_egg(dist)
@@ -325,6 +328,71 @@ def get_installed_packages(self):
             packages = [pkg for pkg in packages]
         return packages
 
+    def get_package_info(self):
+        from .utils import prepare_pip_source_args
+        from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder
+        index_urls = [source.get("url") for source in self.sources]
+
+        class PipCommand(Command):
+            name = "PipCommand"
+
+        dependency_links = []
+        packages = self.get_installed_packages()
+        # This code is borrowed from pip's current implementation
+        for dist in packages:
+            if dist.has_metadata('dependency_links.txt'):
+                dependency_links.extend(dist.get_metadata_lines('dependency_links.txt'))
+
+        pip_command = PipCommand()
+        index_opts = cmdoptions.make_option_group(
+            index_group, pip_command.parser
+        )
+        cmd_opts = pip_command.cmd_opts
+        pip_command.parser.insert_option_group(0, index_opts)
+        pip_command.parser.insert_option_group(0, cmd_opts)
+        pip_args = prepare_pip_source_args(self.sources, [])
+        pip_options, _ = pip_command.parser.parse_args(pip_args)
+        pip_options.cache_dir = PIPENV_CACHE_DIR
+        pip_options.pre = self.settings.get("pre", False)
+        with pip_command._build_session(pip_options) as session:
+            finder = PackageFinder(
+                find_links=pip_options.find_links,
+                index_urls=index_urls, allow_all_prereleases=pip_options.pre,
+                trusted_hosts=pip_options.trusted_hosts,
+                process_dependency_links=pip_options.process_dependency_links,
+                session=session
+            )
+            finder.add_dependency_links(dependency_links)
+
+            for dist in packages:
+                typ = 'unknown'
+                all_candidates = finder.find_all_candidates(dist.key)
+                if not pip_options.pre:
+                    # Remove prereleases
+                    all_candidates = [
+                        candidate for candidate in all_candidates
+                        if not candidate.version.is_prerelease
+                    ]
+
+                if not all_candidates:
+                    continue
+                best_candidate = max(all_candidates, key=finder._candidate_sort_key)
+                remote_version = best_candidate.version
+                if best_candidate.location.is_wheel:
+                    typ = 'wheel'
+                else:
+                    typ = 'sdist'
+                # This is dirty but makes the rest of the code much cleaner
+                dist.latest_version = remote_version
+                dist.latest_filetype = typ
+                yield dist
+
+    def get_outdated_packages(self):
+        return [
+            pkg for pkg in self.get_package_info()
+            if pkg.latest_version._version > pkg.parsed_version._version
+        ]
+
     @classmethod
     def _sanitize(cls, name):
         # Replace dangerous characters into '_'. The length of the sanitized
@@ -975,13 +1043,19 @@ def proper_case_section(self, section):
         # Return whether or not values have been changed.
         return changed_values
 
+    @property
+    def py_version(self):
+        py_path = self.which("python")
+        version = python_version(py_path)
+        return version
+
     @property
     def _pyversion(self):
         include_dir = vistir.compat.Path(self.virtualenv_location) / "include"
         python_path = next((x for x in include_dir.iterdir() if x.name.startswith("python")), None)
         if python_path:
-            python_version = python_path.name.replace("python", "")
-            py_version_short, abiflags = python_version[:3], python_version[3:]
+            py_version = python_path.name.replace("python", "")
+            py_version_short, abiflags = py_version[:3], py_version[3:]
             return {"py_version_short": py_version_short, "abiflags": abiflags}
         return {}
 
@@ -990,8 +1064,10 @@ def env_paths(self):
         location = self.virtualenv_location if self.virtualenv_location else sys.prefix
         prefix = vistir.compat.Path(location)
         import importlib
+        py_version = tuple([int(v) for v in self.py_version.split(".")])
         try:
-            _virtualenv = importlib.import_module("virtualenv")
+            with sys_version(py_version):
+                _virtualenv = importlib.import_module("virtualenv")
         except ImportError:
             with vistir.contextmanagers.temp_path():
                 from string import Formatter
@@ -1015,9 +1091,11 @@ def env_paths(self):
                     sys.path = [
                         os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"),
                     ] + sys.path
-                    six.reload_module(importlib)
-                    _virtualenv = importlib.import_module("virtualenv")
-        home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix())
+                    with sys_version(py_version):
+                        six.reload_module(importlib)
+                        _virtualenv = importlib.import_module("virtualenv")
+        with sys_version(py_version):
+            home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix())
         paths = {
             "lib": lib,
             "include": inc,
@@ -1031,8 +1109,10 @@ def env_paths(self):
     @cached_property
     def finders(self):
         from .vendor.pythonfinder import Finder
+        scripts_dirname = "Scripts" if os.name == "nt" else "bin"
+        scripts_dir = os.path.join(self.virtualenv_location, scripts_dirname)
         finders = [
-            Finder(path=self.env_paths["scripts"], global_search=gs, system=False)
+            Finder(path=scripts_dir, global_search=gs, system=False)
             for gs in (False, True)
         ]
         return finders

pipenv/utils.py

@@ -372,7 +372,7 @@ def venv_resolve_deps(
             result = None
             while True:
                 try:
-                    result = c.expect(u"\n", timeout=-1)
+                    result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT)
                 except (EOF, TIMEOUT):
                     pass
                 if result is None:
@@ -1341,3 +1341,17 @@ def fix_venv_site(venv_lib_dir):
         fp.write(site_contents)
     # Make sure bytecode is up-to-date too.
     assert compileall.compile_file(str(site_py), quiet=1, force=True)
+
+
+@contextmanager
+def sys_version(version_tuple):
+    """
+    Set a temporary sys.version_info tuple
+
+    :param version_tuple: a fake sys.version_info tuple
+    """
+
+    old_version = sys.version_info
+    sys.version_info = version_tuple
+    yield
+    sys.version_info = old_version