Add a proper parser for index urls internally

- Fix invocation of vcs hash retrieval
- Add error handling for pip url retrieval

Signed-off-by: Dan Ryan <dan@danryan.co>

pipenv/core.py

@@ -39,6 +39,7 @@
     rmtree,
     fs_str,
     clean_resolved_dep,
+    parse_indexes
 )
 from ._compat import TemporaryDirectory, Path
 from . import environments, pep508checker, progress
@@ -215,10 +216,14 @@ def import_requirements(r=None, dev=False):
     with open(r, "r") as f:
         contents = f.read()
     indexes = []
+    trusted_hosts = []
     # Find and add extra indexes.
     for line in contents.split("\n"):
-        if line.startswith(("-i ", "--index ", "--index-url ")):
-            indexes.append(line.split()[1])
+        line_indexes, _trusted_hosts, _ = parse_indexes(line.strip())
+        indexes.extend(line_indexes)
+        trusted_hosts.extend(_trusted_hosts)
+    indexes = sorted(set(indexes))
+    trusted_hosts = sorted(set(trusted_hosts))
     reqs = [f for f in parse_requirements(r, session=pip_requests)]
     for package in reqs:
         if package.name not in BAD_PACKAGES:
@@ -232,7 +237,8 @@ def import_requirements(r=None, dev=False):
             else:
                 project.add_package_to_pipfile(str(package.req), dev=dev)
     for index in indexes:
-        project.add_index_to_pipfile(index)
+        trusted = index in trusted_hosts
+        project.add_index_to_pipfile(index, trusted_host=trusted)
     project.recase_pipfile()
 
 
@@ -741,17 +747,14 @@ def cleanup_procs(procs, concurrent):
     for dep, ignore_hash, block in deps_list_bar:
         if len(procs) < PIPENV_MAX_SUBPROCESS:
             # Use a specific index, if specified.
+            indexes, trusted_hosts, dep = parse_indexes(dep)
             index = None
-            if " --index" in dep:
-                dep, _, index = dep.partition(" --index")
-                index = index.lstrip("=")
-            elif " -i " in dep:
-                dep, _, index = dep.partition(" -i ")
             extra_indexes = []
-            if "--extra-index-url" in dep:
-                split_dep = dep.split("--extra-index-url")
-                dep, extra_indexes = split_dep[0], split_dep[1:]
-            dep = Requirement.from_line(dep)
+            if indexes:
+                index = indexes[0]
+                if len(indexes) > 0:
+                    extra_indexes = indexes[1:]
+            dep = Requirement.from_line(" ".join(dep))
             if index:
                 _index = None
                 try:
@@ -1335,7 +1338,7 @@ def pip_install(
         sources = [{"url": index}]
         if extra_indexes:
             if isinstance(extra_indexes, six.string_types):
-                extra_indexes = [extra_indexes]
+                extra_indexes = [extra_indexes,]
             for idx in extra_indexes:
                 try:
                     extra_src = project.find_source(idx).get("url")

pipenv/patched/notpip/_internal/index.py

@@ -477,7 +477,10 @@ def find_all_candidates(self, project_name):
 
         page_versions = []
         for page in self._get_pages(url_locations, project_name):
-            logger.debug('Analyzing links from page %s', page.url)
+            try:
+                logger.debug('Analyzing links from page %s', page.url)
+            except AttributeError:
+                continue
             with indent_log():
                 page_versions.extend(
                     self._package_versions(page.links, search)

pipenv/project.py

@@ -792,11 +792,25 @@ def add_package_to_pipfile(self, package, dev=False):
         # Write Pipfile.
         self.write_toml(p)
 
-    def add_index_to_pipfile(self, index):
+    def add_index_to_pipfile(self, index, verify_ssl=True):
         """Adds a given index to the Pipfile."""
         # Read and append Pipfile.
         p = self.parsed_pipfile
-        source = {"url": index, "verify_ssl": True}
+        try:
+            self.get_source(url=index)
+        except SourceNotFound:
+            source = {"url": index, "verify_ssl": verify_ssl}
+        else:
+            return
+        name, _, tld_guess = six.moves.urllib.parse.urlsplit(index).netloc.rpartition(".")
+        src_name = name.replace(".", "")
+        try:
+            self.get_source(name=src_name)
+        except SourceNotFound:
+            source[name] = src_name
+        else:
+            from random import randint
+            source[name] = "{0}-{1}".format(src_name, randint(1, 1000))
         # Add the package to the group.
         if "source" not in p:
             p["source"] = [source]

pipenv/utils.py

@@ -250,8 +250,10 @@ class PipCommand(basecommand.Command):
         if not dep:
             continue
         url = None
-        if " -i " in dep:
-            dep, url = dep.split(" -i ")
+        indexes, trusted_hosts, remainder = parse_indexes(dep)
+        if indexes:
+            url = indexes[0]
+        dep = " ".join(remainder)
         req = Requirement.from_line(dep)
 
         # extra_constraints = []
@@ -524,15 +526,17 @@ def is_pinned(val):
     return isinstance(val, six.string_types) and val.startswith("==")
 
 
-def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
+def convert_deps_to_pip(deps, project=None, r=True, include_index=True):
     """"Converts a Pipfile-formatted dependency to a pip-formatted one."""
     from ._compat import NamedTemporaryFile
     from .vendor.requirementslib import Requirement
 
     dependencies = []
     for dep_name, dep in deps.items():
-        indexes = project.sources if hasattr(project, "sources") else None
+        indexes = project.sources if hasattr(project, "sources") else []
         new_dep = Requirement.from_pipfile(dep_name, dep)
+        if new_dep.index:
+            include_index = True
         req = new_dep.as_line(sources=indexes if include_index else None).strip()
         dependencies.append(req)
     if not r:
@@ -1169,7 +1173,7 @@ def get_vcs_deps(
         name = requirement.normalized_name
         commit_hash = None
         if requirement.is_vcs:
-            with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
+            with locked_repository(requirement) as repo:
                 commit_hash = repo.get_commit_hash()
                 lockfile[name] = requirement.pipfile_entry[1]
                 lockfile[name]['ref'] = commit_hash
@@ -1307,6 +1311,22 @@ def is_virtual_environment(path):
     return False
 
 
+@contextmanager
+def locked_repository(requirement):
+    from pipenv.vendor.vistir.path import create_tracked_tempdir
+    src_dir = create_tracked_tempdir(prefix="pipenv-src")
+    if not requirement.is_vcs:
+        return
+    original_base = os.environ.pop("PIP_SHIMS_BASE_MODULE", None)
+    os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip")
+    try:
+        with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
+            yield repo
+    finally:
+        if original_base:
+            os.environ["PIP_SHIMS_BASE_MODULE"] = original_base
+
+
 @contextmanager
 def chdir(path):
     """Context manager to change working directories."""
@@ -1326,3 +1346,19 @@ def chdir(path):
 def looks_like_dir(path):
     seps = (sep for sep in (os.path.sep, os.path.altsep) if sep is not None)
     return any(sep in path for sep in seps)
+
+
+def parse_indexes(line):
+    from argparse import ArgumentParser
+    parser = ArgumentParser("indexes")
+    parser.add_argument("--index", "-i", "--index-url", metavar="index_url",
+                                                            action="store", nargs="?",)
+    parser.add_argument("--extra-index-url", "--extra-index", metavar="extra_indexes",
+                                                            action="append")
+    parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append")
+    args, remainder = parser.parse_known_args(line.split())
+    index = [] if not args.index else [args.index,]
+    extra_indexes = [] if not args.extra_index_url else args.extra_index_url
+    indexes = index + extra_indexes
+    trusted_hosts = args.trusted_host if args.trusted_host else []
+    return indexes, trusted_hosts, remainder

tasks/vendoring/patches/patched/pip18.patch

@@ -493,3 +493,19 @@ index fcf9d3d3..d8aff848 100644
              SETUPTOOLS_SHIM % req.setup_py
          ] + list(self.global_options)
 
+diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py
+index 793dd1cb..426880e9 100644
+--- a/pipenv/patched/pip/_internal/index.py
++++ b/pipenv/patched/pip/_internal/index.py
+@@ -477,7 +477,10 @@ class PackageFinder(object):
+
+         page_versions = []
+         for page in self._get_pages(url_locations, project_name):
+-            logger.debug('Analyzing links from page %s', page.url)
++            try:
++                logger.debug('Analyzing links from page %s', page.url)
++            except AttributeError:
++                continue
+             with indent_log():
+                 page_versions.extend(
+                     self._package_versions(page.links, search)