Consolidate pipeline (#3778)

Consolidate pipeline

.azure-pipelines/steps/build-package.yml

@@ -1,4 +1,22 @@
 steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: $(python.version)
+    architecture: '$(python.architecture)'
+    addToPath: true
+  displayName: Use Python $(python.version)
+
+- template: install-dependencies.yml
+
+- script: |
+    echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version)
+  env:
+    PYTHON_VERSION: $(python.version)
+
+- template: create-virtualenv.yml
+  parameters:
+    python_version: $(python.version)
+
 - script: |
     python -m pip install --upgrade wheel pip setuptools twine readme_renderer[md]
     python setup.py sdist bdist_wheel

.azure-pipelines/steps/create-virtualenv.yml

@@ -1,3 +1,6 @@
+parameters:
+  python_version: ''
+
 steps:
 
 - script: |
@@ -29,10 +32,13 @@ steps:
 - script: |
     echo "Python path: $(PY_EXE)"
     echo "GIT_SSL_CAINFO: $(GIT_SSL_CAINFO)"
-    echo "PIPENV PYTHON VERSION: $PIPENV_DEFAULT_PYTHON_VERSION"
+    echo "PIPENV PYTHON VERSION: $(python.version)"
+    echo "python_version: ${{ parameters.python_version }}"
+    git submodule sync
+    git submodule update --init --recursive
     $(PY_EXE) -m pipenv install --deploy --dev --python="$(PY_EXE)"
   env:
-    PIPENV_DEFAULT_PYTHON_VERSION: $(python.version)
+    PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }}
     PYTHONWARNINGS: 'ignore:DEPRECATION'
     PIPENV_NOSPIN: '1'
   displayName: Make Virtualenv

.azure-pipelines/steps/run-tests-linux.yml

@@ -4,15 +4,11 @@ parameters:
 steps:
 - script: |
     # Fix Git SSL errors
-    echo "Using pipenv python version: $PIPENV_DEFAULT_PYTHON_VERSION"
+    echo "Using pipenv python version: $(PIPENV_DEFAULT_PYTHON_VERSION)"
     git submodule sync && git submodule update --init --recursive
     pipenv run pytest --junitxml=test-results.xml
   displayName: Run integration tests
   env:
-    PY_EXE: $(PY_EXE)
-    GIT_SSL_CAINFO: $(GIT_SSL_CAINFO)
-    LANG: $(LANG)
-    PIP_PROCESS_DEPENDENCY_LINKS: '$(PIP_PROCESS_DEPENDENCY_LINKS)'
-    PIPENV_DEFAULT_PYTHON_VERSION: '${{ parameters.python_version }}'
     PYTHONWARNINGS: ignore:DEPRECATION
     PIPENV_NOSPIN: '1'
+    PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }}

.azure-pipelines/steps/run-tests-windows.yml

@@ -6,14 +6,16 @@ steps:
     subst T: "$env:TEMP"
     Write-Host "##vso[task.setvariable variable=TEMP]T:\"
     Write-Host "##vso[task.setvariable variable=TMP]T:\"
+    Write-Host "##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$env:PYTHON_VERSION"
+    Write-Host "##vso[task.setvariable variable=PIPENV_NOSPIN]1"
   displayName: Fix Temp Variable
+  env:
+    PYTHON_VERSION: ${{ parameters.python_version }}
 
 - script: |
-    echo "Using pipenv python version: $PIPENV_DEFAULT_PYTHON_VERSION"
     git submodule sync && git submodule update --init --recursive
     pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests
   displayName: Run integration tests
   env:
-    PIPENV_DEFAULT_PYTHON_VERSION: '${{ parameters.python_version }}'
     PYTHONWARNINGS: 'ignore:DEPRECATION'
     PIPENV_NOSPIN: '1'

.azure-pipelines/steps/run-tests.yml

@@ -9,7 +9,7 @@ steps:
 - template: install-dependencies.yml
 
 - script: |
-    echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$(python.version)'
+    echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version)
   env:
     PYTHON_VERSION: $(python.version)
 

.azure-pipelines/steps/run-vendor-scripts.yml

@@ -2,6 +2,24 @@ parameters:
   python_version: ''
 
 steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: $(python.version)
+    architecture: '$(python.architecture)'
+    addToPath: true
+  displayName: Use Python $(python.version)
+
+- template: install-dependencies.yml
+
+- script: |
+    echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version)
+  env:
+    PYTHON_VERSION: $(python.version)
+
+- template: create-virtualenv.yml
+  parameters:
+    python_version: $(python.version)
+
 - script: |
     python -m pip install --upgrade invoke requests parver bs4 vistir towncrier pip setuptools wheel --upgrade-strategy=eager
     python -m invoke vendoring.update

.gitmodules

@@ -25,3 +25,6 @@
 [submodule "tests/test_artifacts/git/requests-2.18.4"]
 	path = tests/test_artifacts/git/requests-2.18.4
 	url = https://github.com/requests/requests
+[submodule "tests/pypi"]
+	path = tests/pypi
+	url = https://github.com/sarugaku/pipenv-test-artifacts.git

azure-pipelines.yml

@@ -18,6 +18,9 @@ trigger:
     - .gitattributes
     - .editorconfig
 
+variables:
+- group: CI
+
 jobs:
 - job: TestLinux
   pool:
@@ -38,12 +41,28 @@ jobs:
   - template: .azure-pipelines/steps/run-tests.yml
     parameters:
       vmImage: 'Ubuntu-16.04'
-  - template: .azure-pipelines/steps/run-vendor-scripts.yml
-    parameters:
-      vmImage: 'Ubuntu-16.04'
-  - template: .azure-pipelines/steps/build-package.yml
-    parameters:
-      vmImage: 'Ubuntu-16.04'
+
+- job: TestVendoring
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  variables:
+    python.version: '3.7.2'
+    python.architecture: x64
+  steps:
+    - template: .azure-pipelines/steps/run-vendor-scripts.yml
+      parameters:
+        vmImage: 'Ubuntu-16.04'
+
+- job: TestPackaging
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  variables:
+    python.version: '3.7.2'
+    python.architecture: x64
+  steps:
+    - template: .azure-pipelines/steps/build-package.yml
+      parameters:
+        vmImage: 'Ubuntu-16.04'
 
 - job: TestWindows
   pool:

docs/dev/contributing.rst

@@ -128,10 +128,15 @@ Three ways of running the tests are as follows:
 
 1. ``make test`` (which uses ``docker``)
 2. ``./run-tests.sh`` or ``run-tests.bat``
-3. Using pipenv::
+3. Using pipenv:
 
-    pipenv install --dev
-    pipenv run pytest
+.. code-block:: console
+
+    $ git clone https://github.com/pypa/pipenv.git
+    $ cd pipenv
+    $ git submodule sync && git submodule update --init --recursive
+    $ pipenv install --dev
+    $ pipenv run pytest
 
 For the last two, it is important that your environment is setup correctly, and
 this may take some work, for example, on a specific Mac installation, the following

pipenv/cli/command.py

@@ -316,7 +316,10 @@ def lock(
     from ..core import ensure_project, do_init, do_lock
 
     # Ensure that virtualenv is available.
-    ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror)
+    ensure_project(
+        three=state.three, python=state.python, pypi_mirror=state.pypi_mirror,
+        warn=(not state.quiet)
+    )
     if state.installstate.requirementstxt:
         do_init(
             dev=state.installstate.dev,
@@ -330,6 +333,7 @@ def lock(
         pre=state.installstate.pre,
         keep_outdated=state.installstate.keep_outdated,
         pypi_mirror=state.pypi_mirror,
+        write=not state.quiet,
     )
 
 

pipenv/cli/options.py

@@ -6,7 +6,7 @@
 import click.types
 
 from click import (
-    BadParameter, Group, Option, argument, echo, make_pass_decorator, option
+    BadParameter, BadArgumentUsage, Group, Option, argument, echo, make_pass_decorator, option
 )
 from click_didyoumean import DYMMixin
 
@@ -56,6 +56,7 @@ def __init__(self):
         self.index = None
         self.extra_index_urls = []
         self.verbose = False
+        self.quiet = False
         self.pypi_mirror = None
         self.python = None
         self.two = None
@@ -231,12 +232,32 @@ def verbose_option(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
         if value:
+            if state.quiet:
+                raise BadArgumentUsage(
+                    "--verbose and --quiet are mutually exclusive! Please choose one!",
+                    ctx=ctx
+                )
             state.verbose = True
-        setup_verbosity(ctx, param, value)
+            setup_verbosity(ctx, param, 1)
     return option("--verbose", "-v", is_flag=True, expose_value=False,
                   callback=callback, help="Verbose mode.", type=click.types.BOOL)(f)
 
 
+def quiet_option(f):
+    def callback(ctx, param, value):
+        state = ctx.ensure_object(State)
+        if value:
+            if state.verbose:
+                raise BadArgumentUsage(
+                    "--verbose and --quiet are mutually exclusive! Please choose one!",
+                    ctx=ctx
+                )
+            state.quiet = True
+            setup_verbosity(ctx, param, -1)
+    return option("--quiet", "-q", is_flag=True, expose_value=False,
+                  callback=callback, help="Quiet mode.", type=click.types.BOOL)(f)
+
+
 def site_packages_option(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
@@ -313,8 +334,14 @@ def setup_verbosity(ctx, param, value):
     if not value:
         return
     import logging
-    logging.getLogger("pip").setLevel(logging.INFO)
-    environments.PIPENV_VERBOSITY = 1
+    loggers = ("pip", "piptools")
+    if value == 1:
+        for logger in loggers:
+            logging.getLogger(logger).setLevel(logging.INFO)
+    elif value == -1:
+        for logger in loggers:
+            logging.getLogger(logger).setLevel(logging.CRITICAL)
+    environments.PIPENV_VERBOSITY = value
 
 
 def validate_python_path(ctx, param, value):

pipenv/core.py

@@ -35,7 +35,7 @@
     get_canonical_names, is_pinned, is_pypi_url, is_required_version, is_star,
     is_valid_url, parse_indexes, pep423_name, prepare_pip_source_args,
     proper_case, python_version, venv_resolve_deps, run_command,
-    is_python_command, find_python
+    is_python_command, find_python, make_posix, interrupt_handled_subprocess
 )
 
 
@@ -371,6 +371,9 @@ def abort():
     if not python:
         python = PIPENV_DEFAULT_PYTHON_VERSION
     path_to_python = find_a_system_python(python)
+    if environments.is_verbose():
+        click.echo(u"Using python: {0}".format(python), err=True)
+        click.echo(u"Path to python: {0}".format(path_to_python), err=True)
     if not path_to_python and python is not None:
         # We need to install Python.
         click.echo(
@@ -885,11 +888,13 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None):
     )
 
     # Default to using sys.executable, if Python wasn't provided.
+    using_string = u"Using"
     if not python:
         python = sys.executable
+        using_string = "Using default python from"
     click.echo(
         u"{0} {1} {3} {2}".format(
-            crayons.normal("Using", bold=True),
+            crayons.normal(using_string, bold=True),
             crayons.red(python, bold=True),
             crayons.normal(fix_utf8("to create virtualenv…"), bold=True),
             crayons.green("({0})".format(python_version(python))),
@@ -919,21 +924,19 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None):
         pip_config = {}
 
     # Actually create the virtualenv.
+    error = None
     with create_spinner(u"Creating virtual environment...") as sp:
-        c = vistir.misc.run(
-            cmd, verbose=False, return_object=True, write_to_stdout=False,
-            combine_stderr=False, block=True, nospin=True, env=pip_config,
+        with interrupt_handled_subprocess(cmd, combine_stderr=False, env=pip_config) as c:
+            click.echo(crayons.blue(u"{0}".format(c.out)), err=True)
+            if c.returncode != 0:
+                error = c.err if environments.is_verbose() else exceptions.prettify_exc(c.err)
+                sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(u"Failed creating virtual environment"))
+            else:
+                sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format(u"Successfully created virtual environment!"))
+    if error is not None:
+        raise exceptions.VirtualenvCreationException(
+            extra=crayons.red("{0}".format(error))
         )
-        click.echo(crayons.blue(u"{0}".format(c.out)), err=True)
-        if c.returncode != 0:
-            sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(u"Failed creating virtual environment"))
-            error = c.err if environments.is_verbose() else exceptions.prettify_exc(c.err)
-            raise exceptions.VirtualenvCreationException(
-                extra=crayons.red("{0}".format(error))
-            )
-        else:
-
-            sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format(u"Successfully created virtual environment!"))
 
     # Associate project directory with the environment.
     # This mimics Pew's "setproject".

pipenv/environments.py

@@ -126,7 +126,7 @@ def _is_env_truthy(name):
 Default is 16, an arbitrary number that works most of the time.
 """
 
-PIPENV_MAX_SUBPROCESS = int(os.environ.get("PIPENV_MAX_SUBPROCESS", "16"))
+PIPENV_MAX_SUBPROCESS = int(os.environ.get("PIPENV_MAX_SUBPROCESS", "8"))
 """How many subprocesses should Pipenv use when installing.
 
 Default is 16, an arbitrary number that seems to work.
@@ -150,14 +150,12 @@ def _is_env_truthy(name):
 if PIPENV_IS_CI:
     PIPENV_NOSPIN = True
 
-PIPENV_SPINNER = "dots"
+PIPENV_SPINNER = "dots" if not os.name == "nt" else "bouncingBar"
 """Sets the default spinner type.
 
 Spinners are identitcal to the node.js spinners and can be found at
 https://github.com/sindresorhus/cli-spinners
 """
-if os.name == "nt":
-    PIPENV_SPINNER = "bouncingBar"
 
 PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE")
 """If set, this specifies a custom Pipfile location.