Upgrade to pex 1.4.8 and eliminate workarounds.

Pex now allows us to float setuptools high and run on python 3.7 and
it supports sane platform expansion as well as graceful handling of
non-standard `setuptools` platform reporting for Apple system
interpreters allowing us to eliminate several workarounds.

We also pickup a fix for concurrent pex extraction which is particularly
useful when running pants from a pex using `PEX_FORCE_LOCAL`.

Fixes pantsbuild#5922
Fixes pantsbuild#6363

.travis.yml

@@ -241,12 +241,6 @@ matrix:
       osx_image: xcode8.3
       stage: Test Pants
       language: generic
-      env:
-        # Specifically avoid the OSX provided 2.7.10 under xcode8.3 since it returns a platform
-        # of `macosx-*-intel` where the `intel` suffix is bogus but pex has not yet been taught to
-        # deal with this. Can be removed when this issue is resolved:
-        # https://github.com/pantsbuild/pex/issues/523
-        - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>2.7.10,<3']"
       before_install:
         - brew tap caskroom/cask && brew update && brew cask install osxfuse
       before_script:

3rdparty/python/requirements.txt

@@ -14,7 +14,7 @@ mock==2.0.0
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.0
-pex==1.4.5
+pex==1.4.8
 psutil==4.3.0
 pycodestyle==2.4.0
 pyflakes==2.0.0
@@ -27,9 +27,9 @@ pywatchman==1.4.1
 requests[security]>=2.5.0,<2.19
 scandir==1.2
 setproctitle==1.1.10
-setuptools==33.1.1
+setuptools==40.4.3
 six>=1.9.0,<2
 subprocess32==3.2.7 ; python_version<'3'
 thrift>=0.9.1
-wheel==0.29.0
+wheel==0.31.1
 www-authenticate==0.9.2

build-support/bin/ci.sh

@@ -130,9 +130,7 @@ fi
 if [[ "${python_three:-false}" == "true" ]]; then
   # The 3.4 end of this constraint is necessary to jive with the travis ubuntu trusty image.
   banner "Setting interpreter constraints for 3!"
-  # TODO(John Sirois): Allow `<4` when the issues with `3.7` are fixed. See:
-  #   https://github.com/pantsbuild/pants/issues/6363
-  export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<3.7"]'
+  export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<4"]'
   # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache
   # bug between the `./pants binary` and further runs.
   ./pants.pex clean-all

src/python/pants/backend/python/BUILD

@@ -41,7 +41,6 @@ python_library(
   dependencies = [
     '3rdparty/python:future',
     '3rdparty/python:pex',
-    ':pex_util',
     'src/python/pants/backend/python/targets',
     'src/python/pants/base:exceptions',
     'src/python/pants/process',
@@ -88,10 +87,3 @@ python_library(
   ]
 )
 
-python_library(
-  name = 'pex_util',
-  sources = ['pex_util.py'],
-  dependencies = [
-    '3rdparty/python:pex',
-  ]
-)

src/python/pants/backend/python/interpreter_cache.py

@@ -14,7 +14,6 @@
 from pex.resolver import resolve
 from pex.variables import Variables
 
-from pants.backend.python.pex_util import expand_and_maybe_adjust_platform
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
 from pants.process.lock import OwnerPrintingInterProcessFileLock
@@ -225,11 +224,9 @@ def _resolve_and_link(self, interpreter, requirement, target_link):
     distributions = resolve(requirements=[requirement],
                             fetchers=self._python_repos.get_fetchers(),
                             interpreter=interpreter,
-                            platform=expand_and_maybe_adjust_platform(
-                              interpreter=interpreter,
-                              # The local interpreter cache is, by definition, composed of
-                              # interpreters for the 'current' platform.
-                              platform='current'),
+                            # The local interpreter cache is, by definition, composed of
+                            # interpreters for the 'current' platform.
+                            platform='current',
                             context=self._python_repos.get_network_context(),
                             precedence=precedence)
     if not distributions:

src/python/pants/backend/python/subsystems/python_setup.py

@@ -26,9 +26,9 @@ def register_options(cls, register):
                   "or 'PyPy' (A pypy interpreter of any version). Multiple constraint strings will "
                   "be ORed together. These constraints are applied in addition to any "
                   "compatibilities required by the relevant targets.")
-    register('--setuptools-version', advanced=True, default='33.1.1',
+    register('--setuptools-version', advanced=True, default='40.4.3',
              help='The setuptools version for this python environment.')
-    register('--wheel-version', advanced=True, default='0.29.0',
+    register('--wheel-version', advanced=True, default='0.31.1',
              help='The wheel version for this python environment.')
     register('--platforms', advanced=True, type=list, metavar='<platform>', default=['current'],
              help='A list of platforms to be supported by this python environment. Each platform'

src/python/pants/backend/python/tasks/BUILD

@@ -13,7 +13,6 @@ python_library(
     'src/python/pants/backend/native/targets',
     'src/python/pants/backend/native/tasks',
     'src/python/pants/backend/python:interpreter_cache',
-    'src/python/pants/backend/python:pex_util',
     'src/python/pants/backend/python:python_requirement',
     'src/python/pants/backend/python/subsystems',
     'src/python/pants/backend/python/targets',

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -12,7 +12,6 @@
 from pex.resolver import resolve
 from twitter.common.collections import OrderedSet
 
-from pants.backend.python.pex_util import expand_and_maybe_adjust_platform
 from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
@@ -160,7 +159,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links):
       requirements=[req.requirement for req in requirements],
       interpreter=interpreter,
       fetchers=fetchers,
-      platform=expand_and_maybe_adjust_platform(interpreter=interpreter, platform=platform),
+      platform=platform,
       context=python_repos.get_network_context(),
       cache=requirements_cache_dir,
       cache_ttl=python_setup.resolver_cache_ttl,