Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fix] Wheel pip regressions #656

Merged
merged 14 commits into from
Jul 10, 2019
Merged

[Fix] Wheel pip regressions #656

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
02f37a7
fix(rez-pip): ensure the path to bin scripts works cross-platform
lambdaclan Jun 28, 2019
a629e50
fix(rez-pip): ensure only the bin scripts are set as tools
lambdaclan Jun 28, 2019
28f5b8f
fix(rez-pip): correctly handle environment markers in requirements
lambdaclan Jul 2, 2019
d921e36
fix(rez-pip): ensure the pip version check warning is displayed
lambdaclan Jul 2, 2019
2c3ee76
fix(rez-pip): fix typo and add link to environment markers
lambdaclan Jul 5, 2019
cc4248c
refactor(rez-pip): add os.sep to the bin check to avoid naming issues
lambdaclan Jul 5, 2019
59fab19
refactor(rez-pip): use packaging instead of regex for version handling
lambdaclan Jul 5, 2019
8be208f
feat(rez-pip): add attrs library (19.1.0) a packaging lib dependency
lambdaclan Jul 5, 2019
0d35da7
feat(rez-pip): add packaging library to vendor packages
lambdaclan Jul 5, 2019
c6e1bba
feat(rez-pip): update the pydot (1.4.1) vendor package
lambdaclan Jul 5, 2019
bab2083
feat(rez-pip): update pyparsing package (2.4.0) to install packaging
lambdaclan Jul 5, 2019
5d90580
feat(rez-pip): update six library (1.12.0) a packaging lib dependency
lambdaclan Jul 5, 2019
5c44276
feat(rez-pip): add LICENSE file to distlib vendor package
lambdaclan Jul 5, 2019
de37814
fix(rez-pip): update vendor README file changes because of packaging lib
lambdaclan Jul 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 65 additions & 65 deletions src/rez/pip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from rez.vendor.distlib.markers import interpret
from rez.vendor.distlib.util import parse_name_and_version
from rez.vendor.enum.enum import Enum
from rez.vendor.packaging.version import parse, LegacyVersion, InvalidVersion
from rez.resolved_context import ResolvedContext
from rez.utils.system import popen
from rez.utils.logging_ import print_debug, print_info, print_warning
Expand All @@ -25,44 +26,6 @@
import os
import re


VERSION_PATTERN = r"""
v?
(?:
(?:(?P<epoch>[0-9]+)!)? # epoch
(?P<release>[0-9]+(?:\.[0-9]+)*) # release segment
(?P<pre> # pre-release
[-_\.]?
(?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
[-_\.]?
(?P<pre_n>[0-9]+)?
)?
(?P<post> # post release
(?:-(?P<post_n1>[0-9]+))
|
(?:
[-_\.]?
(?P<post_l>post|rev|r)
[-_\.]?
(?P<post_n2>[0-9]+)?
)
)?
(?P<dev> # dev release
[-_\.]?
(?P<dev_l>dev)
[-_\.]?
(?P<dev_n>[0-9]+)?
)?
)
(?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version
"""

CANONICAL_VERSION_RE = re.compile(
r"^\s*" + VERSION_PATTERN + r"\s*$",
re.VERBOSE | re.IGNORECASE,
)


class InstallMode(Enum):
# don't install dependencies. Build may fail, for example the package may
# need to compile against a dependency. Will work for pure python though.
Expand Down Expand Up @@ -102,6 +65,10 @@ def get_distribution_name(pip_name):
version = version.replace("==", "")
name = get_distribution_name(name)
except DistlibException:
# check if package contains extraneous environment info and remove it
# see environment markers: https://www.python.org/dev/peps/pep-0508/#environment-markers
if ";" in package:
package = package.split(";")[0].strip()
n, vs = package.split(' (')
vs = vs[:-1]
versions = []
Expand All @@ -125,7 +92,7 @@ def is_exe(fpath):
return os.path.exists(fpath) and os.access(fpath, os.X_OK)


def pip_to_rez_version(dist_version):
def pip_to_rez_version(dist_version, allow_legacy=True):
"""Convert a distribution version to a rez compatible version.

The python version schema specification isn't 100% compatible with rez.
Expand All @@ -141,38 +108,69 @@ def pip_to_rez_version(dist_version):

Epoch segment: N! - skip
Release segment: N(.N)* 0 as is
Pre-release segment: {a|b|rc}N - always lowercase
Post-release segment: .postN - always lowercase
Pre-release segment: {a|b|c|rc|alpha|beta|pre|preview}N - always lowercase
Post-release segment: .{post|rev|r}N - always lowercase
Development release segment: .devN - always lowercase

Local version identifiers MUST comply with the following scheme:
<public version identifier>[+<local version label>] - use - instead of +

Arguments:
dist_version (str): The distribution version to be converted.
allow_legacy (bool): Flag to allow/disallow PEP440 incompatibility.

Raises:
InvalidVersion: When legacy mode is not allowed and a PEP440
incompatible version is detected.

.. _PEP 440 (all possible matches):
https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions

.. _Core utilities for Python packages:
https://packaging.pypa.io/en/latest/version/

"""
version_match = CANONICAL_VERSION_RE.match(dist_version)
version_segments = version_match.groupdict()

available_segments = dict((k, v) for k, v in version_segments.iteritems() if v)
version = ""
if "release" in available_segments:
release = available_segments["release"]
version += release
if "pre" in available_segments:
pre = available_segments["pre"].lower()
version += pre
if "post" in available_segments:
post = available_segments["post"].lower()
version += post
if "dev" in available_segments:
dev = available_segments["dev"].lower()
version += dev
if "local" in available_segments:
local = available_segments["local"]
version += "-" + local

return version
pkg_version = parse(dist_version)

if isinstance(pkg_version, LegacyVersion):
if allow_legacy:
print("Warning invalid PEP440 version detected: {}. Falling to legacy mode.".format(pkg_version))
# this will always be the entire version string
return pkg_version.base_version.lower()
else:
raise InvalidVersion("Version: {} is not compatible with PEP440.".format(dist_version))

rez_version = ""

if pkg_version.release:
# the components of the release segment excluding epoch or any prerelease/development/postrelease suffixes
rez_version += '.'.join(str(i) for i in pkg_version.release)

if pkg_version.is_prerelease and pkg_version.pre:
# additional check is necessary because dev releases are also considered prereleases
# pair of the prerelease phase (the string "a", "b", or "rc") and the prerelease number
# the following conversions (-->) take place:
# a --> a, alpha --> a, b --> b, beta --> b, c --> c, rc --> rc, pre --> rc, preview --> rc
phase, number = pkg_version.pre
rez_version += phase + str(number)

if pkg_version.is_postrelease:
# this attribute will be the postrelease number (an integer)
# the following conversions (-->) take place:
# post --> post, rev --> post, r --> post
rez_version += ".post" + str(pkg_version.post)

if pkg_version.is_devrelease:
# this attribute will be the development release number (an integer)
rez_version += ".dev" + str(pkg_version.dev)

if pkg_version.local:
# representation of the local version portion is any
# the following conversions (-->) take place:
# 1.0[+ubuntu-1] --> 1.0[-ubuntu.1]
rez_version += "-" + pkg_version.local

return rez_version


def pip_to_rez_package_name(distribution):
Expand Down Expand Up @@ -247,6 +245,8 @@ def find_pip(pip_version=None, python_version=None):

if int(pip_major) < 19:
raise VersionError("pip >= 19 is required! Please update your pip.")
except VersionError:
raise
except:
# silently skip if pip version detection failed, pip itself will show
# a reasonable error message at the least.
Expand Down Expand Up @@ -396,7 +396,7 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = installed_file[0]
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep

if installed_filepath.startswith(bin_prefix):
Expand All @@ -411,7 +411,7 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
destination_file = os.path.relpath(source_file, stagingdir)
exe = False

if is_exe(source_file):
if is_exe(source_file) and destination_file.startswith("bin" + os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
Expand Down
10 changes: 6 additions & 4 deletions src/rez/vendor/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,19 @@ Note that the latest versions column is just to give us an idea of how far back
| argcomplete | ? | 1.9.5 (Apr 2, 2019) | Our version seems patched. |
| argparse | 1.2.1 | Python standard library since 2.7,>=3.2 | We should simply drop support for python <2.6. Note: Can be done now, 2.6 already officially dropped. |
| atomicwrites | 1.2.1 (Aug 30, 2018) | 1.3.0 (Feb 1, 2019) | |
| attrs | 19.1.0 (Mar 3, 2019) | 19.1.0 (Mar 3, 2019) | Added (July 2019) to enable the use of packaging lib that depends on it.
| colorama | 0.3.1 (Apr 19, 2014) | 0.4.1 (Nov 25, 2018) | The newest version probably support Windows :) |
| distlib | 0.2.9.post0 (May 14, 2019) | 0.3.0 (No official release yet) | Updated (June 2019) to enable wheel distribution based installations |
| enum | ? | ? | By looking at the code, it's probably enum34. If so, the latest version is 1.1.6 (May 15, 2016) |
| lockfile | 0.9.1 (Sep 19, 2010) | 0.12.2 (Nov 25, 2015) | |
| memcache (python-memcached) | 1.53 (Jun 7, 2013) | 1.59 (Dec 15, 2017) | We could try to move to a more maintained package like pymemcache from pinterest. NOTE: A port to redis may be a better option, people are more familiar with it and it already has a good python client that supports conn pooling. |
| memcache (python-memcached) | 1.53 (Jun 7, 2013) | 1.59 (Dec 15, 2017) | We could try to move to a more maintained package like pymemcache from pinterest. NOTE: A port to redis may be a better option, people are more familiar with it and it already has a good python client that supports conn pooling.
| packaging | 19.0 (Jan 20, 2019) | 19.0 (Jan 20, 2019) | Added (July 2019) to enable PEP440 compatible versions handling.
| progress | 1.2 (Nov 28, 2013) | 1.5 (Mar 6, 2019) | |
| pydot | 1.0.28 (Jan 1, 2012) | 1.4.1 (Dec 12, 2018) | |
| pydot | 1.4.1 (Dec 12, 2018) | 1.4.1 (Dec 12, 2018) | Updated (July 2019) in order to update pyparsing lib which in turn is required by the packaging library. |
| pygraph (python-graph-core) | 1.8.2 (Jul 14, 2012) | 1.8.2 | Wow, it took me some archeology to find this one. Not entirely sure we need it... We might be able to achieve the same with pydot or graphviz (the pip package)? |
| pyparsing | 2.0.1 (Jul 17, 2013) | 2.4.0 (Apr 8, 2019) | Transitive dependency. Required by pydot and version pydot 1.0.28 specifically require ==2.0.1. Newer versions require >=2.1.4) |
| pyparsing | 2.4.0 (Apr 8, 2019) | 2.4.0 (Apr 8, 2019) | ~~Transitive dependency. Required by pydot and version pydot 1.0.28 specifically require ==2.0.1. Newer versions require >=2.1.4)~~ Updated (July 2019) along with pydot to allow for packaging lib to be used. |
| schema | 0.3.1 (Apr 28, 2014) | 0.7.0 (Feb 27, 2019) | Our version is patched. Updating would probably require quite a lot of work and wouldn't bring that much to rez. |
| six | 1.8.0 (Sep 11, 2014) | 1.12.0 (Dec 9, 2018) | Used only for its exec_ function for now. I guess if we want to support python 2 and 3 we will use it more and more. |
| six | 1.12.0 (Dec 9, 2018) | 1.12.0 (Dec 9, 2018) | Used only for its exec_ function for now. I guess if we want to support python 2 and 3 we will use it more and more. Updated (July 2019) to coincide with packaging lib addition that depends on it. |
| sortedcontainers | 1.5.7 (Dec 22, 2016) | 2.1.0 (Mar 5, 2019) | Used in the resolver. Updating would possibly give us some speed improvements. |
| version | ? | | This is actually part of rez. It's here because I've been intending on releasing it separately for ages. |
| yaml (PyYAML) | 3.10 (May 30, 2011) | 5.1 (Mar 13, 2019) | Not much changes, mostly broken releases that never made it. Between 3.10 and 5.1 there is only 3 versions that made it to the public (3.11 and 3.12, 3.13), the rest were all removed or marked as pre-release. It's safe to update. |
Expand Down
22 changes: 22 additions & 0 deletions src/rez/vendor/attr/LICENSE
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
The MIT License (MIT)

Copyright (c) 2015 Hynek Schlawack

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

65 changes: 65 additions & 0 deletions src/rez/vendor/attr/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
from __future__ import absolute_import, division, print_function

from functools import partial

from . import converters, exceptions, filters, validators
from ._config import get_run_validators, set_run_validators
from ._funcs import asdict, assoc, astuple, evolve, has
from ._make import (
NOTHING,
Attribute,
Factory,
attrib,
attrs,
fields,
fields_dict,
make_class,
validate,
)


__version__ = "19.1.0"

__title__ = "attrs"
__description__ = "Classes Without Boilerplate"
__url__ = "https://www.attrs.org/"
__uri__ = __url__
__doc__ = __description__ + " <" + __uri__ + ">"

__author__ = "Hynek Schlawack"
__email__ = "hs@ox.cx"

__license__ = "MIT"
__copyright__ = "Copyright (c) 2015 Hynek Schlawack"


s = attributes = attrs
ib = attr = attrib
dataclass = partial(attrs, auto_attribs=True) # happy Easter ;)

__all__ = [
"Attribute",
"Factory",
"NOTHING",
"asdict",
"assoc",
"astuple",
"attr",
"attrib",
"attributes",
"attrs",
"converters",
"evolve",
"exceptions",
"fields",
"fields_dict",
"filters",
"get_run_validators",
"has",
"ib",
"make_class",
"s",
"set_run_validators",
"validate",
"validators",
]
Loading