From a017a38460d648420c546754f21800596b588a12 Mon Sep 17 00:00:00 2001 From: lambdaclan <47409392+lambdaclan@users.noreply.github.com> Date: Wed, 18 Dec 2019 09:42:17 +0900 Subject: [PATCH 1/6] Add argument "extra" to rez-pip cli for additional pip install arguments --- src/rez/cli/pip.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/rez/cli/pip.py b/src/rez/cli/pip.py index 40386bc25..e32ead63f 100644 --- a/src/rez/cli/pip.py +++ b/src/rez/cli/pip.py @@ -2,6 +2,7 @@ Install a pip-compatible python package, and its dependencies, as rez packages. """ from __future__ import print_function +from argparse import REMAINDER def setup_parser(parser, completions=False): @@ -30,7 +31,10 @@ def setup_parser(parser, completions=False): parser.add_argument( "PACKAGE", help="package to install or archive/url to install from") - + parser.add_argument( + "-e", "--extra", nargs=REMAINDER, + help="extra args passthrough to pip install" + ) def command(opts, parser, extra_arg_groups=None): from rez.pip import pip_install_package, run_pip_command @@ -59,7 +63,8 @@ def command(opts, parser, extra_arg_groups=None): pip_version=opts.pip_ver, python_version=opts.py_ver, release=opts.release, - prefix=opts.prefix) + prefix=opts.prefix, + extra_args=opts.extra) # print summary # From 0b813752cfb94677f02c1abd0801f40b4129eb5e Mon Sep 17 00:00:00 2001 From: lambdaclan <47409392+lambdaclan@users.noreply.github.com> Date: Wed, 18 Dec 2019 09:44:49 +0900 Subject: [PATCH 2/6] Add "pip_extra_args" to config scheme as optional list --- src/rez/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rez/config.py b/src/rez/config.py index 0a61c7874..cdb7bcd45 100644 --- a/src/rez/config.py +++ b/src/rez/config.py @@ -308,6 +308,7 @@ def _parse_env_var(self, value): "implicit_styles": OptionalStrList, "alias_styles": OptionalStrList, "memcached_uri": OptionalStrList, + "pip_extra_args": OptionalStrList, "local_packages_path": Str, "release_packages_path": Str, "dot_image_format": Str, From 501107970f260cffd0b77ad14330b44984ac4e85 Mon Sep 17 00:00:00 2001 From: lambdaclan <47409392+lambdaclan@users.noreply.github.com> Date: Wed, 18 Dec 2019 09:45:47 +0900 Subject: [PATCH 3/6] Add "pip_extra_args" to default rezconfig (misc) as empty list --- src/rez/rezconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index f9f10a31e..706bfd81c 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -694,6 +694,10 @@ # launched without extension from windows and other systems. create_executable_script_mode = "single" +# Configurable pip extra arguments passed to the rez-pip install command. +# https://pip.pypa.io/en/stable/reference/pip_install/#options +pip_extra_args = [] + ############################################################################### # Rez-1 Compatibility From 71fed89afbf72872c9125ca5ad2aba53d121d786 Mon Sep 17 00:00:00 2001 From: lambdaclan <47409392+lambdaclan@users.noreply.github.com> Date: Wed, 18 Dec 2019 09:47:18 +0900 Subject: [PATCH 4/6] Modify pip install function to use pip extra args from cli or rezconfig --- src/rez/pip.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/rez/pip.py b/src/rez/pip.py index a0db31756..96da4f916 100644 --- a/src/rez/pip.py +++ b/src/rez/pip.py @@ -198,7 +198,8 @@ def find_pip_from_context(python_version, pip_version=None): def pip_install_package(source_name, pip_version=None, python_version=None, - mode=InstallMode.min_deps, release=False, prefix=None): + mode=InstallMode.min_deps, release=False, prefix=None, + extra_args=None): """Install a pip-compatible python package as a rez package. Args: source_name (str): Name of package or archive/url containing the pip @@ -212,6 +213,7 @@ def pip_install_package(source_name, pip_version=None, python_version=None, managed. release (bool): If True, install as a released package; otherwise, it will be installed as a local package. + extra_args (List[str]): Additional options to the pip install command. Returns: 2-tuple: @@ -249,6 +251,9 @@ def pip_install_package(source_name, pip_version=None, python_version=None, context.print_info(buf) _log(buf.getvalue()) + # gather additional pip install options + pip_extra_opts = extra_args if extra_args else config.pip_extra_args + # Build pip commandline cmd = [ py_exe, "-m", "pip", "install", @@ -256,6 +261,8 @@ def pip_install_package(source_name, pip_version=None, python_version=None, "--target=%s" % destpath ] + cmd.extend(pip_extra_opts) + if mode == InstallMode.no_deps: cmd.append("--no-deps") cmd.append(source_name) From 3791173bb5d1e4443526dbfb8c5eeeaf5caecace Mon Sep 17 00:00:00 2001 From: lambdaclan <47409392+lambdaclan@users.noreply.github.com> Date: Fri, 10 Jan 2020 14:42:53 +0900 Subject: [PATCH 5/6] Document which arguments should not be specified in the extra args --- src/rez/cli/pip.py | 2 +- src/rez/rezconfig.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/rez/cli/pip.py b/src/rez/cli/pip.py index e32ead63f..8bb0fb390 100644 --- a/src/rez/cli/pip.py +++ b/src/rez/cli/pip.py @@ -33,7 +33,7 @@ def setup_parser(parser, completions=False): help="package to install or archive/url to install from") parser.add_argument( "-e", "--extra", nargs=REMAINDER, - help="extra args passthrough to pip install" + help="extra args passthrough to pip install (overrides pre-configured args if specified)" ) def command(opts, parser, extra_arg_groups=None): diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index 706bfd81c..0f7fda7be 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -695,6 +695,11 @@ create_executable_script_mode = "single" # Configurable pip extra arguments passed to the rez-pip install command. +# Since the rez-pip install command already includes some pre-configured +# arguments (target, use-pep517) this setting can potentially override the +# default configuration in a way which can cause package installation issues. +# It is recommended to refrain from overriding the default arguments and only +# use this setting for additional arguments that might be needed. # https://pip.pypa.io/en/stable/reference/pip_install/#options pip_extra_args = [] From cbcdaf2c3950c1e98a3ebec06ee808cd8e4e9c9b Mon Sep 17 00:00:00 2001 From: lambdaclan <47409392+lambdaclan@users.noreply.github.com> Date: Fri, 10 Jan 2020 17:43:52 +0900 Subject: [PATCH 6/6] Override pre-configured options specified in extra args --- src/rez/pip.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/rez/pip.py b/src/rez/pip.py index 96da4f916..85879c15c 100644 --- a/src/rez/pip.py +++ b/src/rez/pip.py @@ -251,17 +251,29 @@ def pip_install_package(source_name, pip_version=None, python_version=None, context.print_info(buf) _log(buf.getvalue()) + # preconfigured options + pre_configured = ["--use-pep517", "--target=%s" % destpath] + # gather additional pip install options pip_extra_opts = extra_args if extra_args else config.pip_extra_args + # make sure that the target option is overridden if supplied + if any("--target=" in opt for opt in pip_extra_opts): + # extra args redefines the target option so ignore pre-configured one + pre_configured = pre_configured[0:1] + # disable pep517 if forced + if "--no-use-pep517" in pip_extra_opts: + pre_configured = [] + + # keep unique opts + opts = list(set(pre_configured + pip_extra_opts)) + # Build pip commandline cmd = [ - py_exe, "-m", "pip", "install", - "--use-pep517", - "--target=%s" % destpath + py_exe, "-m", "pip", "install" ] - cmd.extend(pip_extra_opts) + cmd.extend(opts) if mode == InstallMode.no_deps: cmd.append("--no-deps")