diff --git a/src/rez/bind/_utils.py b/src/rez/bind/_utils.py index 12760b8a6..48324ff00 100644 --- a/src/rez/bind/_utils.py +++ b/src/rez/bind/_utils.py @@ -6,7 +6,7 @@ from rez.exceptions import RezBindError from rez.config import config from rez.util import which -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.logging_ import print_debug from rez.vendor.six import six from pipes import quote @@ -128,7 +128,7 @@ def _run_command(args): # https://github.com/nerdvegas/rez/pull/659 use_shell = ("Windows" in platform.system()) - p = popen( + p = Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/src/rez/developer_package.py b/src/rez/developer_package.py index 2e816ee3b..acb6f08a3 100644 --- a/src/rez/developer_package.py +++ b/src/rez/developer_package.py @@ -3,7 +3,7 @@ from rez.serialise import load_from_file, FileFormat, set_objects from rez.packages_ import create_package from rez.exceptions import PackageMetadataError, InvalidPackageError -from rez.utils.system import add_sys_paths +from rez.utils.execution import add_sys_paths from rez.utils.sourcecode import SourceCode from rez.utils.logging_ import print_info, print_error from rez.vendor.enum import Enum diff --git a/src/rez/package_help.py b/src/rez/package_help.py index 5ff01b2b8..89e9a2029 100644 --- a/src/rez/package_help.py +++ b/src/rez/package_help.py @@ -3,7 +3,7 @@ from rez.packages_ import iter_packages from rez.config import config from rez.rex_bindings import VersionBinding -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.backcompat import convert_old_command_expansions from rez.utils.scope import scoped_formatter from rez.vendor.six import six @@ -99,7 +99,7 @@ def open(self, section_index=0): else: if self._verbose: print("running command: %s" % uri) - p = popen(uri, shell=True) + p = Popen(uri, shell=True) p.wait() def print_info(self, buf=None): @@ -120,7 +120,7 @@ def _open_url(cls, url): cmd = [config.browser, url] if not config.quiet: print("running command: %s" % " ".join(cmd)) - p = popen(cmd) + p = Popen(cmd) p.communicate() else: if not config.quiet: diff --git a/src/rez/package_py_utils.py b/src/rez/package_py_utils.py index fbc70bfe6..43130608d 100644 --- a/src/rez/package_py_utils.py +++ b/src/rez/package_py_utils.py @@ -7,7 +7,7 @@ """ # these imports just forward the symbols into this module's namespace -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.exceptions import InvalidPackageError from rez.vendor.six import six @@ -169,7 +169,7 @@ def exec_command(attr, cmd): """ import subprocess - p = popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() if p.returncode: @@ -196,7 +196,7 @@ def exec_python(attr, src, executable="python"): if isinstance(src, basestring): src = [src] - p = popen([executable, "-c", "; ".join(src)], + p = Popen([executable, "-c", "; ".join(src)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() @@ -236,7 +236,7 @@ def find_site_python(module_name, paths=None): py_cmd = 'import {x}; print({x}.__path__)'.format(x=module_name) - p = popen(["python", "-c", py_cmd], stdout=subprocess.PIPE, + p = Popen(["python", "-c", py_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() diff --git a/src/rez/pip.py b/src/rez/pip.py index 0bc5beae1..c15668741 100644 --- a/src/rez/pip.py +++ b/src/rez/pip.py @@ -8,7 +8,7 @@ from rez.vendor.distlib.util import parse_name_and_version from rez.vendor.enum.enum import Enum from rez.resolved_context import ResolvedContext -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.pip import get_rez_requirements, pip_to_rez_package_name, \ pip_to_rez_version from rez.utils.logging_ import print_debug, print_info, print_warning @@ -64,7 +64,7 @@ def run_pip_command(command_args, pip_version=None, python_version=None): command = [py_exe, "-m", "pip"] + list(command_args) if context is None: - return popen(command) + return Popen(command) else: return context.execute_shell(command=command, block=False) @@ -373,7 +373,7 @@ def _cmd(context, command): _log("running: %s" % cmd_str) if context is None: - p = popen(command) + p = Popen(command) else: p = context.execute_shell(command=command, block=False) diff --git a/src/rez/release_vcs.py b/src/rez/release_vcs.py index 82fd8c9b0..8a8ecdfab 100644 --- a/src/rez/release_vcs.py +++ b/src/rez/release_vcs.py @@ -1,7 +1,7 @@ from rez.exceptions import ReleaseVCSError from rez.packages_ import get_developer_package from rez.util import which -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.logging_ import print_debug from rez.utils.filesystem import walk_up_dirs from pipes import quote @@ -207,7 +207,7 @@ def _cmd(self, *nargs): if self.package.config.debug("package_release"): print_debug("Running command: %s" % cmd_str) - p = popen(nargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + p = Popen(nargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.pkg_root, text=True) out, err = p.communicate() diff --git a/src/rez/rex.py b/src/rez/rex.py index 172f9d9d2..1fd3bdd03 100644 --- a/src/rez/rex.py +++ b/src/rez/rex.py @@ -13,7 +13,7 @@ from rez.exceptions import RexError, RexUndefinedVariableError, RezSystemError from rez.util import shlex_join, is_non_string_iterable from rez.utils import reraise -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.sourcecode import SourceCode, SourceCodeError from rez.utils.data_utils import AttrDictWrapper from rez.utils.formatting import expandvars @@ -635,7 +635,7 @@ def subprocess(self, args, **subproc_kwargs): self.target_environ.update(self.manager.environ) shell_mode = isinstance(args, basestring) - return popen(args, + return Popen(args, shell=shell_mode, env=self.target_environ, **subproc_kwargs) diff --git a/src/rez/serialise.py b/src/rez/serialise.py index b7a2ceb40..6b0959ab6 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -16,7 +16,7 @@ from rez.utils.data_utils import ModifyList from rez.exceptions import ResourceError, InvalidPackageError from rez.utils.memcached import memcached -from rez.utils.system import add_sys_paths +from rez.utils.execution import add_sys_paths from rez.utils import py23 from rez.config import config from rez.vendor.atomicwrites import atomic_write diff --git a/src/rez/shells.py b/src/rez/shells.py index 175759c22..188dafb59 100644 --- a/src/rez/shells.py +++ b/src/rez/shells.py @@ -5,7 +5,7 @@ from rez.util import shlex_join, is_non_string_iterable from rez.backport.shutilwhich import which from rez.utils.logging_ import print_warning -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.system import system from rez.exceptions import RezSystemError from rez.rex import EscapedString @@ -475,7 +475,7 @@ def _create_ex(): cmd.extend([self.executable, target_file]) try: - p = popen(cmd, env=env, **Popen_args) + p = Popen(cmd, env=env, **Popen_args) except Exception as e: cmd_str = ' '.join(map(pipes.quote, cmd)) raise RezSystemError("Error running command:\n%s\n%s" diff --git a/src/rez/tests/test_shells.py b/src/rez/tests/test_shells.py index b175eff34..71dddd4bd 100644 --- a/src/rez/tests/test_shells.py +++ b/src/rez/tests/test_shells.py @@ -135,9 +135,9 @@ def test_command_returncode(self): command = "hello_world -q -r 66" commands = (command, command.split()) for cmd in commands: - p = r.execute_shell(command=cmd, stdout=subprocess.PIPE) - p.wait() - self.assertEqual(p.returncode, 66) + with r.execute_shell(command=cmd, stdout=subprocess.PIPE) as p: + p.wait() + self.assertEqual(p.returncode, 66) @per_available_shell() def test_norc(self): diff --git a/src/rez/utils/_version.py b/src/rez/utils/_version.py index 4366f4e32..ab5a2a7b3 100644 --- a/src/rez/utils/_version.py +++ b/src/rez/utils/_version.py @@ -1,7 +1,7 @@ # Update this value to version up Rez. Do not place anything else in this file. -_rez_version = "2.47.4" +_rez_version = "2.47.5" # Copyright 2013-2016 Allan Johns. diff --git a/src/rez/utils/execution.py b/src/rez/utils/execution.py new file mode 100644 index 000000000..f39d5fc85 --- /dev/null +++ b/src/rez/utils/execution.py @@ -0,0 +1,66 @@ +""" +Utilities related to process/script execution. +""" + +from rez.vendor.six import six +from contextlib import contextmanager +import subprocess +import sys + + +@contextmanager +def add_sys_paths(paths): + """Add to sys.path, and revert on scope exit. + """ + original_syspath = sys.path[:] + sys.path.extend(paths) + + try: + yield + finally: + sys.path = original_syspath + + +if six.PY2: + class _PopenBase(subprocess.Popen): + def __enter__(self): + return self + def __exit__(self, exc_type, value, traceback): + pass + +else: # py3 + _PopenBase = subprocess.Popen + + +class Popen(_PopenBase): + """subprocess.Popen wrapper. + + Allows for Popen to be used as a context in both py2 and py3. + """ + def __init__(self, args, **kwargs): + + # Avoids python bug described here: https://bugs.python.org/issue3905. + # This can arise when apps (maya) install a non-standard stdin handler. + # + # In newer version of maya and katana, the sys.stdin object can also + # become replaced by an object with no 'fileno' attribute, this is also + # taken into account. + # + if "stdin" not in kwargs: + try: + file_no = sys.stdin.fileno() + except AttributeError: + file_no = sys.__stdin__.fileno() + + if file_no not in (0, 1, 2): + kwargs["stdin"] = subprocess.PIPE + + # Add support for the new py3 "text" arg, which is equivalent to + # "universal_newlines". + # https://docs.python.org/3/library/subprocess.html#frequently-used-arguments + # + if "text" in kwargs: + kwargs["universal_newlines"] = True + del kwargs["text"] + + super(Popen, self).__init__(args, **kwargs) diff --git a/src/rez/utils/graph_utils.py b/src/rez/utils/graph_utils.py index afb735982..56b864049 100644 --- a/src/rez/utils/graph_utils.py +++ b/src/rez/utils/graph_utils.py @@ -11,7 +11,7 @@ from ast import literal_eval from rez.config import config from rez.vendor.pydot import pydot -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.formatting import PackageRequest from rez.exceptions import PackageRequestError from rez.vendor.pygraph.readwrite.dot import read as read_dot @@ -282,7 +282,7 @@ def view_graph(graph_str, dest_file=None): print("loading image viewer (%s)..." % prog) if config.image_viewer: - proc = popen([config.image_viewer, dest_file]) + proc = Popen([config.image_viewer, dest_file]) proc.wait() viewed = not bool(proc.returncode) diff --git a/src/rez/utils/platform_.py b/src/rez/utils/platform_.py index faa45110f..d0f03f219 100644 --- a/src/rez/utils/platform_.py +++ b/src/rez/utils/platform_.py @@ -5,7 +5,7 @@ import re import subprocess from rez.util import which -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.data_utils import cached_property from rez.utils.platform_mapped import platform_mapped from rez.exceptions import RezSystemError @@ -226,7 +226,7 @@ def _parse(txt, distributor_key, release_key): # next, try getting the output of the lsb_release program import subprocess - p = popen( + p = Popen( ['/usr/bin/env', 'lsb_release', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) @@ -372,7 +372,7 @@ def _physical_cores_from_cpuinfo(self): def _physical_cores_from_lscpu(self): import subprocess try: - p = popen( + p = Popen( ['lscpu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) @@ -442,7 +442,7 @@ def _editor(self): def _physical_cores_from_osx_sysctl(self): import subprocess try: - p = popen( + p = Popen( ['sysctl', '-n', 'hw.physicalcpu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True @@ -534,7 +534,7 @@ def _physical_cores_from_wmic(self): # windows import subprocess try: - p = popen( + p = Popen( 'wmic cpu get NumberOfCores /value'.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True diff --git a/src/rez/utils/py23.py b/src/rez/utils/py23.py index 3fa866b37..0b485110a 100644 --- a/src/rez/utils/py23.py +++ b/src/rez/utils/py23.py @@ -39,19 +39,3 @@ def load_module_from_file(name, filepath): else: from importlib.machinery import SourceFileLoader return SourceFileLoader(name, filepath).load_module() - - -def subprocess_Popen(args, **kwargs): - """subprocess.Popen. - """ - import subprocess - - # Add support for the new py3 "text" arg, which is equivalent to - # "universal_newlines". - # https://docs.python.org/3/library/subprocess.html#frequently-used-arguments - # - if "text" in kwargs: - kwargs["universal_newlines"] = True - del kwargs["text"] - - return subprocess.Popen(args, **kwargs) diff --git a/src/rez/utils/system.py b/src/rez/utils/system.py deleted file mode 100644 index 42c46f153..000000000 --- a/src/rez/utils/system.py +++ /dev/null @@ -1,39 +0,0 @@ -from rez.utils import py23 -from contextlib import contextmanager -import subprocess -import sys - - -@contextmanager -def add_sys_paths(paths): - """Add to sys.path, and revert on scope exit. - """ - original_syspath = sys.path[:] - sys.path.extend(paths) - - try: - yield - finally: - sys.path = original_syspath - - -def popen(args, **kwargs): - """Wrapper for `subprocess.Popen`. - - Avoids python bug described here: https://bugs.python.org/issue3905. This - can arise when apps (maya) install a non-standard stdin handler. - - In newer version of maya and katana, the sys.stdin object can also become - replaced by an object with no 'fileno' attribute, this is also taken into - account. - """ - if "stdin" not in kwargs: - try: - file_no = sys.stdin.fileno() - except AttributeError: - file_no = sys.__stdin__.fileno() - - if file_no not in (0, 1, 2): - kwargs["stdin"] = subprocess.PIPE - - return py23.subprocess_Popen(args, **kwargs) diff --git a/src/rezplugins/shell/_utils/powershell_base.py b/src/rezplugins/shell/_utils/powershell_base.py index 7447e3ca0..d7e25e766 100644 --- a/src/rezplugins/shell/_utils/powershell_base.py +++ b/src/rezplugins/shell/_utils/powershell_base.py @@ -7,7 +7,7 @@ from rez.rex import RexExecutor, OutputStyle, EscapedString from rez.shells import Shell from rez.utils.platform_ import platform_ -from rez.utils.system import popen +from rez.utils.execution import Popen class PowerShellBase(Shell): @@ -92,7 +92,7 @@ def gen_expected_regex(parts): "REG_(EXPAND_)?SZ", "(.*)" ]) - p = popen(cmd, stdout=PIPE, stderr=PIPE, + p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, text=True) out_, _ = p.communicate() out_ = out_.strip() @@ -109,7 +109,7 @@ def gen_expected_regex(parts): "(.*)" ]) - p = popen(cmd, stdout=PIPE, stderr=PIPE, + p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, text=True) out_, _ = p.communicate() out_ = out_.strip() @@ -215,7 +215,7 @@ def _record_shell(ex, files, bind_rez=True, print_msg=False): if shell_command is None: cmd.insert(1, "-NoExit") - p = popen(cmd, env=env, **Popen_args) + p = Popen(cmd, env=env, **Popen_args) return p def get_output(self, style=OutputStyle.file): diff --git a/src/rezplugins/shell/cmd.py b/src/rezplugins/shell/cmd.py index e22946ec9..87f1e52c9 100644 --- a/src/rezplugins/shell/cmd.py +++ b/src/rezplugins/shell/cmd.py @@ -5,7 +5,7 @@ from rez.rex import RexExecutor, expandable, literal, OutputStyle, EscapedString from rez.shells import Shell from rez.system import system -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.platform_ import platform_ from rez.vendor.six import six from functools import partial @@ -97,7 +97,7 @@ def gen_expected_regex(parts): "(.*)" ]) - p = popen(cmd, stdout=subprocess.PIPE, + p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True) out_, _ = p.communicate() out_ = out_.strip() diff --git a/src/rezplugins/shell/csh.py b/src/rezplugins/shell/csh.py index 0dfff2b03..980a2706b 100644 --- a/src/rezplugins/shell/csh.py +++ b/src/rezplugins/shell/csh.py @@ -5,7 +5,7 @@ import os.path import subprocess from rez.config import config -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.platform_ import platform_ from rez.shells import Shell, UnixShell from rez.rex import EscapedString @@ -37,7 +37,7 @@ def get_syspaths(cls): # detect system paths using registry cmd = "cmd=`which %s`; unset PATH; $cmd %s 'echo __PATHS_ $PATH'" \ % (cls.name(), cls.command_arg) - p = popen(cmd, stdout=subprocess.PIPE, + p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True) out_, err_ = p.communicate() if p.returncode: diff --git a/src/rezplugins/shell/sh.py b/src/rezplugins/shell/sh.py index f71d4c72e..49a546c90 100644 --- a/src/rezplugins/shell/sh.py +++ b/src/rezplugins/shell/sh.py @@ -6,7 +6,7 @@ import pipes import subprocess from rez.config import config -from rez.utils.system import popen +from rez.utils.execution import Popen from rez.utils.platform_ import platform_ from rez.shells import Shell, UnixShell from rez.rex import EscapedString @@ -37,7 +37,7 @@ def get_syspaths(cls): # detect system paths using registry cmd = "cmd=`which %s`; unset PATH; $cmd %s %s 'echo __PATHS_ $PATH'" \ % (cls.name(), cls.norc_arg, cls.command_arg) - p = popen(cmd, stdout=subprocess.PIPE, + p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True) out_, err_ = p.communicate() if p.returncode: