From 0d1c2e6ce10529506b8e09c2b3a1cc6921a3ecfa Mon Sep 17 00:00:00 2001 From: Dark Dragon Date: Mon, 18 Sep 2023 21:13:18 +0200 Subject: [PATCH 1/2] Use exec form instead of shell form for subprocess call in cmd(). This avoids the need for proper escaping and fixes running in paths with special characters like spaces or brackets. --- src/appenv.py | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index 94b2c12..40943cb 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -33,10 +33,13 @@ def cmd(c, merge_stderr=True, quiet=False): # XXX better IO management for interactive output and seeing original # errors and output at appropriate places ... try: - kwargs = {"shell": True} + kwargs = {} + if isinstance(c, str): + kwargs["shell"] = True + c = [c] if merge_stderr: kwargs["stderr"] = subprocess.STDOUT - return subprocess.check_output([c], **kwargs) + return subprocess.check_output(c, **kwargs) except subprocess.CalledProcessError as e: print("{} returned with exit code {}".format(c, e.returncode)) print(e.output.decode("utf-8", "replace")) @@ -63,7 +66,7 @@ def ensure_venv(target): if os.path.exists(target): print("Deleting unclean target)") - cmd("rm -rf {target}".format(target=target)) + cmd(["rm", "-rf", target]) version = sys.version.split()[0] python_maj_min = ".".join(str(x) for x in sys.version_info[:2]) @@ -94,7 +97,7 @@ def ensure_venv(target): get("www.python.org", "/ftp/python/{v}/Python-{v}.tgz".format(v=version), f) - cmd("tar xf {} -C {}".format(download, tmp_base)) + cmd(["tar", "xf", download, "-C", tmp_base]) assert os.path.exists( os.path.join(tmp_base, "Python-{}".format(version))) @@ -120,9 +123,12 @@ def ensure_venv(target): shutil.rmtree(tmp_base) print("Ensuring pip ...") - cmd("{target}/bin/python -m ensurepip --default-pip".format(target=target)) - cmd("{target}/bin/python -m pip install --upgrade pip".format( - target=target)) + cmd([ + "{target}/bin/python".format(target=target), "-m", "ensurepip", + "--default-pip"]) + cmd([ + "{target}/bin/python".format(target=target), "-m", "pip", "install", + "--upgrade", "pip"]) def ensure_minimal_python(): @@ -418,7 +424,7 @@ def prepare(self, args=None, remaining=None): raise Exception() except Exception: print("Existing envdir not consistent, deleting") - cmd("rm -rf {env_dir}".format(env_dir=env_dir)) + cmd(["rm", "-rf", env_dir]) if not os.path.exists(env_dir): ensure_venv(env_dir) @@ -427,10 +433,13 @@ def prepare(self, args=None, remaining=None): f.write(requirements) print("Installing ...") - cmd("{env_dir}/bin/python -m pip install --no-deps -r" - " {env_dir}/requirements.lock".format(env_dir=env_dir)) - - cmd("{env_dir}/bin/python -m pip check".format(env_dir=env_dir)) + cmd([ + "{env_dir}/bin/python".format(env_dir=env_dir), "-m", "pip", + "install", "--no-deps", "-r", + "{env_dir}/requirements.lock".format(env_dir=env_dir)]) + cmd([ + "{env_dir}/bin/python".format(env_dir=env_dir), "-m", "pip", + "check"]) with open(os.path.join(env_dir, "appenv.ready"), "w") as f: f.write("Ready or not, here I come, you can't hide\n") @@ -492,7 +501,7 @@ def reset(self, args=None, remaining=None): print( "Resetting ALL application environments in {appenvdir} ...".format( appenvdir=self.appenv_dir)) - cmd("rm -rf {appenvdir}".format(appenvdir=self.appenv_dir)) + cmd(["rm", "-rf", self.appenv_dir]) def update_lockfile(self, args=None, remaining=None): ensure_minimal_python() @@ -500,15 +509,17 @@ def update_lockfile(self, args=None, remaining=None): print("Updating lockfile") tmpdir = os.path.join(self.appenv_dir, "updatelock") if os.path.exists(tmpdir): - cmd("rm -rf {tmpdir}".format(tmpdir=tmpdir)) + cmd(["rm", "-rf", tmpdir]) ensure_venv(tmpdir) print("Installing packages ...") - cmd("{tmpdir}/bin/python -m pip install -r requirements.txt".format( - tmpdir=tmpdir)) + cmd([ + "{tmpdir}/bin/python".format(tmpdir=tmpdir), "-m", "pip", + "install", "-r", "requirements.txt"]) extra_specs = [] - result = cmd( - "{tmpdir}/bin/python -m pip freeze".format(tmpdir=tmpdir), + result = cmd([ + "{tmpdir}/bin/python".format(tmpdir=tmpdir), "-m", "pip", "freeze" + ], merge_stderr=False).decode('ascii') pinned_versions = {} for line in result.splitlines(): @@ -553,7 +564,7 @@ def update_lockfile(self, args=None, remaining=None): self._hash_requirements())) f.write('\n'.join(lines)) f.write('\n') - cmd("rm -rf {tmpdir}".format(tmpdir=tmpdir)) + cmd(["rm", "-rf", tmpdir]) def main(): From 84ff948777cf7448e3beed915fce01b622ff2ccf Mon Sep 17 00:00:00 2001 From: Dark Dragon Date: Mon, 18 Sep 2023 21:20:48 +0200 Subject: [PATCH 2/2] Add shorthand for python and pip --- src/appenv.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index 40943cb..464c44b 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -46,6 +46,14 @@ def cmd(c, merge_stderr=True, quiet=False): raise ValueError(e.output.decode("utf-8", "replace")) +def python(path, c, **kwargs): + return cmd([os.path.join(path, "bin/python")] + c, **kwargs) + + +def pip(path, c, **kwargs): + return python(path, ["-m", "pip"] + c, **kwargs) + + def get(host, path, f): conn = http.client.HTTPSConnection(host) conn.request("GET", path) @@ -123,12 +131,8 @@ def ensure_venv(target): shutil.rmtree(tmp_base) print("Ensuring pip ...") - cmd([ - "{target}/bin/python".format(target=target), "-m", "ensurepip", - "--default-pip"]) - cmd([ - "{target}/bin/python".format(target=target), "-m", "pip", "install", - "--upgrade", "pip"]) + python(target, ["-m", "ensurepip", "--default-pip"]) + pip(target, ["install", "--upgrade", "pip"]) def ensure_minimal_python(): @@ -433,13 +437,10 @@ def prepare(self, args=None, remaining=None): f.write(requirements) print("Installing ...") - cmd([ - "{env_dir}/bin/python".format(env_dir=env_dir), "-m", "pip", + pip(env_dir, [ "install", "--no-deps", "-r", "{env_dir}/requirements.lock".format(env_dir=env_dir)]) - cmd([ - "{env_dir}/bin/python".format(env_dir=env_dir), "-m", "pip", - "check"]) + pip(env_dir, ["check"]) with open(os.path.join(env_dir, "appenv.ready"), "w") as f: f.write("Ready or not, here I come, you can't hide\n") @@ -512,15 +513,10 @@ def update_lockfile(self, args=None, remaining=None): cmd(["rm", "-rf", tmpdir]) ensure_venv(tmpdir) print("Installing packages ...") - cmd([ - "{tmpdir}/bin/python".format(tmpdir=tmpdir), "-m", "pip", - "install", "-r", "requirements.txt"]) + pip(tmpdir, ["install", "-r", "requirements.txt"]) extra_specs = [] - result = cmd([ - "{tmpdir}/bin/python".format(tmpdir=tmpdir), "-m", "pip", "freeze" - ], - merge_stderr=False).decode('ascii') + result = pip(tmpdir, ["freeze"], merge_stderr=False).decode('ascii') pinned_versions = {} for line in result.splitlines(): if line.strip().startswith('-e '):