From 440d3d0c0a0c1e6ec543b77a43d3c7e8f8b49d31 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 02:00:35 +0200 Subject: [PATCH 1/8] Add warning message for Python version >= 3.12 in ensure_minimal_python() function --- src/appenv.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/appenv.py b/src/appenv.py index 15b1478..fd02553 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -153,6 +153,13 @@ def ensure_minimal_python(): print("Update lockfile with with {}.".format(current_python)) print("If you want to use a different version, set it via") print(" `# appenv-python-preference:` in requirements.txt.") + if sys.version_info >= (3, 12): + print("You are using a Python version >= 3.12.") + print( + "Please specify a Python version in the requirements.txt file." + ) + print("Lockfiles created with a Python version lower than 3.12") + print("may create a broken venv with a Python version >= 3.12.") return preferences.sort(key=lambda s: [int(u) for u in s.split('.')]) From d373c5a94704b11aeee342c991d89bf8843f3c33 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 02:11:23 +0200 Subject: [PATCH 2/8] Extract parse_preferences logic --- src/appenv.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index fd02553..d9dcd73 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -134,8 +134,7 @@ def ensure_venv(target): pip(target, ["install", "--upgrade", "pip"]) -def ensure_minimal_python(): - current_python = os.path.realpath(sys.executable) +def parse_preferences(): preferences = None if os.path.exists('requirements.txt'): with open('requirements.txt') as f: @@ -148,6 +147,12 @@ def ensure_minimal_python(): preferences = [x.strip() for x in preferences.split(',')] preferences = list(filter(None, preferences)) break + return preferences + + +def ensure_minimal_python(): + current_python = os.path.realpath(sys.executable) + preferences = parse_preferences() if not preferences: # We have no preferences defined, use the current python. print("Update lockfile with with {}.".format(current_python)) @@ -201,20 +206,11 @@ def ensure_best_python(base): return import shutil - # use newest Python available if nothing else is requested - preferences = ['3.{}'.format(x) for x in reversed(range(4, 20))] + preferences = parse_preferences() - if os.path.exists('requirements.txt'): - with open('requirements.txt') as f: - for line in f: - # Expected format: - # # appenv-python-preference: 3.1,3.9,3.4 - if not line.startswith("# appenv-python-preference: "): - continue - preferences = line.split(':')[1] - preferences = [x.strip() for x in preferences.split(',')] - preferences = list(filter(None, preferences)) - break + if preferences is None: + # use newest Python available if nothing else is requested + preferences = ['3.{}'.format(x) for x in reversed(range(4, 20))] current_python = os.path.realpath(sys.executable) for version in preferences: From 684fa64e5b4f56682059e80e69d16d378e5be3ae Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 02:20:27 +0200 Subject: [PATCH 3/8] Move 3.12 warning message from ensure_minimal (lockfile regenerate) to ensure_best (all runs) --- src/appenv.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index d9dcd73..85346c4 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -158,13 +158,6 @@ def ensure_minimal_python(): print("Update lockfile with with {}.".format(current_python)) print("If you want to use a different version, set it via") print(" `# appenv-python-preference:` in requirements.txt.") - if sys.version_info >= (3, 12): - print("You are using a Python version >= 3.12.") - print( - "Please specify a Python version in the requirements.txt file." - ) - print("Lockfiles created with a Python version lower than 3.12") - print("may create a broken venv with a Python version >= 3.12.") return preferences.sort(key=lambda s: [int(u) for u in s.split('.')]) @@ -209,6 +202,13 @@ def ensure_best_python(base): preferences = parse_preferences() if preferences is None: + if sys.version_info >= (3, 12): + print("You are using a Python version >= 3.12.") + print( + "Please specify a Python version in the requirements.txt file." + ) + print("Lockfiles created with a Python version lower than 3.12") + print("may create a broken venv with a Python version >= 3.12.") # use newest Python available if nothing else is requested preferences = ['3.{}'.format(x) for x in reversed(range(4, 20))] From daa8cefd015df32af932166303faf69154d9a799 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 03:09:21 +0200 Subject: [PATCH 4/8] Fix typo in ensure_minimal_python print statement --- src/appenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/appenv.py b/src/appenv.py index 85346c4..b1b413e 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -155,7 +155,7 @@ def ensure_minimal_python(): preferences = parse_preferences() if not preferences: # We have no preferences defined, use the current python. - print("Update lockfile with with {}.".format(current_python)) + print("Updating lockfile with with {}.".format(current_python)) print("If you want to use a different version, set it via") print(" `# appenv-python-preference:` in requirements.txt.") return From 0beebb244c1d3e8cd28d695c394ded34bc43ef46 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 03:09:40 +0200 Subject: [PATCH 5/8] Display warning message for Python 3.12 compatibility --- src/appenv.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/appenv.py b/src/appenv.py index b1b413e..7ae8321 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -28,6 +28,14 @@ import re +class TColors: + """Terminal colors for pretty output.""" + RED = '\033[91m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RESET = '\033[0m' + + def cmd(c, merge_stderr=True, quiet=False): # TODO revisit the cmd() architecture w/ python 3 # XXX better IO management for interactive output and seeing original @@ -508,6 +516,25 @@ def reset(self, args=None, remaining=None): def update_lockfile(self, args=None, remaining=None): ensure_minimal_python() + preferences = parse_preferences() + if preferences is not None and '3.12' in preferences and any( + f'3.{x}' in preferences for x in range(4, 12)): + print(TColors.RED + + "Warning: mixing Python 3.12 and older versions in the ") + print("requirements.txt file will lead to a broken venv once") + print("batou chooses Python 3.12 as the best (highest preference,") + print( + "available) Python version. Since python/cpython/issues/95299," + ) + print("python >= 3.12 do not install setuptools into the venv by") + print("default anymore. Pip of previous versions of python") + print( + "create a deficient requirements.lock file, as the output of") + print("pip freeze is missing the setuptools requirement, whose") + print( + "behaviour only changed in python 3.12 (pypa/pip/pull/12032)." + + TColors.RESET) + print() os.chdir(self.base) print("Updating lockfile") tmpdir = os.path.join(self.appenv_dir, "updatelock") @@ -519,6 +546,7 @@ def update_lockfile(self, args=None, remaining=None): extra_specs = [] result = pip(tmpdir, ["freeze"], merge_stderr=False).decode('ascii') + # They changed this behaviour in https://github.com/pypa/pip/pull/12032 pinned_versions = {} for line in result.splitlines(): if line.strip().startswith('-e '): From 9511f72841a0c2b44b10e651631386be122e28f2 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 03:42:34 +0200 Subject: [PATCH 6/8] Add Python 3.12 mixed setuptools workaround --- src/appenv.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index 7ae8321..68ce6d1 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -514,27 +514,13 @@ def reset(self, args=None, remaining=None): appenvdir=self.appenv_dir)) cmd(["rm", "-rf", self.appenv_dir]) - def update_lockfile(self, args=None, remaining=None): + def update_lockfile(self, freeze_args=None, remaining=None): ensure_minimal_python() preferences = parse_preferences() + python312_mixed_setuptools_workaround = False if preferences is not None and '3.12' in preferences and any( f'3.{x}' in preferences for x in range(4, 12)): - print(TColors.RED + - "Warning: mixing Python 3.12 and older versions in the ") - print("requirements.txt file will lead to a broken venv once") - print("batou chooses Python 3.12 as the best (highest preference,") - print( - "available) Python version. Since python/cpython/issues/95299," - ) - print("python >= 3.12 do not install setuptools into the venv by") - print("default anymore. Pip of previous versions of python") - print( - "create a deficient requirements.lock file, as the output of") - print("pip freeze is missing the setuptools requirement, whose") - print( - "behaviour only changed in python 3.12 (pypa/pip/pull/12032)." - + TColors.RESET) - print() + python312_mixed_setuptools_workaround = True os.chdir(self.base) print("Updating lockfile") tmpdir = os.path.join(self.appenv_dir, "updatelock") @@ -545,7 +531,11 @@ def update_lockfile(self, args=None, remaining=None): pip(tmpdir, ["install", "-r", "requirements.txt"]) extra_specs = [] - result = pip(tmpdir, ["freeze"], merge_stderr=False).decode('ascii') + pip_freeze_args = ["freeze"] + if python312_mixed_setuptools_workaround: + pip_freeze_args.extend(["--all", "--exclude", "pip"]) + result = pip( + tmpdir, pip_freeze_args, merge_stderr=False).decode('ascii') # They changed this behaviour in https://github.com/pypa/pip/pull/12032 pinned_versions = {} for line in result.splitlines(): From 889fd943c775d915ca679b450d13899b002d45ce Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 03:45:30 +0200 Subject: [PATCH 7/8] Make preference check more general --- src/appenv.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index 68ce6d1..74b28da 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -518,9 +518,10 @@ def update_lockfile(self, freeze_args=None, remaining=None): ensure_minimal_python() preferences = parse_preferences() python312_mixed_setuptools_workaround = False - if preferences is not None and '3.12' in preferences and any( - f'3.{x}' in preferences for x in range(4, 12)): - python312_mixed_setuptools_workaround = True + if preferences is not None: + if any(f'3.{x}' in preferences for x in range(4, 12)): + if any(f'3.{x}' in preferences for x in range(12, 20)): + python312_mixed_setuptools_workaround = True os.chdir(self.base) print("Updating lockfile") tmpdir = os.path.join(self.appenv_dir, "updatelock") From 8de0f8aec85224920f4ee1d5f1bc12cb9e10d394 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Tue, 23 Jul 2024 11:36:28 +0200 Subject: [PATCH 8/8] Fix typo in update_lockfile signature --- src/appenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/appenv.py b/src/appenv.py index 74b28da..58974ef 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -514,7 +514,7 @@ def reset(self, args=None, remaining=None): appenvdir=self.appenv_dir)) cmd(["rm", "-rf", self.appenv_dir]) - def update_lockfile(self, freeze_args=None, remaining=None): + def update_lockfile(self, args=None, remaining=None): ensure_minimal_python() preferences = parse_preferences() python312_mixed_setuptools_workaround = False