Support newer pip versions. (#17555)

Plumbs --pip-version through to pex commands that support it.

Newer pips have perf improvements that may benefit users, in particular for very long-running resolves.

Does not change the default. We can do that via deprecation in a followup.

src/python/pants/backend/python/goals/lockfile.py

@@ -134,7 +134,9 @@ async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAnd
 
 @rule(desc="Generate Python lockfile", level=LogLevel.DEBUG)
 async def generate_lockfile(
-    req: GeneratePythonLockfile, generate_lockfiles_subsystem: GenerateLockfilesSubsystem
+    req: GeneratePythonLockfile,
+    generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
+    python_setup: PythonSetup,
 ) -> GenerateLockfileResult:
     pip_args_setup = await _setup_pip_args_and_constraints_file(req.resolve_name)
 
@@ -150,6 +152,8 @@ async def generate_lockfile(
                 # generate universal locks because they have the best compatibility. We may
                 # want to let users change this, as `style=strict` is safer.
                 "--style=universal",
+                "--pip-version",
+                python_setup.pip_version.value,
                 "--resolver-version",
                 "pip-2020-resolver",
                 # PEX files currently only run on Linux and Mac machines; so we hard code this

src/python/pants/backend/python/subsystems/setup.py

@@ -26,6 +26,13 @@
 logger = logging.getLogger(__name__)
 
 
+@enum.unique
+class PipVersion(enum.Enum):
+    V20_3_4 = "20.3.4-patched"
+    V22_2_2 = "22.2.2"
+    V22_3 = "22.3"
+
+
 @enum.unique
 class InvalidLockfileBehavior(enum.Enum):
     error = "error"
@@ -180,6 +187,11 @@ class PythonSetup(Subsystem):
             """
         ),
     )
+    pip_version = EnumOption(
+        default=PipVersion.V20_3_4,
+        help="Use this version of Pip for resolving requirements and generating lockfiles.",
+        advanced=True,
+    )
     _resolves_to_interpreter_constraints = DictOption["list[str]"](
         help=softwrap(
             """

src/python/pants/backend/python/util_rules/pex_cli.py

@@ -8,6 +8,7 @@
 from typing import Iterable, List, Mapping, Optional, Tuple
 
 from pants.backend.python.subsystems.python_native_code import PythonNativeCodeSubsystem
+from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.util_rules import pex_environment
 from pants.backend.python.util_rules.pex_environment import (
     PexEnvironment,
@@ -37,9 +38,9 @@ class PexCli(TemplatedExternalTool):
     name = "pex"
     help = "The PEX (Python EXecutable) tool (https://github.com/pantsbuild/pex)."
 
-    default_version = "v2.1.111"
+    default_version = "v2.1.113"
     default_url_template = "https://github.com/pantsbuild/pex/releases/download/{version}/pex"
-    version_constraints = ">=2.1.111,<3.0"
+    version_constraints = ">=2.1.113,<3.0"
 
     @classproperty
     def default_known_versions(cls):
@@ -48,8 +49,8 @@ def default_known_versions(cls):
                 (
                     cls.default_version,
                     plat,
-                    "9787e9712aba67ccc019415060a33e850675174be3331d35014e39219212b669",
-                    "4063422",
+                    "cde8eecc5e8e1c9fcca89f36c125a2c4b1c55cdb1073220b1be645e60c0c36e6",
+                    "4067306",
                 )
             )
             for plat in ["macos_arm64", "macos_x86_64", "linux_x86_64", "linux_arm64"]
@@ -125,6 +126,7 @@ async def setup_pex_cli_process(
     python_native_code: PythonNativeCodeSubsystem.EnvironmentAware,
     global_options: GlobalOptions,
     pex_subsystem: PexSubsystem,
+    python_setup: PythonSetup,
 ) -> Process:
     tmpdir = ".tmp"
     gets: List[Get] = [Get(Digest, CreateDigest([Directory(tmpdir)]))]
@@ -165,10 +167,17 @@ async def setup_pex_cli_process(
         if request.set_resolve_args
         else []
     )
+    # All old-style pex runs take the --pip-version flag, but only certain subcommands of the
+    # `pex3` console script do. So if invoked with a subcommand, the caller must selectively
+    # set --pip-version only on subcommands that take it.
+    pip_version_args = (
+        [] if request.subcommand else ["--pip-version", python_setup.pip_version.value]
+    )
     args = [
         *request.subcommand,
         *global_args,
         *verbosity_args,
+        *pip_version_args,
         *resolve_args,
         # NB: This comes at the end because it may use `--` passthrough args, # which must come at
         # the end.

src/python/pants/backend/python/util_rules/pex_cli_test.py

@@ -34,7 +34,7 @@ def test_custom_ca_certs(tmp_path: Path, rule_runner: RuleRunner) -> None:
         Process,
         [PexCliProcess(subcommand=(), extra_args=("some", "--args"), description="")],
     )
-    assert proc.argv[4:6] == ("--cert", "certsfile")
+    assert proc.argv[6:8] == ("--cert", "certsfile")
     files = rule_runner.request(DigestContents, [proc.input_digest])
     chrooted_certs_file = [f for f in files if f.path == "certsfile"]
     assert len(chrooted_certs_file) == 1

src/python/pants/backend/python/util_rules/pex_test.py

@@ -493,6 +493,8 @@ def options(path_mappings_dir: Path) -> tuple[str, ...]:
                 ),
                 f"--python-repos-path-mappings=WHEEL_DIR|{path_mappings_dir}",
                 f"--named-caches-dir={tmp_path}",
+                # Use the vendored pip, so we don't have to set up a wheel for it in dir1_path.
+                "--python-pip-version=20.3.4-patched",
             )
 
         rule_runner.set_options(options(dir1_path), env_inherit=PYTHON_BOOTSTRAP_ENV)

src/python/pants/core/goals/generate_lockfiles.py

@@ -321,7 +321,7 @@ def filter_tool_lockfile_requests(
 
 class GenerateLockfilesSubsystem(GoalSubsystem):
     name = "generate-lockfiles"
-    help = "Generate lockfiles for Python third-party dependencies."
+    help = "Generate lockfiles for third-party dependencies."
 
     @classmethod
     def activated(cls, union_membership: UnionMembership) -> bool: