feat(manager/pip-compile): extract Python version from lock files (#2…

…9145)

lib/modules/manager/pip-compile/artifacts.spec.ts

@@ -1,3 +1,4 @@
+import { codeBlock } from 'common-tags';
 import { mockDeep } from 'jest-mock-extended';
 import { join } from 'upath';
 import { envMock, mockExecAll } from '../../../../test/exec-util';
@@ -233,6 +234,93 @@ describe('modules/manager/pip-compile/artifacts', () => {
     ]);
   });
 
+  it('installs Python version according to the lock file', async () => {
+    GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+    datasource.getPkgReleases.mockResolvedValueOnce({
+      releases: [
+        { version: '3.11.0' },
+        { version: '3.11.1' },
+        { version: '3.12.0' },
+      ],
+    });
+    const execSnapshots = mockExecAll();
+    git.getRepoStatus.mockResolvedValue(
+      partial<StatusResult>({
+        modified: ['requirements.txt'],
+      }),
+    );
+    fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    expect(
+      await updateArtifacts({
+        packageFileName: 'requirements.in',
+        updatedDeps: [],
+        newPackageFileContent: 'some new content',
+        config: {
+          ...config,
+          constraints: { pipTools: '6.13.0' },
+          lockFiles: ['requirements.txt'],
+        },
+      }),
+    ).not.toBeNull();
+
+    expect(execSnapshots).toMatchObject([
+      { cmd: 'install-tool python 3.11.1' },
+      { cmd: 'install-tool pip-tools 6.13.0' },
+      {
+        cmd: 'pip-compile requirements.in',
+        options: { cwd: '/tmp/github/some/repo' },
+      },
+    ]);
+  });
+
+  it('installs latest Python version if no constraints and not in header', async () => {
+    GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+    datasource.getPkgReleases.mockResolvedValueOnce({
+      releases: [
+        { version: '3.11.0' },
+        { version: '3.11.1' },
+        { version: '3.12.0' },
+      ],
+    });
+    const execSnapshots = mockExecAll();
+    git.getRepoStatus.mockResolvedValue(
+      partial<StatusResult>({
+        modified: ['requirements.txt'],
+      }),
+    );
+    // Before 6.2.0, pip-compile didn't include Python version in header
+    const noPythonVersionHeader = codeBlock`
+      #
+      # This file is autogenerated by pip-compile
+      # To update, run:
+      #
+      #    pip-compile requirements.in
+      #
+    `;
+    fs.readLocalFile.mockResolvedValueOnce(noPythonVersionHeader);
+    expect(
+      await updateArtifacts({
+        packageFileName: 'requirements.in',
+        updatedDeps: [],
+        newPackageFileContent: 'some new content',
+        config: {
+          ...config,
+          constraints: { pipTools: '6.13.0' },
+          lockFiles: ['requirements.txt'],
+        },
+      }),
+    ).not.toBeNull();
+
+    expect(execSnapshots).toMatchObject([
+      { cmd: 'install-tool python 3.12.0' },
+      { cmd: 'install-tool pip-tools 6.13.0' },
+      {
+        cmd: 'pip-compile requirements.in',
+        options: { cwd: '/tmp/github/some/repo' },
+      },
+    ]);
+  });
+
   it('catches errors', async () => {
     const execSnapshots = mockExecAll();
     fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt');

lib/modules/manager/pip-compile/artifacts.ts

@@ -12,6 +12,7 @@ import * as pipRequirements from '../pip_requirements';
 import type { UpdateArtifact, UpdateArtifactsResult, Upgrade } from '../types';
 import {
   extractHeaderCommand,
+  extractPythonVersion,
   getExecOptions,
   getRegistryCredVarsFromPackageFile,
 } from './common';
@@ -106,6 +107,10 @@ export async function updateArtifacts({
         await deleteLocalFile(outputFileName);
       }
       const compileArgs = extractHeaderCommand(existingOutput, outputFileName);
+      const pythonVersion = extractPythonVersion(
+        existingOutput,
+        outputFileName,
+      );
       const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile);
       const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate);
       const packageFile = pipRequirements.extractPackageFile(newInputContent);
@@ -114,6 +119,7 @@ export async function updateArtifacts({
         config,
         cwd,
         getRegistryCredVarsFromPackageFile(packageFile),
+        pythonVersion,
       );
       logger.trace({ cwd, cmd }, 'pip-compile command');
       logger.trace({ env: execOptions.extraEnv }, 'pip-compile extra env vars');

lib/modules/manager/pip-compile/common.spec.ts

@@ -4,6 +4,7 @@ import { logger } from '../../../logger';
 import {
   allowedPipOptions,
   extractHeaderCommand,
+  extractPythonVersion,
   getRegistryCredVarsFromPackageFile,
 } from './common';
 import { inferCommandExecDir } from './utils';
@@ -170,6 +171,21 @@ describe('modules/manager/pip-compile/common', () => {
     );
   });
 
+  describe('extractPythonVersion()', () => {
+    it('extracts Python version from valid header', () => {
+      expect(
+        extractPythonVersion(
+          getCommandInHeader('pip-compile reqs.in'),
+          'reqs.txt',
+        ),
+      ).toBe('3.11');
+    });
+
+    it('returns undefined if version cannot be extracted', () => {
+      expect(extractPythonVersion('', 'reqs.txt')).toBeUndefined();
+    });
+  });
+
   describe('getCredentialVarsFromPackageFile()', () => {
     it('handles both registryUrls and additionalRegistryUrls', () => {
       hostRules.find.mockReturnValueOnce({

lib/modules/manager/pip-compile/common.ts

@@ -13,6 +13,7 @@ import type { PipCompileArgs } from './types';
 
 export function getPythonVersionConstraint(
   config: UpdateArtifactsConfig,
+  extractedPythonVersion: string | undefined,
 ): string | undefined | null {
   const { constraints = {} } = config;
   const { python } = constraints;
@@ -22,6 +23,11 @@ export function getPythonVersionConstraint(
     return python;
   }
 
+  if (extractedPythonVersion) {
+    logger.debug('Using python constraint extracted from the lock file');
+    return `==${extractedPythonVersion}`;
+  }
+
   return undefined;
 }
 export function getPipToolsVersionConstraint(
@@ -41,8 +47,9 @@ export async function getExecOptions(
   config: UpdateArtifactsConfig,
   cwd: string,
   extraEnv: ExtraEnv<string>,
+  extractedPythonVersion: string | undefined,
 ): Promise<ExecOptions> {
-  const constraint = getPythonVersionConstraint(config);
+  const constraint = getPythonVersionConstraint(config, extractedPythonVersion);
   const pipToolsConstraint = getPipToolsVersionConstraint(config);
   const execOptions: ExecOptions = {
     cwd: ensureLocalPath(cwd),
@@ -197,6 +204,33 @@ export function extractHeaderCommand(
   return result;
 }
 
+const pythonVersionRegex = regEx(
+  /^(#.*?\r?\n)*# This file is autogenerated by pip-compile with Python (?<pythonVersion>\d+(\.\d+)*)\s/,
+  'i',
+);
+
+export function extractPythonVersion(
+  content: string,
+  fileName: string,
+): string | undefined {
+  const match = pythonVersionRegex.exec(content);
+  if (match?.groups === undefined) {
+    logger.warn(
+      `pip-compile: failed to extract Python version from header in ${fileName} ${content}`,
+    );
+    return undefined;
+  }
+  logger.trace(
+    `pip-compile: found Python version header in ${fileName}: \n${match[0]}`,
+  );
+  const { pythonVersion } = match.groups;
+  logger.debug(
+    { fileName, pythonVersion },
+    `pip-compile: extracted Python version from header`,
+  );
+  return pythonVersion;
+}
+
 function throwForDisallowedOption(arg: string): void {
   if (disallowedPipOptions.includes(arg)) {
     throw new Error(`Option ${arg} not allowed for this manager`);

lib/modules/manager/pip-compile/readme.md

@@ -64,7 +64,7 @@ Because `pip-compile` will update source files with their associated manager you
 
 ### Configuration of Python version
 
-By default Renovate uses the latest version of Python.
+By default Renovate extracts Python version from the header.
 To get Renovate to use another version of Python, add a constraints` rule to the Renovate config:
 
 ```json