diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 0b1b9b3cf01db9..9179ba9371245e 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -2101,7 +2101,7 @@ In the case that a user is automatically added as reviewer (such as Renovate App ## ignoreScripts -Applicable for npm and Composer only for now. Set this to `true` if running scripts causes problems. +Applicable for npm, Composer and Copier only for now. Set this to `true` if running scripts causes problems. ## ignoreTests diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 419039e83b1ac5..e2a8bd2dc768e8 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -888,7 +888,7 @@ const options: RenovateOptions[] = [ 'Set this to `false` if `allowScripts=true` and you wish to run scripts when updating lock files.', type: 'boolean', default: true, - supportedManagers: ['npm', 'composer'], + supportedManagers: ['npm', 'composer', 'copier'], }, { name: 'platform', diff --git a/lib/modules/manager/api.ts b/lib/modules/manager/api.ts index 1505953e30dfe2..db30f841bb4edc 100644 --- a/lib/modules/manager/api.ts +++ b/lib/modules/manager/api.ts @@ -22,6 +22,7 @@ import * as cloudbuild from './cloudbuild'; import * as cocoapods from './cocoapods'; import * as composer from './composer'; import * as conan from './conan'; +import * as copier from './copier'; import * as cpanfile from './cpanfile'; import * as crossplane from './crossplane'; import * as depsEdn from './deps-edn'; @@ -122,6 +123,7 @@ api.set('cloudbuild', cloudbuild); api.set('cocoapods', cocoapods); api.set('composer', composer); api.set('conan', conan); +api.set('copier', copier); api.set('cpanfile', cpanfile); api.set('crossplane', crossplane); api.set('deps-edn', depsEdn); diff --git a/lib/modules/manager/copier/artifacts.spec.ts b/lib/modules/manager/copier/artifacts.spec.ts new file mode 100644 index 00000000000000..4c2c9dc83d5273 --- /dev/null +++ b/lib/modules/manager/copier/artifacts.spec.ts @@ -0,0 +1,387 @@ +import { mockDeep } from 'jest-mock-extended'; +import { join } from 'upath'; +import { mockExecAll } from '../../../../test/exec-util'; +import { fs, git, mocked, partial } from '../../../../test/util'; +import { GlobalConfig } from '../../../config/global'; +import type { RepoGlobalConfig } from '../../../config/types'; +import { logger } from '../../../logger'; +import type { StatusResult } from '../../../util/git/types'; +import * as _datasource from '../../datasource'; +import type { UpdateArtifactsConfig, Upgrade } from '../types'; +import { updateArtifacts } from '.'; + +const datasource = mocked(_datasource); + +jest.mock('../../../util/git'); +jest.mock('../../../util/fs'); +jest.mock('../../datasource', () => mockDeep()); + +process.env.CONTAINERBASE = 'true'; + +const config: UpdateArtifactsConfig = { + ignoreScripts: true, +}; + +const upgrades: Upgrade[] = [ + { + depName: 'https://github.com/foo/bar', + currentValue: '1.0.0', + newValue: '1.1.0', + }, +]; + +const adminConfig: RepoGlobalConfig = { + localDir: join('/tmp/github/some/repo'), + cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), + allowScripts: false, +}; + +describe('modules/manager/copier/artifacts', () => { + beforeEach(() => { + GlobalConfig.set(adminConfig); + + // Mock git repo status + git.getRepoStatus.mockResolvedValue( + partial({ + conflicted: [], + modified: ['.copier-answers.yml'], + not_added: [], + deleted: [], + renamed: [], + }), + ); + }); + + afterEach(() => { + fs.readLocalFile.mockClear(); + git.getRepoStatus.mockClear(); + }); + + describe('updateArtifacts()', () => { + it('returns null if newVersion is not provided', async () => { + const execSnapshots = mockExecAll(); + + const invalidUpgrade = [ + { ...upgrades[0], newValue: undefined, newVersion: undefined }, + ]; + + const result = await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: invalidUpgrade, + newPackageFileContent: '', + config, + }); + + expect(result).toEqual([ + { + artifactError: { + lockFile: '.copier-answers.yml', + stderr: 'Missing copier template version to update to', + }, + }, + ]); + expect(execSnapshots).toEqual([]); + }); + + it('reports an error if no upgrade is specified', async () => { + const execSnapshots = mockExecAll(); + + const result = await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: [], + newPackageFileContent: '', + config, + }); + + expect(result).toEqual([ + { + artifactError: { + lockFile: '.copier-answers.yml', + stderr: 'Unexpected number of dependencies: 0 (should be 1)', + }, + }, + ]); + expect(execSnapshots).toEqual([]); + }); + + it('invokes copier update with the correct options by default', async () => { + const execSnapshots = mockExecAll(); + + await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config: {}, + }); + + expect(execSnapshots).toMatchObject([ + { + cmd: 'copier update --skip-answered --defaults --answers-file .copier-answers.yml --vcs-ref 1.1.0', + }, + ]); + }); + + it.each` + pythonConstraint | copierConstraint + ${null} | ${null} + ${'3.11.3'} | ${null} + ${null} | ${'9.1.0'} + ${'3.11.3'} | ${'9.1.0'} + `( + `supports dynamic install with constraints python=$pythonConstraint copier=$copierConstraint`, + async ({ pythonConstraint, copierConstraint }) => { + GlobalConfig.set({ ...adminConfig, binarySource: 'install' }); + const constraintConfig = { + python: pythonConstraint ?? '', + copier: copierConstraint ?? '', + }; + if (!pythonConstraint) { + datasource.getPkgReleases.mockResolvedValueOnce({ + releases: [{ version: '3.12.4' }], + }); + } + if (!copierConstraint) { + datasource.getPkgReleases.mockResolvedValueOnce({ + releases: [{ version: '9.2.0' }], + }); + } + const execSnapshots = mockExecAll(); + + expect( + await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config: { + ...config, + constraints: constraintConfig, + }, + }), + ).not.toBeNull(); + + expect(execSnapshots).toMatchObject([ + { cmd: `install-tool python ${pythonConstraint ?? '3.12.4'}` }, + { cmd: `install-tool copier ${copierConstraint ?? '9.2.0'}` }, + { + cmd: 'copier update --skip-answered --defaults --answers-file .copier-answers.yml --vcs-ref 1.1.0', + }, + ]); + }, + ); + + it('includes --trust when allowScripts is true and ignoreScripts is false', async () => { + GlobalConfig.set({ ...adminConfig, allowScripts: true }); + const execSnapshots = mockExecAll(); + + const trustConfig = { + ...config, + ignoreScripts: false, + }; + + await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config: trustConfig, + }); + + expect(execSnapshots).toMatchObject([ + { + cmd: 'copier update --skip-answered --defaults --trust --answers-file .copier-answers.yml --vcs-ref 1.1.0', + }, + ]); + }); + + it('does not include --trust when ignoreScripts is true', async () => { + GlobalConfig.set({ ...adminConfig, allowScripts: true }); + const execSnapshots = mockExecAll(); + + await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config, + }); + + expect(execSnapshots).toMatchObject([ + { + cmd: 'copier update --skip-answered --defaults --answers-file .copier-answers.yml --vcs-ref 1.1.0', + }, + ]); + }); + + it('handles exec errors', async () => { + mockExecAll(new Error('exec exception')); + const result = await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config, + }); + + expect(result).toEqual([ + { + artifactError: { + lockFile: '.copier-answers.yml', + stderr: 'exec exception', + }, + }, + ]); + }); + + it('does not report changes if answers-file was not changed', async () => { + mockExecAll(); + + git.getRepoStatus.mockResolvedValueOnce( + partial({ + conflicted: [], + modified: [], + not_added: ['new_file.py'], + deleted: ['old_file.py'], + renamed: [], + }), + ); + + const result = await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config, + }); + + expect(result).toBeNull(); + }); + + it('returns updated artifacts if repo status has changes', async () => { + mockExecAll(); + + git.getRepoStatus.mockResolvedValueOnce( + partial({ + conflicted: [], + modified: ['.copier-answers.yml'], + not_added: ['new_file.py'], + deleted: ['old_file.py'], + renamed: [{ from: 'renamed_old.py', to: 'renamed_new.py' }], + }), + ); + + fs.readLocalFile.mockResolvedValueOnce( + '_src: https://github.com/foo/bar\n_commit: 1.1.0', + ); + fs.readLocalFile.mockResolvedValueOnce('new file contents'); + fs.readLocalFile.mockResolvedValueOnce('renamed file contents'); + + const result = await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config, + }); + + expect(result).toEqual([ + { + file: { + type: 'addition', + path: '.copier-answers.yml', + contents: '_src: https://github.com/foo/bar\n_commit: 1.1.0', + }, + }, + { + file: { + type: 'addition', + path: 'new_file.py', + contents: 'new file contents', + }, + }, + { + file: { + type: 'deletion', + path: 'old_file.py', + }, + }, + { + file: { + type: 'deletion', + path: 'renamed_old.py', + }, + }, + { + file: { + type: 'addition', + path: 'renamed_new.py', + contents: 'renamed file contents', + }, + }, + ]); + }); + + it('warns about, but adds conflicts', async () => { + mockExecAll(); + + git.getRepoStatus.mockResolvedValueOnce( + partial({ + conflicted: ['conflict_file.py'], + modified: ['.copier-answers.yml'], + not_added: ['new_file.py'], + deleted: ['old_file.py'], + renamed: [], + }), + ); + + fs.readLocalFile.mockResolvedValueOnce( + '_src: https://github.com/foo/bar\n_commit: 1.1.0', + ); + fs.readLocalFile.mockResolvedValueOnce('new file contents'); + fs.readLocalFile.mockResolvedValueOnce('conflict file contents'); + + const result = await updateArtifacts({ + packageFileName: '.copier-answers.yml', + updatedDeps: upgrades, + newPackageFileContent: '', + config, + }); + expect(logger.debug).toHaveBeenCalledWith( + { + depName: 'https://github.com/foo/bar', + packageFileName: '.copier-answers.yml', + }, + 'Updating the Copier template yielded 1 merge conflicts. Please check the proposed changes carefully! Conflicting files:\n * conflict_file.py', + ); + expect(result).toEqual([ + { + file: { + type: 'addition', + path: '.copier-answers.yml', + contents: '_src: https://github.com/foo/bar\n_commit: 1.1.0', + }, + }, + { + file: { + type: 'addition', + path: 'new_file.py', + contents: 'new file contents', + }, + }, + { + file: { + type: 'addition', + path: 'conflict_file.py', + contents: 'conflict file contents', + }, + notice: { + file: 'conflict_file.py', + message: + 'This file had merge conflicts. Please check the proposed changes carefully!', + }, + }, + { + file: { + type: 'deletion', + path: 'old_file.py', + }, + }, + ]); + }); + }); +}); diff --git a/lib/modules/manager/copier/artifacts.ts b/lib/modules/manager/copier/artifacts.ts new file mode 100644 index 00000000000000..1b6aa236cf9bfe --- /dev/null +++ b/lib/modules/manager/copier/artifacts.ts @@ -0,0 +1,161 @@ +import { quote } from 'shlex'; +import { GlobalConfig } from '../../../config/global'; +import { logger } from '../../../logger'; +import { exec } from '../../../util/exec'; +import type { ExecOptions } from '../../../util/exec/types'; +import { readLocalFile } from '../../../util/fs'; +import { getRepoStatus } from '../../../util/git'; +import type { + UpdateArtifact, + UpdateArtifactsConfig, + UpdateArtifactsResult, +} from '../types'; +import { + getCopierVersionConstraint, + getPythonVersionConstraint, +} from './utils'; + +const DEFAULT_COMMAND_OPTIONS = ['--skip-answered', '--defaults']; + +function buildCommand( + config: UpdateArtifactsConfig, + packageFileName: string, + newVersion: string, +): string { + const command = ['copier', 'update', ...DEFAULT_COMMAND_OPTIONS]; + if (GlobalConfig.get('allowScripts') && !config.ignoreScripts) { + command.push('--trust'); + } + command.push( + '--answers-file', + quote(packageFileName), + '--vcs-ref', + quote(newVersion), + ); + return command.join(' '); +} + +function artifactError( + packageFileName: string, + message: string, +): UpdateArtifactsResult[] { + return [ + { + artifactError: { + lockFile: packageFileName, + stderr: message, + }, + }, + ]; +} + +export async function updateArtifacts({ + packageFileName, + updatedDeps, + config, +}: UpdateArtifact): Promise { + if (!updatedDeps || updatedDeps.length !== 1) { + // Each answers file (~ packageFileName) has exactly one dependency to update. + return artifactError( + packageFileName, + `Unexpected number of dependencies: ${updatedDeps.length} (should be 1)`, + ); + } + + const newVersion = updatedDeps[0]?.newVersion ?? updatedDeps[0]?.newValue; + if (!newVersion) { + return artifactError( + packageFileName, + 'Missing copier template version to update to', + ); + } + + const command = buildCommand(config, packageFileName, newVersion); + const execOptions: ExecOptions = { + docker: {}, + userConfiguredEnv: config.env, + toolConstraints: [ + { + toolName: 'python', + constraint: getPythonVersionConstraint(config), + }, + { + toolName: 'copier', + constraint: getCopierVersionConstraint(config), + }, + ], + }; + try { + await exec(command, execOptions); + } catch (err) { + logger.debug({ err }, `Failed to update copier template: ${err.message}`); + return artifactError(packageFileName, err.message); + } + + const status = await getRepoStatus(); + // If the answers file didn't change, Copier did not update anything. + if (!status.modified.includes(packageFileName)) { + return null; + } + + if (status.conflicted.length > 0) { + // Sometimes, Copier erroneously reports conflicts. + const msg = + `Updating the Copier template yielded ${status.conflicted.length} merge conflicts. ` + + 'Please check the proposed changes carefully! Conflicting files:\n * ' + + status.conflicted.join('\n * '); + logger.debug({ packageFileName, depName: updatedDeps[0]?.depName }, msg); + } + + const res: UpdateArtifactsResult[] = []; + + for (const f of [ + ...status.modified, + ...status.not_added, + ...status.conflicted, + ]) { + const fileRes: UpdateArtifactsResult = { + file: { + type: 'addition', + path: f, + contents: await readLocalFile(f), + }, + }; + if (status.conflicted.includes(f)) { + // Make the reviewer aware of the conflicts. + // This will be posted in a comment. + fileRes.notice = { + file: f, + message: + 'This file had merge conflicts. Please check the proposed changes carefully!', + }; + } + res.push(fileRes); + } + for (const f of status.deleted) { + res.push({ + file: { + type: 'deletion', + path: f, + }, + }); + } + // `git status` might detect a rename, which is then not contained + // in not_added/deleted. Ensure we respect renames as well if they happen. + for (const f of status.renamed) { + res.push({ + file: { + type: 'deletion', + path: f.from, + }, + }); + res.push({ + file: { + type: 'addition', + path: f.to, + contents: await readLocalFile(f.to), + }, + }); + } + return res; +} diff --git a/lib/modules/manager/copier/extract.spec.ts b/lib/modules/manager/copier/extract.spec.ts new file mode 100644 index 00000000000000..d0bc17dc95d998 --- /dev/null +++ b/lib/modules/manager/copier/extract.spec.ts @@ -0,0 +1,58 @@ +import { extractPackageFile } from '.'; + +describe('modules/manager/copier/extract', () => { + describe('extractPackageFile()', () => { + it('extracts repository and version from .copier-answers.yml', () => { + const content = ` + _commit: v1.0.0 + _src_path: https://github.com/username/template-repo + `; + const result = extractPackageFile(content); + expect(result).toEqual({ + deps: [ + { + depName: 'https://github.com/username/template-repo', + packageName: 'https://github.com/username/template-repo', + currentValue: 'v1.0.0', + datasource: 'git-tags', + depType: 'template', + }, + ], + }); + }); + + it('returns null for invalid .copier-answers.yml', () => { + const content = ` + not_valid: + key: value + `; + const result = extractPackageFile(content); + expect(result).toBeNull(); + }); + + it('returns null for invalid _src_path', () => { + const content = ` + _commit: v1.0.0 + _src_path: notaurl + `; + const result = extractPackageFile(content); + expect(result).toBeNull(); + }); + + it('returns null for missing _commit field', () => { + const content = ` + _src_path: https://github.com/username/template-repo + `; + const result = extractPackageFile(content); + expect(result).toBeNull(); + }); + + it('returns null for missing _src_path field', () => { + const content = ` + _commit: v1.0.0 + `; + const result = extractPackageFile(content); + expect(result).toBeNull(); + }); + }); +}); diff --git a/lib/modules/manager/copier/extract.ts b/lib/modules/manager/copier/extract.ts new file mode 100644 index 00000000000000..bb58c4e43ab649 --- /dev/null +++ b/lib/modules/manager/copier/extract.ts @@ -0,0 +1,31 @@ +import { logger } from '../../../logger'; +import { GitTagsDatasource } from '../../datasource/git-tags'; +import type { PackageDependency, PackageFileContent } from '../types'; +import { CopierAnswersFile } from './schema'; + +export function extractPackageFile( + content: string, + packageFile?: string, +): PackageFileContent | null { + let parsed: CopierAnswersFile; + try { + parsed = CopierAnswersFile.parse(content); + } catch (err) { + logger.debug({ err, packageFile }, `Parsing Copier answers YAML failed`); + return null; + } + + const deps: PackageDependency[] = [ + { + datasource: GitTagsDatasource.id, + depName: parsed._src_path, + packageName: parsed._src_path, + depType: 'template', + currentValue: parsed._commit, + }, + ]; + + return { + deps, + }; +} diff --git a/lib/modules/manager/copier/index.ts b/lib/modules/manager/copier/index.ts new file mode 100644 index 00000000000000..a10f78f09c5d51 --- /dev/null +++ b/lib/modules/manager/copier/index.ts @@ -0,0 +1,12 @@ +import { GitTagsDatasource } from '../../datasource/git-tags'; +import * as pep440 from '../../versioning/pep440'; +export { updateArtifacts } from './artifacts'; +export { extractPackageFile } from './extract'; +export { updateDependency } from './update'; + +export const defaultConfig = { + fileMatch: ['(^|/)\\.copier-answers(\\..+)?\\.ya?ml'], + versioning: pep440.id, +}; + +export const supportedDatasources = [GitTagsDatasource.id]; diff --git a/lib/modules/manager/copier/readme.md b/lib/modules/manager/copier/readme.md new file mode 100644 index 00000000000000..53db498735db24 --- /dev/null +++ b/lib/modules/manager/copier/readme.md @@ -0,0 +1,7 @@ +Keeps Copier templates up to date. +Supports multiple `.copier-answers(...).y(a)ml` files in a single repository. +If a template requires unsafe features, Copier must be invoked with the `--trust` flag. +Enabling this behavior must be allowed in the [self-hosted configuration](../../../self-hosted-configuration.md) via `allowScripts`. +Actually enable it in the [configuration](../../../configuration-options.md) by setting `ignoreScripts` to `false`. + +If you need to change the versioning format, read the [versioning](../../versioning/index.md) documentation to learn more. diff --git a/lib/modules/manager/copier/schema.ts b/lib/modules/manager/copier/schema.ts new file mode 100644 index 00000000000000..27e0db3459ff27 --- /dev/null +++ b/lib/modules/manager/copier/schema.ts @@ -0,0 +1,11 @@ +import { z } from 'zod'; +import { Yaml } from '../../../util/schema-utils'; + +export const CopierAnswersFile = Yaml.pipe( + z.object({ + _commit: z.string(), + _src_path: z.string().url(), + }), +); + +export type CopierAnswersFile = z.infer; diff --git a/lib/modules/manager/copier/update.spec.ts b/lib/modules/manager/copier/update.spec.ts new file mode 100644 index 00000000000000..29c1c344484349 --- /dev/null +++ b/lib/modules/manager/copier/update.spec.ts @@ -0,0 +1,25 @@ +import { codeBlock } from 'common-tags'; +import { updateDependency } from '.'; + +describe('modules/manager/copier/update', () => { + describe('updateDependency', () => { + it('should append a new marking line at the end to trigger the artifact update', () => { + const fileContent = codeBlock` + _src_path: https://foo.bar/baz/quux + _commit: 1.0.0 + `; + const ret = updateDependency({ fileContent, upgrade: {} }); + expect(ret).toBe(`${fileContent}\n#copier updated`); + }); + + it('should not update again if the new line has been appended', () => { + const fileContent = codeBlock` + _src_path: https://foo.bar/baz/quux + _commit: 1.0.0 + #copier updated + `; + const ret = updateDependency({ fileContent, upgrade: {} }); + expect(ret).toBe(fileContent); + }); + }); +}); diff --git a/lib/modules/manager/copier/update.ts b/lib/modules/manager/copier/update.ts new file mode 100644 index 00000000000000..3de43be3a739cc --- /dev/null +++ b/lib/modules/manager/copier/update.ts @@ -0,0 +1,22 @@ +import { logger } from '../../../logger'; +import type { UpdateDependencyConfig } from '../types'; + +const updateLine = '#copier updated'; + +/** + * updateDependency appends a comment line once. + * This is only for the purpose of triggering the artifact update. + * Copier needs to update its answers file itself. + */ +export function updateDependency({ + fileContent, + upgrade, +}: UpdateDependencyConfig): string | null { + logger.trace({ upgrade }, `copier.updateDependency()`); + if (!fileContent.endsWith(updateLine)) { + logger.debug(`append update line to the fileContent if it hasn't been`); + return `${fileContent}\n${updateLine}`; + } + + return fileContent; +} diff --git a/lib/modules/manager/copier/utils.ts b/lib/modules/manager/copier/utils.ts new file mode 100644 index 00000000000000..156507c0981bd0 --- /dev/null +++ b/lib/modules/manager/copier/utils.ts @@ -0,0 +1,31 @@ +import is from '@sindresorhus/is'; +import { logger } from '../../../logger'; +import type { UpdateArtifactsConfig } from '../types'; + +export function getPythonVersionConstraint( + config: UpdateArtifactsConfig, +): string | undefined | null { + const { constraints = {} } = config; + const { python } = constraints; + + if (is.nonEmptyString(python)) { + logger.debug('Using python constraint from config'); + return python; + } + + return undefined; +} + +export function getCopierVersionConstraint( + config: UpdateArtifactsConfig, +): string { + const { constraints = {} } = config; + const { copier } = constraints; + + if (is.nonEmptyString(copier)) { + logger.debug('Using copier constraint from config'); + return copier; + } + + return ''; +} diff --git a/lib/util/exec/containerbase.ts b/lib/util/exec/containerbase.ts index 78678e5efa83cc..29462cad251308 100644 --- a/lib/util/exec/containerbase.ts +++ b/lib/util/exec/containerbase.ts @@ -38,6 +38,11 @@ const allToolConfig: Record = { packageName: 'composer/composer', versioning: composerVersioningId, }, + copier: { + datasource: 'pypi', + packageName: 'copier', + versioning: pep440VersioningId, + }, corepack: { datasource: 'npm', packageName: 'corepack',