From 036858c11f447746c22ab3bdaf97fc2d20397d24 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 07:12:05 +0200 Subject: [PATCH 01/10] feat: disable git-lfs pull temporary --- lib/util/git/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index ac735d03c5d034..7f0f7ae2ed022a 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -299,7 +299,7 @@ export async function syncGit(): Promise { const cloneStart = Date.now(); try { // clone only the default branch - const opts = ['--depth=10']; + const opts = ['--depth=10', '-c', 'lfs.fetchexclude=*']; if (config.extraCloneOpts) { Object.entries(config.extraCloneOpts).forEach((e) => opts.push(e[0], `${e[1]}`) From 45404a91affe0fe99e6004d2ed956ccd34a5de43 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 09:21:37 +0200 Subject: [PATCH 02/10] feat: add git lfs admin option --- docs/usage/self-hosted-configuration.md | 4 ++++ lib/config/admin.ts | 1 + lib/config/definitions.ts | 7 +++++++ lib/config/types.ts | 1 + lib/util/git/index.ts | 8 ++++++-- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 9662206fef2334..2e5ed753acf75f 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -11,6 +11,10 @@ Please also see [Self-Hosted Experimental Options](./self-hosted-experimental.md ## allowCustomCrateRegistries +## allowGitLfs + +You need to also configure `gitNoVerify` to allow commit and push git lfs objects. + ## allowPostUpgradeCommandTemplating Set to true to allow templating of dependency level post-upgrade commands. diff --git a/lib/config/admin.ts b/lib/config/admin.ts index 83eb5bdfcae106..ef2574e338fc55 100644 --- a/lib/config/admin.ts +++ b/lib/config/admin.ts @@ -5,6 +5,7 @@ let adminConfig: RepoAdminConfig = {}; // TODO: once admin config work is complete, add a test to make sure this list includes all options with admin=true (#9603) const repoAdminOptions = [ 'allowCustomCrateRegistries', + 'allowGitLfs', 'allowPostUpgradeCommandTemplating', 'allowScripts', 'allowedPostUpgradeCommands', diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index 26290fa07ebea7..ee4b8fb5a37140 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -7,6 +7,13 @@ import * as pep440Versioning from '../versioning/pep440'; import type { RenovateOptions } from './types'; const options: RenovateOptions[] = [ + { + name: 'allowGitLfs', + description: 'If true allow git-lfs to pull files.', + type: 'boolean', + default: false, + admin: true, + }, { name: 'allowPostUpgradeCommandTemplating', description: 'If true allow templating for post-upgrade commands.', diff --git a/lib/config/types.ts b/lib/config/types.ts index fdd56479c4d8e2..5444c89be68cac 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -88,6 +88,7 @@ export interface GlobalOnlyConfig { // The below should contain config options where admin=true export interface RepoAdminConfig { allowCustomCrateRegistries?: boolean; + allowGitLfs?: boolean; allowPostUpgradeCommandTemplating?: boolean; allowScripts?: boolean; allowedPostUpgradeCommands?: string[]; diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index 7f0f7ae2ed022a..af16b54ecfc9d7 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -267,7 +267,7 @@ export async function syncGit(): Promise { return; } gitInitialized = true; - const { localDir } = getAdminConfig(); + const { localDir, allowGitLfs } = getAdminConfig(); logger.debug('Initializing git repository into ' + localDir); const gitHead = join(localDir, '.git/HEAD'); let clone = true; @@ -299,12 +299,16 @@ export async function syncGit(): Promise { const cloneStart = Date.now(); try { // clone only the default branch - const opts = ['--depth=10', '-c', 'lfs.fetchexclude=*']; + const opts = ['--depth=10']; if (config.extraCloneOpts) { Object.entries(config.extraCloneOpts).forEach((e) => opts.push(e[0], `${e[1]}`) ); } + // istanbul ignore if: Will be changed in future, see #6842 + if (allowGitLfs) { + opts.push('-c', 'lfs.fetchexclude=*'); + } await git.clone(config.url, '.', opts); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'git clone error'); From d3a55c6b72b847727e5254ae1f6cac4f15f466ab Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 09:33:05 +0200 Subject: [PATCH 03/10] fix: wrong condition --- lib/util/git/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index af16b54ecfc9d7..1ee4cbbd01d70f 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -306,7 +306,7 @@ export async function syncGit(): Promise { ); } // istanbul ignore if: Will be changed in future, see #6842 - if (allowGitLfs) { + if (!allowGitLfs) { opts.push('-c', 'lfs.fetchexclude=*'); } await git.clone(config.url, '.', opts); From 73c5e401b57c0daa75b62384ee8e91a13cec278c Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 09:43:17 +0200 Subject: [PATCH 04/10] chore: fix comment --- lib/util/git/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index 1ee4cbbd01d70f..2a0b8feb5c298c 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -305,7 +305,8 @@ export async function syncGit(): Promise { opts.push(e[0], `${e[1]}`) ); } - // istanbul ignore if: Will be changed in future, see #6842 + // will be changed in future, see #6842 + // istanbul ignore if if (!allowGitLfs) { opts.push('-c', 'lfs.fetchexclude=*'); } From 4fe09e717e053c69a89f992a8a2cfdeb7f669374 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 09:48:15 +0200 Subject: [PATCH 05/10] docs: update docs --- docs/usage/self-hosted-configuration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 2e5ed753acf75f..1eac6a2abf0bd5 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -13,7 +13,7 @@ Please also see [Self-Hosted Experimental Options](./self-hosted-experimental.md ## allowGitLfs -You need to also configure `gitNoVerify` to allow commit and push git lfs objects. +**Note** You need to also configure `gitNoVerify` to allow commit and push git lfs objects. ## allowPostUpgradeCommandTemplating @@ -228,6 +228,8 @@ The flag can be passed to `git commit` and/or `git push`. Read the documentation for [git commit --no-verify](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt---no-verify) and [git push --no-verify](https://git-scm.com/docs/git-push#Documentation/git-push.txt---no-verify) to learn exactly what each flag does. To learn more about Git hooks, read the [Pro Git 2 book, section on Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks). +**Note** If you use git-lfs checkout `allowGitLfs` option. + ## gitPrivateKey This should be an armored private key, e.g. the type you get from running `gpg --export-secret-keys --armor 92066A17F0D1707B4E96863955FEF5171C45FAE5 > private.key`. From 0c1c5fdcc934ebcfd258f5716bed72c39844494d Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 11:25:44 +0200 Subject: [PATCH 06/10] Update docs/usage/self-hosted-configuration.md Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- docs/usage/self-hosted-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 1eac6a2abf0bd5..42814357d3e172 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -228,7 +228,7 @@ The flag can be passed to `git commit` and/or `git push`. Read the documentation for [git commit --no-verify](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt---no-verify) and [git push --no-verify](https://git-scm.com/docs/git-push#Documentation/git-push.txt---no-verify) to learn exactly what each flag does. To learn more about Git hooks, read the [Pro Git 2 book, section on Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks). -**Note** If you use git-lfs checkout `allowGitLfs` option. +**Note** If you use Git LFS you also need to configure the `allowGitLfs` option to `true`. ## gitPrivateKey From e3a5c69ad840e4f4c1751763a3513074c84a11d8 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 11:34:08 +0200 Subject: [PATCH 07/10] Update docs/usage/self-hosted-configuration.md Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- docs/usage/self-hosted-configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 42814357d3e172..72c6f6cddecb20 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -13,7 +13,8 @@ Please also see [Self-Hosted Experimental Options](./self-hosted-experimental.md ## allowGitLfs -**Note** You need to also configure `gitNoVerify` to allow commit and push git lfs objects. +**Note** You need to also set the Git commands that Renovate passes to the `gitNoVerify` option. +This way Renovate can commit and push Git LFS objects. ## allowPostUpgradeCommandTemplating From bebde38089c59a18ccc3b49993e3d520682f834d Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 7 Jul 2021 17:18:56 +0200 Subject: [PATCH 08/10] Update docs/usage/self-hosted-configuration.md Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- docs/usage/self-hosted-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 72c6f6cddecb20..20cce4a49ffb9c 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -229,7 +229,7 @@ The flag can be passed to `git commit` and/or `git push`. Read the documentation for [git commit --no-verify](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt---no-verify) and [git push --no-verify](https://git-scm.com/docs/git-push#Documentation/git-push.txt---no-verify) to learn exactly what each flag does. To learn more about Git hooks, read the [Pro Git 2 book, section on Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks). -**Note** If you use Git LFS you also need to configure the `allowGitLfs` option to `true`. +**Note** If you use Git LFS you also need to set the `allowGitLfs` option to `true`. ## gitPrivateKey From 20f6c6d2b086d7182f222a02ba1f68b36670b781 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Fri, 30 Jul 2021 10:31:18 +0200 Subject: [PATCH 09/10] feat: add full git-lfs config options --- docs/usage/configuration-options.md | 12 ++++++++++++ docs/usage/self-hosted-configuration.md | 4 ++-- lib/config/definitions.ts | 17 +++++++++++++++-- lib/config/types.ts | 3 +++ lib/util/git/index.ts | 23 +++++++++++++++++++++-- 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 750e96b8d98c0b..d8fb1c0f33001c 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -705,6 +705,18 @@ If you enabled `automerge` in the Renovate config, you can speed up the automerg Caution (fixed in GitLab >= 12.7): when this option is enabled it is possible due to a bug in GitLab that MRs with failing pipelines might still get merged. This is caused by a race condition in GitLab's Merge Request API - [read the corresponding issue](https://gitlab.com/gitlab-org/gitlab/issues/26293) for details. +## gitLfsExclude + +See also `allowGitLfs` and `gitNoVerify` option. + +**Note** This is not supported in WhiteSource Renovate Hosted App. + +## gitLfsInclude + +See also `allowGitLfs` and `gitNoVerify` option. + +**Note** This is not supported in WhiteSource Renovate Hosted App. + ## golang Configuration added here applies for all Go-related updates, however currently the only supported package manager for Go is the native Go Modules (the `gomod` manager). diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 20cce4a49ffb9c..2e0a3abb7fd7d0 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -16,6 +16,8 @@ Please also see [Self-Hosted Experimental Options](./self-hosted-experimental.md **Note** You need to also set the Git commands that Renovate passes to the `gitNoVerify` option. This way Renovate can commit and push Git LFS objects. +**Note** This currently don't work in official renovate docker images, see [containerbase/buildpack#13](https://github.com/containerbase/buildpack/issues/13) + ## allowPostUpgradeCommandTemplating Set to true to allow templating of dependency level post-upgrade commands. @@ -229,8 +231,6 @@ The flag can be passed to `git commit` and/or `git push`. Read the documentation for [git commit --no-verify](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt---no-verify) and [git push --no-verify](https://git-scm.com/docs/git-push#Documentation/git-push.txt---no-verify) to learn exactly what each flag does. To learn more about Git hooks, read the [Pro Git 2 book, section on Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks). -**Note** If you use Git LFS you also need to set the `allowGitLfs` option to `true`. - ## gitPrivateKey This should be an armored private key, e.g. the type you get from running `gpg --export-secret-keys --armor 92066A17F0D1707B4E96863955FEF5171C45FAE5 > private.key`. diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index ee4b8fb5a37140..f7395f3653e6d9 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -9,9 +9,9 @@ import type { RenovateOptions } from './types'; const options: RenovateOptions[] = [ { name: 'allowGitLfs', - description: 'If true allow git-lfs to pull files.', + description: 'If false disallow git-lfs to pull files.', type: 'boolean', - default: false, + default: true, admin: true, }, { @@ -2031,6 +2031,19 @@ const options: RenovateOptions[] = [ stage: 'global', admin: true, }, + { + name: 'gitLfsInclude', + description: 'Which files should be included in git-lfs checkout', + stage: 'repository', + type: 'string', + default: '.yarn,yarn.lock,**/package-lock.json', + }, + { + name: 'gitLfsExclude', + description: 'Which files should be excluded in git-lfs checkout', + stage: 'repository', + type: 'string', + }, ]; export function getOptions(): RenovateOptions[] { diff --git a/lib/config/types.ts b/lib/config/types.ts index 5444c89be68cac..e61f70bfe50a15 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -168,6 +168,9 @@ export interface RenovateConfig gitAuthor?: string; + gitLfsInclude?: string; + gitLfsExclude?: string; + hostRules?: HostRule[]; ignorePresets?: string[]; diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index 2a0b8feb5c298c..20b9c709e62644 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -1,4 +1,5 @@ import URL from 'url'; +import is from '@sindresorhus/is'; import fs from 'fs-extra'; import Git, { DiffResult as DiffResult_, @@ -56,6 +57,10 @@ interface LocalConfig extends StorageConfig { branchCommits: Record; branchIsModified: Record; branchPrefix: string; + + gitLfsInclude?: string; + gitLfsExclude?: string; + ignoredAuthors: string[]; } @@ -185,6 +190,8 @@ export async function initRepo(args: StorageConfig): Promise { config.ignoredAuthors = []; config.additionalBranches = []; config.branchIsModified = {}; + config.gitLfsInclude = undefined; + config.gitLfsExclude = undefined; const { localDir } = getAdminConfig(); git = Git(localDir); gitInitialized = false; @@ -237,9 +244,13 @@ async function setBranchPrefix(branchPrefix: string): Promise { export async function setUserRepoConfig({ branchPrefix, gitIgnoredAuthors, + gitLfsInclude, + gitLfsExclude, }: RenovateConfig): Promise { await setBranchPrefix(branchPrefix); config.ignoredAuthors = gitIgnoredAuthors ?? []; + config.gitLfsInclude = gitLfsInclude; + config.gitLfsExclude = gitLfsExclude; } export async function getSubmodules(): Promise { @@ -305,11 +316,19 @@ export async function syncGit(): Promise { opts.push(e[0], `${e[1]}`) ); } - // will be changed in future, see #6842 + // istanbul ignore if - if (!allowGitLfs) { + if (allowGitLfs) { + if (is.nonEmptyString(config.gitLfsInclude)) { + opts.push('-c', `lfs.fetchinclude=${config.gitLfsInclude}`); + } + if (is.nonEmptyString(config.gitLfsExclude)) { + opts.push('-c', `lfs.fetchexclude=${config.gitLfsExclude}`); + } + } else { opts.push('-c', 'lfs.fetchexclude=*'); } + await git.clone(config.url, '.', opts); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'git clone error'); From 361f501554def42d5ace9856987c8e54c8db10d5 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 30 Jul 2021 12:21:52 +0200 Subject: [PATCH 10/10] Apply suggestions from code review Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- docs/usage/configuration-options.md | 8 ++++---- docs/usage/self-hosted-configuration.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index d8fb1c0f33001c..276c2d9b7c36f4 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -707,15 +707,15 @@ This is caused by a race condition in GitLab's Merge Request API - [read the cor ## gitLfsExclude -See also `allowGitLfs` and `gitNoVerify` option. +See also the `allowGitLfs` and `gitNoVerify` options. -**Note** This is not supported in WhiteSource Renovate Hosted App. +**Note** This is not supported in the WhiteSource Renovate Hosted App. ## gitLfsInclude -See also `allowGitLfs` and `gitNoVerify` option. +See also the `allowGitLfs` and `gitNoVerify` options. -**Note** This is not supported in WhiteSource Renovate Hosted App. +**Note** This is not supported in the WhiteSource Renovate Hosted App. ## golang diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 2e0a3abb7fd7d0..35b68d0ad9ba2c 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -16,7 +16,7 @@ Please also see [Self-Hosted Experimental Options](./self-hosted-experimental.md **Note** You need to also set the Git commands that Renovate passes to the `gitNoVerify` option. This way Renovate can commit and push Git LFS objects. -**Note** This currently don't work in official renovate docker images, see [containerbase/buildpack#13](https://github.com/containerbase/buildpack/issues/13) +**Note** `allowGitLfs` does not work in the official Renovate docker images, see issue [containerbase/buildpack#13](https://github.com/containerbase/buildpack/issues/13) for more information. ## allowPostUpgradeCommandTemplating