From b0fbaff6ff1b788f02e8ce0a5b7fd0c59aa2e34a Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 24 May 2021 16:28:21 -0400 Subject: [PATCH 1/2] Refactor sparse checkout template: idempotency and no yaml loops --- .../templates/steps/sparse-checkout.yml | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index f98cd60605..92b851ed40 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -16,23 +16,54 @@ steps: - ${{ if not(parameters.SkipDefaultCheckout) }}: - checkout: none - - ${{ each repo in parameters.Repositories }}: - - pwsh: | - $dir = "${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}" - New-Item $dir -ItemType Directory -Force - displayName: Create ${{ repo.Name }} directories - - - pwsh: | - git clone --no-checkout --filter=tree:0 git://github.com/${{ repo.Name }} . - git sparse-checkout init - $paths = ('${{ convertToJson(parameters.Paths) }}' | ConvertFrom-Json) -Join ' ' - Invoke-Expression -Command "git sparse-checkout set eng $paths" - Write-Host "Set sparse checkout paths to:" - Get-Content .git/info/sparse-checkout - displayName: Init sparse checkout ${{ repo.Name }} - workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }} - - - pwsh: | - git checkout ${{ repo.Commitish }} # this will use the default branch if repo.Commitish is empty - displayName: Sparse checkout at ${{ coalesce(repo.Commitish, 'default branch') }} - workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }} + - task: PowerShell@2 + displayName: 'Sparse checkout repositories' + inputs: + targetType: inline + # Define this inline, because of the chicken/egg problem with loading a script when nothing + # has been checked out yet. + script: | + function SparseCheckout([Array]$paths, [Array]$Repositories) + { + $paths = $paths -Join ' ' + + foreach ($repo in $Repositories) { + $dir = $repo.WorkingDirectory + if (!$dir) { + $dir = "./$($repo.Name)" + } + New-Item $dir -ItemType Directory -Force + Push-Location $dir + + if (Test-Path .git/info/sparse-checkout) { + $hasInitialized = $true + Write-Host "Repository $($repo.Name) has already been initialized. Skipping this step." + } else { + Write-Host "Repository $($repo.Name) is being initialized." + git clone --no-checkout --filter=tree:0 git://github.com/$($repo.Name) . + git sparse-checkout init + git sparse-checkout set eng + } + + Invoke-Expression -Command "git sparse-checkout add $paths" + + Write-Host "Set sparse checkout paths to:" + Get-Content .git/info/sparse-checkout + + # sparse-checkout commands after initial checkout will auto-checkout again + if (!$hasInitialized) { + git checkout $($repo.Commitish) # this will use the default branch if repo.Commitish is empty + } + + Pop-Location + } + } + + # Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. + # If the latter, convertToJson will wrap the 'string' in quotes, so remove them. + $paths = '${{ convertToJson(parameters.Paths) }}'.Trim('"') | ConvertFrom-Json + # Escape windows backslash paths + $repositories = '${{ convertToJson(parameters.Repositories) }}' -replace '\\', '\\' | ConvertFrom-Json + SparseCheckout $paths $Repositories + pwsh: true + workingDirectory: $(System.DefaultWorkingDirectory) From f355d529d3a92db2a9878da3409b48f0cb3affc7 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 25 May 2021 15:36:39 -0400 Subject: [PATCH 2/2] Add logging, re-organize sparse checkout script --- .../templates/steps/sparse-checkout.yml | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 92b851ed40..10147521db 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -23,47 +23,52 @@ steps: # Define this inline, because of the chicken/egg problem with loading a script when nothing # has been checked out yet. script: | - function SparseCheckout([Array]$paths, [Array]$Repositories) + function SparseCheckout([Array]$paths, [Hashtable]$repository) { $paths = $paths -Join ' ' - foreach ($repo in $Repositories) { - $dir = $repo.WorkingDirectory - if (!$dir) { - $dir = "./$($repo.Name)" - } - New-Item $dir -ItemType Directory -Force - Push-Location $dir - - if (Test-Path .git/info/sparse-checkout) { - $hasInitialized = $true - Write-Host "Repository $($repo.Name) has already been initialized. Skipping this step." - } else { - Write-Host "Repository $($repo.Name) is being initialized." - git clone --no-checkout --filter=tree:0 git://github.com/$($repo.Name) . - git sparse-checkout init - git sparse-checkout set eng - } + $dir = $repository.WorkingDirectory + if (!$dir) { + $dir = "./$($repository.Name)" + } + New-Item $dir -ItemType Directory -Force + Push-Location $dir - Invoke-Expression -Command "git sparse-checkout add $paths" + if (Test-Path .git/info/sparse-checkout) { + $hasInitialized = $true + Write-Host "Repository $($repository.Name) has already been initialized. Skipping this step." + } else { + Write-Host "Repository $($repository.Name) is being initialized." + git clone --no-checkout --filter=tree:0 git://github.com/$($repository.Name) . + git sparse-checkout init + git sparse-checkout set eng + } - Write-Host "Set sparse checkout paths to:" - Get-Content .git/info/sparse-checkout + $gitsparsecmd = "git sparse-checkout add $paths" + Write-Host $gitsparsecmd + Invoke-Expression -Command $gitsparsecmd - # sparse-checkout commands after initial checkout will auto-checkout again - if (!$hasInitialized) { - git checkout $($repo.Commitish) # this will use the default branch if repo.Commitish is empty - } + Write-Host "Set sparse checkout paths to:" + Get-Content .git/info/sparse-checkout - Pop-Location + # sparse-checkout commands after initial checkout will auto-checkout again + if (!$hasInitialized) { + Write-Host "git checkout $($repository.Commitish)" + git checkout $($repository.Commitish) # this will use the default branch if repo.Commitish is empty + } else { + Write-Host "Skipping checkout as repo has already been initialized" } + + Pop-Location } # Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. # If the latter, convertToJson will wrap the 'string' in quotes, so remove them. $paths = '${{ convertToJson(parameters.Paths) }}'.Trim('"') | ConvertFrom-Json - # Escape windows backslash paths - $repositories = '${{ convertToJson(parameters.Repositories) }}' -replace '\\', '\\' | ConvertFrom-Json - SparseCheckout $paths $Repositories + # Replace windows backslash paths, as Azure Pipelines default directories are sometimes formatted like 'D:\a\1\s' + $repositories = '${{ convertToJson(parameters.Repositories) }}' -replace '\\', '/' | ConvertFrom-Json -AsHashtable + foreach ($repo in $Repositories) { + SparseCheckout $paths $repo + } pwsh: true workingDirectory: $(System.DefaultWorkingDirectory)