From 886d7180b21dd8de7a399c8fb2760937479f3f69 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Thu, 30 Nov 2023 18:59:06 -0800 Subject: [PATCH 01/12] Almost finished with the normal merge. Need to add conflict resolution, continuation from a conflict resolution, and documentation updates --- .../{ => scripts}/override-proxy-version.ps1 | 0 .../testproxy/scripts/tag-merge/README.md | 0 .../scripts/tag-merge/merge-proxy-tags.ps1 | 98 +++++++++++++++++++ eng/common/testproxy/test-proxy-docker.yml | 2 +- eng/common/testproxy/test-proxy-tool.yml | 2 +- 5 files changed, 100 insertions(+), 2 deletions(-) rename eng/common/testproxy/{ => scripts}/override-proxy-version.ps1 (100%) create mode 100644 eng/common/testproxy/scripts/tag-merge/README.md create mode 100644 eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 diff --git a/eng/common/testproxy/override-proxy-version.ps1 b/eng/common/testproxy/scripts/override-proxy-version.ps1 similarity index 100% rename from eng/common/testproxy/override-proxy-version.ps1 rename to eng/common/testproxy/scripts/override-proxy-version.ps1 diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 new file mode 100644 index 000000000000..2ef98d6407c8 --- /dev/null +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -0,0 +1,98 @@ +<# +.SYNOPSIS +Merge multiple asset tagss worth of content into a single asset tag. + +.DESCRIPTION +USAGE: merge-proxy-tags.ps1 TAG1 TAG2 TAG3 + +Attempts to merge the contents of multiple assets tags. The first tag will be checked out, then successive tag's SHAs will be cherry-picked in one at a time. + +In the occurence of a git conflict during a cherry-pick, the process is stopped, and the location of the .assets folder will be returned to the user. + +Users should resolve the conflicts and re-run the script with the SAME ARGUMENTS. + +.PARAMETER TargetTags +The set of tags whos contents should be merged into a single new tag. + +#> +param( + [Parameter(Position=0, ValueFromRemainingArguments=$true)] + [string[]] $TargetTags +) + +function gcmd($CommandString){ + Write-Host "git $CommandString" + $result = Invoke-Expression "git $CommandString" + + if ($lastexitcode -ne 0) { + Write-Host $result + exit 1 + } + + return $result +} + +function GetTagSHA($TagName){ + $results = gcmd "ls-remote $TagName" + + if ($results -and $lastexitcode -eq 0) { + $arr = $results -split '\s+' + + return $arr[0] + } + + Write-Error "Unable to fetch tag SHA for $TagName. The tag does not exist on the repository." + exit 1 +} + +function StartMessage($AssetsRepository, $TargetTags){ + Write-Host "This script will work against " -nonewline + Write-Host $assetsRepository -ForegroundColor Green -nonewline + Write-Host " and attempt to merge tags " -nonewline + Write-Host "$($TargetTags -join ', ')" -ForegroundColor Green +} + +function FinishMessage($AssetsDirectory, $TargetTags){ + $len = $TargetTags.Length + Write-Host "Successfully combined $len tags. Please commit the result found in " -nonewline + Write-Host "$AssetsDirectory." -ForegroundColor Green +} + +function CombineTags($RemainingTags){ + foreach($Tag in $RemainingTags){ + $tagSha = GetTagSHA -TagName $Tag + + gcmd "cherry-pick $tagSha" + } +} + +$ErrorActionPreference = "Stop" + +$assetsRepository = $env:ASSETS_REPOSITORY ?? "azure/azure-sdk-assets" +$assetsDirectory = Resolve-Path -Path "assets" +$assetsUrl = "https://github.com/{0}" -f $assetsRepository + +if (!(Test-Path $assetsDirectory)){ + New-Item -ItemType Directory -Path $assetsDirectory + gcmd "clone -c core.longpaths=true --no-checkout --filter=tree:0 $assetsUrl $assetsDirectory" | Out-null +} +else { + # do nothing, we've already cloned it + # we need a way to be able to tell if we're in a good position to continue from last +} + +StartMessage -AssetsRepository $assetsRepository -RemainingTags $TargetTags + +$startTag = $TargetTags[0] +$remainingTags = $TargetTags[1..($TargetTags.Length - 1)] + +Push-Location $assetsDirectory + +gcmd "-c advice.detachedHead=false checkout $startTag" + +CombineTags -RemainingTags $remainingTags + +FinishMessage -AssetsDirectory $assetsDirectory + +Pop-Location + diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 6e749801f4de..307b5032a494 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -15,7 +15,7 @@ steps: condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', '')) inputs: targetType: filePath - filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/override-proxy-version.ps1' + filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1' arguments: '-TargetVersion "${{ parameters.targetVersion }}"' pwsh: true diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 22e4a7da4346..7aea55d472d2 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -16,7 +16,7 @@ steps: condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', '')) inputs: targetType: filePath - filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/override-proxy-version.ps1' + filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1' arguments: '-TargetVersion "${{ parameters.targetVersion }}"' pwsh: true From 6988733081c91d45f9928d6c9a8b593753466ede Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Fri, 1 Dec 2023 11:59:07 -0800 Subject: [PATCH 02/12] swapped the concept over to something that works in place. It's much much less wonky to do it this way, versus totally separating it from the proxy --- .../onboarding/generate-assets-json.ps1 | 7 +- .../testproxy/scripts/tag-merge/README.md | 8 ++ .../scripts/tag-merge/merge-proxy-tags.ps1 | 108 ++++++++++++------ 3 files changed, 86 insertions(+), 37 deletions(-) diff --git a/eng/common/testproxy/onboarding/generate-assets-json.ps1 b/eng/common/testproxy/onboarding/generate-assets-json.ps1 index 3011a19e57aa..37b9ffbbf832 100644 --- a/eng/common/testproxy/onboarding/generate-assets-json.ps1 +++ b/eng/common/testproxy/onboarding/generate-assets-json.ps1 @@ -216,8 +216,13 @@ Function Get-Repo-Language { return $lang } -Function Get-Repo-Root { +Function Get-Repo-Root($StartDir=$null) { [string] $currentDir = Get-Location + + if ($StartDir){ + $currentDir = $StartDir + } + # -1 to strip off the trialing directory separator return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1) } diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md index e69de29bb2d1..f3854918f78d 100644 --- a/eng/common/testproxy/scripts/tag-merge/README.md +++ b/eng/common/testproxy/scripts/tag-merge/README.md @@ -0,0 +1,8 @@ +# Merge Proxy Tags Script + +This script is intended to allow simpler combination of proxy tags. This is necessary due a few facts: + +- Feature teams often need to combine their efforts while adding features. This means parallel re-recording or addition of tests. +- Instead of recordings being directly alongside the feature work, able to be merged simultaneously, now recordings are a single external reference from `assets.json`. + +This script merely allows the abstraction of some of this "combination" work. diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 2ef98d6407c8..3d89fc67e0dd 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -3,35 +3,93 @@ Merge multiple asset tagss worth of content into a single asset tag. .DESCRIPTION -USAGE: merge-proxy-tags.ps1 TAG1 TAG2 TAG3 +USAGE: merge-proxy-tags.ps1 path/to/target_assets_json. TAG1 TAG2 TAG3 -Attempts to merge the contents of multiple assets tags. The first tag will be checked out, then successive tag's SHAs will be cherry-picked in one at a time. +Attempts to merge the contents of multiple assets tags into a live .assets repository. -In the occurence of a git conflict during a cherry-pick, the process is stopped, and the location of the .assets folder will be returned to the user. +In the case one of the targeted tags exists in the targeted assets.json, that tag will always be the start point. -Users should resolve the conflicts and re-run the script with the SAME ARGUMENTS. +1. test-proxy restore +2. locate recording location +3. walk the incoming tags, cherry-picking their changes directly into the changeset _in context_ +4. In the case of a discovered git conflict, the process ends. A list of which tags merged and which didn't will be presented to the user. + 4a. Users should resolve the git conflicts themselves (don't commit, there's a proxy bug to detect already commit stuff :D) + 4b. Once users have the files into a state they like. They should test-proxy push. + 4c. After pushing the resolved conflict, if there are additional tags, they should re-run this script, just excluding the tags that have been merged. + +This script requires that test-proxy or azure.sdk.tools.testproxy should be on the PATH. .PARAMETER TargetTags -The set of tags whos contents should be merged into a single new tag. +The set of tags whos contents should be merged into the targeted #> param( - [Parameter(Position=0, ValueFromRemainingArguments=$true)] + [Parameter(Position=0)] + [string] $AssetsJson + [Parameter(Position=1, ValueFromRemainingArguments=$true)] [string[]] $TargetTags ) -function gcmd($CommandString){ +. (Join-Path $PSScriptRoot ".." ".." "onboarding" "generate-assets-json.ps1") + +function gcmd($CommandString,$WorkingDirectory=$null){ Write-Host "git $CommandString" + + if ($WorkingDirectory){ + Push-Location $WorkingDirectory + } + $result = Invoke-Expression "git $CommandString" + if ($WorkingDirectory){ + Pop-Location + } + if ($lastexitcode -ne 0) { - Write-Host $result + Write-Error + Write-Error $result exit 1 } return $result } +function ResolveProxy { + # this script requires the presence of git + Test-Exe-In-Path -ExeToLookFor "git" + + $testProxyExe = "test-proxy" + # this script requires the presence of the test-proxy on the PATH + $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false + $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false + + if (-not $proxyToolPresent -and -not $proxyStandalonePresent){ + Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting." + exit 1 + } + + if (-not $proxyToolPresent) { + $testProxyExe = "Azure.Sdk.Tools.TestProxy" + } + + return $testProxyExe +} + +function Call-Proxy { + param( + [string] $TestProxyExe, + [string] $CommandArgs, + [string] $MountDirectory + ) + + $CommandArgs += " --storage-location=$MountDirectory" + Write-Host "$TestProxyExe $CommandArgs" + + [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") + + return $output +} + function GetTagSHA($TagName){ $results = gcmd "ls-remote $TagName" @@ -48,7 +106,7 @@ function GetTagSHA($TagName){ function StartMessage($AssetsRepository, $TargetTags){ Write-Host "This script will work against " -nonewline Write-Host $assetsRepository -ForegroundColor Green -nonewline - Write-Host " and attempt to merge tags " -nonewline + Write-Host " and attempt to merge tags " -noneStartDirwline Write-Host "$($TargetTags -join ', ')" -ForegroundColor Green } @@ -68,31 +126,9 @@ function CombineTags($RemainingTags){ $ErrorActionPreference = "Stop" -$assetsRepository = $env:ASSETS_REPOSITORY ?? "azure/azure-sdk-assets" -$assetsDirectory = Resolve-Path -Path "assets" -$assetsUrl = "https://github.com/{0}" -f $assetsRepository - -if (!(Test-Path $assetsDirectory)){ - New-Item -ItemType Directory -Path $assetsDirectory - gcmd "clone -c core.longpaths=true --no-checkout --filter=tree:0 $assetsUrl $assetsDirectory" | Out-null -} -else { - # do nothing, we've already cloned it - # we need a way to be able to tell if we're in a good position to continue from last -} - -StartMessage -AssetsRepository $assetsRepository -RemainingTags $TargetTags - -$startTag = $TargetTags[0] -$remainingTags = $TargetTags[1..($TargetTags.Length - 1)] - -Push-Location $assetsDirectory - -gcmd "-c advice.detachedHead=false checkout $startTag" - -CombineTags -RemainingTags $remainingTags - -FinishMessage -AssetsDirectory $assetsDirectory - -Pop-Location +$ProxyExe = ResolveProxy +$MountDirectory = Get-Repo-Root -StartDir $AssetsJson +# StartMessage -AssetsRepository $assetsRepository -RemainingTags $TargetTags +# CombineTags -RemainingTags $remainingTags +# FinishMessage -AssetsDirectory $assetsDirectory From df8cd298b3d809d61e7e9e324eb555e8b370a3be Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Fri, 1 Dec 2023 13:55:44 -0800 Subject: [PATCH 03/12] now we just need to cherry-pick and handle conflicts + resume --- .../onboarding/common-asset-functions.ps1 | 273 +++++++++++++++++ .../onboarding/generate-assets-json.ps1 | 274 +----------------- .../scripts/tag-merge/merge-proxy-tags.ps1 | 99 +++++-- 3 files changed, 352 insertions(+), 294 deletions(-) create mode 100644 eng/common/testproxy/onboarding/common-asset-functions.ps1 diff --git a/eng/common/testproxy/onboarding/common-asset-functions.ps1 b/eng/common/testproxy/onboarding/common-asset-functions.ps1 new file mode 100644 index 000000000000..28c817fd341d --- /dev/null +++ b/eng/common/testproxy/onboarding/common-asset-functions.ps1 @@ -0,0 +1,273 @@ +class Assets { + [string]$AssetsRepo = $DefaultAssetsRepo + [string]$AssetsRepoPrefixPath = "" + [string]$TagPrefix = "" + [string]$Tag = "" + Assets( + [string]$AssetsRepoPrefixPath, + [string]$TagPrefix + ) { + $this.TagPrefix = $TagPrefix + $this.AssetsRepoPrefixPath = $AssetsRepoPrefixPath + } +} + +class Version { + [int]$Year + [int]$Month + [int]$Day + [int]$Revision + Version( + [string]$VersionString + ) { + if ($VersionString -match "(?20\d{2})(?\d{2})(?\d{2}).(?\d+)") { + $this.Year = [int]$Matches["year"] + $this.Month = [int]$Matches["month"] + $this.Day = [int]$Matches["day"] + $this.Revision = [int]$Matches["revision"] + } + else { + # This should be a Write-Error however powershell apparently cannot utilize that + # in the constructor in certain cases + Write-Warning "Version String '$($VersionString)' is invalid and cannot be parsed" + exit 1 + } + } + [bool] IsGreaterEqual([string]$OtherVersionString) { + [Version]$OtherVersion = [Version]::new($OtherVersionString) + if ($this.Year -lt $OtherVersion.Year) { + return $false + } + elseif ($this.Year -eq $OtherVersion.Year) { + if ($this.Month -lt $OtherVersion.Month) { + return $false + } + elseif ($this.Month -eq $OtherVersion.Month) { + if ($this.Day -lt $OtherVersion.Day) { + return $false + } + elseif ($this.Day -eq $OtherVersion.Day) { + if ($this.Revision -lt $OtherVersion.Revision) { + return $false + } + } + } + } + return $true + } +} + +Function Test-Exe-In-Path { + Param([string] $ExeToLookFor, [bool]$ExitOnError = $true) + if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) { + if ($ExitOnError) { + Write-Error "Unable to find $ExeToLookFor in your PATH" + exit 1 + } + else { + return $false + } + } + + return $true +} + +Function Test-TestProxyVersion { + param( + [string] $TestProxyExe + ) + + Write-Host "$TestProxyExe --version" + [string] $output = & "$TestProxyExe" --version + + [Version]$CurrentProxyVersion = [Version]::new($output) + if (!$CurrentProxyVersion.IsGreaterEqual($MinTestProxyVersion)) { + Write-Error "$TestProxyExe version, $output, is less than the minimum version $MinTestProxyVersion" + Write-Error "Please refer to https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md#installation to upgrade your $TestProxyExe" + exit 1 + } +} + +Function Get-Repo-Language { + + $GitRepoOnDiskErr = "This script can only be called from within an azure-sdk-for- repository on disk." + # Git remote -v is going to give us output similar to the following + # origin git@github.com:Azure/azure-sdk-for-java.git (fetch) + # origin git@github.com:Azure/azure-sdk-for-java.git (push) + # upstream git@github.com:Azure/azure-sdk-for-java (fetch) + # upstream git@github.com:Azure/azure-sdk-for-java (push) + # We're really only trying to get the language from the git remote + Write-Host "git remote -v" + [array] $remotes = & git remote -v + foreach ($line in $remotes) { + Write-Host "$line" + } + + # Git remote -v returned "fatal: not a git repository (or any of the parent directories): .git" + # and the list of remotes will be null + if (-not $remotes) { + Write-Error $GitRepoOnDiskErr + exit 1 + } + + # The regular expression needed to be updated to handle the following types of input: + # origin git@github.com:Azure/azure-sdk-for-python.git (fetch) + # origin git@github.com:Azure/azure-sdk-for-python-pr.git (fetch) + # fork git@github.com:UserName/azure-sdk-for-python (fetch) + # azure-sdk https://github.com/azure-sdk/azure-sdk-for-net.git (fetch) + # origin https://github.com/Azure/azure-sdk-for-python/ (fetch) + # ForEach-Object splits the string on whitespace so each of the above strings is actually + # 3 different strings. The first and last pieces won't match anything, the middle string + # will match what is below. If the regular expression needs to be updated the following + # link below will go to a regex playground + # https://regex101.com/r/auOnAr/1 + $lang = $remotes[0] | ForEach-Object { if ($_ -match "azure-sdk-for-(?[^\-\.\/ ]+)") { + #Return the named language match + return $Matches["lang"] + } + } + + if ([String]::IsNullOrWhitespace($lang)) { + Write-Error $GitRepoOnDiskErr + exit 1 + } + + Write-Host "Current language=$lang" + return $lang +} + +Function Get-Repo-Root($StartDir=$null) { + [string] $currentDir = Get-Location + + if ($StartDir){ + $currentDir = $StartDir + } + + # -1 to strip off the trialing directory separator + return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1) +} + +Function New-Assets-Json-File { + param( + [Parameter(Mandatory = $true)] + [string] $Language + ) + $AssetsRepoPrefixPath = $Language + + [string] $currentDir = Get-Location + + $sdkDir = "$([IO.Path]::DirectorySeparatorChar)sdk$([IO.Path]::DirectorySeparatorChar)" + + # if we're not in a /sdk/ or deeper then this script isn't + # being run in the right place + if (-not $currentDir.contains($sdkDir)) { + Write-Error "This script needs to be run at an sdk/ or deeper." + exit 1 + } + + $TagPrefix = $currentDir.Substring($currentDir.LastIndexOf("sdk") + 4) + $TagPrefix = $TagPrefix.Replace("\", "/") + $TagPrefix = "$($AssetsRepoPrefixPath)/$($TagPrefix)" + [Assets]$Assets = [Assets]::new($AssetsRepoPrefixPath, $TagPrefix) + + $AssetsJson = $Assets | ConvertTo-Json + + $AssetsFileName = Join-Path -Path $currentDir -ChildPath "assets.json" + Write-Host "Writing file $AssetsFileName with the following contents" + Write-Host $AssetsJson + $Assets | ConvertTo-Json | Out-File $AssetsFileName + + return $AssetsFileName +} + +# Invoke the proxy command and echo the output. +Function Invoke-ProxyCommand { + param( + [string] $TestProxyExe, + [string] $CommandArgs, + [string] $TargetDirectory + ) + $updatedDirectory = $TargetDirectory.Replace("`\", "/") + + if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){ + $token = $env:GIT_TOKEN + $committer = $env:GIT_COMMIT_OWNER + $email = $env:GIT_COMMIT_EMAIL + + if (-not $committer) { + $committer = & git config --global user.name + } + + if (-not $email) { + $email = & git config --global user.email + } + + if(-not $token -or -not $committer -or -not $email){ + Write-Error ("When running this transition script in `"docker`" or `"podman`" mode, " ` + + "the environment variables GIT_TOKEN, GIT_COMMIT_OWNER, and GIT_COMMIT_EMAIL must be set to reflect the appropriate user. ") + exit 1 + } + + $targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" } + + $CommandArgs = @( + "run --rm --name transition.test.proxy", + "-v `"${updatedDirectory}:/srv/testproxy`"", + "-e `"GIT_TOKEN=${token}`"", + "-e `"GIT_COMMIT_OWNER=${committer}`"", + "-e `"GIT_COMMIT_EMAIL=${email}`"", + $targetImage, + "test-proxy", + $CommandArgs + ) -join " " + } + + Write-Host "$TestProxyExe $CommandArgs" + [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") --storage-location="$updatedDirectory" + # echo the command output + foreach ($line in $output) { + Write-Host "$line" + } +} + +# Get the shorthash directory under PROXY_ASSETS_FOLDER +Function Get-AssetsRoot { + param( + [string] $AssetsJsonFile, + [string] $TestProxyExe + ) + $repoRoot = Get-Repo-Root + $relPath = [IO.Path]::GetRelativePath($repoRoot, $AssetsJsonFile).Replace("`\", "/") + $assetsJsonDirectory = Split-Path $relPath + + [array] $output = & "$TestProxyExe" config locate -a "$relPath" --storage-location="$repoRoot" + $assetsDirectory = $output[-1] + + return Join-Path $assetsDirectory $assetsJsonDirectory +} + +Function Move-AssetsFromLangRepo { + param( + [string] $AssetsRoot + ) + $filter = $LangRecordingDirs[$language] + Write-Host "Language recording directory name=$filter" + Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }" + $filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } } + [string] $currentDir = Get-Location + + foreach ($fromFile in $filesToMove) { + $relPath = [IO.Path]::GetRelativePath($currentDir, $fromFile) + + $toFile = Join-Path -Path $AssetsRoot -ChildPath $relPath + # Write-Host "Moving from=$fromFile" + # Write-Host " to=$toFile" + $toPath = Split-Path -Path $toFile + + Write-Host $toFile + if (!(Test-Path $toPath)) { + New-Item -Path $toPath -ItemType Directory -Force | Out-Null + } + Move-Item -LiteralPath $fromFile -Destination $toFile -Force + } +} \ No newline at end of file diff --git a/eng/common/testproxy/onboarding/generate-assets-json.ps1 b/eng/common/testproxy/onboarding/generate-assets-json.ps1 index 37b9ffbbf832..7f9ed1cbc071 100644 --- a/eng/common/testproxy/onboarding/generate-assets-json.ps1 +++ b/eng/common/testproxy/onboarding/generate-assets-json.ps1 @@ -78,279 +78,7 @@ $LangRecordingDirs = @{"cpp" = "recordings"; "python" = "recordings"; }; -class Assets { - [string]$AssetsRepo = $DefaultAssetsRepo - [string]$AssetsRepoPrefixPath = "" - [string]$TagPrefix = "" - [string]$Tag = "" - Assets( - [string]$AssetsRepoPrefixPath, - [string]$TagPrefix - ) { - $this.TagPrefix = $TagPrefix - $this.AssetsRepoPrefixPath = $AssetsRepoPrefixPath - } -} - -class Version { - [int]$Year - [int]$Month - [int]$Day - [int]$Revision - Version( - [string]$VersionString - ) { - if ($VersionString -match "(?20\d{2})(?\d{2})(?\d{2}).(?\d+)") { - $this.Year = [int]$Matches["year"] - $this.Month = [int]$Matches["month"] - $this.Day = [int]$Matches["day"] - $this.Revision = [int]$Matches["revision"] - } - else { - # This should be a Write-Error however powershell apparently cannot utilize that - # in the constructor in certain cases - Write-Warning "Version String '$($VersionString)' is invalid and cannot be parsed" - exit 1 - } - } - [bool] IsGreaterEqual([string]$OtherVersionString) { - [Version]$OtherVersion = [Version]::new($OtherVersionString) - if ($this.Year -lt $OtherVersion.Year) { - return $false - } - elseif ($this.Year -eq $OtherVersion.Year) { - if ($this.Month -lt $OtherVersion.Month) { - return $false - } - elseif ($this.Month -eq $OtherVersion.Month) { - if ($this.Day -lt $OtherVersion.Day) { - return $false - } - elseif ($this.Day -eq $OtherVersion.Day) { - if ($this.Revision -lt $OtherVersion.Revision) { - return $false - } - } - } - } - return $true - } -} - -Function Test-Exe-In-Path { - Param([string] $ExeToLookFor, [bool]$ExitOnError = $true) - if ($null -eq (Get-Command $ExeToLookFor -ErrorAction SilentlyContinue)) { - if ($ExitOnError) { - Write-Error "Unable to find $ExeToLookFor in your PATH" - exit 1 - } - else { - return $false - } - } - - return $true -} - -Function Test-TestProxyVersion { - param( - [string] $TestProxyExe - ) - - Write-Host "$TestProxyExe --version" - [string] $output = & "$TestProxyExe" --version - - [Version]$CurrentProxyVersion = [Version]::new($output) - if (!$CurrentProxyVersion.IsGreaterEqual($MinTestProxyVersion)) { - Write-Error "$TestProxyExe version, $output, is less than the minimum version $MinTestProxyVersion" - Write-Error "Please refer to https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md#installation to upgrade your $TestProxyExe" - exit 1 - } -} - -Function Get-Repo-Language { - - $GitRepoOnDiskErr = "This script can only be called from within an azure-sdk-for- repository on disk." - # Git remote -v is going to give us output similar to the following - # origin git@github.com:Azure/azure-sdk-for-java.git (fetch) - # origin git@github.com:Azure/azure-sdk-for-java.git (push) - # upstream git@github.com:Azure/azure-sdk-for-java (fetch) - # upstream git@github.com:Azure/azure-sdk-for-java (push) - # We're really only trying to get the language from the git remote - Write-Host "git remote -v" - [array] $remotes = & git remote -v - foreach ($line in $remotes) { - Write-Host "$line" - } - - # Git remote -v returned "fatal: not a git repository (or any of the parent directories): .git" - # and the list of remotes will be null - if (-not $remotes) { - Write-Error $GitRepoOnDiskErr - exit 1 - } - - # The regular expression needed to be updated to handle the following types of input: - # origin git@github.com:Azure/azure-sdk-for-python.git (fetch) - # origin git@github.com:Azure/azure-sdk-for-python-pr.git (fetch) - # fork git@github.com:UserName/azure-sdk-for-python (fetch) - # azure-sdk https://github.com/azure-sdk/azure-sdk-for-net.git (fetch) - # origin https://github.com/Azure/azure-sdk-for-python/ (fetch) - # ForEach-Object splits the string on whitespace so each of the above strings is actually - # 3 different strings. The first and last pieces won't match anything, the middle string - # will match what is below. If the regular expression needs to be updated the following - # link below will go to a regex playground - # https://regex101.com/r/auOnAr/1 - $lang = $remotes[0] | ForEach-Object { if ($_ -match "azure-sdk-for-(?[^\-\.\/ ]+)") { - #Return the named language match - return $Matches["lang"] - } - } - - if ([String]::IsNullOrWhitespace($lang)) { - Write-Error $GitRepoOnDiskErr - exit 1 - } - - Write-Host "Current language=$lang" - return $lang -} - -Function Get-Repo-Root($StartDir=$null) { - [string] $currentDir = Get-Location - - if ($StartDir){ - $currentDir = $StartDir - } - - # -1 to strip off the trialing directory separator - return $currentDir.Substring(0, $currentDir.LastIndexOf("sdk") - 1) -} - -Function New-Assets-Json-File { - param( - [Parameter(Mandatory = $true)] - [string] $Language - ) - $AssetsRepoPrefixPath = $Language - - [string] $currentDir = Get-Location - - $sdkDir = "$([IO.Path]::DirectorySeparatorChar)sdk$([IO.Path]::DirectorySeparatorChar)" - - # if we're not in a /sdk/ or deeper then this script isn't - # being run in the right place - if (-not $currentDir.contains($sdkDir)) { - Write-Error "This script needs to be run at an sdk/ or deeper." - exit 1 - } - - $TagPrefix = $currentDir.Substring($currentDir.LastIndexOf("sdk") + 4) - $TagPrefix = $TagPrefix.Replace("\", "/") - $TagPrefix = "$($AssetsRepoPrefixPath)/$($TagPrefix)" - [Assets]$Assets = [Assets]::new($AssetsRepoPrefixPath, $TagPrefix) - - $AssetsJson = $Assets | ConvertTo-Json - - $AssetsFileName = Join-Path -Path $currentDir -ChildPath "assets.json" - Write-Host "Writing file $AssetsFileName with the following contents" - Write-Host $AssetsJson - $Assets | ConvertTo-Json | Out-File $AssetsFileName - - return $AssetsFileName -} - -# Invoke the proxy command and echo the output. -Function Invoke-ProxyCommand { - param( - [string] $TestProxyExe, - [string] $CommandArgs, - [string] $TargetDirectory - ) - $updatedDirectory = $TargetDirectory.Replace("`\", "/") - - if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){ - $token = $env:GIT_TOKEN - $committer = $env:GIT_COMMIT_OWNER - $email = $env:GIT_COMMIT_EMAIL - - if (-not $committer) { - $committer = & git config --global user.name - } - - if (-not $email) { - $email = & git config --global user.email - } - - if(-not $token -or -not $committer -or -not $email){ - Write-Error ("When running this transition script in `"docker`" or `"podman`" mode, " ` - + "the environment variables GIT_TOKEN, GIT_COMMIT_OWNER, and GIT_COMMIT_EMAIL must be set to reflect the appropriate user. ") - exit 1 - } - - $targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" } - - $CommandArgs = @( - "run --rm --name transition.test.proxy", - "-v `"${updatedDirectory}:/srv/testproxy`"", - "-e `"GIT_TOKEN=${token}`"", - "-e `"GIT_COMMIT_OWNER=${committer}`"", - "-e `"GIT_COMMIT_EMAIL=${email}`"", - $targetImage, - "test-proxy", - $CommandArgs - ) -join " " - } - - Write-Host "$TestProxyExe $CommandArgs" - [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") --storage-location="$updatedDirectory" - # echo the command output - foreach ($line in $output) { - Write-Host "$line" - } -} - -# Get the shorthash directory under PROXY_ASSETS_FOLDER -Function Get-AssetsRoot { - param( - [string] $AssetsJsonFile, - [string] $TestProxyExe - ) - $repoRoot = Get-Repo-Root - $relPath = [IO.Path]::GetRelativePath($repoRoot, $AssetsJsonFile).Replace("`\", "/") - $assetsJsonDirectory = Split-Path $relPath - - [array] $output = & "$TestProxyExe" config locate -a "$relPath" --storage-location="$repoRoot" - $assetsDirectory = $output[-1] - - return Join-Path $assetsDirectory $assetsJsonDirectory -} - -Function Move-AssetsFromLangRepo { - param( - [string] $AssetsRoot - ) - $filter = $LangRecordingDirs[$language] - Write-Host "Language recording directory name=$filter" - Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }" - $filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } } - [string] $currentDir = Get-Location - - foreach ($fromFile in $filesToMove) { - $relPath = [IO.Path]::GetRelativePath($currentDir, $fromFile) - - $toFile = Join-Path -Path $AssetsRoot -ChildPath $relPath - # Write-Host "Moving from=$fromFile" - # Write-Host " to=$toFile" - $toPath = Split-Path -Path $toFile - - Write-Host $toFile - if (!(Test-Path $toPath)) { - New-Item -Path $toPath -ItemType Directory -Force | Out-Null - } - Move-Item -LiteralPath $fromFile -Destination $toFile -Force - } -} +. (Join-Path $PSScriptRoot "common-asset-functions.ps1") Test-Exe-In-Path -ExeToLookFor $GitExe $language = Get-Repo-Language diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 3d89fc67e0dd..9833779b5fce 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -25,14 +25,14 @@ The set of tags whos contents should be merged into the targeted #> param( [Parameter(Position=0)] - [string] $AssetsJson + [string] $AssetsJson, [Parameter(Position=1, ValueFromRemainingArguments=$true)] [string[]] $TargetTags ) -. (Join-Path $PSScriptRoot ".." ".." "onboarding" "generate-assets-json.ps1") +. (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1") -function gcmd($CommandString,$WorkingDirectory=$null){ +function Git-Command($CommandString,$WorkingDirectory=$null){ Write-Host "git $CommandString" if ($WorkingDirectory){ @@ -54,9 +54,9 @@ function gcmd($CommandString,$WorkingDirectory=$null){ return $result } -function ResolveProxy { +function Resolve-Proxy { # this script requires the presence of git - Test-Exe-In-Path -ExeToLookFor "git" + Test-Exe-In-Path -ExeToLookFor "git" | Out-Null $testProxyExe = "test-proxy" # this script requires the presence of the test-proxy on the PATH @@ -79,19 +79,37 @@ function Call-Proxy { param( [string] $TestProxyExe, [string] $CommandArgs, - [string] $MountDirectory + [string] $MountDirectory, + [boolean] $Output = $true ) $CommandArgs += " --storage-location=$MountDirectory" - Write-Host "$TestProxyExe $CommandArgs" + + if ($Output -eq $true){ + Write-Host "$TestProxyExe $CommandArgs" + } [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") - + + if ($Output -eq $true){ + foreach($line in $output) { + Write-Host $line + } + } + return $output } +function Locate-Assets-Slice($ProxyExe, $AssetsJson, $MountDirectory) { + $CommandString = "config locate -a $AssetsJson" + + $output = Call-Proxy -TestProxyExe $ProxyExe -CommandArgs $CommandString -MountDirectory $MountDirectory -Output $false + + return $output[-1].Trim() +} + function GetTagSHA($TagName){ - $results = gcmd "ls-remote $TagName" + $results = Git-Command "ls-remote $TagName" if ($results -and $lastexitcode -eq 0) { $arr = $results -split '\s+' @@ -103,32 +121,71 @@ function GetTagSHA($TagName){ exit 1 } -function StartMessage($AssetsRepository, $TargetTags){ - Write-Host "This script will work against " -nonewline - Write-Host $assetsRepository -ForegroundColor Green -nonewline - Write-Host " and attempt to merge tags " -noneStartDirwline - Write-Host "$($TargetTags -join ', ')" -ForegroundColor Green +function StartMessage($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory){ + Write-Host "`nThis script will attempt to merge the following tag" -nonewline + if ($TargetTags.Length -gt 1){ + Write-Host "s" -nonewline + } + Write-Host ":" + foreach($Tag in $TargetTags){ + Write-Host " - " -nonewline + Write-Host "$Tag" -ForegroundColor Green + } + Write-Host "`nTargeting the assets slice targeted by " -nonewline + Write-Host "$AssetsJson." -ForegroundColor Green + Write-Host "`nThe work will be completed in " -nonewline + Write-Host $AssetsRepoLocation -ForegroundColor Green -nonewline + Write-Host "." } -function FinishMessage($AssetsDirectory, $TargetTags){ +function FinishMessage($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory){ $len = $TargetTags.Length - Write-Host "Successfully combined $len tags. Please commit the result found in " -nonewline - Write-Host "$AssetsDirectory." -ForegroundColor Green + Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -nonewline + Write-Host $AssetsJson -ForegroundColor Green -nonewline + Write-Host "`" to push the results as a new tag." } function CombineTags($RemainingTags){ foreach($Tag in $RemainingTags){ $tagSha = GetTagSHA -TagName $Tag - gcmd "cherry-pick $tagSha" + Git-Command "cherry-pick $tagSha" + } +} + +function Resolve-Target-Tags($AssetsJson, $TargetTags) { + $jsonContent = Get-Content -Raw -Path $AssetsJson + $jsonObj = $JsonContent | ConvertFrom-Json + + $existingTarget = $jsonObj.Tag + + return $TargetTags | Where-Object { + if ($_ -eq $existingTarget) { + Write-Host "Excluding tag $($_) due from tag list, it is present in assets.json." + } + $_ -ne $existingTarget } } $ErrorActionPreference = "Stop" -$ProxyExe = ResolveProxy +# resolve the proxy location so that we can invoke it easily +$ProxyExe = Resolve-Proxy + +# figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting +# directory so that the test-proxy restore operations work IN PLACE with existing tooling $MountDirectory = Get-Repo-Root -StartDir $AssetsJson -# StartMessage -AssetsRepository $assetsRepository -RemainingTags $TargetTags +# using the MountingDirectory and the assets.json location, we can figure out where the assets slice actually lives within the .assets folder. +# we will use this to invoke individual cherry-picks before pushing up the result +$AssetsRepoLocation = Locate-Assets-Slice $ProxyExe $AssetsJson $MountDirectory + +# resolve the tags that we will go after. If the target assets.json contains one of these tags, it will be run _first_ in the first restore. +# This script is intended to run in context of an existing repo, so this is just the way it's gotta be for consistency. +$Tags = Resolve-Target-Tags $AssetsJson $TargetTags + +StartMessage $AssetsJson $Tags $AssetsRepoLocation $MountDirectory + # CombineTags -RemainingTags $remainingTags -# FinishMessage -AssetsDirectory $assetsDirectory + +FinishMessage $AssetsJson $Tags $AssetsRepoLocation $MountDirectory From 092d2d89d9077c7de3ce1f08a917ab198f260fba Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Fri, 1 Dec 2023 18:31:12 -0800 Subject: [PATCH 04/12] only thing remaining is testing graceful conflict resolution --- .../testproxy/scripts/tag-merge/README.md | 34 +++++ .../scripts/tag-merge/merge-proxy-tags.ps1 | 123 ++++++++++++++---- 2 files changed, 132 insertions(+), 25 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md index f3854918f78d..68496ff6e83b 100644 --- a/eng/common/testproxy/scripts/tag-merge/README.md +++ b/eng/common/testproxy/scripts/tag-merge/README.md @@ -6,3 +6,37 @@ This script is intended to allow simpler combination of proxy tags. This is nece - Instead of recordings being directly alongside the feature work, able to be merged simultaneously, now recordings are a single external reference from `assets.json`. This script merely allows the abstraction of some of this "combination" work. + +## Usage + +### PreReqs + +- Must have `git` available on your PATH +- Must have the `test-proxy` available on your PATH + - `test-proxy` is honored when the proxy is installed as a `dotnet tool` + - `Azure.Sdk.Tools.TestProxy` is honored when the standalone executable is on your PATH + - Preference for `dotnet tool` if present + +### Call the script + +```powershell +cd "path/to/language/repo/root" +./eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 sdk/storage/azure-storage-blob/assets.json integration/example/storage_feature_addition2 integration/example/storage_feature_addition1 +# ^ Combined Tag 1 ^ Combined Tag 2 +test-proxy push -a sdk/storage/azure-storage-blob/assets.json +``` + +### Resolve Conflicts + +If the script ends early to a `git conflict` occurring, the script leaves the asset repo in a resolvable state. + +- `cd` to the working directory described in the output from the script before it starts working. +- `git status`, then resolve the conflicts in the files. you don't need to commit the result +- Invoke the script **with the original arguments** + - If the script stopped to resolve a conflict, it drops a tiny file (`.mergeprogress`) at repo root to remember where it stopped progressing. It will utilize this file to avoid cherry-picking already completed commits. + +### Push the result + +Once the script has completed successfully, `test-proxy push` the results! + +C:\repo\azure-sdk-tools\eng\common\testproxy\scripts\tag-merge\merge-proxy-tags.ps1 "C:/repo/azure-sdk-for-python/sdk/storage/azure-storage-blob/assets.json" integration/example/storage_feature_addition2 integration/example/storage_feature_addition1 \ No newline at end of file diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 9833779b5fce..68add463ced9 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -32,7 +32,7 @@ param( . (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1") -function Git-Command($CommandString,$WorkingDirectory=$null){ +function Git-Command($CommandString,$WorkingDirectory=$null) { Write-Host "git $CommandString" if ($WorkingDirectory){ @@ -46,7 +46,6 @@ function Git-Command($CommandString,$WorkingDirectory=$null){ } if ($lastexitcode -ne 0) { - Write-Error Write-Error $result exit 1 } @@ -91,6 +90,14 @@ function Call-Proxy { [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") + if ($lastexitcode -ne 0){ + foreach($line in $output) { + Write-Host $line + } + Write-Error "Proxy exe exited with unexpected non-zero exit code." + exit 1 + } + if ($Output -eq $true){ foreach($line in $output) { Write-Host $line @@ -108,20 +115,33 @@ function Locate-Assets-Slice($ProxyExe, $AssetsJson, $MountDirectory) { return $output[-1].Trim() } -function GetTagSHA($TagName){ - $results = Git-Command "ls-remote $TagName" +function Get-Tag-SHA($TagName, $WorkingDirectory) { + $results = Git-Command "ls-remote origin $TagName" $WorkingDirectory if ($results -and $lastexitcode -eq 0) { $arr = $results -split '\s+' return $arr[0] } - + Write-Error "Unable to fetch tag SHA for $TagName. The tag does not exist on the repository." exit 1 } -function StartMessage($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory){ +function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { + $alreadyCombinedTags = Load-Incomplete-Progress $MountDirectory + + if ($alreadyCombinedTags) { + Write-Host "This script has detected the presence of a .mergeprogress file with folder $MountDirectory." + Write-Host "Attempting to continue from a previous run, and excluding: " + foreach($Tag in $alreadyCombinedTags){ + Write-Host " - " -nonewline + Write-Host "$Tag" -ForegroundColor Green + } + } + + $TargetTags = $TargetTags | Where-Object { $_ -notin $alreadyCombinedTags } + Write-Host "`nThis script will attempt to merge the following tag" -nonewline if ($TargetTags.Length -gt 1){ Write-Host "s" -nonewline @@ -136,23 +156,17 @@ function StartMessage($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirec Write-Host "`nThe work will be completed in " -nonewline Write-Host $AssetsRepoLocation -ForegroundColor Green -nonewline Write-Host "." + + Read-Host -Prompt "If the above looks correct, press enter, otherwise, ctrl-c" } -function FinishMessage($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory){ +function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { $len = $TargetTags.Length Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -nonewline Write-Host $AssetsJson -ForegroundColor Green -nonewline Write-Host "`" to push the results as a new tag." } -function CombineTags($RemainingTags){ - foreach($Tag in $RemainingTags){ - $tagSha = GetTagSHA -TagName $Tag - - Git-Command "cherry-pick $tagSha" - } -} - function Resolve-Target-Tags($AssetsJson, $TargetTags) { $jsonContent = Get-Content -Raw -Path $AssetsJson $jsonObj = $JsonContent | ConvertFrom-Json @@ -167,25 +181,84 @@ function Resolve-Target-Tags($AssetsJson, $TargetTags) { } } +function Save-Incomplete-Progress($Tag, $MountDirectory) { + $progressFile = (Join-Path $MountDirectory ".mergeprogress") + [array] $existingTags = @() + if (Test-Path $progressFile) { + $existingTags = (Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() } + } + + $existingTags = $existingTags + $Tag | Select-Object -Unique + + Set-Content -Path $progressFile -Value ($existingTags -join "`n") | Out-Null + + return $existingTags +} + +function Load-Incomplete-Progress($MountDirectory) { + $progressFile = (Join-Path $MountDirectory ".mergeprogress") + [array] $existingTags = @() + if (Test-Path $progressFile) { + $existingTags = (Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() } + } + + return $existingTags +} + +function Cleanup-Incomplete-Progress($MountDirectory) { + $progressFile = (Join-Path $MountDirectory ".mergeprogress") + + if (Test-Path $progressFile) { + Remove-Item $progressFile | Out-Null + } +} + +function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { + $inprogress = Load-Incomplete-Progress $MountDirectory + + if ($inprogress.Length -eq 0) { + Call-Proxy -TestProxyExe $ProxyExe -CommandArgs "reset -y -a $AssetsJson" -MountDirectory $MountDirectory -Output $false + } +} + +function CombineTags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ + foreach($Tag in $RemainingTags){ + $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation + + $cherryPickOut = Git-Command "cherry-pick $tagSha" $AssetsRepoLocation + + Save-Incomplete-Progress $Tag $MountDirectory + } + + # todo: if we managed to get here without any errors, we need to touch a file that all the cherry-picked commits will push up so that test-proxy push + # properly sees the additional commits + + # if we have successfully gotten to the end without any non-zero exit codes...delete the mergeprogress file, we're g2g + Cleanup-Incomplete-Progress $MountDirectory +} + $ErrorActionPreference = "Stop" # resolve the proxy location so that we can invoke it easily -$ProxyExe = Resolve-Proxy +$proxyExe = Resolve-Proxy # figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting # directory so that the test-proxy restore operations work IN PLACE with existing tooling -$MountDirectory = Get-Repo-Root -StartDir $AssetsJson +$mountDirectory = Get-Repo-Root -StartDir $AssetsJson + +# ensure we actually have the .assets folder that we can cherry-pick on top of +Prepare-Assets $proxyExe $mountDirectory $AssetsJson -# using the MountingDirectory and the assets.json location, we can figure out where the assets slice actually lives within the .assets folder. +# using the mountingDirectory and the assets.json location, we can figure out where the assets slice actually lives within the .assets folder. # we will use this to invoke individual cherry-picks before pushing up the result -$AssetsRepoLocation = Locate-Assets-Slice $ProxyExe $AssetsJson $MountDirectory +$assetsRepoLocation = Locate-Assets-Slice $proxyExe $AssetsJson $mountDirectory -# resolve the tags that we will go after. If the target assets.json contains one of these tags, it will be run _first_ in the first restore. -# This script is intended to run in context of an existing repo, so this is just the way it's gotta be for consistency. -$Tags = Resolve-Target-Tags $AssetsJson $TargetTags +# resolve the tags that we will go after. If the target assets.json contains one of these tags, that tag is _already present_ +# because the entire point of this script is to run in context of a set of recordings in the repo +$tags = Resolve-Target-Tags $AssetsJson $TargetTags -StartMessage $AssetsJson $Tags $AssetsRepoLocation $MountDirectory +Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory -# CombineTags -RemainingTags $remainingTags +$CombinedTags = CombineTags $Tags $AssetsRepoLocation $mountDirectory -FinishMessage $AssetsJson $Tags $AssetsRepoLocation $MountDirectory +Finish-Message $AssetsJson $CombinedTags $AssetsRepoLocation $mountDirectory From 67294c70936a8e4a9a8238d2f513f19bf5dda7df Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 4 Dec 2023 18:31:02 -0800 Subject: [PATCH 05/12] allow resumption of cherry-picking after fixing a conflict! --- .../scripts/tag-merge/merge-proxy-tags.ps1 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 68add463ced9..2b99727e8be5 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -167,7 +167,9 @@ function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDir Write-Host "`" to push the results as a new tag." } -function Resolve-Target-Tags($AssetsJson, $TargetTags) { +function Resolve-Target-Tags($AssetsJson, $TargetTags, $MountDirectory) { + $inprogress = Load-Incomplete-Progress $MountDirectory + $jsonContent = Get-Content -Raw -Path $AssetsJson $jsonObj = $JsonContent | ConvertFrom-Json @@ -175,9 +177,14 @@ function Resolve-Target-Tags($AssetsJson, $TargetTags) { return $TargetTags | Where-Object { if ($_ -eq $existingTarget) { - Write-Host "Excluding tag $($_) due from tag list, it is present in assets.json." + Write-Host "Excluding tag $($_) from tag input list, it is present in assets.json." } $_ -ne $existingTarget + } | Where-Object { + if ($_ -in $inprogress) { + Write-Host "Excluding tag $($_) because we have already done work against it in a previous script invocation." + } + $_ -notin $inprogress } } @@ -224,10 +231,8 @@ function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { function CombineTags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ foreach($Tag in $RemainingTags){ $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation - - $cherryPickOut = Git-Command "cherry-pick $tagSha" $AssetsRepoLocation - Save-Incomplete-Progress $Tag $MountDirectory + $cherryPickOut = Git-Command "cherry-pick $tagSha" $AssetsRepoLocation } # todo: if we managed to get here without any errors, we need to touch a file that all the cherry-picked commits will push up so that test-proxy push @@ -255,7 +260,7 @@ $assetsRepoLocation = Locate-Assets-Slice $proxyExe $AssetsJson $mountDirectory # resolve the tags that we will go after. If the target assets.json contains one of these tags, that tag is _already present_ # because the entire point of this script is to run in context of a set of recordings in the repo -$tags = Resolve-Target-Tags $AssetsJson $TargetTags +$tags = Resolve-Target-Tags $AssetsJson $TargetTags $mountDirectory Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory From 8dff574f691b9ae8dbed839df240515b0b72a9bd Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 4 Dec 2023 19:07:10 -0800 Subject: [PATCH 06/12] handle resuming after pause for reconciliation --- .../scripts/tag-merge/merge-proxy-tags.ps1 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 2b99727e8be5..d47b2d07b94a 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -162,6 +162,7 @@ function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDire function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { $len = $TargetTags.Length + Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -nonewline Write-Host $AssetsJson -ForegroundColor Green -nonewline Write-Host "`" to push the results as a new tag." @@ -228,15 +229,15 @@ function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { } } -function CombineTags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ +function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ foreach($Tag in $RemainingTags){ $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation Save-Incomplete-Progress $Tag $MountDirectory $cherryPickOut = Git-Command "cherry-pick $tagSha" $AssetsRepoLocation } - - # todo: if we managed to get here without any errors, we need to touch a file that all the cherry-picked commits will push up so that test-proxy push - # properly sees the additional commits + + $testFile = Get-ChildItem -Recurse -Path $AssetsRepoLocation | Where-Object { !$_.PSIsContainer } | Select-Object -First 1 + Add-Content -Path $testFile -Value "`n" # if we have successfully gotten to the end without any non-zero exit codes...delete the mergeprogress file, we're g2g Cleanup-Incomplete-Progress $MountDirectory @@ -247,6 +248,8 @@ $ErrorActionPreference = "Stop" # resolve the proxy location so that we can invoke it easily $proxyExe = Resolve-Proxy +$AssetsJson = Resolve-Path $AssetsJson + # figure out where the root of the repo for the passed assets.json is. We need it to properly set the mounting # directory so that the test-proxy restore operations work IN PLACE with existing tooling $mountDirectory = Get-Repo-Root -StartDir $AssetsJson @@ -264,6 +267,6 @@ $tags = Resolve-Target-Tags $AssetsJson $TargetTags $mountDirectory Start-Message $AssetsJson $Tags $AssetsRepoLocation $mountDirectory -$CombinedTags = CombineTags $Tags $AssetsRepoLocation $mountDirectory +$CombinedTags = Combine-Tags $Tags $AssetsRepoLocation $mountDirectory Finish-Message $AssetsJson $CombinedTags $AssetsRepoLocation $mountDirectory From 92cc742bd96443cffb7b2c78b508b2cec49e2c04 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 5 Dec 2023 17:39:08 -0800 Subject: [PATCH 07/12] conflict resolution. adding some messaging so that users can actually SEE the failure --- .../scripts/tag-merge/merge-proxy-tags.ps1 | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index d47b2d07b94a..6bab864c7b61 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -5,20 +5,23 @@ Merge multiple asset tagss worth of content into a single asset tag. .DESCRIPTION USAGE: merge-proxy-tags.ps1 path/to/target_assets_json. TAG1 TAG2 TAG3 -Attempts to merge the contents of multiple assets tags into a live .assets repository. +Attempts to merge the contents of multiple assets tags into a single new local changeset, which can be `test-proxy push`-ed. In the case one of the targeted tags exists in the targeted assets.json, that tag will always be the start point. -1. test-proxy restore -2. locate recording location +1. test-proxy restore -a -> populate .assets +2. test-proxy config locate -a -> get location of cloned git repo 3. walk the incoming tags, cherry-picking their changes directly into the changeset _in context_ 4. In the case of a discovered git conflict, the process ends. A list of which tags merged and which didn't will be presented to the user. - 4a. Users should resolve the git conflicts themselves (don't commit, there's a proxy bug to detect already commit stuff :D) - 4b. Once users have the files into a state they like. They should test-proxy push. - 4c. After pushing the resolved conflict, if there are additional tags, they should re-run this script, just excluding the tags that have been merged. + 4a. Users should resolve the git conflicts themselves. + 4b. If the conflict was on the final tag, resolve the conflict (leaving it uncommitted tyvm), and test-proxy push, you're done. + 4c. If the conflict was _not_ on the final tag, resolve the conflict, commit it, and then re-run this script with the SAME arguments as before. This script requires that test-proxy or azure.sdk.tools.testproxy should be on the PATH. +.PARAMETER AssetsJson +The script uses a target assets.json to resolve where specifically on disk the tag merging should take place. + .PARAMETER TargetTags The set of tags whos contents should be merged into the targeted @@ -32,7 +35,7 @@ param( . (Join-Path $PSScriptRoot ".." ".." "onboarding" "common-asset-functions.ps1") -function Git-Command($CommandString,$WorkingDirectory=$null) { +function Git-Command-With-Result($CommandString, $WorkingDirectory) { Write-Host "git $CommandString" if ($WorkingDirectory){ @@ -45,12 +48,21 @@ function Git-Command($CommandString,$WorkingDirectory=$null) { Pop-Location } - if ($lastexitcode -ne 0) { + return [PSCustomObject]@{ + ExitCode = $lastexitcode, + Output = $result + } +} + +function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) { + $result = Git-Command-With-Result $CommandString $WorkingDirectory + + if ($lastexitcode -ne 0 -and $HardExit) { Write-Error $result exit 1 } - return $result + return $result.Output } function Resolve-Proxy { @@ -233,7 +245,21 @@ function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ foreach($Tag in $RemainingTags){ $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation Save-Incomplete-Progress $Tag $MountDirectory - $cherryPickOut = Git-Command "cherry-pick $tagSha" $AssetsRepoLocation + $cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false + + if ($cherryPickResult.ExitCode -ne 0) { + Write-Error $cherryPickResult.Output + # not last tag + if ($Tag -ne $RemainingTags[-1]) { + Write-Error "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", commit the result, and re-run this script with the same arguments as before." + exit 1 + } + # last tag + elseif ($Tag -eq $RemainingTags[-1]) { + Write-Error "Conflicts while cherry-picking $Tag. Resolve the conflict over in `"$AssetsRepoLocation`", leave the result uncommitted, ``test-proxy push`` the assets.json you ran this script against!" + exit 1 + } + } } $testFile = Get-ChildItem -Recurse -Path $AssetsRepoLocation | Where-Object { !$_.PSIsContainer } | Select-Object -First 1 From f1ff7d137d75bdba29e02420fadf72b5fe1879fb Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 5 Dec 2023 17:56:06 -0800 Subject: [PATCH 08/12] documentation update. now handling the conflict resolution a bit more smartly --- .../testproxy/scripts/tag-merge/README.md | 38 ++++++++++++++++--- .../scripts/tag-merge/merge-proxy-tags.ps1 | 14 ++----- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md index 68496ff6e83b..d66b19b18968 100644 --- a/eng/common/testproxy/scripts/tag-merge/README.md +++ b/eng/common/testproxy/scripts/tag-merge/README.md @@ -30,13 +30,41 @@ test-proxy push -a sdk/storage/azure-storage-blob/assets.json If the script ends early to a `git conflict` occurring, the script leaves the asset repo in a resolvable state. -- `cd` to the working directory described in the output from the script before it starts working. -- `git status`, then resolve the conflicts in the files. you don't need to commit the result -- Invoke the script **with the original arguments** - - If the script stopped to resolve a conflict, it drops a tiny file (`.mergeprogress`) at repo root to remember where it stopped progressing. It will utilize this file to avoid cherry-picking already completed commits. +- `cd` to the working directory described in the output from the script before it starts working. ("The work will be complete in...") +- `git status` to identify which files are conflicted + +You will see something along these lines: + +```bash +C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python |>git status +HEAD detached from python/storage/azure-storage-blob_12c8154ae2 +You are currently cherry-picking commit 1fd0865. + (fix conflicts and run "git cherry-pick --continue") + (use "git cherry-pick --skip" to skip this patch) + (use "git cherry-pick --abort" to cancel the cherry-pick operation) + +You are in a sparse checkout with 100% of tracked files present. + +Unmerged paths: + (use "git add ..." to mark resolution) + both added: sdk/storage/azure-storage-blob/tests/recordings/test_append_blob_async.pyTestStorageAppendBlobAsynctest_append_blob_from_text_new.json + +no changes added to commit (use "git add" and/or "git commit -a") +``` + +Resolve the conflicts in the file, then add it using `git add `. Once the conflict is fully resolved, use + +```bash +C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python [???]|>git cherry-pick --continue +[detached HEAD 236e234] add the same file names as what was present in tag integration/example/storage_feature_addition2. In this case, the files themselves are just different enough from integration/example/storage_feature_addition2 that we should intentionally cause a conflict + Date: Fri Dec 1 16:57:52 2023 -0800 + 1 file changed, 2 insertions(+), 2 deletions(-) +``` ### Push the result Once the script has completed successfully, `test-proxy push` the results! -C:\repo\azure-sdk-tools\eng\common\testproxy\scripts\tag-merge\merge-proxy-tags.ps1 "C:/repo/azure-sdk-for-python/sdk/storage/azure-storage-blob/assets.json" integration/example/storage_feature_addition2 integration/example/storage_feature_addition1 \ No newline at end of file +```bash +test-proxy push sdk/storage/azure-storage-blob/assets.json +``` diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 6bab864c7b61..bdbdf68a822f 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -49,7 +49,7 @@ function Git-Command-With-Result($CommandString, $WorkingDirectory) { } return [PSCustomObject]@{ - ExitCode = $lastexitcode, + ExitCode = $lastexitcode Output = $result } } @@ -57,8 +57,8 @@ function Git-Command-With-Result($CommandString, $WorkingDirectory) { function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) { $result = Git-Command-With-Result $CommandString $WorkingDirectory - if ($lastexitcode -ne 0 -and $HardExit) { - Write-Error $result + if ($result.ExitCode -ne 0 -and $HardExit) { + Write-Error $result.Output exit 1 } @@ -249,14 +249,8 @@ function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ if ($cherryPickResult.ExitCode -ne 0) { Write-Error $cherryPickResult.Output - # not last tag if ($Tag -ne $RemainingTags[-1]) { - Write-Error "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", commit the result, and re-run this script with the same arguments as before." - exit 1 - } - # last tag - elseif ($Tag -eq $RemainingTags[-1]) { - Write-Error "Conflicts while cherry-picking $Tag. Resolve the conflict over in `"$AssetsRepoLocation`", leave the result uncommitted, ``test-proxy push`` the assets.json you ran this script against!" + Write-Error "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." exit 1 } } From a7b3d78abc0565b6acb4b4b436d197fb6f6ed796 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 5 Dec 2023 18:57:01 -0800 Subject: [PATCH 09/12] lots more detail in docs. handling the conflict modes --- .../testproxy/scripts/tag-merge/README.md | 24 +++++++ .../scripts/tag-merge/merge-proxy-tags.ps1 | 64 +++++++++++-------- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/README.md b/eng/common/testproxy/scripts/tag-merge/README.md index d66b19b18968..aa524282e8a7 100644 --- a/eng/common/testproxy/scripts/tag-merge/README.md +++ b/eng/common/testproxy/scripts/tag-merge/README.md @@ -61,6 +61,30 @@ C:/repo/azure-sdk-for-python/.assets/eDscgL1p9G/python [???]|>git cherry-pick -- 1 file changed, 2 insertions(+), 2 deletions(-) ``` +Once you've resolved the conflict, re-run the same script. The results of the cherry-pick resolution will be visible. + +```bash +C:/repo/azure-sdk-for-python [test-storage-tag-combination]|>eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 sdk/storage/azure-storage-blob/assets.json integration/example/storage_feature_addition2 integration/example/storage_feature_addition2_conflict integration/example/storage_feature_addition1 +Excluding tag integration/example/storage_feature_addition2 because we have already done work against it in a previous script invocation. +Excluding tag integration/example/storage_feature_addition2_conflict because we have already done work against it in a previous script invocation. +This script has detected the presence of a .mergeprogress file within folder C:\repo\azure-sdk-for-python. +If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again. +Attempting to continue from a previous run, and excluding: + - integration/example/storage_feature_addition2 + - integration/example/storage_feature_addition2_conflict +But continuing with: + - integration/example/storage_feature_addition1 +If the above looks correct, press enter, otherwise, ctrl-c: +``` + +On successful result, the following will be present: + +``` +Successfully combined 3 tags. Invoke "test-proxy push C:\repo\azure-sdk-for-python\sdk\storage\azure-storage-blob\assets.json" to push the results as a new tag. +``` + +Just follow the instructions to push your combined tag! + ### Push the result Once the script has completed successfully, `test-proxy push` the results! diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index bdbdf68a822f..2fae6b88d692 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -142,32 +142,41 @@ function Get-Tag-SHA($TagName, $WorkingDirectory) { function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { $alreadyCombinedTags = Load-Incomplete-Progress $MountDirectory + + $TargetTags = $TargetTags | Where-Object { $_ -notin $alreadyCombinedTags } if ($alreadyCombinedTags) { - Write-Host "This script has detected the presence of a .mergeprogress file with folder $MountDirectory." - Write-Host "Attempting to continue from a previous run, and excluding: " + Write-Host "This script has detected the presence of a .mergeprogress file within folder $MountDirectory." + Write-Host "If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again." + Write-Host "Attempting to continue from a previous run, and excluding:" + foreach($Tag in $alreadyCombinedTags){ - Write-Host " - " -nonewline + Write-Host " - " -NoNewLine Write-Host "$Tag" -ForegroundColor Green } - } + Write-Host "But continuing with:" - $TargetTags = $TargetTags | Where-Object { $_ -notin $alreadyCombinedTags } - - Write-Host "`nThis script will attempt to merge the following tag" -nonewline - if ($TargetTags.Length -gt 1){ - Write-Host "s" -nonewline + foreach($Tag in $TargetTags){ + Write-Host " - " -NoNewLine + Write-Host "$Tag" -ForegroundColor Green + } } - Write-Host ":" - foreach($Tag in $TargetTags){ - Write-Host " - " -nonewline - Write-Host "$Tag" -ForegroundColor Green + else { + Write-Host "`nThis script will attempt to merge the following tag" -NoNewLine + if ($TargetTags.Length -gt 1){ + Write-Host "s" -NoNewLine + } + Write-Host ":" + foreach($Tag in $TargetTags){ + Write-Host " - " -NoNewLine + Write-Host "$Tag" -ForegroundColor Green + } + Write-Host "`nTargeting the assets slice targeted by " -NoNewLine + Write-Host "$AssetsJson." -ForegroundColor Green + Write-Host "`nThe work will be completed in " -NoNewLine + Write-Host $AssetsRepoLocation -ForegroundColor Green -NoNewLine + Write-Host "." } - Write-Host "`nTargeting the assets slice targeted by " -nonewline - Write-Host "$AssetsJson." -ForegroundColor Green - Write-Host "`nThe work will be completed in " -nonewline - Write-Host $AssetsRepoLocation -ForegroundColor Green -nonewline - Write-Host "." Read-Host -Prompt "If the above looks correct, press enter, otherwise, ctrl-c" } @@ -175,8 +184,8 @@ function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDire function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) { $len = $TargetTags.Length - Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -nonewline - Write-Host $AssetsJson -ForegroundColor Green -nonewline + Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -NoNewLine + Write-Host $AssetsJson -ForegroundColor Green -NoNewLine Write-Host "`" to push the results as a new tag." } @@ -219,7 +228,7 @@ function Load-Incomplete-Progress($MountDirectory) { $progressFile = (Join-Path $MountDirectory ".mergeprogress") [array] $existingTags = @() if (Test-Path $progressFile) { - $existingTags = (Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() } + $existingTags = ((Get-Content -Path $progressFile) -split "`n" | ForEach-Object { $_.Trim() }) } return $existingTags @@ -244,23 +253,24 @@ function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ foreach($Tag in $RemainingTags){ $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation - Save-Incomplete-Progress $Tag $MountDirectory + $existingTags = Save-Incomplete-Progress $Tag $MountDirectory $cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false if ($cherryPickResult.ExitCode -ne 0) { - Write-Error $cherryPickResult.Output - if ($Tag -ne $RemainingTags[-1]) { - Write-Error "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." - exit 1 - } + Write-Host "Conflicts while cherry-picking $Tag. Resolve the the conflict over in `"$AssetsRepoLocation`", and re-run this script with the same arguments as before." -ForegroundColor Red + exit 1 } } + $pushedTags = Load-Incomplete-Progress $MountDirectory + $testFile = Get-ChildItem -Recurse -Path $AssetsRepoLocation | Where-Object { !$_.PSIsContainer } | Select-Object -First 1 Add-Content -Path $testFile -Value "`n" # if we have successfully gotten to the end without any non-zero exit codes...delete the mergeprogress file, we're g2g Cleanup-Incomplete-Progress $MountDirectory + + return $pushedTags } $ErrorActionPreference = "Stop" From 9b926c89ed99ba28bcba96fe1d48b60bed654f0d Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 6 Dec 2023 11:32:46 -0800 Subject: [PATCH 10/12] update spacing and comment text --- .../scripts/tag-merge/merge-proxy-tags.ps1 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 index 2fae6b88d692..866a5f787de4 100644 --- a/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 +++ b/eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 @@ -23,7 +23,7 @@ This script requires that test-proxy or azure.sdk.tools.testproxy should be on t The script uses a target assets.json to resolve where specifically on disk the tag merging should take place. .PARAMETER TargetTags -The set of tags whos contents should be merged into the targeted +The set of tags whose contents should be combined. Any number of tags > 1 is allowed. #> param( @@ -44,7 +44,7 @@ function Git-Command-With-Result($CommandString, $WorkingDirectory) { $result = Invoke-Expression "git $CommandString" - if ($WorkingDirectory){ + if ($WorkingDirectory) { Pop-Location } @@ -74,7 +74,7 @@ function Resolve-Proxy { $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false $proxyStandalonePresent = Test-Exe-In-Path -ExeToLookFor "Azure.Sdk.Tools.TestProxy" -ExitOnError $false - if (-not $proxyToolPresent -and -not $proxyStandalonePresent){ + if (-not $proxyToolPresent -and -not $proxyStandalonePresent) { Write-Error "This script requires the presence of a test-proxy executable to complete its operations. Exiting." exit 1 } @@ -96,13 +96,13 @@ function Call-Proxy { $CommandArgs += " --storage-location=$MountDirectory" - if ($Output -eq $true){ + if ($Output -eq $true) { Write-Host "$TestProxyExe $CommandArgs" } [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") - if ($lastexitcode -ne 0){ + if ($lastexitcode -ne 0) { foreach($line in $output) { Write-Host $line } @@ -110,7 +110,7 @@ function Call-Proxy { exit 1 } - if ($Output -eq $true){ + if ($Output -eq $true) { foreach($line in $output) { Write-Host $line } @@ -150,7 +150,7 @@ function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDire Write-Host "If the presence of a previous execution of this script is surprising, delete the .assets folder and .mergeprogress file before invoking the script again." Write-Host "Attempting to continue from a previous run, and excluding:" - foreach($Tag in $alreadyCombinedTags){ + foreach($Tag in $alreadyCombinedTags) { Write-Host " - " -NoNewLine Write-Host "$Tag" -ForegroundColor Green } @@ -163,11 +163,11 @@ function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDire } else { Write-Host "`nThis script will attempt to merge the following tag" -NoNewLine - if ($TargetTags.Length -gt 1){ + if ($TargetTags.Length -gt 1) { Write-Host "s" -NoNewLine } Write-Host ":" - foreach($Tag in $TargetTags){ + foreach($Tag in $TargetTags) { Write-Host " - " -NoNewLine Write-Host "$Tag" -ForegroundColor Green } @@ -251,7 +251,7 @@ function Prepare-Assets($ProxyExe, $MountDirectory, $AssetsJson) { } function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){ - foreach($Tag in $RemainingTags){ + foreach($Tag in $RemainingTags) { $tagSha = Get-Tag-SHA $Tag $AssetsRepoLocation $existingTags = Save-Incomplete-Progress $Tag $MountDirectory $cherryPickResult = Git-Command-With-Result "cherry-pick $tagSha" - $AssetsRepoLocation -HardExit $false From 285768710d07ba9c75de160c390082cdfcbd554a Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 6 Dec 2023 11:57:53 -0800 Subject: [PATCH 11/12] update $CommandArgs to $CommandString to better reflect what it actually is --- .../testproxy/onboarding/common-asset-functions.ps1 | 12 +++++++----- .../testproxy/onboarding/generate-assets-json.ps1 | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/eng/common/testproxy/onboarding/common-asset-functions.ps1 b/eng/common/testproxy/onboarding/common-asset-functions.ps1 index 28c817fd341d..3caa6654c133 100644 --- a/eng/common/testproxy/onboarding/common-asset-functions.ps1 +++ b/eng/common/testproxy/onboarding/common-asset-functions.ps1 @@ -184,11 +184,13 @@ Function New-Assets-Json-File { Function Invoke-ProxyCommand { param( [string] $TestProxyExe, - [string] $CommandArgs, + [string] $CommandString, [string] $TargetDirectory ) $updatedDirectory = $TargetDirectory.Replace("`\", "/") + # CommandString just a string indicating the proxy arguments. In the default case of running against the proxy tool, can just be used directly. + # However, in the case of docker, we need to append a bunch more arguments to the string. if ($TestProxyExe -eq "docker" -or $TestProxyExe -eq "podman"){ $token = $env:GIT_TOKEN $committer = $env:GIT_COMMIT_OWNER @@ -210,7 +212,7 @@ Function Invoke-ProxyCommand { $targetImage = if ($env:TRANSITION_SCRIPT_DOCKER_TAG) { $env:TRANSITION_SCRIPT_DOCKER_TAG } else { "azsdkengsys.azurecr.io/engsys/test-proxy:latest" } - $CommandArgs = @( + $CommandString = @( "run --rm --name transition.test.proxy", "-v `"${updatedDirectory}:/srv/testproxy`"", "-e `"GIT_TOKEN=${token}`"", @@ -218,12 +220,12 @@ Function Invoke-ProxyCommand { "-e `"GIT_COMMIT_EMAIL=${email}`"", $targetImage, "test-proxy", - $CommandArgs + $CommandString ) -join " " } - Write-Host "$TestProxyExe $CommandArgs" - [array] $output = & "$TestProxyExe" $CommandArgs.Split(" ") --storage-location="$updatedDirectory" + Write-Host "$TestProxyExe $CommandString" + [array] $output = & "$TestProxyExe" $CommandString.Split(" ") --storage-location="$updatedDirectory" # echo the command output foreach ($line in $output) { Write-Host "$line" diff --git a/eng/common/testproxy/onboarding/generate-assets-json.ps1 b/eng/common/testproxy/onboarding/generate-assets-json.ps1 index 7f9ed1cbc071..bd825eebbe14 100644 --- a/eng/common/testproxy/onboarding/generate-assets-json.ps1 +++ b/eng/common/testproxy/onboarding/generate-assets-json.ps1 @@ -135,7 +135,7 @@ if ($InitialPush) { # Execute a restore on the current assets.json, it'll prep the root directory that # the recordings need to be copied into $CommandArgs = "restore --assets-json-path $assetsJsonRelPath" - Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandArgs $CommandArgs -TargetDirectory $repoRoot + Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandString $CommandArgs -TargetDirectory $repoRoot $assetsRoot = (Get-AssetsRoot -AssetsJsonFile $assetsJsonFile -TestProxyExe $TestProxyExe) Write-Host "assetsRoot=$assetsRoot" @@ -143,7 +143,7 @@ if ($InitialPush) { Move-AssetsFromLangRepo -AssetsRoot $assetsRoot $CommandArgs = "push --assets-json-path $assetsJsonRelPath" - Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandArgs $CommandArgs -TargetDirectory $repoRoot + Invoke-ProxyCommand -TestProxyExe $TestProxyExe -CommandString $CommandArgs -TargetDirectory $repoRoot # Verify that the assets.json file was updated $updatedAssets = Get-Content $assetsJsonFile | Out-String | ConvertFrom-Json From 02e14cface889958ad10f7c31fdfba3688697344 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 6 Dec 2023 14:00:11 -0800 Subject: [PATCH 12/12] whoops! didn't update the path to the target_version.txt. --- eng/common/testproxy/scripts/override-proxy-version.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/scripts/override-proxy-version.ps1 b/eng/common/testproxy/scripts/override-proxy-version.ps1 index d1e6ff5bfa37..031b0c17f083 100644 --- a/eng/common/testproxy/scripts/override-proxy-version.ps1 +++ b/eng/common/testproxy/scripts/override-proxy-version.ps1 @@ -10,7 +10,7 @@ param( [Parameter(mandatory=$true)] [string] $TargetVersion ) -$versionFile = Join-Path $PSScriptRoot "target_version.txt" +$versionFile = Join-Path $PSScriptRoot ".." "target_version.txt" $existingVersionText = Get-Content -Raw -Path $versionFile $existingVersion = $existingVersionText.Trim()