From 29099c632419cf4528e31b561589ab110c1fcf3c Mon Sep 17 00:00:00 2001 From: "Dustin L. Howett" Date: Fri, 8 Mar 2024 14:22:11 -0600 Subject: [PATCH] Check the localizations into the project nightly (#16835) Right now, the localization submission pipeline runs every night and sends our localizable resources over to Touchdown. Later, release builds pick up the localizations directly from Touchdown, move them into place, and consume them. This allowed us to avoid having localized content in the repository, but it came with too many downsides: - Users could not contribute additional localizations very easily - We use the same release pipeline and Touchdown configuration for every branch, so strings needed to either slightly match or _entirely match_ across an entire set of active release branches - Building from day to day can pull in different strings, making the product not reproduceable - Calling TDBuild during release builds requires network access from the build machine (so does restoring NuGet packages, but that's neither here nor there) - Local developers and users could not test out other languages This pull request moves all localization processing into the nightly build phase and introduces support for checking loc in and submitting a pull request. The pull request will not be created anew if one already exists which has not been merged. Anything we needed to do on release is now done overnight. This includes moving loc files into the right places and merging the Cascadia resources with the Context Menu resources (so that we can work around a relatively lower amount of translations being chosen for the app versus the context menu; see #12491 for more info.) There are some smaller downsides to this approach and its implementation: - The first commit is going to be huge - Right now, it only manages a single branch and uses a force push; if a PR is not reviewed timely, it will be force-pushed and you cannot see the day-to-day changes in the strings. Hopefully there won't be any. I've taken great care to ensure that repeated runs of this new pipeline will not result in unnecessary whitespace changes. This required changing how we merge ContextMenu.resw into CascadiaPackage to always use the .NET XmlWriter with specific flags. NOTE that this does not allow users to _contribute_ translation fixes for the 10 languages which we are importing. We will still need to pull changes out of those files and submit them as bugs to the localization team separately, and hope they come back around in another nightly build. However, there is no reason users cannot contribute _non-Touchdown_ languages. (cherry picked from commit ab4b140aa48c3a97d0c2761b89c5857f10ca6317) Service-Card-Id: 92019664 Service-Version: 1.19 --- .github/actions/spelling/allow/allow.txt | 1 + .github/actions/spelling/expect/expect.txt | 1 + build/pipelines/daily-loc-submission.yml | 60 ++++++++++++++++--- .../pipeline-full-release-build.yml | 4 -- .../pipeline-onebranch-full-release-build.yml | 4 -- .../steps-fetch-and-prepare-localizations.yml | 27 --------- ...-ContextMenuResourcesToCascadiaPackage.ps1 | 18 +++++- .../Resources/en-US/Resources.resw | 48 +++++++++------ 8 files changed, 101 insertions(+), 62 deletions(-) delete mode 100644 build/pipelines/templates-v2/steps-fetch-and-prepare-localizations.yml diff --git a/.github/actions/spelling/allow/allow.txt b/.github/actions/spelling/allow/allow.txt index a5d5bfcab3e..6d9f76b73a7 100644 --- a/.github/actions/spelling/allow/allow.txt +++ b/.github/actions/spelling/allow/allow.txt @@ -14,6 +14,7 @@ changelog clickable clig CMMI +consvc copyable Counterintuitively CtrlDToClose diff --git a/.github/actions/spelling/expect/expect.txt b/.github/actions/spelling/expect/expect.txt index bc1633f6908..87306cd74c4 100644 --- a/.github/actions/spelling/expect/expect.txt +++ b/.github/actions/spelling/expect/expect.txt @@ -77,6 +77,7 @@ aumid Authenticode AUTOBUDDY AUTOCHECKBOX +autocrlf autohide AUTOHSCROLL automagically diff --git a/build/pipelines/daily-loc-submission.yml b/build/pipelines/daily-loc-submission.yml index 890c1f86a47..93964c9b8a3 100644 --- a/build/pipelines/daily-loc-submission.yml +++ b/build/pipelines/daily-loc-submission.yml @@ -8,6 +8,12 @@ schedules: - main always: false # only run if there's code changes! + +parameters: + - name: targetBranch + type: string + default: "automated/loc-update" + pool: vmImage: windows-2019 @@ -38,6 +44,13 @@ steps: persistCredentials: true path: s/Terminal.Internal +- pwsh: |- + Install-Module PSGitHub -Scope CurrentUser -Force + git config --local user.email "consvc@microsoft.com" + git config --local user.name "Console Service Bot" + git config --local core.autocrlf true + displayName: Prepare git submission environment + - task: MicrosoftTDBuild.tdbuild-task.tdbuild-task.TouchdownBuildTask@1 displayName: 'Touchdown Build - 7105, PRODEXT' inputs: @@ -51,13 +64,44 @@ steps: outputDirectoryRoot: LocOutput appendRelativeDir: true pseudoSetting: Included + localizationTarget: true -# Saving one of these makes it really easy to inspect the loc output... -- powershell: 'tar czf LocOutput.tar.gz LocOutput' - displayName: 'Archive Loc Output for Submission' +- pwsh: |- + Remove-Item -EA:Ignore -R -Force LocOutput\Terminal.Internal + $Files = Get-ChildItem LocOutput -R -Include 'ContextMenu.resw','Resources.resw' | ? FullName -Like '*en-US\*\*.resw' + $Files | % { Move-Item -Verbose $_.Directory $_.Directory.Parent.Parent -EA:Ignore } + & tar.exe -c -f LocOutputMunged.tar -C LocOutput . + & tar.exe -x -v -f LocOutputMunged.tar + rm LocOutputMunged.tar + rm -r -fo LocOutput + & ./build/scripts/Copy-ContextMenuResourcesToCascadiaPackage.ps1 + displayName: Move Loc files to the right places -- task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: LocOutput' - inputs: - PathtoPublish: LocOutput.tar.gz - ArtifactName: LocOutput +- pwsh: |- + git add **/*.resw + git status + git diff --quiet --cached --exit-code + If ($LASTEXITCODE -Ne 0) { + $Now = Get-Date + git commit -m "Localization Updates - $Now" + git push origin HEAD:refs/heads/${{parameters.targetBranch}} -f + Write-Host "##vso[task.setvariable variable=ChangesPushedToRepo]1" + } Else { + Write-Host "##vso[task.setvariable variable=ChangesPushedToRepo]0" + } + displayName: git commit and push + +- pwsh: |- + Import-Module PSGitHub + $BaseBranch = "$(Build.SourceBranch)" -Replace "^refs/heads/","" + Write-Host "Preparing PR against $BaseBranch" + $PSDefaultParameterValues['*GitHub*:Owner'] = "microsoft" + $PSDefaultParameterValues['*GitHub*:RepositoryName'] = "terminal" + $PSDefaultParameterValues['*GitHub*:Token'] = ("$(GithubPullRequestToken)" | ConvertTo-SecureString -AsPlainText -Force) + $existingPr = Get-GitHubPullRequest -HeadBranch "${{parameters.targetBranch}}" -BaseBranch $BaseBranch + If ($null -Eq $existingPr) { + $Now = Get-Date + New-GitHubPullRequest -Head "${{parameters.targetBranch}}" -Base $BaseBranch -Title "Localization Updates - $BaseBranch - $Now" -Verbose + } + displayName: Publish pull request + condition: and(eq(variables['ChangesPushedToRepo'], '1'), succeeded()) diff --git a/build/pipelines/templates-v2/pipeline-full-release-build.yml b/build/pipelines/templates-v2/pipeline-full-release-build.yml index 06358de5c5b..92800efe189 100644 --- a/build/pipelines/templates-v2/pipeline-full-release-build.yml +++ b/build/pipelines/templates-v2/pipeline-full-release-build.yml @@ -104,10 +104,6 @@ stages: packageListDownload: e82d490c-af86-4733-9dc4-07b772033204 versionListDownload: ${{ parameters.terminalInternalPackageVersion }} - - template: ./steps-fetch-and-prepare-localizations.yml - parameters: - includePseudoLoc: true - - ${{ if eq(parameters.buildWPF, true) }}: # Add an Any CPU build flavor for the WPF control bits - template: ./job-build-project.yml diff --git a/build/pipelines/templates-v2/pipeline-onebranch-full-release-build.yml b/build/pipelines/templates-v2/pipeline-onebranch-full-release-build.yml index 991ec1b262b..e05b373f6b8 100644 --- a/build/pipelines/templates-v2/pipeline-onebranch-full-release-build.yml +++ b/build/pipelines/templates-v2/pipeline-onebranch-full-release-build.yml @@ -128,10 +128,6 @@ extends: packageListDownload: e82d490c-af86-4733-9dc4-07b772033204 versionListDownload: ${{ parameters.terminalInternalPackageVersion }} - - template: ./build/pipelines/templates-v2/steps-fetch-and-prepare-localizations.yml@self - parameters: - includePseudoLoc: true - - ${{ if eq(parameters.buildWPF, true) }}: # Add an Any CPU build flavor for the WPF control bits - template: ./build/pipelines/templates-v2/job-build-project.yml@self diff --git a/build/pipelines/templates-v2/steps-fetch-and-prepare-localizations.yml b/build/pipelines/templates-v2/steps-fetch-and-prepare-localizations.yml deleted file mode 100644 index 8ace357aad3..00000000000 --- a/build/pipelines/templates-v2/steps-fetch-and-prepare-localizations.yml +++ /dev/null @@ -1,27 +0,0 @@ -parameters: - - name: includePseudoLoc - type: boolean - default: true - -steps: - - task: TouchdownBuildTask@1 - displayName: Download Localization Files - inputs: - teamId: 7105 - authId: $(TouchdownAppId) - authKey: $(TouchdownAppKey) - resourceFilePath: | - src\cascadia\**\en-US\*.resw - appendRelativeDir: true - localizationTarget: false - ${{ if eq(parameters.includePseudoLoc, true) }}: - pseudoSetting: Included - - - pwsh: |- - $Files = Get-ChildItem . -R -Filter 'Resources.resw' | ? FullName -Like '*en-US\*\Resources.resw' - $Files | % { Move-Item -Verbose $_.Directory $_.Directory.Parent.Parent -EA:Ignore } - displayName: Move Loc files into final locations - - - pwsh: |- - ./build/scripts/Copy-ContextMenuResourcesToCascadiaPackage.ps1 - displayName: Copy the Context Menu Loc Resources to CascadiaPackage diff --git a/build/scripts/Copy-ContextMenuResourcesToCascadiaPackage.ps1 b/build/scripts/Copy-ContextMenuResourcesToCascadiaPackage.ps1 index 8111b3fc66e..bbccdaf57d1 100644 --- a/build/scripts/Copy-ContextMenuResourcesToCascadiaPackage.ps1 +++ b/build/scripts/Copy-ContextMenuResourcesToCascadiaPackage.ps1 @@ -10,11 +10,12 @@ $LocalizationsFromContextMenu | ForEach-Object { ForEach ($pair in $Languages.GetEnumerator()) { $LanguageDir = "./src/cascadia/CascadiaPackage/Resources/$($pair.Key)" $ResPath = "$LanguageDir/Resources.resw" + $XmlDocument = $null $PreexistingResw = Get-Item $ResPath -EA:Ignore If ($null -eq $PreexistingResw) { Write-Host "Copying $($pair.Value.FullName) to $ResPath" + $XmlDocument = [xml](Get-Content $pair.Value.FullName) New-Item -type Directory $LanguageDir -EA:Ignore - Copy-Item $pair.Value.FullName $ResPath } Else { # Merge Them! Write-Host "Merging $($pair.Value.FullName) into $ResPath" @@ -29,6 +30,19 @@ ForEach ($pair in $Languages.GetEnumerator()) { $newXml.root.data | % { $null = $existingXml.root.AppendChild($existingXml.ImportNode($_, $true)) } - $existingXml.Save($PreexistingResw.FullName) + $XmlDocument = $existingXml # (which has been updated) } + + # Reset paths to be absolute (for .NET) + $LanguageDir = (Get-Item $LanguageDir).FullName + $ResPath = "$LanguageDir/Resources.resw" + # Force the "new" and "preexisting" paths to serialize with XmlWriter, + # to ensure consistency. + $writerSettings = [System.Xml.XmlWriterSettings]::new() + $writerSettings.NewLineChars = "`r`n" + $writerSettings.Indent = $true + $writer = [System.Xml.XmlWriter]::Create($ResPath, $writerSettings) + $XmlDocument.Save($writer) + $writer.Flush() + $writer.Close() } diff --git a/src/cascadia/CascadiaPackage/Resources/en-US/Resources.resw b/src/cascadia/CascadiaPackage/Resources/en-US/Resources.resw index f7d5a7010e9..b66bd1d55f3 100644 --- a/src/cascadia/CascadiaPackage/Resources/en-US/Resources.resw +++ b/src/cascadia/CascadiaPackage/Resources/en-US/Resources.resw @@ -1,4 +1,4 @@ - +