From c2b3824aaaa077cf1fea9e2bb4556ea27d4928da Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Thu, 2 Dec 2021 09:16:50 -0800 Subject: [PATCH 1/8] Use docker validation on release pipeline --- .../steps/update-docsms-metadata.yml | 19 ++++++-- eng/common/scripts/Update-DocsMsMetadata.ps1 | 45 +++++++++++++++---- eng/common/scripts/common.ps1 | 1 + 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml index 5b4a92f3a6ca..1ee879d3a4f4 100644 --- a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml +++ b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml @@ -26,7 +26,12 @@ parameters: type: object default: - '**' - + - name: PackageSourceOverride + type: string + default: '' + - name: DocValidationImageId + type: string + default: '' steps: - template: /eng/common/pipelines/templates/steps/enable-long-path-support.yml @@ -73,7 +78,13 @@ steps: parameters: WorkingDirectory: $(DocRepoLocation) DefaultBranchVariableName: TargetBranchName - +# Pull and build the docker image. +- ${{ if ne(parameters.DocValidationImageId, '') }}: +- template: /eng/common/pipelines/templates/steps/docker-pull-image.yml + parameters: + ContainerRegistryClientId: $(azuresdkimages-cr-clientid) + ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret) + ImageId: '${{ parameters.DocValidationImageId }}' - pwsh: | $packageInfoJson = '${{ convertToJson(parameters.PackageInfoLocations) }}'.Trim('"').Replace("\\", "/") $packageInfoLocations = ConvertFrom-Json $packageInfoJson @@ -81,7 +92,9 @@ steps: -PackageInfoJsonLocations $packageInfoLocations ` -DocRepoLocation "$(DocRepoLocation)" ` -Language '${{parameters.Language}}' ` - -RepoId '${{ parameters.RepoId }}' + -RepoId '${{ parameters.RepoId }}' ` + -DocValidationImageId '${{ parameters.DocValidationImageId }}' ` + -PackageSourceOverride '${{ parameters.PackageSourceOverride }}' displayName: Apply Documentation Updates - template: /eng/common/pipelines/templates/steps/git-push-changes.yml diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index fcb1756082aa..8456879b5f50 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -26,14 +26,23 @@ path information is provided by $GetDocsMsMetadataForPackageFn Programming language to supply to metadata .PARAMETER RepoId -GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js' +GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js' +.PARAMETER DocValidationImageId +The docker image id in format of '$containerRegistry/$imageName:$tag' +e.g. azuresdkimages.azurecr.io/jsrefautocr:latest + +.PARAMETER PackageSourceOverride +Optional parameter to supply a different package source (useful for daily dev +docs generation from pacakges which are not published to the default feed). This +variable is meant to be used in the domain-specific business logic in +&$ValidateDocsMsPackagesFn #> param( [Parameter(Mandatory = $true)] [array]$PackageInfoJsonLocations, - + [Parameter(Mandatory = $true)] [string]$DocRepoLocation, @@ -41,7 +50,13 @@ param( [string]$Language, [Parameter(Mandatory = $true)] - [string]$RepoId + [string]$RepoId, + + [Parameter(Mandatory = $false)] + [string]$DocValidationImageId, + + [Parameter(Mandatory = $false)] + [string]$PackageSourceOverride ) . (Join-Path $PSScriptRoot common.ps1) @@ -104,13 +119,12 @@ ms.technology: azure ms.devlang: $Language ms.service: $service --- - "@ return "$header`n$ReadmeContent" } -function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { +function GetPackageInfoJson ($packageInfoJsonLocation) { if (!(Test-Path $packageInfoJsonLocation)) { LogWarning "Package metadata not found for $packageInfoJsonLocation" return @@ -118,7 +132,6 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { $packageInfoJson = Get-Content $packageInfoJsonLocation -Raw $packageInfo = ConvertFrom-Json $packageInfoJson - $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version) if ($packageInfo.DevVersion) { # If the package is of a dev version there may be language-specific needs to # specify the appropriate version. For example, in the case of JS, the dev @@ -131,6 +144,11 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { $packageInfo.Version = $packageInfo.DevVersion } } + return $packageInfo +} + +function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation, $packageInfo) { + $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version) $packageMetadataArray = (Get-CSVMetadata).Where({ $_.Package -eq $packageInfo.Name -and $_.GroupId -eq $packageInfo.Group -and $_.Hide -ne 'true' -and $_.New -eq 'true' }) if ($packageMetadataArray.Count -eq 0) { @@ -176,7 +194,18 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { -Value $packageInfoJson } -foreach ($packageInfo in $PackageInfoJsonLocations) { +foreach ($packageInfoLocation in $PackageInfoJsonLocations) { Write-Host "Updating metadata for package: $packageInfo" - UpdateDocsMsMetadataForPackage $packageInfo + + # Convert package metadata json file to metadata json property. + $packageInfo = GetPackageInfoJson $packageInfoLocation + # Add validation step for daily update and release + if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) { + &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo $PackageSourceOverride $DocValidationImageId + if ($LASTEXITCODE -ne 0) { + LogError "The package failed Doc.Ms validation. Please fixed the doc and republish to Doc.Ms." + exit 1 + } + } + UpdateDocsMsMetadataForPackage $packageInfoLocation $packageInfo } diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 4f31c92c3d20..0f2dfe2f76f5 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -45,3 +45,4 @@ $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageS $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview" $TestProxyTrustCertFn = "Import-Dev-Cert-${Language}" +$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages" \ No newline at end of file From d4b61a55f0a5a819fdc0a7cc7d4290ff849667d0 Mon Sep 17 00:00:00 2001 From: sima-zhu Date: Thu, 2 Dec 2021 09:21:16 -0800 Subject: [PATCH 2/8] format --- .../templates/steps/update-docsms-metadata.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml index 1ee879d3a4f4..ce57f620c90b 100644 --- a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml +++ b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml @@ -80,11 +80,11 @@ steps: DefaultBranchVariableName: TargetBranchName # Pull and build the docker image. - ${{ if ne(parameters.DocValidationImageId, '') }}: -- template: /eng/common/pipelines/templates/steps/docker-pull-image.yml - parameters: - ContainerRegistryClientId: $(azuresdkimages-cr-clientid) - ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret) - ImageId: '${{ parameters.DocValidationImageId }}' + - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml + parameters: + ContainerRegistryClientId: $(azuresdkimages-cr-clientid) + ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret) + ImageId: '${{ parameters.DocValidationImageId }}' - pwsh: | $packageInfoJson = '${{ convertToJson(parameters.PackageInfoLocations) }}'.Trim('"').Replace("\\", "/") $packageInfoLocations = ConvertFrom-Json $packageInfoJson From 92e5c780972a5c25dd165bbaeadcdc0463235eaa Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Thu, 2 Dec 2021 12:52:26 -0800 Subject: [PATCH 3/8] Update Update-DocsMsMetadata.ps1 --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index 8456879b5f50..a2c7dc9f45d7 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -42,7 +42,7 @@ variable is meant to be used in the domain-specific business logic in param( [Parameter(Mandatory = $true)] [array]$PackageInfoJsonLocations, - + [Parameter(Mandatory = $true)] [string]$DocRepoLocation, From e26c0fb21554451f7932e5ae70f298e2520dd861 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Thu, 2 Dec 2021 12:52:51 -0800 Subject: [PATCH 4/8] Update Update-DocsMsMetadata.ps1 --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index a2c7dc9f45d7..f2e72e48d255 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -26,7 +26,7 @@ path information is provided by $GetDocsMsMetadataForPackageFn Programming language to supply to metadata .PARAMETER RepoId -GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js' +GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js' .PARAMETER DocValidationImageId The docker image id in format of '$containerRegistry/$imageName:$tag' From fe41e63bca2a12a3d76cf15c42c1e9c71f80806c Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Mon, 6 Dec 2021 08:57:44 -0800 Subject: [PATCH 5/8] Update Update-DocsMsMetadata.ps1 --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index f2e72e48d255..27cbed23eaed 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -203,7 +203,7 @@ foreach ($packageInfoLocation in $PackageInfoJsonLocations) { if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) { &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo $PackageSourceOverride $DocValidationImageId if ($LASTEXITCODE -ne 0) { - LogError "The package failed Doc.Ms validation. Please fixed the doc and republish to Doc.Ms." + LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for how to reproduce the errors" exit 1 } } From 2634b61234bf6967ff38cd54df80730d77f03fd3 Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Mon, 6 Dec 2021 09:30:35 -0800 Subject: [PATCH 6/8] For testing null purpose --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index 27cbed23eaed..a60f4377ba85 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -201,7 +201,7 @@ foreach ($packageInfoLocation in $PackageInfoJsonLocations) { $packageInfo = GetPackageInfoJson $packageInfoLocation # Add validation step for daily update and release if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) { - &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo $PackageSourceOverride $DocValidationImageId + &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId if ($LASTEXITCODE -ne 0) { LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for how to reproduce the errors" exit 1 From cf2ffea47b8e011d3eff3778ae49d5083de507ab Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Mon, 6 Dec 2021 09:53:57 -0800 Subject: [PATCH 7/8] Update eng/common/scripts/Update-DocsMsMetadata.ps1 --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index a60f4377ba85..52eace4ae02a 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -203,7 +203,7 @@ foreach ($packageInfoLocation in $PackageInfoJsonLocations) { if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) { &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId if ($LASTEXITCODE -ne 0) { - LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for how to reproduce the errors" + LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for more details on how to diagnose this issue." exit 1 } } From ab8e99dc5a38d247861ceeaac6800ba108baa76c Mon Sep 17 00:00:00 2001 From: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Date: Mon, 6 Dec 2021 10:59:22 -0800 Subject: [PATCH 8/8] Update eng/common/scripts/Update-DocsMsMetadata.ps1 Co-authored-by: Ben Broderick Phillips --- eng/common/scripts/Update-DocsMsMetadata.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1 index 52eace4ae02a..a77ec890a3d0 100644 --- a/eng/common/scripts/Update-DocsMsMetadata.ps1 +++ b/eng/common/scripts/Update-DocsMsMetadata.ps1 @@ -202,9 +202,9 @@ foreach ($packageInfoLocation in $PackageInfoJsonLocations) { # Add validation step for daily update and release if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) { &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId - if ($LASTEXITCODE -ne 0) { + if ($LASTEXITCODE) { LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for more details on how to diagnose this issue." - exit 1 + exit $LASTEXITCODE } } UpdateDocsMsMetadataForPackage $packageInfoLocation $packageInfo