From bf1a37d317531fb3bfd13f83975cc3e5853f8c89 Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Tue, 17 Nov 2020 16:55:49 -0800 Subject: [PATCH] Refactor Update-docs-ci.ps1 --- .../scripts/artifact-metadata-parsing.ps1 | 2 + eng/common/scripts/common.ps1 | 3 +- eng/common/scripts/update-docs-ci.ps1 | 220 +----------------- 3 files changed, 11 insertions(+), 214 deletions(-) diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 6f9db9b88ee..20541f53b4c 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -1,3 +1,5 @@ +. (Join-Path $EngCommonScriptsDir SemVer.ps1) + $SDIST_PACKAGE_REGEX = "^(?.*)\-(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" # Posts a github release for each item of the pkgList variable. SilentlyContinue diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 9b2f2bce375..e9a4e070999 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -32,4 +32,5 @@ if (-not $LanguageShort) # Transformed Functions $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" -$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" \ No newline at end of file +$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" +$UpdateDocCIFn = "Update-${Language}-CIConfig" \ No newline at end of file diff --git a/eng/common/scripts/update-docs-ci.ps1 b/eng/common/scripts/update-docs-ci.ps1 index f0df3b375af..d13942b7f15 100644 --- a/eng/common/scripts/update-docs-ci.ps1 +++ b/eng/common/scripts/update-docs-ci.ps1 @@ -32,198 +32,6 @@ param ( . (Join-Path $PSScriptRoot common.ps1) -# Updates a python CI configuration json. -# For "latest", the version attribute is cleared, as default behavior is to pull latest "non-preview". -# For "preview", we update to >= the target releasing package version. -function UpdateParamsJsonPython($pkgs, $ciRepo, $locationInDocRepo){ - $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) - - if (-not (Test-Path $pkgJsonLoc)) { - Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." - exit(1) - } - - $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json - $visibleInCI = @{} - - for ($i=0; $i -lt $allJson.packages.Length; $i++) { - $pkgDef = $allJson.packages[$i] - - if ($pkgDef.package_info.name) { - $visibleInCI[$pkgDef.package_info.name] = $i - } - } - - foreach ($releasingPkg in $pkgs) { - if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { - $packagesIndex = $visibleInCI[$releasingPkg.PackageId] - $existingPackageDef = $allJson.packages[$packagesIndex] - - if ($releasingPkg.IsPrerelease) { - if (-not $existingPackageDef.package_info.version) { - $existingPackageDef.package_info | Add-Member -NotePropertyName version -NotePropertyValue "" - } - - $existingPackageDef.package_info.version = ">=$($releasingPkg.PackageVersion)" - } - else { - if ($def.version) { - $def.PSObject.Properties.Remove('version') - } - } - } - else { - $newItem = New-Object PSObject -Property @{ - package_info = New-Object PSObject -Property @{ - prefer_source_distribution = "true" - install_type = "pypi" - name=$releasingPkg.PackageId - } - excludePath = @("test*","example*","sample*","doc*") - } - $allJson.packages += $newItem - } - } - - $jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } - - Set-Content -Path $pkgJsonLoc -Value $jsonContent -} - -# Updates a js CI configuration json. -# For "latest", we simply set a target package name -# For "preview", we add @next to the target package name -function UpdateParamsJsonJS($pkgs, $ciRepo, $locationInDocRepo){ - $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) - - if (-not (Test-Path $pkgJsonLoc)) { - Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." - exit(1) - } - - $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json - - $visibleInCI = @{} - - for ($i=0; $i -lt $allJson.npm_package_sources.Length; $i++) { - $pkgDef = $allJson.npm_package_sources[$i] - $accessor = ($pkgDef.name).Replace("`@next", "") - $visibleInCI[$accessor] = $i - } - - foreach ($releasingPkg in $pkgs) { - $name = $releasingPkg.PackageId - - if ($releasingPkg.IsPrerelease) { - $name += "`@next" - } - - if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { - $packagesIndex = $visibleInCI[$releasingPkg.PackageId] - $existingPackageDef = $allJson.npm_package_sources[$packagesIndex] - $existingPackageDef.name = $name - } - else { - $newItem = New-Object PSObject -Property @{ - name = $name - } - - if ($newItem) { $allJson.npm_package_sources += $newItem } - } - } - - $jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } - - Set-Content -Path $pkgJsonLoc -Value $jsonContent -} - -# details on CSV schema can be found here -# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job -function UpdateCSVBasedCI($pkgs, $ciRepo, $locationInDocRepo){ - $csvLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) - - if (-not (Test-Path $csvLoc)) { - Write-Error "Unable to locate package csv at location $csvLoc, exiting." - exit(1) - } - - $allCSVRows = Get-Content $csvLoc - $visibleInCI = @{} - - # first pull what's already available - for ($i=0; $i -lt $allCSVRows.Length; $i++) { - $pkgDef = $allCSVRows[$i] - - # get rid of the modifiers to get just the package id - $id = $pkgDef.split(",")[1] -replace "\[.*?\]", "" - - $visibleInCI[$id] = $i - } - - foreach ($releasingPkg in $pkgs) { - $installModifiers = "tfm=netstandard2.0" - if ($releasingPkg.IsPrerelease) { - $installModifiers += ";isPrerelease=true" - } - $lineId = $releasingPkg.PackageId.Replace(".","").ToLower() - - if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { - $packagesIndex = $visibleInCI[$releasingPkg.PackageId] - $allCSVRows[$packagesIndex] = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" - } - else { - $newItem = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" - $allCSVRows += ($newItem) - } - } - - Set-Content -Path $csvLoc -Value $allCSVRows -} - -# a "package.json configures target packages for all the monikers in a Repository, it also has a slightly different -# schema than the moniker-specific json config that is seen in python and js -function UpdatePackageJson($pkgs, $ciRepo, $locationInDocRepo, $monikerId){ - $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) - - if (-not (Test-Path $pkgJsonLoc)) { - Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." - exit(1) - } - - $allJsonData = Get-Content $pkgJsonLoc | ConvertFrom-Json - - $visibleInCI = @{} - - for ($i=0; $i -lt $allJsonData[$monikerId].packages.Length; $i++) { - $pkgDef = $allJsonData[$monikerId].packages[$i] - $visibleInCI[$pkgDef.packageArtifactId] = $i - } - - foreach ($releasingPkg in $pkgs) { - if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { - $packagesIndex = $visibleInCI[$releasingPkg.PackageId] - $existingPackageDef = $allJsonData[$monikerId].packages[$packagesIndex] - $existingPackageDef.packageVersion = $releasingPkg.PackageVersion - } - else { - $newItem = New-Object PSObject -Property @{ - packageDownloadUrl = "https://repo1.maven.org/maven2" - packageGroupId = $releasingPkg.GroupId - packageArtifactId = $releasingPkg.PackageId - packageVersion = $releasingPkg.PackageVersion - inputPath = @() - excludePath = @() - } - - $allJsonData[$monikerId].packages += $newItem - } - } - - $jsonContent = $allJsonData | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } - - Set-Content -Path $pkgJsonLoc -Value $jsonContent -} - $targets = ($Configs | ConvertFrom-Json).targets #{ @@ -246,27 +54,13 @@ foreach ($config in $targets) { Write-Host "Given the visible artifacts, CI updates against $($config.path_to_config) will be processed for the following packages." Write-Host ($pkgsFiltered | % { $_.PackageId + " " + $_.PackageVersion }) - switch ($Repository) { - "Nuget" { - UpdateCSVBasedCI -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config - break - } - "NPM" { - UpdateParamsJsonJS -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config - break - } - "PyPI" { - UpdateParamsJsonPython -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config - break - } - "Maven" { - UpdatePackageJson -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid - break - } - default { - Write-Host "Unrecognized target: $Repository" - exit(1) - } + if ($UpdateDocCIFn -and (Test-Path "Function:$UpdateDocCIFn")) + { + &$UpdateDocCIFn -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid + } + else + { + LogWarning "The function '$UpdateDocCIFn' was not found." } } } \ No newline at end of file