.NET-specific changes: Move deprecated packages to legacy (Azure#36194)

* First cut at migrating to legacy moniker * Update-PackageMonikers.ps1 * Refinement * Update docindex.yml for testing * Rename * ToC fixes * Refactor namespace discovery * Remove EnsureNamespaces * Add "legacy" moniker * Undo Service-Level-Readme-Automation.ps1 because it is generating incorrect package paths * Better edge case handling, documentation * Add fixes to service level overview generation * Reset eng/common * docindex changes * Comment cleanup * Fix ToC generation tests * Test cleanup
yaotongms · Oct 12, 2023 · 8044c3e · 8044c3e
1 parent 9f3b61a
commit 8044c3e
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 127 deletions.
diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml
@@ -38,12 +38,22 @@ jobs:
           Repositories:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
+
       # Install the tool for toc step to use
       - task: NuGetToolInstaller@1
         inputs:
           versionSpec: 6.3.1
-      # Call update docs ci script to onboard packages
+
       - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Move deprecated packages to legacy moniker
+        condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
+
       - task: Powershell@2
         inputs:
           pwsh: true
@@ -109,6 +119,14 @@ jobs:
         displayName: Set package source variable
         workingDirectory: $(DocRepoLocation)
 
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Move deprecated packages to legacy moniker
+        condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
+
       - task: Powershell@2
         inputs:
           pwsh: true

diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
@@ -298,6 +298,7 @@ function Get-dotnet-DocsMsMetadataForPackage($PackageInfo) {
     DocsMsReadMeName = $readmeName
     LatestReadMeLocation = 'api/overview/azure/latest'
     PreviewReadMeLocation = 'api/overview/azure/preview'
+    LegacyReadMeLocation = 'api/overview/azure/legacy'
     Suffix = ''
   }
 }
@@ -494,6 +495,25 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
       continue
     }
 
+    if ($matchingPublishedPackage.Support -eq 'deprecated') { 
+      if ($Mode -eq 'legacy') { 
+
+        # Select the GA version, if none use the preview version
+        $updatedVersion = $matchingPublishedPackage.VersionGA.Trim()
+        if (!$updatedVersion) { 
+          $updatedVersion = $matchingPublishedPackage.VersionPreview.Trim()
+        }
+        $package.Versions = @($updatedVersion)
+
+        Write-Host "Add deprecated package to legacy moniker: $($package.Name)"
+        $outputPackages += $package
+      } else { 
+        Write-Host "Removing deprecated package: $($package.Name)"
+      }
+
+      continue
+    }
+
     $updatedVersion = $matchingPublishedPackage.VersionGA.Trim()
     if ($Mode -eq 'preview') {
       $updatedVersion = $matchingPublishedPackage.VersionPreview.Trim()

diff --git a/eng/scripts/docs/Docs-ToC.ps1 b/eng/scripts/docs/Docs-ToC.ps1
@@ -110,7 +110,10 @@ function Get-dotnet-OnboardedDocsMsPackagesForMoniker ($DocRepoLocation, $monike
     $onboardingSpec = "$DocRepoLocation/bundlepackages/azure-dotnet.csv"
     if ("preview" -eq $moniker) {
         $onboardingSpec = "$DocRepoLocation/bundlepackages/azure-dotnet-preview.csv"
+    } elseif ('legacy' -eq $moniker) { 
+        $onboardingSpec = "$DocRepoLocation/bundlepackages/azure-dotnet-legacy.csv"
     }
+
     $onboardedPackages = @{}
     $packageInfos = Get-DocsCiConfig $onboardingSpec
     foreach ($packageInfo in $packageInfos) {
@@ -147,92 +150,65 @@ function GetPackageReadmeName($packageMetadata) {
     return $packageLevelReadmeName
 }
 
+# Defined in common.ps1
+# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
 function Get-dotnet-DocsMsTocData($packageMetadata, $docRepoLocation, $PackageSourceOverride) {
     $packageLevelReadmeName = GetPackageReadmeName -packageMetadata $packageMetadata
     $packageTocHeader = GetDocsTocDisplayName $packageMetadata
 
-    $children = @()
-    # Children here combine namespaces in both preview and GA.
-    if($packageMetadata.VersionPreview) {
-        $children += Get-Toc-Children -package $packageMetadata.Package -version $packageMetadata.VersionPreview `
-            -docRepoLocation $docRepoLocation -folderName "preview"
-    }
-    if($packageMetadata.VersionGA) {
-        $children += Get-Toc-Children -package $packageMetadata.Package -version $packageMetadata.VersionGA `
-            -docRepoLocation $docRepoLocation -folderName "latest"
-    }
-    $children = @($children | Sort-Object -Unique)
+    $children = Get-Toc-Children `
+        -package $packageMetadata.Package `
+        -docRepoLocation $docRepoLocation 
+
     if (!$children) {
-        if ($packageMetadata.VersionPreview) {
-            LogDebug "Did not find the package namespaces for $($packageMetadata.Package):$($packageMetadata.VersionPreview)"
-        }
-        if ($packageMetadata.VersionGA) {
-            LogDebug "Did not find the package namespaces for $($packageMetadata.Package):$($packageMetadata.VersionGA)"
-        }
+        LogWarning "Did not find the package namespaces for $($packageMetadata.Package)"
     }
     $output = [PSCustomObject]@{
       PackageLevelReadmeHref = "~/api/overview/azure/{moniker}/$packageLevelReadmeName-readme.md"
       PackageTocHeader       = $packageTocHeader
-      TocChildren            = $children
+      TocChildren            = @($children)
     }
 
     return $output
 }
 
 
-# This is a helper function to fetch the dotnet package namespaces.
-# Here is the major workflow:
-# 1. Read the ${package}.json under /metadata folder. If json file exists, Namespaces property exists and version match, return the array. 
-# 2. If the json file exist but version mismatch, fetch the namespaces from nuget and udpate the json namespaces property.
-# 3. If file not found, then fetch namespaces from nuget and create new package.json file with very basic info, like package name, version, namespaces. 
-function Get-Toc-Children($package, $version, $docRepoLocation, $folderName) {
-    # Looking for the txt
-    $packageJsonPath = "$docRepoLocation/metadata/$folderName/$package.json" 
-    $packageJsonObject = [PSCustomObject]@{
-        Name = $package
-        Version = $version
-        Namespaces = @()
-    }
-    # We will download package and parse out namespaces for the following cases.
-    # 1. Package json file doesn't exist. Create a new json file with $packageJsonObject
-    # 2. Package json file exists, but no Namespaces property. Add the Namespaces property with right values.
-    # 3. Both package json file and Namespaces property exist, but the version mismatch. Update the nameapace values.
-    $jsonExists = Test-Path $packageJsonPath
-    $namespacesExist = $false
-    $versionMismatch = $false
-    $packageJson = $null
-    # Add backup if no namespaces fetched out from new packages.
-    $originalNamespaces = @()
-    # Collect the information and return the booleans of updates or not.
-    if ($jsonExists) {
-        $packageJson = Get-Content $packageJsonPath | ConvertFrom-Json
-        $namespacesExist = $packageJson.PSObject.Members.Name -contains "Namespaces"
-        if ($namespacesExist) {
-            # Fallback to original ones if failed to fetch updated namespaces.
-            $originalNamespaces = $packageJson.Namespaces
-        }
-        $versionMismatch = $version -ne $packageJson.Version
-    }
-    if ($jsonExists -and $namespacesExist -and !$versionMismatch) {
-        return $originalNamespaces
-    }
-    $namespaces = Fetch-NamespacesFromNupkg -package $package -version $version
-    if (!$namespaces) {
-        $namespaces = $originalNamespaces
-    }
-    if (!$packageJson) {
-        $packageJsonObject.Namespaces = $namespaces
-        $packageJson = ($packageJsonObject | ConvertTo-Json | ConvertFrom-Json)
+# This function is called within a loop. To prevent multiple reads of the same 
+# file data, this uses a script-scoped cache variable.
+$script:PackageMetadataJsonLookup = $null
+function GetPackageMetadataJsonLookup($docRepoLocation) { 
+    if ($script:PackageMetadataJsonLookup) {
+        return $script:PackageMetadataJsonLookup
     }
-    elseif(!$namespacesExist) {
-        $packageJson = $packageJson | Add-Member -MemberType NoteProperty -Name Namespaces -Value $namespaces -PassThru
+
+    $script:PackageMetadataJsonLookup = @{}
+    $packageJsonFiles = Get-ChildItem $docRepoLocation/metadata/ -Filter *.json -Recurse
+    foreach ($packageJsonFile in $packageJsonFiles) {
+        $packageJson = Get-Content $packageJsonFile -Raw | ConvertFrom-Json -AsHashtable
+
+        if (!$script:PackageMetadataJsonLookup.ContainsKey($packageJson.Name)) { 
+            $script:PackageMetadataJsonLookup[$packageJson.Name] = @($packageJson)
+        } else { 
+            $script:PackageMetadataJsonLookup[$packageJson.Name] += $packageJson
+        }
     }
-    else{
-        $packageJson.Namespaces = $namespaces
+
+    return $script:PackageMetadataJsonLookup
+}
+
+function Get-Toc-Children($package, $docRepoLocation) {
+    $packageTable = GetPackageMetadataJsonLookup $docRepoLocation
+
+    $namespaces = @()
+    if ($packageTable.ContainsKey($package)) { 
+        foreach ($entry in $packageTable[$package]) {
+            if ($entry.ContainsKey('Namespaces')) {
+                $namespaces += $entry['Namespaces']
+            }
+        }
     }
-    # Keep the json file up to date.
-    Set-Content $packageJsonPath -Value ($packageJson | ConvertTo-Json)
-    return $namespaces
+
+    return $namespaces | Sort-Object -Unique
 }
 
 function Get-dotnet-PackageLevelReadme ($packageMetadata) {

diff --git a/eng/scripts/docs/tests/Toc-Generation.Tests.ps1 b/eng/scripts/docs/tests/Toc-Generation.Tests.ps1
@@ -9,27 +9,12 @@ Import-Module Pester
 BeforeAll {
     . $PSScriptRoot/../Docs-ToC.ps1
     . $PSScriptRoot/logging.ps1
-    function Backup-File($targetPath, $backupFolder) {
-        if (!(Test-Path $targetPath)) {
-            return $null
-        }
-        $fileName = (Split-Path $targetPath -leaf)
-        $backupFile = "$backupFolder/temp-$fileName"
-        $null = New-Item $backupFile -ItemType "file" -Force
-        $null = Copy-Item $targetPath -Destination $backupFile
-        return $backupFile
-    }
-    function Reset-File($targetPath, $backupFile) {
-        if ($backupFile) {
-            $null = Copy-Item $backupFile -Destination $targetPath
-        }
-    }
 }
 
 AfterAll {
     $tempLocation = (Join-Path ([System.IO.Path]::GetTempPath()) "extractNupkg")
-    Remove-Item "$tempLocation/*" -Recurse -Force
-    Remove-Item "$PSScriptRoot/outputs" -Recurse -Force
+    Remove-Item "$tempLocation/*" -Recurse -Force -ErrorAction Ignore
+    Remove-Item "$PSScriptRoot/outputs" -Recurse -Force -ErrorAction Ignore
 }
 # Test plan:
 # 1. Tests on Fetch-NamespacesFromNupkg from nuget source. 
@@ -86,33 +71,28 @@ Describe "Fetch-NamespacesFromNupkg-PublicFeeds" -Tag "UnitTest" {
     }
 }
 
-# Test scenarios:
-# 1. No package json found. Expect generate and update the json.
-# 2. Json exists, but no namespaces. Expect to add new property and update the json
-# 3. Json and namespaces exists, but version mismatch. Expect to update to the most recent namespaces.
-# 4. Otherwise, return namespaces in json file.
 Describe "Get-Toc-Children" -Tag "UnitTest" {
-    # passed cases
     It "Get toc children from package json" -TestCases @(
-        @{ package = "Azure.Security.KeyVault.Secrets"; version="4.4.0"; folder="latest"; expectNamespaces = @('Azure.Security.KeyVault.Secrets', 'Microsoft.Extensions.Azure') }
-        @{ package = "Azure.Security.KeyVault.Certificates"; version="4.3.0-beta.4"; folder="preview"; expectNamespaces = @('Azure.Security.KeyVault.Certificates', 'Microsoft.Extensions.Azure') }
-        @{ package = "Azure.Security.KeyVault.Keys"; version="4.4.0"; folder="latest"; expectNamespaces = @('Azure.Security.KeyVault.Keys', 'Azure.Security.KeyVault.Keys.Cryptography', 'Microsoft.Extensions.Azure') }
+        @{ 
+            package = "Azure.Security.KeyVault.Secrets";
+            expectNamespaces = @('Azure.Security.KeyVault.Secrets', 'Microsoft.Extensions.Azure') 
+        }
     ) {
-        $targetFile = "$PSScriptRoot/inputs/metadata/$folder/$package.json"
-        $backupFolder = "$PSScriptRoot/outputs"
-        $backupFile = Backup-File $targetFile $backupFolder
-        $namespaces = Get-Toc-Children -package $package -version $version -docRepoLocation "$PSScriptRoot/inputs" -folder $folder
+        $namespaces = Get-Toc-Children `
+            -package $package `
+            -docRepoLocation "$PSScriptRoot/inputs"
         $namespaces | Should -Be $expectNamespaces
-        Reset-File $targetFile $backupFile
     }
-    # No json case
-    It "Get toc children when no package json" -TestCases @(
-        @{ package = "Azure.Data.AppConfiguration"; version="1.3.0-beta.1"; folder="latest"; expectNamespaces = @('Azure.Data.AppConfiguration', 'Microsoft.Extensions.Azure') }
-    ) {
-        $namespaces = Get-Toc-Children -package $package -version $version -docRepoLocation "$PSScriptRoot/inputs" -folder $folder
-        $namespaces | Should -Be $expectNamespaces
-        $expectFile = "$PSScriptRoot/inputs/metadata/$folder/$package.json"
-        (Test-Path $expectFile) | Should -BeTrue
-        Remove-Item $expectFile
+
+    It "Combines ToC children from each moniker" { 
+        $namespaces = Get-Toc-Children `
+            -package 'Azure.Security.KeyVault.Certificates' `
+            -docRepoLocation "$PSScriptRoot/inputs"
+
+        $namespaces | Should -be @(
+            'Azure.Security.KeyVault.Certificates',
+            'Microsoft.Extensions.Azure'
+            'Some.New.Namespace'
+        )
     }
 }
diff --git a/eng/scripts/docs/tests/inputs/bundlepackages/azure-dotnet.csv b/eng/scripts/docs/tests/inputs/bundlepackages/azure-dotnet.csv
@@ -1,2 +1,2 @@
-azurekeyvaultkeys,[tfm=netstandard2.0]Azure.Security.KeyVault.Keys,4.4.0
+azuresecuritykeyvaultcertificates,[tfm=netstandard2.0]Azure.Security.KeyVault.Certificates,4.2.0
 azurekeyvaultsecrets,[tfm=netstandard2.0]Azure.Security.KeyVault.Secrets,4.4.0
diff --git a/eng/scripts/docs/tests/inputs/metadata/latest/Azure.Security.KeyVault.Certificates.json b/eng/scripts/docs/tests/inputs/metadata/latest/Azure.Security.KeyVault.Certificates.json
@@ -0,0 +1,18 @@
+{
+  "Name": "Azure.Security.KeyVault.Certificates",
+  "Version": "4.2.0",
+  "DevVersion": null,
+  "DirectoryPath": "sdk/keyvault/Azure.Security.KeyVault.Certificates",
+  "ServiceDirectory": "keyvault",
+  "ReadMePath": "sdk/keyvault/Azure.Security.KeyVault.Certificates/README.md",
+  "ChangeLogPath": "sdk/keyvault/Azure.Security.KeyVault.Certificates/CHANGELOG.md",
+  "Group": null,
+  "SdkType": "client",
+  "IsNewSdk": true,
+  "ArtifactName": "Azure.Security.KeyVault.Certificates",
+  "ReleaseStatus": "2022-09-20",
+  "Namespaces": [
+    "Azure.Security.KeyVault.Certificates",
+    "Microsoft.Extensions.Azure"
+  ]
+}
diff --git a/eng/scripts/docs/tests/inputs/metadata/latest/Azure.Security.KeyVault.Keys.json b/eng/scripts/docs/tests/inputs/metadata/latest/Azure.Security.KeyVault.Keys.json
diff --git a/eng/scripts/docs/tests/inputs/metadata/preview/Azure.Security.KeyVault.Certificates.json b/eng/scripts/docs/tests/inputs/metadata/preview/Azure.Security.KeyVault.Certificates.json
@@ -12,8 +12,6 @@
   "ArtifactName": "Azure.Security.KeyVault.Certificates",
   "ReleaseStatus": "2022-09-20",
   "Namespaces": [
-    "Azure.Security.KeyVault.Certificates",
-    "Microsoft.Extensions.Azure",
-    "OutOfDate.Namespaces"
+    "Some.New.Namespace"
   ]
 }