Sync eng/common directory with azure-sdk-tools repository for Tools P…

…R 926

eng/common/pipelines/templates/steps/publish-blobs.yml

@@ -4,6 +4,8 @@ parameters:
   TargetLanguage: ''
   BlobName: ''
   ScriptPath: ''
+  ArtifactLocation: ''
+  RepoId: $(Build.Repository.Name)
 
 steps:
 - pwsh: |
@@ -21,6 +23,8 @@ steps:
       -SASKey "${{ parameters.BlobSASKey }}"
       -Language "${{ parameters.TargetLanguage }}"
       -BlobName "${{ parameters.BlobName }}"
+      -PublicArtifactLocation "${{ parameters.ArtifactLocation }}"
+      -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master"
     pwsh: true
     workingDirectory: $(Pipeline.Workspace)
   displayName: Copy Docs to Blob

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -69,6 +69,7 @@ function ParseMavenPackage($pkg, $workingDirectory) {
     PackageId      = $pkgId
     GroupId        = $groupId
     PackageVersion = $pkgVersion
+    ReleaseTag     = "$($pkgId)_$($pkgVersion)"
     Deployable     = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/"))
     ReleaseNotes   = $releaseNotes
     ReadmeContent  = $readmeContent
@@ -150,6 +151,7 @@ function ParseNPMPackage($pkg, $workingDirectory) {
   $resultObj = New-Object PSObject -Property @{
     PackageId      = $pkgId
     PackageVersion = $pkgVersion
+    ReleaseTag     = "$($pkgId)_$($pkgVersion)"
     Deployable     = $forceCreate -or !(IsNPMPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
     ReleaseNotes   = $releaseNotes
     ReadmeContent  = $readmeContent
@@ -208,6 +210,7 @@ function ParseNugetPackage($pkg, $workingDirectory) {
   return New-Object PSObject -Property @{
     PackageId      = $pkgId
     PackageVersion = $pkgVersion
+    ReleaseTag     = "$($pkgId)_$($pkgVersion)"
     Deployable     = $forceCreate -or !(IsNugetPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
     ReleaseNotes   = $releaseNotes
     ReadmeContent  = $readmeContent
@@ -272,6 +275,7 @@ function ParsePyPIPackage($pkg, $workingDirectory) {
   return New-Object PSObject -Property @{
     PackageId      = $pkgId
     PackageVersion = $pkgVersion
+    ReleaseTag     = "$($pkgId)_$($pkgVersion)"
     Deployable     = $forceCreate -or !(IsPythonPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion)
     ReleaseNotes   = $releaseNotes
     ReadmeContent  = $readmeContent
@@ -300,6 +304,7 @@ function ParseCArtifact($pkg, $workingDirectory) {
   return New-Object PSObject -Property @{
     PackageId      = ''
     PackageVersion = $pkgVersion
+    ReleaseTag     = $pkgVersion
     # Artifact info is always considered deployable for C becasue it is not
     # deployed anywhere. Dealing with duplicate tags happens downstream in
     # CheckArtifactShaAgainstTagsList
@@ -331,6 +336,7 @@ function ParseCppArtifact($pkg, $workingDirectory) {
   return New-Object PSObject -Property @{
     PackageId      = $pkgName
     PackageVersion = $pkgVersion
+    ReleaseTag     = "$($pkgId)_$($pkgVersion)"
     # Artifact info is always considered deployable for now becasue it is not
     # deployed anywhere. Dealing with duplicate tags happens downstream in
     # CheckArtifactShaAgainstTagsList
@@ -387,52 +393,81 @@ function GetExistingTags($apiUrl) {
   }
 }
 
-# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
-function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha,  $continueOnError = $false) {
-  $pkgList = [array]@()
-  $ParsePkgInfoFn = ""
+# Retrieve release tag for artiface package. If multiple packages, then output the first one.
+function RetrieveReleaseTag($pkgRepository, $artifactLocation, $pkgName, $continueOnError = $true) {
+  if (!$artifactLocation) {
+    return ""
+  }
+  try {
+    $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation -pkgName $pkgName
+    if (!$pkgs -or !$pkgs[0]) {
+      return ""
+    }
+    $parsedPackage = &$parsePkgInfoFn -pkg $pkgs[0]
+    return $parsedPackage.ReleaseTag
+  }
+  catch {
+    if ($continueOnError) {
+      return ""
+    }
+    Write-Error "No release tag retrieved from $artifactLocation"
+  }
+}
+function RetrievePackages($pkgRepository, $artifactLocation, $pkgName) {
+  $parsePkgInfoFn = ""
   $packagePattern = ""
-
+  $pkgRepository = $pkgRepository.Trim()
   switch ($pkgRepository) {
     "Maven" {
-      $ParsePkgInfoFn = "ParseMavenPackage"
+      $parsePkgInfoFn = "ParseMavenPackage"
       $packagePattern = "*.pom"
       break
     }
     "Nuget" {
-      $ParsePkgInfoFn = "ParseNugetPackage"
+      $parsePkgInfoFn = "ParseNugetPackage"
       $packagePattern = "*.nupkg"
       break
     }
     "NPM" {
-      $ParsePkgInfoFn = "ParseNPMPackage"
+      $parsePkgInfoFn = "ParseNPMPackage"
       $packagePattern = "*.tgz"
       break
     }
     "PyPI" {
-      $ParsePkgInfoFn = "ParsePyPIPackage"
+      $parsePkgInfoFn = "ParsePyPIPackage"
       $packagePattern = "*.zip"
       break
     }
     "C" {
-      $ParsePkgInfoFn = "ParseCArtifact"
+      $parsePkgInfoFn = "ParseCArtifact"
       $packagePattern = "*.json"
     }
     "CPP" {
-      $ParsePkgInfoFn = "ParseCppArtifact"
+      $parsePkgInfoFn = "ParseCppArtifact"
       $packagePattern = "*.json"
     }
     default {
       Write-Host "Unrecognized Language: $language"
       exit(1)
     }
   }
+  $file_regex = if (!$pkgName) {
+    $packagePattern
+  } else {
+    "${pkgName}${packagePattern}"
+  }
+  $pkgs = Get-ChildItem -Path $artifactLocation -Include $file_regex -Recurse -File
+  return $pkgs, $parsePkgInfoFn
+}
 
-  $pkgs = (Get-ChildItem -Path $artifactLocation -Include $packagePattern -Recurse -File)
+# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
+function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha,  $continueOnError = $false) {
+  $pkgList = [array]@()
+  $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation
 
   foreach ($pkg in $pkgs) {
     try {
-      $parsedPackage = &$ParsePkgInfoFn -pkg $pkg -workingDirectory $workingDirectory
+      $parsedPackage = &$parsePkgInfoFn -pkg $pkg -workingDirectory $workingDirectory
 
       if ($parsedPackage -eq $null) {
         continue
@@ -444,17 +479,11 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
         exit(1)
       }
 
-      $tag = if ($parsedPackage.packageId) {
-        "$($parsedPackage.packageId)_$($parsedPackage.PackageVersion)"
-      } else {
-        $parsedPackage.PackageVersion
-      }
-
       $pkgList += New-Object PSObject -Property @{
         PackageId      = $parsedPackage.PackageId
         PackageVersion = $parsedPackage.PackageVersion
         GroupId        = $parsedPackage.GroupId
-        Tag            = $tag
+        Tag            = $parsedPackage.ReleaseTag
         ReleaseNotes   = $parsedPackage.ReleaseNotes
         ReadmeContent  = $parsedPackage.ReadmeContent
         IsPrerelease   = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease
@@ -511,4 +540,4 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap
     Write-Host "Tags already existing with different SHA versions. Exiting."
     exit(1)
   }
-}
+}

eng/common/scripts/copy-docs-to-blobstorage.ps1

@@ -7,8 +7,11 @@ param (
   $Language,
   $BlobName,
   $ExitOnError=1,
-  $UploadLatest=1
+  $UploadLatest=1,
+  $PublicArtifactLocation = "",
+  $RepoReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master"
 )
+. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
 
 $Language = $Language.ToLower()
 
@@ -186,7 +189,8 @@ function Upload-Blobs
     Param (
         [Parameter(Mandatory=$true)] [String]$DocDir,
         [Parameter(Mandatory=$true)] [String]$PkgName,
-        [Parameter(Mandatory=$true)] [String]$DocVersion
+        [Parameter(Mandatory=$true)] [String]$DocVersion,
+        [Parameter(Mandatory=$false)] [String]$ReleaseTag
     )
     #eg : $BlobName = "https://azuresdkdocs.blob.core.windows.net"
     $DocDest = "$($BlobName)/`$web/$($Language)"
@@ -196,7 +200,23 @@ function Upload-Blobs
     Write-Host "DocVersion $($DocVersion)"
     Write-Host "DocDir $($DocDir)"
     Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)"
+    Write-Host "Release Tag $($ReleaseTag)"
 
+    # Use the step to replace master link to release tag link 
+    if ($ReleaseTag) {
+        foreach ($htmlFile in (Get-ChildItem $DocDir -include *.html -r)) 
+        {
+            $fileContent = Get-Content -Path $htmlFile
+            $updatedFileContent = $fileContent -replace $RepoReplaceRegex, "`${1}$ReleaseTag"
+            if ($updatedFileContent -ne $fileContent) {
+                Set-Content -Path $htmlFile -Value $updatedFileContent
+            }
+        }
+    } 
+    else {
+        Write-Warning "Not able to do the master link replacement, since no release tag found for the release. Please manually check."
+    } 
+
     Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..."
     & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true
 
@@ -213,7 +233,6 @@ function Upload-Blobs
     }
 }
 
-
 if ($Language -eq "javascript")
 {
     $PublishedDocs = Get-ChildItem "$($DocLocation)/documentation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")}
@@ -227,7 +246,8 @@ if ($Language -eq "javascript")
         if($dirList.Length -eq 1){
             $DocVersion = $dirList[0].Name
             Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..."
-            Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion
+            $releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation $PkgName
+            Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag
         }
         else{
             Write-Host "found more than 1 folder under the documentation for package - $($Item.Name)"
@@ -252,7 +272,8 @@ if ($Language -eq "dotnet")
             Write-Host "DocDir $($Item)"
             Write-Host "PkgName $($PkgName)"
             Write-Host "DocVersion $($DocVersion)"
-            Upload-Blobs -DocDir "$($Item)" -PkgName $PkgName -DocVersion $DocVersion
+            $releaseTag = RetrieveReleaseTag "Nuget" $PublicArtifactLocation $PkgName
+            Upload-Blobs -DocDir "$($Item)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag
         }
         else
         {
@@ -279,8 +300,8 @@ if ($Language -eq "python")
         Write-Host "Discovered Package Name: $PkgName"
         Write-Host "Discovered Package Version: $Version"
         Write-Host "Directory for Upload: $UnzippedDocumentationPath"
-
-        Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version
+        $releaseTag = RetrieveReleaseTag "PyPI" $PublicArtifactLocation $PkgName
+        Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version -ReleaseTag $releaseTag
     }
 }
 
@@ -326,8 +347,8 @@ if ($Language -eq "java")
             Write-Host "DocDir $($UnjarredDocumentationPath)"
             Write-Host "PkgName $($ArtifactId)"
             Write-Host "DocVersion $($Version)"
-
-            Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version
+            $releaseTag = RetrieveReleaseTag "Maven" $PublicArtifactLocation $ArtifactId
+            Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version -ReleaseTag $releaseTag
 
         } Finally {
             if (![string]::IsNullOrEmpty($UnjarredDocumentationPath)) {
@@ -349,11 +370,13 @@ if ($Language -eq "c")
     # Those loops are left over from previous versions of this script which were
     # used to publish multiple docs packages in a single invocation.
     $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json
-    Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version
+    $releaseTag = RetrieveReleaseTag "C" $PublicArtifactLocation 
+    Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version -ReleaseTag $releaseTag
 }
 
 if ($Language -eq "cpp")
 {
     $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json)
-    Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version
-}
+    $releaseTag = RetrieveReleaseTag "CPP" $PublicArtifactLocation $packageInfo.name
+    Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -ReleaseTag $releaseTag
+}

eng/common/scripts/update-docs-metadata.ps1

@@ -17,6 +17,8 @@ param (
 . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1)
 . (Join-Path $PSScriptRoot SemVer.ps1)
 
+$releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)master"
+
 function GetMetaData($lang){
   switch ($lang) {
     "java" {
@@ -75,6 +77,10 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
     if ($headerContentMatches) {
       $foundTitle = $headerContentMatches.Matches[0]
       $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n"
+
+      # Replace github master link with release tag.
+      $ReplacementPattern = "`${1}$($pkgInfo.Tag)"
+      $fileContent = $fileContent -replace $releaseReplaceRegex, $ReplacementPattern
     }
 
     $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"