Update devops workitem helpers (#938)

- Switch to using rest instead of cli for querying work items as cli is limited to 1000 items only. - Fix issues with Generated fields not being set. - Correctly sort older workitems by version isntead of string. Co-authored-by: Wes Haggard <Wes.Haggard@microsoft.com>
Azure · Jun 30, 2021 · e4146fd · e4146fd
1 parent 2190d83
commit e4146fd
Showing 1 changed file with 105 additions and 36 deletions.
diff --git a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1
@@ -12,6 +12,77 @@ function Invoke-AzBoardsCmd($subCmd, $parameters, $output = $true)
   return Invoke-Expression "$azCmdStr" | ConvertFrom-Json -AsHashTable
 }
 
+function Invoke-Query($fields, $wiql, $output = $true)
+{
+  #POST https://dev.azure.com/{organization}/{project}/{team}/_apis/wit/wiql?timePrecision={timePrecision}&$top={$top}&api-version=6.1-preview.2
+
+  $body = @"
+{
+  "query": "$wiql"
+}
+"@
+
+  if ($output) {
+    Write-Host "Executing query $wiql"
+  }
+
+  $headers = $null
+  if (Get-Variable -Name "devops_pat" -ValueOnly -ErrorAction "Ignore")
+  {
+    $encodedToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes([string]::Format("{0}:{1}", "", $devops_pat)))
+    $headers = @{ Authorization = "Basic $encodedToken" }
+  }
+  else
+  {
+    # Get a temp access token from the logged in az cli user for azure devops resource
+    $jwt_accessToken = (az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv)
+    $headers = @{ Authorization = "Bearer $jwt_accessToken" }
+  }
+  $response = Invoke-RestMethod -Method POST `
+    -Uri "https://dev.azure.com/azure-sdk/Release/_apis/wit/wiql/?`$top=10000&api-version=6.0" `
+    -Headers $headers -Body $body -ContentType "application/json" | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashTable
+
+  if (!$response.workItems) {
+    Write-Verbose "Query returned no items. $wiql"
+    return ,@()
+  }
+
+  $workItems = @()
+  $i = 0
+  do
+  {
+    $idBatch = @()
+    while ($idBatch.Count -lt 200 -and $i -lt $response.workItems.Count)
+    {
+      $idBatch += $response.workItems[$i].id
+      $i++
+    }
+
+    $uri = "https://dev.azure.com/azure-sdk/Release/_apis/wit/workitems?ids=$($idBatch -join ',')&fields=$($fields -join ',')&api-version=6.0"
+
+    Write-Verbose "Pulling work items $uri "
+
+    $batchResponse = Invoke-RestMethod -Method GET -Uri $uri `
+      -Headers $headers -ContentType "application/json" -MaximumRetryCount 3 | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashTable
+
+      if ($batchResponse.value)
+      {
+        $batchResponse.value | % { $workItems += $_ }
+      }
+      else
+      {
+        Write-Warning "Batch return no items from $uri"
+      }
+  }
+  while ($i -lt $response.workItems.Count)
+
+  if ($output) {
+    Write-Host "Query return $($workItems.Count) items"
+  }
+
+  return $workItems
+}
+
 function LoginToAzureDevops([string]$devops_pat)
 {
   if (!$devops_pat) {
@@ -64,13 +135,14 @@ function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand =
     $serviceCondition = "[ServiceName] <> ''"
   }
 
-  $parameters = $ReleaseDevOpsCommonParametersWithProject
-  $parameters += "--wiql"
-  $parameters += "`"SELECT [ID], [ServiceName], [PackageDisplayName], [Parent] FROM WorkItems WHERE [Work Item Type] = 'Epic' AND ${serviceCondition}`""
+  $query = "SELECT [ID], [ServiceName], [PackageDisplayName], [Parent] FROM WorkItems WHERE [Work Item Type] = 'Epic' AND ${serviceCondition}"
 
-  $workItems = Invoke-AzBoardsCmd "query" $parameters $outputCommand
+  $fields = @("System.Id", "Custom.ServiceName", "Custom.PackageDisplayName", "System.Parent")
 
-  foreach ($wi in $workItems) {
+  $workItems = Invoke-Query $fields $query $outputCommand
+
+  foreach ($wi in $workItems)
+  {
     $localKey = BuildHashKey $wi.fields["Custom.ServiceName"] $wi.fields["Custom.PackageDisplayName"]
     if (!$localKey) { continue }
     if ($parentWorkItems.ContainsKey($localKey) -and $parentWorkItems[$localKey].id -ne $wi.id) {
@@ -107,9 +179,7 @@ function FindLatestPackageWorkItem($lang, $packageName, $outputCommand = $true)
       continue
     }
 
-    # Note this only does string sorting which is enough for our current usages
-    # if we need absolute sorting at some point we would need to parse these versions
-    if ($wi.fields["Custom.PackageVersionMajorMinor"] -gt $latestWI.fields["Custom.PackageVersionMajorMinor"]) {
+    if (($wi.fields["Custom.PackageVersionMajorMinor"] -as [Version]) -gt ($latestWI.fields["Custom.PackageVersionMajorMinor"] -as [Version])) {
       $latestWI = $wi
     }
   }
@@ -124,26 +194,26 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr
   }
 
   $fields = @()
-  $fields += "ID"
-  $fields += "State"
+  $fields += "System.ID"
+  $fields += "System.State"
   $fields += "System.AssignedTo"
-  $fields += "Parent"
-  $fields += "Language"
-  $fields += "Package"
-  $fields += "PackageDisplayName"
-  $fields += "Title"
-  $fields += "PackageType"
-  $fields += "PackageTypeNewLibrary"
-  $fields += "PackageVersionMajorMinor"
-  $fields += "PackageRepoPath"
-  $fields += "ServiceName"
-  $fields += "Planned Packages"
-  $fields += "Shipped Packages"
-  $fields += "PackageBetaVersions"
-  $fields += "PackageGAVersion"
-  $fields += "PackagePatchVersions"
-  $fields += "Generated"
-  $fields += "RoadmapState"
+  $fields += "System.Parent"
+  $fields += "Custom.Language"
+  $fields += "Custom.Package"
+  $fields += "Custom.PackageDisplayName"
+  $fields += "System.Title"
+  $fields += "Custom.PackageType"
+  $fields += "Custom.PackageTypeNewLibrary"
+  $fields += "Custom.PackageVersionMajorMinor"
+  $fields += "Custom.PackageRepoPath"
+  $fields += "Custom.ServiceName"
+  $fields += "Custom.PlannedPackages"
+  $fields += "Custom.ShippedPackages"
+  $fields += "Custom.PackageBetaVersions"
+  $fields += "Custom.PackageGAVersion"
+  $fields += "Custom.PackagePatchVersions"
+  $fields += "Custom.Generated"
+  $fields += "Custom.RoadmapState"
 
   $fieldList = ($fields | ForEach-Object { "[$_]"}) -join ", "
   $query = "SELECT ${fieldList} FROM WorkItems WHERE [Work Item Type] = 'Package'"
@@ -160,14 +230,8 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr
   if ($version) {
     $query += " AND [PackageVersionMajorMinor] = '${version}'"
   }
-  $parameters = $ReleaseDevOpsCommonParametersWithProject
-  $parameters += "--wiql", "`"${query}`""
-
-  $workItems = Invoke-AzBoardsCmd "query" $parameters $outputCommand
 
-  if ($workItems -and $workItems.Count -eq 1000) {
-    Write-Warning "Retrieved the max of 1000 items so item list might not be complete."
-  }
+  $workItems = Invoke-Query $fields $query $outputCommand
 
   foreach ($wi in $workItems)
   {
@@ -316,8 +380,13 @@ function FindOrCreateClonePackageWorkItem($lang, $pkg, $verMajorMinor, $allowPro
         $pkg.RepoPath = $pkg.fields["Custom.PackageRepoPath"]
       }
 
-      $extraFields += "`"Generated=" + $latestVersionItem.fields["Custom.Generated"] + "`""
-      $extraFields += "`"RoadmapState=" +  $latestVersionItem.fields["Custom.RoadmapState"] + "`""
+      if ($latestVersionItem.fields["Custom.Generated"]) {
+        $extraFields += "`"Generated=" + $latestVersionItem.fields["Custom.Generated"] + "`""
+      }
+
+      if ($latestVersionItem.fields["Custom.RoadmapState"]) {
+        $extraFields += "`"RoadmapState=" +  $latestVersionItem.fields["Custom.RoadmapState"] + "`""
+      }
     }
 
     if ($allowPrompt) {