Sync eng/common directory with azure-sdk-tools for PR 1912 (#20340)

* Attempt to purge all vaults, managed HSMs Reverts #1910. Vaults and managed HSMs are automatically purged on their purge date. The point was to purge them daily to preserve capacity. The default purge date is +90 days. * Add timeout and more logging * Pass required -Resource * Fix log message * Ensure the $Resource is correctly captured Added comment to new code explaining why, since ScriptBlock.GetNewClosure() is not working as expected. * Add -ErrorAction to Receive-Job Worked without terminating when run locally, but failed on the first error in the AzDO agent. * Use $using:r instead of creating ScriptBlock More idiomatic for passing ScriptBlocks to jobs. * Resolve PR feedback * Change default DeleteAfterHours to 120 Resolves #1917 * Use the Az cmdlets built-in -AsJob Co-authored-by: Heath Stewart <heaths@microsoft.com>
Azure · Sep 29, 2021 · 1f88942 · 1f88942
1 parent 54cf263
commit 1f88942
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 43 deletions.
diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1
@@ -50,7 +50,7 @@ param (
 
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
-    [int] $DeleteAfterHours = 48,
+    [int] $DeleteAfterHours = 120,
 
     [Parameter()]
     [string] $Location = '',

diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md
@@ -307,7 +307,7 @@ Aliases:
 
 Required: False
 Position: Named
-Default value: 48
+Default value: 120
 Accept pipeline input: False
 Accept wildcard characters: False
 ```

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -7,48 +7,42 @@ function Get-PurgeableGroupResources {
   )
   $purgeableResources = @()
 
+  # Discover Managed HSMs first since they are a premium resource.
+  Write-Verbose "Retrieving deleted Managed HSMs from resource group $ResourceGroupName"
+
+  # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
+  $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru `
+    | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
+
+  if ($deletedHsms) {
+    Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
+    $purgeableResources += $deletedHsms
+  }
+
   Write-Verbose "Retrieving deleted Key Vaults from resource group $ResourceGroupName"
 
   # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled.
   $deletedKeyVaults = Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object {
     # Enumerating vaults from a resource group does not return all properties we required.
     Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } `
-      | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
-  }
+      | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
+      | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
+    }
 
   if ($deletedKeyVaults) {
     Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
     $purgeableResources += $deletedKeyVaults
   }
 
-  Write-Verbose "Retrieving deleted Managed HSMs from resource group $ResourceGroupName"
-
-  # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
-  $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
-
-  if ($deletedHsms) {
-    Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
-    $purgeableResources += $deletedHsms
-  }
-
   return $purgeableResources
 }
+
 function Get-PurgeableResources {
   $purgeableResources = @()
   $subscriptionId = (Get-AzContext).Subscription.Id
 
-  Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
-
-  # Get deleted Key Vaults for the current subscription.
-  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
-
-  if ($deletedKeyVaults) {
-    Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
-    $purgeableResources += $deletedKeyVaults
-  }
-
+  # Discover Managed HSMs first since they are a premium resource.
   Write-Verbose "Retrieving deleted Managed HSMs from subscription $subscriptionId"
 
   # Get deleted Managed HSMs for the current subscription.
@@ -60,6 +54,7 @@ function Get-PurgeableResources {
     foreach ($r in $content.value) {
       $deletedHsms += [pscustomobject] @{
         AzsdkResourceType = 'Managed HSM'
+        AzsdkName = $r.name
         Id = $r.id
         Name = $r.name
         Location = $r.properties.location
@@ -75,6 +70,18 @@ function Get-PurgeableResources {
     }
   }
 
+  Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
+
+  # Get deleted Key Vaults for the current subscription.
+  $deletedKeyVaults = Get-AzKeyVault -InRemovedState `
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
+    | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
+
+  if ($deletedKeyVaults) {
+    Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
+    $purgeableResources += $deletedKeyVaults
+  }
+
   return $purgeableResources
 }
 
@@ -83,7 +90,14 @@ function Get-PurgeableResources {
 filter Remove-PurgeableResources {
   param (
     [Parameter(Position=0, ValueFromPipeline=$true)]
-    [object[]] $Resource
+    [object[]] $Resource,
+
+    [Parameter()]
+    [ValidateRange(1, [int]::MaxValue)]
+    [int] $Timeout = 30,
+
+    [Parameter()]
+    [switch] $PassThru
   )
 
   if (!$Resource) {
@@ -93,38 +107,43 @@ filter Remove-PurgeableResources {
   $subscriptionId = (Get-AzContext).Subscription.Id
 
   foreach ($r in $Resource) {
+    Log "Attempting to purge $($r.AzsdkResourceType) '$($r.AzsdkName)'"
     switch ($r.AzsdkResourceType) {
       'Key Vault' {
-        Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'"
         if ($r.EnablePurgeProtection) {
-          # We will try anyway but will ignore errors
+          # We will try anyway but will ignore errors.
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue
+        # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior.
+        Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob `
+          | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru
       }
 
       'Managed HSM' {
-        Log "Attempting to purge $($r.AzsdkResourceType) '$($r.Name)'"
         if ($r.EnablePurgeProtection) {
-          # We will try anyway but will ignore errors
+          # We will try anyway but will ignore errors.
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
-        if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-          Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
-        } elseif ($response.Content) {
-          $content = $response.Content | ConvertFrom-Json
-          if ($content.error) {
-            $err = $content.error
-            Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
-          }
-        }
+        # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured.
+        Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore -AsJob `
+          | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action {
+              param ( $response )
+              if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+                Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+              } elseif ($response.Content) {
+                $content = $response.Content | ConvertFrom-Json
+                if ($content.error) {
+                  $err = $content.error
+                  Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+                }
+              }
+            }.GetNewClosure()
       }
 
       default {
-        Write-Warning "Cannot purge resource type $($r.AzsdkResourceType). Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
+        Write-Warning "Cannot purge $($r.AzsdkResourceType) '$($r.AzsdkName)'. Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
       }
     }
   }
@@ -134,3 +153,43 @@ filter Remove-PurgeableResources {
 function Log($Message) {
   Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
+
+function Wait-PurgeableResourceJob {
+  param (
+    [Parameter(Mandatory=$true, ValueFromPipeline=$true)]
+    $Job,
+
+    # The resource is used for logging and to return if `-PassThru` is specified
+    # so we can easily see all resources that may be in a bad state when the script has completed.
+    [Parameter(Mandatory=$true)]
+    $Resource,
+
+    # Optional ScriptBlock should define params corresponding to the associated job's `Output` property.
+    [Parameter()]
+    [scriptblock] $Action,
+
+    [Parameter()]
+    [ValidateRange(1, [int]::MaxValue)]
+    [int] $Timeout = 30,
+
+    [Parameter()]
+    [switch] $PassThru
+  )
+
+  $null = Wait-Job -Job $Job -Timeout $Timeout
+
+  if ($Job.State -eq 'Completed' -or $Job.State -eq 'Failed') {
+    $result = Receive-Job -Job $Job -ErrorAction Continue
+
+    if ($Action) {
+      $null = $Action.Invoke($result)
+    }
+  } else {
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job."
+    $Job.Cancel()
+
+    if ($PassThru) {
+      $Resource
+    }
+  }
+}