Enable Caching of PS Modules (Azure#18991)

- Remove copied AzPowershell utilities
- Add latest AZ module path already on hosted agents to PSModulePath
- Rename setup-az-modules template setup-environments to reflect what is is doing
- Add support for Caching the current user PS Module folder
- Add support for install-module if not already present in module folder
- Organize the live test clean-up script to be in the standard location

Co-authored-by: Wes Haggard <Wes.Haggard@microsoft.com>

eng/common/TestResources/deploy-test-resources.yml

@@ -33,10 +33,12 @@ parameters:
 
 
 steps:
-  - template: /eng/common/TestResources/setup-az-modules.yml
+  - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml
+
+  - template: /eng/common/TestResources/setup-environments.yml
 
   - pwsh: |
-      eng/common/TestResources/Import-AzModules.ps1
+      eng/common/scripts/Import-AzModules.ps1
 
       $subscriptionConfiguration = @'
         ${{ parameters.SubscriptionConfiguration }}

eng/common/TestResources/remove-test-resources.yml

@@ -21,7 +21,7 @@ parameters:
 
 steps:
   - pwsh: |
-      eng/common/TestResources/Import-AzModules.ps1
+      eng/common/scripts/Import-AzModules.ps1
 
       $subscriptionConfiguration = @"
         ${{ parameters.SubscriptionConfiguration }}

eng/common/TestResources/setup-az-modules.yml → eng/common/TestResources/setup-environments.yml

@@ -21,12 +21,12 @@ steps:
     condition: contains(variables['OSVmImage'], 'mac')
 
   - task: Powershell@2
+    displayName: Register Dogfood environment
     inputs:
-      displayName: Register Dogfood environment
       targetType: inline
       pwsh: true
       script: |
-        eng/common/TestResources/Import-AzModules.ps1
+        eng/common/scripts/Import-AzModules.ps1
 
         $environmentSpec = @"
           $(env-config-dogfood)

eng/common/pipelines/templates/steps/cache-ps-modules.yml

@@ -0,0 +1,10 @@
+steps:
+- pwsh: |
+    . ./eng/common/scripts/Helpers/PSModule-Helpers.ps1
+    Write-Host "##vso[task.setvariable variable=CachedPSModulePath]$global:CurrentUserModulePath"
+  displayName: Set PS Modules Cache Directory
+- task: Cache@2
+  inputs:
+    key: 'PSModulePath | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/common/scripts/Import-AzModules.ps1'
+    path: $(CachedPSModulePath)
+  displayName: Cache PS Modules

eng/common/scripts/Helpers/PSModule-Helpers.ps1

@@ -0,0 +1,97 @@
+$DefaultPSRepositoryUrl = "https://www.powershellgallery.com/api/v2"
+$global:CurrentUserModulePath = ""
+
+function Update-PSModulePath()
+{
+  # Information on PSModulePath taken from docs
+  # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_psmodulepath
+
+  # Information on Az custom module paths on hosted agents taken from
+  # https://github.com/microsoft/azure-pipelines-tasks/blob/c9771bc064cd60f47587c68e5c871b7cd13f0f28/Tasks/AzurePowerShellV5/Utility.ps1
+
+  if ($IsWindows) {
+    $hostedAgentModulePath = $env:SystemDrive + "\Modules"
+    $moduleSeperator = ";"
+  } else {
+    $hostedAgentModulePath = "/usr/share"
+    $moduleSeperator = ":"
+  }
+  $modulePaths = $env:PSModulePath -split $moduleSeperator
+
+  # Remove any hosted agent paths (needed to remove old default azure/azurerm paths which cause conflicts)
+  $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) })
+
+  # Add any "az_" paths from the agent which is the lastest set of azure modules
+  $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
+  if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) {
+    $modulePaths += $AzModuleCachPath
+  }
+
+  $env:PSModulePath = $modulePaths -join $moduleSeperator
+
+  # Find the path that is under user home directory
+  $homeDirectories = $modulePaths.Where({ $_.StartsWith($home) })
+  if ($homeDirectories.Count -gt 0) {
+    $global:CurrentUserModulePath = $homeDirectories[0]
+    if ($homeDirectories.Count -gt 1) {
+      Write-Verbose "Found more then one module path starting with $home so selecting the first one $global:CurrentUserModulePath"
+    }
+
+    # In some cases the directory might not exist so we need to create it otherwise caching an empty directory will fail
+    if (!(Test-Path $global:CurrentUserModulePath)) {
+      New-Item $global:CurrentUserModulePath -ItemType Directory > $null
+    }
+  }
+  else {
+    Write-Error "Did not find a module path starting with $home to set up a user module path in $env:PSModulePath"
+  }
+}
+
+# If we want to use another default repository other then PSGallery we can update the default parameters
+function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $DefaultPSRepositoryUrl)
+{
+  # Check installed modules
+  $modules = (Get-Module -ListAvailable $moduleName)
+  if ($version -as [Version]) {
+    $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version })
+  }
+
+  if ($modules.Count -eq 0)
+  {
+    $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl })
+    if ($repositories.Count -eq 0)
+    {
+      Register-PSRepository -Name $repositoryUrl -SourceLocation $repositoryUrl -InstallationPolicy Trusted
+      $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl })
+      if ($repositories.Count -eq 0) {
+        Write-Error "Failed to registory package repository $repositoryUrl."
+        return
+      }
+    }
+    $repository = $repositories[0]
+
+    if ($repository.InstallationPolicy -ne "Trusted") {
+      Set-PSRepository -Name $repository.Name -InstallationPolicy "Trusted"
+    }
+
+    Write-Host "Installing module $moduleName with min version $version from $repositoryUrl"
+    # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching
+    Install-Module $moduleName -MinimumVersion $version -Repository $repository.Name -Scope CurrentUser -Force
+
+    # Ensure module installed
+    $modules = (Get-Module -ListAvailable $moduleName)
+    if ($version -as [Version]) {
+      $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version })
+    }
+
+    if ($modules.Count -eq 0) {
+      Write-Error "Failed to install module $moduleName with version $version"
+      return
+    }
+  }
+
+  Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)."
+  return $modules[0]
+}
+
+Update-PSModulePath

eng/common/scripts/Import-AzModules.ps1

@@ -0,0 +1,8 @@
+[CmdletBinding()]
+param (
+  [string]$AzModuleVersion = "5.7.0" # Current version cached on agents
+)
+
+. (Join-Path $PSScriptRoot Helpers PSModule-Helpers.ps1)
+
+Install-ModuleIfNotInstalled "Az" $AzModuleVersion | Import-Module