diff --git a/eng/common/TestResources/AzurePowerShellV4/Utility.ps1 b/eng/common/TestResources/AzurePowerShellV4/Utility.ps1
deleted file mode 100644
index 0bac797fb14b..000000000000
--- a/eng/common/TestResources/AzurePowerShellV4/Utility.ps1
+++ /dev/null
@@ -1,153 +0,0 @@
-# Copied from https://github.com/microsoft/azure-pipelines-tasks/blob/a1502bbe67561f5bec8402f32c997406f798a019/Tasks/AzurePowerShellV4/Utility.ps1
-
-function Get-SavedModulePath {
-    [CmdletBinding()]
-    param([string] $azurePowerShellVersion)
-    $savedModulePath = $($env:SystemDrive + "\Modules\az_" + $azurePowerShellVersion)
-    Write-Verbose "The value of the module path is: $savedModulePath"
-    return $savedModulePath 
-}
-
-function Get-SavedModulePathLinux {
-    [CmdletBinding()]
-    param([string] $azurePowerShellVersion)
-    $savedModulePath =  $("/usr/share/az_" + $azurePowerShellVersion)
-    Write-Verbose "The value of the module path is: $savedModulePath"
-    return $savedModulePath
-}
-
-function Update-PSModulePathForHostedAgent {
-    [CmdletBinding()]
-    param([string] $targetAzurePs)
-    try {
-        if ($targetAzurePs) {
-            $hostedAgentAzModulePath = Get-SavedModulePath -azurePowerShellVersion $targetAzurePs
-        }
-        else {
-            $hostedAgentAzModulePath = Get-LatestModule -patternToMatch "^az_[0-9]+\.[0-9]+\.[0-9]+$" -patternToExtract "[0-9]+\.[0-9]+\.[0-9]+$"
-        }
-        $env:PSModulePath = $hostedAgentAzModulePath + ";" + $env:PSModulePath
-        $env:PSModulePath = $env:PSModulePath.TrimStart(';') 
-    } finally {
-        Write-Verbose "The updated value of the PSModulePath is: $($env:PSModulePath)"
-    }
-}
-
-function Update-PSModulePathForHostedAgentLinux {
-    [CmdletBinding()]
-    param([string] $targetAzurePs)
-    try {
-        if ($targetAzurePs) {
-            $hostedAgentAzModulePath = Get-SavedModulePathLinux -azurePowerShellVersion $targetAzurePs
-            if(!(Test-Path $hostedAgentAzModulePath)) {
-                Write-Verbose "No module path found with this name"
-                throw ("Could not find the module path with given version.")
-            }
-        }
-        else {
-            $hostedAgentAzModulePath = Get-LatestModuleLinux -patternToMatch "^az_[0-9]+\.[0-9]+\.[0-9]+$" -patternToExtract "[0-9]+\.[0-9]+\.[0-9]+$"
-        }
-        $env:PSModulePath = $hostedAgentAzModulePath + ":" + $env:PSModulePath
-        $env:PSModulePath = $env:PSModulePath.TrimStart(':') 
-    } finally {
-        Write-Verbose "The updated value of the PSModulePath is: $($env:PSModulePath)"
-    }
-}
-
-function Get-LatestModule {
-    [CmdletBinding()]
-    param([string] $patternToMatch,
-          [string] $patternToExtract)
-    
-    $resultFolder = ""
-    $regexToMatch = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToMatch
-    $regexToExtract = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToExtract
-    $maxVersion = [version] "0.0.0"
-    $modulePath = $env:SystemDrive + "\Modules";
-
-    try {
-        if (-not (Test-Path -Path $modulePath)) {
-            return $resultFolder
-        }
-
-        $moduleFolders = Get-ChildItem -Directory -Path $modulePath | Where-Object { $regexToMatch.IsMatch($_.Name) }
-        foreach ($moduleFolder in $moduleFolders) {
-            $moduleVersion = [version] $($regexToExtract.Match($moduleFolder.Name).Groups[0].Value)
-            if($moduleVersion -gt $maxVersion) {
-                $modulePath = [System.IO.Path]::Combine($moduleFolder.FullName,"Az\$moduleVersion\Az.psm1")
-
-                if(Test-Path -LiteralPath $modulePath -PathType Leaf) {
-                    $maxVersion = $moduleVersion
-                    $resultFolder = $moduleFolder.FullName
-                } else {
-                    Write-Verbose "A folder matching the module folder pattern was found at $($moduleFolder.FullName) but didn't contain a valid module file"
-                }
-            }
-        }
-    }
-    catch {
-        Write-Verbose "Attempting to find the Latest Module Folder failed with the error: $($_.Exception.Message)"
-        $resultFolder = ""
-    }
-    Write-Verbose "Latest module folder detected: $resultFolder"
-    return $resultFolder
-}
-
-function Get-LatestModuleLinux {
-    [CmdletBinding()]
-    param([string] $patternToMatch,
-          [string] $patternToExtract)
-    
-    $resultFolder = ""
-    $regexToMatch = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToMatch
-    $regexToExtract = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToExtract
-    $maxVersion = [version] "0.0.0"
-
-    try {
-        $moduleFolders = Get-ChildItem -Directory -Path $("/usr/share") | Where-Object { $regexToMatch.IsMatch($_.Name) }
-        foreach ($moduleFolder in $moduleFolders) {
-            $moduleVersion = [version] $($regexToExtract.Match($moduleFolder.Name).Groups[0].Value)
-            if($moduleVersion -gt $maxVersion) {
-                $modulePath = [System.IO.Path]::Combine($moduleFolder.FullName,"Az/$moduleVersion/Az.psm1")
-
-                if(Test-Path -LiteralPath $modulePath -PathType Leaf) {
-                    $maxVersion = $moduleVersion
-                    $resultFolder = $moduleFolder.FullName
-                } else {
-                    Write-Verbose "A folder matching the module folder pattern was found at $($moduleFolder.FullName) but didn't contain a valid module file"
-                }
-            }
-        }
-    }
-    catch {
-        Write-Verbose "Attempting to find the Latest Module Folder failed with the error: $($_.Exception.Message)"
-        $resultFolder = ""
-    }
-    Write-Verbose "Latest module folder detected: $resultFolder"
-    return $resultFolder
-}
-
-function CleanUp-PSModulePathForHostedAgent {
-    # Clean up PSModulePath for hosted agent
-    $azureRMModulePath = "C:\Modules\azurerm_2.1.0"
-    $azureModulePath = "C:\Modules\azure_2.1.0"
-    $azPSModulePath = $env:PSModulePath
-
-    if ($azPSModulePath.split(";") -contains $azureRMModulePath) {
-        $azPSModulePath = (($azPSModulePath).Split(";") | ? { $_ -ne $azureRMModulePath }) -join ";"
-        write-verbose "$azureRMModulePath removed. Restart the prompt for the changes to take effect."
-    }
-    else {
-        write-verbose "$azureRMModulePath is not present in $azPSModulePath"
-    }
-
-    if ($azPSModulePath.split(";") -contains $azureModulePath) {
-        $azPSModulePath = (($azPSModulePath).Split(";") | ? { $_ -ne $azureModulePath }) -join ";"
-        write-verbose "$azureModulePath removed. Restart the prompt for the changes to take effect."
-    }
-    else {
-        write-verbose "$azureModulePath is not present in $azPSModulePath"
-    }
-
-    $env:PSModulePath = $azPSModulePath
-}
diff --git a/eng/common/TestResources/Import-AzModules.ps1 b/eng/common/TestResources/Import-AzModules.ps1
deleted file mode 100644
index 3cd7b3124489..000000000000
--- a/eng/common/TestResources/Import-AzModules.ps1
+++ /dev/null
@@ -1,11 +0,0 @@
-. "$PSScriptRoot/AzurePowerShellV4/Utility.ps1"
-
-if ($IsWindows) {
-    # Copied from https://github.com/microsoft/azure-pipelines-tasks/blob/9cc8e1b3ee37dc023c81290de1dd522b77faccf7/Tasks/AzurePowerShellV4/AzurePowerShell.ps1#L57-L58
-    CleanUp-PSModulePathForHostedAgent
-    Update-PSModulePathForHostedAgent
-}
-else {
-    # Copied from https://github.com/microsoft/azure-pipelines-tasks/blob/9cc8e1b3ee37dc023c81290de1dd522b77faccf7/Tasks/AzurePowerShellV4/InitializeAz.ps1#L16
-    Update-PSModulePathForHostedAgentLinux
-}
diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml
index a9d693b99420..ac05e14c7934 100644
--- a/eng/common/TestResources/deploy-test-resources.yml
+++ b/eng/common/TestResources/deploy-test-resources.yml
@@ -33,10 +33,12 @@ parameters:
 
 
 steps:
-  - template: /eng/common/TestResources/setup-az-modules.yml
+  - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml
+
+  - template: /eng/common/TestResources/setup-environments.yml
 
   - pwsh: |
-      eng/common/TestResources/Import-AzModules.ps1
+      eng/common/scripts/Import-AzModules.ps1
 
       $subscriptionConfiguration = @'
         ${{ parameters.SubscriptionConfiguration }}
diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml
index a0871a2ff7ad..0c059f51a032 100644
--- a/eng/common/TestResources/remove-test-resources.yml
+++ b/eng/common/TestResources/remove-test-resources.yml
@@ -21,7 +21,7 @@ parameters:
 
 steps:
   - pwsh: |
-      eng/common/TestResources/Import-AzModules.ps1
+      eng/common/scripts/Import-AzModules.ps1
 
       $subscriptionConfiguration = @"
         ${{ parameters.SubscriptionConfiguration }}
diff --git a/eng/common/TestResources/setup-az-modules.yml b/eng/common/TestResources/setup-environments.yml
similarity index 92%
rename from eng/common/TestResources/setup-az-modules.yml
rename to eng/common/TestResources/setup-environments.yml
index b5369f433b90..ef400bcb1d12 100644
--- a/eng/common/TestResources/setup-az-modules.yml
+++ b/eng/common/TestResources/setup-environments.yml
@@ -21,12 +21,12 @@ steps:
     condition: contains(variables['OSVmImage'], 'mac')
 
   - task: Powershell@2
+    displayName: Register Dogfood environment
     inputs:
-      displayName: Register Dogfood environment
       targetType: inline
       pwsh: true
       script: |
-        eng/common/TestResources/Import-AzModules.ps1
+        eng/common/scripts/Import-AzModules.ps1
 
         $environmentSpec = @"
           $(env-config-dogfood)
diff --git a/eng/common/pipelines/templates/steps/cache-ps-modules.yml b/eng/common/pipelines/templates/steps/cache-ps-modules.yml
new file mode 100644
index 000000000000..563dba40294f
--- /dev/null
+++ b/eng/common/pipelines/templates/steps/cache-ps-modules.yml
@@ -0,0 +1,10 @@
+steps:
+- pwsh: |
+    . ./eng/common/scripts/Helpers/PSModule-Helpers.ps1
+    Write-Host "##vso[task.setvariable variable=CachedPSModulePath]$global:CurrentUserModulePath"
+  displayName: Set PS Modules Cache Directory
+- task: Cache@2
+  inputs:
+    key: 'PSModulePath | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/common/scripts/Import-AzModules.ps1'
+    path: $(CachedPSModulePath)
+  displayName: Cache PS Modules
\ No newline at end of file
diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1
new file mode 100644
index 000000000000..96f34ff71f4a
--- /dev/null
+++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1
@@ -0,0 +1,97 @@
+$DefaultPSRepositoryUrl = "https://www.powershellgallery.com/api/v2"
+$global:CurrentUserModulePath = ""
+
+function Update-PSModulePath()
+{
+  # Information on PSModulePath taken from docs
+  # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_psmodulepath
+
+  # Information on Az custom module paths on hosted agents taken from
+  # https://github.com/microsoft/azure-pipelines-tasks/blob/c9771bc064cd60f47587c68e5c871b7cd13f0f28/Tasks/AzurePowerShellV5/Utility.ps1
+
+  if ($IsWindows) {
+    $hostedAgentModulePath = $env:SystemDrive + "\Modules"
+    $moduleSeperator = ";"
+  } else {
+    $hostedAgentModulePath = "/usr/share"
+    $moduleSeperator = ":"
+  }
+  $modulePaths = $env:PSModulePath -split $moduleSeperator
+
+  # Remove any hosted agent paths (needed to remove old default azure/azurerm paths which cause conflicts)
+  $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) })
+
+  # Add any "az_" paths from the agent which is the lastest set of azure modules
+  $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator
+  if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) {
+    $modulePaths += $AzModuleCachPath
+  }
+
+  $env:PSModulePath = $modulePaths -join $moduleSeperator
+
+  # Find the path that is under user home directory
+  $homeDirectories = $modulePaths.Where({ $_.StartsWith($home) })
+  if ($homeDirectories.Count -gt 0) {
+    $global:CurrentUserModulePath = $homeDirectories[0]
+    if ($homeDirectories.Count -gt 1) {
+      Write-Verbose "Found more then one module path starting with $home so selecting the first one $global:CurrentUserModulePath"
+    }
+
+    # In some cases the directory might not exist so we need to create it otherwise caching an empty directory will fail
+    if (!(Test-Path $global:CurrentUserModulePath)) {
+      New-Item $global:CurrentUserModulePath -ItemType Directory > $null
+    }
+  }
+  else {
+    Write-Error "Did not find a module path starting with $home to set up a user module path in $env:PSModulePath"
+  }
+}
+
+# If we want to use another default repository other then PSGallery we can update the default parameters
+function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $DefaultPSRepositoryUrl)
+{
+  # Check installed modules
+  $modules = (Get-Module -ListAvailable $moduleName)
+  if ($version -as [Version]) {
+    $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version })
+  }
+
+  if ($modules.Count -eq 0)
+  {
+    $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl })
+    if ($repositories.Count -eq 0)
+    {
+      Register-PSRepository -Name $repositoryUrl -SourceLocation $repositoryUrl -InstallationPolicy Trusted
+      $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl })
+      if ($repositories.Count -eq 0) {
+        Write-Error "Failed to registory package repository $repositoryUrl."
+        return
+      }
+    }
+    $repository = $repositories[0]
+
+    if ($repository.InstallationPolicy -ne "Trusted") {
+      Set-PSRepository -Name $repository.Name -InstallationPolicy "Trusted"
+    }
+
+    Write-Host "Installing module $moduleName with min version $version from $repositoryUrl"
+    # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching
+    Install-Module $moduleName -MinimumVersion $version -Repository $repository.Name -Scope CurrentUser -Force
+
+    # Ensure module installed
+    $modules = (Get-Module -ListAvailable $moduleName)
+    if ($version -as [Version]) {
+      $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version })
+    }
+
+    if ($modules.Count -eq 0) {
+      Write-Error "Failed to install module $moduleName with version $version"
+      return
+    }
+  }
+
+  Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)."
+  return $modules[0]
+}
+
+Update-PSModulePath
\ No newline at end of file
diff --git a/eng/common/scripts/Import-AzModules.ps1 b/eng/common/scripts/Import-AzModules.ps1
new file mode 100644
index 000000000000..6e97ba970289
--- /dev/null
+++ b/eng/common/scripts/Import-AzModules.ps1
@@ -0,0 +1,8 @@
+[CmdletBinding()]
+param (
+  [string]$AzModuleVersion = "5.7.0" # Current version cached on agents
+)
+
+. (Join-Path $PSScriptRoot Helpers PSModule-Helpers.ps1)
+
+Install-ModuleIfNotInstalled "Az" $AzModuleVersion | Import-Module
\ No newline at end of file