From 4e951cd5330e71d9c0ec7663a53b076217e78ed9 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Fri, 25 Sep 2020 10:29:28 -0700 Subject: [PATCH 1/4] Added post deployment step to create SAS with additional permission --- sdk/storage/test-resources-post.ps1 | 40 ++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/sdk/storage/test-resources-post.ps1 b/sdk/storage/test-resources-post.ps1 index 9a46733dc9c6..6b19fffa8425 100644 --- a/sdk/storage/test-resources-post.ps1 +++ b/sdk/storage/test-resources-post.ps1 @@ -44,4 +44,42 @@ Write-Verbose "CORS rule set for $storageAccountName" $datalakeStorageAccountName = $DeploymentOutputs['DFS_ACCOUNT_NAME'] $context = New-AzStorageContext -StorageAccountName $datalakeStorageAccountName Set-AzStorageCORSRule -ServiceType 'Blob' -CorsRules $corsRules -Context $context -Write-Verbose "CORS rule set for $datalakeStorageAccountName" \ No newline at end of file +Write-Verbose "CORS rule set for $datalakeStorageAccountName" + +# Run any post deployment script and set any additional keys to set in Env +$AdditionalEnvKeys = @{} + +# Create SAS for storage account with additional permissions that are not supported by SRP deployment +$storageAccount = $DeploymentOutputs['ACCOUNT_NAME'] +Write-Host "Creating SAS for storage account $storageAccount" +$storageContext = New-AzStorageContext -StorageAccountName $storageAccount -StorageAccountKey $DeploymentOutputs['ACCOUNT_KEY'] +$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $storageContext +$AdditionalEnvKeys["ACCOUNT_SAS"] = $storageSas + +# Try to detect the shell based on the parent process name (e.g. launch via shebang). +$shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { + 'cmd', 'set {0}={1}' +} +elseif (@('bash', 'csh', 'tcsh', 'zsh') -contains $parentProcessName) { + 'shell', 'export {0}={1}' +} +else { + 'PowerShell', '$env:{0} = ''{1}''' +} + +$CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) +foreach ($key in $AdditionalEnvKeys.Keys) { + $value = $AdditionalEnvKeys[$key] + $environmentVariables[$key] = $value + + if ($CI) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + Write-Host "Setting variable '$key': ***" + Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($value)" + Write-Host "##vso[task.setvariable variable=$key;]$($value)" + } + else { + Write-Host ($shellExportFormat -f $key, $value) + } +} From 3d856b3e84378dc3219f03d13383152f9fb1bc26 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Fri, 25 Sep 2020 10:45:07 -0700 Subject: [PATCH 2/4] Updated storage context script --- sdk/storage/test-resources-post.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/storage/test-resources-post.ps1 b/sdk/storage/test-resources-post.ps1 index 6b19fffa8425..097d70035597 100644 --- a/sdk/storage/test-resources-post.ps1 +++ b/sdk/storage/test-resources-post.ps1 @@ -50,9 +50,8 @@ Write-Verbose "CORS rule set for $datalakeStorageAccountName" $AdditionalEnvKeys = @{} # Create SAS for storage account with additional permissions that are not supported by SRP deployment -$storageAccount = $DeploymentOutputs['ACCOUNT_NAME'] -Write-Host "Creating SAS for storage account $storageAccount" -$storageContext = New-AzStorageContext -StorageAccountName $storageAccount -StorageAccountKey $DeploymentOutputs['ACCOUNT_KEY'] +Write-Host "Creating SAS for storage account $storageAccountName" +$storageContext = New-AzStorageContext -StorageAccountName $storageAccountName $storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $storageContext $AdditionalEnvKeys["ACCOUNT_SAS"] = $storageSas From e0d631f29e304457260961e4cda9603daf098fe0 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Fri, 25 Sep 2020 11:03:36 -0700 Subject: [PATCH 3/4] Updated to create SAS for datalake storage account --- sdk/storage/test-resources-post.ps1 | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/sdk/storage/test-resources-post.ps1 b/sdk/storage/test-resources-post.ps1 index 097d70035597..0725a4a1aaef 100644 --- a/sdk/storage/test-resources-post.ps1 +++ b/sdk/storage/test-resources-post.ps1 @@ -16,8 +16,15 @@ param ( [string] $TestApplicationSecret ) +# Run any post deployment script and set any additional keys to set in Env +$AdditionalEnvKeys = @{} + $storageAccountName = $DeploymentOutputs['ACCOUNT_NAME'] $context = New-AzStorageContext -StorageAccountName $storageAccountName +# Create SAS for storage account with additional permissions that are not supported by SRP deployment +Write-Host "Creating SAS for storage account $storageAccountName" +$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $context +$AdditionalEnvKeys["ACCOUNT_SAS"] = $storageSas # https://docs.microsoft.com/en-us/powershell/module/az.storage/set-azstoragecorsrule?view=azps-3.3.0 $corsRules = (@{ @@ -37,23 +44,15 @@ $corsRules = (@{ }) Set-AzStorageCORSRule -ServiceType 'Queue' -CorsRules $corsRules -Context $context - Write-Verbose "CORS rule set for $storageAccountName" - $datalakeStorageAccountName = $DeploymentOutputs['DFS_ACCOUNT_NAME'] $context = New-AzStorageContext -StorageAccountName $datalakeStorageAccountName Set-AzStorageCORSRule -ServiceType 'Blob' -CorsRules $corsRules -Context $context Write-Verbose "CORS rule set for $datalakeStorageAccountName" - -# Run any post deployment script and set any additional keys to set in Env -$AdditionalEnvKeys = @{} - -# Create SAS for storage account with additional permissions that are not supported by SRP deployment -Write-Host "Creating SAS for storage account $storageAccountName" -$storageContext = New-AzStorageContext -StorageAccountName $storageAccountName -$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $storageContext -$AdditionalEnvKeys["ACCOUNT_SAS"] = $storageSas +Write-Host "Creating SAS for datalake storage account $datalakeStorageAccountName" +$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $context +$AdditionalEnvKeys["DFS_ACCOUNT_SAS"] = $storageSas # Try to detect the shell based on the parent process name (e.g. launch via shebang). $shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { @@ -67,6 +66,8 @@ else { } $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) + +# Set additional keys as ENV variables foreach ($key in $AdditionalEnvKeys.Keys) { $value = $AdditionalEnvKeys[$key] $environmentVariables[$key] = $value From 70b6634cdb54e4135925d90d27b7876b721b5a4e Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Fri, 25 Sep 2020 11:51:27 -0700 Subject: [PATCH 4/4] update script to create SAS for datalake also --- sdk/storage/test-resources-post.ps1 | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/sdk/storage/test-resources-post.ps1 b/sdk/storage/test-resources-post.ps1 index 0725a4a1aaef..c2604c770827 100644 --- a/sdk/storage/test-resources-post.ps1 +++ b/sdk/storage/test-resources-post.ps1 @@ -16,15 +16,8 @@ param ( [string] $TestApplicationSecret ) -# Run any post deployment script and set any additional keys to set in Env -$AdditionalEnvKeys = @{} - $storageAccountName = $DeploymentOutputs['ACCOUNT_NAME'] $context = New-AzStorageContext -StorageAccountName $storageAccountName -# Create SAS for storage account with additional permissions that are not supported by SRP deployment -Write-Host "Creating SAS for storage account $storageAccountName" -$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $context -$AdditionalEnvKeys["ACCOUNT_SAS"] = $storageSas # https://docs.microsoft.com/en-us/powershell/module/az.storage/set-azstoragecorsrule?view=azps-3.3.0 $corsRules = (@{ @@ -44,16 +37,32 @@ $corsRules = (@{ }) Set-AzStorageCORSRule -ServiceType 'Queue' -CorsRules $corsRules -Context $context + Write-Verbose "CORS rule set for $storageAccountName" + $datalakeStorageAccountName = $DeploymentOutputs['DFS_ACCOUNT_NAME'] $context = New-AzStorageContext -StorageAccountName $datalakeStorageAccountName Set-AzStorageCORSRule -ServiceType 'Blob' -CorsRules $corsRules -Context $context Write-Verbose "CORS rule set for $datalakeStorageAccountName" -Write-Host "Creating SAS for datalake storage account $datalakeStorageAccountName" -$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $context + +# Run any post deployment script and set any additional keys to set in Env +$AdditionalEnvKeys = @{} + +# Create SAS for storage account with additional permissions that are not supported by SRP deployment +$storageAccount = $DeploymentOutputs['ACCOUNT_NAME'] +Write-Host "Creating SAS for storage account $storageAccount" +$storageContext = New-AzStorageContext -StorageAccountName $storageAccount -StorageAccountKey $DeploymentOutputs['ACCOUNT_KEY'] +$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $storageContext +$AdditionalEnvKeys["ACCOUNT_SAS"] = $storageSas + +$datalakeStorageAccount = $DeploymentOutputs['DFS_ACCOUNT_NAME'] +Write-Host "Creating SAS for datalake storage account $datalakeStorageAccount" +$storageContext = New-AzStorageContext -StorageAccountName $datalakeStorageAccount -StorageAccountKey $DeploymentOutputs['DFS_ACCOUNT_KEY'] +$storageSas = New-AzStorageAccountSASToken -ResourceType Service, Container, Object -Service Blob, File, Queue, Table -Permission "rwdxftlacup" -Context $storageContext $AdditionalEnvKeys["DFS_ACCOUNT_SAS"] = $storageSas + # Try to detect the shell based on the parent process name (e.g. launch via shebang). $shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { 'cmd', 'set {0}={1}'