[CI] Remove Azure Storage usage (#21217)

tools/devops/automation/build-pipeline.yml

@@ -263,8 +263,6 @@ extends:
             testConfigurations: ${{ parameters.testConfigurations }}
           deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }}
           macTestsConfigurations: ${{ parameters.macTestsConfigurations }}
-          azureStorage: ${{ variables['azureStorage'] }}
-          azureContainer: ${{ variables['azureContainer'] }}
           signingSetupSteps:
             - template: ./templates/sign-and-notarized/setup.yml
               parameters:

tools/devops/automation/build-pull-request.yml

@@ -251,8 +251,6 @@ extends:
           testConfigurations: ${{ parameters.testConfigurations }}
         deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }}
         macTestsConfigurations: ${{ parameters.macTestsConfigurations }}
-        azureStorage: ${{ variables['azureStorage'] }}
-        azureContainer: ${{ variables['azureContainer'] }}
         signingSetupSteps:
         - template: ./templates/sign-and-notarized/setup.yml
           parameters:

tools/devops/automation/templates/main-stage.yml

@@ -256,12 +256,6 @@ parameters:
       }
     ]
 
-  - name: azureStorage
-    type: string
-
-  - name: azureContainer
-    type: string
-
   - name: stageDisplayNamePrefix
     type: string
     default: ''
@@ -438,8 +432,6 @@ stages:
           isPR: ${{ parameters.isPR }}
           repositoryAlias: ${{ parameters.repositoryAlias }}
           commit: ${{ parameters.commit }}
-          azureStorage: ${{ parameters.azureStorage }}
-          azureContainer: ${{ parameters.azureContainer }}
           packages:  # flatten the pkgs for the parameter
             - ${{ each pkg in parameters.legacyPackageJobs }}:
                 - ${{ pkg }}

tools/devops/automation/templates/pipelines/run-tests-pipeline.yml

@@ -196,6 +196,4 @@ stages:
         testConfigurations: ${{ parameters.testConfigurations }}
       deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }}
       macTestsConfigurations: ${{ parameters.macTestsConfigurations }}
-      azureStorage: ${{ variables['azureStorage'] }}
-      azureContainer: ${{ variables['azureContainer'] }}
 

tools/devops/automation/templates/sign-and-notarized/funnel.yml

@@ -14,12 +14,6 @@ parameters:
   type: string
   default: HEAD
 
-- name: azureStorage
-  type: string
-
-- name: azureContainer
-  type: string
-
 - name: uploadPrefix
   type: string
   default: '$(MaciosUploadPrefix)'
@@ -175,8 +169,6 @@ jobs:
       repositoryAlias: ${{ parameters.repositoryAlias }}
       commit: ${{ parameters.commit }}
       sbomFilter: '*.nupkg;*.pkg;*.msi'
-      azureStorage: ${{ parameters.azureStorage }}
-      azureContainer: ${{ parameters.azureContainer }}
 
 # Job that runs on a vm that downloads the artifacts information and adds a github comment pointing to the results of the build.
 - job: artifacts_github_comment

tools/devops/automation/templates/sign-and-notarized/upload-azure.yml

@@ -3,12 +3,6 @@ parameters:
   type: string
   default: '*'            # Supports multiple filters separated by semi-colon such as *.msi;*.nupkg
 
-- name: azureStorage
-  type: string
-
-- name: azureContainer
-  type: string
-
 - name: repositoryAlias
   type: string
   default: self
@@ -68,17 +62,25 @@ steps:
   displayName: "Show downloaded artifacts"
 
 - pwsh: |
-    $packagePrefix = "https://dl.internalx.com/$Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/package"
+    $DropServiceUrl = "https://devdiv.artifacts.visualstudio.com/DefaultCollection"
+    $StatusDropUrl = "https://vsdrop.microsoft.com/file/v1"
+    $BuildShortHash = "$(Build.SourceVersion)".subString(0, 7)
+    $ProjectPathUrl = "Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$BuildShortHash/$(Build.BuildId)"
+    $PackagePrefix = "$StatusDropUrl/$ProjectPathUrl/${{ parameters.uploadPrefix }}package"
+
+    Write-Host "##vso[task.setvariable variable=PackagePrefix]$PackagePrefix"
+  displayName: "Set drop status URL variables"
+
+- pwsh: |
     $files =  Get-ChildItem -Path "$(Build.SourcesDirectory)\\artifacts\\${{ parameters.uploadPrefix }}package" -File -Force -Name
     $manifestFile = "$(Build.SourcesDirectory)\\artifacts\${{ parameters.uploadPrefix }}package\\manifest"
     foreach ($f in $files) {
-      Add-Content -Path "$manifestFile" -Value "$packagePrefix/$f"
+      Add-Content -Path "$manifestFile" -Value "$(PackagePrefix);$f"
     }
-    Add-Content -Path "$manifestFile" -Value "$packagePrefix/$artifacts.json"
-    Add-Content -Path "$manifestFile" -Value "$packagePrefix/manifest"
+    Add-Content -Path "$manifestFile" -Value "$(PackagePrefix);$artifacts.json"
+    Add-Content -Path "$manifestFile" -Value "$(PackagePrefix);manifest"
   env:
     VIRTUAL_PATH: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId)
-    AZURE_CONTAINER: ${{ parameters.azureContainer }}
   displayName: "Build manifest"
 
 # Important needed for the next step
@@ -87,91 +89,12 @@ steps:
     GitHubToken: $(GitHub.Token)
     ArtifactDirectory: $(Build.SourcesDirectory)/${{ parameters.uploadPrefix }}package-internal
 
-- task: AzureFileCopy@6
-  displayName: 'Publish package to Azure'
-  name: upload
-  inputs:
-    SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package
-    azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)'
-    Destination: AzureBlob
-    storage: ${{ parameters.azureStorage }}
-    ContainerName: ${{ parameters.azureContainer }}
-    BlobPrefix: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId)/${{ parameters.uploadPrefix }}package  # ideally, we would use a variable for this
-
-- task: AzureFileCopy@6
-  displayName: 'Publish manifest to Azure'
-  inputs:
-    SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest
-    azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)'
-    Destination: AzureBlob
-    storage: ${{ parameters.azureStorage }}
-    ContainerName: ${{ parameters.azureContainer }}
-    BlobPrefix: jenkins/$(Build.SourceBranchName)/$(Build.SourceVersion)
-
-- task: AzureFileCopy@6
-  displayName: 'Publish manifest to Azure as latest'
-  inputs:
-    SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest
-    azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)'
-    Destination: AzureBlob
-    storage: ${{ parameters.azureStorage }}
-    ContainerName: ${{ parameters.azureContainer }}
-    BlobPrefix: jenkins/$(Build.SourceBranchName)/latest
-
-- task: AzureFileCopy@6
-  displayName: 'Publish manifest to Azure per commit'
-  inputs:
-    SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest
-    azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)'
-    Destination: AzureBlob
-    storage: ${{ parameters.azureStorage }}
-    ContainerName: ${{ parameters.azureContainer }}
-    BlobPrefix: jenkins/$(Build.SourceVersion)
-
-- pwsh: |
-    $execPath="$Env:BUILD_SOURCESDIRECTORY\Xamarin.Build.Tasks\tools\BuildTasks\build-tasks.exe"
-
-    if (-not (Test-Path $execPath -PathType Leaf)) {
-        Write-Host "Build task not found at $execPath!"
-    }
-
-    $maciosPath="$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios"
-
-    Write-Host "Exect path is $execPath"
-    Write-Host "Macios path is $maciosPath"
-    Write-Host "$Env:VIRTUAL_PATH"
-    Write-Host "Artifacts url $Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/${{ parameters.uploadPrefix }}package"
-
-    $execExpression = (
-                    "$execPath artifacts" +
-                    " -s `"$maciosPath`"" +
-                    " -a $Env:AZURE_STORAGE" +
-                    " -c $Env:STORAGE_PASS" +
-                    " -u `"$Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/${{ parameters.uploadPrefix }}package`"" +
-                    " -d `"$(Build.SourcesDirectory)\\artifacts\${{ parameters.uploadPrefix }}package`"" +
-                    " -o `"$(Build.SourcesDirectory)\\artifacts\${{ parameters.uploadPrefix }}package`""
-                    )
-
-    Write-Host "Expression is:"
-    Write-Host $execExpression
-
-    Invoke-Expression $execExpression
-  env:
-    VIRTUAL_PATH: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId)
-    GITHUB_AUTH_TOKEN: $(GitHub.Token)
-    STORAGE_PASS: $(auth-xamarin-bosstoragemirror-account-key)
-    AZURE_STORAGE: ${{ parameters.azureStorage }}
-    AZURE_CONTAINER: ${{ parameters.azureContainer }}
-  displayName: 'Generate artifacts.json'
-
-# upload the artifacts.json to the build pipeline artifacts so that it can be consumed by other jobs to
-# get the required urls
-- task: 1ES.PublishPipelineArtifact@1
-  displayName: 'Publish Build Artifacts'
-  inputs:
-    path: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/artifacts.json
-    artifact: '${{ parameters.uploadPrefix }}pkg-info'
-  continueOnError: true
+- template: upload-to-drop/v1.yml@yaml-templates
+  parameters:
+    SourcesDirectory: $(Build.SourcesDirectory)/xamarin-macios
+    ArtifactsDirectory: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package
+    Azure.DropPrefix: ${{ parameters.uploadPrefix }}package
+    GitHub.Context: 'vsts-devdiv artifacts'
 
 - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1
   displayName: 'Show Environment'
@@ -184,7 +107,7 @@ steps:
 
     # the azure step sets the env variable, but it points to the wrong url since it uses the one in bosstorage, we
     # need to use the dl version.
-    $pkgsVirtualUrl =  "https://dl.internalx.com/$Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/${{ parameters.uploadPrefix }}package"
+    $pkgsVirtualUrl = "$(PackagePrefix)"
     Write-Host "Urls is $pkgsVirtualUrl"
 
     $pkgsPath = "$(Build.SourcesDirectory)\\artifacts\\${{ parameters.uploadPrefix }}package"
@@ -209,15 +132,15 @@ steps:
           Path = "$pkgsPath\\$iOSPkg";
           Context = "PKG-Xamarin.iOS";
           Description = $iOSPkg;
-          TargetUrl = "$pkgsVirtualUrl/$iOSPkg";
+          TargetUrl = "$pkgsVirtualUrl;$iOSPkg";
           Error = "xamarin.ios pkg not found";
           ShouldExist = $true;
         },
         @{
           Path = "$pkgsPath\\notarized\\xamarin.ios-*.pkg";
           Context = "PKG-Xamarin.iOS-notarized"
           Description = "$iOSPkg (Notarized)" ;
-          TargetUrl = "$pkgsVirtualUrl/notarized/$iOSPkg" ;
+          TargetUrl = "$pkgsVirtualUrl/notarized;$iOSPkg" ;
           Error = "Notarized xamarin.ios pkg not found" ;
           ShouldExist = $notarizedShouldExist;
         }
@@ -235,15 +158,15 @@ steps:
           Path = "$pkgsPath\\xamarin.mac-*.pkg" ;
           Context = "PKG-Xamarin.Mac" ;
           Description = "$macPkg" ;
-          TargetUrl = "$pkgsVirtualUrl/$macPkg" ;
+          TargetUrl = "$pkgsVirtualUrl;$macPkg" ;
           Error = "xamarin.mac pkg not found." ;
           ShouldExist = $true;
         },
         @{
           Path = "$pkgsPath\\notarized\\xamarin.mac-*.pkg" ;
           Context = "PKG-Xamarin.Mac-notarized" ;
           Description = "$macPkg (Notarized)" ;
-          TargetUrl = "$pkgsVirtualUrl/notarized/$macPkg" ;
+          TargetUrl = "$pkgsVirtualUrl/notarized;$macPkg" ;
           Error = "Notarized xamarin.mac pkg not found." ;
           ShouldExist = $notarizedShouldExist;
         }
@@ -258,15 +181,15 @@ steps:
         Path = "$pkgsPath\\bundle.zip" ;
         Context = "bundle.zip" ;
         Description = "bundle.zip" ;
-        TargetUrl = "$pkgsVirtualUrl/bundle.zip" ;
+        TargetUrl = "$pkgsVirtualUrl;bundle.zip" ;
         Error = "bundle.zip not found." ;
         ShouldExist = $true;
       },
       @{
         Path = "$pkgsPath\msbuild.zip" ;
         Context = "msbuild.zip" ;
         Description = "msbuild.zip" ;
-        TargetUrl = "$pkgsVirtualUrl/msbuild.zip" ;
+        TargetUrl = "$pkgsVirtualUrl;msbuild.zip" ;
         Error = "msbuild.zip not found." ;
         ShouldExist = $true;
       }
@@ -283,15 +206,14 @@ steps:
     $msi = Get-ChildItem -Path $pkgsPath -Filter *.msi -File -Name
 
     foreach ($n in $msi) {
-      $statuses.SetStatus("success", "$n", "$n", "$pkgsVirtualUrl/$n")
+      $statuses.SetStatus("success", "$n", "$n", "$pkgsVirtualUrl;$n")
     }
 
     Write-Host "Setting SBOM.PackagesPath: ${pkgsPath}"
     Write-Host "##vso[task.setvariable variable=SBOM.PackagesPath]$pkgsPath"
   env:
     GITHUB_TOKEN: $(GitHub.Token)
     ACCESSTOKEN: $(System.AccessToken)
-    AZURE_CONTAINER: ${{ parameters.azureContainer }}
     VIRTUAL_PATH: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId)
   displayName: 'Set GithubStatus'
 

tools/devops/automation/templates/tests-stage.yml

@@ -231,12 +231,6 @@ parameters:
 - name: macTestsConfigurations
   type: object
 
-- name: azureStorage
-  type: string
-
-- name: azureContainer
-  type: string
-
 - name: stageDisplayNamePrefix
   type: string
   default: ''

tools/devops/automation/templates/variables.yml

@@ -34,10 +34,6 @@ variables:
   value: 'https://devdiv.visualstudio.com/_settings/agentpools?poolId=367&view=agents'
 - name: CIBuild
   value: true
-- name: azureStorage
-  value: 'bosstoragemirror'
-- name: azureContainer
-  value: 'wrench'
 - name: BUILD_REVISION
   value: azure-devops-$(Build.SourceVersion)
 - name: MacPoolAccessToken