From 270a50231ea8bd9a75cb88a2c65abdc052da9f62 Mon Sep 17 00:00:00 2001 From: Matt Sylvia Date: Wed, 11 Sep 2024 14:26:33 -0400 Subject: [PATCH] [CI] Remove Azure Storage usage (#21217) --- tools/devops/automation/build-pipeline.yml | 2 - .../devops/automation/build-pull-request.yml | 2 - .../automation/templates/main-stage.yml | 8 -- .../pipelines/run-tests-pipeline.yml | 2 - .../templates/sign-and-notarized/funnel.yml | 8 -- .../sign-and-notarized/upload-azure.yml | 132 ++++-------------- .../automation/templates/tests-stage.yml | 6 - .../devops/automation/templates/variables.yml | 4 - 8 files changed, 27 insertions(+), 137 deletions(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index 5111b1481221..eeb88f0df7fc 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -263,8 +263,6 @@ extends: testConfigurations: ${{ parameters.testConfigurations }} deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }} macTestsConfigurations: ${{ parameters.macTestsConfigurations }} - azureStorage: ${{ variables['azureStorage'] }} - azureContainer: ${{ variables['azureContainer'] }} signingSetupSteps: - template: ./templates/sign-and-notarized/setup.yml parameters: diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index 7cd145296483..3e3e1eb425e9 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -251,8 +251,6 @@ extends: testConfigurations: ${{ parameters.testConfigurations }} deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }} macTestsConfigurations: ${{ parameters.macTestsConfigurations }} - azureStorage: ${{ variables['azureStorage'] }} - azureContainer: ${{ variables['azureContainer'] }} signingSetupSteps: - template: ./templates/sign-and-notarized/setup.yml parameters: diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index aec04a612159..4bf02630ff93 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -256,12 +256,6 @@ parameters: } ] - - name: azureStorage - type: string - - - name: azureContainer - type: string - - name: stageDisplayNamePrefix type: string default: '' @@ -438,8 +432,6 @@ stages: isPR: ${{ parameters.isPR }} repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} - azureStorage: ${{ parameters.azureStorage }} - azureContainer: ${{ parameters.azureContainer }} packages: # flatten the pkgs for the parameter - ${{ each pkg in parameters.legacyPackageJobs }}: - ${{ pkg }} diff --git a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml index b21ef5af1148..a1b127a464f5 100644 --- a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml @@ -196,6 +196,4 @@ stages: testConfigurations: ${{ parameters.testConfigurations }} deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }} macTestsConfigurations: ${{ parameters.macTestsConfigurations }} - azureStorage: ${{ variables['azureStorage'] }} - azureContainer: ${{ variables['azureContainer'] }} diff --git a/tools/devops/automation/templates/sign-and-notarized/funnel.yml b/tools/devops/automation/templates/sign-and-notarized/funnel.yml index 90ab46b39667..36ed9035e0be 100644 --- a/tools/devops/automation/templates/sign-and-notarized/funnel.yml +++ b/tools/devops/automation/templates/sign-and-notarized/funnel.yml @@ -14,12 +14,6 @@ parameters: type: string default: HEAD -- name: azureStorage - type: string - -- name: azureContainer - type: string - - name: uploadPrefix type: string default: '$(MaciosUploadPrefix)' @@ -175,8 +169,6 @@ jobs: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} sbomFilter: '*.nupkg;*.pkg;*.msi' - azureStorage: ${{ parameters.azureStorage }} - azureContainer: ${{ parameters.azureContainer }} # Job that runs on a vm that downloads the artifacts information and adds a github comment pointing to the results of the build. - job: artifacts_github_comment diff --git a/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml b/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml index 94af45c53a12..24aa23ad50d0 100644 --- a/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml +++ b/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml @@ -3,12 +3,6 @@ parameters: type: string default: '*' # Supports multiple filters separated by semi-colon such as *.msi;*.nupkg -- name: azureStorage - type: string - -- name: azureContainer - type: string - - name: repositoryAlias type: string default: self @@ -68,17 +62,25 @@ steps: displayName: "Show downloaded artifacts" - pwsh: | - $packagePrefix = "https://dl.internalx.com/$Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/package" + $DropServiceUrl = "https://devdiv.artifacts.visualstudio.com/DefaultCollection" + $StatusDropUrl = "https://vsdrop.microsoft.com/file/v1" + $BuildShortHash = "$(Build.SourceVersion)".subString(0, 7) + $ProjectPathUrl = "Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$BuildShortHash/$(Build.BuildId)" + $PackagePrefix = "$StatusDropUrl/$ProjectPathUrl/${{ parameters.uploadPrefix }}package" + + Write-Host "##vso[task.setvariable variable=PackagePrefix]$PackagePrefix" + displayName: "Set drop status URL variables" + +- pwsh: | $files = Get-ChildItem -Path "$(Build.SourcesDirectory)\\artifacts\\${{ parameters.uploadPrefix }}package" -File -Force -Name $manifestFile = "$(Build.SourcesDirectory)\\artifacts\${{ parameters.uploadPrefix }}package\\manifest" foreach ($f in $files) { - Add-Content -Path "$manifestFile" -Value "$packagePrefix/$f" + Add-Content -Path "$manifestFile" -Value "$(PackagePrefix);$f" } - Add-Content -Path "$manifestFile" -Value "$packagePrefix/$artifacts.json" - Add-Content -Path "$manifestFile" -Value "$packagePrefix/manifest" + Add-Content -Path "$manifestFile" -Value "$(PackagePrefix);$artifacts.json" + Add-Content -Path "$manifestFile" -Value "$(PackagePrefix);manifest" env: VIRTUAL_PATH: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId) - AZURE_CONTAINER: ${{ parameters.azureContainer }} displayName: "Build manifest" # Important needed for the next step @@ -87,91 +89,12 @@ steps: GitHubToken: $(GitHub.Token) ArtifactDirectory: $(Build.SourcesDirectory)/${{ parameters.uploadPrefix }}package-internal -- task: AzureFileCopy@6 - displayName: 'Publish package to Azure' - name: upload - inputs: - SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package - azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' - Destination: AzureBlob - storage: ${{ parameters.azureStorage }} - ContainerName: ${{ parameters.azureContainer }} - BlobPrefix: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId)/${{ parameters.uploadPrefix }}package # ideally, we would use a variable for this - -- task: AzureFileCopy@6 - displayName: 'Publish manifest to Azure' - inputs: - SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest - azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' - Destination: AzureBlob - storage: ${{ parameters.azureStorage }} - ContainerName: ${{ parameters.azureContainer }} - BlobPrefix: jenkins/$(Build.SourceBranchName)/$(Build.SourceVersion) - -- task: AzureFileCopy@6 - displayName: 'Publish manifest to Azure as latest' - inputs: - SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest - azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' - Destination: AzureBlob - storage: ${{ parameters.azureStorage }} - ContainerName: ${{ parameters.azureContainer }} - BlobPrefix: jenkins/$(Build.SourceBranchName)/latest - -- task: AzureFileCopy@6 - displayName: 'Publish manifest to Azure per commit' - inputs: - SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest - azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' - Destination: AzureBlob - storage: ${{ parameters.azureStorage }} - ContainerName: ${{ parameters.azureContainer }} - BlobPrefix: jenkins/$(Build.SourceVersion) - -- pwsh: | - $execPath="$Env:BUILD_SOURCESDIRECTORY\Xamarin.Build.Tasks\tools\BuildTasks\build-tasks.exe" - - if (-not (Test-Path $execPath -PathType Leaf)) { - Write-Host "Build task not found at $execPath!" - } - - $maciosPath="$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios" - - Write-Host "Exect path is $execPath" - Write-Host "Macios path is $maciosPath" - Write-Host "$Env:VIRTUAL_PATH" - Write-Host "Artifacts url $Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/${{ parameters.uploadPrefix }}package" - - $execExpression = ( - "$execPath artifacts" + - " -s `"$maciosPath`"" + - " -a $Env:AZURE_STORAGE" + - " -c $Env:STORAGE_PASS" + - " -u `"$Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/${{ parameters.uploadPrefix }}package`"" + - " -d `"$(Build.SourcesDirectory)\\artifacts\${{ parameters.uploadPrefix }}package`"" + - " -o `"$(Build.SourcesDirectory)\\artifacts\${{ parameters.uploadPrefix }}package`"" - ) - - Write-Host "Expression is:" - Write-Host $execExpression - - Invoke-Expression $execExpression - env: - VIRTUAL_PATH: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId) - GITHUB_AUTH_TOKEN: $(GitHub.Token) - STORAGE_PASS: $(auth-xamarin-bosstoragemirror-account-key) - AZURE_STORAGE: ${{ parameters.azureStorage }} - AZURE_CONTAINER: ${{ parameters.azureContainer }} - displayName: 'Generate artifacts.json' - -# upload the artifacts.json to the build pipeline artifacts so that it can be consumed by other jobs to -# get the required urls -- task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Build Artifacts' - inputs: - path: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/artifacts.json - artifact: '${{ parameters.uploadPrefix }}pkg-info' - continueOnError: true +- template: upload-to-drop/v1.yml@yaml-templates + parameters: + SourcesDirectory: $(Build.SourcesDirectory)/xamarin-macios + ArtifactsDirectory: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package + Azure.DropPrefix: ${{ parameters.uploadPrefix }}package + GitHub.Context: 'vsts-devdiv artifacts' - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 displayName: 'Show Environment' @@ -184,7 +107,7 @@ steps: # the azure step sets the env variable, but it points to the wrong url since it uses the one in bosstorage, we # need to use the dl version. - $pkgsVirtualUrl = "https://dl.internalx.com/$Env:AZURE_CONTAINER/$Env:VIRTUAL_PATH/${{ parameters.uploadPrefix }}package" + $pkgsVirtualUrl = "$(PackagePrefix)" Write-Host "Urls is $pkgsVirtualUrl" $pkgsPath = "$(Build.SourcesDirectory)\\artifacts\\${{ parameters.uploadPrefix }}package" @@ -209,7 +132,7 @@ steps: Path = "$pkgsPath\\$iOSPkg"; Context = "PKG-Xamarin.iOS"; Description = $iOSPkg; - TargetUrl = "$pkgsVirtualUrl/$iOSPkg"; + TargetUrl = "$pkgsVirtualUrl;$iOSPkg"; Error = "xamarin.ios pkg not found"; ShouldExist = $true; }, @@ -217,7 +140,7 @@ steps: Path = "$pkgsPath\\notarized\\xamarin.ios-*.pkg"; Context = "PKG-Xamarin.iOS-notarized" Description = "$iOSPkg (Notarized)" ; - TargetUrl = "$pkgsVirtualUrl/notarized/$iOSPkg" ; + TargetUrl = "$pkgsVirtualUrl/notarized;$iOSPkg" ; Error = "Notarized xamarin.ios pkg not found" ; ShouldExist = $notarizedShouldExist; } @@ -235,7 +158,7 @@ steps: Path = "$pkgsPath\\xamarin.mac-*.pkg" ; Context = "PKG-Xamarin.Mac" ; Description = "$macPkg" ; - TargetUrl = "$pkgsVirtualUrl/$macPkg" ; + TargetUrl = "$pkgsVirtualUrl;$macPkg" ; Error = "xamarin.mac pkg not found." ; ShouldExist = $true; }, @@ -243,7 +166,7 @@ steps: Path = "$pkgsPath\\notarized\\xamarin.mac-*.pkg" ; Context = "PKG-Xamarin.Mac-notarized" ; Description = "$macPkg (Notarized)" ; - TargetUrl = "$pkgsVirtualUrl/notarized/$macPkg" ; + TargetUrl = "$pkgsVirtualUrl/notarized;$macPkg" ; Error = "Notarized xamarin.mac pkg not found." ; ShouldExist = $notarizedShouldExist; } @@ -258,7 +181,7 @@ steps: Path = "$pkgsPath\\bundle.zip" ; Context = "bundle.zip" ; Description = "bundle.zip" ; - TargetUrl = "$pkgsVirtualUrl/bundle.zip" ; + TargetUrl = "$pkgsVirtualUrl;bundle.zip" ; Error = "bundle.zip not found." ; ShouldExist = $true; }, @@ -266,7 +189,7 @@ steps: Path = "$pkgsPath\msbuild.zip" ; Context = "msbuild.zip" ; Description = "msbuild.zip" ; - TargetUrl = "$pkgsVirtualUrl/msbuild.zip" ; + TargetUrl = "$pkgsVirtualUrl;msbuild.zip" ; Error = "msbuild.zip not found." ; ShouldExist = $true; } @@ -283,7 +206,7 @@ steps: $msi = Get-ChildItem -Path $pkgsPath -Filter *.msi -File -Name foreach ($n in $msi) { - $statuses.SetStatus("success", "$n", "$n", "$pkgsVirtualUrl/$n") + $statuses.SetStatus("success", "$n", "$n", "$pkgsVirtualUrl;$n") } Write-Host "Setting SBOM.PackagesPath: ${pkgsPath}" @@ -291,7 +214,6 @@ steps: env: GITHUB_TOKEN: $(GitHub.Token) ACCESSTOKEN: $(System.AccessToken) - AZURE_CONTAINER: ${{ parameters.azureContainer }} VIRTUAL_PATH: $(Build.SourceBranchName)/$(Build.SourceVersion)/$(Build.BuildId) displayName: 'Set GithubStatus' diff --git a/tools/devops/automation/templates/tests-stage.yml b/tools/devops/automation/templates/tests-stage.yml index 10a451545f82..904ede195003 100644 --- a/tools/devops/automation/templates/tests-stage.yml +++ b/tools/devops/automation/templates/tests-stage.yml @@ -231,12 +231,6 @@ parameters: - name: macTestsConfigurations type: object -- name: azureStorage - type: string - -- name: azureContainer - type: string - - name: stageDisplayNamePrefix type: string default: '' diff --git a/tools/devops/automation/templates/variables.yml b/tools/devops/automation/templates/variables.yml index a9513ef295c1..38e6ddc50d75 100644 --- a/tools/devops/automation/templates/variables.yml +++ b/tools/devops/automation/templates/variables.yml @@ -34,10 +34,6 @@ variables: value: 'https://devdiv.visualstudio.com/_settings/agentpools?poolId=367&view=agents' - name: CIBuild value: true -- name: azureStorage - value: 'bosstoragemirror' -- name: azureContainer - value: 'wrench' - name: BUILD_REVISION value: azure-devops-$(Build.SourceVersion) - name: MacPoolAccessToken