From 6bfce887550065f506b1867057cb38727ab383ff Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 2 Nov 2021 10:18:26 -0700 Subject: [PATCH] Update from WindowsAzure.Storage (no longer supported) to Azure.Storage.Blobs. Plumb through a retry-heavy retry policy to the uploads as well. (#8118) --- eng/Versions.props | 5 +- ...osoft.DotNet.Build.Tasks.Feed.Tests.csproj | 2 +- .../Microsoft.DotNet.Helix.Client.csproj | 2 +- .../Microsoft.DotNet.Helix.JobSender.csproj | 2 +- .../JobSender/StorageHelpers/ApiBlobHelper.cs | 20 ++++--- .../ConnectionStringBlobHelper.cs | 59 +++++++------------ .../JobSender/StorageHelpers/ContainerBase.cs | 17 +++--- .../StorageHelpers/IBlobContainer.cs | 9 +-- .../JobSender/StorageHelpers/IBlobHelper.cs | 12 ++-- .../StorageHelpers/StorageRetryPolicy.cs | 25 ++++++++ .../Sdk/DownloadFromResultsContainer.cs | 15 +++-- 11 files changed, 84 insertions(+), 84 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/StorageRetryPolicy.cs diff --git a/eng/Versions.props b/eng/Versions.props index 3534d792600..a6807430fac 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -13,7 +13,8 @@ 0.25.2 2.0.10 4.3.4 - 12.3.0 + 1.19.0 + 12.10.0 5.10.3 2.16.0 3.0.0 @@ -62,7 +63,7 @@ 4.3.0 4.7.2 4.5.2 - 4.4.0 + 4.5.0 8.5.0 2.4.2-pre.9 2.0.3 diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj index 36d4aa93871..b845996cd3a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj index 531ba3203fa..a743b31b379 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj b/src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj index 1c57d6d76a7..5a77604e1aa 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj +++ b/src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj @@ -9,10 +9,10 @@ + - diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ApiBlobHelper.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ApiBlobHelper.cs index 64c57c2fc79..a8fe9d2608d 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ApiBlobHelper.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ApiBlobHelper.cs @@ -1,9 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; using System.Threading; using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; using Microsoft.DotNet.Helix.Client.Models; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; namespace Microsoft.DotNet.Helix.Client { @@ -20,17 +23,18 @@ public ApiBlobHelper(IStorage helixApiStorage) public async Task GetContainerAsync(string requestedName, string targetQueue, CancellationToken cancellationToken) { ContainerInformation info = await _helixApiStorage.NewAsync(new ContainerCreationRequest(30, requestedName, targetQueue), cancellationToken).ConfigureAwait(false); - var client = new CloudBlobClient(new Uri($"https://{info.StorageAccountName}.blob.core.windows.net/"), new StorageCredentials(info.WriteToken)); - CloudBlobContainer container = client.GetContainerReference(info.ContainerName); + Uri containerUri = new Uri($"https://{info.StorageAccountName}.blob.core.windows.net/{info.ContainerName}"); + AzureSasCredential creds = new AzureSasCredential(info.WriteToken); + var container = new BlobContainerClient(containerUri, creds, StorageRetryPolicy.GetBlobClientOptionsRetrySettings()); return new Container(container, info); } private class Container : ContainerBase { - private readonly CloudBlobContainer _container; + private readonly BlobContainerClient _container; private readonly ContainerInformation _info; - public Container(CloudBlobContainer container, ContainerInformation info) + public Container(BlobContainerClient container, ContainerInformation info) { _container = container; _info = info; @@ -40,7 +44,7 @@ public Container(CloudBlobContainer container, ContainerInformation info) public override string ReadSas => _info.ReadToken; public override string WriteSas => _info.WriteToken; - protected override (CloudBlockBlob blob, string sasToken) GetBlob(string blobName) + protected override (BlobClient blob, string sasToken) GetBlob(string blobName) { string sasToken = _info.ReadToken; if (sasToken.StartsWith("?")) @@ -48,7 +52,7 @@ protected override (CloudBlockBlob blob, string sasToken) GetBlob(string blobNam sasToken = sasToken.Substring(1); } - CloudBlockBlob blob = _container.GetBlockBlobReference(blobName); + BlobClient blob = _container.GetBlobClient(blobName); return (blob, sasToken); } } diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ConnectionStringBlobHelper.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ConnectionStringBlobHelper.cs index bfb6901cfd0..39078975bbc 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ConnectionStringBlobHelper.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ConnectionStringBlobHelper.cs @@ -1,13 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; -using System.Diagnostics; -using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.DotNet.Helix.Client.Models; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; +using Azure.Storage.Sas; namespace Microsoft.DotNet.Helix.Client { @@ -23,60 +21,43 @@ public ConnectionStringBlobHelper(string connectionString) public async Task GetContainerAsync(string requestedName, string targetQueue, CancellationToken cancellationToken) { - CloudStorageAccount account = CloudStorageAccount.Parse(_connectionString); - CloudBlobClient client = account.CreateCloudBlobClient(); - CloudBlobContainer container = client.GetContainerReference(requestedName); + BlobServiceClient account = new BlobServiceClient(_connectionString, StorageRetryPolicy.GetBlobClientOptionsRetrySettings()); + + BlobContainerClient container = account.GetBlobContainerClient(requestedName); await container.CreateIfNotExistsAsync(); return new Container(container); } private class Container : ContainerBase { - private readonly CloudBlobContainer _container; + private readonly BlobContainerClient _container; - public Container(CloudBlobContainer container) + public Container(BlobContainerClient container) { _container = container; } public override string Uri => _container.Uri.ToString(); - public override string ReadSas => _container.GetSharedAccessSignature(SasReadOnly); - public override string WriteSas => _container.GetSharedAccessSignature(SasReadWrite); - - - private SharedAccessBlobPolicy SasReadOnly - { - get - { - return new SharedAccessBlobPolicy - { - SharedAccessExpiryTime = DateTime.UtcNow.AddDays(30), - Permissions = SharedAccessBlobPermissions.Read - }; - } - } + public override string ReadSas => GetSasTokenForPermissions(BlobContainerSasPermissions.Read, DateTime.UtcNow.AddDays(30)); + public override string WriteSas => GetSasTokenForPermissions(BlobContainerSasPermissions.Write | + BlobContainerSasPermissions.Read, + DateTime.UtcNow.AddDays(30)); - private SharedAccessBlobPolicy SasReadWrite + private string GetSasTokenForPermissions(BlobContainerSasPermissions permissions, DateTime expiration) { - get - { - return new SharedAccessBlobPolicy - { - SharedAccessExpiryTime = DateTime.UtcNow.AddDays(30), - Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read - }; - } + string sas = _container.GenerateSasUri(permissions, expiration).ToString(); + return sas.Substring(sas.IndexOf('?')); } - protected override (CloudBlockBlob blob, string sasToken) GetBlob(string blobName) + protected override (BlobClient blob, string sasToken) GetBlob(string blobName) { - string sasToken = _container.GetSharedAccessSignature(SasReadOnly); + string sasToken = GetSasTokenForPermissions(BlobContainerSasPermissions.Read, DateTime.UtcNow.AddDays(30)); if (sasToken.StartsWith("?")) { sasToken = sasToken.Substring(1); } - CloudBlockBlob blob = _container.GetBlockBlobReference(blobName); + BlobClient blob = _container.GetBlobClient(blobName); return (blob, sasToken); } } diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs index eed45d6a851..eca23eb5302 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs @@ -1,26 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; -using System.Diagnostics; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.DotNet.Helix.Client.Models; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; namespace Microsoft.DotNet.Helix.Client { internal abstract class ContainerBase : IBlobContainer { - protected abstract (CloudBlockBlob blob, string sasToken) GetBlob(string blobName); + protected abstract (BlobClient blob, string sasToken) GetBlob(string blobName); public async Task UploadFileAsync(Stream stream, string blobName, CancellationToken cancellationToken) { var (pageBlob, sasToken) = GetBlob(blobName); - - await pageBlob.UploadFromStreamAsync(stream, default(AccessCondition), default(BlobRequestOptions), default(OperationContext), cancellationToken); + await pageBlob.UploadAsync(stream, cancellationToken); return new UriBuilder(pageBlob.Uri) { Query = sasToken }.Uri; } @@ -29,9 +27,8 @@ public async Task UploadTextAsync(string text, string blobName, Cancellatio { var (pageBlob, sasToken) = GetBlob(blobName); byte[] bytes = Encoding.UTF8.GetBytes(text); - await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length, default(AccessCondition), default(BlobRequestOptions), default(OperationContext), cancellationToken); - return new UriBuilder(pageBlob.Uri) { Query = sasToken }.Uri; + return await UploadFileAsync(new MemoryStream(bytes), blobName, cancellationToken); } public abstract string Uri { get; } diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobContainer.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobContainer.cs index 09b9985a2f0..596ed2b18a4 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobContainer.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobContainer.cs @@ -1,13 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; -using System.Diagnostics; using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.DotNet.Helix.Client.Models; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; namespace Microsoft.DotNet.Helix.Client { diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobHelper.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobHelper.cs index 994cad9ee62..833e0ff596f 100644 --- a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobHelper.cs +++ b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobHelper.cs @@ -1,13 +1,9 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Text; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System.Threading; using System.Threading.Tasks; -using Microsoft.DotNet.Helix.Client.Models; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; + namespace Microsoft.DotNet.Helix.Client { diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/StorageRetryPolicy.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/StorageRetryPolicy.cs new file mode 100644 index 00000000000..0add98b4c88 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/StorageRetryPolicy.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Azure.Core; +using Azure.Storage.Blobs; + +namespace Microsoft.DotNet.Helix.Client +{ + class StorageRetryPolicy + { + internal static BlobClientOptions GetBlobClientOptionsRetrySettings() + { + // If this takes way too long in failure mode, or doesn't retry enough, change the settings here to impact all *BlobHelper classes + + BlobClientOptions options = new BlobClientOptions(); + options.Retry.Delay = TimeSpan.FromSeconds(5); + options.Retry.MaxDelay = TimeSpan.FromSeconds(30); + options.Retry.MaxRetries = 10; + options.Retry.Mode = RetryMode.Exponential; + + return options; + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs index 9130f079b8e..27406e73b83 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs @@ -1,12 +1,11 @@ -using Microsoft.Build.Framework; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; using System; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Microsoft.Build.Framework; namespace Microsoft.DotNet.Helix.Sdk { @@ -100,12 +99,12 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory // 1 until helix provides an API to get result files from the "good" iteration run. // https://github.com/dotnet/core-eng/issues/13983 var uri = new Uri($"{ResultsContainer}{workItemName}/1/{file}"); - CloudBlob blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new CloudBlob(uri) : new CloudBlob(uri, new StorageCredentials(ResultsContainerReadSAS)); - await blob.DownloadToFileAsync(destinationFile, FileMode.Create); + BlobClient blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new BlobClient(uri) : new BlobClient(uri, new AzureSasCredential(ResultsContainerReadSAS)); + await blob.DownloadToAsync(destinationFile); } - catch (StorageException e) + catch (RequestFailedException rfe) { - Log.LogWarning($"Failed to download {workItemName}/1/{file} blob from results container: {e.Message}"); + Log.LogWarning($"Failed to download {workItemName}/1/{file} blob from results container: {rfe.Message}"); } } };