Update from WindowsAzure.Storage (no longer supported) to Azure.Stora…

…ge.Blobs. Plumb through a retry-heavy retry policy to the uploads as well. (#8118)
dotnet · Nov 2, 2021 · 6bfce88 · 6bfce88
1 parent 45d6c38
commit 6bfce88
Show file tree

Hide file tree

Showing 11 changed files with 84 additions and 84 deletions.
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -13,7 +13,8 @@
     <LibGit2SharpVersion>0.25.2</LibGit2SharpVersion>
     <log4netVersion>2.0.10</log4netVersion>
     <SystemNetHttpVersion>4.3.4</SystemNetHttpVersion>
-    <AzureStorageBlobsVersion>12.3.0</AzureStorageBlobsVersion>
+    <AzureCoreVersion>1.19.0</AzureCoreVersion>
+    <AzureStorageBlobsVersion>12.10.0</AzureStorageBlobsVersion>
     <FluentAssertionsVersion>5.10.3</FluentAssertionsVersion>
     <MicrosoftApplicationInsightsVersion>2.16.0</MicrosoftApplicationInsightsVersion>
     <MicrosoftAzureKeyVaultVersion>3.0.0</MicrosoftAzureKeyVaultVersion>
@@ -62,7 +63,7 @@
     <SystemRuntimeInteropServicesRuntimeInformation>4.3.0</SystemRuntimeInteropServicesRuntimeInformation>
     <SystemTextEncodingsWebVersion>4.7.2</SystemTextEncodingsWebVersion>
     <SystemThreadingTasksExtensionVersion>4.5.2</SystemThreadingTasksExtensionVersion>
-    <SystemValueTupleVersion>4.4.0</SystemValueTupleVersion>
+    <SystemValueTupleVersion>4.5.0</SystemValueTupleVersion>
     <WindowsAzureStorageVersion>8.5.0</WindowsAzureStorageVersion>
     <XUnitVersion>2.4.2-pre.9</XUnitVersion>
     <XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>

diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj
@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Core" Version="1.0.2.0" />
+    <PackageReference Include="Azure.Core" Version="$(AzureCoreVersion)" />
     <PackageReference Include="Azure.Storage.Blobs" Version="$(AzureStorageBlobsVersion)" />
     <PackageReference Include="DotNet.SleetLib" Version="$(DotNetSleetLibVersion)" />
     <PackageReference Include="FluentAssertions" Version="$(FluentAssertionsVersion)" />

diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Core" Version="1.5.0" />
+    <PackageReference Include="Azure.Core" Version="$(AzureCoreVersion)" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="1.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
     <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />

diff --git a/src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj b/src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj
@@ -9,10 +9,10 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="$(AzureStorageBlobsVersion)" /> 
     <PackageReference Include="Microsoft.Data.OData" Version="$(MicrosoftDataODataVersion)" />
     <PackageReference Include="System.IO.Compression" Version="$(SystemIOCompressionVersion)" />
     <PackageReference Include="System.ValueTuple" Version="$(SystemValueTupleVersion)" />
-    <PackageReference Include="WindowsAzure.Storage" Version="$(WindowsAzureStorageVersion)" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ApiBlobHelper.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ApiBlobHelper.cs
@@ -1,9 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Blobs;
 using Microsoft.DotNet.Helix.Client.Models;
-using Microsoft.WindowsAzure.Storage.Auth;
-using Microsoft.WindowsAzure.Storage.Blob;
 
 namespace Microsoft.DotNet.Helix.Client
 {
@@ -20,17 +23,18 @@ public ApiBlobHelper(IStorage helixApiStorage)
         public async Task<IBlobContainer> GetContainerAsync(string requestedName, string targetQueue, CancellationToken cancellationToken)
         {
             ContainerInformation info = await _helixApiStorage.NewAsync(new ContainerCreationRequest(30, requestedName, targetQueue), cancellationToken).ConfigureAwait(false);
-            var client = new CloudBlobClient(new Uri($"https://{info.StorageAccountName}.blob.core.windows.net/"), new StorageCredentials(info.WriteToken));
-            CloudBlobContainer container = client.GetContainerReference(info.ContainerName);
+            Uri containerUri = new Uri($"https://{info.StorageAccountName}.blob.core.windows.net/{info.ContainerName}");
+            AzureSasCredential creds = new AzureSasCredential(info.WriteToken);
+            var container = new BlobContainerClient(containerUri, creds, StorageRetryPolicy.GetBlobClientOptionsRetrySettings());
             return new Container(container, info);
         }
 
         private class Container : ContainerBase
         {
-            private readonly CloudBlobContainer _container;
+            private readonly BlobContainerClient _container;
             private readonly ContainerInformation _info;
 
-            public Container(CloudBlobContainer container, ContainerInformation info)
+            public Container(BlobContainerClient container, ContainerInformation info)
             {
                 _container = container;
                 _info = info;
@@ -40,15 +44,15 @@ public Container(CloudBlobContainer container, ContainerInformation info)
             public override string ReadSas => _info.ReadToken;
             public override string WriteSas => _info.WriteToken;
 
-            protected override (CloudBlockBlob blob, string sasToken) GetBlob(string blobName)
+            protected override (BlobClient blob, string sasToken) GetBlob(string blobName)
             {
                 string sasToken = _info.ReadToken;
                 if (sasToken.StartsWith("?"))
                 {
                     sasToken = sasToken.Substring(1);
                 }
 
-                CloudBlockBlob blob = _container.GetBlockBlobReference(blobName);
+                BlobClient blob = _container.GetBlobClient(blobName);
                 return (blob, sasToken);
             }
         }

diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ConnectionStringBlobHelper.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ConnectionStringBlobHelper.cs
@@ -1,13 +1,11 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
 using System;
-using System.Diagnostics;
-using System.IO;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.DotNet.Helix.Client.Models;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Auth;
-using Microsoft.WindowsAzure.Storage.Blob;
+using Azure.Storage.Blobs;
+using Azure.Storage.Sas;
 
 namespace Microsoft.DotNet.Helix.Client
 {
@@ -23,60 +21,43 @@ public ConnectionStringBlobHelper(string connectionString)
 
         public async Task<IBlobContainer> GetContainerAsync(string requestedName, string targetQueue, CancellationToken cancellationToken)
         {
-            CloudStorageAccount account = CloudStorageAccount.Parse(_connectionString);
-            CloudBlobClient client = account.CreateCloudBlobClient();
-            CloudBlobContainer container = client.GetContainerReference(requestedName);
+            BlobServiceClient account = new BlobServiceClient(_connectionString, StorageRetryPolicy.GetBlobClientOptionsRetrySettings());
+
+            BlobContainerClient container = account.GetBlobContainerClient(requestedName);
             await container.CreateIfNotExistsAsync();
             return new Container(container);
         }
 
         private class Container : ContainerBase
         {
-            private readonly CloudBlobContainer _container;
+            private readonly BlobContainerClient _container;
 
-            public Container(CloudBlobContainer container)
+            public Container(BlobContainerClient container)
             {
                 _container = container;
             }
 
             public override string Uri => _container.Uri.ToString();
-            public override string ReadSas => _container.GetSharedAccessSignature(SasReadOnly);
-            public override string WriteSas => _container.GetSharedAccessSignature(SasReadWrite);
-
-
-            private SharedAccessBlobPolicy SasReadOnly
-            {
-                get
-                {
-                    return new SharedAccessBlobPolicy
-                    {
-                        SharedAccessExpiryTime = DateTime.UtcNow.AddDays(30),
-                        Permissions = SharedAccessBlobPermissions.Read
-                    };
-                }
-            }
+            public override string ReadSas => GetSasTokenForPermissions(BlobContainerSasPermissions.Read, DateTime.UtcNow.AddDays(30));
+            public override string WriteSas => GetSasTokenForPermissions(BlobContainerSasPermissions.Write | 
+                                                                         BlobContainerSasPermissions.Read,
+                                                                         DateTime.UtcNow.AddDays(30));
 
-            private SharedAccessBlobPolicy SasReadWrite
+            private string GetSasTokenForPermissions(BlobContainerSasPermissions permissions, DateTime expiration)
             {
-                get
-                {
-                    return new SharedAccessBlobPolicy
-                    {
-                        SharedAccessExpiryTime = DateTime.UtcNow.AddDays(30),
-                        Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read
-                    };
-                }
+                string sas = _container.GenerateSasUri(permissions, expiration).ToString();
+                return sas.Substring(sas.IndexOf('?'));
             }
 
-            protected override (CloudBlockBlob blob, string sasToken) GetBlob(string blobName)
+            protected override (BlobClient blob, string sasToken) GetBlob(string blobName)
             {
-                string sasToken = _container.GetSharedAccessSignature(SasReadOnly);
+                string sasToken = GetSasTokenForPermissions(BlobContainerSasPermissions.Read, DateTime.UtcNow.AddDays(30));
                 if (sasToken.StartsWith("?"))
                 {
                     sasToken = sasToken.Substring(1);
                 }
 
-                CloudBlockBlob blob = _container.GetBlockBlobReference(blobName);
+                BlobClient blob = _container.GetBlobClient(blobName);
                 return (blob, sasToken);
             }
         }

diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs
@@ -1,26 +1,24 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
 using System;
-using System.Diagnostics;
 using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.DotNet.Helix.Client.Models;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Auth;
-using Microsoft.WindowsAzure.Storage.Blob;
+using Azure.Storage.Blobs;
 
 namespace Microsoft.DotNet.Helix.Client
 {
 
     internal abstract class ContainerBase : IBlobContainer
     {
-        protected abstract (CloudBlockBlob blob, string sasToken) GetBlob(string blobName);
+        protected abstract (BlobClient blob, string sasToken) GetBlob(string blobName);
 
         public async Task<Uri> UploadFileAsync(Stream stream, string blobName, CancellationToken cancellationToken)
         {
             var (pageBlob, sasToken) = GetBlob(blobName);
-
-            await pageBlob.UploadFromStreamAsync(stream, default(AccessCondition), default(BlobRequestOptions), default(OperationContext), cancellationToken);
+            await pageBlob.UploadAsync(stream, cancellationToken);
 
             return new UriBuilder(pageBlob.Uri) { Query = sasToken }.Uri;
         }
@@ -29,9 +27,8 @@ public async Task<Uri> UploadTextAsync(string text, string blobName, Cancellatio
         {
             var (pageBlob, sasToken) = GetBlob(blobName);
             byte[] bytes = Encoding.UTF8.GetBytes(text);
-            await pageBlob.UploadFromByteArrayAsync(bytes, 0, bytes.Length, default(AccessCondition), default(BlobRequestOptions), default(OperationContext), cancellationToken);
 
-            return new UriBuilder(pageBlob.Uri) { Query = sasToken }.Uri;
+            return await UploadFileAsync(new MemoryStream(bytes), blobName, cancellationToken);
         }
 
         public abstract string Uri { get; }

diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobContainer.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobContainer.cs
@@ -1,13 +1,10 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
 using System;
-using System.Diagnostics;
 using System.IO;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.DotNet.Helix.Client.Models;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Auth;
-using Microsoft.WindowsAzure.Storage.Blob;
 
 namespace Microsoft.DotNet.Helix.Client
 {

diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobHelper.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/IBlobHelper.cs
@@ -1,13 +1,9 @@
-using System;
-using System.Diagnostics;
-using System.IO;
-using System.Text;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.DotNet.Helix.Client.Models;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Auth;
-using Microsoft.WindowsAzure.Storage.Blob;
+
 
 namespace Microsoft.DotNet.Helix.Client
 {

diff --git a/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/StorageRetryPolicy.cs b/src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/StorageRetryPolicy.cs
@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using Azure.Core;
+using Azure.Storage.Blobs;
+
+namespace Microsoft.DotNet.Helix.Client
+{
+    class StorageRetryPolicy
+    {
+        internal static BlobClientOptions GetBlobClientOptionsRetrySettings()
+        {
+            // If this takes way too long in failure mode, or doesn't retry enough, change the settings here to impact all *BlobHelper classes
+
+            BlobClientOptions options = new BlobClientOptions();
+            options.Retry.Delay = TimeSpan.FromSeconds(5);
+            options.Retry.MaxDelay = TimeSpan.FromSeconds(30);
+            options.Retry.MaxRetries = 10;
+            options.Retry.Mode = RetryMode.Exponential;
+
+            return options;
+        }
+    }
+}
diff --git a/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs b/src/Microsoft.DotNet.Helix/Sdk/DownloadFromResultsContainer.cs
@@ -1,12 +1,11 @@
-using Microsoft.Build.Framework;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Auth;
-using Microsoft.WindowsAzure.Storage.Blob;
 using System;
 using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Blobs;
+using Microsoft.Build.Framework;
 
 namespace Microsoft.DotNet.Helix.Sdk
 {
@@ -100,12 +99,12 @@ private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directory
                         // 1 until helix provides an API to get result files from the "good" iteration run.
                         // https://github.com/dotnet/core-eng/issues/13983
                         var uri = new Uri($"{ResultsContainer}{workItemName}/1/{file}");
-                        CloudBlob blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new CloudBlob(uri) : new CloudBlob(uri, new StorageCredentials(ResultsContainerReadSAS));
-                        await blob.DownloadToFileAsync(destinationFile, FileMode.Create);
+                        BlobClient blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new BlobClient(uri) : new BlobClient(uri, new AzureSasCredential(ResultsContainerReadSAS));
+                        await blob.DownloadToAsync(destinationFile);
                     }
-                    catch (StorageException e)
+                    catch (RequestFailedException rfe)
                     {
-                        Log.LogWarning($"Failed to download {workItemName}/1/{file} blob from results container: {e.Message}");
+                        Log.LogWarning($"Failed to download {workItemName}/1/{file} blob from results container: {rfe.Message}");
                     }
                 }
             };