From 2a5c344449253a51099eaea75282387f5c80af07 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:41:00 -0700 Subject: [PATCH] [Storage][Webjobs] Bind to BlobClient (#16336) * can bind [Blob] to BlobClient. * this works. * update snippet to use blobclient. * remove todo. * add doc strings. * scope new converter to blobtrigger. * remove unused part. --- .../README.md | 8 +-- .../samples/BlobExtensionSamples.cs | 11 ++-- .../StorageBlobContainerExtensions.cs | 7 ++- .../src/BlobAttribute.cs | 2 + .../src/BlobTriggerAttribute.cs | 2 + .../Config/BlobsExtensionConfigProvider.cs | 55 +++++++++++----- .../tests/BlobTests.cs | 40 ++++++++++++ .../tests/BlobTriggerTests.cs | 2 +- .../tests/HostCallTests.cs | 63 +++++++++++++++++++ .../tests/BlobBindingEndToEndTests.cs | 24 +++++++ 10 files changed, 189 insertions(+), 25 deletions(-) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/README.md b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/README.md index de913d33ae439..45683f93d0948 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/README.md +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/README.md @@ -102,13 +102,13 @@ public static class BlobFunction_WriteStream ### Binding to Azure Storage Blob SDK types -```C# Snippet:BlobFunction_BlobBaseClient -public static class BlobFunction_BlobBaseClient +```C# Snippet:BlobFunction_BlobClient +public static class BlobFunction_BlobClient { [FunctionName("BlobFunction")] public static async Task Run( - [BlobTrigger("sample-container/sample-blob-1")] BlobBaseClient blobTriggerClient, - [Blob("sample-container/sample-blob-2")] BlobBaseClient blobClient, + [BlobTrigger("sample-container/sample-blob-1")] BlobClient blobTriggerClient, + [Blob("sample-container/sample-blob-2")] BlobClient blobClient, ILogger logger) { BlobProperties blobTriggerProperties = await blobTriggerClient.GetPropertiesAsync(); diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/samples/BlobExtensionSamples.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/samples/BlobExtensionSamples.cs index e086ff3d89a98..c2c9989f3549e 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/samples/BlobExtensionSamples.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/samples/BlobExtensionSamples.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Azure.WebJobs.Extensions.Storage.Common.Tests; @@ -22,7 +23,7 @@ public class BlobExtensionSamples [TestCase(typeof(BlobFunction_String))] [TestCase(typeof(BlobFunction_ReadStream))] [TestCase(typeof(BlobFunction_WriteStream))] - [TestCase(typeof(BlobFunction_BlobBaseClient))] + [TestCase(typeof(BlobFunction_BlobClient))] public async Task Run_BlobFunction(Type programType) { var containerClient = AzuriteNUnitFixture.Instance.GetBlobServiceClient().GetBlobContainerClient("sample-container"); @@ -96,13 +97,13 @@ public static async Task Run( } #endregion - #region Snippet:BlobFunction_BlobBaseClient - public static class BlobFunction_BlobBaseClient + #region Snippet:BlobFunction_BlobClient + public static class BlobFunction_BlobClient { [FunctionName("BlobFunction")] public static async Task Run( - [BlobTrigger("sample-container/sample-blob-1")] BlobBaseClient blobTriggerClient, - [Blob("sample-container/sample-blob-2")] BlobBaseClient blobClient, + [BlobTrigger("sample-container/sample-blob-1")] BlobClient blobTriggerClient, + [Blob("sample-container/sample-blob-2")] BlobClient blobClient, ILogger logger) { BlobProperties blobTriggerProperties = await blobTriggerClient.GetPropertiesAsync(); diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Bindings/StorageBlobContainerExtensions.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Bindings/StorageBlobContainerExtensions.cs index 7f67a221c1963..fab92e6f552ce 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Bindings/StorageBlobContainerExtensions.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Bindings/StorageBlobContainerExtensions.cs @@ -16,7 +16,12 @@ internal static class StorageBlobContainerExtensions public static Task GetBlobReferenceForArgumentTypeAsync(this BlobContainerClient container, string blobName, Type argumentType, CancellationToken cancellationToken) { - if (argumentType == typeof(BlockBlobClient)) + if (argumentType == typeof(BlobClient)) + { + BlobBaseClient blob = container.GetBlobClient(blobName); + return Task.FromResult(blob); + } + else if (argumentType == typeof(BlockBlobClient)) { BlobBaseClient blob = container.GetBlockBlobClient(blobName); return Task.FromResult(blob); diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobAttribute.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobAttribute.cs index 8a6668a38bac9..083e023423d86 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobAttribute.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobAttribute.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; +using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.WebJobs.Description; @@ -40,6 +41,7 @@ namespace Microsoft.Azure.WebJobs /// The parameter type can be CloudBlobContainer, CloudBlobDirectory or /// of one of the following element types: /// + /// /// /// /// diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs index 1d7a77500590e..e96cc8619c3c0 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobTriggerAttribute.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.IO; +using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.WebJobs.Description; @@ -18,6 +19,7 @@ namespace Microsoft.Azure.WebJobs /// /// The method parameter type can be one of the following: /// + /// /// /// /// diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Config/BlobsExtensionConfigProvider.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Config/BlobsExtensionConfigProvider.cs index a3af341d39b42..d04bebab1cb83 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Config/BlobsExtensionConfigProvider.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/Config/BlobsExtensionConfigProvider.cs @@ -67,17 +67,17 @@ private void InitilizeBlobBindings(ExtensionConfigContext context) // Normal blob // These are not converters because Blob/Page/Append affects how we *create* the blob. rule.BindToInput((attr, cts) => CreateBlobReference(attr, cts)); - rule.BindToInput((attr, cts) => CreateBlobReference(attr, cts)); - rule.BindToInput((attr, cts) => CreateBlobReference(attr, cts)); - - // TODO (kasobol-msft) figure out how to add binding to BlobClient. + rule.BindToInput((attr, cts) => CreateBlobReference(attr, cts)); rule.BindToInput((attr, cts) => CreateBlobReference(attr, cts)); // CloudBlobStream's derived functionality is only relevant to writing. check derived functionality rule.When("Access", FileAccess.Write). BindToInput(ConvertToCloudBlobStreamAsync); + + RegisterCommonConverters(rule); + rule.AddConverter(new StorageBlobConverter()); } private void InitializeBlobTriggerBindings(ExtensionConfigContext context) @@ -88,17 +88,20 @@ private void InitializeBlobTriggerBindings(ExtensionConfigContext context) rule.AddConverter(blob => new DirectInvokeString(blob.GetBlobPath())); rule.AddConverter(ConvertFromInvokeString); - // Common converters shared between [Blob] and [BlobTrigger] + RegisterCommonConverters(rule); + rule.AddConverter(ConvertBlobBaseClientToBlobClient); + } - // Trigger already has the IStorageBlob. Whereas BindToInput defines: Attr-->Stream. +#pragma warning disable CS0618 // Type or member is obsolete. FluentBindingRule is "Not ready for public consumption." + private void RegisterCommonConverters(FluentBindingRule rule) where T : Attribute +#pragma warning restore CS0618 // Type or member is obsolete + { // Converter manager already has Stream-->Byte[],String,TextReader - context.AddConverter(ConvertToStreamAsync); - + rule.AddConverter(ConvertToStreamAsync); // Blob type is a property of an existing blob. - // $$$ did we lose CloudBlob. That's a base class for Cloud*Blob, but does not implement ICloudBlob? - context.AddConverter(new StorageBlobConverter()); - context.AddConverter(new StorageBlobConverter()); - context.AddConverter(new StorageBlobConverter()); + rule.AddConverter(new StorageBlobConverter()); + rule.AddConverter(new StorageBlobConverter()); + rule.AddConverter(new StorageBlobConverter()); } #region Container rules @@ -134,8 +137,8 @@ private class BlobCollectionType : OpenType { private static readonly Type[] _types = new Type[] { - // TODO (kasobol-msft) figure out how to introduce BlobClient binding typeof(BlobBaseClient), + typeof(BlobClient), typeof(BlockBlobClient), typeof(PageBlobClient), typeof(AppendBlobClient), @@ -191,7 +194,15 @@ private async Task> ConvertBlobs(IAsyncEnumerable blobI switch (blobItem.Properties.BlobType) { case BlobType.Block: - src = blobContainerClient.GetBlockBlobClient(blobItem.Name); + if (typeof(T) == typeof(BlobClient)) + { + // BlobClient is simplified version of BlockBlobClient, i.e. upload results in creation of block blob. + src = blobContainerClient.GetBlobClient(blobItem.Name); + } + else + { + src = blobContainerClient.GetBlockBlobClient(blobItem.Name); + } break; case BlobType.Append: src = blobContainerClient.GetAppendBlobClient(blobItem.Name); @@ -254,6 +265,22 @@ private async Task ConvertFromInvokeString(DirectInvokeString in return blob.Item1; } + // BlobClient is simplified version of BlockBlobClient, i.e. upload results in creation of block blob. + private BlobClient ConvertBlobBaseClientToBlobClient(BlobBaseClient input, BlobTriggerAttribute attr) + { + if (input is BlockBlobClient) + { + var client = _blobServiceClientProvider.Get(attr.Connection); + var blob = client.GetBlobContainerClient(input.BlobContainerName).GetBlobClient(input.Name); + + return blob; + } + else + { + throw new InvalidOperationException($"The blob is not an {typeof(BlockBlobClient).Name}."); + } + } + private async Task CreateStreamAsync( BlobAttribute blobAttribute, ValueBindingContext context) diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTests.cs index 8c62e02ab8dc1..cbb1f2fdfbb77 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTests.cs @@ -64,6 +64,35 @@ public async Task Blob_IfBoundToCloudBlockBlob_BindsAndCreatesContainerButNotBlo Assert.False(await blob.ExistsAsync()); } + [Test] + public async Task Blob_IfBoundToBlobClient_BindsAndCreatesContainerButNotBlob() + { + // Act + var prog = new BindToBlobClientProgram(); + IHost host = new HostBuilder() + .ConfigureDefaultTestHost(prog, builder => + { + builder.AddAzureStorageBlobs() + .UseStorageServices(blobServiceClient, queueServiceClient); + }) + .Build(); + + var jobHost = host.GetJobHost(); + await jobHost.CallAsync(nameof(BindToBlobClientProgram.Run)); + + var result = prog.Result; + + // Assert + Assert.NotNull(result); + Assert.AreEqual(BlobName, result.Name); + Assert.NotNull(result.BlobContainerName); + Assert.AreEqual(ContainerName, result.BlobContainerName); + var container = GetContainerReference(blobServiceClient, ContainerName); + Assert.True(await container.ExistsAsync()); + var blob = container.GetBlockBlobClient(BlobName); + Assert.False(await blob.ExistsAsync()); + } + [Test] public async Task Blob_IfBoundToTextWriter_CreatesBlob() { @@ -123,6 +152,17 @@ public void Run( } } + private class BindToBlobClientProgram + { + public BlobClient Result { get; set; } + + public void Run( + [Blob(BlobPath)] BlobClient blob) + { + this.Result = blob; + } + } + private class BindToTextWriterProgram { public static void Run([QueueTrigger(TriggerQueueName)] string message, diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTriggerTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTriggerTests.cs index b8d4630be8283..53811f90c6efb 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTriggerTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobTriggerTests.cs @@ -54,7 +54,7 @@ private class BindToCloudBlobProgram { public static TaskCompletionSource TaskSource { get; set; } - public static void Run([BlobTrigger(BlobPath)] BlobBaseClient blob) // TODO (kasobol-msft how about binding to BlobClient?? + public static void Run([BlobTrigger(BlobPath)] BlobBaseClient blob) { TaskSource.TrySetResult(blob); } diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/HostCallTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/HostCallTests.cs index 86cdd2d91d381..7b3283f321c68 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/HostCallTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/HostCallTests.cs @@ -122,6 +122,19 @@ public async Task Blob_IfBoundToCloudBlockBlob_CanCall() await CallAsync(typeof(BlobProgram), "BindToCloudBlockBlob"); } + [Test] + public async Task Blob_IfBoundToBlobClient_CanCall() + { + // Arrange + var container = blobServiceClient.GetBlobContainerClient(ContainerName); + var inputBlob = container.GetBlockBlobClient(BlobName); + await container.CreateIfNotExistsAsync(); + await inputBlob.UploadTextAsync("ignore"); + + // Act + await CallAsync(typeof(BlobProgram), "BindToBlobClient"); + } + [Test] public async Task Blob_IfBoundToString_CanCall() { @@ -281,6 +294,50 @@ public static void Call([BlobTrigger(BlobPath)] BlockBlobClient blob) } } + [Test] + public async Task BlobTrigger_IfBoundToBlobClient_CanCall() + { + // Arrange + var container = blobServiceClient.GetBlobContainerClient(ContainerName); + var blob = container.GetBlockBlobClient(BlobName); + await container.CreateIfNotExistsAsync(); + await blob.UploadTextAsync("ignore"); + + // TODO: Remove argument once host.Call supports more flexibility. + IDictionary arguments = new Dictionary + { + { "blob", BlobPath } + }; + + // Act + var result = await CallAsync(typeof(BlobTriggerBindToBlobClientProgram), + "Call", arguments, (s) => BlobTriggerBindToBlobClientProgram.TaskSource = s); + + // Assert + Assert.NotNull(result); + } + + [Test] + public async Task BlobTrigger_IfBoundToBlobClientAndTriggerArgumentIsMissing_CallThrows() + { + // Act + Exception exception = await CallFailureAsync(typeof(BlobTriggerBindToBlobClientProgram), "Call"); + + // Assert + Assert.IsInstanceOf(exception); + Assert.AreEqual("Missing value for trigger parameter 'blob'.", exception.Message); + } + + private class BlobTriggerBindToBlobClientProgram + { + public static TaskCompletionSource TaskSource { get; set; } + + public static void Call([BlobTrigger(BlobPath)] BlobClient blob) + { + TaskSource.TrySetResult(blob); + } + } + [Test] public async Task BlobTrigger_IfBoundToCloudPageBlob_CanCall() { @@ -606,6 +663,12 @@ public static void BindToCloudBlockBlob([Blob(BlobPath)] BlockBlobClient blob) Assert.AreEqual(BlobName, blob.Name); } + public static void BindToBlobClient([Blob(BlobPath)] BlobClient blob) + { + Assert.NotNull(blob); + Assert.AreEqual(BlobName, blob.Name); + } + public static void BindToString([Blob(BlobPath)] string content) { Assert.NotNull(content); diff --git a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/BlobBindingEndToEndTests.cs b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/BlobBindingEndToEndTests.cs index 24f0768416c1f..3bdb4174c36eb 100644 --- a/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/BlobBindingEndToEndTests.cs +++ b/sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Scenario.Tests/tests/BlobBindingEndToEndTests.cs @@ -221,6 +221,14 @@ public async Task BindToIEnumerableICloudBlob() Assert.AreEqual(6, _numBlobsRead); } + [Test] + public async Task BindToIEnumerableBlobClient() + { + await _fixture.JobHost.CallAsync(typeof(BlobBindingEndToEndTests).GetMethod("IEnumerableBlobClientBinding")); + + Assert.AreEqual(6, _numBlobsRead); + } + [Test] public async Task BindToByteArray() { @@ -514,6 +522,22 @@ public static async Task IEnumerableICloudBlobBinding( _numBlobsRead = blobs.Count(); } + [NoAutomaticTrigger] + public static async Task IEnumerableBlobClientBinding( + [Blob(ContainerName)] IEnumerable blobs) + { + foreach (var blob in blobs) + { + Stream stream = await blob.OpenReadAsync(); + using (StreamReader reader = new StreamReader(stream)) + { + string content = reader.ReadToEnd(); + Assert.AreEqual(TestData, content); + } + } + _numBlobsRead = blobs.Count(); + } + [NoAutomaticTrigger] public static void StringBinding_Block( [Blob(ContainerName + "/blob1")] string blob)