[Storage] [DataMovement] [Blobs] Stress Scenarios (#46400)

* Added separate sln file for Storage Data Movement and start of stress testing for DM Blobs

* Fix to Stress Test README

* Fix path for .slnf file

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP - builds

* WIP - docker issues

* WIP - build issue in docker

* WIP - builds locally, not in Docker

* WIP - runs until hitting excuting yaml file

* WIP - helm won't commit run

* WIP - fixing upload scenario test

* WIP - helm still not pushing commit; added more options for scenarios

* WIP

* Cleanup

* WIP

* More cleanup and some PR Comments

* Part 2 PR comments

* WIP

* WIP

* WIP - Created all scenario types and implementation

* Cleanup

* Update script to include all new scenarios

* Cleanup

* PR Comments: Updated Create Blob Helper methods; removed file close; short hand

* PR comments: Removed unnecessary init transfermanager; reapplied options

sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml

@@ -1,7 +1,92 @@
 scenarios:
 - image: Dockerfile
   imageBuildDir: ../../../..
-  testScenario: uploadsingleblockblob
   Scenario: uploadsingleblockblob
+  testScenario: uploadsingleblockblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: uploaddirectoryblockBlob
+  testScenario: uploaddirectoryblockBlob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: downloadsingleblockblob
+  testScenario: downloadsingleblockblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: downloaddirectoryblockblob
+  testScenario: downloaddirectoryblockblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: copysingleblockblob
+  testScenario: copysingleblockblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: copydirectoryblockblob
+  testScenario: copydirectoryblockblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: uploadsingleappendblob
+  testScenario: uploadsingleappendblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: uploaddirectoryappendblob
+  testScenario: uploaddirectoryappendblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: downloadsingleappendblob
+  testScenario: downloadsingleappendblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: downloaddirectoryappendblob
+  testScenario: downloaddirectoryappendblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: copysingleappendblob
+  testScenario: copysingleappendblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: copydirectoryappendblob
+  testScenario: copydirectoryappendblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: uploadsinglepageblob
+  testScenario: uploadsinglepageblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: uploaddirectorypageblob
+  testScenario: uploaddirectorypageblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: downloadsinglepageblob
+  testScenario: downloadsinglepageblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: downloaddirectorypageblob
+  testScenario: downloaddirectorypageblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: copysinglepageblob
+  testScenario: copysinglepageblob
+  imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
+- image: Dockerfile
+  imageBuildDir: ../../../..
+  Scenario: copydirectorypageblob
+  testScenario: copydirectorypageblob
   imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
 

sdk/storage/Azure.Storage.DataMovement.Blobs/stress/scenarios-matrix.yaml

@@ -6,4 +6,72 @@ matrix:
     uploadsingleblockblob:
       testScenario: uploadsingleblockblob
       image: Dockerfile
+      imageBuildDir: "../../../.."
+    uploaddirectoryblockblob:
+      testScenario: uploaddirectoryblockblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    downloadsingleblockblob:
+      testScenario: downloadsingleblockblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    downloaddirectoryblockblob:
+      testScenario: downloaddirectoryblockblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    copysingleblockblob:
+      testScenario: copysingleblockblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    copydirectoryblockblob:
+      testScenario: copydirectoryblockblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    uploadsingleappendblob:
+      testScenario: uploadsingleappendblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    uploaddirectoryappendblob:
+      testScenario: uploaddirectoryappendblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    downloadsingleappendblob:
+      testScenario: downloadsingleappendblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    downloaddirectoryappendblob:
+      testScenario: downloaddirectoryappendblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    copysingleappendblob:
+      testScenario: copysingleappendblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    copydirectoryappendblob:
+      testScenario: copydirectoryappendblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    uploadsinglepageblob:
+      testScenario: uploadsinglepageblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    uploaddirectorypageblob:
+      testScenario: uploaddirectorypageblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    downloadsinglepageblob:
+      testScenario: downloadsinglepageblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    downloaddirectorypageblob:
+      testScenario: downloaddirectorypageblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    copysinglepageblob:
+      testScenario: copysinglepageblob
+      image: Dockerfile
+      imageBuildDir: "../../../.."
+    copydirectorypageblob:
+      testScenario: copydirectorypageblob
+      image: Dockerfile
       imageBuildDir: "../../../.."

sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj

@@ -34,6 +34,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="$(AzureStorageSharedSources)Constants.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
+    <Compile Include="$(AzureStorageSharedSources)WindowStream.cs" LinkBase="Shared\Storage" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="$(AzureStorageSharedTestSources)TemporaryFileStream.cs" LinkBase="SharedSource\Storage" />

sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs

@@ -1,18 +1,25 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure.Storage.Shared;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Specialized;
+using NUnit.Framework;
+using Azure.Storage.Blobs.Models;
 
 namespace Azure.Storage.DataMovement.Blobs.Stress
 {
     public static class BlobTestSetupHelper
     {
-        public static async Task CreateDirectoryBlockBlobsAsync(
+        private const int DefaultBufferSize = 4 * Constants.KB * Constants.KB;
+
+        public static async Task CreateBlobsInDirectoryAsync(
             BlobContainerClient container,
+            BlobType blobType,
             string pathPrefix,
             int blobCount = 2,
             long objectLength = Constants.KB * 4,
@@ -21,10 +28,27 @@ public static async Task CreateDirectoryBlockBlobsAsync(
             for (int i = 0; i < blobCount; i++)
             {
                 string blobName = $"{pathPrefix}/{TestSetupHelper.Randomize("blob")}";
-                await CreateBlockBlobAsync(
+                if (blobType == BlobType.Append)
+                {
+                    await CreateAppendBlobAsync(
+                        container.GetAppendBlobClient(blobName),
+                        objectLength,
+                        cancellationToken);
+                }
+                else if (blobType == BlobType.Page)
+                {
+                    await CreatePageBlobAsync(
+                        container.GetPageBlobClient(blobName),
+                        objectLength,
+                        cancellationToken);
+                }
+                else
+                {
+                    await CreateBlockBlobAsync(
                     container.GetBlockBlobClient(blobName),
                     objectLength,
                     cancellationToken);
+                }
             }
         }
 
@@ -34,18 +58,52 @@ public static async Task CreateBlockBlobAsync(
             CancellationToken cancellationToken = default)
         {
             using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
-            if (originalStream != default)
+            var data = new byte[0];
+            using (var stream = new MemoryStream(data))
             {
-                await blockBlobClient.UploadAsync(originalStream, cancellationToken: cancellationToken);
+                await blockBlobClient.UploadAsync(
+                    content: stream,
+                    cancellationToken: cancellationToken);
             }
-            else
+        }
+
+        public static async Task CreateAppendBlobAsync(
+            AppendBlobClient appendBlobClient,
+            long? objectLength = Constants.KB * 4,
+            CancellationToken cancellationToken = default)
+        {
+            await appendBlobClient.CreateIfNotExistsAsync();
+            if (objectLength.Value > 0)
+            {
+                using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
+                long offset = 0;
+                long blockSize = Math.Min(DefaultBufferSize, objectLength.Value);
+                while (offset < objectLength.Value)
+                {
+                    Stream partStream = WindowStream.GetWindow(originalStream, blockSize);
+                    await appendBlobClient.AppendBlockAsync(partStream);
+                    offset += blockSize;
+                }
+            }
+        }
+
+        public static async Task CreatePageBlobAsync(
+            PageBlobClient pageBlobClient,
+            long? objectLength = Constants.KB * 4,
+            CancellationToken cancellationToken = default)
+        {
+            Assert.IsTrue(objectLength.Value % (Constants.KB / 2) == 0, "Cannot create page blob that's not a multiple of 512");
+            await pageBlobClient.CreateIfNotExistsAsync(objectLength.Value);
+            if (objectLength.Value > 0)
             {
-                var data = new byte[0];
-                using (var stream = new MemoryStream(data))
+                using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
+                long offset = 0;
+                long blockSize = Math.Min(DefaultBufferSize, objectLength.Value);
+                while (offset < objectLength.Value)
                 {
-                    await blockBlobClient.UploadAsync(
-                        content: stream,
-                        cancellationToken: cancellationToken);
+                    Stream partStream = WindowStream.GetWindow(originalStream, blockSize);
+                    await pageBlobClient.UploadPagesAsync(partStream, offset);
+                    offset += blockSize;
                 }
             }
         }

sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/TestSetupHelper.cs

@@ -72,10 +72,8 @@ public static async Task<StorageResource> GetTemporaryFileStorageResourceAsync(
             string localSourceFile = Path.Combine(prefixPath, fileName);
             // create a new file and copy contents of stream into it, and then close the FileStream
             // so the StagedUploadAsync call is not prevented from reading using its FileStream.
-            Console.Out.Write($"Creating File: {localSourceFile}..");
             using (FileStream fileStream = File.Create(localSourceFile))
             {
-                Console.Out.WriteLine("Copying Stream to File..");
                 await originalStream.CopyToAsync(
                     fileStream,
                     bufferSize,

sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs

@@ -14,29 +14,32 @@ namespace Azure.Storage.DataMovement.Blobs.Stress
     public abstract class BlobScenarioBase : TestScenarioBase
     {
         protected internal readonly Uri _destinationBlobUri;
+        protected internal int _blobSize;
         protected internal readonly TokenCredential _tokenCredential;
         protected internal BlobsStorageResourceProvider _blobsStorageResourceProvider;
         protected internal LocalFilesStorageResourceProvider _localFilesStorageResourceProvider;
-        protected internal BlobServiceClient _destinationServiceClient;
+        protected internal BlobServiceClient _blobServiceClient;
         protected internal readonly TransferManagerOptions _transferManagerOptions;
         protected internal readonly DataTransferOptions _dataTransferOptions;
 
         public BlobScenarioBase(
-            Uri destinationBlobUri,
+            Uri blobUri,
+            int? blobSize,
             TransferManagerOptions transferManagerOptions,
             DataTransferOptions dataTransferOptions,
             TokenCredential tokenCredential,
             Metrics metrics,
             string testRunId)
             : base(metrics, testRunId)
         {
-            _destinationBlobUri = destinationBlobUri;
+            _destinationBlobUri = blobUri;
+            _blobSize = blobSize != default ? blobSize.Value : DataMovementBlobStressConstants.DefaultObjectSize;
             _transferManagerOptions = transferManagerOptions;
             _dataTransferOptions = dataTransferOptions;
             _tokenCredential = tokenCredential;
             _blobsStorageResourceProvider = new BlobsStorageResourceProvider(tokenCredential);
             _localFilesStorageResourceProvider = new LocalFilesStorageResourceProvider();
-            _destinationServiceClient = new BlobServiceClient(destinationBlobUri, tokenCredential);
+            _blobServiceClient = new BlobServiceClient(blobUri, tokenCredential);
         }
     }
 }