Skip to content

Commit

Permalink
[Storage] [DataMovement] [Blobs] Stress Scenarios (#46400)
Browse files Browse the repository at this point in the history
* Added separate sln file for Storage Data Movement and start of stress testing for DM Blobs

* Fix to Stress Test README

* Fix path for .slnf file

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP - builds

* WIP - docker issues

* WIP - build issue in docker

* WIP - builds locally, not in Docker

* WIP - runs until hitting excuting yaml file

* WIP - helm won't commit run

* WIP - fixing upload scenario test

* WIP - helm still not pushing commit; added more options for scenarios

* WIP

* Cleanup

* WIP

* More cleanup and some PR Comments

* Part 2 PR comments

* WIP

* WIP

* WIP - Created all scenario types and implementation

* Cleanup

* Update script to include all new scenarios

* Cleanup

* PR Comments: Updated Create Blob Helper methods; removed file close; short hand

* PR comments: Removed unnecessary init transfermanager; reapplied options
  • Loading branch information
amnguye authored Oct 3, 2024
1 parent 1a1b91b commit f5d4fe2
Show file tree
Hide file tree
Showing 36 changed files with 1,461 additions and 139 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,92 @@
scenarios:
- image: Dockerfile
imageBuildDir: ../../../..
testScenario: uploadsingleblockblob
Scenario: uploadsingleblockblob
testScenario: uploadsingleblockblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: uploaddirectoryblockBlob
testScenario: uploaddirectoryblockBlob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: downloadsingleblockblob
testScenario: downloadsingleblockblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: downloaddirectoryblockblob
testScenario: downloaddirectoryblockblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: copysingleblockblob
testScenario: copysingleblockblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: copydirectoryblockblob
testScenario: copydirectoryblockblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: uploadsingleappendblob
testScenario: uploadsingleappendblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: uploaddirectoryappendblob
testScenario: uploaddirectoryappendblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: downloadsingleappendblob
testScenario: downloadsingleappendblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: downloaddirectoryappendblob
testScenario: downloaddirectoryappendblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: copysingleappendblob
testScenario: copysingleappendblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: copydirectoryappendblob
testScenario: copydirectoryappendblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: uploadsinglepageblob
testScenario: uploadsinglepageblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: uploaddirectorypageblob
testScenario: uploaddirectorypageblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: downloadsinglepageblob
testScenario: downloadsinglepageblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: downloaddirectorypageblob
testScenario: downloaddirectorypageblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: copysinglepageblob
testScenario: copysinglepageblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
- image: Dockerfile
imageBuildDir: ../../../..
Scenario: copydirectorypageblob
testScenario: copydirectorypageblob
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye

Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,72 @@ matrix:
uploadsingleblockblob:
testScenario: uploadsingleblockblob
image: Dockerfile
imageBuildDir: "../../../.."
uploaddirectoryblockblob:
testScenario: uploaddirectoryblockblob
image: Dockerfile
imageBuildDir: "../../../.."
downloadsingleblockblob:
testScenario: downloadsingleblockblob
image: Dockerfile
imageBuildDir: "../../../.."
downloaddirectoryblockblob:
testScenario: downloaddirectoryblockblob
image: Dockerfile
imageBuildDir: "../../../.."
copysingleblockblob:
testScenario: copysingleblockblob
image: Dockerfile
imageBuildDir: "../../../.."
copydirectoryblockblob:
testScenario: copydirectoryblockblob
image: Dockerfile
imageBuildDir: "../../../.."
uploadsingleappendblob:
testScenario: uploadsingleappendblob
image: Dockerfile
imageBuildDir: "../../../.."
uploaddirectoryappendblob:
testScenario: uploaddirectoryappendblob
image: Dockerfile
imageBuildDir: "../../../.."
downloadsingleappendblob:
testScenario: downloadsingleappendblob
image: Dockerfile
imageBuildDir: "../../../.."
downloaddirectoryappendblob:
testScenario: downloaddirectoryappendblob
image: Dockerfile
imageBuildDir: "../../../.."
copysingleappendblob:
testScenario: copysingleappendblob
image: Dockerfile
imageBuildDir: "../../../.."
copydirectoryappendblob:
testScenario: copydirectoryappendblob
image: Dockerfile
imageBuildDir: "../../../.."
uploadsinglepageblob:
testScenario: uploadsinglepageblob
image: Dockerfile
imageBuildDir: "../../../.."
uploaddirectorypageblob:
testScenario: uploaddirectorypageblob
image: Dockerfile
imageBuildDir: "../../../.."
downloadsinglepageblob:
testScenario: downloadsinglepageblob
image: Dockerfile
imageBuildDir: "../../../.."
downloaddirectorypageblob:
testScenario: downloaddirectorypageblob
image: Dockerfile
imageBuildDir: "../../../.."
copysinglepageblob:
testScenario: copysinglepageblob
image: Dockerfile
imageBuildDir: "../../../.."
copydirectorypageblob:
testScenario: copydirectorypageblob
image: Dockerfile
imageBuildDir: "../../../.."
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageSharedSources)Constants.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
<Compile Include="$(AzureStorageSharedSources)WindowStream.cs" LinkBase="Shared\Storage" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageSharedTestSources)TemporaryFileStream.cs" LinkBase="SharedSource\Storage" />
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Azure.Storage.Shared;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Specialized;
using NUnit.Framework;
using Azure.Storage.Blobs.Models;

namespace Azure.Storage.DataMovement.Blobs.Stress
{
public static class BlobTestSetupHelper
{
public static async Task CreateDirectoryBlockBlobsAsync(
private const int DefaultBufferSize = 4 * Constants.KB * Constants.KB;

public static async Task CreateBlobsInDirectoryAsync(
BlobContainerClient container,
BlobType blobType,
string pathPrefix,
int blobCount = 2,
long objectLength = Constants.KB * 4,
Expand All @@ -21,10 +28,27 @@ public static async Task CreateDirectoryBlockBlobsAsync(
for (int i = 0; i < blobCount; i++)
{
string blobName = $"{pathPrefix}/{TestSetupHelper.Randomize("blob")}";
await CreateBlockBlobAsync(
if (blobType == BlobType.Append)
{
await CreateAppendBlobAsync(
container.GetAppendBlobClient(blobName),
objectLength,
cancellationToken);
}
else if (blobType == BlobType.Page)
{
await CreatePageBlobAsync(
container.GetPageBlobClient(blobName),
objectLength,
cancellationToken);
}
else
{
await CreateBlockBlobAsync(
container.GetBlockBlobClient(blobName),
objectLength,
cancellationToken);
}
}
}

Expand All @@ -34,18 +58,52 @@ public static async Task CreateBlockBlobAsync(
CancellationToken cancellationToken = default)
{
using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
if (originalStream != default)
var data = new byte[0];
using (var stream = new MemoryStream(data))
{
await blockBlobClient.UploadAsync(originalStream, cancellationToken: cancellationToken);
await blockBlobClient.UploadAsync(
content: stream,
cancellationToken: cancellationToken);
}
else
}

public static async Task CreateAppendBlobAsync(
AppendBlobClient appendBlobClient,
long? objectLength = Constants.KB * 4,
CancellationToken cancellationToken = default)
{
await appendBlobClient.CreateIfNotExistsAsync();
if (objectLength.Value > 0)
{
using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
long offset = 0;
long blockSize = Math.Min(DefaultBufferSize, objectLength.Value);
while (offset < objectLength.Value)
{
Stream partStream = WindowStream.GetWindow(originalStream, blockSize);
await appendBlobClient.AppendBlockAsync(partStream);
offset += blockSize;
}
}
}

public static async Task CreatePageBlobAsync(
PageBlobClient pageBlobClient,
long? objectLength = Constants.KB * 4,
CancellationToken cancellationToken = default)
{
Assert.IsTrue(objectLength.Value % (Constants.KB / 2) == 0, "Cannot create page blob that's not a multiple of 512");
await pageBlobClient.CreateIfNotExistsAsync(objectLength.Value);
if (objectLength.Value > 0)
{
var data = new byte[0];
using (var stream = new MemoryStream(data))
using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
long offset = 0;
long blockSize = Math.Min(DefaultBufferSize, objectLength.Value);
while (offset < objectLength.Value)
{
await blockBlobClient.UploadAsync(
content: stream,
cancellationToken: cancellationToken);
Stream partStream = WindowStream.GetWindow(originalStream, blockSize);
await pageBlobClient.UploadPagesAsync(partStream, offset);
offset += blockSize;
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,8 @@ public static async Task<StorageResource> GetTemporaryFileStorageResourceAsync(
string localSourceFile = Path.Combine(prefixPath, fileName);
// create a new file and copy contents of stream into it, and then close the FileStream
// so the StagedUploadAsync call is not prevented from reading using its FileStream.
Console.Out.Write($"Creating File: {localSourceFile}..");
using (FileStream fileStream = File.Create(localSourceFile))
{
Console.Out.WriteLine("Copying Stream to File..");
await originalStream.CopyToAsync(
fileStream,
bufferSize,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,32 @@ namespace Azure.Storage.DataMovement.Blobs.Stress
public abstract class BlobScenarioBase : TestScenarioBase
{
protected internal readonly Uri _destinationBlobUri;
protected internal int _blobSize;
protected internal readonly TokenCredential _tokenCredential;
protected internal BlobsStorageResourceProvider _blobsStorageResourceProvider;
protected internal LocalFilesStorageResourceProvider _localFilesStorageResourceProvider;
protected internal BlobServiceClient _destinationServiceClient;
protected internal BlobServiceClient _blobServiceClient;
protected internal readonly TransferManagerOptions _transferManagerOptions;
protected internal readonly DataTransferOptions _dataTransferOptions;

public BlobScenarioBase(
Uri destinationBlobUri,
Uri blobUri,
int? blobSize,
TransferManagerOptions transferManagerOptions,
DataTransferOptions dataTransferOptions,
TokenCredential tokenCredential,
Metrics metrics,
string testRunId)
: base(metrics, testRunId)
{
_destinationBlobUri = destinationBlobUri;
_destinationBlobUri = blobUri;
_blobSize = blobSize != default ? blobSize.Value : DataMovementBlobStressConstants.DefaultObjectSize;
_transferManagerOptions = transferManagerOptions;
_dataTransferOptions = dataTransferOptions;
_tokenCredential = tokenCredential;
_blobsStorageResourceProvider = new BlobsStorageResourceProvider(tokenCredential);
_localFilesStorageResourceProvider = new LocalFilesStorageResourceProvider();
_destinationServiceClient = new BlobServiceClient(destinationBlobUri, tokenCredential);
_blobServiceClient = new BlobServiceClient(blobUri, tokenCredential);
}
}
}
Loading

0 comments on commit f5d4fe2

Please sign in to comment.