Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Storage][DataMovement] Add ProviderId to resources and job plan file #39163

Merged
merged 6 commits into from
Oct 10, 2023
Merged

[Storage][DataMovement] Add ProviderId to resources and job plan file #39163

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a3bf987
Add source/dest provider id to job plan file and resources
jalauzon-msft Oct 6, 2023
8718846
Consume ProviderId
jalauzon-msft Oct 9, 2023
f1b3f6e
Export API
jalauzon-msft Oct 9, 2023
a271823
Re-ignore test
jalauzon-msft Oct 9, 2023
dec68fb
More tests
jalauzon-msft Oct 9, 2023
bd9322f
Small fixes
jalauzon-msft Oct 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorag
public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
public BlobsStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
protected override string TypeId { get { throw null; } }
protected override string ProviderId { get { throw null; } }
public Azure.Storage.DataMovement.StorageResource FromBlob(string blobUri, Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions options = null) { throw null; }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.BlobContainerClient client, Azure.Storage.DataMovement.Blobs.BlobStorageResourceContainerOptions options = null) { throw null; }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.Specialized.AppendBlobClient client, Azure.Storage.DataMovement.Blobs.AppendBlobStorageResourceOptions options = null) { throw null; }
Expand Down
2 changes: 1 addition & 1 deletion ...e/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorag
public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
public BlobsStorageResourceProvider(Azure.Storage.DataMovement.Blobs.BlobsStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
public BlobsStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
protected override string TypeId { get { throw null; } }
protected override string ProviderId { get { throw null; } }
public Azure.Storage.DataMovement.StorageResource FromBlob(string blobUri, Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions options = null) { throw null; }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.BlobContainerClient client, Azure.Storage.DataMovement.Blobs.BlobStorageResourceContainerOptions options = null) { throw null; }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Blobs.Specialized.AppendBlobClient client, Azure.Storage.DataMovement.Blobs.AppendBlobStorageResourceOptions options = null) { throw null; }
Expand Down
8 changes: 2 additions & 6 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,12 @@ internal class AppendBlobStorageResource : StorageResourceItemInternal
internal long? _length;
internal ETag? _etagDownloadLock = default;

/// <summary>
/// The identifier for the type of storage resource.
/// </summary>
protected override string ResourceId => "AppendBlob";

/// <summary>
/// Gets the Uri of the Storage Resource
/// </summary>
public override Uri Uri => BlobClient.Uri;

public override string ProviderId => "blob";

/// <summary>
/// Defines the recommended Transfer Type for the storage resource.
/// </summary>
Expand Down
2 changes: 2 additions & 0 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ internal class BlobStorageResourceContainer : StorageResourceContainerInternal
/// </summary>
public override Uri Uri => _uri;

public override string ProviderId => "blob";

/// <summary>
/// The constructor to create an instance of the BlobStorageResourceContainer.
/// </summary>
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobsStorageResourceProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ private enum CredentialType
}

/// <inheritdoc/>
protected override string TypeId => "blob";
protected override string ProviderId => "blob";

private readonly CredentialType _credentialType;
private readonly GetStorageSharedKeyCredential _getStorageSharedKeyCredential;
Expand Down
8 changes: 2 additions & 6 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,12 @@ internal class BlockBlobStorageResource : StorageResourceItemInternal
/// </summary>
private ConcurrentDictionary<long, string> _blocks;

/// <summary>
/// The identifier for the type of storage resource.
/// </summary>
protected override string ResourceId => "BlockBlob";

/// <summary>
/// Gets the Uri of the StorageResource
/// </summary>
public override Uri Uri => BlobClient.Uri;

public override string ProviderId => "blob";

/// <summary>
/// Defines the recommended Transfer Type of the storage resource.
/// </summary>
Expand Down
8 changes: 2 additions & 6 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,12 @@ internal class PageBlobStorageResource : StorageResourceItemInternal
internal long? _length;
internal ETag? _etagDownloadLock = default;

/// <summary>
/// The identifier for the type of storage resource.
/// </summary>
protected override string ResourceId => "PageBlob";

/// <summary>
/// Gets the Uri of the Storage Resource
/// </summary>
public override Uri Uri => BlobClient.Uri;

public override string ProviderId => "blob";

/// <summary>
/// Defines the recommended Transfer Type for the storage resource.
/// </summary>
Expand Down
32 changes: 32 additions & 0 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,27 @@ private static string ToResourceId(StorageResourceType type)
};
}

private static string ToProviderId(StorageResourceType type)
{
return type switch
{
StorageResourceType.BlockBlob => "blob",
StorageResourceType.PageBlob => "blob",
StorageResourceType.AppendBlob => "blob",
StorageResourceType.Local => "local",
_ => throw new NotImplementedException(),
};
}

private static Mock<DataTransferProperties> GetProperties(
string checkpointerPath,
string transferId,
string sourcePath,
string destinationPath,
string sourceResourceId,
string destinationResourceId,
string sourceProviderId,
string destinationProviderId,
bool isContainer)
{
var mock = new Mock<DataTransferProperties>(MockBehavior.Strict);
Expand All @@ -57,6 +71,8 @@ private static Mock<DataTransferProperties> GetProperties(
mock.Setup(p => p.DestinationPath).Returns(destinationPath);
mock.Setup(p => p.SourceTypeId).Returns(sourceResourceId);
mock.Setup(p => p.DestinationTypeId).Returns(destinationResourceId);
mock.Setup(p => p.SourceProviderId).Returns(sourceProviderId);
mock.Setup(p => p.DestinationProviderId).Returns(destinationProviderId);
mock.Setup(p => p.IsContainer).Returns(isContainer);
return mock;
}
Expand Down Expand Up @@ -112,8 +128,10 @@ private async Task AddJobPartToCheckpointer(
// Use mock resources that don't correspond to correct paths
var sourceMock = new Mock<StorageResource>();
sourceMock.Setup(s => s.Uri).Returns(new Uri(CheckpointerTesting.DefaultWebSourcePath));
sourceMock.Setup(s => s.ProviderId).Returns(ToProviderId(sourceType));
var destMock = new Mock<StorageResource>();
destMock.Setup(s => s.Uri).Returns(new Uri(CheckpointerTesting.DefaultWebDestinationPath));
destMock.Setup(s => s.ProviderId).Returns(ToProviderId(destinationType));
await checkpointer.AddNewJobAsync(transferId, sourceMock.Object, destMock.Object);

for (int currentPart = 0; currentPart < partCount; currentPart++)
Expand Down Expand Up @@ -159,6 +177,8 @@ public async Task RehydrateBlockBlob(
destinationPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: false).Object;

await AddJobPartToCheckpointer(
Expand Down Expand Up @@ -196,6 +216,8 @@ public async Task RehydrateBlockBlob_Options()
destinationPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: false).Object;

IDictionary<string, string> metadata = DataProvider.BuildMetadata();
Expand Down Expand Up @@ -250,6 +272,8 @@ public async Task RehydratePageBlob(
destinationPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: false).Object;

await AddJobPartToCheckpointer(
Expand Down Expand Up @@ -287,6 +311,8 @@ public async Task RehydratePageBlob_Options()
destinationPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: false).Object;

IDictionary<string, string> metadata = DataProvider.BuildMetadata();
Expand Down Expand Up @@ -341,6 +367,8 @@ public async Task RehydrateAppendBlob(
destinationPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: false).Object;

await AddJobPartToCheckpointer(
Expand Down Expand Up @@ -378,6 +406,8 @@ public async Task RehydrateAppendBlob_Options()
destinationPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: false).Object;

IDictionary<string, string> metadata = DataProvider.BuildMetadata();
Expand Down Expand Up @@ -441,6 +471,8 @@ public async Task RehydrateBlobContainer(
destinationParentPath,
ToResourceId(sourceType),
ToResourceId(destinationType),
ToProviderId(sourceType),
ToProviderId(destinationType),
isContainer: true).Object;

await AddJobPartToCheckpointer(
Expand Down
2 changes: 1 addition & 1 deletion ...e.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares
public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
public ShareFilesStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
protected override string TypeId { get { throw null; } }
protected override string ProviderId { get { throw null; } }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareDirectoryClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareFileClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
protected override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }
Expand Down
2 changes: 1 addition & 1 deletion ...e.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares
public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetStorageSharedKeyCredential getStorageSharedKeyCredentialAsync) { }
public ShareFilesStorageResourceProvider(Azure.Storage.DataMovement.Files.Shares.ShareFilesStorageResourceProvider.GetTokenCredential getTokenCredentialAsync) { }
public ShareFilesStorageResourceProvider(Azure.Storage.StorageSharedKeyCredential credential) { }
protected override string TypeId { get { throw null; } }
protected override string ProviderId { get { throw null; } }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareDirectoryClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
public Azure.Storage.DataMovement.StorageResource FromClient(Azure.Storage.Files.Shares.ShareFileClient client, Azure.Storage.DataMovement.Files.Shares.ShareFileStorageResourceOptions options = null) { throw null; }
protected override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }
Expand Down
2 changes: 2 additions & 0 deletions ...age/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ internal class ShareDirectoryStorageResourceContainer : StorageResourceContainer

public override Uri Uri => ShareDirectoryClient.Uri;

public override string ProviderId => "share";

internal ShareDirectoryStorageResourceContainer(ShareDirectoryClient shareDirectoryClient, ShareFileStorageResourceOptions options)
{
ShareDirectoryClient = shareDirectoryClient;
Expand Down
2 changes: 2 additions & 0 deletions sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ internal class ShareFileStorageResource : StorageResourceItemInternal

public override Uri Uri => ShareFileClient.Uri;

public override string ProviderId => "share";

protected override string ResourceId => "ShareFile";

protected override DataTransferOrder TransferType => DataTransferOrder.Sequential;
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ private enum CredentialType
}

/// <inheritdoc/>
protected override string TypeId => "share";
protected override string ProviderId => "share";

private readonly CredentialType _credentialType;
private readonly GetStorageSharedKeyCredential _getStorageSharedKeyCredential;
Expand Down
7 changes: 5 additions & 2 deletions sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,11 @@ public partial class DataTransferProperties
protected internal DataTransferProperties() { }
public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } }
public virtual string DestinationPath { get { throw null; } }
public virtual string DestinationProviderId { get { throw null; } }
public virtual string DestinationTypeId { get { throw null; } }
public virtual bool IsContainer { get { throw null; } }
public virtual string SourcePath { get { throw null; } }
public virtual string SourceProviderId { get { throw null; } }
public virtual string SourceTypeId { get { throw null; } }
public virtual string TransferId { get { throw null; } }
}
Expand Down Expand Up @@ -97,7 +99,7 @@ protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferSta
public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider
{
public LocalFilesStorageResourceProvider() { }
protected internal override string TypeId { get { throw null; } }
protected internal override string ProviderId { get { throw null; } }
protected internal override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }
public Azure.Storage.DataMovement.StorageResourceContainer FromDirectory(string directoryPath) { throw null; }
public Azure.Storage.DataMovement.StorageResourceItem FromFile(string filePath) { throw null; }
Expand All @@ -113,6 +115,7 @@ public abstract partial class StorageResource
{
protected StorageResource() { }
protected internal abstract bool IsContainer { get; }
public abstract string ProviderId { get; }
public abstract System.Uri Uri { get; }
}
public abstract partial class StorageResourceCheckpointData
Expand Down Expand Up @@ -168,7 +171,7 @@ public StorageResourceProperties(System.DateTimeOffset lastModified, System.Date
public abstract partial class StorageResourceProvider
{
protected StorageResourceProvider() { }
protected internal abstract string TypeId { get; }
protected internal abstract string ProviderId { get; }
protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromSourceAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
}
Expand Down
7 changes: 5 additions & 2 deletions sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,11 @@ public partial class DataTransferProperties
protected internal DataTransferProperties() { }
public virtual Azure.Storage.DataMovement.TransferCheckpointStoreOptions Checkpointer { get { throw null; } }
public virtual string DestinationPath { get { throw null; } }
public virtual string DestinationProviderId { get { throw null; } }
public virtual string DestinationTypeId { get { throw null; } }
public virtual bool IsContainer { get { throw null; } }
public virtual string SourcePath { get { throw null; } }
public virtual string SourceProviderId { get { throw null; } }
public virtual string SourceTypeId { get { throw null; } }
public virtual string TransferId { get { throw null; } }
}
Expand Down Expand Up @@ -97,7 +99,7 @@ protected internal DataTransferStatus(Azure.Storage.DataMovement.DataTransferSta
public partial class LocalFilesStorageResourceProvider : Azure.Storage.DataMovement.StorageResourceProvider
{
public LocalFilesStorageResourceProvider() { }
protected internal override string TypeId { get { throw null; } }
protected internal override string ProviderId { get { throw null; } }
protected internal override System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken) { throw null; }
public Azure.Storage.DataMovement.StorageResourceContainer FromDirectory(string directoryPath) { throw null; }
public Azure.Storage.DataMovement.StorageResourceItem FromFile(string filePath) { throw null; }
Expand All @@ -113,6 +115,7 @@ public abstract partial class StorageResource
{
protected StorageResource() { }
protected internal abstract bool IsContainer { get; }
public abstract string ProviderId { get; }
jalauzon-msft marked this conversation as resolved.
Show resolved Hide resolved
public abstract System.Uri Uri { get; }
}
public abstract partial class StorageResourceCheckpointData
Expand Down Expand Up @@ -168,7 +171,7 @@ public StorageResourceProperties(System.DateTimeOffset lastModified, System.Date
public abstract partial class StorageResourceProvider
{
protected StorageResourceProvider() { }
protected internal abstract string TypeId { get; }
protected internal abstract string ProviderId { get; }
protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromDestinationAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
protected internal abstract System.Threading.Tasks.Task<Azure.Storage.DataMovement.StorageResource> FromSourceAsync(Azure.Storage.DataMovement.DataTransferProperties properties, System.Threading.CancellationToken cancellationToken);
}
Expand Down
Loading
Loading