Skip to content

Commit

Permalink
Added DataLake SDK (Azure#8235)
Browse files Browse the repository at this point in the history
* Added DataLake SDK
  • Loading branch information
seanmcc-msft authored Oct 25, 2019
1 parent 2e0bc39 commit 50cfba4
Show file tree
Hide file tree
Showing 535 changed files with 181,855 additions and 7 deletions.
31 changes: 31 additions & 0 deletions sdk/storage/Azure.Storage.Common/src/Constants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,37 @@ internal static class Share
}
}

/// <summary>
/// Data Lake constant values;
/// </summary>
internal static class DataLake
{
/// <summary>
/// The blob URI suffex.
/// </summary>
public const string BlobUriSuffix = "blob";

/// <summary>
/// The DFS URI suffex.
/// </summary>
public const string DfsUriSuffix = "dfs";

/// <summary>
/// The key of the object json object returned for errors.
/// </summary>
public const string ErrorKey = "error";

/// <summary>
/// The key of the error code returned for errors.
/// </summary>
public const string ErrorCodeKey = "code";

/// <summary>
/// The key of the error message returned for errors.
/// </summary>
public const string ErrorMessageKey = "message";
}

/// <summary>
/// Queue constant values
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@ public StorageTestBase(bool async, RecordedTestMode? mode = null)
"Storage_TestConfigOAuth",
() => TestConfigurations.DefaultTargetOAuthTenant);

/// <summary>
/// Gets the tenant to use for any tests that require authentication
/// with Azure AD.
/// </summary>
public TenantConfiguration TestConfigHierarchicalNamespace => GetTestConfig(
"Storage_TestConfigHierarchicalNamespace",
() => TestConfigurations.DefaultTargetHierarchicalNamespaceTenant);

/// <summary>
/// Gets a cache used for storing serialized tenant configurations. Do
/// not get values from this directly; use GetTestConfig.
Expand Down Expand Up @@ -186,7 +194,7 @@ public TokenCredential GetOAuthCredential(string tenantId, string appId, string
Recording.InstrumentClientOptions(
new TokenCredentialOptions() { AuthorityHost = authorityHost }));

public void AssertMetadataEquality(IDictionary<string, string> expected, IDictionary<string, string> actual)
public virtual void AssertMetadataEquality(IDictionary<string, string> expected, IDictionary<string, string> actual)
{
Assert.IsNotNull(expected, "Expected metadata is null");
Assert.IsNotNull(actual, "Actual metadata is null");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ public class TestConfigurations
/// </summary>
private string TargetOAuthTenantName { get; set; }

/// <summary>
/// Gets the name of the tenant in the Tenants dictionary to use for
/// any tests that require hierarchical namespace.
/// </summary>
private string TargetHierarchicalNamespaceTenantName { get; set; }

/// <summary>
/// Gets the tenant to use by default for our tests.
/// </summary>
Expand Down Expand Up @@ -98,6 +104,12 @@ public class TestConfigurations
public static TenantConfiguration DefaultTargetOAuthTenant =>
GetTenant("TargetOAuthTenant", s_configurations.Value.TargetOAuthTenantName);

/// <summary>
/// Gets a tenant to use for any tests that require hierarchical namespace
/// </summary>
public static TenantConfiguration DefaultTargetHierarchicalNamespaceTenant =>
GetTenant("TargetHierarchicalNamespaceTenant", s_configurations.Value.TargetHierarchicalNamespaceTenantName);

/// <summary>
/// When loading our test configuration, we'll check the
/// AZ_STORAGE_CONFIG_PATH first.
Expand Down Expand Up @@ -193,6 +205,7 @@ private static TestConfigurations ReadFromXml(XDocument doc)
TargetPremiumBlobTenantName = Get("TargetPremiumBlobTenant"),
TargetPreviewBlobTenantName = Get("TargetPreviewBlobTenant"),
TargetOAuthTenantName = Get("TargetOAuthTenant"),
TargetHierarchicalNamespaceTenantName = Get("TargetHierarchicalNamespaceTenant"),
Tenants =
config.Element("TenantConfigurations").Elements("TenantConfiguration")
.Select(TenantConfiguration.Parse)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<TargetSecondaryTestTenant>SecondaryTenant</TargetSecondaryTestTenant>
<TargetPreviewBlobTenant>NotInPreview</TargetPreviewBlobTenant> <!-- Internal only -->
<TargetOAuthTenant>OAuthTenant</TargetOAuthTenant>
<TargetHierarchicalNamespaceTenant>NamespaceTenant</TargetHierarchicalNamespaceTenant>

<TenantConfigurations>
<!-- Replace [ACCOUNT] and [ACCOUNT-KEY] -->
Expand All @@ -23,6 +24,7 @@
<ConnectionString>DefaultEndpointsProtocol=https;AccountName=[ACCOUNT];AccountKey=[ACCOUNT-KEY];EndpointSuffix=core.windows.net</ConnectionString>
</TenantConfiguration>

<!-- A storage account with Read-only geo-redundant access (RA-GRS) enabled -->
<!-- Replace [RAGRS-ACCOUNT] and [RAGRS-ACCOUNT-KEY] -->
<TenantConfiguration>
<TenantName>SecondaryTenant</TenantName>
Expand All @@ -39,16 +41,19 @@
<TableServiceSecondaryEndpoint>http://[RAGRS-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
</TenantConfiguration>

<!-- A storage account with Oauth access -->
<!-- Replace [OAUTH-ACCOUNT] and [OAUTH-ACCOUNT-KEY] -->
<TenantConfiguration>
<TenantName>OAuthTenant</TenantName>
<TenantType>Cloud</TenantType>
<AccountName>[OAUTH-ACCOUNT]</AccountName>
<AccountKey>[OAUTH-ACCOUNT-KEY]</AccountKey>

<ActiveDirectoryApplicationId>[ActiveDirectoryApplicationId]</ActiveDirectoryApplicationId>
<ActiveDirectoryApplicationSecret>[ActiveDirectoryApplicationSecret]</ActiveDirectoryApplicationSecret>
<ActiveDirectoryTenantId>[ActiveDirectoryTenantId]</ActiveDirectoryTenantId>
<ActiveDirectoryAuthEndpoint>https://login.microsoftonline.com/</ActiveDirectoryAuthEndpoint>

<BlobServiceEndpoint>http://[OAUTH-ACCOUNT].blob.core.windows.net</BlobServiceEndpoint>
<QueueServiceEndpoint>http://[OAUTH-ACCOUNT].queue.core.windows.net</QueueServiceEndpoint>
<TableServiceEndpoint>http://[OAUTH-ACCOUNT].table.core.windows.net</TableServiceEndpoint>
Expand All @@ -59,7 +64,8 @@
<TableServiceSecondaryEndpoint>http://[OAUTH-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
</TenantConfiguration>

<!-- Replace [RAGRS-ACCOUNT] and [RAGRS-ACCOUNT-KEY] -->
<!-- A storage account with premium preformance enabled -->
<!-- Replace [RAGRS-ACCOUNT] and [RAGRS-ACCOUNT-KEY] -->
<TenantConfiguration>
<TenantName>PremiumBlobTenant</TenantName>
<TenantType>Cloud</TenantType>
Expand All @@ -74,5 +80,28 @@
<FileServiceSecondaryEndpoint>http://[PREMIUM-ACCOUNT]-secondary.file.core.windows.net</FileServiceSecondaryEndpoint>
<TableServiceSecondaryEndpoint>http://[PREMIUM-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
</TenantConfiguration>

<!-- A storge account with hierarchical namespace enabled and OAuth access -->
<!-- Replace [NAMESPACE-ACCOUNT] and [NAMESPACE-ACCOUNT-KEY] -->
<TenantConfiguration>
<TenantName>NamespaceTenant</TenantName>
<TenantType>Cloud</TenantType>
<AccountName>[NAMESPACE-ACCOUNT]</AccountName>
<AccountKey>[NAMESPACE-ACCOUNT-KEY]</AccountKey>

<ActiveDirectoryApplicationId>[ActiveDirectoryApplicationId]</ActiveDirectoryApplicationId>
<ActiveDirectoryApplicationSecret>[ActiveDirectoryApplicationSecret]</ActiveDirectoryApplicationSecret>
<ActiveDirectoryTenantId>[ActiveDirectoryTenantId]</ActiveDirectoryTenantId>
<ActiveDirectoryAuthEndpoint>https://login.microsoftonline.com/</ActiveDirectoryAuthEndpoint>

<BlobServiceEndpoint>http://[NAMESPACE-ACCOUNT].blob.core.windows.net</BlobServiceEndpoint>
<QueueServiceEndpoint>http://[NAMESPACE-ACCOUNT].queue.core.windows.net</QueueServiceEndpoint>
<TableServiceEndpoint>http://[NAMESPACE-ACCOUNT].table.core.windows.net</TableServiceEndpoint>
<FileServiceEndpoint>http://[NAMESPACE-ACCOUNT].file.core.windows.net</FileServiceEndpoint>
<BlobServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.blob.core.windows.net</BlobServiceSecondaryEndpoint>
<QueueServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.queue.core.windows.net</QueueServiceSecondaryEndpoint>
<FileServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.file.core.windows.net</FileServiceSecondaryEndpoint>
<TableServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
</TenantConfiguration>
</TenantConfigurations>
</TestConfigurations>
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@
"012ea67d2479751a0b8c087a4185541b851bd8b16f8d91b840e51b1cb0ba6fe647997e57429265" +
"e85ef62d565db50a69ae1647d54d7bd855e4db3d8a91510e5bcbd0edfbbecaa20a7bd9ae74593d" +
"aa7b11b4")]
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@
REST API Reference for Data Lake - https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem
</Description>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Text.Json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Common\src\Azure.Storage.Common.csproj" />
<ProjectReference Include="..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Storage.Blobs.Models;
using Azure.Storage.Files.DataLake.Models;

namespace Azure.Storage.Files.DataLake
{
internal static class BlobContainerItemExtensions
{
internal static FileSystemItem ToFileSystemItem(this BlobContainerItem containerItem) =>
new FileSystemItem()
{
Name = containerItem.Name,
Properties = new FileSystemProperties()
{
LastModified = containerItem.Properties.LastModified,
LeaseStatus = containerItem.Properties.LeaseStatus.HasValue
? (Models.LeaseStatus)containerItem.Properties.LeaseStatus : default,
LeaseState = containerItem.Properties.LeaseState.HasValue
? (Models.LeaseState)containerItem.Properties.LeaseState : default,
LeaseDuration = containerItem.Properties.LeaseDuration.HasValue
? (Models.LeaseDurationType)containerItem.Properties.LeaseDuration : default,
PublicAccess = containerItem.Properties.PublicAccess.HasValue
? (Models.PublicAccessType)containerItem.Properties.PublicAccess : default,
HasImmutabilityPolicy = containerItem.Properties.HasImmutabilityPolicy,
HasLegalHold = containerItem.Properties.HasLegalHold,
ETag = containerItem.Properties.ETag
},
Metadata = containerItem.Metadata
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Storage.Blobs.Models;
using Azure.Storage.Files.DataLake.Models;

namespace Azure.Storage.Files.DataLake
{
internal static class BlobDownloadDetailsExtensions
{
internal static FileDownloadDetails ToFileDownloadDetails(this BlobDownloadDetails blobDownloadProperties) =>
new FileDownloadDetails()
{
LastModified = blobDownloadProperties.LastModified,
Metadata = blobDownloadProperties.Metadata,
ContentRange = blobDownloadProperties.ContentRange,
ETag = blobDownloadProperties.ETag,
ContentEncoding = blobDownloadProperties.ContentEncoding,
ContentDisposition = blobDownloadProperties.ContentDisposition,
ContentLanguage = blobDownloadProperties.ContentLanguage,
CopyCompletedOn = blobDownloadProperties.CopyCompletedOn,
CopyStatusDescription = blobDownloadProperties.CopyStatusDescription,
CopyId = blobDownloadProperties.CopyId,
CopyProgress = blobDownloadProperties.CopyProgress,
CopyStatus = (Models.CopyStatus)blobDownloadProperties.CopyStatus,
LeaseDuration = (Models.LeaseDurationType)blobDownloadProperties.LeaseDuration,
LeaseState = (Models.LeaseState)blobDownloadProperties.LeaseState,
LeaseStatus = (Models.LeaseStatus)blobDownloadProperties.LeaseStatus,
AcceptRanges = blobDownloadProperties.AcceptRanges,
IsServerEncrypted = blobDownloadProperties.IsServerEncrypted,
EncryptionKeySha256 = blobDownloadProperties.EncryptionKeySha256,
ContentHash = blobDownloadProperties.BlobContentHash
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Storage.Blobs.Models;
using Azure.Storage.Files.DataLake.Models;

namespace Azure.Storage.Files.DataLake
{
internal static class BlobDownloadInfoExtensions
{
internal static FileDownloadInfo ToFileDownloadInfo(this BlobDownloadInfo blobDownloadInfo) =>
new FileDownloadInfo()
{
ContentLength = blobDownloadInfo.ContentLength,
Content = blobDownloadInfo.Content,
ContentHash = blobDownloadInfo.ContentHash,
Properties = blobDownloadInfo.Details.ToFileDownloadDetails()
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Storage.Blobs.Models;
using Azure.Storage.Files.DataLake.Models;

namespace Azure.Storage.Files.DataLake
{
internal static class BlobPropertiesExtensions
{
internal static PathProperties ToPathProperties(this BlobProperties blobProperties) =>
new PathProperties()
{
LastModified = blobProperties.LastModified,
CreatedOn = blobProperties.CreatedOn,
Metadata = blobProperties.Metadata,
CopyCompletedOn = blobProperties.CopyCompletedOn,
CopyStatusDescription = blobProperties.CopyStatusDescription,
CopyId = blobProperties.CopyId,
CopyProgress = blobProperties.CopyProgress,
CopySource = blobProperties.CopySource,
IsIncrementalCopy = blobProperties.IsIncrementalCopy,
LeaseDuration = (Models.LeaseDurationType)blobProperties.LeaseDuration,
LeaseStatus = (Models.LeaseStatus)blobProperties.LeaseStatus,
LeaseState = (Models.LeaseState)blobProperties.LeaseState,
ContentLength = blobProperties.ContentLength,
ContentType = blobProperties.ContentType,
ETag = blobProperties.ETag,
ContentHash = blobProperties.ContentHash,
ContentEncoding = blobProperties.ContentEncoding,
ContentDisposition = blobProperties.ContentDisposition,
ContentLanguage = blobProperties.ContentLanguage,
CacheControl = blobProperties.CacheControl,
AcceptRanges = blobProperties.AcceptRanges,
IsServerEncrypted = blobProperties.IsServerEncrypted,
EncryptionKeySha256 = blobProperties.EncryptionKeySha256,
AccessTier = blobProperties.AccessTier,
ArchiveStatus = blobProperties.ArchiveStatus,
AccessTierChangedOn = blobProperties.AccessTierChangedOn
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Core;
using Azure.Storage.Blobs;
using Azure.Storage.Files.DataLake.Models;

namespace Azure.Storage.Files.DataLake
{
#pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion
/// <summary>
/// Provides the client configuration options for connecting to Azure Data Lake service.
/// </summary>
public class DataLakeClientOptions : BlobClientOptions
#pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion
{

}
}
Loading

0 comments on commit 50cfba4

Please sign in to comment.