From 2496c9740dd68081a7125e3c5bdb1cca59189e1c Mon Sep 17 00:00:00 2001 From: Jingshu Pan Date: Thu, 6 Aug 2020 10:02:23 +0800 Subject: [PATCH] [DataFactory]Support source sink level staging --- .../datafactory_resource-manager.txt | 4 +- .../src/CHANGELOG.md | 1 + .../src/Generated/Models/DataFlowSink.cs | 15 +++++- .../src/Generated/Models/DataFlowSource.cs | 15 +++++- ...rosoft.Azure.Management.DataFactory.csproj | 1 + .../tests/JsonSamples/DataFlowJsonSamples.cs | 46 +++++++++++++++++++ 6 files changed, 78 insertions(+), 4 deletions(-) diff --git a/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt b/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt index 26811d6930226..ea0948071a12f 100644 --- a/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt +++ b/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt @@ -4,11 +4,11 @@ Commencing code generation Generating CSharp code Executing AutoRest command cmd.exe /c autorest.cmd https://github.com/Azure/azure-rest-api-specs/blob/master/specification/datafactory/resource-manager/readme.md --csharp --version=v2 --reflect-api-versions --tag=package-2018-06 --csharp-sdks-folder=D:\Projects\azure-sdk-for-net\sdk -2020-07-30 05:05:28 UTC +2020-08-06 02:01:09 UTC Azure-rest-api-specs repository information GitHub fork: Azure Branch: master -Commit: b8630cc7b5869fbb764eeca3a618b23141e612db +Commit: 9ea4dc2d936d8eaae61d507242771b1d6447a96e AutoRest information Requested version: v2 Bootstrapper version: autorest@2.0.4413 diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md index 992ea8cf4dd7a..50e876a11ae4d 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md @@ -6,6 +6,7 @@ - Added column delimiter for SAP Table/open hub - Added azureCloudType for Aad Auth - Added snapshot to Azure File Storage linked service +- Added source/sink level staging support in dataflow ## Version 4.10.0 ### Feature Additions diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSink.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSink.cs index 7fde44a13732b..5e73cd8cb4031 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSink.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSink.cs @@ -35,12 +35,15 @@ public DataFlowSink() /// Linked service reference. /// Schema linked service /// reference. - public DataFlowSink(string name, string description = default(string), DatasetReference dataset = default(DatasetReference), LinkedServiceReference linkedService = default(LinkedServiceReference), LinkedServiceReference schemaLinkedService = default(LinkedServiceReference)) + /// Staging info for execute data flow + /// activity. + public DataFlowSink(string name, string description = default(string), DatasetReference dataset = default(DatasetReference), LinkedServiceReference linkedService = default(LinkedServiceReference), LinkedServiceReference schemaLinkedService = default(LinkedServiceReference), DataFlowStagingInfo staging = default(DataFlowStagingInfo)) : base(name, description) { Dataset = dataset; LinkedService = linkedService; SchemaLinkedService = schemaLinkedService; + Staging = staging; CustomInit(); } @@ -67,6 +70,12 @@ public DataFlowSink() [JsonProperty(PropertyName = "schemaLinkedService")] public LinkedServiceReference SchemaLinkedService { get; set; } + /// + /// Gets or sets staging info for execute data flow activity. + /// + [JsonProperty(PropertyName = "staging")] + public DataFlowStagingInfo Staging { get; set; } + /// /// Validate the object. /// @@ -88,6 +97,10 @@ public override void Validate() { SchemaLinkedService.Validate(); } + if (Staging != null) + { + Staging.Validate(); + } } } } diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSource.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSource.cs index 838e69cc4a251..cd976967cba19 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSource.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DataFlowSource.cs @@ -35,12 +35,15 @@ public DataFlowSource() /// Linked service reference. /// Schema linked service /// reference. - public DataFlowSource(string name, string description = default(string), DatasetReference dataset = default(DatasetReference), LinkedServiceReference linkedService = default(LinkedServiceReference), LinkedServiceReference schemaLinkedService = default(LinkedServiceReference)) + /// Staging info for execute data flow + /// activity. + public DataFlowSource(string name, string description = default(string), DatasetReference dataset = default(DatasetReference), LinkedServiceReference linkedService = default(LinkedServiceReference), LinkedServiceReference schemaLinkedService = default(LinkedServiceReference), DataFlowStagingInfo staging = default(DataFlowStagingInfo)) : base(name, description) { Dataset = dataset; LinkedService = linkedService; SchemaLinkedService = schemaLinkedService; + Staging = staging; CustomInit(); } @@ -67,6 +70,12 @@ public DataFlowSource() [JsonProperty(PropertyName = "schemaLinkedService")] public LinkedServiceReference SchemaLinkedService { get; set; } + /// + /// Gets or sets staging info for execute data flow activity. + /// + [JsonProperty(PropertyName = "staging")] + public DataFlowStagingInfo Staging { get; set; } + /// /// Validate the object. /// @@ -88,6 +97,10 @@ public override void Validate() { SchemaLinkedService.Validate(); } + if (Staging != null) + { + Staging.Validate(); + } } } } diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj index 39c9bf30f468a..28cfe7cef4c71 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj @@ -14,6 +14,7 @@ - Added column delimiter for SAP Table/open hub - Added azureCloudType for Aad Auth - Added snapshot to Azure File Storage linked service +- Added source/sink level staging support in dataflow ]]> diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DataFlowJsonSamples.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DataFlowJsonSamples.cs index a37aa53bfd054..55f8f1321ab9f 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DataFlowJsonSamples.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DataFlowJsonSamples.cs @@ -92,5 +92,51 @@ public class DataFlowJsonSamples : JsonSampleCollection } } "; + + [JsonSample] + public const string MappingDataFlowWithSourceSinkStaging = @" + { + ""name"": ""exampleDataFlow"", + ""properties"": { + ""description"": ""Sample demo data flow to convert currencies showing usage of union, derive and conditional split transformation."", + ""type"": ""MappingDataFlow"", + ""typeProperties"": { + ""sources"": [ + { + ""dataset"": { + ""referenceName"": ""CurrencyDatasetUSD"", + ""type"": ""DatasetReference"" + }, + ""name"": ""USDCurrency"", + ""staging"": { + ""linkedService"": { + ""referenceName"": ""blob_store_sasToken01"", + ""type"": ""LinkedServiceReference"" + }, + ""folderPath"":""testcontainer01"" + } + } + ], + ""sinks"": [ + { + ""dataset"": { + ""referenceName"": ""USDOutput"", + ""type"": ""DatasetReference"" + }, + ""name"": ""USDSink"", + ""staging"": { + ""linkedService"": { + ""referenceName"": ""blob_store_sasToken02"", + ""type"": ""LinkedServiceReference"" + }, + ""folderPath"":""testcontainer02"" + } + } + ], + ""script"": ""some script"" + } + } + } +"; } }