From ea073b2c56d62e166efc4689ecc012175b782dfa Mon Sep 17 00:00:00 2001 From: Jingshu Pan Date: Thu, 27 Aug 2020 10:28:39 +0800 Subject: [PATCH 1/2] Added logLevel/enableReliableLogging to LogStorageSettings --- .../datafactory_resource-manager.txt | 4 ++-- .../src/CHANGELOG.md | 4 ++++ .../Generated/Models/LogStorageSettings.cs | 24 ++++++++++++++++++- ...rosoft.Azure.Management.DataFactory.csproj | 12 ++-------- .../src/Properties/AssemblyInfo.cs | 4 ++-- .../tests/JsonSamples/PipelineJsonSamples.cs | 4 +++- 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt b/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt index 3764f4d232e61..351df7e580d40 100644 --- a/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt +++ b/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt @@ -4,11 +4,11 @@ Commencing code generation Generating CSharp code Executing AutoRest command cmd.exe /c autorest.cmd https://github.com/Azure/azure-rest-api-specs/blob/master/specification/datafactory/resource-manager/readme.md --csharp --version=v2 --reflect-api-versions --tag=package-2018-06 --csharp-sdks-folder=D:\Projects\azure-sdk-for-net\sdk -2020-08-18 10:43:15 UTC +2020-08-27 02:27:19 UTC Azure-rest-api-specs repository information GitHub fork: Azure Branch: master -Commit: 271086fdddc2d13910e8fa1373d285b1512ed032 +Commit: 49ebeaecbaac5d81399152c1584db1e540ae0bc3 AutoRest information Requested version: v2 Bootstrapper version: autorest@2.0.4413 diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md index 5f7fbc33372ec..bae40ada48ebe 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for the Azure Data Factory V2 .NET SDK +## Version 4.12.0 +### Feature Additions +- Added logLevel/enableReliableLogging to LogStorageSettings + ## Version 4.11.0 ### Feature Additions - Added support for Trigger Run Cancel API diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/LogStorageSettings.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/LogStorageSettings.cs index 633975044eec1..777cc15863a80 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/LogStorageSettings.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/LogStorageSettings.cs @@ -40,11 +40,19 @@ public LogStorageSettings() /// The path to storage for storing detailed logs of /// activity execution. Type: string (or Expression with resultType /// string). - public LogStorageSettings(LinkedServiceReference linkedServiceName, IDictionary additionalProperties = default(IDictionary), object path = default(object)) + /// Gets or sets the log level, support: Info, + /// Warning. Type: string (or Expression with resultType + /// string). + /// Specifies whether to enable + /// reliable logging. Type: boolean (or Expression with resultType + /// boolean). + public LogStorageSettings(LinkedServiceReference linkedServiceName, IDictionary additionalProperties = default(IDictionary), object path = default(object), object logLevel = default(object), object enableReliableLogging = default(object)) { AdditionalProperties = additionalProperties; LinkedServiceName = linkedServiceName; Path = path; + LogLevel = logLevel; + EnableReliableLogging = enableReliableLogging; CustomInit(); } @@ -74,6 +82,20 @@ public LogStorageSettings() [JsonProperty(PropertyName = "path")] public object Path { get; set; } + /// + /// Gets or sets the log level, support: Info, Warning. Type: string + /// (or Expression with resultType string). + /// + [JsonProperty(PropertyName = "logLevel")] + public object LogLevel { get; set; } + + /// + /// Gets or sets specifies whether to enable reliable logging. Type: + /// boolean (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "enableReliableLogging")] + public object EnableReliableLogging { get; set; } + /// /// Validate the object. /// diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj index 64d4746ee4c97..99b1008d0d757 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj @@ -5,20 +5,12 @@ Microsoft.Azure.Management.DataFactory Azure Data Factory V2 is the data integration platform that goes beyond Azure Data Factory V1's orchestration and batch-processing of time-series data, with a general purpose app model supporting modern data warehousing patterns and scenarios, lift-and-shift SSIS, and data-driven SaaS applications. Compose and manage reliable and secure data integration workflows at scale. Use native ADF data connectors and Integration Runtimes to move and transform cloud and on-premises data that can be unstructured, semi-structured, and structured with Hadoop, Azure Data Lake, Spark, SQL Server, Cosmos DB and many other data platforms. - 4.11.0 + 4.12.0 Microsoft.Azure.Management.DataFactory Microsoft Azure resource management;Data Factory;ADF; diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs index f45daac9c0152..f3a6073061118 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs @@ -6,8 +6,8 @@ [assembly: AssemblyTitle("Microsoft Azure Data Factory Management Library")] [assembly: AssemblyDescription("Provides management functionality for Microsoft Azure Data Factory Resources.")] -[assembly: AssemblyVersion("4.11.0.0")] -[assembly: AssemblyFileVersion("4.11.0.0")] +[assembly: AssemblyVersion("4.12.0.0")] +[assembly: AssemblyFileVersion("4.12.0.0")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Microsoft Azure .NET SDK")] diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs index 3bc362d48a4cc..394262b2bbe9b 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs @@ -4387,7 +4387,9 @@ public class PipelineJsonSamples : JsonSampleCollection ""referenceName"": ""exampleLinkedService"", ""type"": ""LinkedServiceReference"" }, - ""path"": ""test"" + ""path"": ""test"", + ""logLevel"": ""exampleLogLevel"", + ""enableReliableLogging"": true } }, ""inputs"": [ From e48ff780089428f11660cce520a5cd9c7c09b3c0 Mon Sep 17 00:00:00 2001 From: Jingshu Pan Date: Tue, 1 Sep 2020 13:25:40 +0800 Subject: [PATCH 2/2] Update Tar GZip --- .../datafactory_resource-manager.txt | 4 +- .../src/CHANGELOG.md | 1 + .../Generated/Models/DatasetTarCompression.cs | 49 ++++++++++ .../Models/DatasetTarGZipCompression.cs | 57 +++++++++++ .../Generated/Models/TarGZipReadSettings.cs | 59 ++++++++++++ .../src/Generated/Models/TarReadSettings.cs | 59 ++++++++++++ ...rosoft.Azure.Management.DataFactory.csproj | 1 + .../tests/JsonSamples/DatasetJsonSamples.cs | 25 +++++ .../tests/JsonSamples/PipelineJsonSamples.cs | 94 +++++++++++++++++++ 9 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarCompression.cs create mode 100644 sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarGZipCompression.cs create mode 100644 sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarGZipReadSettings.cs create mode 100644 sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarReadSettings.cs diff --git a/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt b/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt index 351df7e580d40..851c0094f1a15 100644 --- a/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt +++ b/eng/mgmt/mgmtmetadata/datafactory_resource-manager.txt @@ -4,11 +4,11 @@ Commencing code generation Generating CSharp code Executing AutoRest command cmd.exe /c autorest.cmd https://github.com/Azure/azure-rest-api-specs/blob/master/specification/datafactory/resource-manager/readme.md --csharp --version=v2 --reflect-api-versions --tag=package-2018-06 --csharp-sdks-folder=D:\Projects\azure-sdk-for-net\sdk -2020-08-27 02:27:19 UTC +2020-09-01 05:25:03 UTC Azure-rest-api-specs repository information GitHub fork: Azure Branch: master -Commit: 49ebeaecbaac5d81399152c1584db1e540ae0bc3 +Commit: aa96f138f37c06bdbf3458a4fa327f08a593594c AutoRest information Requested version: v2 Bootstrapper version: autorest@2.0.4413 diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md index bae40ada48ebe..9143e2d4c91c5 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md @@ -3,6 +3,7 @@ ## Version 4.12.0 ### Feature Additions - Added logLevel/enableReliableLogging to LogStorageSettings +- Support Tar GZip compression type in Data Factory ## Version 4.11.0 ### Feature Additions diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarCompression.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarCompression.cs new file mode 100644 index 0000000000000..4b1fac814a0fc --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarCompression.cs @@ -0,0 +1,49 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using Newtonsoft.Json; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// The Tar archive method used on a dataset. + /// + [Newtonsoft.Json.JsonObject("Tar")] + public partial class DatasetTarCompression : DatasetCompression + { + /// + /// Initializes a new instance of the DatasetTarCompression class. + /// + public DatasetTarCompression() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the DatasetTarCompression class. + /// + /// Unmatched properties from the + /// message are deserialized this collection + public DatasetTarCompression(IDictionary additionalProperties = default(IDictionary)) + : base(additionalProperties) + { + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarGZipCompression.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarGZipCompression.cs new file mode 100644 index 0000000000000..e8fa46e3cd997 --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/DatasetTarGZipCompression.cs @@ -0,0 +1,57 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using Newtonsoft.Json; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// The TarGZip compression method used on a dataset. + /// + [Newtonsoft.Json.JsonObject("TarGZip")] + public partial class DatasetTarGZipCompression : DatasetCompression + { + /// + /// Initializes a new instance of the DatasetTarGZipCompression class. + /// + public DatasetTarGZipCompression() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the DatasetTarGZipCompression class. + /// + /// Unmatched properties from the + /// message are deserialized this collection + /// The TarGZip compression level. + public DatasetTarGZipCompression(IDictionary additionalProperties = default(IDictionary), object level = default(object)) + : base(additionalProperties) + { + Level = level; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + /// + /// Gets or sets the TarGZip compression level. + /// + [JsonProperty(PropertyName = "level")] + public object Level { get; set; } + + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarGZipReadSettings.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarGZipReadSettings.cs new file mode 100644 index 0000000000000..af66ed4573f9c --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarGZipReadSettings.cs @@ -0,0 +1,59 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using Newtonsoft.Json; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// The TarGZip compression read settings. + /// + public partial class TarGZipReadSettings : CompressionReadSettings + { + /// + /// Initializes a new instance of the TarGZipReadSettings class. + /// + public TarGZipReadSettings() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the TarGZipReadSettings class. + /// + /// Unmatched properties from the + /// message are deserialized this collection + /// Preserve the + /// compression file name as folder path. Type: boolean (or Expression + /// with resultType boolean). + public TarGZipReadSettings(IDictionary additionalProperties = default(IDictionary), object preserveCompressionFileNameAsFolder = default(object)) + : base(additionalProperties) + { + PreserveCompressionFileNameAsFolder = preserveCompressionFileNameAsFolder; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + /// + /// Gets or sets preserve the compression file name as folder path. + /// Type: boolean (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "preserveCompressionFileNameAsFolder")] + public object PreserveCompressionFileNameAsFolder { get; set; } + + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarReadSettings.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarReadSettings.cs new file mode 100644 index 0000000000000..173e9a28aaea6 --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/TarReadSettings.cs @@ -0,0 +1,59 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using Newtonsoft.Json; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// The Tar compression read settings. + /// + public partial class TarReadSettings : CompressionReadSettings + { + /// + /// Initializes a new instance of the TarReadSettings class. + /// + public TarReadSettings() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the TarReadSettings class. + /// + /// Unmatched properties from the + /// message are deserialized this collection + /// Preserve the + /// compression file name as folder path. Type: boolean (or Expression + /// with resultType boolean). + public TarReadSettings(IDictionary additionalProperties = default(IDictionary), object preserveCompressionFileNameAsFolder = default(object)) + : base(additionalProperties) + { + PreserveCompressionFileNameAsFolder = preserveCompressionFileNameAsFolder; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + /// + /// Gets or sets preserve the compression file name as folder path. + /// Type: boolean (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "preserveCompressionFileNameAsFolder")] + public object PreserveCompressionFileNameAsFolder { get; set; } + + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj index 99b1008d0d757..fdd9b23a11078 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj @@ -11,6 +11,7 @@ diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DatasetJsonSamples.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DatasetJsonSamples.cs index 86a97d0be5476..d3d6cda306b04 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DatasetJsonSamples.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/DatasetJsonSamples.cs @@ -1259,6 +1259,31 @@ public class DatasetJsonSamples : JsonSampleCollection } }"; + [JsonSample] + public const string BinaryWithTarGZipDataset = @" +{ + ""name"": ""BinaryDataset"", + ""properties"": { + ""type"": ""Binary"", + ""linkedServiceName"": { + ""referenceName"": ""AzureBlobStorageLinkedService"", + ""type"": ""LinkedServiceReference"" + }, + ""typeProperties"": { + ""location"": { + ""type"": ""AzureBlobStorageLocation"", + ""container"": ""ContainerName"", + ""folderPath"": ""dataflow/test/input"", + ""fileName"": ""testTgz01"" + }, + ""compression"": { + ""type"": ""TarGZip"", + ""level"": ""Optimal"" + } + } + } +}"; + [JsonSample] public const string OrcDataset = @" { diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs index 394262b2bbe9b..bb0096491a71d 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs @@ -4507,6 +4507,100 @@ public class PipelineJsonSamples : JsonSampleCollection } }"; + [JsonSample] + public const string CopyActivity_DelimitedText_AzureBlob_UntarGZip = @"{ + ""properties"": { + ""activities"": [ + { + ""type"": ""Copy"", + ""typeProperties"": { + ""source"": { + ""type"": ""DelimitedTextSource"", + ""storeSettings"": { + ""type"": ""AzureBlobStorageReadSettings"", + ""recursive"": true + }, + ""formatSettings"": { + ""type"": ""DelimitedTextReadSettings"", + ""compressionProperties"": { + ""type"": ""TarGZipReadSettings"", + ""preserveCompressionFileNameAsFolder"": false + } + } + }, + ""sink"": { + ""type"": ""DelimitedTextSink"", + ""storeSettings"": { + ""type"": ""AzureDataLakeStoreWriteSettings"", + ""recursive"": true + }, + ""formatSettings"": { + ""type"": ""DelimitedTextWriteSettings"", + ""fileExtension"": "".txt"" + } + } + }, + ""inputs"": [ + { + ""referenceName"": ""exampleDataset"", + ""type"": ""DatasetReference"" + } + ], + ""outputs"": [ + { + ""referenceName"": ""exampleDataset"", + ""type"": ""DatasetReference"" + } + ], + ""name"": ""DelimitedTextToBlob_UntarGZip"" + }, + { + ""type"": ""Copy"", + ""typeProperties"": { + ""source"": { + ""type"": ""DelimitedTextSource"", + ""storeSettings"": { + ""type"": ""AzureBlobStorageReadSettings"", + ""recursive"": true + }, + ""formatSettings"": { + ""type"": ""DelimitedTextReadSettings"", + ""compressionProperties"": { + ""type"": ""ZipDeflateReadSettings"", + ""preserveZipFileNameAsFolder"": false + } + } + }, + ""sink"": { + ""type"": ""DelimitedTextSink"", + ""storeSettings"": { + ""type"": ""AzureBlobStorageWriteSettings"", + ""recursive"": true + }, + ""formatSettings"": { + ""type"": ""DelimitedTextWriteSettings"", + ""fileExtension"": "".txt"" + } + } + }, + ""name"": ""DelimitedTextToBlob_Unzip"", + ""inputs"": [ + { + ""referenceName"": ""SourceBlobDataset"", + ""type"": ""DatasetReference"" + } + ], + ""outputs"": [ + { + ""referenceName"": ""SinkBlobDataset"", + ""type"": ""DatasetReference"" + } + ] + } + ] + } +}"; + [JsonSample] public const string CopyActivity_DelimitedText_BlobFS = @"{ ""properties"": {