From f756788799f5681f7ccc10d0e2ec15a41f030bfe Mon Sep 17 00:00:00 2001 From: Yiliu Date: Thu, 14 Mar 2019 15:56:38 +0800 Subject: [PATCH 01/11] Modify the public swagger by reference to private swagger --- .../2018-06-01/entityTypes/Dataset.json | 514 ++++++++---------- 1 file changed, 233 insertions(+), 281 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index 294ad19ca54d..b300fbe474f9 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -398,10 +398,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/AzureBlobDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "AzureBlobDatasetTypeProperties": { "description": "Azure Blob dataset properties.", @@ -483,10 +480,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/AzureSqlTableDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "AzureSqlTableDatasetTypeProperties": { "description": "Azure SQL dataset properties.", @@ -515,10 +509,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/AzureSqlDWTableDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "AzureSqlDWTableDatasetTypeProperties": { "description": "Azure SQL Data Warehouse dataset properties.", @@ -547,10 +538,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/CassandraTableDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "CassandraTableDatasetTypeProperties": { "description": "Cassandra dataset properties.", @@ -579,10 +567,7 @@ "x-ms-client-flatten": true, "type": "object" } - }, - "required": [ - "typeProperties" - ] + } }, "DocumentDbCollectionDataset": { "x-ms-discriminator-value": "DocumentDbCollection", @@ -631,10 +616,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/DynamicsEntityDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "DynamicsEntityDatasetTypeProperties": { "description": "Dynamics entity dataset properties.", @@ -660,10 +642,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/AzureDataLakeStoreDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "AzureDataLakeStoreDatasetTypeProperties": { "description": "Azure Data Lake Store dataset properties.", @@ -684,10 +663,7 @@ "description": "The data compression method used for the item(s) in the Azure Data Lake Store.", "$ref": "#/definitions/DatasetCompression" } - }, - "required": [ - "folderPath" - ] + } }, "AzureBlobFSDataset": { "x-ms-discriminator-value": "AzureBlobFSFile", @@ -819,10 +795,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/FileShareDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "FileShareDatasetTypeProperties": { "description": "On-premises file system dataset properties.", @@ -968,10 +941,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/ODataResourceDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "ODataResourceDatasetTypeProperties": { "description": "OData dataset properties.", @@ -997,10 +967,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/OracleTableDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "OracleTableDatasetTypeProperties": { "description": "On-premises Oracle dataset properties.", @@ -1009,10 +976,7 @@ "type": "object", "description": "The table name of the on-premises Oracle database. Type: string (or Expression with resultType string)." } - }, - "required": [ - "tableName" - ] + } }, "AzureMySqlTableDataset": { "x-ms-discriminator-value": "AzureMySqlTable", @@ -1058,10 +1022,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/RelationalTableDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "RelationalTableDatasetTypeProperties": { "description": "Relational table dataset properties.", @@ -1087,10 +1048,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/SalesforceObjectDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "SalesforceObjectDatasetTypeProperties": { "description": "Salesforce object dataset properties.", @@ -1220,10 +1178,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/SqlServerTableDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "SqlServerTableDatasetTypeProperties": { "description": "On-premises SQL Server dataset properties.", @@ -1362,10 +1317,7 @@ "x-ms-client-flatten": true, "$ref": "#/definitions/HttpDatasetTypeProperties" } - }, - "required": [ - "typeProperties" - ] + } }, "HttpDatasetTypeProperties": { "description": "Properties specific to this dataset type.", @@ -1406,14 +1358,14 @@ } }, "AmazonMWSObjectDataset" : { - "x-ms-discriminator-value": "AmazonMWSObject", - "description": "Amazon Marketplace Web Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "AmazonMWSObject", + "description": "Amazon Marketplace Web Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1424,13 +1376,13 @@ }, "AzurePostgreSqlTableDataset" : { "x-ms-discriminator-value": "AzurePostgreSqlTable", - "description": "Azure PostgreSQL dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "Azure PostgreSQL dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1440,14 +1392,14 @@ } }, "ConcurObjectDataset" : { - "x-ms-discriminator-value": "ConcurObject", - "description": "Concur Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "ConcurObject", + "description": "Concur Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1458,13 +1410,13 @@ }, "CouchbaseTableDataset" : { "x-ms-discriminator-value": "CouchbaseTable", - "description": "Couchbase server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "Couchbase server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1475,13 +1427,13 @@ }, "DrillTableDataset" : { "x-ms-discriminator-value": "DrillTable", - "description": "Drill server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "Drill server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1491,14 +1443,14 @@ } }, "EloquaObjectDataset" : { - "x-ms-discriminator-value": "EloquaObject", - "description": "Eloqua server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "EloquaObject", + "description": "Eloqua server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1508,14 +1460,14 @@ } }, "GoogleBigQueryObjectDataset" : { - "x-ms-discriminator-value": "GoogleBigQueryObject", - "description": "Google BigQuery service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "GoogleBigQueryObject", + "description": "Google BigQuery service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1526,13 +1478,13 @@ }, "GreenplumTableDataset" : { "x-ms-discriminator-value": "GreenplumTable", - "description": "Greenplum Database dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "Greenplum Database dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1542,14 +1494,14 @@ } }, "HBaseObjectDataset" : { - "x-ms-discriminator-value": "HBaseObject", - "description": "HBase server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "HBaseObject", + "description": "HBase server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1559,14 +1511,14 @@ } }, "HiveObjectDataset" : { - "x-ms-discriminator-value": "HiveObject", - "description": "Hive Server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "HiveObject", + "description": "Hive Server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1576,14 +1528,14 @@ } }, "HubspotObjectDataset" : { - "x-ms-discriminator-value": "HubspotObject", - "description": "Hubspot Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "HubspotObject", + "description": "Hubspot Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1593,14 +1545,14 @@ } }, "ImpalaObjectDataset" : { - "x-ms-discriminator-value": "ImpalaObject", - "description": "Impala server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "ImpalaObject", + "description": "Impala server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1610,14 +1562,14 @@ } }, "JiraObjectDataset" : { - "x-ms-discriminator-value": "JiraObject", - "description": "Jira Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "JiraObject", + "description": "Jira Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1627,14 +1579,14 @@ } }, "MagentoObjectDataset" : { - "x-ms-discriminator-value": "MagentoObject", - "description": "Magento server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "MagentoObject", + "description": "Magento server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1645,13 +1597,13 @@ }, "MariaDBTableDataset" : { "x-ms-discriminator-value": "MariaDBTable", - "description": "MariaDB server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "MariaDB server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1661,14 +1613,14 @@ } }, "MarketoObjectDataset" : { - "x-ms-discriminator-value": "MarketoObject", - "description": "Marketo server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "MarketoObject", + "description": "Marketo server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1678,14 +1630,14 @@ } }, "PaypalObjectDataset" : { - "x-ms-discriminator-value": "PaypalObject", - "description": "Paypal Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "PaypalObject", + "description": "Paypal Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1695,14 +1647,14 @@ } }, "PhoenixObjectDataset" : { - "x-ms-discriminator-value": "PhoenixObject", - "description": "Phoenix server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "PhoenixObject", + "description": "Phoenix server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1712,14 +1664,14 @@ } }, "PrestoObjectDataset" : { - "x-ms-discriminator-value": "PrestoObject", - "description": "Presto server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "PrestoObject", + "description": "Presto server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1729,14 +1681,14 @@ } }, "QuickBooksObjectDataset" : { - "x-ms-discriminator-value": "QuickBooksObject", - "description": "QuickBooks server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "QuickBooksObject", + "description": "QuickBooks server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1746,14 +1698,14 @@ } }, "ServiceNowObjectDataset" : { - "x-ms-discriminator-value": "ServiceNowObject", - "description": "ServiceNow server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "ServiceNowObject", + "description": "ServiceNow server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1763,14 +1715,14 @@ } }, "ShopifyObjectDataset" : { - "x-ms-discriminator-value": "ShopifyObject", - "description": "Shopify Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "ShopifyObject", + "description": "Shopify Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1780,14 +1732,14 @@ } }, "SparkObjectDataset" : { - "x-ms-discriminator-value": "SparkObject", - "description": "Spark Server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "SparkObject", + "description": "Spark Server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1797,14 +1749,14 @@ } }, "SquareObjectDataset" : { - "x-ms-discriminator-value": "SquareObject", - "description": "Square Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "SquareObject", + "description": "Square Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1814,14 +1766,14 @@ } }, "XeroObjectDataset" : { - "x-ms-discriminator-value": "XeroObject", - "description": "Xero Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "XeroObject", + "description": "Xero Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1831,14 +1783,14 @@ } }, "ZohoObjectDataset" : { - "x-ms-discriminator-value": "ZohoObject", - "description": "Zoho server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "x-ms-discriminator-value": "ZohoObject", + "description": "Zoho server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1849,13 +1801,13 @@ }, "NetezzaTableDataset" : { "x-ms-discriminator-value": "NetezzaTable", - "description": "Netezza dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "Netezza dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", @@ -1866,13 +1818,13 @@ }, "VerticaTableDataset" : { "x-ms-discriminator-value": "VerticaTable", - "description": "Vertica dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], + "description": "Vertica dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], "properties": { "typeProperties": { "description": "Properties specific to this dataset type.", From 47c70acbf9f2e6f731a678e23fc38823d9dbca28 Mon Sep 17 00:00:00 2001 From: Yiliu Date: Wed, 22 May 2019 15:54:45 +0800 Subject: [PATCH 02/11] update --- .../entityTypes/Dataset.json | 496 +++++----- .../entityTypes/IntegrationRuntime.json | 46 +- .../entityTypes/Pipeline.json | 931 +++++++++--------- .../2018-06-01/entityTypes/Dataset.json | 84 +- .../entityTypes/IntegrationRuntime.json | 44 + .../2018-06-01/entityTypes/Pipeline.json | 321 +++++- 6 files changed, 1181 insertions(+), 741 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json index 2c8eb0db1629..b25e34df3ce0 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json @@ -166,7 +166,7 @@ }, "JsonFormatFilePattern": { "description": "JSON format file pattern. A property of JsonFormat.", - "type": "string", + "type": "object", "enum": [ "setOfObjects", "arrayOfObjects" @@ -282,7 +282,7 @@ "CompressionLevel": { "title": "Microsoft.Azure.Management.DataFactories.Models.CompressionLevel", "description": "All available compression levels.", - "type": "string", + "type": "object", "enum": [ "Optimal", "Fastest" @@ -923,7 +923,7 @@ "description": "Sap ECC OData resource dataset properties.", "properties": { "path": { - "type": "string", + "type": "object", "description": "The path of the SAP ECC OData entity. Type: string (or Expression with resultType string)." } }, @@ -1081,339 +1081,339 @@ } }, "AmazonMWSObjectDataset" : { - "x-ms-discriminator-value": "AmazonMWSObject", - "description": "Amazon Marketplace Web Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "AmazonMWSObject", + "description": "Amazon Marketplace Web Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "AzurePostgreSqlTableDataset" : { "x-ms-discriminator-value": "AzurePostgreSqlTable", - "description": "Azure PostgreSQL dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "Azure PostgreSQL dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "ConcurObjectDataset" : { - "x-ms-discriminator-value": "ConcurObject", - "description": "Concur Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "ConcurObject", + "description": "Concur Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "CouchbaseTableDataset" : { "x-ms-discriminator-value": "CouchbaseTable", - "description": "Couchbase server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "Couchbase server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "DrillTableDataset" : { "x-ms-discriminator-value": "DrillTable", - "description": "Drill server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "Drill server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "EloquaObjectDataset" : { - "x-ms-discriminator-value": "EloquaObject", - "description": "Eloqua server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "EloquaObject", + "description": "Eloqua server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "GoogleBigQueryObjectDataset" : { - "x-ms-discriminator-value": "GoogleBigQueryObject", - "description": "Google BigQuery service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "GoogleBigQueryObject", + "description": "Google BigQuery service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "GreenplumTableDataset" : { "x-ms-discriminator-value": "GreenplumTable", - "description": "Greenplum Database dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "Greenplum Database dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "HBaseObjectDataset" : { - "x-ms-discriminator-value": "HBaseObject", - "description": "HBase server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "HBaseObject", + "description": "HBase server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "HiveObjectDataset" : { - "x-ms-discriminator-value": "HiveObject", - "description": "Hive Server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "HiveObject", + "description": "Hive Server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "HubspotObjectDataset" : { - "x-ms-discriminator-value": "HubspotObject", - "description": "Hubspot Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "HubspotObject", + "description": "Hubspot Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "ImpalaObjectDataset" : { - "x-ms-discriminator-value": "ImpalaObject", - "description": "Impala server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "ImpalaObject", + "description": "Impala server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "JiraObjectDataset" : { - "x-ms-discriminator-value": "JiraObject", - "description": "Jira Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "JiraObject", + "description": "Jira Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "MagentoObjectDataset" : { - "x-ms-discriminator-value": "MagentoObject", - "description": "Magento server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "MagentoObject", + "description": "Magento server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "MariaDBTableDataset" : { "x-ms-discriminator-value": "MariaDBTable", - "description": "MariaDB server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "MariaDB server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "MarketoObjectDataset" : { - "x-ms-discriminator-value": "MarketoObject", - "description": "Marketo server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "MarketoObject", + "description": "Marketo server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "PaypalObjectDataset" : { - "x-ms-discriminator-value": "PaypalObject", - "description": "Paypal Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "PaypalObject", + "description": "Paypal Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "PhoenixObjectDataset" : { - "x-ms-discriminator-value": "PhoenixObject", - "description": "Phoenix server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "PhoenixObject", + "description": "Phoenix server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "PrestoObjectDataset" : { - "x-ms-discriminator-value": "PrestoObject", - "description": "Presto server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "PrestoObject", + "description": "Presto server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "QuickBooksObjectDataset" : { - "x-ms-discriminator-value": "QuickBooksObject", - "description": "QuickBooks server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "QuickBooksObject", + "description": "QuickBooks server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "ServiceNowObjectDataset" : { - "x-ms-discriminator-value": "ServiceNowObject", - "description": "ServiceNow server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "ServiceNowObject", + "description": "ServiceNow server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "ShopifyObjectDataset" : { - "x-ms-discriminator-value": "ShopifyObject", - "description": "Shopify Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "ShopifyObject", + "description": "Shopify Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "SparkObjectDataset" : { - "x-ms-discriminator-value": "SparkObject", - "description": "Spark Server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "SparkObject", + "description": "Spark Server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "SquareObjectDataset" : { - "x-ms-discriminator-value": "SquareObject", - "description": "Square Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "SquareObject", + "description": "Square Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "XeroObjectDataset" : { - "x-ms-discriminator-value": "XeroObject", - "description": "Xero Service dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "XeroObject", + "description": "Xero Service dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "ZohoObjectDataset" : { - "x-ms-discriminator-value": "ZohoObject", - "description": "Zoho server dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "x-ms-discriminator-value": "ZohoObject", + "description": "Zoho server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "NetezzaTableDataset" : { "x-ms-discriminator-value": "NetezzaTable", - "description": "Netezza dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "Netezza dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "VerticaTableDataset" : { "x-ms-discriminator-value": "VerticaTable", - "description": "Vertica dataset.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Dataset" - } - ], - "properties": { + "description": "Vertica dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { } }, "SalesforceMarketingCloudObjectDataset": { diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/IntegrationRuntime.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/IntegrationRuntime.json index dbb3f8502999..7ef54f396836 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/IntegrationRuntime.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/IntegrationRuntime.json @@ -150,6 +150,10 @@ "description": "Custom setup script properties for a managed dedicated integration runtime.", "$ref": "#/definitions/IntegrationRuntimeCustomSetupScriptProperties" }, + "dataProxyProperties": { + "description": "Data proxy properties for a managed dedicated integration runtime.", + "$ref": "#/definitions/IntegrationRuntimeDataProxyProperties" + }, "edition": { "description": "The edition for the SSIS Integration Runtime", "type": "string", @@ -208,6 +212,46 @@ } } }, + "IntegrationRuntimeDataProxyProperties": { + "description": "Data proxy properties for a managed dedicated integration runtime.", + "type": "object", + "properties": { + "connectVia": { + "description": "The self-hosted integration runtime reference.", + "$ref": "#/definitions/EntityReference" + }, + "stagingLinkedService": { + "description": "The staging linked service reference.", + "$ref": "#/definitions/EntityReference" + }, + "path": { + "description": "The path to contain the staged data in the Blob storage.", + "type": "string" + } + } + }, + "EntityReference": { + "description": "The entity reference.", + "type": "object", + "properties": { + "type": { + "description": "The type of this referenced entity.", + "type": "string", + "enum": [ + "IntegrationRuntimeReference", + "LinkedServiceReference" + ], + "x-ms-enum": { + "name": "IntegrationRuntimeEntityReferenceType", + "modelAsString": true + } + }, + "referenceName": { + "description": "The name of this referenced entity.", + "type": "string" + } + } + }, "SelfHostedIntegrationRuntime": { "x-ms-discriminator-value": "SelfHosted", "description": "Self-hosted integration runtime.", @@ -301,7 +345,7 @@ "dataFactoryName": { "description": "The data factory name which the integration runtime belong to.", "type": "string", - "readOnly": true + "readOnly": true }, "state": { "description": "The state of integration runtime.", diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json index 952721439015..6e21ea7e04ff 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json @@ -36,7 +36,7 @@ "items": { "type": "object" } - } + } } }, "Activity": { @@ -208,7 +208,7 @@ }, "translator": { "description": "Copy activity translator. If not specified, tabular translator is used.", - "$ref": "#/definitions/CopyTranslator" + "type": "object" }, "enableStaging": { "type": "object", @@ -371,7 +371,7 @@ }, "readBehavior": { "description": "The read behavior for the operation. Default is Query.", - "type": "string", + "type": "object", "enum": [ "Query", "QueryAll" @@ -408,7 +408,7 @@ ], "properties": { "query": { - "type": "string", + "type": "object", "description": "SAP ECC OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." } } @@ -577,7 +577,7 @@ }, "consistencyLevel": { "description": "The consistency level specifies how many Cassandra servers must respond to a read request before returning data to the client application. Cassandra checks the specified number of Cassandra servers for data to satisfy the read request. Must be one of cassandraSourceReadConsistencyLevels. The default value is 'ONE'. It is case-insensitive.", - "type": "string", + "type": "object", "enum": [ "ALL", "EACH_QUORUM", @@ -642,455 +642,455 @@ } } }, - "AmazonMWSSource": { - "description": "A copy activity Amazon Marketplace Web Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "AzurePostgreSqlSource": { - "description": "A copy activity Azure PostgreSQL source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "ConcurSource": { - "description": "A copy activity Concur Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "CouchbaseSource": { - "description": "A copy activity Couchbase server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "DrillSource": { - "description": "A copy activity Drill server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "EloquaSource": { - "description": "A copy activity Eloqua server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "GoogleBigQuerySource": { - "description": "A copy activity Google BigQuery service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "GreenplumSource": { - "description": "A copy activity Greenplum Database source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "HBaseSource": { - "description": "A copy activity HBase server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "HiveSource": { - "description": "A copy activity Hive Server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "HubspotSource": { - "description": "A copy activity Hubspot Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "ImpalaSource": { - "description": "A copy activity Impala server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "JiraSource": { - "description": "A copy activity Jira Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "MagentoSource": { - "description": "A copy activity Magento server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "MariaDBSource": { - "description": "A copy activity MariaDB server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "MarketoSource": { - "description": "A copy activity Marketo server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "PaypalSource": { - "description": "A copy activity Paypal Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "PhoenixSource": { - "description": "A copy activity Phoenix server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "PrestoSource": { - "description": "A copy activity Presto server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "QuickBooksSource": { - "description": "A copy activity QuickBooks server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "ServiceNowSource": { - "description": "A copy activity ServiceNow server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "ShopifySource": { - "description": "A copy activity Shopify Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "SparkSource": { - "description": "A copy activity Spark Server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "SquareSource": { - "description": "A copy activity Square Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "XeroSource": { - "description": "A copy activity Xero Service source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "ZohoSource": { - "description": "A copy activity Zoho server source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "NetezzaSource": { - "description": "A copy activity Netezza source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "VerticaSource": { - "description": "A copy activity Vertica source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "SalesforceMarketingCloudSource": { - "description": "A copy activity Salesforce Marketing Cloud source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } - }, - "ResponsysSource": { - "description": "A copy activity Responsys source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": { - "query": { - "type": "object", - "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." - } - } + "AmazonMWSSource": { + "description": "A copy activity Amazon Marketplace Web Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "AzurePostgreSqlSource": { + "description": "A copy activity Azure PostgreSQL source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "ConcurSource": { + "description": "A copy activity Concur Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "CouchbaseSource": { + "description": "A copy activity Couchbase server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "DrillSource": { + "description": "A copy activity Drill server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "EloquaSource": { + "description": "A copy activity Eloqua server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "GoogleBigQuerySource": { + "description": "A copy activity Google BigQuery service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "GreenplumSource": { + "description": "A copy activity Greenplum Database source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "HBaseSource": { + "description": "A copy activity HBase server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "HiveSource": { + "description": "A copy activity Hive Server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "HubspotSource": { + "description": "A copy activity Hubspot Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "ImpalaSource": { + "description": "A copy activity Impala server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "JiraSource": { + "description": "A copy activity Jira Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "MagentoSource": { + "description": "A copy activity Magento server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "MariaDBSource": { + "description": "A copy activity MariaDB server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "MarketoSource": { + "description": "A copy activity Marketo server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "PaypalSource": { + "description": "A copy activity Paypal Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "PhoenixSource": { + "description": "A copy activity Phoenix server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "PrestoSource": { + "description": "A copy activity Presto server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "QuickBooksSource": { + "description": "A copy activity QuickBooks server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "ServiceNowSource": { + "description": "A copy activity ServiceNow server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "ShopifySource": { + "description": "A copy activity Shopify Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "SparkSource": { + "description": "A copy activity Spark Server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "SquareSource": { + "description": "A copy activity Square Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "XeroSource": { + "description": "A copy activity Xero Service source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "ZohoSource": { + "description": "A copy activity Zoho server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "NetezzaSource": { + "description": "A copy activity Netezza source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "VerticaSource": { + "description": "A copy activity Vertica source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "SalesforceMarketingCloudSource": { + "description": "A copy activity Salesforce Marketing Cloud source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } + }, + "ResponsysSource": { + "description": "A copy activity Responsys source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + } + } } , "AmazonRedshiftSource": { "description": "A copy activity source for Amazon Redshift Source.", @@ -1148,10 +1148,11 @@ }, "StoredProcedureParameterType": { "description": "Stored procedure parameter type.", - "type": "string", + "type": "object", "enum": [ "String", "Int", + "Int64", "Decimal", "Guid", "Boolean", @@ -1206,7 +1207,7 @@ "properties": { "writeBehavior": { "description": "The write behavior for the operation. Default is 'Insert'.", - "type": "string", + "type": "object", "enum": [ "Insert", "Update" @@ -1230,7 +1231,7 @@ }, "CopyBehaviorType": { "description": "All available types of copy behavior.", - "type": "string", + "type": "object", "enum": [ "PreserveHierarchy", "FlattenHierarchy", @@ -1291,7 +1292,7 @@ }, "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" } } }, @@ -1306,7 +1307,7 @@ "properties": { "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" } } }, @@ -1405,7 +1406,7 @@ }, "PolybaseSettingsRejectType": { "description": "Indicates whether the RejectValue property is specified as a literal value or a percentage.", - "type": "string", + "type": "object", "enum": [ "value", "percentage" @@ -1485,7 +1486,7 @@ "properties": { "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" } } }, @@ -1500,7 +1501,7 @@ "properties": { "writeBehavior": { "description": "Specify the write behavior when upserting documents into Azure Search Index.", - "type": "string", + "type": "object", "enum": [ "Merge", "Upload" @@ -1538,7 +1539,7 @@ "properties": { "writeBehavior": { "description": "The write behavior for the operation.", - "type": "string", + "type": "object", "enum": [ "Upsert" ], @@ -1567,7 +1568,7 @@ "properties": { "writeBehavior": { "description": "The write behavior for the operation. Default is Insert.", - "type": "string", + "type": "object", "enum": [ "Insert", "Upsert" @@ -1593,7 +1594,7 @@ "type": "object", "properties": { "type": { - "type": "string", + "type": "object", "description": "Copy translator type." } }, diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index fbd70667a33f..d292e76a59a4 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -197,7 +197,7 @@ }, "JsonFormatFilePattern": { "description": "JSON format file pattern. A property of JsonFormat.", - "type": "string", + "type": "object", "enum": [ "setOfObjects", "arrayOfObjects" @@ -313,7 +313,7 @@ "CompressionLevel": { "title": "Microsoft.Azure.Management.DataFactories.Models.CompressionLevel", "description": "All available compression levels.", - "type": "string", + "type": "object", "enum": [ "Optimal", "Fastest" @@ -433,6 +433,82 @@ } } }, + "DelimitedTextDataset": { + "x-ms-discriminator-value": "DelimitedText", + "description": "Delimited text dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { + "typeProperties": { + "description": "Delimited text dataset properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/DelimitedTextDatasetTypeProperties" + } + } + }, + "DelimitedTextDatasetTypeProperties": { + "description": "DelimitedText dataset properties.", + "properties": { + "location": { + "$ref": "#/definitions/DatasetLocation", + "description": "The location of the delimited text storage." + }, + "columnDelimiter": { + "type": "object", + "description": "The column delimiter. Type: string (or Expression with resultType string)." + }, + "rowDelimiter": { + "type": "object", + "description": "The row delimiter. Type: string (or Expression with resultType string)." + }, + "encodingName": { + "type": "object", + "description": "The code page name of the preferred encoding. If miss, the default value is UTF-8, unless BOM denotes another Unicode encoding. Refer to the name column of the table in the following link to set supported values: https://msdn.microsoft.com/library/system.text.encoding.aspx. Type: string (or Expression with resultType string)." + }, + "compressionCodec": { + "type": "object", + "enum": [ + "bzip2", + "gzip", + "deflate", + "zipDeflate", + "snappy", + "lz4" + ], + "x-ms-enum": { + "name": "compressionCodec", + "modelAsString": true + } + }, + "compressionLevel": { + "description": "The data compression method used for DelimitedText.", + "$ref": "#/definitions/CompressionLevel" + }, + "quoteChar": { + "type": "object", + "description": "The quote character. Type: string (or Expression with resultType string)." + }, + "escapeChar": { + "type": "object", + "description": "The escape character. Type: string (or Expression with resultType string)." + }, + "firstRowAsHeader": { + "type": "object", + "description": "When used as input, treat the first row of data as headers. When used as output,write the headers into the output as the first row of data. The default value is false. Type: boolean (or Expression with resultType boolean)." + }, + "nullValue": { + "type": "object", + "description": "The null value string. Type: string (or Expression with resultType string)." + } + }, + "required": [ + "location" + ] + }, "AzureTableDataset": { "x-ms-discriminator-value": "AzureTable", "description": "The Azure Table storage dataset.", @@ -1074,7 +1150,7 @@ "description": "Sap ECC OData resource dataset properties.", "properties": { "path": { - "type": "string", + "type": "object", "description": "The path of the SAP ECC OData entity. Type: string (or Expression with resultType string)." } }, @@ -1850,7 +1926,7 @@ "description": "Dynamics AX OData resource dataset properties.", "properties": { "path": { - "type": "string", + "type": "object", "description": "The path of the Dynamics AX OData entity. Type: string (or Expression with resultType string)." } }, diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/IntegrationRuntime.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/IntegrationRuntime.json index 8ea81d3bb1d2..2f9c9ca48e09 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/IntegrationRuntime.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/IntegrationRuntime.json @@ -150,6 +150,10 @@ "description": "Custom setup script properties for a managed dedicated integration runtime.", "$ref": "#/definitions/IntegrationRuntimeCustomSetupScriptProperties" }, + "dataProxyProperties": { + "description": "Data proxy properties for a managed dedicated integration runtime.", + "$ref": "#/definitions/IntegrationRuntimeDataProxyProperties" + }, "edition": { "description": "The edition for the SSIS Integration Runtime", "type": "string", @@ -218,6 +222,46 @@ } } }, + "IntegrationRuntimeDataProxyProperties": { + "description": "Data proxy properties for a managed dedicated integration runtime.", + "type": "object", + "properties": { + "connectVia": { + "description": "The self-hosted integration runtime reference.", + "$ref": "#/definitions/EntityReference" + }, + "stagingLinkedService": { + "description": "The staging linked service reference.", + "$ref": "#/definitions/EntityReference" + }, + "path": { + "description": "The path to contain the staged data in the Blob storage.", + "type": "string" + } + } + }, + "EntityReference": { + "description": "The entity reference.", + "type": "object", + "properties": { + "type": { + "description": "The type of this referenced entity.", + "type": "string", + "enum": [ + "IntegrationRuntimeReference", + "LinkedServiceReference" + ], + "x-ms-enum": { + "name": "IntegrationRuntimeEntityReferenceType", + "modelAsString": true + } + }, + "referenceName": { + "description": "The name of this referenced entity.", + "type": "string" + } + } + }, "SelfHostedIntegrationRuntime": { "x-ms-discriminator-value": "SelfHosted", "description": "Self-hosted integration runtime.", diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index a28f3d82c131..9ab2f0a3ee0b 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -205,6 +205,89 @@ "type": "object" } }, + "DelimitedTextReadSetting": { + "description": "Delimited text read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/FormatReadSetting" + } + ], + "properties": { + "skipLineCount": { + "type": "object", + "description": "Indicates the number of non-empty rows to skip when reading data from input files. Type: integer (or Expression with resultType integer)." + }, + "addtionalNullValues": { + "description": "List of strings each of which will be treated as a null value as well as that in dataset.", + "type": "array", + "items": { + "type": "object", + "description": "Type: string (or Expression with resultType string)." + } + } + } + }, + "DelimitedTextWriteSetting": { + "description": "Delimited text write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/FormatWriteSetting" + } + ], + "properties": { + "quoteAllText": { + "type": "object", + "description": "Indicates whether string values should always be enclosed with quotes. Type: boolean (or Expression with resultType boolean)." + }, + "fileExtension": { + "type": "object", + "description": "The file extension used to create the files. Type: string (or Expression with resultType string)." + } + }, + "required": [ + "fileExtension" + ] + }, + "DelimitedTextSource": { + "description": "A copy activity DelimitedText source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "storeSettings": { + "$ref": "#/definitions/ConnectorReadSetting", + "description": "DelimitedText store settings." + }, + "formatSettings": { + "$ref": "#/definitions/DelimitedTextReadSetting", + "description": "DelimitedText format settings." + } + } + }, + "DelimitedTextSink": { + "description": "A copy activity DelimitedText sink.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySink" + } + ], + "properties": { + "storeSettings": { + "$ref": "#/definitions/ConnectorWriteSetting", + "description": "DelimitedText store settings." + }, + "formatSettings": { + "$ref": "#/definitions/DelimitedTextWriteSetting", + "description": "DelimitedText format settings." + } + } + }, "CopyActivity": { "x-ms-discriminator-value": "Copy", "description": "Copy activity.", @@ -251,7 +334,7 @@ }, "translator": { "description": "Copy activity translator. If not specified, tabular translator is used.", - "$ref": "#/definitions/CopyTranslator" + "type": "object" }, "enableStaging": { "type": "object", @@ -284,6 +367,14 @@ "description": "Type: string (or Expression with resultType string)." }, "description": "Preserve Rules." + }, + "preserve": { + "type": "array", + "items": { + "type": "object", + "description": "Type: string (or Expression with resultType string)." + }, + "description": "Preserve." } }, "required": [ @@ -426,7 +517,7 @@ }, "readBehavior": { "description": "The read behavior for the operation. Default is Query.", - "type": "string", + "type": "object", "enum": [ "Query", "QueryAll" @@ -463,21 +554,11 @@ ], "properties": { "query": { - "type": "string", + "type": "object", "description": "SAP ECC OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." } } }, - "SapOpenHubSource": { - "description": "A copy activity source for SAP Business Warehouse Open Hub Destination source.", - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/CopySource" - } - ], - "properties": {} - }, "RestSource": { "description": "A copy activity Rest service source.", "type": "object", @@ -523,6 +604,66 @@ } } }, + "SqlServerSource": { + "description": "A copy activity SQL server source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "sqlReaderQuery": { + "type": "object", + "description": "SQL reader query. Type: string (or Expression with resultType string)." + }, + "sqlReaderStoredProcedureName": { + "type": "object", + "description": "Name of the stored procedure for a SQL Database source. This cannot be used at the same time as SqlReaderQuery. Type: string (or Expression with resultType string)." + }, + "storedProcedureParameters": { + "description": "Value and type setting for stored procedure parameters. Example: \"{Parameter1: {value: \"1\", type: \"int\"}}\".", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/StoredProcedureParameter" + } + }, + "produceAdditionalTypes": { + "description": "Which additional types to produce.", + "type": "object" + } + } + }, + "AzureSqlSource": { + "description": "A copy activity Azure SQL source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "sqlReaderQuery": { + "type": "object", + "description": "SQL reader query. Type: string (or Expression with resultType string)." + }, + "sqlReaderStoredProcedureName": { + "type": "object", + "description": "Name of the stored procedure for a SQL Database source. This cannot be used at the same time as SqlReaderQuery. Type: string (or Expression with resultType string)." + }, + "storedProcedureParameters": { + "description": "Value and type setting for stored procedure parameters. Example: \"{Parameter1: {value: \"1\", type: \"int\"}}\".", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/StoredProcedureParameter" + } + }, + "produceAdditionalTypes": { + "description": "Which additional types to produce.", + "type": "object" + } + } + }, "SqlDWSource": { "description": "A copy activity SQL Data Warehouse source.", "type": "object", @@ -659,6 +800,49 @@ "queryTimeout": { "type": "object", "description": "Query timeout. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + }, + "partitionOption": { + "description": "The partition mechanism that will be used for oracle read in parallel.", + "type": "object", + "enum": [ + "None", + "PhysicalPartitionsOfTable", + "DynamicRange" + ], + "x-ms-enum": { + "name": "TabularPartitionOption", + "modelAsString": true + } + }, + "partitionSettings": { + "description": "The settings that will be leveraged for oracle source partitioning.", + "$ref": "#/definitions/OraclePartitionSettings" + } + } + }, + "OraclePartitionSettings": { + "description": "The settings that will be leveraged for oracle source partitioning.", + "type": "object", + "properties": { + "partitionNames": { + "type": "array", + "items": { + "type": "object", + "description": "Type: string (or Expression with resultType string)." + }, + "description": "Names of the physical partitions of oracle table. " + }, + "partitionColumnName": { + "type": "object", + "description": "The name of the column in integer type that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + }, + "partitionUpperBound": { + "type": "object", + "description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + }, + "partitionLowerBound": { + "type": "object", + "description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." } } }, @@ -687,7 +871,7 @@ }, "consistencyLevel": { "description": "The consistency level specifies how many Cassandra servers must respond to a read request before returning data to the client application. Cassandra checks the specified number of Cassandra servers for data to satisfy the read request. Must be one of cassandraSourceReadConsistencyLevels. The default value is 'ONE'. It is case-insensitive.", - "type": "string", + "type": "object", "enum": [ "ALL", "EACH_QUORUM", @@ -1408,6 +1592,7 @@ "enum": [ "String", "Int", + "Int64", "Decimal", "Guid", "Boolean", @@ -1466,7 +1651,7 @@ "properties": { "writeBehavior": { "description": "The write behavior for the operation. Default is 'Insert'.", - "type": "string", + "type": "object", "enum": [ "Insert", "Update" @@ -1490,7 +1675,7 @@ }, "CopyBehaviorType": { "description": "All available types of copy behavior.", - "type": "string", + "type": "object", "enum": [ "PreserveHierarchy", "FlattenHierarchy", @@ -1528,6 +1713,16 @@ } } }, + "ParquetSink": { + "description": "A copy activity Parquet sink.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySink" + } + ], + "properties": {} + }, "BlobSink": { "description": "A copy activity Azure Blob sink.", "type": "object", @@ -1551,7 +1746,7 @@ }, "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" } } }, @@ -1566,7 +1761,7 @@ "properties": { "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" } } }, @@ -1582,6 +1777,10 @@ "nestingSeparator": { "type": "object", "description": "Nested properties separator. Default is . (dot). Type: string (or Expression with resultType string)." + }, + "writeBehavior": { + "type": "object", + "description": "Describes how to write data to Azure Cosmos DB. Allowed values: insert and upsert." } } }, @@ -1612,6 +1811,78 @@ "additionalProperties": { "$ref": "#/definitions/StoredProcedureParameter" } + }, + "storedProcedureTableTypeParameterName": { + "type": "object", + "description": "The stored procedure parameter name of the table type. Type: string (or Expression with resultType string)." + } + } + }, + "SqlServerSink": { + "description": "A copy activity SQL server sink.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySink" + } + ], + "properties": { + "sqlWriterStoredProcedureName": { + "type": "object", + "description": "SQL writer stored procedure name. Type: string (or Expression with resultType string)." + }, + "sqlWriterTableType": { + "type": "object", + "description": "SQL writer table type. Type: string (or Expression with resultType string)." + }, + "preCopyScript": { + "type": "object", + "description": "SQL pre-copy script. Type: string (or Expression with resultType string)." + }, + "storedProcedureParameters": { + "description": "SQL stored procedure parameters.", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/StoredProcedureParameter" + } + }, + "storedProcedureTableTypeParameterName": { + "type": "object", + "description": "The stored procedure parameter name of the table type. Type: string (or Expression with resultType string)." + } + } + }, + "AzureSqlSink": { + "description": "A copy activity Azure SQL sink.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySink" + } + ], + "properties": { + "sqlWriterStoredProcedureName": { + "type": "object", + "description": "SQL writer stored procedure name. Type: string (or Expression with resultType string)." + }, + "sqlWriterTableType": { + "type": "object", + "description": "SQL writer table type. Type: string (or Expression with resultType string)." + }, + "preCopyScript": { + "type": "object", + "description": "SQL pre-copy script. Type: string (or Expression with resultType string)." + }, + "storedProcedureParameters": { + "description": "SQL stored procedure parameters.", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/StoredProcedureParameter" + } + }, + "storedProcedureTableTypeParameterName": { + "type": "object", + "description": "The stored procedure parameter name of the table type. Type: string (or Expression with resultType string)." } } }, @@ -1765,7 +2036,11 @@ "properties": { "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" + }, + "enableAdlsSingleFileParallel": { + "description": "Single File Parallel.", + "type": "object" } } }, @@ -1780,7 +2055,7 @@ "properties": { "copyBehavior": { "description": "The type of copy behavior for copy sink.", - "$ref": "#/definitions/CopyBehaviorType" + "type": "object" } } }, @@ -1795,7 +2070,7 @@ "properties": { "writeBehavior": { "description": "Specify the write behavior when upserting documents into Azure Search Index.", - "type": "string", + "type": "object", "enum": [ "Merge", "Upload" @@ -1833,7 +2108,7 @@ "properties": { "writeBehavior": { "description": "The write behavior for the operation.", - "type": "string", + "type": "object", "enum": [ "Upsert" ], @@ -1885,7 +2160,7 @@ "properties": { "writeBehavior": { "description": "The write behavior for the operation. Default is Insert.", - "type": "string", + "type": "object", "enum": [ "Insert", "Upsert" From 4e1af64b8fd59b4c121cb27978372ef9f516fcbb Mon Sep 17 00:00:00 2001 From: Yiliu Date: Wed, 29 May 2019 15:37:54 +0800 Subject: [PATCH 03/11] finally modify --- .../entityTypes/Dataset.json | 2 +- .../2018-06-01/entityTypes/Dataset.json | 241 +++++++-- .../2018-06-01/entityTypes/Pipeline.json | 466 +++++++++++++++++- 3 files changed, 662 insertions(+), 47 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json index b25e34df3ce0..506b80d1769a 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Dataset.json @@ -166,7 +166,7 @@ }, "JsonFormatFilePattern": { "description": "JSON format file pattern. A property of JsonFormat.", - "type": "object", + "type": "string", "enum": [ "setOfObjects", "arrayOfObjects" diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index d292e76a59a4..10c5339b7646 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -61,6 +61,139 @@ "linkedServiceName" ] }, + "DatasetLocation": { + "description": "Dataset location.", + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of dataset storage location." + }, + "folderPath": { + "type": "object", + "description": "Specify the folder path of dataset. Type: string (or Expression with resultType string)" + }, + "fileName": { + "type": "object", + "description": "Specify the file name of dataset. Type: string (or Expression with resultType string)." + } + }, + "additionalProperties": { + "type": "object" + }, + "required": [ + "type" + ] + }, + "AzureBlobStorageLocation": { + "description": "The location of azure blob dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ], + "properties": { + "container": { + "type": "object", + "description": "Specify the container of azure blob. Type: string (or Expression with resultType string)." + } + } + }, + "AzureBlobFSLocation": { + "description": "The location of azure blobFS dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ], + "properties": { + "fileSystem": { + "type": "object", + "description": "Specify the fileSystem of azure blobFS. Type: string (or Expression with resultType string)." + } + } + }, + "AzureDataLakeStoreLocation": { + "description": "The location of azure data lake store dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, + "AmazonS3Location": { + "description": "The location of amazon S3 dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ], + "properties": { + "bucketName": { + "type": "object", + "description": "Specify the bucketName of amazon S3. Type: string (or Expression with resultType string)" + }, + "version": { + "type": "object", + "description": "Specify the version of amazon S3. Type: string (or Expression with resultType string)." + } + } + }, + "FileServerLocation": { + "description": "The location of file server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, + "FtpServerLocation": { + "description": "The location of ftp server dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, + "SftpLocation": { + "description": "The location of SFTP dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, + "HttpServerLocation": { + "description": "The location of http server.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ], + "properties": { + "relativeUrl": { + "type": "object", + "description": "Specify the relativeUrl of http server. Type: string (or Expression with resultType string)" + } + } + }, + "HdfsLocation": { + "description": "The location of HDFS.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, "DatasetDataElement": { "description": "Columns that define the structure of the dataset.", "type": "object", @@ -383,9 +516,9 @@ "bucketName" ] }, - "AzureBlobDataset": { - "x-ms-discriminator-value": "AzureBlob", - "description": "The Azure Blob storage.", + "ParquetDataset": { + "x-ms-discriminator-value": "Parquet", + "description": "Parquet dataset.", "type": "object", "allOf": [ { @@ -394,44 +527,36 @@ ], "properties": { "typeProperties": { - "description": "Azure Blob dataset properties.", + "description": "Parquet dataset properties.", "x-ms-client-flatten": true, - "$ref": "#/definitions/AzureBlobDatasetTypeProperties" + "$ref": "#/definitions/ParquetDatasetTypeProperties" } } }, - "AzureBlobDatasetTypeProperties": { - "description": "Azure Blob dataset properties.", + "ParquetDatasetTypeProperties": { + "description": "Parquet dataset properties.", "properties": { - "folderPath": { - "type": "object", - "description": "The path of the Azure Blob storage. Type: string (or Expression with resultType string)." - }, - "tableRootLocation": { - "type": "object", - "description": "The root of blob path. Type: string (or Expression with resultType string)." - }, - "fileName": { - "type": "object", - "description": "The name of the Azure Blob. Type: string (or Expression with resultType string)." - }, - "modifiedDatetimeStart": { - "type": "object", - "description": "The start of Azure Blob's modified datetime. Type: string (or Expression with resultType string)." + "location": { + "$ref": "#/definitions/DatasetLocation", + "description": "The location of the parquet storage." }, - "modifiedDatetimeEnd": { + "compressionCodec": { "type": "object", - "description": "The end of Azure Blob's modified datetime. Type: string (or Expression with resultType string)." - }, - "format": { - "description": "The format of the Azure Blob storage.", - "$ref": "#/definitions/DatasetStorageFormat" - }, - "compression": { - "description": "The data compression method used for the blob storage.", - "$ref": "#/definitions/DatasetCompression" + "enum": [ + "none", + "gzip", + "snappy", + "lzo" + ], + "x-ms-enum": { + "name": "compressionCodec", + "modelAsString": true + } } - } + }, + "required": [ + "location" + ] }, "DelimitedTextDataset": { "x-ms-discriminator-value": "DelimitedText", @@ -509,6 +634,56 @@ "location" ] }, + "AzureBlobDataset": { + "x-ms-discriminator-value": "AzureBlob", + "description": "The Azure Blob storage.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { + "typeProperties": { + "description": "Azure Blob dataset properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/AzureBlobDatasetTypeProperties" + } + } + }, + "AzureBlobDatasetTypeProperties": { + "description": "Azure Blob dataset properties.", + "properties": { + "folderPath": { + "type": "object", + "description": "The path of the Azure Blob storage. Type: string (or Expression with resultType string)." + }, + "tableRootLocation": { + "type": "object", + "description": "The root of blob path. Type: string (or Expression with resultType string)." + }, + "fileName": { + "type": "object", + "description": "The name of the Azure Blob. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of Azure Blob's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of Azure Blob's modified datetime. Type: string (or Expression with resultType string)." + }, + "format": { + "description": "The format of the Azure Blob storage.", + "$ref": "#/definitions/DatasetStorageFormat" + }, + "compression": { + "description": "The data compression method used for the blob storage.", + "$ref": "#/definitions/DatasetCompression" + } + } + }, "AzureTableDataset": { "x-ms-discriminator-value": "AzureTable", "description": "The Azure Table storage dataset.", diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 9ab2f0a3ee0b..d72c782e30ec 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -205,6 +205,405 @@ "type": "object" } }, + "ConnectorReadSetting": { + "description": "Connector read settings.", + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "The read setting type." + }, + "maxConcurrentConnections": { + "type": "object", + "description": "The maximum concurrent connection count for the source data store. Type: integer (or Expression with resultType integer)." + } + }, + "additionalProperties": { + "type": "object" + }, + "required": [ + "type" + ] + }, + "AzureBlobStorageReadSetting": { + "description": "Azure blob read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "Azure blob wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "Azure blob wildcardFileName. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "boolean", + "description": "Indicates whether to enable partition discovery." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, + "AzureBlobFSReadSetting": { + "description": "Azure blobFS read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "Azure blobFS wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "Azure blobFS wildcardFileName. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "boolean", + "description": "Indicates whether to enable partition discovery." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, + "AzureDataLakeStoreReadSetting": { + "description": "Azure data lake store read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "ADLS wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "ADLS wildcardFileName. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "boolean", + "description": "Indicates whether to enable partition discovery." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, + "AmazonS3ReadSetting": { + "description": "Azure data lake store read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "AmazonS3 wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "AmazonS3 wildcardFileName. Type: string (or Expression with resultType string)." + }, + "prefix": { + "type": "object", + "description": "The prefix filter for the S3 object name. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "boolean", + "description": "Indicates whether to enable partition discovery." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, + "FileServerReadSetting": { + "description": "File server read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "FileServer wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "FileServer wildcardFileName. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "boolean", + "description": "Indicates whether to enable partition discovery." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, + "FtpReadSetting": { + "description": "Ftp read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "Ftp wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "Ftp wildcardFileName. Type: string (or Expression with resultType string)." + }, + "useBinaryTransfer": { + "type": "boolean", + "description": "Specify whether to use binary transfer mode for FTP stores." + } + } + }, + "SftpReadSetting": { + "description": "Sftp read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "Sftp wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "Sftp wildcardFileName. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, + "HttpReadSetting": { + "description": "Sftp read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "requestMethod": { + "type": "object", + "description": "The HTTP method used to call the RESTful API. The default is GET. Type: string (or Expression with resultType string)." + }, + "requestBody": { + "type": "object", + "description": "The HTTP request body to the RESTful API if requestMethod is POST. Type: string (or Expression with resultType string)." + }, + "additionalHeaders": { + "type": "object", + "description": "The additional HTTP headers in the request to the RESTful API. Type: string (or Expression with resultType string)." + }, + "requestTimeout": { + "type": "object", + "description": "Specifies the timeout for a HTTP client to get HTTP response from HTTP server." + } + } + }, + "HdfsReadSetting": { + "description": "HDFS read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorReadSetting" + } + ], + "properties": { + "recursive": { + "type": "object", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "description": "HDFS wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "description": "HDFS wildcardFileName. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "boolean", + "description": "Indicates whether to enable partition discovery." + }, + "modifiedDatetimeStart": { + "type": "object", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "distcpSettings": { + "description": "Specifies Distcp-related settings.", + "$ref": "#/definitions/DistcpSettings" + } + } + }, + "ConnectorWriteSetting": { + "description": "Connector write settings.", + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "The write setting type." + }, + "maxConcurrentConnections": { + "type": "object", + "description": "The maximum concurrent connection count for the source data store. Type: integer (or Expression with resultType integer)." + }, + "copyBehavior": { + "description": "The type of copy behavior for copy sink.", + "type": "object" + } + }, + "additionalProperties": { + "type": "object" + }, + "required": [ + "type" + ] + }, + "AzureBlobStorageWriteSetting": { + "description": "Azure blob write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorWriteSetting" + } + ] + }, + "AzureBlobFSWriteSetting": { + "description": "Azure blobFS write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorWriteSetting" + } + ] + }, + "AzureDataLakeStoreWriteSetting": { + "description": "Azure data lake store write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorWriteSetting" + } + ] + }, + "FileServerWriteSetting": { + "description": "File server write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/ConnectorWriteSetting" + } + ] + }, + "FormatReadSetting": { + "description": "Format read settings.", + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "The read setting type." + } + }, + "additionalProperties": { + "type": "object" + }, + "required": [ + "type" + ] + }, "DelimitedTextReadSetting": { "description": "Delimited text read settings.", "type": "object", @@ -218,16 +617,28 @@ "type": "object", "description": "Indicates the number of non-empty rows to skip when reading data from input files. Type: integer (or Expression with resultType integer)." }, - "addtionalNullValues": { - "description": "List of strings each of which will be treated as a null value as well as that in dataset.", - "type": "array", - "items": { - "type": "object", - "description": "Type: string (or Expression with resultType string)." - } + "treatEmptyAsNull": { + "type": "boolean", + "description": "Specify whether to treat null or empty string as a null value when reading data from an input file." } } }, + "FormatWriteSetting": { + "description": "Format write settings.", + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "The write setting type." + } + }, + "additionalProperties": { + "type": "object" + }, + "required": [ + "type" + ] + }, "DelimitedTextWriteSetting": { "description": "Delimited text write settings.", "type": "object", @@ -244,12 +655,27 @@ "fileExtension": { "type": "object", "description": "The file extension used to create the files. Type: string (or Expression with resultType string)." - } - }, - "required": [ - "fileExtension" - ] + } }, + "required": [ + "fileExtension" + ] + }, + "ParquetSource": { + "description": "A copy activity Parquet source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "storeSettings": { + "$ref": "#/definitions/ConnectorReadSetting", + "description": "Parquet store settings." + } + } + }, "DelimitedTextSource": { "description": "A copy activity DelimitedText source.", "type": "object", @@ -374,7 +800,7 @@ "type": "object", "description": "Type: string (or Expression with resultType string)." }, - "description": "Preserve." + "description": "Preserve rules." } }, "required": [ @@ -559,6 +985,16 @@ } } }, + "SapOpenHubSource": { + "description": "A copy activity source for SAP Business Warehouse Open Hub Destination source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": {} + }, "RestSource": { "description": "A copy activity Rest service source.", "type": "object", @@ -2232,6 +2668,10 @@ "collectionReference": { "type": "object", "description": "The JSON Path of the Nested Array that is going to do cross-apply. Type: object (or Expression with resultType object)." + }, + "mappings": { + "type": "object", + "description": "Column mappings with logical types" } } }, From 316b6eb6a6b9e9390a7c5eb6baa3c419947e7265 Mon Sep 17 00:00:00 2001 From: Yiliu Date: Wed, 29 May 2019 17:08:01 +0800 Subject: [PATCH 04/11] modify ParquetSink --- .../stable/2018-06-01/entityTypes/Pipeline.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index d72c782e30ec..9c81ee5a0418 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -2157,7 +2157,12 @@ "$ref": "#/definitions/CopySink" } ], - "properties": {} + "properties": { + "storeSettings": { + "$ref": "#/definitions/ConnectorWriteSetting", + "description": "Parquet store settings." + } + } }, "BlobSink": { "description": "A copy activity Azure Blob sink.", From ee70850679e3171b01674bc23820b1270ae444f9 Mon Sep 17 00:00:00 2001 From: Yiliu Date: Wed, 29 May 2019 17:48:54 +0800 Subject: [PATCH 05/11] delete treatEmptyAsNull --- .../stable/2018-06-01/entityTypes/Pipeline.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 9c81ee5a0418..15ba1ff781a8 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -616,10 +616,6 @@ "skipLineCount": { "type": "object", "description": "Indicates the number of non-empty rows to skip when reading data from input files. Type: integer (or Expression with resultType integer)." - }, - "treatEmptyAsNull": { - "type": "boolean", - "description": "Specify whether to treat null or empty string as a null value when reading data from an input file." } } }, From 4b230d1bb5edcbbed626e7547dbb07f392fe2dcc Mon Sep 17 00:00:00 2001 From: Yiliu Date: Thu, 30 May 2019 11:02:37 +0800 Subject: [PATCH 06/11] remove tablename --- .../stable/2018-06-01/entityTypes/Dataset.json | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index 10c5339b7646..d2976dc6f9cb 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -740,10 +740,7 @@ "type": "object", "description": "The table name of the Azure SQL database. Type: string (or Expression with resultType string)." } - }, - "required": [ - "tableName" - ] + } }, "AzureSqlDWTableDataset": { "x-ms-discriminator-value": "AzureSqlDWTable", @@ -769,10 +766,7 @@ "type": "object", "description": "The table name of the Azure SQL Data Warehouse. Type: string (or Expression with resultType string)." } - }, - "required": [ - "tableName" - ] + } }, "CassandraTableDataset": { "x-ms-discriminator-value": "CassandraTable", @@ -1397,10 +1391,7 @@ "type": "object", "description": "The table name of the SQL Server dataset. Type: string (or Expression with resultType string)." } - }, - "required": [ - "tableName" - ] + } }, "RestResourceDataset": { "x-ms-discriminator-value": "RestResource", From eab7ea9795f6de6a674cd28b6d8342dc8d2254de Mon Sep 17 00:00:00 2001 From: Yiliu Date: Fri, 31 May 2019 13:07:17 +0800 Subject: [PATCH 07/11] finally modify --- .../2018-06-01/entityTypes/LinkedService.json | 89 +++++++++++++++++++ .../2018-06-01/entityTypes/Pipeline.json | 80 ++++++++++++++++- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json index 52ba169775af..e98ec77898ad 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json @@ -4861,6 +4861,95 @@ "authenticationType" ] }, + "SapTableLinkedService": { + "x-ms-discriminator-value": "SapOpenHub", + "description": "SAP Business Warehouse Open Hub Destination Linked Service.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/LinkedService" + } + ], + "properties": { + "typeProperties": { + "description": "Properties specific to this linked service type.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/SapOpenHubLinkedServiceTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "SapTableLinkedServiceTypeProperties": { + "description": "Properties specific to this linked service type.", + "properties": { + "server": { + "type": "object", + "description": "Host name of the SAP instance where the table is located. Type: string (or Expression with resultType string)." + }, + "systemNumber": { + "type": "object", + "description": "System number of the SAP system where the table is located. (Usually a two-digit decimal number represented as a string.) Type: string (or Expression with resultType string)." + }, + "clientId": { + "type": "object", + "description": "Client ID of the client on the SAP system where the table is located. (Usually a three-digit decimal number represented as a string) Type: string (or Expression with resultType string)." + }, + "language": { + "type": "object", + "description": "Language of the SAP system where the table is located. The default value is EN. Type: string (or Expression with resultType string)." + }, + "systemId": { + "type": "object", + "description": "SystemID of the SAP system where the table is located. Type: string (or Expression with resultType string)." + }, + "userName": { + "type": "object", + "description": "Username to access the SAP server where the table is located. Type: string (or Expression with resultType string)." + }, + "password": { + "description": "Password to access the SAP server where the table is located.", + "$ref": "../datafactory.json#/definitions/SecretBase" + }, + "messageServer": { + "type": "object", + "description": "The hostname of the SAP Message Server. Type: string (or Expression with resultType string)." + }, + "messageServerService": { + "type": "object", + "description": "The service name or port number of the Message Server. Type: string (or Expression with resultType string)." + }, + "sncMode": { + "type": "object", + "description": "SNC activation indicator to access the SAP server where the table is located. Must be either 0 (off) or 1 (on). Type: string (or Expression with resultType string)." + }, + "sncMyName": { + "type": "object", + "description": "Initiator's SNC name to access the SAP server where the table is located. Type: string (or Expression with resultType string)." + }, + "sncPartnerName": { + "type": "object", + "description": "Communication partner's SNC name to access the SAP server where the table is located. Type: string (or Expression with resultType string)." + }, + "sncLibraryPath": { + "type": "object", + "description": "External security product's library to access the SAP server where the table is located. Type: string (or Expression with resultType string)." + }, + "sncQop": { + "type": "object", + "description": "SNC Quality of Protection. Allowed value include: 1, 2, 3, 8, 9. Type: string (or Expression with resultType string)." + }, + "logonGroup": { + "type": "object", + "description": "The Logon Group for the SAP System. Type: string (or Expression with resultType string)." + }, + "encryptedCredential": { + "type": "object", + "description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)." + } + } + }, "AzureDataExplorerLinkedService": { "x-ms-discriminator-value": "AzureDataExplorer", "description": "Azure Data Explorer (Kusto) linked service.", diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 15ba1ff781a8..5c4214985c29 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -991,6 +991,82 @@ ], "properties": {} }, + "SapTableSource": { + "description": "A copy activity source for SAP Table source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "rowCount": { + "type": "object", + "description": "The number of rows to be retrieved. Type: integer(or Expression with resultType integer)." + }, + "rowSkips": { + "type": "object", + "description": "The number of rows that will be skipped. Type: integer (or Expression with resultType integer)." + }, + "rfcTableFields": { + "type": "object", + "description": "The fields of the SAP table that will be retrieved. For example, column0, column1. Type: string (or Expression with resultType string)." + }, + "rfcTableOptions": { + "type": "object", + "description": "The options for the filtering of the SAP Table. For example, COLUMN0 EQ SOME VALUE. Type: string (or Expression with resultType string)." + }, + "batchSize": { + "type": "object", + "description": "Specifies the maximum number of rows that will be retrieved at a time when retrieving data from SAP Table. Type: integer (or Expression with resultType integer)." + }, + "customRfcReadTableFunctionModule": { + "type": "object", + "description": "Specifies the custom RFC function module that will be used to read data from SAP Table. Type: string (or Expression with resultType string)." + }, + "partitionOption": { + "description": "The partition mechanism that will be used for SAP table read in parallel.", + "type": "object", + "enum": [ + "None", + "PartitionOnInt", + "PartitionOnCalendarYear", + "PartitionOnCalendarMonth", + "PartitionOnCalendarDate" + ], + "x-ms-enum": { + "name": "TabularPartitionOption", + "modelAsString": true + } + }, + "partitionSettings": { + "description": "The settings that will be leveraged for SAP table source partitioning.", + "$ref": "#/definitions/SapTablePartitionSettings" + } + } + }, + "SapTablePartitionSettings": { + "description": "The settings that will be leveraged for SAP table source partitioning.", + "type": "object", + "properties": { + "partitionColumnName": { + "type": "object", + "description": "The name of the column that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + }, + "partitionUpperBound": { + "type": "object", + "description": "The maximum value of column specificed in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + }, + "partitionLowerBound": { + "type": "object", + "description": "The minimum value of column specificed in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + }, + "maxPartitionsNumber": { + "type": "object", + "description": "The maximum value of partitions the table will be splitted into. Type: integer (or Expression with resultType string)." + } + } + }, "RestSource": { "description": "A copy activity Rest service source.", "type": "object", @@ -1257,7 +1333,7 @@ "type": "object", "properties": { "partitionNames": { - "type": "array", + "type": "object", "items": { "type": "object", "description": "Type: string (or Expression with resultType string)." @@ -1274,7 +1350,7 @@ }, "partitionLowerBound": { "type": "object", - "description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + "description": "The minimum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." } } }, From e64566f74d125f7f97e194f83b73293e53318bb8 Mon Sep 17 00:00:00 2001 From: Yiliu Date: Fri, 31 May 2019 13:58:58 +0800 Subject: [PATCH 08/11] modify --- .../2018-06-01/entityTypes/Dataset.json | 32 +++++++++++++++++++ .../2018-06-01/entityTypes/LinkedService.json | 6 ++-- .../2018-06-01/entityTypes/Pipeline.json | 6 ++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index d2976dc6f9cb..ee9b5f560e90 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -1435,6 +1435,38 @@ } } }, + "SapTableResourceDataset": { + "x-ms-discriminator-value": "SapTableResource", + "description": "SAP Table Resource properties.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { + "typeProperties": { + "description": "SAP Table Resource properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/SapTableResourceDatasetTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "SapTableResourceDatasetTypeProperties": { + "description": "SAP Table Resource properties.", + "properties": { + "tableName": { + "type": "object", + "description": "The name of the SAP Table. Type: string (or Expression with resultType string)." + } + }, + "required": [ + "tableName" + ] + }, "WebTableDataset": { "x-ms-discriminator-value": "WebTable", "description": "The dataset points to a HTML table in the web page.", diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json index e98ec77898ad..30fe39388f96 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json @@ -4861,9 +4861,9 @@ "authenticationType" ] }, - "SapTableLinkedService": { + "SapTable": { "x-ms-discriminator-value": "SapOpenHub", - "description": "SAP Business Warehouse Open Hub Destination Linked Service.", + "description": "SAP Table Linked Service.", "type": "object", "allOf": [ { @@ -4874,7 +4874,7 @@ "typeProperties": { "description": "Properties specific to this linked service type.", "x-ms-client-flatten": true, - "$ref": "#/definitions/SapOpenHubLinkedServiceTypeProperties" + "$ref": "#/definitions/SapTableLinkedServiceTypeProperties" } }, "required": [ diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 5c4214985c29..2d91f7375073 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1055,15 +1055,15 @@ }, "partitionUpperBound": { "type": "object", - "description": "The maximum value of column specificed in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + "description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." }, "partitionLowerBound": { "type": "object", - "description": "The minimum value of column specificed in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + "description": "The minimum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." }, "maxPartitionsNumber": { "type": "object", - "description": "The maximum value of partitions the table will be splitted into. Type: integer (or Expression with resultType string)." + "description": "The maximum value of partitions the table will be split into. Type: integer (or Expression with resultType string)." } } }, From e3102063492120312b2c10629d4cb6022685db2d Mon Sep 17 00:00:00 2001 From: Yiliu Date: Tue, 4 Jun 2019 10:54:39 +0800 Subject: [PATCH 09/11] modify mappings schemaMappihng columnMapping description --- .../stable/2018-06-01/entityTypes/Pipeline.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 2d91f7375073..07a67fa9726f 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -2736,11 +2736,11 @@ "properties": { "columnMappings": { "type": "object", - "description": "Column mappings. Example: \"UserId: MyUserId, Group: MyGroup, Name: MyName\" Type: string (or Expression with resultType string)." + "description": "Column mappings. Example: \"UserId: MyUserId, Group: MyGroup, Name: MyName\" Type: string (or Expression with resultType string). This property will be retired. Please use mappings property." }, "schemaMapping": { "type": "object", - "description": "The schema mapping to map between tabular data and hierarchical data. Example: {\"Column1\": \"$.Column1\", \"Column2\": \"$.Column2.Property1\", \"Column3\": \"$.Column2.Property2\"}. Type: object (or Expression with resultType object)." + "description": "The schema mapping to map between tabular data and hierarchical data. Example: {\"Column1\": \"$.Column1\", \"Column2\": \"$.Column2.Property1\", \"Column3\": \"$.Column2.Property2\"}. Type: object (or Expression with resultType object). This property will be retired. Please use mappings property." }, "collectionReference": { "type": "object", @@ -2748,7 +2748,7 @@ }, "mappings": { "type": "object", - "description": "Column mappings with logical types" + "description": "Column mappings with logical types. Tabular->tabular example: [{\"source\":{\"name\":\"CustomerName\",\"type\":\"String\"},\"sink\":{\"name\":\"ClientName\",\"type\":\"String\"}},{\"source\":{\"name\":\"CustomerAddress\",\"type\":\"String\"},\"sink\":{\"name\":\"ClientAddress\",\"type\":\"String\"}}]. Hierarchical->tabular example: [{\"source\":{\"path\":\"$.CustomerName\",\"type\":\"String\"},\"sink\":{\"name\":\"ClientName\",\"type\":\"String\"}},{\"source\":{\"path\":\"$.CustomerAddress\",\"type\":\"String\"},\"sink\":{\"name\":\"ClientAddress\",\"type\":\"String\"}}]. Type: object (or Expression with resultType object)." } } }, From 922aef877f01d388454d3f3b102a116ecfddca60 Mon Sep 17 00:00:00 2001 From: Yiliu Date: Tue, 11 Jun 2019 11:25:33 +0800 Subject: [PATCH 10/11] modify SapTable --- .../stable/2018-06-01/entityTypes/LinkedService.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json index 30fe39388f96..fd5d1c5d067b 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json @@ -4861,8 +4861,8 @@ "authenticationType" ] }, - "SapTable": { - "x-ms-discriminator-value": "SapOpenHub", + "SapTableLinkedService": { + "x-ms-discriminator-value": "SapTable", "description": "SAP Table Linked Service.", "type": "object", "allOf": [ From 41ae155f3d9f2bb1256a9790626d3a936146c1d7 Mon Sep 17 00:00:00 2001 From: Yiliu Date: Wed, 12 Jun 2019 10:51:21 +0800 Subject: [PATCH 11/11] modify ConnectorReadSetting --- .../stable/2018-06-01/entityTypes/Pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 07a67fa9726f..c311f8bd8f7c 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -206,7 +206,7 @@ } }, "ConnectorReadSetting": { - "description": "Connector read settings.", + "description": "Connector read setting.", "type": "object", "properties": { "type": {