Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DataFactory]Add AzurePostgreSql Sink and Oracle/SapTable/Netezza Partitions and Several Relational sets #6617

Merged
merged 8 commits into from
Jul 19, 2019
Merged

[DataFactory]Add AzurePostgreSql Sink and Oracle/SapTable/Netezza Partitions and Several Relational sets #6617

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b20c8b8
[DataFactory]Add AzurePostgreSql Sink and Oracle/SapTable/Netezza Par…
wenbof Jul 15, 2019
80fd3f1
Fix typo.
wenbof Jul 15, 2019
dbe9b5f
update SapBwCube
wenbof Jul 17, 2019
057f2e3
Merge branch 'master' into b0715
wenbof Jul 17, 2019
8b1eb70
fix issue brought by merging from azure branch
wenbof Jul 17, 2019
286bc89
remove SapBwCube and Sybase for investigation
wenbof Jul 18, 2019
75bd5a9
will postpone SapBwSource in another coming PR.
wenbof Jul 19, 2019
7f632e8
refine case of Oracle in description
wenbof Jul 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 96 additions & 1 deletion ...factory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1379,6 +1379,84 @@
}
}
},
"OdbcTableDataset": {
"x-ms-discriminator-value": "OdbcTable",
"description": "The ODBC table dataset.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/Dataset"
}
],
"properties": {
"typeProperties": {
"description": "ODBC table dataset properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/OdbcTableDatasetTypeProperties"
}
}
},
"OdbcTableDatasetTypeProperties": {
"description": "ODBC table dataset properties.",
"properties": {
"tableName": {
"type": "object",
"description": "The ODBC table name. Type: string (or Expression with resultType string)."
}
}
},
"MySqlTableDataset": {
"x-ms-discriminator-value": "MySqlTable",
"description": "The MySQL table dataset.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/Dataset"
}
],
"properties": {
"typeProperties": {
"description": "MySQL table dataset properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/MySqlTableDatasetTypeProperties"
}
}
},
"MySqlTableDatasetTypeProperties": {
"description": "MySql table dataset properties.",
"properties": {
"tableName": {
"type": "object",
"description": "The MySQL table name. Type: string (or Expression with resultType string)."
}
}
},
"PostgreSqlTableDataset": {
"x-ms-discriminator-value": "PostgreSqlTable",
"description": "The PostgreSQL table dataset.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/Dataset"
}
],
"properties": {
"typeProperties": {
"description": "PostgreSQL table dataset properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/PostgreSqlTableDatasetTypeProperties"
}
}
},
"PostgreSqlTableDatasetTypeProperties": {
"description": "PostgreSQL table dataset properties.",
"properties": {
"tableName": {
"type": "object",
"description": "The PostgreSQL table name. Type: string (or Expression with resultType string)."
}
}
},
"MicrosoftAccessTableDataset": {
"x-ms-discriminator-value": "MicrosoftAccessTable",
"description": "The Microsoft Access table dataset.",
Expand Down Expand Up @@ -1844,7 +1922,24 @@
"typeProperties": {
"description": "Properties specific to this dataset type.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/GenericDatasetTypeProperties"
"$ref": "#/definitions/AzurePostgreSqlTableDatasetTypeProperties"
}
}
},
"AzurePostgreSqlTableDatasetTypeProperties": {
"description": "Azure PostgreSQL dataset properties.",
"properties": {
"tableName": {
"type": "object",
"description": "The table name of the Azure PostgreSQL database which includes both schema and table. Type: string (or Expression with resultType string)."
},
"table": {
"type": "object",
"description": "The table name of the Azure PostgreSQL database. Type: string (or Expression with resultType string)."
},
"schema": {
"type": "object",
"description": "The schema name of the Azure PostgreSQL database. Type: string (or Expression with resultType string)."
}
}
},
Expand Down
240 changes: 240 additions & 0 deletions ...actory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -984,6 +984,111 @@
}
}
},
"Db2Source": {
"description": "A copy activity source for Db2 databases.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "Database query. Type: string (or Expression with resultType string)."
}
}
},
"OdbcSource": {
"description": "A copy activity source for ODBC databases.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "Database query. Type: string (or Expression with resultType string)."
}
}
},
"MySqlSource": {
"description": "A copy activity source for MySQL databases.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "Database query. Type: string (or Expression with resultType string)."
}
}
},
"PostgreSqlSource": {
"description": "A copy activity source for PostgreSQL databases.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "Database query. Type: string (or Expression with resultType string)."
}
}
},
"SapBwSource": {
wenbof-zz marked this conversation as resolved.
Show resolved Hide resolved
"description": "A copy activity source for SapBW server via MDX.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "MDX query. Type: string (or Expression with resultType string)."
}
}
},
"SybaseSource": {
"description": "A copy activity source for Sybase databases.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "Database query. Type: string (or Expression with resultType string)."
}
}
},
"ODataSource": {
"description": "A copy activity source for OData source.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySource"
}
],
"properties": {
"query": {
"type": "object",
"description": "OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)."
}
}
},
"SalesforceSource": {
"description": "A copy activity Salesforce source.",
"type": "object",
Expand Down Expand Up @@ -1129,6 +1234,48 @@
"customRfcReadTableFunctionModule": {
"type": "object",
"description": "Specifies the custom RFC function module that will be used to read data from SAP Table. Type: string (or Expression with resultType string)."
},
"partitionOption": {
"description": "The partition mechanism that will be used for SAP table read in parallel.",
"type": "string",
"enum": [
"None",
"PartitionOnInt",
"PartitionOnCalendarYear",
"PartitionOnCalendarMonth",
"PartitionOnCalendarDate",
"PartitionOnTime"
],
"x-ms-enum": {
"name": "SapTablePartitionOption",
"modelAsString": true
}
},
"partitionSettings": {
"description": "The settings that will be leveraged for SAP table source partitioning.",
"$ref": "#/definitions/SapTablePartitionSettings"
}
}
},
"SapTablePartitionSettings": {
wenbof-zz marked this conversation as resolved.
Show resolved Hide resolved
"description": "The settings that will be leveraged for SAP table source partitioning.",
"type": "object",
"properties": {
"partitionColumnName": {
"type": "object",
"description": "The name of the column that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"partitionUpperBound": {
"type": "object",
"description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"partitionLowerBound": {
"type": "object",
"description": "The minimum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"maxPartitionsNumber": {
"type": "object",
"description": "The maximum value of partitions the table will be split into. Type: integer (or Expression with resultType string)."
}
}
},
Expand Down Expand Up @@ -1403,6 +1550,49 @@
"queryTimeout": {
"type": "object",
"description": "Query timeout. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))."
},
"partitionOption": {
"description": "The partition mechanism that will be used for oracle read in parallel.",
"type": "string",
"enum": [
"None",
"PhysicalPartitionsOfTable",
"DynamicRange"
],
"x-ms-enum": {
"name": "OraclePartitionOption",
"modelAsString": true
}
},
"partitionSettings": {
"description": "The settings that will be leveraged for oracle source partitioning.",
"$ref": "#/definitions/OraclePartitionSettings"
}
}
},
"OraclePartitionSettings": {
"description": "The settings that will be leveraged for oracle source partitioning.",
"type": "object",
"properties": {
"partitionNames": {
"type": "object",
"items": {
"type": "object",
"description": "Type: string (or Expression with resultType string)."
},
"description": "Names of the physical partitions of oracle table. "
wenbof-zz marked this conversation as resolved.
Show resolved Hide resolved
},
"partitionColumnName": {
"type": "object",
"description": "The name of the column in integer type that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"partitionUpperBound": {
"type": "object",
"description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"partitionLowerBound": {
"type": "object",
"description": "The minimum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
}
}
},
Expand Down Expand Up @@ -1680,6 +1870,21 @@
}
}
},
"AzurePostgreSqlSink": {
"description": "A copy activity Azure PostgreSQL sink.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/CopySink"
}
],
"properties": {
"preCopyScript": {
"type": "object",
"description": "A query to execute before starting the copy. Type: string (or Expression with resultType string)."
}
}
},
"ConcurSource": {
"description": "A copy activity Concur Service source.",
"type": "object",
Expand Down Expand Up @@ -2052,6 +2257,41 @@
"query": {
"type": "object",
"description": "A query to retrieve data from source. Type: string (or Expression with resultType string)."
},
"partitionOption": {
"description": "The partition mechanism that will be used for Netezza read in parallel.",
"type": "string",
"enum": [
"None",
"DataSlice",
"DynamicRange"
],
"x-ms-enum": {
"name": "NetezzaPartitionOption",
"modelAsString": true
}
},
"partitionSettings": {
"description": "The settings that will be leveraged for Netezza source partitioning.",
"$ref": "#/definitions/NetezzaPartitionSettings"
}
}
},
"NetezzaPartitionSettings": {
"description": "The settings that will be leveraged for Netezza source partitioning.",
"type": "object",
"properties": {
"partitionColumnName": {
"type": "object",
"description": "The name of the column in integer type that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"partitionUpperBound": {
"type": "object",
"description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
},
"partitionLowerBound": {
"type": "object",
"description": "The minimum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)."
}
}
},
Expand Down