Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[AutoPR datafactory] Add scan folder and spark config support in Sparkjob activity #1613

Closed
wants to merge 1 commit into from
Closed

[AutoPR datafactory] Add scan folder and spark config support in Sparkjob activity #1613

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 78 additions & 7 deletions schemas/2018-06-01/Microsoft.DataFactory.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27270,6 +27270,35 @@
],
"description": "A copy activity snowflake source."
},
"SparkConfigurationParametrizationReference": {
"type": "object",
"properties": {
"referenceName": {
"type": "object",
"properties": {},
"description": "Reference spark configuration name. Type: string (or Expression with resultType string)."
},
"type": {
"oneOf": [
{
"type": "string",
"enum": [
"SparkConfigurationReference"
]
},
{
"$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
}
],
"description": "Spark configuration reference type."
}
},
"required": [
"referenceName",
"type"
],
"description": "Spark configuration reference."
},
"SparkDatasetTypeProperties": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -29246,6 +29275,22 @@
"properties": {},
"description": "Spark configuration properties, which will override the 'conf' of the spark job definition you provide."
},
"configurationType": {
"oneOf": [
{
"type": "string",
"enum": [
"Default",
"Customized",
"Artifact"
]
},
{
"$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
}
],
"description": "The type of the spark config."
},
"driverSize": {
"type": "object",
"properties": {},
Expand Down Expand Up @@ -29292,30 +29337,45 @@
"description": "Additional files used for reference in the main definition file, which will override the 'jars' and 'files' of the spark job definition you provide."
},
"numExecutors": {
"type": "object",
"properties": {},
"description": "Number of executors to launch for this job, which will override the 'numExecutors' of the spark job definition you provide. Type: integer (or Expression with resultType integer)."
},
"pythonCodeReference": {
"oneOf": [
{
"type": "integer"
"type": "array",
"items": {
"type": "object",
"properties": {}
}
},
{
"$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
}
],
"description": "Number of executors to launch for this job, which will override the 'numExecutors' of the spark job definition you provide."
"description": "Additional python code files used for reference in the main definition file, which will override the 'pyFiles' of the spark job definition you provide."
},
"pythonCodeReference": {
"scanFolder": {
"type": "object",
"properties": {},
"description": "Scanning subfolders from the root folder of the main definition file, these files will be added as reference files. The folders named 'jars', 'pyFiles', 'files' or 'archives' will be scanned, and the folders name are case sensitive. Type: boolean (or Expression with resultType boolean)."
},
"sparkConfig": {
"oneOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {}
}
},
"properties": {}
},
{
"$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
}
],
"description": "Additional python code files used for reference in the main definition file, which will override the 'pyFiles' of the spark job definition you provide."
"description": "Spark configuration property."
},
"sparkJob": {
"oneOf": [
Expand All @@ -29338,6 +29398,17 @@
}
],
"description": "Big data pool reference type."
},
"targetSparkConfiguration": {
"oneOf": [
{
"$ref": "#/definitions/SparkConfigurationParametrizationReference"
},
{
"$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
}
],
"description": "Spark configuration reference."
}
},
"required": [
Expand Down