Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Option to select Processing Service #87

Merged
merged 5 commits into from
Sep 7, 2021
Merged

Add Option to select Processing Service #87

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bbdf7bf
Add Option to select Processing Service
marvinbuss Sep 7, 2021
04fba1a
Updated portal experience
marvinbuss Sep 7, 2021
276655c
fixed bug in portal wrt password
marvinbuss Sep 7, 2021
e76526c
Updated SQL server collation
marvinbuss Sep 7, 2021
6d89dab
Updated SQL Server Admin username
marvinbuss Sep 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 68 additions & 42 deletions docs/reference/portal.dataProduct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,52 +329,46 @@
"bladeSubtitle": "General Settings",
"elements": [
{
"name": "synapseDeploymentSettings",
"label": "Synapse Deployment Settings",
"name": "dataProcessingServiceDeploymentSettings",
"label": "Data Processing Service Deployment Settings",
"type": "Microsoft.Common.Section",
"visible": true,
"elements": [
{
"name": "synapseDeploymentSettingsText",
"type": "Microsoft.Common.TextBlock",
"name": "infoBoxProcessingService",
"type": "Microsoft.Common.InfoBox",
"visible": true,
"options": {
"text": "Specify settings for your Synapse workspace.",
"link": {
"label": "Learn more",
"uri": "https://docs.microsoft.com/en-us/azure/synapse-analytics/overview-what-is"
}
"text": "Select a processing engine that you want to deploy as part of the Data Product. The service can not only be used as an additional data processing engine, but also as a workflow orchestration engine to orchestrate and trigger workflows and pipelines within Machine Learning.",
"style": "Info"
}
},
{
"name": "administratorPassword",
"label": {
"password": "Password",
"confirmPassword": "Confirm password"
},
"type": "Microsoft.Compute.CredentialsCombo",
"name": "processingService",
"label": "Create Data Processing Service",
"type": "Microsoft.Common.DropDown",
"visible": true,
"defaultValue": "",
"toolTip": {
"password": "Specify an administrator password for the Synapse workspace."
},
"defaultValue": "Data Factory",
"toolTip": "Specify an additional data processing engine that you want to deploy. This can be used as Spark engine as well as for workflow orchestration.",
"multiselect": false,
"selectAll": false,
"filter": true,
"filterPlaceholder": "Filter items ...",
"multiLine": true,
"constraints": {
"required": true,
"customPasswordRegex": "^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9])(?=.*?[#?!@$%^&*-]).{8,128}$",
"customValidationMessage": "The password must be alphanumeric, contain at least 8 characters, and have at least 1 letter, 1 number and one special character."
},
"options": {
"hideConfirmation": false
},
"osPlatform": "Windows"
},
{
"name": "infoBoxSynapseDefaultStorageAccount",
"type": "Microsoft.Common.InfoBox",
"visible": true,
"options": {
"text": "We are recommending to use a dedicated container per Data Product in the workspace Data Lake and per Data Integration in the Raw as well as Enriched & Curated Data Lake. Please select the respective Data Lake in your Data Landing Zone (e.g. Raw - '{prefix}{environment}raw', Enriched & Curated - '{prefix}{environment}encur', Workspace - '{prefix}{environment}work') and select one of the pre-existing storage account containers (e.g. 'di001', 'di002', 'dp001', 'dp002') or create and select a new one for this deployment if all of them are already in use.",
"style": "Info"
"allowedValues": [
{
"label": "Data Factory",
"description": "Select if you want to deploy a Data Factory.",
"value": "dataFactory"
},
{
"label": "Synapse",
"description": "Select if you want to deploy a Synapse workspace.",
"value": "synapse"
}
],
"required": true
}
},
{
Expand All @@ -385,11 +379,20 @@
"path": "[concat(steps('basics').deploymentDetails.subscriptionId, '/providers/Microsoft.Storage/storageAccounts?api-version=2021-04-01')]"
}
},
{
"name": "infoBoxSynapseDefaultStorageAccount",
"type": "Microsoft.Common.InfoBox",
"visible": "[equals(steps('generalSettings').dataProcessingServiceDeploymentSettings.processingService, 'synapse')]",
"options": {
"text": "We are recommending to use a dedicated container per Data Product in the workspace Data Lake and per Data Integration in the Raw as well as Enriched & Curated Data Lake. Please select the respective Data Lake in your Data Landing Zone (e.g. Raw - '{prefix}{environment}raw', Enriched & Curated - '{prefix}{environment}encur', Workspace - '{prefix}{environment}work') and select one of the pre-existing storage account containers (e.g. 'di001', 'di002', 'dp001', 'dp002') or create and select a new one for this deployment if all of them are already in use.",
"style": "Info"
}
},
{
"name": "synapseDefaultStorageAccountId",
"label": "Storage Account",
"type": "Microsoft.Common.DropDown",
"visible": true,
"visible": "[equals(steps('generalSettings').dataProcessingServiceDeploymentSettings.processingService, 'synapse')]",
"defaultValue": "",
"toolTip": "Select the Storage account to which you want to connect Synapse.",
"multiselect": false,
Expand All @@ -398,7 +401,7 @@
"filterPlaceholder": "Filter items ...",
"multiLine": true,
"constraints": {
"allowedValues": "[map(filter(steps('generalSettings').synapseDeploymentSettings.synapseDefaultStorageAccountApi.value,(item) => and(equals(item.location, steps('basics').deploymentDetails.locationName), equals(item.properties.isHnsEnabled, true))),(item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.id, '\",\"description\":\"', 'Resource Group: ', last(take(split(item.id, '/'), 5)), '\"}')))]",
"allowedValues": "[map(filter(steps('generalSettings').dataProcessingServiceDeploymentSettings.synapseDefaultStorageAccountApi.value,(item) => and(equals(item.location, steps('basics').deploymentDetails.locationName), equals(item.properties.isHnsEnabled, true))),(item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.id, '\",\"description\":\"', 'Resource Group: ', last(take(split(item.id, '/'), 5)), '\"}')))]",
"required": true
}
},
Expand All @@ -407,14 +410,14 @@
"type": "Microsoft.Solutions.ArmApiControl",
"request": {
"method": "GET",
"path": "[concat(steps('generalSettings').synapseDeploymentSettings.synapseDefaultStorageAccountId, '/blobServices/default/containers?api-version=2021-04-01')]"
"path": "[concat(steps('generalSettings').dataProcessingServiceDeploymentSettings.synapseDefaultStorageAccountId, '/blobServices/default/containers?api-version=2021-04-01')]"
}
},
{
"name": "synapseDefaultStorageAccountFileSystemId",
"label": "Synapse Default Storage Account Container",
"type": "Microsoft.Common.DropDown",
"visible": true,
"visible": "[equals(steps('generalSettings').dataProcessingServiceDeploymentSettings.processingService, 'synapse')]",
"defaultValue": "",
"toolTip": "Default storage container for the Synapse workspace. If no option is shown, please make sure that a container exists within the storage account selected above.",
"multiselect": false,
Expand All @@ -423,7 +426,7 @@
"filterPlaceholder": "Filter items ...",
"multiLine": true,
"constraints": {
"allowedValues": "[map(steps('generalSettings').synapseDeploymentSettings.synapseDefaultStorageAccountFileSystemApi.value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.id, '\"}')))]",
"allowedValues": "[map(steps('generalSettings').dataProcessingServiceDeploymentSettings.synapseDefaultStorageAccountFileSystemApi.value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.id, '\"}')))]",
"required": true
}
}
Expand Down Expand Up @@ -520,6 +523,28 @@
]
}
},
{
"name": "administratorPassword",
"label": {
"password": "Password",
"confirmPassword": "Confirm password"
},
"type": "Microsoft.Compute.CredentialsCombo",
"visible": true,
"defaultValue": "",
"toolTip": {
"password": "Specify an administrator password for the SQL Server and Synapse workspace, if you have selected Synapse as processing sercice."
},
"constraints": {
"required": true,
"customPasswordRegex": "^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9])(?=.*?[#?!@$%^&*-]).{8,128}$",
"customValidationMessage": "The password must be alphanumeric, contain at least 8 characters, and have at least 1 letter, 1 number and one special character."
},
"options": {
"hideConfirmation": false
},
"osPlatform": "Windows"
},
{
"name": "infoBoxRoleAssignment",
"type": "Microsoft.Common.InfoBox",
Expand Down Expand Up @@ -921,8 +946,9 @@
"environment": "[if(empty(steps('basics').dataProductName.environment), '', steps('basics').dataProductName.environment)]",
"prefix": "[if(empty(steps('basics').dataProductName.dataProductPrefix), '', steps('basics').dataProductName.dataProductPrefix)]",
"sqlFlavour": "[if(empty(steps('generalSettings').generalSettings.sqlFlavour), '', steps('generalSettings').generalSettings.sqlFlavour)]",
"administratorPassword": "[if(empty(steps('generalSettings').synapseDeploymentSettings.administratorPassword.password), '', steps('generalSettings').synapseDeploymentSettings.administratorPassword.password)]",
"synapseDefaultStorageAccountFileSystemId": "[if(empty(steps('generalSettings').synapseDeploymentSettings.synapseDefaultStorageAccountFileSystemId), '', steps('generalSettings').synapseDeploymentSettings.synapseDefaultStorageAccountFileSystemId)]",
"administratorPassword": "[if(empty(steps('generalSettings').generalSettings.administratorPassword.password), '', steps('generalSettings').generalSettings.administratorPassword.password)]",
"processingService": "[if(empty(steps('generalSettings').dataProcessingServiceDeploymentSettings.processingService), '', steps('generalSettings').dataProcessingServiceDeploymentSettings.processingService)]",
"synapseDefaultStorageAccountFileSystemId": "[if(empty(steps('generalSettings').dataProcessingServiceDeploymentSettings.synapseDefaultStorageAccountFileSystemId), '', steps('generalSettings').dataProcessingServiceDeploymentSettings.synapseDefaultStorageAccountFileSystemId)]",
"purviewId": "[if(empty(steps('generalSettings').dataGovernanceSettings.purviewId.id), '', steps('generalSettings').dataGovernanceSettings.purviewId.id)]",
"enableRoleAssignments": "[steps('generalSettings').generalSettings.enableRoleAssignments]",
"subnetId": "[if(empty(steps('connectivitySettings').virtualNetwork.subnetId), '', steps('connectivitySettings').virtualNetwork.subnetId)]",
Expand Down
18 changes: 12 additions & 6 deletions infra/main.bicep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,14 @@ param sqlFlavour string = 'sql'
@secure()
@description('Specifies the administrator password of the sql servers.')
param administratorPassword string
@description('Specifies the resource ID of the default storage account file system for synapse.')
param synapseDefaultStorageAccountFileSystemId string
@allowed([
'dataFactory'
'synapse'
])
@description('Specifies the data engineering service that will be deployed (Data Factory, Synapse).')
param processingService string = 'dataFactory'
@description('Specifies the resource ID of the default storage account file system for synapse. If you selected dataFactory as processingService, leave this value empty as is.')
param synapseDefaultStorageAccountFileSystemId string = ''
@description('Specifies the resource ID of the central purview instance.')
param purviewId string = ''
@description('Specifies whether role assignments should be enabled.')
Expand Down Expand Up @@ -101,7 +107,7 @@ module keyVault001 'modules/services/keyvault.bicep' = {
}
}

module synapse001 'modules/services/synapse.bicep' = {
module synapse001 'modules/services/synapse.bicep' = if (processingService == 'synapse') {
name: 'synapse001'
scope: resourceGroup()
params: {
Expand All @@ -121,16 +127,16 @@ module synapse001 'modules/services/synapse.bicep' = {
}
}

module synapse001RoleAssignmentStorage 'modules/auxiliary/synapseRoleAssignmentStorage.bicep' = if (enableRoleAssignments) {
module synapse001RoleAssignmentStorage 'modules/auxiliary/synapseRoleAssignmentStorage.bicep' = if (processingService == 'synapse' && enableRoleAssignments) {
name: 'synapse001RoleAssignmentStorage'
scope: resourceGroup(synapseDefaultStorageAccountSubscriptionId, synapseDefaultStorageAccountResourceGroupName)
params: {
storageAccountFileSystemId: synapseDefaultStorageAccountFileSystemId
synapseId: synapse001.outputs.synapseId
synapseId: processingService == 'synapse' ? synapse001.outputs.synapseId : ''
}
}

module datafactory001 'modules/services/datafactory.bicep' = {
module datafactory001 'modules/services/datafactory.bicep' = if (processingService == 'dataFactory') {
name: 'datafactory001'
scope: resourceGroup()
params: {
Expand Down
Loading