From e5764a2960a9d4776a8b02b448cf3580efc70ce6 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Mon, 7 Aug 2023 22:17:46 +0200
Subject: [PATCH 01/26] First draft & test

---
 .../.test/common/dependencies.bicep           |  45 ++++
 .../workspaces/.test/common/main.test.bicep   |  10 +
 modules/databricks/workspaces/main.bicep      | 199 +++++++++++++++++-
 3 files changed, 245 insertions(+), 9 deletions(-)

diff --git a/modules/databricks/workspaces/.test/common/dependencies.bicep b/modules/databricks/workspaces/.test/common/dependencies.bicep
index a7f42aee7b..c6d0cf5024 100644
--- a/modules/databricks/workspaces/.test/common/dependencies.bicep
+++ b/modules/databricks/workspaces/.test/common/dependencies.bicep
@@ -4,10 +4,55 @@ param location string = resourceGroup().location
 @description('Required. The name of the Managed Identity to create.')
 param managedIdentityName string
 
+@description('Required. The name of the Key Vault to create.')
+param keyVaultName string
+
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = {
     name: managedIdentityName
     location: location
 }
 
+resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = {
+    name: keyVaultName
+    location: location
+    properties: {
+        sku: {
+            family: 'A'
+            name: 'standard'
+        }
+        tenantId: tenant().tenantId
+        enablePurgeProtection: true // Required by batch account
+        softDeleteRetentionInDays: 7
+        enabledForTemplateDeployment: true
+        enabledForDiskEncryption: true
+        enabledForDeployment: true
+        enableRbacAuthorization: true
+        accessPolicies: []
+    }
+
+    resource key 'keys@2022-07-01' = {
+        name: 'keyEncryptionKey'
+        properties: {
+            kty: 'RSA'
+        }
+    }
+}
+
+resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+    name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-Key-Key-Vault-Crypto-User-RoleAssignment')
+    scope: keyVault::key
+    properties: {
+        principalId: '2ff814a6-3304-4ab8-85cb-cd0e6f879c1d' // AzureDatabricks enterprise application
+        roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // Key Vault Crypto User
+        principalType: 'ServicePrincipal'
+    }
+}
+
+@description('The resource ID of the created Key Vault.')
+output keyVaultResourceId string = keyVault.id
+
+@description('The name of the created Key Vault encryption key.')
+output keyVaultKeyName string = keyVault::key.name
+
 @description('The principal ID of the created Managed Identity.')
 output managedIdentityPrincipalId string = managedIdentity.properties.principalId
diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index cad178b102..b30ab37dd9 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -17,6 +17,9 @@ param serviceShort string = 'dwcom'
 @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).')
 param enableDefaultTelemetry bool = true
 
+@description('Generated. Used as a basis for unique resource names.')
+param baseTime string = utcNow('u')
+
 @description('Optional. A token to inject into the name of each resource.')
 param namePrefix string = '[[namePrefix]]'
 
@@ -36,6 +39,9 @@ module nestedDependencies 'dependencies.bicep' = {
   name: '${uniqueString(deployment().name, location)}-nestedDependencies'
   params: {
     managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}'
+    // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total)
+    // keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
+    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-alsehr'
   }
 }
 
@@ -82,5 +88,9 @@ module testDeployment '../../main.bicep' = {
       Environment: 'Non-Prod'
       Role: 'DeploymentValidation'
     }
+    cMKManagedDisksKeyName: nestedDependencies.outputs.keyVaultKeyName
+    cMKManagedDisksKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
+    cMKManagedServicesKeyName: nestedDependencies.outputs.keyVaultKeyName
+    cMKManagedServicesKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
   }
 }
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 5c306dbc5e..1839b2a811 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -22,9 +22,6 @@ param location string = resourceGroup().location
 @description('Optional. Array of role assignment objects that contain the \'roleDefinitionIdOrName\' and \'principalId\' to define RBAC role assignments on this resource. In the roleDefinitionIdOrName attribute, you can provide either the display name of the role definition, or its fully qualified ID in the following format: \'/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11\'.')
 param roleAssignments array = []
 
-@description('Optional. The workspace\'s custom parameters.')
-param parameters object = {}
-
 @description('Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely.')
 @minValue(0)
 @maxValue(365)
@@ -56,6 +53,95 @@ param tags object = {}
 @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).')
 param enableDefaultTelemetry bool = true
 
+@description('Optional. The resource ID of a Virtual Network where this Databricks Cluster should be created.')
+param customVirtualNetworkResourceId string = ''
+
+@description('Optional. The resource ID of a Azure Machine Learning workspace to link with Databricks workspace.')
+param amlWorkspaceResourceId string = ''
+
+@description('Optional. The name of the Private Subnet within the Virtual Network.')
+param customPrivateSubnetName string = ''
+
+@description('Optional. The name of a Public Subnet within the Virtual Network')
+param customPublicSubnetName string = ''
+
+@description('Optional. Disable Public IP.')
+param disablePublicIp bool = false
+
+@description('Conditional. The resource ID of a key vault to reference a customer managed key for encryption from. Required if \'cMKKeyName\' is not empty.')
+param cMKManagedServicesKeyVaultResourceId string = ''
+
+@description('Optional. The name of the customer managed key to use for encryption.')
+param cMKManagedServicesKeyName string = ''
+
+@description('Optional. The version of the customer managed key to reference for encryption. If not provided, the latest key version is used.')
+param cMKManagedServicesKeyVersion string = ''
+
+@description('Conditional. The resource ID of a key vault to reference a customer managed key for encryption from. Required if \'cMKKeyName\' is not empty.')
+param cMKManagedDisksKeyVaultResourceId string = ''
+
+@description('Optional. The name of the customer managed key to use for encryption.')
+param cMKManagedDisksKeyName string = ''
+
+@description('Optional. The version of the customer managed key to reference for encryption. If not provided, the latest key version is used.')
+param cMKManagedDisksKeyVersion string = ''
+
+@description('Optional. Enable Auto Rotation of Key.')
+param cMKManagedDisksKeyRotationToLatestKeyVersionEnabled bool = true
+
+@description('Optional. Name of the outbound Load Balancer Backend Pool for Secure Cluster Connectivity (No Public IP).')
+param loadBalancerBackendPoolName string = ''
+
+@description('Optional. Name of the outbound Load Balancer Backend Pool for Secure Cluster Connectivity (No Public IP).')
+param loadBalancerResourceId string = ''
+
+@description('Optional. Name of the NAT gateway for Secure Cluster Connectivity (No Public IP) workspace subnets.')
+param natGatewayName string = ''
+
+@description('Optional. Prepare the workspace for encryption. Enables the Managed Identity for managed storage account.')
+param prepareEncryption bool = false
+
+@description('Optional. Name of the Public IP for No Public IP workspace with managed vNet.')
+param publicIpName string = ''
+
+@description('Optional. A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest.')
+param requireInfrastructureEncryption bool = true
+
+@description('Optional. Default DBFS storage account name.')
+param storageAccountName string = ''
+
+@description('Optional. Storage account SKU name.')
+param storageAccountSkuName string = ''
+
+@description('Optional. Address prefix for Managed virtual network.')
+param vnetAddressPrefix string = ''
+
+@description('Optional. The workspace provider authorizations.')
+param authorizations array = []
+
+@description('Optional. The details of Managed Identity of Disk Encryption Set used for Managed Disk Encryption.')
+param managedDiskIdentity object = {}
+
+@description('Optional. 	The network access type for accessing workspace. Set value to disabled to access workspace only via private link.')
+@allowed([
+  'Disabled'
+  'Enabled'
+])
+param publicNetworkAccess string = 'Enabled'
+
+@description('Optional. Gets or sets a value indicating whether data plane (clusters) to control plane communication happen over private endpoint.')
+@allowed([
+  'AllRules'
+  'NoAzureDatabricksRules'
+])
+param requiredNsgRules string = 'AllRules'
+
+@description('Optional. The details of Managed Identity of Storage Account.')
+param storageAccountIdentity object = {}
+
+@description('Optional. The blob URI where the UI definition file is located.')
+param uiDefinitionUri string = ''
+
 @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to \'\' to disable log collection.')
 @allowed([
   ''
@@ -98,9 +184,6 @@ var diagnosticsLogs = contains(diagnosticLogCategoriesToEnable, 'allLogs') ? [
   }
 ] : contains(diagnosticLogCategoriesToEnable, '') ? [] : diagnosticsLogsSpecified
 
-var managedResourceGroupName = '${name}-rg'
-var managedResourceGroupIdVar = '${subscription().id}/resourceGroups/${managedResourceGroupName}'
-
 resource defaultTelemetry 'Microsoft.Resources/deployments@2021-04-01' = if (enableDefaultTelemetry) {
   name: 'pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-${uniqueString(deployment().name, location)}'
   properties: {
@@ -113,7 +196,27 @@ resource defaultTelemetry 'Microsoft.Resources/deployments@2021-04-01' = if (ena
   }
 }
 
-resource workspace 'Microsoft.Databricks/workspaces@2018-04-01' = {
+resource cMKManagedDisksKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(cMKManagedDisksKeyVaultResourceId)) {
+  name: last(split(cMKManagedDisksKeyVaultResourceId, '/'))!
+  scope: resourceGroup(split(cMKManagedDisksKeyVaultResourceId, '/')[2], split(cMKManagedDisksKeyVaultResourceId, '/')[4])
+}
+
+resource cMKManagedDisksKeyVaultKey 'Microsoft.KeyVault/vaults/keys@2023-02-01' existing = if (!empty(cMKManagedDisksKeyVaultResourceId) && !empty(cMKManagedDisksKeyName)) {
+  name: '${last(split(cMKManagedDisksKeyVaultResourceId, '/'))}/${cMKManagedDisksKeyName}'!
+  scope: resourceGroup(split(cMKManagedDisksKeyVaultResourceId, '/')[2], split(cMKManagedDisksKeyVaultResourceId, '/')[4])
+}
+
+resource cMKManagedServicesKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(cMKManagedServicesKeyVaultResourceId)) {
+  name: last(split(cMKManagedServicesKeyVaultResourceId, '/'))!
+  scope: resourceGroup(split(cMKManagedServicesKeyVaultResourceId, '/')[2], split(cMKManagedServicesKeyVaultResourceId, '/')[4])
+}
+
+resource cMKManagedServicesKeyVaultKey 'Microsoft.KeyVault/vaults/keys@2023-02-01' existing = if (!empty(cMKManagedServicesKeyVaultResourceId) && !empty(cMKManagedServicesKeyName)) {
+  name: '${last(split(cMKManagedServicesKeyVaultResourceId, '/'))}/${cMKManagedServicesKeyName}'!
+  scope: resourceGroup(split(cMKManagedServicesKeyVaultResourceId, '/')[2], split(cMKManagedServicesKeyVaultResourceId, '/')[4])
+}
+
+resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
   name: name
   location: location
   tags: tags
@@ -121,8 +224,86 @@ resource workspace 'Microsoft.Databricks/workspaces@2018-04-01' = {
     name: pricingTier
   }
   properties: {
-    managedResourceGroupId: (empty(managedResourceGroupId) ? managedResourceGroupIdVar : managedResourceGroupId)
-    parameters: parameters
+    managedResourceGroupId: !empty(managedResourceGroupId) ? managedResourceGroupId : '${subscription().id}/resourceGroups/${name}-rg'
+    parameters: {
+      customVirtualNetworkId: {
+        value: customVirtualNetworkResourceId
+      }
+      amlWorkspaceId: {
+        value: amlWorkspaceResourceId
+      }
+      customPrivateSubnetName: {
+        value: customPrivateSubnetName
+      }
+      customPublicSubnetName: {
+        value: customPublicSubnetName
+      }
+      enableNoPublicIp: {
+        value: disablePublicIp
+      }
+      // encryption: !empty(cMKKeyName) ? {
+      //   keySource: 'Microsoft.KeyVault'
+      //   keyVaultProperties: {
+      //     keyVaultUri: cMKKeyVault.properties.vaultUri
+      //     keyName: cMKKeyName
+      //     keyVersion: !empty(cMKKeyVersion) ? cMKKeyVersion : last(split(cMKKeyVaultKey.properties.keyUriWithVersion, '/'))
+      //   }
+      // } : null
+      loadBalancerBackendPoolName: {
+        value: loadBalancerBackendPoolName
+      }
+      loadBalancerId: {
+        value: loadBalancerResourceId
+      }
+      natGatewayName: {
+        value: natGatewayName
+      }
+      prepareEncryption: {
+        value: prepareEncryption
+      }
+      publicIpName: {
+        value: publicIpName
+      }
+      requireInfrastructureEncryption: {
+        value: requireInfrastructureEncryption
+      }
+      storageAccountName: {
+        value: storageAccountName
+      }
+      storageAccountSkuName: {
+        value: storageAccountSkuName
+      }
+      vnetAddressPrefix: {
+        value: vnetAddressPrefix
+      }
+    }
+    authorizations: authorizations
+    managedDiskIdentity: managedDiskIdentity
+    publicNetworkAccess: publicNetworkAccess
+    requiredNsgRules: requiredNsgRules
+    storageAccountIdentity: storageAccountIdentity
+    uiDefinitionUri: uiDefinitionUri
+    encryption: !empty(cMKManagedServicesKeyName) || !empty(cMKManagedServicesKeyName) ? {
+      entities: {
+        managedServices: !empty(cMKManagedServicesKeyName) ? {
+          keySource: 'Microsoft.Keyvault'
+          keyVaultProperties: {
+            keyVaultUri: cMKManagedServicesKeyVault.properties.vaultUri
+            keyName: cMKManagedServicesKeyName
+            keyVersion: !empty(cMKManagedServicesKeyVersion) ? cMKManagedServicesKeyVersion : last(split(cMKManagedServicesKeyVaultKey.properties.keyUriWithVersion, '/'))
+          }
+        } : null
+        managedDisk: !empty(cMKManagedDisksKeyName) ? {
+          keySource: 'Microsoft.Keyvault'
+          keyVaultProperties: {
+            keyVaultUri: cMKManagedDisksKeyVault.properties.vaultUri
+            keyName: cMKManagedDisksKeyName
+            keyVersion: !empty(cMKManagedDisksKeyVersion) ? cMKManagedDisksKeyVersion : last(split(cMKManagedDisksKeyVaultKey.properties.keyUriWithVersion, '/'))
+          }
+          rotationToLatestKeyVersionEnabled: cMKManagedDisksKeyRotationToLatestKeyVersionEnabled
+        } : null
+      }
+    } : null
   }
 }
 

From b266c9e8adfb0adc9122f08e34b99d051d8144c2 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Tue, 8 Aug 2023 19:07:45 +0200
Subject: [PATCH 02/26] Latest draft

---
 .../.test/common/dependencies.bicep           | 60 ++++++++++++++-
 .../workspaces/.test/common/main.test.bicep   | 14 +++-
 modules/databricks/workspaces/main.bicep      | 74 +++++++++----------
 3 files changed, 104 insertions(+), 44 deletions(-)

diff --git a/modules/databricks/workspaces/.test/common/dependencies.bicep b/modules/databricks/workspaces/.test/common/dependencies.bicep
index c6d0cf5024..6ab76269a6 100644
--- a/modules/databricks/workspaces/.test/common/dependencies.bicep
+++ b/modules/databricks/workspaces/.test/common/dependencies.bicep
@@ -7,6 +7,15 @@ param managedIdentityName string
 @description('Required. The name of the Key Vault to create.')
 param keyVaultName string
 
+@description('Required. The name of the Azure Machine Learning Workspace to create.')
+param amlWorkspaceName string
+
+@description('Required. The name of the Storage Account to create.')
+param storageAccountName string
+
+@description('Required. The name of the Application Insights Instanec to create.')
+param applicationInsightsName string
+
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = {
     name: managedIdentityName
     location: location
@@ -42,12 +51,61 @@ resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
     name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-Key-Key-Vault-Crypto-User-RoleAssignment')
     scope: keyVault::key
     properties: {
-        principalId: '2ff814a6-3304-4ab8-85cb-cd0e6f879c1d' // AzureDatabricks enterprise application
+        principalId: '5167ea7a-355a-466f-ae8b-8ea60f718b35' // AzureDatabricks Enterprise Application Object Id
         roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // Key Vault Crypto User
         principalType: 'ServicePrincipal'
     }
 }
 
+resource amlPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+    name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-Key-Vault-Contributor')
+    scope: keyVault::key
+    properties: {
+        principalId: managedIdentity.properties.principalId
+        roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') // Contributor
+        principalType: 'ServicePrincipal'
+    }
+}
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = {
+    name: storageAccountName
+    location: location
+    sku: {
+        name: 'Standard_ZRS'
+    }
+    kind: 'StorageV2'
+    properties: {}
+}
+
+resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
+    name: applicationInsightsName
+    location: location
+    kind: 'web'
+    properties: {
+        Application_Type: 'web'
+    }
+}
+
+resource machineLearningWorkspace 'Microsoft.MachineLearningServices/workspaces@2023-04-01' = {
+    name: amlWorkspaceName
+    location: location
+    identity: {
+        type: 'UserAssigned'
+        userAssignedIdentities: {
+            '${managedIdentity.id}': {}
+        }
+    }
+    properties: {
+        storageAccount: storageAccount.id
+        keyVault: keyVault.id
+        applicationInsights: applicationInsights.id
+        primaryUserAssignedIdentity: managedIdentity.id
+    }
+}
+
+@description('The resource ID of the created Azure Machine Learning Workspace.')
+output machineLearningWorkspaceResourceId string = machineLearningWorkspace.id
+
 @description('The resource ID of the created Key Vault.')
 output keyVaultResourceId string = keyVault.id
 
diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index b30ab37dd9..837046e20f 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -39,9 +39,12 @@ module nestedDependencies 'dependencies.bicep' = {
   name: '${uniqueString(deployment().name, location)}-nestedDependencies'
   params: {
     managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}'
+    amlWorkspaceName: 'dep-${namePrefix}-aml-${serviceShort}'
+    applicationInsightsName: 'dep-${namePrefix}-appi-${serviceShort}'
+    storageAccountName: 'dep${namePrefix}sa${serviceShort}'
     // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total)
     // keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
-    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-alsehr'
+    keyVaultName: 'dep${namePrefix}kv${serviceShort}al'
   }
 }
 
@@ -74,7 +77,7 @@ module testDeployment '../../main.bicep' = {
     diagnosticEventHubAuthorizationRuleId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId
     diagnosticEventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName
     diagnosticLogsRetentionInDays: 7
-    lock: 'CanNotDelete'
+    // lock: 'CanNotDelete'
     roleAssignments: [
       {
         roleDefinitionIdOrName: 'Reader'
@@ -92,5 +95,12 @@ module testDeployment '../../main.bicep' = {
     cMKManagedDisksKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
     cMKManagedServicesKeyName: nestedDependencies.outputs.keyVaultKeyName
     cMKManagedServicesKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
+    storageAccountName: '${namePrefix}${serviceShort}001'
+    publicIpName: 'nat-gw-public-ip'
+    natGatewayName: 'nat-gateway'
+    prepareEncryption: true
+    requiredNsgRules: 'AllRules'
+    skuName: 'premium'
+    amlWorkspaceResourceId: nestedDependencies.outputs.machineLearningWorkspaceResourceId
   }
 }
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 1839b2a811..fb1218c912 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -14,7 +14,7 @@ param managedResourceGroupId string = ''
   'standard'
   'premium'
 ])
-param pricingTier string = 'premium'
+param skuName string = 'premium'
 
 @description('Optional. Location for all Resources.')
 param location string = resourceGroup().location
@@ -111,10 +111,10 @@ param requireInfrastructureEncryption bool = true
 param storageAccountName string = ''
 
 @description('Optional. Storage account SKU name.')
-param storageAccountSkuName string = ''
+param storageAccountSkuName string = 'Standard_GRS'
 
 @description('Optional. Address prefix for Managed virtual network.')
-param vnetAddressPrefix string = ''
+param vnetAddressPrefix string = '10.139'
 
 @description('Optional. The workspace provider authorizations.')
 param authorizations array = []
@@ -221,68 +221,60 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
   location: location
   tags: tags
   sku: {
-    name: pricingTier
+    name: skuName
   }
   properties: {
     managedResourceGroupId: !empty(managedResourceGroupId) ? managedResourceGroupId : '${subscription().id}/resourceGroups/${name}-rg'
     parameters: {
-      customVirtualNetworkId: {
+      customVirtualNetworkId: !empty(customVirtualNetworkResourceId) ? {
         value: customVirtualNetworkResourceId
-      }
-      amlWorkspaceId: {
+      } : null
+      amlWorkspaceId: !empty(amlWorkspaceResourceId) ? {
         value: amlWorkspaceResourceId
-      }
-      customPrivateSubnetName: {
+      } : null
+      customPrivateSubnetName: !empty(customPrivateSubnetName) ? {
         value: customPrivateSubnetName
-      }
-      customPublicSubnetName: {
+      } : null
+      customPublicSubnetName: !empty(customPublicSubnetName) ? {
         value: customPublicSubnetName
-      }
+      } : null
       enableNoPublicIp: {
         value: disablePublicIp
       }
-      // encryption: !empty(cMKKeyName) ? {
-      //   keySource: 'Microsoft.KeyVault'
-      //   keyVaultProperties: {
-      //     keyVaultUri: cMKKeyVault.properties.vaultUri
-      //     keyName: cMKKeyName
-      //     keyVersion: !empty(cMKKeyVersion) ? cMKKeyVersion : last(split(cMKKeyVaultKey.properties.keyUriWithVersion, '/'))
-      //   }
-      // } : null
-      loadBalancerBackendPoolName: {
+      loadBalancerBackendPoolName: !empty(loadBalancerBackendPoolName) ? {
         value: loadBalancerBackendPoolName
-      }
-      loadBalancerId: {
+      } : null
+      loadBalancerId: !empty(loadBalancerResourceId) ? {
         value: loadBalancerResourceId
-      }
-      natGatewayName: {
+      } : null
+      natGatewayName: !empty(natGatewayName) ? {
         value: natGatewayName
-      }
+      } : null
       prepareEncryption: {
         value: prepareEncryption
       }
-      publicIpName: {
+      publicIpName: !empty(publicIpName) ? {
         value: publicIpName
-      }
+      } : null
       requireInfrastructureEncryption: {
         value: requireInfrastructureEncryption
       }
-      storageAccountName: {
+      storageAccountName: !empty(storageAccountName) ? {
         value: storageAccountName
-      }
-      storageAccountSkuName: {
+      } : null
+      storageAccountSkuName: !empty(storageAccountSkuName) ? {
         value: storageAccountSkuName
-      }
-      vnetAddressPrefix: {
+      } : null
+      vnetAddressPrefix: !empty(vnetAddressPrefix) ? {
         value: vnetAddressPrefix
-      }
+      } : null
     }
-    authorizations: authorizations
-    managedDiskIdentity: managedDiskIdentity
-    publicNetworkAccess: publicNetworkAccess
-    requiredNsgRules: requiredNsgRules
-    storageAccountIdentity: storageAccountIdentity
-    uiDefinitionUri: uiDefinitionUri
+    // authorizations: authorizations
+    // managedDiskIdentity: managedDiskIdentity
+    // publicNetworkAccess: publicNetworkAccess
+    // requiredNsgRules: requiredNsgRules
+    // storageAccountIdentity: storageAccountIdentity
+    // uiDefinitionUri: uiDefinitionUri
     encryption: !empty(cMKManagedServicesKeyName) || !empty(cMKManagedServicesKeyName) ? {
       entities: {
         managedServices: !empty(cMKManagedServicesKeyName) ? {
@@ -317,7 +309,7 @@ resource workspace_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(
 }
 
 // Note: Diagnostic Settings are only supported by the premium tier
-resource workspace_diagnosticSettings 'Microsoft.Insights/diagnosticsettings@2021-05-01-preview' = if (pricingTier == 'premium' && ((!empty(diagnosticStorageAccountId)) || (!empty(diagnosticWorkspaceId)) || (!empty(diagnosticEventHubAuthorizationRuleId)) || (!empty(diagnosticEventHubName)))) {
+resource workspace_diagnosticSettings 'Microsoft.Insights/diagnosticsettings@2021-05-01-preview' = if (skuName == 'premium' && ((!empty(diagnosticStorageAccountId)) || (!empty(diagnosticWorkspaceId)) || (!empty(diagnosticEventHubAuthorizationRuleId)) || (!empty(diagnosticEventHubName)))) {
   name: !empty(diagnosticSettingsName) ? diagnosticSettingsName : '${name}-diagnosticSettings'
   properties: {
     storageAccountId: !empty(diagnosticStorageAccountId) ? diagnosticStorageAccountId : null

From 64afc9eb413a4e29e023d65caa1d4dc521333b51 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Tue, 8 Aug 2023 20:41:39 +0200
Subject: [PATCH 03/26] Latest working draft

---
 modules/databricks/workspaces/.test/common/dependencies.bicep | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/databricks/workspaces/.test/common/dependencies.bicep b/modules/databricks/workspaces/.test/common/dependencies.bicep
index 6ab76269a6..9c3fd6bcc8 100644
--- a/modules/databricks/workspaces/.test/common/dependencies.bicep
+++ b/modules/databricks/workspaces/.test/common/dependencies.bicep
@@ -58,8 +58,8 @@ resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
 }
 
 resource amlPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
-    name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-Key-Vault-Contributor')
-    scope: keyVault::key
+    name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-Key-Vault-Contributor')
+    scope: keyVault
     properties: {
         principalId: managedIdentity.properties.principalId
         roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') // Contributor

From 4f5d766cd8394d093e75d6f55447aed3c02f9a7d Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Tue, 8 Aug 2023 22:41:29 +0200
Subject: [PATCH 04/26] Latest working draft

---
 .../.test/common/dependencies.bicep           | 220 ++++++++++++++++++
 .../workspaces/.test/common/main.test.bicep   |  30 ++-
 modules/databricks/workspaces/main.bicep      |  32 ++-
 3 files changed, 278 insertions(+), 4 deletions(-)

diff --git a/modules/databricks/workspaces/.test/common/dependencies.bicep b/modules/databricks/workspaces/.test/common/dependencies.bicep
index 9c3fd6bcc8..f80ee55ca5 100644
--- a/modules/databricks/workspaces/.test/common/dependencies.bicep
+++ b/modules/databricks/workspaces/.test/common/dependencies.bicep
@@ -10,12 +10,26 @@ param keyVaultName string
 @description('Required. The name of the Azure Machine Learning Workspace to create.')
 param amlWorkspaceName string
 
+@description('Required. The name of the Load Balancer to create.')
+param loadBalancerName string
+
+@description('Required. The name of the Network Security Group to create.')
+param networkSecurityGroupName string
+
 @description('Required. The name of the Storage Account to create.')
 param storageAccountName string
 
 @description('Required. The name of the Application Insights Instanec to create.')
 param applicationInsightsName string
 
+@description('Required. The name of the Virtual Network to create.')
+param virtualNetworkName string
+
+var addressPrefix = '10.0.0.0/16'
+var defaultAddressPrefix = '10.0.0.0/20'
+var publicAddressPrefix = '10.0.16.0/20'
+var privateAddressPrefix = '10.0.32.0/20'
+
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = {
     name: managedIdentityName
     location: location
@@ -103,12 +117,218 @@ resource machineLearningWorkspace 'Microsoft.MachineLearningServices/workspaces@
     }
 }
 
+resource loadBalancer 'Microsoft.Network/loadBalancers@2023-04-01' = {
+    name: loadBalancerName
+    location: location
+    properties: {
+        backendAddressPools: [
+            {
+                name: 'default'
+            }
+        ]
+        frontendIPConfigurations: [
+            {
+                name: 'privateIPConfig1'
+                properties: {
+                    subnet: {
+                        id: virtualNetwork.properties.subnets[0].id
+                    }
+                }
+            }
+        ]
+    }
+}
+
+resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-04-01' = {
+    name: networkSecurityGroupName
+    location: location
+    properties: {
+        securityRules: [
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-inbound'
+                properties: {
+                    description: 'Required for worker nodes communication within a cluster.'
+                    protocol: '*'
+                    sourcePortRange: '*'
+                    destinationPortRange: '*'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'VirtualNetwork'
+                    access: 'Allow'
+                    priority: 100
+                    direction: 'Inbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-databricks-webapp'
+                properties: {
+                    description: 'Required for workers communication with Databricks Webapp.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '443'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'AzureDatabricks'
+                    access: 'Allow'
+                    priority: 100
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-sql'
+                properties: {
+                    description: 'Required for workers communication with Azure SQL services.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '3306'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'Sql'
+                    access: 'Allow'
+                    priority: 101
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-storage'
+                properties: {
+                    description: 'Required for workers communication with Azure Storage services.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '443'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'Storage'
+                    access: 'Allow'
+                    priority: 102
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-outbound'
+                properties: {
+                    description: 'Required for worker nodes communication within a cluster.'
+                    protocol: '*'
+                    sourcePortRange: '*'
+                    destinationPortRange: '*'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'VirtualNetwork'
+                    access: 'Allow'
+                    priority: 103
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-eventhub'
+                properties: {
+                    description: 'Required for worker communication with Azure Eventhub services.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '9093'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'EventHub'
+                    access: 'Allow'
+                    priority: 104
+                    direction: 'Outbound'
+                }
+            }
+        ]
+    }
+}
+
+resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = {
+    name: virtualNetworkName
+    location: location
+    properties: {
+        addressSpace: {
+            addressPrefixes: [
+                addressPrefix
+            ]
+        }
+        subnets: [
+            {
+                name: 'defaultSubnet'
+                properties: {
+                    addressPrefix: defaultAddressPrefix
+                }
+            }
+            {
+                name: 'custom-public-subnet'
+                properties: {
+                    addressPrefix: publicAddressPrefix
+                    networkSecurityGroup: {
+                        id: networkSecurityGroup.id
+                    }
+                    delegations: [
+                        {
+                            name: 'databricksDelegation'
+                            properties: {
+                                serviceName: 'Microsoft.Databricks/workspaces'
+                            }
+                        }
+                    ]
+                }
+            }
+            {
+                name: 'custom-private-subnet'
+                properties: {
+                    addressPrefix: privateAddressPrefix
+                    networkSecurityGroup: {
+                        id: networkSecurityGroup.id
+                    }
+                    delegations: [
+                        {
+                            name: 'databricksDelegation'
+                            properties: {
+                                serviceName: 'Microsoft.Databricks/workspaces'
+                            }
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+}
+
+resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
+    name: 'privatelink.azuredatabricks.net'
+    location: 'global'
+
+    resource virtualNetworkLinks 'virtualNetworkLinks@2020-06-01' = {
+        name: '${virtualNetwork.name}-vnetlink'
+        location: 'global'
+        properties: {
+            virtualNetwork: {
+                id: virtualNetwork.id
+            }
+            registrationEnabled: false
+        }
+    }
+}
+
+@description('The resource ID of the created Virtual Network Default Subnet.')
+output defaultSubnetResourceId string = virtualNetwork.properties.subnets[0].id
+
+@description('The resource ID of the created Virtual Network Public Subnet.')
+output customPublicSubnetName string = virtualNetwork.properties.subnets[1].name
+
+@description('The resource ID of the created Virtual Network Private Subnet.')
+output customPrivateSubnetName string = virtualNetwork.properties.subnets[2].name
+
+@description('The resource ID of the created Virtual Network.')
+output virtualNetworkResourceId string = virtualNetwork.id
+
+@description('The resource ID of the created Private DNS Zone.')
+output privateDNSResourceId string = privateDNSZone.id
+
 @description('The resource ID of the created Azure Machine Learning Workspace.')
 output machineLearningWorkspaceResourceId string = machineLearningWorkspace.id
 
 @description('The resource ID of the created Key Vault.')
 output keyVaultResourceId string = keyVault.id
 
+@description('The resource ID of the created Load Balancer.')
+output loadBalancerResourceId string = loadBalancer.id
+
+@description('The name of the created Load Balancer Backend Pool.')
+output loadBalancerBackendPoolName string = loadBalancer.properties.backendAddressPools[0].name
+
 @description('The name of the created Key Vault encryption key.')
 output keyVaultKeyName string = keyVault::key.name
 
diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index 837046e20f..826d720d16 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -41,7 +41,10 @@ module nestedDependencies 'dependencies.bicep' = {
     managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}'
     amlWorkspaceName: 'dep-${namePrefix}-aml-${serviceShort}'
     applicationInsightsName: 'dep-${namePrefix}-appi-${serviceShort}'
+    loadBalancerName: 'dep-${namePrefix}-lb-${serviceShort}'
     storageAccountName: 'dep${namePrefix}sa${serviceShort}'
+    virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}'
+    networkSecurityGroupName: 'dep-${namePrefix}-nsg-${serviceShort}'
     // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total)
     // keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
     keyVaultName: 'dep${namePrefix}kv${serviceShort}al'
@@ -91,10 +94,11 @@ module testDeployment '../../main.bicep' = {
       Environment: 'Non-Prod'
       Role: 'DeploymentValidation'
     }
-    cMKManagedDisksKeyName: nestedDependencies.outputs.keyVaultKeyName
-    cMKManagedDisksKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
     cMKManagedServicesKeyName: nestedDependencies.outputs.keyVaultKeyName
     cMKManagedServicesKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
+    cMKManagedDisksKeyName: nestedDependencies.outputs.keyVaultKeyName
+    cMKManagedDisksKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
+    cMKManagedDisksKeyRotationToLatestKeyVersionEnabled: true
     storageAccountName: '${namePrefix}${serviceShort}001'
     publicIpName: 'nat-gw-public-ip'
     natGatewayName: 'nat-gateway'
@@ -102,5 +106,27 @@ module testDeployment '../../main.bicep' = {
     requiredNsgRules: 'AllRules'
     skuName: 'premium'
     amlWorkspaceResourceId: nestedDependencies.outputs.machineLearningWorkspaceResourceId
+    customPrivateSubnetName: nestedDependencies.outputs.customPrivateSubnetName
+    customPublicSubnetName: nestedDependencies.outputs.customPublicSubnetName
+    publicNetworkAccess: 'Disabled'
+    disablePublicIp: true
+    loadBalancerResourceId: nestedDependencies.outputs.loadBalancerResourceId
+    loadBalancerBackendPoolName: nestedDependencies.outputs.loadBalancerBackendPoolName
+    customVirtualNetworkResourceId: nestedDependencies.outputs.virtualNetworkResourceId
+    privateEndpoints: [
+      {
+        privateDnsZoneGroup: {
+          privateDNSResourceIds: [
+            nestedDependencies.outputs.privateDNSResourceId
+          ]
+        }
+        service: 'databricks_ui_api'
+        subnetResourceId: nestedDependencies.outputs.defaultSubnetResourceId
+        tags: {
+          Environment: 'Non-Prod'
+          Role: 'DeploymentValidation'
+        }
+      }
+    ]
   }
 }
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index fb1218c912..e0b4fa8b6d 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -92,7 +92,7 @@ param cMKManagedDisksKeyRotationToLatestKeyVersionEnabled bool = true
 @description('Optional. Name of the outbound Load Balancer Backend Pool for Secure Cluster Connectivity (No Public IP).')
 param loadBalancerBackendPoolName string = ''
 
-@description('Optional. Name of the outbound Load Balancer Backend Pool for Secure Cluster Connectivity (No Public IP).')
+@description('Optional. Resource URI of Outbound Load balancer for Secure Cluster Connectivity (No Public IP) workspace.')
 param loadBalancerResourceId string = ''
 
 @description('Optional. Name of the NAT gateway for Secure Cluster Connectivity (No Public IP) workspace subnets.')
@@ -142,6 +142,9 @@ param storageAccountIdentity object = {}
 @description('Optional. The blob URI where the UI definition file is located.')
 param uiDefinitionUri string = ''
 
+@description('Optional. Configuration details for private endpoints. For security reasons, it is recommended to use private endpoints whenever possible.')
+param privateEndpoints array = []
+
 @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to \'\' to disable log collection.')
 @allowed([
   ''
@@ -184,6 +187,8 @@ var diagnosticsLogs = contains(diagnosticLogCategoriesToEnable, 'allLogs') ? [
   }
 ] : contains(diagnosticLogCategoriesToEnable, '') ? [] : diagnosticsLogsSpecified
 
+var enableReferencedModulesTelemetry = false
+
 resource defaultTelemetry 'Microsoft.Resources/deployments@2021-04-01' = if (enableDefaultTelemetry) {
   name: 'pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-${uniqueString(deployment().name, location)}'
   properties: {
@@ -322,7 +327,7 @@ resource workspace_diagnosticSettings 'Microsoft.Insights/diagnosticsettings@202
 }
 
 module workspace_roleAssignments '.bicep/nested_roleAssignments.bicep' = [for (roleAssignment, index) in roleAssignments: {
-  name: '${uniqueString(deployment().name, location)}-DataBricks-Rbac-${index}'
+  name: '${uniqueString(deployment().name, location)}-Databricks-Rbac-${index}'
   params: {
     description: contains(roleAssignment, 'description') ? roleAssignment.description : ''
     principalIds: roleAssignment.principalIds
@@ -334,6 +339,29 @@ module workspace_roleAssignments '.bicep/nested_roleAssignments.bicep' = [for (r
   }
 }]
 
+module workspace_privateEndpoints '../../network/private-endpoints/main.bicep' = [for (privateEndpoint, index) in privateEndpoints: {
+  name: '${uniqueString(deployment().name, location)}-Databricks-PrivateEndpoint-${index}'
+  params: {
+    groupIds: [
+      privateEndpoint.service
+    ]
+    name: contains(privateEndpoint, 'name') ? privateEndpoint.name : 'pe-${last(split(workspace.id, '/'))}-${privateEndpoint.service}-${index}'
+    serviceResourceId: workspace.id
+    subnetResourceId: privateEndpoint.subnetResourceId
+    enableDefaultTelemetry: enableReferencedModulesTelemetry
+    location: reference(split(privateEndpoint.subnetResourceId, '/subnets/')[0], '2020-06-01', 'Full').location
+    lock: contains(privateEndpoint, 'lock') ? privateEndpoint.lock : lock
+    privateDnsZoneGroup: contains(privateEndpoint, 'privateDnsZoneGroup') ? privateEndpoint.privateDnsZoneGroup : {}
+    roleAssignments: contains(privateEndpoint, 'roleAssignments') ? privateEndpoint.roleAssignments : []
+    tags: contains(privateEndpoint, 'tags') ? privateEndpoint.tags : {}
+    manualPrivateLinkServiceConnections: contains(privateEndpoint, 'manualPrivateLinkServiceConnections') ? privateEndpoint.manualPrivateLinkServiceConnections : []
+    customDnsConfigs: contains(privateEndpoint, 'customDnsConfigs') ? privateEndpoint.customDnsConfigs : []
+    ipConfigurations: contains(privateEndpoint, 'ipConfigurations') ? privateEndpoint.ipConfigurations : []
+    applicationSecurityGroups: contains(privateEndpoint, 'applicationSecurityGroups') ? privateEndpoint.applicationSecurityGroups : []
+    customNetworkInterfaceName: contains(privateEndpoint, 'customNetworkInterfaceName') ? privateEndpoint.customNetworkInterfaceName : ''
+  }
+}]
+
 @description('The name of the deployed databricks workspace.')
 output name string = workspace.name
 

From 7567becde9b0aedb65f50751d2f840135b252056 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 11:12:05 +0200
Subject: [PATCH 05/26] Latest working draft

---
 .../workspaces/.test/common/main.test.bicep   |  7 ++++--
 modules/databricks/workspaces/main.bicep      | 22 ++++++-------------
 2 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index 826d720d16..4ff5ab2967 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -47,7 +47,7 @@ module nestedDependencies 'dependencies.bicep' = {
     networkSecurityGroupName: 'dep-${namePrefix}-nsg-${serviceShort}'
     // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total)
     // keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
-    keyVaultName: 'dep${namePrefix}kv${serviceShort}al'
+    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-al'
   }
 }
 
@@ -103,7 +103,7 @@ module testDeployment '../../main.bicep' = {
     publicIpName: 'nat-gw-public-ip'
     natGatewayName: 'nat-gateway'
     prepareEncryption: true
-    requiredNsgRules: 'AllRules'
+    requiredNsgRules: 'NoAzureDatabricksRules'
     skuName: 'premium'
     amlWorkspaceResourceId: nestedDependencies.outputs.machineLearningWorkspaceResourceId
     customPrivateSubnetName: nestedDependencies.outputs.customPrivateSubnetName
@@ -128,5 +128,8 @@ module testDeployment '../../main.bicep' = {
         }
       }
     ]
+    storageAccountSkuName: 'Standard_ZRS'
+    managedResourceGroupResourceId: '${subscription().id}/resourceGroups/rg-${resourceGroupName}-managed'
+
   }
 }
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index e0b4fa8b6d..6e423acf3f 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -5,8 +5,8 @@ metadata owner = 'Azure/module-maintainers'
 @description('Required. The name of the Azure Databricks workspace to create.')
 param name string
 
-@description('Optional. The managed resource group ID.')
-param managedResourceGroupId string = ''
+@description('Optional. The managed resource group ID. Is created by the module as per the to-be resource ID you provide.')
+param managedResourceGroupResourceId string = ''
 
 @description('Optional. The pricing tier of workspace.')
 @allowed([
@@ -119,9 +119,6 @@ param vnetAddressPrefix string = '10.139'
 @description('Optional. The workspace provider authorizations.')
 param authorizations array = []
 
-@description('Optional. The details of Managed Identity of Disk Encryption Set used for Managed Disk Encryption.')
-param managedDiskIdentity object = {}
-
 @description('Optional. 	The network access type for accessing workspace. Set value to disabled to access workspace only via private link.')
 @allowed([
   'Disabled'
@@ -136,9 +133,6 @@ param publicNetworkAccess string = 'Enabled'
 ])
 param requiredNsgRules string = 'AllRules'
 
-@description('Optional. The details of Managed Identity of Storage Account.')
-param storageAccountIdentity object = {}
-
 @description('Optional. The blob URI where the UI definition file is located.')
 param uiDefinitionUri string = ''
 
@@ -229,7 +223,7 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
     name: skuName
   }
   properties: {
-    managedResourceGroupId: !empty(managedResourceGroupId) ? managedResourceGroupId : '${subscription().id}/resourceGroups/${name}-rg'
+    managedResourceGroupId: !empty(managedResourceGroupResourceId) ? managedResourceGroupResourceId : '${subscription().id}/resourceGroups/${name}-rg'
     parameters: {
       customVirtualNetworkId: !empty(customVirtualNetworkResourceId) ? {
         value: customVirtualNetworkResourceId
@@ -274,12 +268,10 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
         value: vnetAddressPrefix
       } : null
     }
-    // authorizations: authorizations
-    // managedDiskIdentity: managedDiskIdentity
-    // publicNetworkAccess: publicNetworkAccess
-    // requiredNsgRules: requiredNsgRules
-    // storageAccountIdentity: storageAccountIdentity
-    // uiDefinitionUri: uiDefinitionUri
+    //authorizations: authorizations
+    publicNetworkAccess: publicNetworkAccess
+    requiredNsgRules: requiredNsgRules
+    //uiDefinitionUri: uiDefinitionUri
     encryption: !empty(cMKManagedServicesKeyName) || !empty(cMKManagedServicesKeyName) ? {
       entities: {
         managedServices: !empty(cMKManagedServicesKeyName) ? {

From cfd3db77006de5881e4262e8a0642403e729f8d4 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 11:13:51 +0200
Subject: [PATCH 06/26] Latest working draft

---
 modules/databricks/workspaces/.test/common/main.test.bicep | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index 4ff5ab2967..5711282be0 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -100,6 +100,7 @@ module testDeployment '../../main.bicep' = {
     cMKManagedDisksKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
     cMKManagedDisksKeyRotationToLatestKeyVersionEnabled: true
     storageAccountName: '${namePrefix}${serviceShort}001'
+    storageAccountSkuName: 'Standard_ZRS'
     publicIpName: 'nat-gw-public-ip'
     natGatewayName: 'nat-gateway'
     prepareEncryption: true
@@ -128,7 +129,6 @@ module testDeployment '../../main.bicep' = {
         }
       }
     ]
-    storageAccountSkuName: 'Standard_ZRS'
     managedResourceGroupResourceId: '${subscription().id}/resourceGroups/rg-${resourceGroupName}-managed'
 
   }

From 575cc482e0961f89315ca955caf455626a6e67b6 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 19:44:19 +0200
Subject: [PATCH 07/26] Latest working draft

---
 .../workspaces/.test/common/main.test.bicep   | 17 ++++---
 .../workspaces/.test/min/main.test.bicep      | 45 +++++++++++++++++++
 modules/databricks/workspaces/main.bicep      |  8 ----
 3 files changed, 57 insertions(+), 13 deletions(-)
 create mode 100644 modules/databricks/workspaces/.test/min/main.test.bicep

diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index 5711282be0..977f9d2618 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -47,7 +47,7 @@ module nestedDependencies 'dependencies.bicep' = {
     networkSecurityGroupName: 'dep-${namePrefix}-nsg-${serviceShort}'
     // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total)
     // keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
-    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-al'
+    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-ad'
   }
 }
 
@@ -57,7 +57,7 @@ module diagnosticDependencies '../../../../.shared/.templates/diagnostic.depende
   scope: resourceGroup
   name: '${uniqueString(deployment().name, location)}-diagnosticDependencies'
   params: {
-    storageAccountName: 'dep${namePrefix}diasa${serviceShort}01'
+    storageAccountName: 'dep${namePrefix}diasa${serviceShort}'
     logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}'
     eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}'
     eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}'
@@ -80,7 +80,7 @@ module testDeployment '../../main.bicep' = {
     diagnosticEventHubAuthorizationRuleId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId
     diagnosticEventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName
     diagnosticLogsRetentionInDays: 7
-    // lock: 'CanNotDelete'
+    lock: 'CanNotDelete'
     roleAssignments: [
       {
         roleDefinitionIdOrName: 'Reader'
@@ -99,7 +99,7 @@ module testDeployment '../../main.bicep' = {
     cMKManagedDisksKeyName: nestedDependencies.outputs.keyVaultKeyName
     cMKManagedDisksKeyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
     cMKManagedDisksKeyRotationToLatestKeyVersionEnabled: true
-    storageAccountName: '${namePrefix}${serviceShort}001'
+    storageAccountName: 'sa${namePrefix}${serviceShort}001'
     storageAccountSkuName: 'Standard_ZRS'
     publicIpName: 'nat-gw-public-ip'
     natGatewayName: 'nat-gateway'
@@ -130,6 +130,13 @@ module testDeployment '../../main.bicep' = {
       }
     ]
     managedResourceGroupResourceId: '${subscription().id}/resourceGroups/rg-${resourceGroupName}-managed'
-
+    diagnosticLogCategoriesToEnable: [
+      'jobs'
+      'notebook'
+    ]
+    diagnosticSettingsName: 'diag${namePrefix}${serviceShort}001'
+    requireInfrastructureEncryption: true
+    vnetAddressPrefix: '10.100'
+    location: resourceGroup.location
   }
 }
diff --git a/modules/databricks/workspaces/.test/min/main.test.bicep b/modules/databricks/workspaces/.test/min/main.test.bicep
new file mode 100644
index 0000000000..b9bd4a5703
--- /dev/null
+++ b/modules/databricks/workspaces/.test/min/main.test.bicep
@@ -0,0 +1,45 @@
+targetScope = 'subscription'
+
+// ========== //
+// Parameters //
+// ========== //
+
+@description('Optional. The name of the resource group to deploy for testing purposes.')
+@maxLength(90)
+param resourceGroupName string = 'ms.databricks.workspaces-${serviceShort}-rg'
+
+@description('Optional. The location to deploy resources to.')
+param location string = deployment().location
+
+@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.')
+param serviceShort string = 'dwmin'
+
+@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).')
+param enableDefaultTelemetry bool = true
+
+@description('Optional. A token to inject into the name of each resource.')
+param namePrefix string = 'carml'
+
+// ============ //
+// Dependencies //
+// ============ //
+
+// General resources
+// =================
+resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = {
+  name: resourceGroupName
+  location: location
+}
+
+// ============== //
+// Test Execution //
+// ============== //
+
+module testDeployment '../../main.bicep' = {
+  scope: resourceGroup
+  name: '${uniqueString(deployment().name, location)}-test-${serviceShort}'
+  params: {
+    enableDefaultTelemetry: enableDefaultTelemetry
+    name: '${namePrefix}${serviceShort}001'
+  }
+}
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 6e423acf3f..60307c76bc 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -116,9 +116,6 @@ param storageAccountSkuName string = 'Standard_GRS'
 @description('Optional. Address prefix for Managed virtual network.')
 param vnetAddressPrefix string = '10.139'
 
-@description('Optional. The workspace provider authorizations.')
-param authorizations array = []
-
 @description('Optional. 	The network access type for accessing workspace. Set value to disabled to access workspace only via private link.')
 @allowed([
   'Disabled'
@@ -133,9 +130,6 @@ param publicNetworkAccess string = 'Enabled'
 ])
 param requiredNsgRules string = 'AllRules'
 
-@description('Optional. The blob URI where the UI definition file is located.')
-param uiDefinitionUri string = ''
-
 @description('Optional. Configuration details for private endpoints. For security reasons, it is recommended to use private endpoints whenever possible.')
 param privateEndpoints array = []
 
@@ -268,10 +262,8 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
         value: vnetAddressPrefix
       } : null
     }
-    //authorizations: authorizations
     publicNetworkAccess: publicNetworkAccess
     requiredNsgRules: requiredNsgRules
-    //uiDefinitionUri: uiDefinitionUri
     encryption: !empty(cMKManagedServicesKeyName) || !empty(cMKManagedServicesKeyName) ? {
       entities: {
         managedServices: !empty(cMKManagedServicesKeyName) ? {

From f30c7f27fd576d9133a8d73571e15a5cb11e0df3 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 20:56:08 +0200
Subject: [PATCH 08/26] Updated template & readme

---
 modules/databricks/workspaces/README.md  | 331 ++++++++-
 modules/databricks/workspaces/main.bicep |   2 +-
 modules/databricks/workspaces/main.json  | 819 ++++++++++++++++++++++-
 3 files changed, 1117 insertions(+), 35 deletions(-)

diff --git a/modules/databricks/workspaces/README.md b/modules/databricks/workspaces/README.md
index b8f6b1a20d..d744c91748 100644
--- a/modules/databricks/workspaces/README.md
+++ b/modules/databricks/workspaces/README.md
@@ -16,8 +16,10 @@ This module deploys an Azure Databricks Workspace.
 | :-- | :-- |
 | `Microsoft.Authorization/locks` | [2020-05-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2020-05-01/locks) |
 | `Microsoft.Authorization/roleAssignments` | [2022-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2022-04-01/roleAssignments) |
-| `Microsoft.Databricks/workspaces` | [2018-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Databricks/2018-04-01/workspaces) |
+| `Microsoft.Databricks/workspaces` | [2023-02-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Databricks/2023-02-01/workspaces) |
 | `Microsoft.Insights/diagnosticSettings` | [2021-05-01-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings) |
+| `Microsoft.Network/privateEndpoints` | [2022-07-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2022-07-01/privateEndpoints) |
+| `Microsoft.Network/privateEndpoints/privateDnsZoneGroups` | [2022-07-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2022-07-01/privateEndpoints/privateDnsZoneGroups) |
 
 ## Parameters
 
@@ -27,10 +29,26 @@ This module deploys an Azure Databricks Workspace.
 | :-- | :-- | :-- |
 | `name` | string | The name of the Azure Databricks workspace to create. |
 
+**Conditional parameters**
+
+| Parameter Name | Type | Default Value | Description |
+| :-- | :-- | :-- | :-- |
+| `cMKManagedDisksKeyVaultResourceId` | string | `''` | The resource ID of a key vault to reference a customer managed key for encryption from. Required if 'cMKKeyName' is not empty. |
+| `cMKManagedServicesKeyVaultResourceId` | string | `''` | The resource ID of a key vault to reference a customer managed key for encryption from. Required if 'cMKKeyName' is not empty. |
+
 **Optional parameters**
 
 | Parameter Name | Type | Default Value | Allowed Values | Description |
 | :-- | :-- | :-- | :-- | :-- |
+| `amlWorkspaceResourceId` | string | `''` |  | The resource ID of a Azure Machine Learning workspace to link with Databricks workspace. |
+| `cMKManagedDisksKeyName` | string | `''` |  | The name of the customer managed key to use for encryption. |
+| `cMKManagedDisksKeyRotationToLatestKeyVersionEnabled` | bool | `True` |  | Enable Auto Rotation of Key. |
+| `cMKManagedDisksKeyVersion` | string | `''` |  | The version of the customer managed key to reference for encryption. If not provided, the latest key version is used. |
+| `cMKManagedServicesKeyName` | string | `''` |  | The name of the customer managed key to use for encryption. |
+| `cMKManagedServicesKeyVersion` | string | `''` |  | The version of the customer managed key to reference for encryption. If not provided, the latest key version is used. |
+| `customPrivateSubnetName` | string | `''` |  | The name of the Private Subnet within the Virtual Network. |
+| `customPublicSubnetName` | string | `''` |  | The name of a Public Subnet within the Virtual Network |
+| `customVirtualNetworkResourceId` | string | `''` |  | The resource ID of a Virtual Network where this Databricks Cluster should be created. |
 | `diagnosticEventHubAuthorizationRuleId` | string | `''` |  | Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. |
 | `diagnosticEventHubName` | string | `''` |  | Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. |
 | `diagnosticLogCategoriesToEnable` | array | `[allLogs]` | `['', accounts, allLogs, clusters, dbfs, instancePools, jobs, notebook, secrets, sqlPermissions, ssh, workspace]` | The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to '' to disable log collection. |
@@ -38,14 +56,26 @@ This module deploys an Azure Databricks Workspace.
 | `diagnosticSettingsName` | string | `''` |  | The name of the diagnostic setting, if deployed. If left empty, it defaults to "<resourceName>-diagnosticSettings". |
 | `diagnosticStorageAccountId` | string | `''` |  | Resource ID of the diagnostic storage account. |
 | `diagnosticWorkspaceId` | string | `''` |  | Resource ID of the diagnostic log analytics workspace. |
+| `disablePublicIp` | bool | `False` |  | Disable Public IP. |
 | `enableDefaultTelemetry` | bool | `True` |  | Enable telemetry via a Globally Unique Identifier (GUID). |
+| `loadBalancerBackendPoolName` | string | `''` |  | Name of the outbound Load Balancer Backend Pool for Secure Cluster Connectivity (No Public IP). |
+| `loadBalancerResourceId` | string | `''` |  | Resource URI of Outbound Load balancer for Secure Cluster Connectivity (No Public IP) workspace. |
 | `location` | string | `[resourceGroup().location]` |  | Location for all Resources. |
 | `lock` | string | `''` | `['', CanNotDelete, ReadOnly]` | Specify the type of lock. |
-| `managedResourceGroupId` | string | `''` |  | The managed resource group ID. |
-| `parameters` | object | `{object}` |  | The workspace's custom parameters. |
-| `pricingTier` | string | `'premium'` | `[premium, standard, trial]` | The pricing tier of workspace. |
+| `managedResourceGroupResourceId` | string | `''` |  | The managed resource group ID. Is created by the module as per the to-be resource ID you provide. |
+| `natGatewayName` | string | `''` |  | Name of the NAT gateway for Secure Cluster Connectivity (No Public IP) workspace subnets. |
+| `prepareEncryption` | bool | `False` |  | Prepare the workspace for encryption. Enables the Managed Identity for managed storage account. |
+| `privateEndpoints` | array | `[]` |  | Configuration details for private endpoints. For security reasons, it is recommended to use private endpoints whenever possible. |
+| `publicIpName` | string | `''` |  | Name of the Public IP for No Public IP workspace with managed vNet. |
+| `publicNetworkAccess` | string | `'Enabled'` | `[Disabled, Enabled]` | 	The network access type for accessing workspace. Set value to disabled to access workspace only via private link. |
+| `requiredNsgRules` | string | `'AllRules'` | `[AllRules, NoAzureDatabricksRules]` | Gets or sets a value indicating whether data plane (clusters) to control plane communication happen over private endpoint. |
+| `requireInfrastructureEncryption` | bool | `False` |  | A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest. |
 | `roleAssignments` | array | `[]` |  | Array of role assignment objects that contain the 'roleDefinitionIdOrName' and 'principalId' to define RBAC role assignments on this resource. In the roleDefinitionIdOrName attribute, you can provide either the display name of the role definition, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'. |
+| `skuName` | string | `'premium'` | `[premium, standard, trial]` | The pricing tier of workspace. |
+| `storageAccountName` | string | `''` |  | Default DBFS storage account name. |
+| `storageAccountSkuName` | string | `'Standard_GRS'` |  | Storage account SKU name. |
 | `tags` | object | `{object}` |  | Tags of the resource. |
+| `vnetAddressPrefix` | string | `'10.139'` |  | Address prefix for Managed virtual network. |
 
 
 ### Parameter Usage: `roleAssignments`
@@ -213,6 +243,106 @@ tags: {
 </details>
 <p>
 
+### Parameter Usage: `privateEndpoints`
+
+To use Private Endpoint the following dependencies must be deployed:
+
+- Destination subnet must be created with the following configuration option - `"privateEndpointNetworkPolicies": "Disabled"`. Setting this option acknowledges that NSG rules are not applied to Private Endpoints (this capability is coming soon). A full example is available in the Virtual Network Module.
+- Although not strictly required, it is highly recommended to first create a private DNS Zone to host Private Endpoint DNS records. See [Azure Private Endpoint DNS configuration](https://learn.microsoft.com/en-us/azure/private-link/private-endpoint-dns) for more information.
+
+<details>
+
+<summary>Parameter JSON format</summary>
+
+```json
+"privateEndpoints": {
+    "value": [
+        // Example showing all available fields
+        {
+            "name": "sxx-az-pe", // Optional: Name will be automatically generated if one is not provided here
+            "subnetResourceId": "/subscriptions/[[subscriptionId]]/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/sxx-az-vnet-x-001/subnets/sxx-az-subnet-x-001",
+            "service": "<serviceName>", // e.g. vault, registry, blob
+            "privateDnsZoneGroup": {
+                "privateDNSResourceIds": [ // Optional: No DNS record will be created if a private DNS zone Resource ID is not specified
+                    "/subscriptions/[[subscriptionId]]/resourceGroups/validation-rg/providers/Microsoft.Network/privateDnsZones/<privateDnsZoneName>" // e.g. privatelink.vaultcore.azure.net, privatelink.azurecr.io, privatelink.blob.core.windows.net
+                ]
+            },
+            "ipConfigurations":[
+                {
+                    "name": "myIPconfigTest02",
+                    "properties": {
+                        "groupId": "blob",
+                        "memberName": "blob",
+                        "privateIPAddress": "10.0.0.30"
+                    }
+                }
+            ],
+            "customDnsConfigs": [
+                {
+                    "fqdn": "customname.test.local",
+                    "ipAddresses": [
+                        "10.10.10.10"
+                    ]
+                }
+            ]
+        },
+        // Example showing only mandatory fields
+        {
+            "subnetResourceId": "/subscriptions/[[subscriptionId]]/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/sxx-az-vnet-x-001/subnets/sxx-az-subnet-x-001",
+            "service": "<serviceName>" // e.g. vault, registry, blob
+        }
+    ]
+}
+```
+
+</details>
+
+<details>
+
+<summary>Bicep format</summary>
+
+```bicep
+privateEndpoints:  [
+    // Example showing all available fields
+    {
+        name: 'sxx-az-pe' // Optional: Name will be automatically generated if one is not provided here
+        subnetResourceId: '/subscriptions/[[subscriptionId]]/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/sxx-az-vnet-x-001/subnets/sxx-az-subnet-x-001'
+        service: '<serviceName>' // e.g. vault, registry, blob
+        privateDnsZoneGroup: {
+            privateDNSResourceIds: [ // Optional: No DNS record will be created if a private DNS zone Resource ID is not specified
+                '/subscriptions/[[subscriptionId]]/resourceGroups/validation-rg/providers/Microsoft.Network/privateDnsZones/<privateDnsZoneName>' // e.g. privatelink.vaultcore.azure.net, privatelink.azurecr.io, privatelink.blob.core.windows.net
+            ]
+        }
+        customDnsConfigs: [
+            {
+                fqdn: 'customname.test.local'
+                ipAddresses: [
+                    '10.10.10.10'
+                ]
+            }
+        ]
+        ipConfigurations:[
+          {
+            name: 'myIPconfigTest02'
+            properties: {
+              groupId: 'blob'
+              memberName: 'blob'
+              privateIPAddress: '10.0.0.30'
+            }
+          }
+        ]
+    }
+    // Example showing only mandatory fields
+    {
+        subnetResourceId: '/subscriptions/[[subscriptionId]]/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/sxx-az-vnet-x-001/subnets/sxx-az-subnet-x-001'
+        service: '<serviceName>' // e.g. vault, registry, blob
+    }
+]
+```
+
+</details>
+<p>
+
 ## Outputs
 
 | Output Name | Type | Description |
@@ -224,7 +354,11 @@ tags: {
 
 ## Cross-referenced modules
 
-_None_
+This section gives you an overview of all local-referenced module files (i.e., other CARML modules that are referenced in this module) and all remote-referenced files (i.e., Bicep modules that are referenced from a Bicep Registry or Template Specs).
+
+| Reference | Type |
+| :-- | :-- |
+| `network/private-endpoints` | Local reference |
 
 ## Deployment examples
 
@@ -246,13 +380,53 @@ module workspaces './databricks/workspaces/main.bicep' = {
     // Required parameters
     name: 'dwcom001'
     // Non-required parameters
+    amlWorkspaceResourceId: '<amlWorkspaceResourceId>'
+    cMKManagedDisksKeyName: '<cMKManagedDisksKeyName>'
+    cMKManagedDisksKeyRotationToLatestKeyVersionEnabled: true
+    cMKManagedDisksKeyVaultResourceId: '<cMKManagedDisksKeyVaultResourceId>'
+    cMKManagedServicesKeyName: '<cMKManagedServicesKeyName>'
+    cMKManagedServicesKeyVaultResourceId: '<cMKManagedServicesKeyVaultResourceId>'
+    customPrivateSubnetName: '<customPrivateSubnetName>'
+    customPublicSubnetName: '<customPublicSubnetName>'
+    customVirtualNetworkResourceId: '<customVirtualNetworkResourceId>'
     diagnosticEventHubAuthorizationRuleId: '<diagnosticEventHubAuthorizationRuleId>'
     diagnosticEventHubName: '<diagnosticEventHubName>'
+    diagnosticLogCategoriesToEnable: [
+      'jobs'
+      'notebook'
+    ]
     diagnosticLogsRetentionInDays: 7
+    diagnosticSettingsName: 'diagdwcom001'
     diagnosticStorageAccountId: '<diagnosticStorageAccountId>'
     diagnosticWorkspaceId: '<diagnosticWorkspaceId>'
+    disablePublicIp: true
     enableDefaultTelemetry: '<enableDefaultTelemetry>'
+    loadBalancerBackendPoolName: '<loadBalancerBackendPoolName>'
+    loadBalancerResourceId: '<loadBalancerResourceId>'
+    location: '<location>'
     lock: 'CanNotDelete'
+    managedResourceGroupResourceId: '<managedResourceGroupResourceId>'
+    natGatewayName: 'nat-gateway'
+    prepareEncryption: true
+    privateEndpoints: [
+      {
+        privateDnsZoneGroup: {
+          privateDNSResourceIds: [
+            '<privateDNSResourceId>'
+          ]
+        }
+        service: 'databricks_ui_api'
+        subnetResourceId: '<subnetResourceId>'
+        tags: {
+          Environment: 'Non-Prod'
+          Role: 'DeploymentValidation'
+        }
+      }
+    ]
+    publicIpName: 'nat-gw-public-ip'
+    publicNetworkAccess: 'Disabled'
+    requiredNsgRules: 'NoAzureDatabricksRules'
+    requireInfrastructureEncryption: true
     roleAssignments: [
       {
         principalIds: [
@@ -262,10 +436,14 @@ module workspaces './databricks/workspaces/main.bicep' = {
         roleDefinitionIdOrName: 'Reader'
       }
     ]
+    skuName: 'premium'
+    storageAccountName: 'sadwcom001'
+    storageAccountSkuName: 'Standard_ZRS'
     tags: {
       Environment: 'Non-Prod'
       Role: 'DeploymentValidation'
     }
+    vnetAddressPrefix: '10.100'
   }
 }
 ```
@@ -287,27 +465,113 @@ module workspaces './databricks/workspaces/main.bicep' = {
       "value": "dwcom001"
     },
     // Non-required parameters
+    "amlWorkspaceResourceId": {
+      "value": "<amlWorkspaceResourceId>"
+    },
+    "cMKManagedDisksKeyName": {
+      "value": "<cMKManagedDisksKeyName>"
+    },
+    "cMKManagedDisksKeyRotationToLatestKeyVersionEnabled": {
+      "value": true
+    },
+    "cMKManagedDisksKeyVaultResourceId": {
+      "value": "<cMKManagedDisksKeyVaultResourceId>"
+    },
+    "cMKManagedServicesKeyName": {
+      "value": "<cMKManagedServicesKeyName>"
+    },
+    "cMKManagedServicesKeyVaultResourceId": {
+      "value": "<cMKManagedServicesKeyVaultResourceId>"
+    },
+    "customPrivateSubnetName": {
+      "value": "<customPrivateSubnetName>"
+    },
+    "customPublicSubnetName": {
+      "value": "<customPublicSubnetName>"
+    },
+    "customVirtualNetworkResourceId": {
+      "value": "<customVirtualNetworkResourceId>"
+    },
     "diagnosticEventHubAuthorizationRuleId": {
       "value": "<diagnosticEventHubAuthorizationRuleId>"
     },
     "diagnosticEventHubName": {
       "value": "<diagnosticEventHubName>"
     },
+    "diagnosticLogCategoriesToEnable": {
+      "value": [
+        "jobs",
+        "notebook"
+      ]
+    },
     "diagnosticLogsRetentionInDays": {
       "value": 7
     },
+    "diagnosticSettingsName": {
+      "value": "diagdwcom001"
+    },
     "diagnosticStorageAccountId": {
       "value": "<diagnosticStorageAccountId>"
     },
     "diagnosticWorkspaceId": {
       "value": "<diagnosticWorkspaceId>"
     },
+    "disablePublicIp": {
+      "value": true
+    },
     "enableDefaultTelemetry": {
       "value": "<enableDefaultTelemetry>"
     },
+    "loadBalancerBackendPoolName": {
+      "value": "<loadBalancerBackendPoolName>"
+    },
+    "loadBalancerResourceId": {
+      "value": "<loadBalancerResourceId>"
+    },
+    "location": {
+      "value": "<location>"
+    },
     "lock": {
       "value": "CanNotDelete"
     },
+    "managedResourceGroupResourceId": {
+      "value": "<managedResourceGroupResourceId>"
+    },
+    "natGatewayName": {
+      "value": "nat-gateway"
+    },
+    "prepareEncryption": {
+      "value": true
+    },
+    "privateEndpoints": {
+      "value": [
+        {
+          "privateDnsZoneGroup": {
+            "privateDNSResourceIds": [
+              "<privateDNSResourceId>"
+            ]
+          },
+          "service": "databricks_ui_api",
+          "subnetResourceId": "<subnetResourceId>",
+          "tags": {
+            "Environment": "Non-Prod",
+            "Role": "DeploymentValidation"
+          }
+        }
+      ]
+    },
+    "publicIpName": {
+      "value": "nat-gw-public-ip"
+    },
+    "publicNetworkAccess": {
+      "value": "Disabled"
+    },
+    "requiredNsgRules": {
+      "value": "NoAzureDatabricksRules"
+    },
+    "requireInfrastructureEncryption": {
+      "value": true
+    },
     "roleAssignments": {
       "value": [
         {
@@ -319,11 +583,68 @@ module workspaces './databricks/workspaces/main.bicep' = {
         }
       ]
     },
+    "skuName": {
+      "value": "premium"
+    },
+    "storageAccountName": {
+      "value": "sadwcom001"
+    },
+    "storageAccountSkuName": {
+      "value": "Standard_ZRS"
+    },
     "tags": {
       "value": {
         "Environment": "Non-Prod",
         "Role": "DeploymentValidation"
       }
+    },
+    "vnetAddressPrefix": {
+      "value": "10.100"
+    }
+  }
+}
+```
+
+</details>
+<p>
+
+<h3>Example 2: Min</h3>
+
+<details>
+
+<summary>via Bicep module</summary>
+
+```bicep
+module workspaces './databricks/workspaces/main.bicep' = {
+  name: '${uniqueString(deployment().name, location)}-test-dwmin'
+  params: {
+    // Required parameters
+    name: 'dwmin001'
+    // Non-required parameters
+    enableDefaultTelemetry: '<enableDefaultTelemetry>'
+  }
+}
+```
+
+</details>
+<p>
+
+<details>
+
+<summary>via JSON Parameter file</summary>
+
+```json
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    // Required parameters
+    "name": {
+      "value": "dwmin001"
+    },
+    // Non-required parameters
+    "enableDefaultTelemetry": {
+      "value": "<enableDefaultTelemetry>"
     }
   }
 }
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 60307c76bc..74d8880bd2 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -105,7 +105,7 @@ param prepareEncryption bool = false
 param publicIpName string = ''
 
 @description('Optional. A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest.')
-param requireInfrastructureEncryption bool = true
+param requireInfrastructureEncryption bool = false
 
 @description('Optional. Default DBFS storage account name.')
 param storageAccountName string = ''
diff --git a/modules/databricks/workspaces/main.json b/modules/databricks/workspaces/main.json
index 72214add59..debc6e594e 100644
--- a/modules/databricks/workspaces/main.json
+++ b/modules/databricks/workspaces/main.json
@@ -4,9 +4,12 @@
   "metadata": {
     "_generator": {
       "name": "bicep",
-      "version": "0.19.5.34762",
-      "templateHash": "15914648850876491078"
-    }
+      "version": "0.20.4.51522",
+      "templateHash": "5183537567802884455"
+    },
+    "name": "Azure Databricks Workspaces",
+    "description": "This module deploys an Azure Databricks Workspace.",
+    "owner": "Azure/module-maintainers"
   },
   "parameters": {
     "name": {
@@ -15,14 +18,14 @@
         "description": "Required. The name of the Azure Databricks workspace to create."
       }
     },
-    "managedResourceGroupId": {
+    "managedResourceGroupResourceId": {
       "type": "string",
       "defaultValue": "",
       "metadata": {
-        "description": "Optional. The managed resource group ID."
+        "description": "Optional. The managed resource group ID. Is created by the module as per the to-be resource ID you provide."
       }
     },
-    "pricingTier": {
+    "skuName": {
       "type": "string",
       "defaultValue": "premium",
       "allowedValues": [
@@ -48,18 +51,11 @@
         "description": "Optional. Array of role assignment objects that contain the 'roleDefinitionIdOrName' and 'principalId' to define RBAC role assignments on this resource. In the roleDefinitionIdOrName attribute, you can provide either the display name of the role definition, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'."
       }
     },
-    "parameters": {
-      "type": "object",
-      "defaultValue": {},
-      "metadata": {
-        "description": "Optional. The workspace's custom parameters."
-      }
-    },
     "diagnosticLogsRetentionInDays": {
       "type": "int",
       "defaultValue": 365,
-      "maxValue": 365,
       "minValue": 0,
+      "maxValue": 365,
       "metadata": {
         "description": "Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely."
       }
@@ -95,14 +91,14 @@
     "lock": {
       "type": "string",
       "defaultValue": "",
-      "metadata": {
-        "description": "Optional. Specify the type of lock."
-      },
       "allowedValues": [
         "",
         "CanNotDelete",
         "ReadOnly"
-      ]
+      ],
+      "metadata": {
+        "description": "Optional. Specify the type of lock."
+      }
     },
     "tags": {
       "type": "object",
@@ -118,6 +114,182 @@
         "description": "Optional. Enable telemetry via a Globally Unique Identifier (GUID)."
       }
     },
+    "customVirtualNetworkResourceId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The resource ID of a Virtual Network where this Databricks Cluster should be created."
+      }
+    },
+    "amlWorkspaceResourceId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The resource ID of a Azure Machine Learning workspace to link with Databricks workspace."
+      }
+    },
+    "customPrivateSubnetName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The name of the Private Subnet within the Virtual Network."
+      }
+    },
+    "customPublicSubnetName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The name of a Public Subnet within the Virtual Network"
+      }
+    },
+    "disablePublicIp": {
+      "type": "bool",
+      "defaultValue": false,
+      "metadata": {
+        "description": "Optional. Disable Public IP."
+      }
+    },
+    "cMKManagedServicesKeyVaultResourceId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Conditional. The resource ID of a key vault to reference a customer managed key for encryption from. Required if 'cMKKeyName' is not empty."
+      }
+    },
+    "cMKManagedServicesKeyName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The name of the customer managed key to use for encryption."
+      }
+    },
+    "cMKManagedServicesKeyVersion": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The version of the customer managed key to reference for encryption. If not provided, the latest key version is used."
+      }
+    },
+    "cMKManagedDisksKeyVaultResourceId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Conditional. The resource ID of a key vault to reference a customer managed key for encryption from. Required if 'cMKKeyName' is not empty."
+      }
+    },
+    "cMKManagedDisksKeyName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The name of the customer managed key to use for encryption."
+      }
+    },
+    "cMKManagedDisksKeyVersion": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. The version of the customer managed key to reference for encryption. If not provided, the latest key version is used."
+      }
+    },
+    "cMKManagedDisksKeyRotationToLatestKeyVersionEnabled": {
+      "type": "bool",
+      "defaultValue": true,
+      "metadata": {
+        "description": "Optional. Enable Auto Rotation of Key."
+      }
+    },
+    "loadBalancerBackendPoolName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. Name of the outbound Load Balancer Backend Pool for Secure Cluster Connectivity (No Public IP)."
+      }
+    },
+    "loadBalancerResourceId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. Resource URI of Outbound Load balancer for Secure Cluster Connectivity (No Public IP) workspace."
+      }
+    },
+    "natGatewayName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. Name of the NAT gateway for Secure Cluster Connectivity (No Public IP) workspace subnets."
+      }
+    },
+    "prepareEncryption": {
+      "type": "bool",
+      "defaultValue": false,
+      "metadata": {
+        "description": "Optional. Prepare the workspace for encryption. Enables the Managed Identity for managed storage account."
+      }
+    },
+    "publicIpName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. Name of the Public IP for No Public IP workspace with managed vNet."
+      }
+    },
+    "requireInfrastructureEncryption": {
+      "type": "bool",
+      "defaultValue": false,
+      "metadata": {
+        "description": "Optional. A boolean indicating whether or not the DBFS root file system will be enabled with secondary layer of encryption with platform managed keys for data at rest."
+      }
+    },
+    "storageAccountName": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Optional. Default DBFS storage account name."
+      }
+    },
+    "storageAccountSkuName": {
+      "type": "string",
+      "defaultValue": "Standard_GRS",
+      "metadata": {
+        "description": "Optional. Storage account SKU name."
+      }
+    },
+    "vnetAddressPrefix": {
+      "type": "string",
+      "defaultValue": "10.139",
+      "metadata": {
+        "description": "Optional. Address prefix for Managed virtual network."
+      }
+    },
+    "publicNetworkAccess": {
+      "type": "string",
+      "defaultValue": "Enabled",
+      "allowedValues": [
+        "Disabled",
+        "Enabled"
+      ],
+      "metadata": {
+        "description": "Optional. \tThe network access type for accessing workspace. Set value to disabled to access workspace only via private link."
+      }
+    },
+    "requiredNsgRules": {
+      "type": "string",
+      "defaultValue": "AllRules",
+      "allowedValues": [
+        "AllRules",
+        "NoAzureDatabricksRules"
+      ],
+      "metadata": {
+        "description": "Optional. Gets or sets a value indicating whether data plane (clusters) to control plane communication happen over private endpoint."
+      }
+    },
+    "privateEndpoints": {
+      "type": "array",
+      "defaultValue": [],
+      "metadata": {
+        "description": "Optional. Configuration details for private endpoints. For security reasons, it is recommended to use private endpoints whenever possible."
+      }
+    },
     "diagnosticLogCategoriesToEnable": {
       "type": "array",
       "defaultValue": [
@@ -165,8 +337,7 @@
       }
     ],
     "diagnosticsLogs": "[if(contains(parameters('diagnosticLogCategoriesToEnable'), 'allLogs'), createArray(createObject('categoryGroup', 'allLogs', 'enabled', true(), 'retentionPolicy', createObject('enabled', true(), 'days', parameters('diagnosticLogsRetentionInDays')))), if(contains(parameters('diagnosticLogCategoriesToEnable'), ''), createArray(), variables('diagnosticsLogsSpecified')))]",
-    "managedResourceGroupName": "[format('{0}-rg', parameters('name'))]",
-    "managedResourceGroupIdVar": "[format('{0}/resourceGroups/{1}', subscription().id, variables('managedResourceGroupName'))]"
+    "enableReferencedModulesTelemetry": false
   },
   "resources": [
     {
@@ -185,16 +356,40 @@
     },
     {
       "type": "Microsoft.Databricks/workspaces",
-      "apiVersion": "2018-04-01",
+      "apiVersion": "2023-02-01",
       "name": "[parameters('name')]",
       "location": "[parameters('location')]",
       "tags": "[parameters('tags')]",
       "sku": {
-        "name": "[parameters('pricingTier')]"
+        "name": "[parameters('skuName')]"
       },
       "properties": {
-        "managedResourceGroupId": "[if(empty(parameters('managedResourceGroupId')), variables('managedResourceGroupIdVar'), parameters('managedResourceGroupId'))]",
-        "parameters": "[parameters('parameters')]"
+        "managedResourceGroupId": "[if(not(empty(parameters('managedResourceGroupResourceId'))), parameters('managedResourceGroupResourceId'), format('{0}/resourceGroups/{1}-rg', subscription().id, parameters('name')))]",
+        "parameters": {
+          "customVirtualNetworkId": "[if(not(empty(parameters('customVirtualNetworkResourceId'))), createObject('value', parameters('customVirtualNetworkResourceId')), null())]",
+          "amlWorkspaceId": "[if(not(empty(parameters('amlWorkspaceResourceId'))), createObject('value', parameters('amlWorkspaceResourceId')), null())]",
+          "customPrivateSubnetName": "[if(not(empty(parameters('customPrivateSubnetName'))), createObject('value', parameters('customPrivateSubnetName')), null())]",
+          "customPublicSubnetName": "[if(not(empty(parameters('customPublicSubnetName'))), createObject('value', parameters('customPublicSubnetName')), null())]",
+          "enableNoPublicIp": {
+            "value": "[parameters('disablePublicIp')]"
+          },
+          "loadBalancerBackendPoolName": "[if(not(empty(parameters('loadBalancerBackendPoolName'))), createObject('value', parameters('loadBalancerBackendPoolName')), null())]",
+          "loadBalancerId": "[if(not(empty(parameters('loadBalancerResourceId'))), createObject('value', parameters('loadBalancerResourceId')), null())]",
+          "natGatewayName": "[if(not(empty(parameters('natGatewayName'))), createObject('value', parameters('natGatewayName')), null())]",
+          "prepareEncryption": {
+            "value": "[parameters('prepareEncryption')]"
+          },
+          "publicIpName": "[if(not(empty(parameters('publicIpName'))), createObject('value', parameters('publicIpName')), null())]",
+          "requireInfrastructureEncryption": {
+            "value": "[parameters('requireInfrastructureEncryption')]"
+          },
+          "storageAccountName": "[if(not(empty(parameters('storageAccountName'))), createObject('value', parameters('storageAccountName')), null())]",
+          "storageAccountSkuName": "[if(not(empty(parameters('storageAccountSkuName'))), createObject('value', parameters('storageAccountSkuName')), null())]",
+          "vnetAddressPrefix": "[if(not(empty(parameters('vnetAddressPrefix'))), createObject('value', parameters('vnetAddressPrefix')), null())]"
+        },
+        "publicNetworkAccess": "[parameters('publicNetworkAccess')]",
+        "requiredNsgRules": "[parameters('requiredNsgRules')]",
+        "encryption": "[if(or(not(empty(parameters('cMKManagedServicesKeyName'))), not(empty(parameters('cMKManagedServicesKeyName')))), createObject('entities', createObject('managedServices', if(not(empty(parameters('cMKManagedServicesKeyName'))), createObject('keySource', 'Microsoft.Keyvault', 'keyVaultProperties', createObject('keyVaultUri', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(parameters('cMKManagedServicesKeyVaultResourceId'), '/'))), '2023-02-01').vaultUri, 'keyName', parameters('cMKManagedServicesKeyName'), 'keyVersion', if(not(empty(parameters('cMKManagedServicesKeyVersion'))), parameters('cMKManagedServicesKeyVersion'), last(split(reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults/keys', split(format('{0}/{1}', last(split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')), parameters('cMKManagedServicesKeyName')), '/')[0], split(format('{0}/{1}', last(split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')), parameters('cMKManagedServicesKeyName')), '/')[1]), '2023-02-01').keyUriWithVersion, '/'))))), null()), 'managedDisk', if(not(empty(parameters('cMKManagedDisksKeyName'))), createObject('keySource', 'Microsoft.Keyvault', 'keyVaultProperties', createObject('keyVaultUri', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(parameters('cMKManagedDisksKeyVaultResourceId'), '/'))), '2023-02-01').vaultUri, 'keyName', parameters('cMKManagedDisksKeyName'), 'keyVersion', if(not(empty(parameters('cMKManagedDisksKeyVersion'))), parameters('cMKManagedDisksKeyVersion'), last(split(reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults/keys', split(format('{0}/{1}', last(split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')), parameters('cMKManagedDisksKeyName')), '/')[0], split(format('{0}/{1}', last(split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')), parameters('cMKManagedDisksKeyName')), '/')[1]), '2023-02-01').keyUriWithVersion, '/')))), 'rotationToLatestKeyVersionEnabled', parameters('cMKManagedDisksKeyRotationToLatestKeyVersionEnabled')), null()))), null())]"
       }
     },
     {
@@ -212,7 +407,7 @@
       ]
     },
     {
-      "condition": "[and(equals(parameters('pricingTier'), 'premium'), or(or(or(not(empty(parameters('diagnosticStorageAccountId'))), not(empty(parameters('diagnosticWorkspaceId')))), not(empty(parameters('diagnosticEventHubAuthorizationRuleId')))), not(empty(parameters('diagnosticEventHubName')))))]",
+      "condition": "[and(equals(parameters('skuName'), 'premium'), or(or(or(not(empty(parameters('diagnosticStorageAccountId'))), not(empty(parameters('diagnosticWorkspaceId')))), not(empty(parameters('diagnosticEventHubAuthorizationRuleId')))), not(empty(parameters('diagnosticEventHubName')))))]",
       "type": "Microsoft.Insights/diagnosticSettings",
       "apiVersion": "2021-05-01-preview",
       "scope": "[format('Microsoft.Databricks/workspaces/{0}', parameters('name'))]",
@@ -235,7 +430,7 @@
       },
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2022-09-01",
-      "name": "[format('{0}-DataBricks-Rbac-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]",
+      "name": "[format('{0}-Databricks-Rbac-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]",
       "properties": {
         "expressionEvaluationOptions": {
           "scope": "inner"
@@ -262,8 +457,8 @@
           "metadata": {
             "_generator": {
               "name": "bicep",
-              "version": "0.19.5.34762",
-              "templateHash": "13613915507747675582"
+              "version": "0.20.4.51522",
+              "templateHash": "11920634368378558628"
             }
           },
           "parameters": {
@@ -375,6 +570,572 @@
       "dependsOn": [
         "[resourceId('Microsoft.Databricks/workspaces', parameters('name'))]"
       ]
+    },
+    {
+      "copy": {
+        "name": "workspace_privateEndpoints",
+        "count": "[length(parameters('privateEndpoints'))]"
+      },
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2022-09-01",
+      "name": "[format('{0}-Databricks-PrivateEndpoint-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]",
+      "properties": {
+        "expressionEvaluationOptions": {
+          "scope": "inner"
+        },
+        "mode": "Incremental",
+        "parameters": {
+          "groupIds": {
+            "value": [
+              "[parameters('privateEndpoints')[copyIndex()].service]"
+            ]
+          },
+          "name": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'name'), createObject('value', parameters('privateEndpoints')[copyIndex()].name), createObject('value', format('pe-{0}-{1}-{2}', last(split(resourceId('Microsoft.Databricks/workspaces', parameters('name')), '/')), parameters('privateEndpoints')[copyIndex()].service, copyIndex())))]",
+          "serviceResourceId": {
+            "value": "[resourceId('Microsoft.Databricks/workspaces', parameters('name'))]"
+          },
+          "subnetResourceId": {
+            "value": "[parameters('privateEndpoints')[copyIndex()].subnetResourceId]"
+          },
+          "enableDefaultTelemetry": {
+            "value": "[variables('enableReferencedModulesTelemetry')]"
+          },
+          "location": {
+            "value": "[reference(split(parameters('privateEndpoints')[copyIndex()].subnetResourceId, '/subnets/')[0], '2020-06-01', 'Full').location]"
+          },
+          "lock": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'lock'), createObject('value', parameters('privateEndpoints')[copyIndex()].lock), createObject('value', parameters('lock')))]",
+          "privateDnsZoneGroup": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'privateDnsZoneGroup'), createObject('value', parameters('privateEndpoints')[copyIndex()].privateDnsZoneGroup), createObject('value', createObject()))]",
+          "roleAssignments": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'roleAssignments'), createObject('value', parameters('privateEndpoints')[copyIndex()].roleAssignments), createObject('value', createArray()))]",
+          "tags": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'tags'), createObject('value', parameters('privateEndpoints')[copyIndex()].tags), createObject('value', createObject()))]",
+          "manualPrivateLinkServiceConnections": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'manualPrivateLinkServiceConnections'), createObject('value', parameters('privateEndpoints')[copyIndex()].manualPrivateLinkServiceConnections), createObject('value', createArray()))]",
+          "customDnsConfigs": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'customDnsConfigs'), createObject('value', parameters('privateEndpoints')[copyIndex()].customDnsConfigs), createObject('value', createArray()))]",
+          "ipConfigurations": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'ipConfigurations'), createObject('value', parameters('privateEndpoints')[copyIndex()].ipConfigurations), createObject('value', createArray()))]",
+          "applicationSecurityGroups": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'applicationSecurityGroups'), createObject('value', parameters('privateEndpoints')[copyIndex()].applicationSecurityGroups), createObject('value', createArray()))]",
+          "customNetworkInterfaceName": "[if(contains(parameters('privateEndpoints')[copyIndex()], 'customNetworkInterfaceName'), createObject('value', parameters('privateEndpoints')[copyIndex()].customNetworkInterfaceName), createObject('value', ''))]"
+        },
+        "template": {
+          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+          "contentVersion": "1.0.0.0",
+          "metadata": {
+            "_generator": {
+              "name": "bicep",
+              "version": "0.20.4.51522",
+              "templateHash": "609426544615556519"
+            },
+            "name": "Private Endpoints",
+            "description": "This module deploys a Private Endpoint.",
+            "owner": "Azure/module-maintainers"
+          },
+          "parameters": {
+            "name": {
+              "type": "string",
+              "metadata": {
+                "description": "Required. Name of the private endpoint resource to create."
+              }
+            },
+            "subnetResourceId": {
+              "type": "string",
+              "metadata": {
+                "description": "Required. Resource ID of the subnet where the endpoint needs to be created."
+              }
+            },
+            "serviceResourceId": {
+              "type": "string",
+              "metadata": {
+                "description": "Required. Resource ID of the resource that needs to be connected to the network."
+              }
+            },
+            "applicationSecurityGroups": {
+              "type": "array",
+              "defaultValue": [],
+              "metadata": {
+                "description": "Optional. Application security groups in which the private endpoint IP configuration is included."
+              }
+            },
+            "customNetworkInterfaceName": {
+              "type": "string",
+              "defaultValue": "",
+              "metadata": {
+                "description": "Optional. The custom name of the network interface attached to the private endpoint."
+              }
+            },
+            "ipConfigurations": {
+              "type": "array",
+              "defaultValue": [],
+              "metadata": {
+                "description": "Optional. A list of IP configurations of the private endpoint. This will be used to map to the First Party Service endpoints."
+              }
+            },
+            "groupIds": {
+              "type": "array",
+              "metadata": {
+                "description": "Required. Subtype(s) of the connection to be created. The allowed values depend on the type serviceResourceId refers to."
+              }
+            },
+            "privateDnsZoneGroup": {
+              "type": "object",
+              "defaultValue": {},
+              "metadata": {
+                "description": "Optional. The private DNS zone group configuration used to associate the private endpoint with one or multiple private DNS zones. A DNS zone group can support up to 5 DNS zones."
+              }
+            },
+            "location": {
+              "type": "string",
+              "defaultValue": "[resourceGroup().location]",
+              "metadata": {
+                "description": "Optional. Location for all Resources."
+              }
+            },
+            "lock": {
+              "type": "string",
+              "defaultValue": "",
+              "allowedValues": [
+                "",
+                "CanNotDelete",
+                "ReadOnly"
+              ],
+              "metadata": {
+                "description": "Optional. Specify the type of lock."
+              }
+            },
+            "roleAssignments": {
+              "type": "array",
+              "defaultValue": [],
+              "metadata": {
+                "description": "Optional. Array of role assignment objects that contain the 'roleDefinitionIdOrName' and 'principalId' to define RBAC role assignments on this resource. In the roleDefinitionIdOrName attribute, you can provide either the display name of the role definition, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'."
+              }
+            },
+            "tags": {
+              "type": "object",
+              "defaultValue": {},
+              "metadata": {
+                "description": "Optional. Tags to be applied on all resources/resource groups in this deployment."
+              }
+            },
+            "customDnsConfigs": {
+              "type": "array",
+              "defaultValue": [],
+              "metadata": {
+                "description": "Optional. Custom DNS configurations."
+              }
+            },
+            "manualPrivateLinkServiceConnections": {
+              "type": "array",
+              "defaultValue": [],
+              "metadata": {
+                "description": "Optional. Manual PrivateLink Service Connections."
+              }
+            },
+            "enableDefaultTelemetry": {
+              "type": "bool",
+              "defaultValue": true,
+              "metadata": {
+                "description": "Optional. Enable telemetry via a Globally Unique Identifier (GUID)."
+              }
+            }
+          },
+          "variables": {
+            "enableReferencedModulesTelemetry": false
+          },
+          "resources": [
+            {
+              "condition": "[parameters('enableDefaultTelemetry')]",
+              "type": "Microsoft.Resources/deployments",
+              "apiVersion": "2021-04-01",
+              "name": "[format('pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-{0}', uniqueString(deployment().name, parameters('location')))]",
+              "properties": {
+                "mode": "Incremental",
+                "template": {
+                  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+                  "contentVersion": "1.0.0.0",
+                  "resources": []
+                }
+              }
+            },
+            {
+              "type": "Microsoft.Network/privateEndpoints",
+              "apiVersion": "2022-07-01",
+              "name": "[parameters('name')]",
+              "location": "[parameters('location')]",
+              "tags": "[parameters('tags')]",
+              "properties": {
+                "applicationSecurityGroups": "[parameters('applicationSecurityGroups')]",
+                "customDnsConfigs": "[parameters('customDnsConfigs')]",
+                "customNetworkInterfaceName": "[parameters('customNetworkInterfaceName')]",
+                "ipConfigurations": "[parameters('ipConfigurations')]",
+                "manualPrivateLinkServiceConnections": "[parameters('manualPrivateLinkServiceConnections')]",
+                "privateLinkServiceConnections": [
+                  {
+                    "name": "[parameters('name')]",
+                    "properties": {
+                      "privateLinkServiceId": "[parameters('serviceResourceId')]",
+                      "groupIds": "[parameters('groupIds')]"
+                    }
+                  }
+                ],
+                "subnet": {
+                  "id": "[parameters('subnetResourceId')]"
+                }
+              }
+            },
+            {
+              "condition": "[not(empty(parameters('lock')))]",
+              "type": "Microsoft.Authorization/locks",
+              "apiVersion": "2020-05-01",
+              "scope": "[format('Microsoft.Network/privateEndpoints/{0}', parameters('name'))]",
+              "name": "[format('{0}-{1}-lock', parameters('name'), parameters('lock'))]",
+              "properties": {
+                "level": "[parameters('lock')]",
+                "notes": "[if(equals(parameters('lock'), 'CanNotDelete'), 'Cannot delete resource or child resources.', 'Cannot modify the resource or child resources.')]"
+              },
+              "dependsOn": [
+                "[resourceId('Microsoft.Network/privateEndpoints', parameters('name'))]"
+              ]
+            },
+            {
+              "condition": "[not(empty(parameters('privateDnsZoneGroup')))]",
+              "type": "Microsoft.Resources/deployments",
+              "apiVersion": "2022-09-01",
+              "name": "[format('{0}-PrivateEndpoint-PrivateDnsZoneGroup', uniqueString(deployment().name, parameters('location')))]",
+              "properties": {
+                "expressionEvaluationOptions": {
+                  "scope": "inner"
+                },
+                "mode": "Incremental",
+                "parameters": {
+                  "privateDNSResourceIds": {
+                    "value": "[parameters('privateDnsZoneGroup').privateDNSResourceIds]"
+                  },
+                  "privateEndpointName": {
+                    "value": "[parameters('name')]"
+                  },
+                  "enableDefaultTelemetry": {
+                    "value": "[variables('enableReferencedModulesTelemetry')]"
+                  }
+                },
+                "template": {
+                  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+                  "contentVersion": "1.0.0.0",
+                  "metadata": {
+                    "_generator": {
+                      "name": "bicep",
+                      "version": "0.20.4.51522",
+                      "templateHash": "18114841529204248478"
+                    },
+                    "name": "Private Endpoint Private DNS Zone Groups",
+                    "description": "This module deploys a Private Endpoint Private DNS Zone Group.",
+                    "owner": "Azure/module-maintainers"
+                  },
+                  "parameters": {
+                    "privateEndpointName": {
+                      "type": "string",
+                      "metadata": {
+                        "description": "Conditional. The name of the parent private endpoint. Required if the template is used in a standalone deployment."
+                      }
+                    },
+                    "privateDNSResourceIds": {
+                      "type": "array",
+                      "minLength": 1,
+                      "maxLength": 5,
+                      "metadata": {
+                        "description": "Required. Array of private DNS zone resource IDs. A DNS zone group can support up to 5 DNS zones."
+                      }
+                    },
+                    "name": {
+                      "type": "string",
+                      "defaultValue": "default",
+                      "metadata": {
+                        "description": "Optional. The name of the private DNS zone group."
+                      }
+                    },
+                    "enableDefaultTelemetry": {
+                      "type": "bool",
+                      "defaultValue": true,
+                      "metadata": {
+                        "description": "Optional. Enable telemetry via a Globally Unique Identifier (GUID)."
+                      }
+                    }
+                  },
+                  "variables": {
+                    "copy": [
+                      {
+                        "name": "privateDnsZoneConfigs",
+                        "count": "[length(parameters('privateDNSResourceIds'))]",
+                        "input": {
+                          "name": "[last(split(parameters('privateDNSResourceIds')[copyIndex('privateDnsZoneConfigs')], '/'))]",
+                          "properties": {
+                            "privateDnsZoneId": "[parameters('privateDNSResourceIds')[copyIndex('privateDnsZoneConfigs')]]"
+                          }
+                        }
+                      }
+                    ]
+                  },
+                  "resources": [
+                    {
+                      "condition": "[parameters('enableDefaultTelemetry')]",
+                      "type": "Microsoft.Resources/deployments",
+                      "apiVersion": "2021-04-01",
+                      "name": "[format('pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-{0}', uniqueString(deployment().name))]",
+                      "properties": {
+                        "mode": "Incremental",
+                        "template": {
+                          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+                          "contentVersion": "1.0.0.0",
+                          "resources": []
+                        }
+                      }
+                    },
+                    {
+                      "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
+                      "apiVersion": "2022-07-01",
+                      "name": "[format('{0}/{1}', parameters('privateEndpointName'), parameters('name'))]",
+                      "properties": {
+                        "privateDnsZoneConfigs": "[variables('privateDnsZoneConfigs')]"
+                      }
+                    }
+                  ],
+                  "outputs": {
+                    "name": {
+                      "type": "string",
+                      "metadata": {
+                        "description": "The name of the private endpoint DNS zone group."
+                      },
+                      "value": "[parameters('name')]"
+                    },
+                    "resourceId": {
+                      "type": "string",
+                      "metadata": {
+                        "description": "The resource ID of the private endpoint DNS zone group."
+                      },
+                      "value": "[resourceId('Microsoft.Network/privateEndpoints/privateDnsZoneGroups', parameters('privateEndpointName'), parameters('name'))]"
+                    },
+                    "resourceGroupName": {
+                      "type": "string",
+                      "metadata": {
+                        "description": "The resource group the private endpoint DNS zone group was deployed into."
+                      },
+                      "value": "[resourceGroup().name]"
+                    }
+                  }
+                }
+              },
+              "dependsOn": [
+                "[resourceId('Microsoft.Network/privateEndpoints', parameters('name'))]"
+              ]
+            },
+            {
+              "copy": {
+                "name": "privateEndpoint_roleAssignments",
+                "count": "[length(parameters('roleAssignments'))]"
+              },
+              "type": "Microsoft.Resources/deployments",
+              "apiVersion": "2022-09-01",
+              "name": "[format('{0}-PrivateEndpoint-Rbac-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]",
+              "properties": {
+                "expressionEvaluationOptions": {
+                  "scope": "inner"
+                },
+                "mode": "Incremental",
+                "parameters": {
+                  "description": "[if(contains(parameters('roleAssignments')[copyIndex()], 'description'), createObject('value', parameters('roleAssignments')[copyIndex()].description), createObject('value', ''))]",
+                  "principalIds": {
+                    "value": "[parameters('roleAssignments')[copyIndex()].principalIds]"
+                  },
+                  "principalType": "[if(contains(parameters('roleAssignments')[copyIndex()], 'principalType'), createObject('value', parameters('roleAssignments')[copyIndex()].principalType), createObject('value', ''))]",
+                  "roleDefinitionIdOrName": {
+                    "value": "[parameters('roleAssignments')[copyIndex()].roleDefinitionIdOrName]"
+                  },
+                  "condition": "[if(contains(parameters('roleAssignments')[copyIndex()], 'condition'), createObject('value', parameters('roleAssignments')[copyIndex()].condition), createObject('value', ''))]",
+                  "delegatedManagedIdentityResourceId": "[if(contains(parameters('roleAssignments')[copyIndex()], 'delegatedManagedIdentityResourceId'), createObject('value', parameters('roleAssignments')[copyIndex()].delegatedManagedIdentityResourceId), createObject('value', ''))]",
+                  "resourceId": {
+                    "value": "[resourceId('Microsoft.Network/privateEndpoints', parameters('name'))]"
+                  }
+                },
+                "template": {
+                  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+                  "contentVersion": "1.0.0.0",
+                  "metadata": {
+                    "_generator": {
+                      "name": "bicep",
+                      "version": "0.20.4.51522",
+                      "templateHash": "11548486149222715894"
+                    }
+                  },
+                  "parameters": {
+                    "principalIds": {
+                      "type": "array",
+                      "metadata": {
+                        "description": "Required. The IDs of the principals to assign the role to."
+                      }
+                    },
+                    "roleDefinitionIdOrName": {
+                      "type": "string",
+                      "metadata": {
+                        "description": "Required. The name of the role to assign. If it cannot be found you can specify the role definition ID instead."
+                      }
+                    },
+                    "resourceId": {
+                      "type": "string",
+                      "metadata": {
+                        "description": "Required. The resource ID of the resource to apply the role assignment to."
+                      }
+                    },
+                    "principalType": {
+                      "type": "string",
+                      "defaultValue": "",
+                      "allowedValues": [
+                        "ServicePrincipal",
+                        "Group",
+                        "User",
+                        "ForeignGroup",
+                        "Device",
+                        ""
+                      ],
+                      "metadata": {
+                        "description": "Optional. The principal type of the assigned principal ID."
+                      }
+                    },
+                    "description": {
+                      "type": "string",
+                      "defaultValue": "",
+                      "metadata": {
+                        "description": "Optional. The description of the role assignment."
+                      }
+                    },
+                    "condition": {
+                      "type": "string",
+                      "defaultValue": "",
+                      "metadata": {
+                        "description": "Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase \"foo_storage_container\"."
+                      }
+                    },
+                    "conditionVersion": {
+                      "type": "string",
+                      "defaultValue": "2.0",
+                      "allowedValues": [
+                        "2.0"
+                      ],
+                      "metadata": {
+                        "description": "Optional. Version of the condition."
+                      }
+                    },
+                    "delegatedManagedIdentityResourceId": {
+                      "type": "string",
+                      "defaultValue": "",
+                      "metadata": {
+                        "description": "Optional. Id of the delegated managed identity resource."
+                      }
+                    }
+                  },
+                  "variables": {
+                    "builtInRoleNames": {
+                      "Avere Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f8fab4f-1852-4a58-a46a-8eaf358af14a')]",
+                      "Avere Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c025889f-8102-4ebf-b32c-fc0c6f0c6bd9')]",
+                      "Azure Center for SAP solutions administrator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7b0c7e81-271f-4c71-90bf-e30bdfdbc2f7')]",
+                      "Azure Center for SAP solutions reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '05352d14-a920-4328-a0de-4cbe7430e26b')]",
+                      "Azure Center for SAP solutions service role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'aabbc5dd-1af0-458b-a942-81af88f9c138')]",
+                      "Azure Kubernetes Service Policy Add-on Deployment": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18ed5180-3e48-46fd-8541-4ea054d57064')]",
+                      "Backup Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5e467623-bb1f-42f4-a55d-6e525e11384b')]",
+                      "Backup Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '00c29273-979b-4161-815c-10b084fb9324')]",
+                      "Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
+                      "Cosmos DB Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '230815da-be43-4aae-9cb4-875f7bd000aa')]",
+                      "Desktop Virtualization Virtual Machine Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a959dbd1-f747-45e3-8ba6-dd80f235f97c')]",
+                      "DevTest Labs User": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '76283e04-6283-4c54-8f91-bcf1374a3c64')]",
+                      "DNS Resolver Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0f2ebee7-ffd4-4fc0-b3b7-664099fdad5d')]",
+                      "DNS Zone Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314')]",
+                      "DocumentDB Account Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5bd9cd88-fe45-4216-938b-f97437e15450')]",
+                      "Domain Services Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'eeaeda52-9324-47f6-8069-5d5bade478b2')]",
+                      "Domain Services Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '361898ef-9ed1-48c2-849c-a832951106bb')]",
+                      "LocalNGFirewallAdministrator role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a8835c7d-b5cb-47fa-b6f0-65ea10ce07a2')]",
+                      "Log Analytics Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '92aaf0da-9dab-42b6-94a3-d43ce8d16293')]",
+                      "Log Analytics Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '73c42c96-874c-492b-b04d-ab87d138a893')]",
+                      "Managed Application Contributor Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '641177b8-a67a-45b9-a033-47bc880bb21e')]",
+                      "Managed Application Operator Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c7393b34-138c-406f-901b-d8cf2b17e6ae')]",
+                      "Managed Applications Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b9331d33-8a36-4f8c-b097-4f54124fdb44')]",
+                      "Monitoring Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '749f88d5-cbae-40b8-bcfc-e573ddc772fa')]",
+                      "Monitoring Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '43d0d8ad-25c7-4714-9337-8ba259a9fe05')]",
+                      "Network Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7')]",
+                      "Owner": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]",
+                      "Private DNS Zone Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b12aa53e-6015-4669-85d0-8515ebb3ae7f')]",
+                      "Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]",
+                      "Resource Policy Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '36243c78-bf99-498c-9df9-86d9f8d28608')]",
+                      "Role Based Access Control Administrator (Preview)": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f58310d9-a9f6-439a-9e8d-f62e7b41a168')]",
+                      "Site Recovery Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '6670b86e-a3f7-4917-ac9b-5d6ab1be4567')]",
+                      "Site Recovery Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '494ae006-db33-4328-bf46-533a6560a3ca')]",
+                      "SQL Managed Instance Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4939a1f6-9ae0-4e48-a1e0-f2cbe897382d')]",
+                      "SQL Security Manager": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '056cd41c-7e88-42e1-933e-88ba6a50c9c3')]",
+                      "Storage Account Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '17d1049b-9a84-46fb-8f53-869881c3d3ab')]",
+                      "Traffic Manager Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a4b10055-b0c7-44c2-b00f-c7b5b3550cf7')]",
+                      "User Access Administrator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9')]",
+                      "Virtual Machine Administrator Login": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '1c0163c0-47e6-4577-8991-ea5c82e286e4')]",
+                      "Virtual Machine Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '9980e02c-c2be-4d73-94e8-173b1dc7cf3c')]",
+                      "Virtual Machine User Login": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'fb879df8-f326-4884-b1cf-06f3ad86be52')]",
+                      "Windows Admin Center Administrator Login": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a6333a3e-0164-44c3-b281-7a577aff287f')]"
+                    }
+                  },
+                  "resources": [
+                    {
+                      "copy": {
+                        "name": "roleAssignment",
+                        "count": "[length(parameters('principalIds'))]"
+                      },
+                      "type": "Microsoft.Authorization/roleAssignments",
+                      "apiVersion": "2022-04-01",
+                      "scope": "[format('Microsoft.Network/privateEndpoints/{0}', last(split(parameters('resourceId'), '/')))]",
+                      "name": "[guid(resourceId('Microsoft.Network/privateEndpoints', last(split(parameters('resourceId'), '/'))), parameters('principalIds')[copyIndex()], parameters('roleDefinitionIdOrName'))]",
+                      "properties": {
+                        "description": "[parameters('description')]",
+                        "roleDefinitionId": "[if(contains(variables('builtInRoleNames'), parameters('roleDefinitionIdOrName')), variables('builtInRoleNames')[parameters('roleDefinitionIdOrName')], parameters('roleDefinitionIdOrName'))]",
+                        "principalId": "[parameters('principalIds')[copyIndex()]]",
+                        "principalType": "[if(not(empty(parameters('principalType'))), parameters('principalType'), null())]",
+                        "condition": "[if(not(empty(parameters('condition'))), parameters('condition'), null())]",
+                        "conditionVersion": "[if(and(not(empty(parameters('conditionVersion'))), not(empty(parameters('condition')))), parameters('conditionVersion'), null())]",
+                        "delegatedManagedIdentityResourceId": "[if(not(empty(parameters('delegatedManagedIdentityResourceId'))), parameters('delegatedManagedIdentityResourceId'), null())]"
+                      }
+                    }
+                  ]
+                }
+              },
+              "dependsOn": [
+                "[resourceId('Microsoft.Network/privateEndpoints', parameters('name'))]"
+              ]
+            }
+          ],
+          "outputs": {
+            "resourceGroupName": {
+              "type": "string",
+              "metadata": {
+                "description": "The resource group the private endpoint was deployed into."
+              },
+              "value": "[resourceGroup().name]"
+            },
+            "resourceId": {
+              "type": "string",
+              "metadata": {
+                "description": "The resource ID of the private endpoint."
+              },
+              "value": "[resourceId('Microsoft.Network/privateEndpoints', parameters('name'))]"
+            },
+            "name": {
+              "type": "string",
+              "metadata": {
+                "description": "The name of the private endpoint."
+              },
+              "value": "[parameters('name')]"
+            },
+            "location": {
+              "type": "string",
+              "metadata": {
+                "description": "The location the resource was deployed into."
+              },
+              "value": "[reference(resourceId('Microsoft.Network/privateEndpoints', parameters('name')), '2022-07-01', 'full').location]"
+            }
+          }
+        }
+      },
+      "dependsOn": [
+        "[resourceId('Microsoft.Databricks/workspaces', parameters('name'))]"
+      ]
     }
   ],
   "outputs": {
@@ -404,7 +1165,7 @@
       "metadata": {
         "description": "The location the resource was deployed into."
       },
-      "value": "[reference(resourceId('Microsoft.Databricks/workspaces', parameters('name')), '2018-04-01', 'full').location]"
+      "value": "[reference(resourceId('Microsoft.Databricks/workspaces', parameters('name')), '2023-02-01', 'full').location]"
     }
   }
 }
\ No newline at end of file

From 5ba02f0b1c7b6c4118debf4e5b5a794d5c39d63c Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 21:05:38 +0200
Subject: [PATCH 09/26] Updated test

---
 modules/databricks/workspaces/.test/common/main.test.bicep | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/modules/databricks/workspaces/.test/common/main.test.bicep b/modules/databricks/workspaces/.test/common/main.test.bicep
index 977f9d2618..3ff9408e3f 100644
--- a/modules/databricks/workspaces/.test/common/main.test.bicep
+++ b/modules/databricks/workspaces/.test/common/main.test.bicep
@@ -46,8 +46,7 @@ module nestedDependencies 'dependencies.bicep' = {
     virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}'
     networkSecurityGroupName: 'dep-${namePrefix}-nsg-${serviceShort}'
     // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total)
-    // keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
-    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-ad'
+    keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}'
   }
 }
 

From 9099f1dd4b0707c27df2769a455dc6c352b85fc8 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 21:13:38 +0200
Subject: [PATCH 10/26] Attempted alternative property usage

---
 .../workspaces/.test/min/main.test.bicep      |  2 +-
 modules/databricks/workspaces/main.bicep      | 44 +++++++++---------
 modules/databricks/workspaces/main.json       | 46 ++++++++++++++-----
 3 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/modules/databricks/workspaces/.test/min/main.test.bicep b/modules/databricks/workspaces/.test/min/main.test.bicep
index b9bd4a5703..85cd2ef8dc 100644
--- a/modules/databricks/workspaces/.test/min/main.test.bicep
+++ b/modules/databricks/workspaces/.test/min/main.test.bicep
@@ -18,7 +18,7 @@ param serviceShort string = 'dwmin'
 param enableDefaultTelemetry bool = true
 
 @description('Optional. A token to inject into the name of each resource.')
-param namePrefix string = 'carml'
+param namePrefix string = '[[namePrefix]]'
 
 // ============ //
 // Dependencies //
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 74d8880bd2..94fded6da3 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -219,48 +219,48 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
   properties: {
     managedResourceGroupId: !empty(managedResourceGroupResourceId) ? managedResourceGroupResourceId : '${subscription().id}/resourceGroups/${name}-rg'
     parameters: {
-      customVirtualNetworkId: !empty(customVirtualNetworkResourceId) ? {
+      customVirtualNetworkId: {
         value: customVirtualNetworkResourceId
-      } : null
-      amlWorkspaceId: !empty(amlWorkspaceResourceId) ? {
+      }
+      amlWorkspaceId: {
         value: amlWorkspaceResourceId
-      } : null
-      customPrivateSubnetName: !empty(customPrivateSubnetName) ? {
+      }
+      customPrivateSubnetName: {
         value: customPrivateSubnetName
-      } : null
-      customPublicSubnetName: !empty(customPublicSubnetName) ? {
+      }
+      customPublicSubnetName: {
         value: customPublicSubnetName
-      } : null
+      }
       enableNoPublicIp: {
         value: disablePublicIp
       }
-      loadBalancerBackendPoolName: !empty(loadBalancerBackendPoolName) ? {
+      loadBalancerBackendPoolName: {
         value: loadBalancerBackendPoolName
-      } : null
-      loadBalancerId: !empty(loadBalancerResourceId) ? {
+      }
+      loadBalancerId: {
         value: loadBalancerResourceId
-      } : null
-      natGatewayName: !empty(natGatewayName) ? {
+      }
+      natGatewayName: {
         value: natGatewayName
-      } : null
+      }
       prepareEncryption: {
         value: prepareEncryption
       }
-      publicIpName: !empty(publicIpName) ? {
+      publicIpName: {
         value: publicIpName
-      } : null
+      }
       requireInfrastructureEncryption: {
         value: requireInfrastructureEncryption
       }
-      storageAccountName: !empty(storageAccountName) ? {
+      storageAccountName: {
         value: storageAccountName
-      } : null
-      storageAccountSkuName: !empty(storageAccountSkuName) ? {
+      }
+      storageAccountSkuName: {
         value: storageAccountSkuName
-      } : null
-      vnetAddressPrefix: !empty(vnetAddressPrefix) ? {
+      }
+      vnetAddressPrefix: {
         value: vnetAddressPrefix
-      } : null
+      }
     }
     publicNetworkAccess: publicNetworkAccess
     requiredNsgRules: requiredNsgRules
diff --git a/modules/databricks/workspaces/main.json b/modules/databricks/workspaces/main.json
index debc6e594e..fa11d4aa15 100644
--- a/modules/databricks/workspaces/main.json
+++ b/modules/databricks/workspaces/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.20.4.51522",
-      "templateHash": "5183537567802884455"
+      "templateHash": "9480568301304847644"
     },
     "name": "Azure Databricks Workspaces",
     "description": "This module deploys an Azure Databricks Workspace.",
@@ -366,26 +366,48 @@
       "properties": {
         "managedResourceGroupId": "[if(not(empty(parameters('managedResourceGroupResourceId'))), parameters('managedResourceGroupResourceId'), format('{0}/resourceGroups/{1}-rg', subscription().id, parameters('name')))]",
         "parameters": {
-          "customVirtualNetworkId": "[if(not(empty(parameters('customVirtualNetworkResourceId'))), createObject('value', parameters('customVirtualNetworkResourceId')), null())]",
-          "amlWorkspaceId": "[if(not(empty(parameters('amlWorkspaceResourceId'))), createObject('value', parameters('amlWorkspaceResourceId')), null())]",
-          "customPrivateSubnetName": "[if(not(empty(parameters('customPrivateSubnetName'))), createObject('value', parameters('customPrivateSubnetName')), null())]",
-          "customPublicSubnetName": "[if(not(empty(parameters('customPublicSubnetName'))), createObject('value', parameters('customPublicSubnetName')), null())]",
+          "customVirtualNetworkId": {
+            "value": "[parameters('customVirtualNetworkResourceId')]"
+          },
+          "amlWorkspaceId": {
+            "value": "[parameters('amlWorkspaceResourceId')]"
+          },
+          "customPrivateSubnetName": {
+            "value": "[parameters('customPrivateSubnetName')]"
+          },
+          "customPublicSubnetName": {
+            "value": "[parameters('customPublicSubnetName')]"
+          },
           "enableNoPublicIp": {
             "value": "[parameters('disablePublicIp')]"
           },
-          "loadBalancerBackendPoolName": "[if(not(empty(parameters('loadBalancerBackendPoolName'))), createObject('value', parameters('loadBalancerBackendPoolName')), null())]",
-          "loadBalancerId": "[if(not(empty(parameters('loadBalancerResourceId'))), createObject('value', parameters('loadBalancerResourceId')), null())]",
-          "natGatewayName": "[if(not(empty(parameters('natGatewayName'))), createObject('value', parameters('natGatewayName')), null())]",
+          "loadBalancerBackendPoolName": {
+            "value": "[parameters('loadBalancerBackendPoolName')]"
+          },
+          "loadBalancerId": {
+            "value": "[parameters('loadBalancerResourceId')]"
+          },
+          "natGatewayName": {
+            "value": "[parameters('natGatewayName')]"
+          },
           "prepareEncryption": {
             "value": "[parameters('prepareEncryption')]"
           },
-          "publicIpName": "[if(not(empty(parameters('publicIpName'))), createObject('value', parameters('publicIpName')), null())]",
+          "publicIpName": {
+            "value": "[parameters('publicIpName')]"
+          },
           "requireInfrastructureEncryption": {
             "value": "[parameters('requireInfrastructureEncryption')]"
           },
-          "storageAccountName": "[if(not(empty(parameters('storageAccountName'))), createObject('value', parameters('storageAccountName')), null())]",
-          "storageAccountSkuName": "[if(not(empty(parameters('storageAccountSkuName'))), createObject('value', parameters('storageAccountSkuName')), null())]",
-          "vnetAddressPrefix": "[if(not(empty(parameters('vnetAddressPrefix'))), createObject('value', parameters('vnetAddressPrefix')), null())]"
+          "storageAccountName": {
+            "value": "[parameters('storageAccountName')]"
+          },
+          "storageAccountSkuName": {
+            "value": "[parameters('storageAccountSkuName')]"
+          },
+          "vnetAddressPrefix": {
+            "value": "[parameters('vnetAddressPrefix')]"
+          }
         },
         "publicNetworkAccess": "[parameters('publicNetworkAccess')]",
         "requiredNsgRules": "[parameters('requiredNsgRules')]",

From 30ae393614b75af744ccec33fb6bc7d5b052b78b Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 21:20:28 +0200
Subject: [PATCH 11/26] Addressed static issues

---
 .azuredevops/modulePipelines/ms.databricks.workspaces.yml | 1 +
 .github/workflows/ms.databricks.workspaces.yml            | 1 +
 modules/databricks/workspaces/.test/main.test.bicep       | 8 ++++++++
 modules/databricks/workspaces/README.md                   | 2 +-
 modules/databricks/workspaces/main.bicep                  | 2 +-
 5 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/.azuredevops/modulePipelines/ms.databricks.workspaces.yml b/.azuredevops/modulePipelines/ms.databricks.workspaces.yml
index 4b71657b7c..24e39d6fe2 100644
--- a/.azuredevops/modulePipelines/ms.databricks.workspaces.yml
+++ b/.azuredevops/modulePipelines/ms.databricks.workspaces.yml
@@ -30,6 +30,7 @@ trigger:
       - '/.azuredevops/modulePipelines/ms.databricks.workspaces.yml'
       - '/.azuredevops/pipelineTemplates/*.yml'
       - '/modules/databricks/workspaces/*'
+      - '/modules/network/private-endpoints/*'
       - '/utilities/pipelines/*'
     exclude:
       - '/utilities/pipelines/deploymentRemoval/*'
diff --git a/.github/workflows/ms.databricks.workspaces.yml b/.github/workflows/ms.databricks.workspaces.yml
index c85a7b0cfa..67cdabdaf4 100644
--- a/.github/workflows/ms.databricks.workspaces.yml
+++ b/.github/workflows/ms.databricks.workspaces.yml
@@ -31,6 +31,7 @@ on:
       - '.github/workflows/template.module.yml'
       - '.github/workflows/ms.databricks.workspaces.yml'
       - 'modules/databricks/workspaces/**'
+      - 'modules/network/private-endpoints/**'
       - 'utilities/pipelines/**'
       - '!utilities/pipelines/deploymentRemoval/**'
       - '!*/**/README.md'
diff --git a/modules/databricks/workspaces/.test/main.test.bicep b/modules/databricks/workspaces/.test/main.test.bicep
index dff85d99b8..ef99b3dc03 100644
--- a/modules/databricks/workspaces/.test/main.test.bicep
+++ b/modules/databricks/workspaces/.test/main.test.bicep
@@ -13,3 +13,11 @@ module common 'common/main.test.bicep' = {
     namePrefix: namePrefix
   }
 }
+
+// TEST 2 - MIN
+module min 'min/main.test.bicep' = {
+  name: '${uniqueString(deployment().name)}-min-test'
+  params: {
+    namePrefix: namePrefix
+  }
+}
diff --git a/modules/databricks/workspaces/README.md b/modules/databricks/workspaces/README.md
index d744c91748..ae0de2ae87 100644
--- a/modules/databricks/workspaces/README.md
+++ b/modules/databricks/workspaces/README.md
@@ -47,7 +47,7 @@ This module deploys an Azure Databricks Workspace.
 | `cMKManagedServicesKeyName` | string | `''` |  | The name of the customer managed key to use for encryption. |
 | `cMKManagedServicesKeyVersion` | string | `''` |  | The version of the customer managed key to reference for encryption. If not provided, the latest key version is used. |
 | `customPrivateSubnetName` | string | `''` |  | The name of the Private Subnet within the Virtual Network. |
-| `customPublicSubnetName` | string | `''` |  | The name of a Public Subnet within the Virtual Network |
+| `customPublicSubnetName` | string | `''` |  | The name of a Public Subnet within the Virtual Network. |
 | `customVirtualNetworkResourceId` | string | `''` |  | The resource ID of a Virtual Network where this Databricks Cluster should be created. |
 | `diagnosticEventHubAuthorizationRuleId` | string | `''` |  | Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. |
 | `diagnosticEventHubName` | string | `''` |  | Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. |
diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 94fded6da3..c895965d1c 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -62,7 +62,7 @@ param amlWorkspaceResourceId string = ''
 @description('Optional. The name of the Private Subnet within the Virtual Network.')
 param customPrivateSubnetName string = ''
 
-@description('Optional. The name of a Public Subnet within the Virtual Network')
+@description('Optional. The name of a Public Subnet within the Virtual Network.')
 param customPublicSubnetName string = ''
 
 @description('Optional. Disable Public IP.')

From d13197e7df88c7fac25c0fb8b25b33a192818c4e Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 21:23:39 +0200
Subject: [PATCH 12/26] Update to latest

---
 modules/databricks/workspaces/main.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/databricks/workspaces/main.json b/modules/databricks/workspaces/main.json
index fa11d4aa15..50244e7a24 100644
--- a/modules/databricks/workspaces/main.json
+++ b/modules/databricks/workspaces/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.20.4.51522",
-      "templateHash": "9480568301304847644"
+      "templateHash": "6885797445606439561"
     },
     "name": "Azure Databricks Workspaces",
     "description": "This module deploys an Azure Databricks Workspace.",
@@ -139,7 +139,7 @@
       "type": "string",
       "defaultValue": "",
       "metadata": {
-        "description": "Optional. The name of a Public Subnet within the Virtual Network"
+        "description": "Optional. The name of a Public Subnet within the Virtual Network."
       }
     },
     "disablePublicIp": {

From abea78614980075a3ab08d8a36c62b369b44e941 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 9 Aug 2023 21:55:56 +0200
Subject: [PATCH 13/26] Latest working draft

---
 modules/databricks/workspaces/main.bicep | 20 ++++++++++----------
 modules/databricks/workspaces/main.json  | 22 ++++++----------------
 2 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index c895965d1c..1cc5d39e0e 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -219,12 +219,12 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
   properties: {
     managedResourceGroupId: !empty(managedResourceGroupResourceId) ? managedResourceGroupResourceId : '${subscription().id}/resourceGroups/${name}-rg'
     parameters: {
-      customVirtualNetworkId: {
+      customVirtualNetworkId: !empty(customVirtualNetworkResourceId) ? {
         value: customVirtualNetworkResourceId
-      }
-      amlWorkspaceId: {
+      } : null
+      amlWorkspaceId: !empty(amlWorkspaceResourceId) ? {
         value: amlWorkspaceResourceId
-      }
+      } : null
       customPrivateSubnetName: {
         value: customPrivateSubnetName
       }
@@ -240,21 +240,21 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
       loadBalancerId: {
         value: loadBalancerResourceId
       }
-      natGatewayName: {
+      natGatewayName: !empty(natGatewayName) ? {
         value: natGatewayName
-      }
+      } : null
       prepareEncryption: {
         value: prepareEncryption
       }
-      publicIpName: {
+      publicIpName: !empty(publicIpName) ? {
         value: publicIpName
-      }
+      } : null
       requireInfrastructureEncryption: {
         value: requireInfrastructureEncryption
       }
-      storageAccountName: {
+      storageAccountName: !empty(storageAccountName) ? {
         value: storageAccountName
-      }
+      } : null
       storageAccountSkuName: {
         value: storageAccountSkuName
       }
diff --git a/modules/databricks/workspaces/main.json b/modules/databricks/workspaces/main.json
index 50244e7a24..d8b3eb7842 100644
--- a/modules/databricks/workspaces/main.json
+++ b/modules/databricks/workspaces/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.20.4.51522",
-      "templateHash": "6885797445606439561"
+      "templateHash": "3704611994759977421"
     },
     "name": "Azure Databricks Workspaces",
     "description": "This module deploys an Azure Databricks Workspace.",
@@ -366,12 +366,8 @@
       "properties": {
         "managedResourceGroupId": "[if(not(empty(parameters('managedResourceGroupResourceId'))), parameters('managedResourceGroupResourceId'), format('{0}/resourceGroups/{1}-rg', subscription().id, parameters('name')))]",
         "parameters": {
-          "customVirtualNetworkId": {
-            "value": "[parameters('customVirtualNetworkResourceId')]"
-          },
-          "amlWorkspaceId": {
-            "value": "[parameters('amlWorkspaceResourceId')]"
-          },
+          "customVirtualNetworkId": "[if(not(empty(parameters('customVirtualNetworkResourceId'))), createObject('value', parameters('customVirtualNetworkResourceId')), null())]",
+          "amlWorkspaceId": "[if(not(empty(parameters('amlWorkspaceResourceId'))), createObject('value', parameters('amlWorkspaceResourceId')), null())]",
           "customPrivateSubnetName": {
             "value": "[parameters('customPrivateSubnetName')]"
           },
@@ -387,21 +383,15 @@
           "loadBalancerId": {
             "value": "[parameters('loadBalancerResourceId')]"
           },
-          "natGatewayName": {
-            "value": "[parameters('natGatewayName')]"
-          },
+          "natGatewayName": "[if(not(empty(parameters('natGatewayName'))), createObject('value', parameters('natGatewayName')), null())]",
           "prepareEncryption": {
             "value": "[parameters('prepareEncryption')]"
           },
-          "publicIpName": {
-            "value": "[parameters('publicIpName')]"
-          },
+          "publicIpName": "[if(not(empty(parameters('publicIpName'))), createObject('value', parameters('publicIpName')), null())]",
           "requireInfrastructureEncryption": {
             "value": "[parameters('requireInfrastructureEncryption')]"
           },
-          "storageAccountName": {
-            "value": "[parameters('storageAccountName')]"
-          },
+          "storageAccountName": "[if(not(empty(parameters('storageAccountName'))), createObject('value', parameters('storageAccountName')), null())]",
           "storageAccountSkuName": {
             "value": "[parameters('storageAccountSkuName')]"
           },

From 54ec5a439a44b512de5b85c7258d64bd7409509c Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Fri, 11 Aug 2023 08:41:07 +0200
Subject: [PATCH 14/26] Updated preview removal flag

---
 .../pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 b/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1
index 3885febb63..b6b0202f2c 100644
--- a/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1
+++ b/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1
@@ -154,7 +154,7 @@ function Invoke-ResourceRemoval {
             $subscriptionId = $resourceId.Split('/')[2]
             $resourceGroupName = $resourceId.Split('/')[4]
             $resourceName = Split-Path $resourceId -Leaf
-            $purgePath = 'subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.MachineLearningServices/workspaces/{2}?api-version=2023-04-01-preview&forceToPurge={3}' -f $subscriptionId, $resourceGroupName, $resourceName, $true
+            $purgePath = 'subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.MachineLearningServices/workspaces/{2}?api-version=2023-04-01&forceToPurge={3}' -f $subscriptionId, $resourceGroupName, $resourceName, $true
             $purgeRequestInputObject = @{
                 Method = 'DELETE'
                 Path   = $purgePath

From c72e9cfb5574acd813cda8b6b4bca350c200e722 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Fri, 11 Aug 2023 09:46:46 +0200
Subject: [PATCH 15/26] Updated parameter handling

---
 modules/databricks/workspaces/main.bicep | 111 ++++++++++++++---------
 modules/databricks/workspaces/main.json  |  37 +-------
 2 files changed, 69 insertions(+), 79 deletions(-)

diff --git a/modules/databricks/workspaces/main.bicep b/modules/databricks/workspaces/main.bicep
index 1cc5d39e0e..1917de7c51 100644
--- a/modules/databricks/workspaces/main.bicep
+++ b/modules/databricks/workspaces/main.bicep
@@ -218,50 +218,73 @@ resource workspace 'Microsoft.Databricks/workspaces@2023-02-01' = {
   }
   properties: {
     managedResourceGroupId: !empty(managedResourceGroupResourceId) ? managedResourceGroupResourceId : '${subscription().id}/resourceGroups/${name}-rg'
-    parameters: {
-      customVirtualNetworkId: !empty(customVirtualNetworkResourceId) ? {
-        value: customVirtualNetworkResourceId
-      } : null
-      amlWorkspaceId: !empty(amlWorkspaceResourceId) ? {
-        value: amlWorkspaceResourceId
-      } : null
-      customPrivateSubnetName: {
-        value: customPrivateSubnetName
-      }
-      customPublicSubnetName: {
-        value: customPublicSubnetName
-      }
-      enableNoPublicIp: {
-        value: disablePublicIp
-      }
-      loadBalancerBackendPoolName: {
-        value: loadBalancerBackendPoolName
-      }
-      loadBalancerId: {
-        value: loadBalancerResourceId
-      }
-      natGatewayName: !empty(natGatewayName) ? {
-        value: natGatewayName
-      } : null
-      prepareEncryption: {
-        value: prepareEncryption
-      }
-      publicIpName: !empty(publicIpName) ? {
-        value: publicIpName
-      } : null
-      requireInfrastructureEncryption: {
-        value: requireInfrastructureEncryption
-      }
-      storageAccountName: !empty(storageAccountName) ? {
-        value: storageAccountName
-      } : null
-      storageAccountSkuName: {
-        value: storageAccountSkuName
-      }
-      vnetAddressPrefix: {
-        value: vnetAddressPrefix
-      }
-    }
+    parameters: union(
+      // Always added parameters
+      {
+        enableNoPublicIp: {
+          value: disablePublicIp
+        }
+        prepareEncryption: {
+          value: prepareEncryption
+        }
+        vnetAddressPrefix: {
+          value: vnetAddressPrefix
+        }
+        requireInfrastructureEncryption: {
+          value: requireInfrastructureEncryption
+        }
+      },
+      // Parameters only added if not empty
+      !empty(customVirtualNetworkResourceId) ? {
+        customVirtualNetworkId: {
+          value: customVirtualNetworkResourceId
+        }
+      } : {},
+      !empty(amlWorkspaceResourceId) ? {
+        amlWorkspaceId: {
+          value: amlWorkspaceResourceId
+        }
+      } : {},
+      !empty(customPrivateSubnetName) ? {
+        customPrivateSubnetName: {
+          value: customPrivateSubnetName
+        }
+      } : {},
+      !empty(customPublicSubnetName) ? {
+        customPublicSubnetName: {
+          value: customPublicSubnetName
+        }
+      } : {},
+      !empty(loadBalancerBackendPoolName) ? {
+        loadBalancerBackendPoolName: {
+          value: loadBalancerBackendPoolName
+        }
+      } : {},
+      !empty(loadBalancerResourceId) ? {
+        loadBalancerId: {
+          value: loadBalancerResourceId
+        }
+      } : {},
+      !empty(natGatewayName) ? {
+        natGatewayName: {
+          value: natGatewayName
+        }
+      } : {},
+      !empty(publicIpName) ? {
+        publicIpName: {
+          value: publicIpName
+        }
+      } : {},
+      !empty(storageAccountName) ? {
+        storageAccountName: {
+          value: storageAccountName
+        }
+      } : {},
+      !empty(storageAccountSkuName) ? {
+        storageAccountSkuName: {
+          value: storageAccountSkuName
+        }
+      } : {})
     publicNetworkAccess: publicNetworkAccess
     requiredNsgRules: requiredNsgRules
     encryption: !empty(cMKManagedServicesKeyName) || !empty(cMKManagedServicesKeyName) ? {
diff --git a/modules/databricks/workspaces/main.json b/modules/databricks/workspaces/main.json
index d8b3eb7842..87cdcdb965 100644
--- a/modules/databricks/workspaces/main.json
+++ b/modules/databricks/workspaces/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.20.4.51522",
-      "templateHash": "3704611994759977421"
+      "templateHash": "6917573805504793323"
     },
     "name": "Azure Databricks Workspaces",
     "description": "This module deploys an Azure Databricks Workspace.",
@@ -365,40 +365,7 @@
       },
       "properties": {
         "managedResourceGroupId": "[if(not(empty(parameters('managedResourceGroupResourceId'))), parameters('managedResourceGroupResourceId'), format('{0}/resourceGroups/{1}-rg', subscription().id, parameters('name')))]",
-        "parameters": {
-          "customVirtualNetworkId": "[if(not(empty(parameters('customVirtualNetworkResourceId'))), createObject('value', parameters('customVirtualNetworkResourceId')), null())]",
-          "amlWorkspaceId": "[if(not(empty(parameters('amlWorkspaceResourceId'))), createObject('value', parameters('amlWorkspaceResourceId')), null())]",
-          "customPrivateSubnetName": {
-            "value": "[parameters('customPrivateSubnetName')]"
-          },
-          "customPublicSubnetName": {
-            "value": "[parameters('customPublicSubnetName')]"
-          },
-          "enableNoPublicIp": {
-            "value": "[parameters('disablePublicIp')]"
-          },
-          "loadBalancerBackendPoolName": {
-            "value": "[parameters('loadBalancerBackendPoolName')]"
-          },
-          "loadBalancerId": {
-            "value": "[parameters('loadBalancerResourceId')]"
-          },
-          "natGatewayName": "[if(not(empty(parameters('natGatewayName'))), createObject('value', parameters('natGatewayName')), null())]",
-          "prepareEncryption": {
-            "value": "[parameters('prepareEncryption')]"
-          },
-          "publicIpName": "[if(not(empty(parameters('publicIpName'))), createObject('value', parameters('publicIpName')), null())]",
-          "requireInfrastructureEncryption": {
-            "value": "[parameters('requireInfrastructureEncryption')]"
-          },
-          "storageAccountName": "[if(not(empty(parameters('storageAccountName'))), createObject('value', parameters('storageAccountName')), null())]",
-          "storageAccountSkuName": {
-            "value": "[parameters('storageAccountSkuName')]"
-          },
-          "vnetAddressPrefix": {
-            "value": "[parameters('vnetAddressPrefix')]"
-          }
-        },
+        "parameters": "[union(createObject('enableNoPublicIp', createObject('value', parameters('disablePublicIp')), 'prepareEncryption', createObject('value', parameters('prepareEncryption')), 'vnetAddressPrefix', createObject('value', parameters('vnetAddressPrefix')), 'requireInfrastructureEncryption', createObject('value', parameters('requireInfrastructureEncryption'))), if(not(empty(parameters('customVirtualNetworkResourceId'))), createObject('customVirtualNetworkId', createObject('value', parameters('customVirtualNetworkResourceId'))), createObject()), if(not(empty(parameters('amlWorkspaceResourceId'))), createObject('amlWorkspaceId', createObject('value', parameters('amlWorkspaceResourceId'))), createObject()), if(not(empty(parameters('customPrivateSubnetName'))), createObject('customPrivateSubnetName', createObject('value', parameters('customPrivateSubnetName'))), createObject()), if(not(empty(parameters('customPublicSubnetName'))), createObject('customPublicSubnetName', createObject('value', parameters('customPublicSubnetName'))), createObject()), if(not(empty(parameters('loadBalancerBackendPoolName'))), createObject('loadBalancerBackendPoolName', createObject('value', parameters('loadBalancerBackendPoolName'))), createObject()), if(not(empty(parameters('loadBalancerResourceId'))), createObject('loadBalancerId', createObject('value', parameters('loadBalancerResourceId'))), createObject()), if(not(empty(parameters('natGatewayName'))), createObject('natGatewayName', createObject('value', parameters('natGatewayName'))), createObject()), if(not(empty(parameters('publicIpName'))), createObject('publicIpName', createObject('value', parameters('publicIpName'))), createObject()), if(not(empty(parameters('storageAccountName'))), createObject('storageAccountName', createObject('value', parameters('storageAccountName'))), createObject()), if(not(empty(parameters('storageAccountSkuName'))), createObject('storageAccountSkuName', createObject('value', parameters('storageAccountSkuName'))), createObject()))]",
         "publicNetworkAccess": "[parameters('publicNetworkAccess')]",
         "requiredNsgRules": "[parameters('requiredNsgRules')]",
         "encryption": "[if(or(not(empty(parameters('cMKManagedServicesKeyName'))), not(empty(parameters('cMKManagedServicesKeyName')))), createObject('entities', createObject('managedServices', if(not(empty(parameters('cMKManagedServicesKeyName'))), createObject('keySource', 'Microsoft.Keyvault', 'keyVaultProperties', createObject('keyVaultUri', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(parameters('cMKManagedServicesKeyVaultResourceId'), '/'))), '2023-02-01').vaultUri, 'keyName', parameters('cMKManagedServicesKeyName'), 'keyVersion', if(not(empty(parameters('cMKManagedServicesKeyVersion'))), parameters('cMKManagedServicesKeyVersion'), last(split(reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults/keys', split(format('{0}/{1}', last(split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')), parameters('cMKManagedServicesKeyName')), '/')[0], split(format('{0}/{1}', last(split(parameters('cMKManagedServicesKeyVaultResourceId'), '/')), parameters('cMKManagedServicesKeyName')), '/')[1]), '2023-02-01').keyUriWithVersion, '/'))))), null()), 'managedDisk', if(not(empty(parameters('cMKManagedDisksKeyName'))), createObject('keySource', 'Microsoft.Keyvault', 'keyVaultProperties', createObject('keyVaultUri', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(parameters('cMKManagedDisksKeyVaultResourceId'), '/'))), '2023-02-01').vaultUri, 'keyName', parameters('cMKManagedDisksKeyName'), 'keyVersion', if(not(empty(parameters('cMKManagedDisksKeyVersion'))), parameters('cMKManagedDisksKeyVersion'), last(split(reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[2], split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')[4]), 'Microsoft.KeyVault/vaults/keys', split(format('{0}/{1}', last(split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')), parameters('cMKManagedDisksKeyName')), '/')[0], split(format('{0}/{1}', last(split(parameters('cMKManagedDisksKeyVaultResourceId'), '/')), parameters('cMKManagedDisksKeyName')), '/')[1]), '2023-02-01').keyUriWithVersion, '/')))), 'rotationToLatestKeyVersionEnabled', parameters('cMKManagedDisksKeyRotationToLatestKeyVersionEnabled')), null()))), null())]"

From 1362c87e8fa84a2ee7a421b9fe5eff585537a1f0 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Sat, 26 Aug 2023 11:55:38 +0200
Subject: [PATCH 16/26] Rollback unrelated change

---
 .../pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 b/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1
index b6b0202f2c..3885febb63 100644
--- a/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1
+++ b/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1
@@ -154,7 +154,7 @@ function Invoke-ResourceRemoval {
             $subscriptionId = $resourceId.Split('/')[2]
             $resourceGroupName = $resourceId.Split('/')[4]
             $resourceName = Split-Path $resourceId -Leaf
-            $purgePath = 'subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.MachineLearningServices/workspaces/{2}?api-version=2023-04-01&forceToPurge={3}' -f $subscriptionId, $resourceGroupName, $resourceName, $true
+            $purgePath = 'subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.MachineLearningServices/workspaces/{2}?api-version=2023-04-01-preview&forceToPurge={3}' -f $subscriptionId, $resourceGroupName, $resourceName, $true
             $purgeRequestInputObject = @{
                 Method = 'DELETE'
                 Path   = $purgePath

From bba0781159e162ce2c3f45a485b054cac41e1935 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Sat, 26 Aug 2023 11:59:31 +0200
Subject: [PATCH 17/26] Moved dislocated code

---
 .../workspace/.test/common/dependencies.bicep | 323 ++++++++++++++++++
 .../.test/min/main.test.bicep                 |   0
 modules/databricks/workspace/main.bicep       |   2 +-
 .../.test/common/dependencies.bicep           | 323 ------------------
 4 files changed, 324 insertions(+), 324 deletions(-)
 rename modules/databricks/{workspaces => workspace}/.test/min/main.test.bicep (100%)

diff --git a/modules/databricks/workspace/.test/common/dependencies.bicep b/modules/databricks/workspace/.test/common/dependencies.bicep
index a7f42aee7b..f80ee55ca5 100644
--- a/modules/databricks/workspace/.test/common/dependencies.bicep
+++ b/modules/databricks/workspace/.test/common/dependencies.bicep
@@ -4,10 +4,333 @@ param location string = resourceGroup().location
 @description('Required. The name of the Managed Identity to create.')
 param managedIdentityName string
 
+@description('Required. The name of the Key Vault to create.')
+param keyVaultName string
+
+@description('Required. The name of the Azure Machine Learning Workspace to create.')
+param amlWorkspaceName string
+
+@description('Required. The name of the Load Balancer to create.')
+param loadBalancerName string
+
+@description('Required. The name of the Network Security Group to create.')
+param networkSecurityGroupName string
+
+@description('Required. The name of the Storage Account to create.')
+param storageAccountName string
+
+@description('Required. The name of the Application Insights Instanec to create.')
+param applicationInsightsName string
+
+@description('Required. The name of the Virtual Network to create.')
+param virtualNetworkName string
+
+var addressPrefix = '10.0.0.0/16'
+var defaultAddressPrefix = '10.0.0.0/20'
+var publicAddressPrefix = '10.0.16.0/20'
+var privateAddressPrefix = '10.0.32.0/20'
+
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = {
     name: managedIdentityName
     location: location
 }
 
+resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = {
+    name: keyVaultName
+    location: location
+    properties: {
+        sku: {
+            family: 'A'
+            name: 'standard'
+        }
+        tenantId: tenant().tenantId
+        enablePurgeProtection: true // Required by batch account
+        softDeleteRetentionInDays: 7
+        enabledForTemplateDeployment: true
+        enabledForDiskEncryption: true
+        enabledForDeployment: true
+        enableRbacAuthorization: true
+        accessPolicies: []
+    }
+
+    resource key 'keys@2022-07-01' = {
+        name: 'keyEncryptionKey'
+        properties: {
+            kty: 'RSA'
+        }
+    }
+}
+
+resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+    name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-Key-Key-Vault-Crypto-User-RoleAssignment')
+    scope: keyVault::key
+    properties: {
+        principalId: '5167ea7a-355a-466f-ae8b-8ea60f718b35' // AzureDatabricks Enterprise Application Object Id
+        roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // Key Vault Crypto User
+        principalType: 'ServicePrincipal'
+    }
+}
+
+resource amlPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+    name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-Key-Vault-Contributor')
+    scope: keyVault
+    properties: {
+        principalId: managedIdentity.properties.principalId
+        roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') // Contributor
+        principalType: 'ServicePrincipal'
+    }
+}
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = {
+    name: storageAccountName
+    location: location
+    sku: {
+        name: 'Standard_ZRS'
+    }
+    kind: 'StorageV2'
+    properties: {}
+}
+
+resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
+    name: applicationInsightsName
+    location: location
+    kind: 'web'
+    properties: {
+        Application_Type: 'web'
+    }
+}
+
+resource machineLearningWorkspace 'Microsoft.MachineLearningServices/workspaces@2023-04-01' = {
+    name: amlWorkspaceName
+    location: location
+    identity: {
+        type: 'UserAssigned'
+        userAssignedIdentities: {
+            '${managedIdentity.id}': {}
+        }
+    }
+    properties: {
+        storageAccount: storageAccount.id
+        keyVault: keyVault.id
+        applicationInsights: applicationInsights.id
+        primaryUserAssignedIdentity: managedIdentity.id
+    }
+}
+
+resource loadBalancer 'Microsoft.Network/loadBalancers@2023-04-01' = {
+    name: loadBalancerName
+    location: location
+    properties: {
+        backendAddressPools: [
+            {
+                name: 'default'
+            }
+        ]
+        frontendIPConfigurations: [
+            {
+                name: 'privateIPConfig1'
+                properties: {
+                    subnet: {
+                        id: virtualNetwork.properties.subnets[0].id
+                    }
+                }
+            }
+        ]
+    }
+}
+
+resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-04-01' = {
+    name: networkSecurityGroupName
+    location: location
+    properties: {
+        securityRules: [
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-inbound'
+                properties: {
+                    description: 'Required for worker nodes communication within a cluster.'
+                    protocol: '*'
+                    sourcePortRange: '*'
+                    destinationPortRange: '*'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'VirtualNetwork'
+                    access: 'Allow'
+                    priority: 100
+                    direction: 'Inbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-databricks-webapp'
+                properties: {
+                    description: 'Required for workers communication with Databricks Webapp.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '443'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'AzureDatabricks'
+                    access: 'Allow'
+                    priority: 100
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-sql'
+                properties: {
+                    description: 'Required for workers communication with Azure SQL services.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '3306'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'Sql'
+                    access: 'Allow'
+                    priority: 101
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-storage'
+                properties: {
+                    description: 'Required for workers communication with Azure Storage services.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '443'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'Storage'
+                    access: 'Allow'
+                    priority: 102
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-outbound'
+                properties: {
+                    description: 'Required for worker nodes communication within a cluster.'
+                    protocol: '*'
+                    sourcePortRange: '*'
+                    destinationPortRange: '*'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'VirtualNetwork'
+                    access: 'Allow'
+                    priority: 103
+                    direction: 'Outbound'
+                }
+            }
+            {
+                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-eventhub'
+                properties: {
+                    description: 'Required for worker communication with Azure Eventhub services.'
+                    protocol: 'Tcp'
+                    sourcePortRange: '*'
+                    destinationPortRange: '9093'
+                    sourceAddressPrefix: 'VirtualNetwork'
+                    destinationAddressPrefix: 'EventHub'
+                    access: 'Allow'
+                    priority: 104
+                    direction: 'Outbound'
+                }
+            }
+        ]
+    }
+}
+
+resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = {
+    name: virtualNetworkName
+    location: location
+    properties: {
+        addressSpace: {
+            addressPrefixes: [
+                addressPrefix
+            ]
+        }
+        subnets: [
+            {
+                name: 'defaultSubnet'
+                properties: {
+                    addressPrefix: defaultAddressPrefix
+                }
+            }
+            {
+                name: 'custom-public-subnet'
+                properties: {
+                    addressPrefix: publicAddressPrefix
+                    networkSecurityGroup: {
+                        id: networkSecurityGroup.id
+                    }
+                    delegations: [
+                        {
+                            name: 'databricksDelegation'
+                            properties: {
+                                serviceName: 'Microsoft.Databricks/workspaces'
+                            }
+                        }
+                    ]
+                }
+            }
+            {
+                name: 'custom-private-subnet'
+                properties: {
+                    addressPrefix: privateAddressPrefix
+                    networkSecurityGroup: {
+                        id: networkSecurityGroup.id
+                    }
+                    delegations: [
+                        {
+                            name: 'databricksDelegation'
+                            properties: {
+                                serviceName: 'Microsoft.Databricks/workspaces'
+                            }
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+}
+
+resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
+    name: 'privatelink.azuredatabricks.net'
+    location: 'global'
+
+    resource virtualNetworkLinks 'virtualNetworkLinks@2020-06-01' = {
+        name: '${virtualNetwork.name}-vnetlink'
+        location: 'global'
+        properties: {
+            virtualNetwork: {
+                id: virtualNetwork.id
+            }
+            registrationEnabled: false
+        }
+    }
+}
+
+@description('The resource ID of the created Virtual Network Default Subnet.')
+output defaultSubnetResourceId string = virtualNetwork.properties.subnets[0].id
+
+@description('The resource ID of the created Virtual Network Public Subnet.')
+output customPublicSubnetName string = virtualNetwork.properties.subnets[1].name
+
+@description('The resource ID of the created Virtual Network Private Subnet.')
+output customPrivateSubnetName string = virtualNetwork.properties.subnets[2].name
+
+@description('The resource ID of the created Virtual Network.')
+output virtualNetworkResourceId string = virtualNetwork.id
+
+@description('The resource ID of the created Private DNS Zone.')
+output privateDNSResourceId string = privateDNSZone.id
+
+@description('The resource ID of the created Azure Machine Learning Workspace.')
+output machineLearningWorkspaceResourceId string = machineLearningWorkspace.id
+
+@description('The resource ID of the created Key Vault.')
+output keyVaultResourceId string = keyVault.id
+
+@description('The resource ID of the created Load Balancer.')
+output loadBalancerResourceId string = loadBalancer.id
+
+@description('The name of the created Load Balancer Backend Pool.')
+output loadBalancerBackendPoolName string = loadBalancer.properties.backendAddressPools[0].name
+
+@description('The name of the created Key Vault encryption key.')
+output keyVaultKeyName string = keyVault::key.name
+
 @description('The principal ID of the created Managed Identity.')
 output managedIdentityPrincipalId string = managedIdentity.properties.principalId
diff --git a/modules/databricks/workspaces/.test/min/main.test.bicep b/modules/databricks/workspace/.test/min/main.test.bicep
similarity index 100%
rename from modules/databricks/workspaces/.test/min/main.test.bicep
rename to modules/databricks/workspace/.test/min/main.test.bicep
diff --git a/modules/databricks/workspace/main.bicep b/modules/databricks/workspace/main.bicep
index 1917de7c51..3ae68d26a3 100644
--- a/modules/databricks/workspace/main.bicep
+++ b/modules/databricks/workspace/main.bicep
@@ -346,7 +346,7 @@ module workspace_roleAssignments '.bicep/nested_roleAssignments.bicep' = [for (r
   }
 }]
 
-module workspace_privateEndpoints '../../network/private-endpoints/main.bicep' = [for (privateEndpoint, index) in privateEndpoints: {
+module workspace_privateEndpoints '../../network/private-endpoint/main.bicep' = [for (privateEndpoint, index) in privateEndpoints: {
   name: '${uniqueString(deployment().name, location)}-Databricks-PrivateEndpoint-${index}'
   params: {
     groupIds: [
diff --git a/modules/web/connection/.test/common/dependencies.bicep b/modules/web/connection/.test/common/dependencies.bicep
index f80ee55ca5..a7f42aee7b 100644
--- a/modules/web/connection/.test/common/dependencies.bicep
+++ b/modules/web/connection/.test/common/dependencies.bicep
@@ -4,333 +4,10 @@ param location string = resourceGroup().location
 @description('Required. The name of the Managed Identity to create.')
 param managedIdentityName string
 
-@description('Required. The name of the Key Vault to create.')
-param keyVaultName string
-
-@description('Required. The name of the Azure Machine Learning Workspace to create.')
-param amlWorkspaceName string
-
-@description('Required. The name of the Load Balancer to create.')
-param loadBalancerName string
-
-@description('Required. The name of the Network Security Group to create.')
-param networkSecurityGroupName string
-
-@description('Required. The name of the Storage Account to create.')
-param storageAccountName string
-
-@description('Required. The name of the Application Insights Instanec to create.')
-param applicationInsightsName string
-
-@description('Required. The name of the Virtual Network to create.')
-param virtualNetworkName string
-
-var addressPrefix = '10.0.0.0/16'
-var defaultAddressPrefix = '10.0.0.0/20'
-var publicAddressPrefix = '10.0.16.0/20'
-var privateAddressPrefix = '10.0.32.0/20'
-
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = {
     name: managedIdentityName
     location: location
 }
 
-resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = {
-    name: keyVaultName
-    location: location
-    properties: {
-        sku: {
-            family: 'A'
-            name: 'standard'
-        }
-        tenantId: tenant().tenantId
-        enablePurgeProtection: true // Required by batch account
-        softDeleteRetentionInDays: 7
-        enabledForTemplateDeployment: true
-        enabledForDiskEncryption: true
-        enabledForDeployment: true
-        enableRbacAuthorization: true
-        accessPolicies: []
-    }
-
-    resource key 'keys@2022-07-01' = {
-        name: 'keyEncryptionKey'
-        properties: {
-            kty: 'RSA'
-        }
-    }
-}
-
-resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
-    name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-Key-Key-Vault-Crypto-User-RoleAssignment')
-    scope: keyVault::key
-    properties: {
-        principalId: '5167ea7a-355a-466f-ae8b-8ea60f718b35' // AzureDatabricks Enterprise Application Object Id
-        roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // Key Vault Crypto User
-        principalType: 'ServicePrincipal'
-    }
-}
-
-resource amlPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
-    name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-Key-Vault-Contributor')
-    scope: keyVault
-    properties: {
-        principalId: managedIdentity.properties.principalId
-        roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') // Contributor
-        principalType: 'ServicePrincipal'
-    }
-}
-
-resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = {
-    name: storageAccountName
-    location: location
-    sku: {
-        name: 'Standard_ZRS'
-    }
-    kind: 'StorageV2'
-    properties: {}
-}
-
-resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
-    name: applicationInsightsName
-    location: location
-    kind: 'web'
-    properties: {
-        Application_Type: 'web'
-    }
-}
-
-resource machineLearningWorkspace 'Microsoft.MachineLearningServices/workspaces@2023-04-01' = {
-    name: amlWorkspaceName
-    location: location
-    identity: {
-        type: 'UserAssigned'
-        userAssignedIdentities: {
-            '${managedIdentity.id}': {}
-        }
-    }
-    properties: {
-        storageAccount: storageAccount.id
-        keyVault: keyVault.id
-        applicationInsights: applicationInsights.id
-        primaryUserAssignedIdentity: managedIdentity.id
-    }
-}
-
-resource loadBalancer 'Microsoft.Network/loadBalancers@2023-04-01' = {
-    name: loadBalancerName
-    location: location
-    properties: {
-        backendAddressPools: [
-            {
-                name: 'default'
-            }
-        ]
-        frontendIPConfigurations: [
-            {
-                name: 'privateIPConfig1'
-                properties: {
-                    subnet: {
-                        id: virtualNetwork.properties.subnets[0].id
-                    }
-                }
-            }
-        ]
-    }
-}
-
-resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-04-01' = {
-    name: networkSecurityGroupName
-    location: location
-    properties: {
-        securityRules: [
-            {
-                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-inbound'
-                properties: {
-                    description: 'Required for worker nodes communication within a cluster.'
-                    protocol: '*'
-                    sourcePortRange: '*'
-                    destinationPortRange: '*'
-                    sourceAddressPrefix: 'VirtualNetwork'
-                    destinationAddressPrefix: 'VirtualNetwork'
-                    access: 'Allow'
-                    priority: 100
-                    direction: 'Inbound'
-                }
-            }
-            {
-                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-databricks-webapp'
-                properties: {
-                    description: 'Required for workers communication with Databricks Webapp.'
-                    protocol: 'Tcp'
-                    sourcePortRange: '*'
-                    destinationPortRange: '443'
-                    sourceAddressPrefix: 'VirtualNetwork'
-                    destinationAddressPrefix: 'AzureDatabricks'
-                    access: 'Allow'
-                    priority: 100
-                    direction: 'Outbound'
-                }
-            }
-            {
-                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-sql'
-                properties: {
-                    description: 'Required for workers communication with Azure SQL services.'
-                    protocol: 'Tcp'
-                    sourcePortRange: '*'
-                    destinationPortRange: '3306'
-                    sourceAddressPrefix: 'VirtualNetwork'
-                    destinationAddressPrefix: 'Sql'
-                    access: 'Allow'
-                    priority: 101
-                    direction: 'Outbound'
-                }
-            }
-            {
-                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-storage'
-                properties: {
-                    description: 'Required for workers communication with Azure Storage services.'
-                    protocol: 'Tcp'
-                    sourcePortRange: '*'
-                    destinationPortRange: '443'
-                    sourceAddressPrefix: 'VirtualNetwork'
-                    destinationAddressPrefix: 'Storage'
-                    access: 'Allow'
-                    priority: 102
-                    direction: 'Outbound'
-                }
-            }
-            {
-                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-outbound'
-                properties: {
-                    description: 'Required for worker nodes communication within a cluster.'
-                    protocol: '*'
-                    sourcePortRange: '*'
-                    destinationPortRange: '*'
-                    sourceAddressPrefix: 'VirtualNetwork'
-                    destinationAddressPrefix: 'VirtualNetwork'
-                    access: 'Allow'
-                    priority: 103
-                    direction: 'Outbound'
-                }
-            }
-            {
-                name: 'Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-eventhub'
-                properties: {
-                    description: 'Required for worker communication with Azure Eventhub services.'
-                    protocol: 'Tcp'
-                    sourcePortRange: '*'
-                    destinationPortRange: '9093'
-                    sourceAddressPrefix: 'VirtualNetwork'
-                    destinationAddressPrefix: 'EventHub'
-                    access: 'Allow'
-                    priority: 104
-                    direction: 'Outbound'
-                }
-            }
-        ]
-    }
-}
-
-resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = {
-    name: virtualNetworkName
-    location: location
-    properties: {
-        addressSpace: {
-            addressPrefixes: [
-                addressPrefix
-            ]
-        }
-        subnets: [
-            {
-                name: 'defaultSubnet'
-                properties: {
-                    addressPrefix: defaultAddressPrefix
-                }
-            }
-            {
-                name: 'custom-public-subnet'
-                properties: {
-                    addressPrefix: publicAddressPrefix
-                    networkSecurityGroup: {
-                        id: networkSecurityGroup.id
-                    }
-                    delegations: [
-                        {
-                            name: 'databricksDelegation'
-                            properties: {
-                                serviceName: 'Microsoft.Databricks/workspaces'
-                            }
-                        }
-                    ]
-                }
-            }
-            {
-                name: 'custom-private-subnet'
-                properties: {
-                    addressPrefix: privateAddressPrefix
-                    networkSecurityGroup: {
-                        id: networkSecurityGroup.id
-                    }
-                    delegations: [
-                        {
-                            name: 'databricksDelegation'
-                            properties: {
-                                serviceName: 'Microsoft.Databricks/workspaces'
-                            }
-                        }
-                    ]
-                }
-            }
-        ]
-    }
-}
-
-resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
-    name: 'privatelink.azuredatabricks.net'
-    location: 'global'
-
-    resource virtualNetworkLinks 'virtualNetworkLinks@2020-06-01' = {
-        name: '${virtualNetwork.name}-vnetlink'
-        location: 'global'
-        properties: {
-            virtualNetwork: {
-                id: virtualNetwork.id
-            }
-            registrationEnabled: false
-        }
-    }
-}
-
-@description('The resource ID of the created Virtual Network Default Subnet.')
-output defaultSubnetResourceId string = virtualNetwork.properties.subnets[0].id
-
-@description('The resource ID of the created Virtual Network Public Subnet.')
-output customPublicSubnetName string = virtualNetwork.properties.subnets[1].name
-
-@description('The resource ID of the created Virtual Network Private Subnet.')
-output customPrivateSubnetName string = virtualNetwork.properties.subnets[2].name
-
-@description('The resource ID of the created Virtual Network.')
-output virtualNetworkResourceId string = virtualNetwork.id
-
-@description('The resource ID of the created Private DNS Zone.')
-output privateDNSResourceId string = privateDNSZone.id
-
-@description('The resource ID of the created Azure Machine Learning Workspace.')
-output machineLearningWorkspaceResourceId string = machineLearningWorkspace.id
-
-@description('The resource ID of the created Key Vault.')
-output keyVaultResourceId string = keyVault.id
-
-@description('The resource ID of the created Load Balancer.')
-output loadBalancerResourceId string = loadBalancer.id
-
-@description('The name of the created Load Balancer Backend Pool.')
-output loadBalancerBackendPoolName string = loadBalancer.properties.backendAddressPools[0].name
-
-@description('The name of the created Key Vault encryption key.')
-output keyVaultKeyName string = keyVault::key.name
-
 @description('The principal ID of the created Managed Identity.')
 output managedIdentityPrincipalId string = managedIdentity.properties.principalId

From e282f10df5ec9e3cbc2690fe008687628f1a7e66 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Sat, 26 Aug 2023 12:00:26 +0200
Subject: [PATCH 18/26] Update to latest

---
 modules/databricks/workspace/.test/main.test.bicep | 8 ++++++++
 modules/web/connection/.test/main.test.bicep       | 8 --------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/modules/databricks/workspace/.test/main.test.bicep b/modules/databricks/workspace/.test/main.test.bicep
index dff85d99b8..ef99b3dc03 100644
--- a/modules/databricks/workspace/.test/main.test.bicep
+++ b/modules/databricks/workspace/.test/main.test.bicep
@@ -13,3 +13,11 @@ module common 'common/main.test.bicep' = {
     namePrefix: namePrefix
   }
 }
+
+// TEST 2 - MIN
+module min 'min/main.test.bicep' = {
+  name: '${uniqueString(deployment().name)}-min-test'
+  params: {
+    namePrefix: namePrefix
+  }
+}
diff --git a/modules/web/connection/.test/main.test.bicep b/modules/web/connection/.test/main.test.bicep
index ef99b3dc03..dff85d99b8 100644
--- a/modules/web/connection/.test/main.test.bicep
+++ b/modules/web/connection/.test/main.test.bicep
@@ -13,11 +13,3 @@ module common 'common/main.test.bicep' = {
     namePrefix: namePrefix
   }
 }
-
-// TEST 2 - MIN
-module min 'min/main.test.bicep' = {
-  name: '${uniqueString(deployment().name)}-min-test'
-  params: {
-    namePrefix: namePrefix
-  }
-}

From 6a0cdbeaf3bcca97fd6341caf6dee245dfdfbd88 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Sat, 26 Aug 2023 12:02:11 +0200
Subject: [PATCH 19/26] Updated docs

---
 modules/databricks/workspace/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/databricks/workspace/README.md b/modules/databricks/workspace/README.md
index a4414d9165..1aa1ee4915 100644
--- a/modules/databricks/workspace/README.md
+++ b/modules/databricks/workspace/README.md
@@ -358,7 +358,7 @@ This section gives you an overview of all local-referenced module files (i.e., o
 
 | Reference | Type |
 | :-- | :-- |
-| `network/private-endpoints` | Local reference |
+| `network/private-endpoint` | Local reference |
 
 ## Deployment examples
 
@@ -615,7 +615,7 @@ module workspace './databricks/workspace/main.bicep' = {
 <summary>via Bicep module</summary>
 
 ```bicep
-module workspaces './databricks/workspaces/main.bicep' = {
+module workspace './databricks/workspace/main.bicep' = {
   name: '${uniqueString(deployment().name, location)}-test-dwmin'
   params: {
     // Required parameters

From 288c463290b28cba75009df710160278b696a9e0 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Sat, 26 Aug 2023 12:12:39 +0200
Subject: [PATCH 20/26] Updated PE module API version to unblock PR

---
 modules/network/private-endpoint/README.md    |  4 +-
 modules/network/private-endpoint/main.bicep   |  2 +-
 modules/network/private-endpoint/main.json    | 38 +++++++++++--------
 .../private-dns-zone-group/README.md          |  2 +-
 .../private-dns-zone-group/main.bicep         |  4 +-
 .../private-dns-zone-group/main.json          | 13 ++++---
 6 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/modules/network/private-endpoint/README.md b/modules/network/private-endpoint/README.md
index 18541322bd..1480d9ba84 100644
--- a/modules/network/private-endpoint/README.md
+++ b/modules/network/private-endpoint/README.md
@@ -16,8 +16,8 @@ This module deploys a Private Endpoint.
 | :-- | :-- |
 | `Microsoft.Authorization/locks` | [2020-05-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2020-05-01/locks) |
 | `Microsoft.Authorization/roleAssignments` | [2022-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2022-04-01/roleAssignments) |
-| `Microsoft.Network/privateEndpoints` | [2022-07-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2022-07-01/privateEndpoints) |
-| `Microsoft.Network/privateEndpoints/privateDnsZoneGroups` | [2022-07-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2022-07-01/privateEndpoints/privateDnsZoneGroups) |
+| `Microsoft.Network/privateEndpoints` | [2023-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2023-04-01/privateEndpoints) |
+| `Microsoft.Network/privateEndpoints/privateDnsZoneGroups` | [2023-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2023-04-01/privateEndpoints/privateDnsZoneGroups) |
 
 ### Resource dependency
 
diff --git a/modules/network/private-endpoint/main.bicep b/modules/network/private-endpoint/main.bicep
index b0420647e1..c43e6c55ef 100644
--- a/modules/network/private-endpoint/main.bicep
+++ b/modules/network/private-endpoint/main.bicep
@@ -66,7 +66,7 @@ resource defaultTelemetry 'Microsoft.Resources/deployments@2021-04-01' = if (ena
   }
 }
 
-resource privateEndpoint 'Microsoft.Network/privateEndpoints@2022-07-01' = {
+resource privateEndpoint 'Microsoft.Network/privateEndpoints@2023-04-01' = {
   name: name
   location: location
   tags: tags
diff --git a/modules/network/private-endpoint/main.json b/modules/network/private-endpoint/main.json
index 8bb7cfb2b2..ab7eacf336 100644
--- a/modules/network/private-endpoint/main.json
+++ b/modules/network/private-endpoint/main.json
@@ -4,9 +4,12 @@
   "metadata": {
     "_generator": {
       "name": "bicep",
-      "version": "0.19.5.34762",
-      "templateHash": "11825715538543749879"
-    }
+      "version": "0.20.4.51522",
+      "templateHash": "13560297539192628062"
+    },
+    "name": "Private Endpoints",
+    "description": "This module deploys a Private Endpoint.",
+    "owner": "Azure/module-maintainers"
   },
   "parameters": {
     "name": {
@@ -71,14 +74,14 @@
     "lock": {
       "type": "string",
       "defaultValue": "",
-      "metadata": {
-        "description": "Optional. Specify the type of lock."
-      },
       "allowedValues": [
         "",
         "CanNotDelete",
         "ReadOnly"
-      ]
+      ],
+      "metadata": {
+        "description": "Optional. Specify the type of lock."
+      }
     },
     "roleAssignments": {
       "type": "array",
@@ -136,7 +139,7 @@
     },
     {
       "type": "Microsoft.Network/privateEndpoints",
-      "apiVersion": "2022-07-01",
+      "apiVersion": "2023-04-01",
       "name": "[parameters('name')]",
       "location": "[parameters('location')]",
       "tags": "[parameters('tags')]",
@@ -201,9 +204,12 @@
           "metadata": {
             "_generator": {
               "name": "bicep",
-              "version": "0.19.5.34762",
-              "templateHash": "12684511314187066258"
-            }
+              "version": "0.20.4.51522",
+              "templateHash": "17831763001460207830"
+            },
+            "name": "Private Endpoint Private DNS Zone Groups",
+            "description": "This module deploys a Private Endpoint Private DNS Zone Group.",
+            "owner": "Azure/module-maintainers"
           },
           "parameters": {
             "privateEndpointName": {
@@ -214,8 +220,8 @@
             },
             "privateDNSResourceIds": {
               "type": "array",
-              "maxLength": 5,
               "minLength": 1,
+              "maxLength": 5,
               "metadata": {
                 "description": "Required. Array of private DNS zone resource IDs. A DNS zone group can support up to 5 DNS zones."
               }
@@ -266,7 +272,7 @@
             },
             {
               "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
-              "apiVersion": "2022-07-01",
+              "apiVersion": "2023-04-01",
               "name": "[format('{0}/{1}', parameters('privateEndpointName'), parameters('name'))]",
               "properties": {
                 "privateDnsZoneConfigs": "[variables('privateDnsZoneConfigs')]"
@@ -336,8 +342,8 @@
           "metadata": {
             "_generator": {
               "name": "bicep",
-              "version": "0.19.5.34762",
-              "templateHash": "18055161250379920591"
+              "version": "0.20.4.51522",
+              "templateHash": "11548486149222715894"
             }
           },
           "parameters": {
@@ -507,7 +513,7 @@
       "metadata": {
         "description": "The location the resource was deployed into."
       },
-      "value": "[reference(resourceId('Microsoft.Network/privateEndpoints', parameters('name')), '2022-07-01', 'full').location]"
+      "value": "[reference(resourceId('Microsoft.Network/privateEndpoints', parameters('name')), '2023-04-01', 'full').location]"
     }
   }
 }
\ No newline at end of file
diff --git a/modules/network/private-endpoint/private-dns-zone-group/README.md b/modules/network/private-endpoint/private-dns-zone-group/README.md
index 08764e80c8..e1d46b8986 100644
--- a/modules/network/private-endpoint/private-dns-zone-group/README.md
+++ b/modules/network/private-endpoint/private-dns-zone-group/README.md
@@ -13,7 +13,7 @@ This module deploys a Private Endpoint Private DNS Zone Group.
 
 | Resource Type | API Version |
 | :-- | :-- |
-| `Microsoft.Network/privateEndpoints/privateDnsZoneGroups` | [2022-07-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2022-07-01/privateEndpoints/privateDnsZoneGroups) |
+| `Microsoft.Network/privateEndpoints/privateDnsZoneGroups` | [2023-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Network/2023-04-01/privateEndpoints/privateDnsZoneGroups) |
 
 ## Parameters
 
diff --git a/modules/network/private-endpoint/private-dns-zone-group/main.bicep b/modules/network/private-endpoint/private-dns-zone-group/main.bicep
index 04ad6b03e2..316f0800b6 100644
--- a/modules/network/private-endpoint/private-dns-zone-group/main.bicep
+++ b/modules/network/private-endpoint/private-dns-zone-group/main.bicep
@@ -35,11 +35,11 @@ var privateDnsZoneConfigs = [for privateDNSResourceId in privateDNSResourceIds:
   }
 }]
 
-resource privateEndpoint 'Microsoft.Network/privateEndpoints@2022-07-01' existing = {
+resource privateEndpoint 'Microsoft.Network/privateEndpoints@2023-04-01' existing = {
   name: privateEndpointName
 }
 
-resource privateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2022-07-01' = {
+resource privateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-04-01' = {
   name: name
   parent: privateEndpoint
   properties: {
diff --git a/modules/network/private-endpoint/private-dns-zone-group/main.json b/modules/network/private-endpoint/private-dns-zone-group/main.json
index 9ec37c9710..93baa64a6d 100644
--- a/modules/network/private-endpoint/private-dns-zone-group/main.json
+++ b/modules/network/private-endpoint/private-dns-zone-group/main.json
@@ -4,9 +4,12 @@
   "metadata": {
     "_generator": {
       "name": "bicep",
-      "version": "0.19.5.34762",
-      "templateHash": "12684511314187066258"
-    }
+      "version": "0.20.4.51522",
+      "templateHash": "17831763001460207830"
+    },
+    "name": "Private Endpoint Private DNS Zone Groups",
+    "description": "This module deploys a Private Endpoint Private DNS Zone Group.",
+    "owner": "Azure/module-maintainers"
   },
   "parameters": {
     "privateEndpointName": {
@@ -17,8 +20,8 @@
     },
     "privateDNSResourceIds": {
       "type": "array",
-      "maxLength": 5,
       "minLength": 1,
+      "maxLength": 5,
       "metadata": {
         "description": "Required. Array of private DNS zone resource IDs. A DNS zone group can support up to 5 DNS zones."
       }
@@ -69,7 +72,7 @@
     },
     {
       "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
-      "apiVersion": "2022-07-01",
+      "apiVersion": "2023-04-01",
       "name": "[format('{0}/{1}', parameters('privateEndpointName'), parameters('name'))]",
       "properties": {
         "privateDnsZoneConfigs": "[variables('privateDnsZoneConfigs')]"

From b8db825a2b7aa518e2e0cfec8835d56887621eca Mon Sep 17 00:00:00 2001
From: Alexander Sehr <ASehr@hotmail.de>
Date: Wed, 6 Sep 2023 13:56:20 +0200
Subject: [PATCH 21/26] Update
 modules/databricks/workspace/.test/common/dependencies.bicep

Co-authored-by: Ahmad Abdalla <28486158+ahmadabdalla@users.noreply.github.com>
---
 modules/databricks/workspace/.test/common/dependencies.bicep | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/databricks/workspace/.test/common/dependencies.bicep b/modules/databricks/workspace/.test/common/dependencies.bicep
index f80ee55ca5..dc7bd0c573 100644
--- a/modules/databricks/workspace/.test/common/dependencies.bicep
+++ b/modules/databricks/workspace/.test/common/dependencies.bicep
@@ -245,7 +245,7 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = {
             {
                 name: 'defaultSubnet'
                 properties: {
-                    addressPrefix: defaultAddressPrefix
+                    addressPrefix: cidrSubnet(addressPrefix, 16, 0)
                 }
             }
             {

From 701b23a6c0ef39de8857cc237fa4c403ed8805c8 Mon Sep 17 00:00:00 2001
From: Alexander Sehr <ASehr@hotmail.de>
Date: Wed, 6 Sep 2023 13:56:51 +0200
Subject: [PATCH 22/26] Update
 modules/databricks/workspace/.test/common/dependencies.bicep

Co-authored-by: Ahmad Abdalla <28486158+ahmadabdalla@users.noreply.github.com>
---
 modules/databricks/workspace/.test/common/dependencies.bicep | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/databricks/workspace/.test/common/dependencies.bicep b/modules/databricks/workspace/.test/common/dependencies.bicep
index dc7bd0c573..a4e5160f9d 100644
--- a/modules/databricks/workspace/.test/common/dependencies.bicep
+++ b/modules/databricks/workspace/.test/common/dependencies.bicep
@@ -305,7 +305,7 @@ resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = {
 @description('The resource ID of the created Virtual Network Default Subnet.')
 output defaultSubnetResourceId string = virtualNetwork.properties.subnets[0].id
 
-@description('The resource ID of the created Virtual Network Public Subnet.')
+@description('The name of the created Virtual Network Public Subnet.')
 output customPublicSubnetName string = virtualNetwork.properties.subnets[1].name
 
 @description('The resource ID of the created Virtual Network Private Subnet.')

From 30a03ac0e82e7d8d297966bab306e6711b87e094 Mon Sep 17 00:00:00 2001
From: Alexander Sehr <ASehr@hotmail.de>
Date: Wed, 6 Sep 2023 13:57:58 +0200
Subject: [PATCH 23/26] Update
 modules/databricks/workspace/.test/common/dependencies.bicep

Co-authored-by: Ahmad Abdalla <28486158+ahmadabdalla@users.noreply.github.com>
---
 modules/databricks/workspace/.test/common/dependencies.bicep | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/databricks/workspace/.test/common/dependencies.bicep b/modules/databricks/workspace/.test/common/dependencies.bicep
index a4e5160f9d..de13a72ce6 100644
--- a/modules/databricks/workspace/.test/common/dependencies.bicep
+++ b/modules/databricks/workspace/.test/common/dependencies.bicep
@@ -308,7 +308,7 @@ output defaultSubnetResourceId string = virtualNetwork.properties.subnets[0].id
 @description('The name of the created Virtual Network Public Subnet.')
 output customPublicSubnetName string = virtualNetwork.properties.subnets[1].name
 
-@description('The resource ID of the created Virtual Network Private Subnet.')
+@description('The name of the created Virtual Network Private Subnet.')
 output customPrivateSubnetName string = virtualNetwork.properties.subnets[2].name
 
 @description('The resource ID of the created Virtual Network.')

From c4d078e1b4524839229548c373cd875d4fa65820 Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 6 Sep 2023 14:00:05 +0200
Subject: [PATCH 24/26] Added CIDR function

---
 .../databricks/workspace/.test/common/dependencies.bicep | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/modules/databricks/workspace/.test/common/dependencies.bicep b/modules/databricks/workspace/.test/common/dependencies.bicep
index de13a72ce6..7030a8aa0a 100644
--- a/modules/databricks/workspace/.test/common/dependencies.bicep
+++ b/modules/databricks/workspace/.test/common/dependencies.bicep
@@ -26,9 +26,6 @@ param applicationInsightsName string
 param virtualNetworkName string
 
 var addressPrefix = '10.0.0.0/16'
-var defaultAddressPrefix = '10.0.0.0/20'
-var publicAddressPrefix = '10.0.16.0/20'
-var privateAddressPrefix = '10.0.32.0/20'
 
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = {
     name: managedIdentityName
@@ -245,13 +242,13 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = {
             {
                 name: 'defaultSubnet'
                 properties: {
-                    addressPrefix: cidrSubnet(addressPrefix, 16, 0)
+                    addressPrefix: cidrSubnet(addressPrefix, 20, 0)
                 }
             }
             {
                 name: 'custom-public-subnet'
                 properties: {
-                    addressPrefix: publicAddressPrefix
+                    addressPrefix: cidrSubnet(addressPrefix, 20, 1)
                     networkSecurityGroup: {
                         id: networkSecurityGroup.id
                     }
@@ -268,7 +265,7 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = {
             {
                 name: 'custom-private-subnet'
                 properties: {
-                    addressPrefix: privateAddressPrefix
+                    addressPrefix: cidrSubnet(addressPrefix, 20, 2)
                     networkSecurityGroup: {
                         id: networkSecurityGroup.id
                     }

From e3b84a97fa7e6c0700992adb2299ab286e0f54c9 Mon Sep 17 00:00:00 2001
From: Alexander Sehr <ASehr@hotmail.de>
Date: Wed, 6 Sep 2023 14:09:01 +0200
Subject: [PATCH 25/26] Update modules/databricks/workspace/main.bicep

Co-authored-by: Ahmad Abdalla <28486158+ahmadabdalla@users.noreply.github.com>
---
 modules/databricks/workspace/main.bicep | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/databricks/workspace/main.bicep b/modules/databricks/workspace/main.bicep
index fdab9996ae..e88461ce0a 100644
--- a/modules/databricks/workspace/main.bicep
+++ b/modules/databricks/workspace/main.bicep
@@ -5,7 +5,7 @@ metadata owner = 'Azure/module-maintainers'
 @description('Required. The name of the Azure Databricks workspace to create.')
 param name string
 
-@description('Optional. The managed resource group ID. Is created by the module as per the to-be resource ID you provide.')
+@description('Optional. The managed resource group ID. It is created by the module as per the to-be resource ID you provide.')
 param managedResourceGroupResourceId string = ''
 
 @description('Optional. The pricing tier of workspace.')

From 4cf8c49635aecc400c01567e3c9da8e5a202bddc Mon Sep 17 00:00:00 2001
From: AlexanderSehr <asehr@hotmail.de>
Date: Wed, 6 Sep 2023 14:13:28 +0200
Subject: [PATCH 26/26] Updated docs

---
 modules/databricks/workspace/README.md | 2 +-
 modules/databricks/workspace/main.json | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/databricks/workspace/README.md b/modules/databricks/workspace/README.md
index 7b86263ed7..5f1ba4b232 100644
--- a/modules/databricks/workspace/README.md
+++ b/modules/databricks/workspace/README.md
@@ -61,7 +61,7 @@ This module deploys an Azure Databricks Workspace.
 | `loadBalancerResourceId` | string | `''` |  | Resource URI of Outbound Load balancer for Secure Cluster Connectivity (No Public IP) workspace. |
 | `location` | string | `[resourceGroup().location]` |  | Location for all Resources. |
 | `lock` | string | `''` | `['', CanNotDelete, ReadOnly]` | Specify the type of lock. |
-| `managedResourceGroupResourceId` | string | `''` |  | The managed resource group ID. Is created by the module as per the to-be resource ID you provide. |
+| `managedResourceGroupResourceId` | string | `''` |  | The managed resource group ID. It is created by the module as per the to-be resource ID you provide. |
 | `natGatewayName` | string | `''` |  | Name of the NAT gateway for Secure Cluster Connectivity (No Public IP) workspace subnets. |
 | `prepareEncryption` | bool | `False` |  | Prepare the workspace for encryption. Enables the Managed Identity for managed storage account. |
 | `privateEndpoints` | array | `[]` |  | Configuration details for private endpoints. For security reasons, it is recommended to use private endpoints whenever possible. |
diff --git a/modules/databricks/workspace/main.json b/modules/databricks/workspace/main.json
index cf8ef194a3..4437110d43 100644
--- a/modules/databricks/workspace/main.json
+++ b/modules/databricks/workspace/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.20.4.51522",
-      "templateHash": "9156434929960030548"
+      "templateHash": "14136317346083571214"
     },
     "name": "Azure Databricks Workspaces",
     "description": "This module deploys an Azure Databricks Workspace.",
@@ -22,7 +22,7 @@
       "type": "string",
       "defaultValue": "",
       "metadata": {
-        "description": "Optional. The managed resource group ID. Is created by the module as per the to-be resource ID you provide."
+        "description": "Optional. The managed resource group ID. It is created by the module as per the to-be resource ID you provide."
       }
     },
     "skuName": {