Merge pull request microsoft#2670 from sebassem/arcbox_3.0_patches

Arcbox 3.0 patches

azure_jumpstart_arcbox/ARM/azuredeploy.json

@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.29.47.4906",
-      "templateHash": "13913810108142988750"
+      "templateHash": "2738768575994422412"
     }
   },
   "parameters": {
@@ -78,7 +78,7 @@
     },
     "githubBranch": {
       "type": "string",
-      "defaultValue": "arcbox_3.0",
+      "defaultValue": "main",
       "metadata": {
         "description": "Target GitHub branch"
       }
@@ -511,7 +511,7 @@
         "name": "ubuntuRancherK3sDataSvcNodesDeployment",
         "count": "[length(range(0, variables('k3sClusterNodesCount')))]"
       },
-      "condition": "[or(or(equals(parameters('flavor'), 'Full'), equals(parameters('flavor'), 'DataOps')), equals(parameters('flavor'), 'DevOps'))]",
+      "condition": "[or(equals(parameters('flavor'), 'DataOps'), equals(parameters('flavor'), 'DevOps'))]",
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2022-09-01",
       "name": "[format('ubuntuRancherK3sDataSvcNodesDeployment-{0}', range(0, variables('k3sClusterNodesCount'))[copyIndex()])]",
@@ -1704,6 +1704,8 @@
       "dependsOn": [
         "[resourceId('Microsoft.Resources/deployments', 'mgmtArtifactsAndPolicyDeployment')]",
         "[resourceId('Microsoft.Resources/deployments', 'stagingStorageAccountDeployment')]",
+        "[resourceId('Microsoft.Resources/deployments', 'ubuntuRancherK3sDataSvcDeployment')]",
+        "[resourceId('Microsoft.Resources/deployments', 'ubuntuRancherK3sDeployment')]",
         "[resourceId('Microsoft.Resources/deployments', 'updateVNetDNSServers')]"
       ]
     },

azure_jumpstart_arcbox/artifacts/DataOpsLogonScript.ps1

@@ -12,7 +12,7 @@ $aksArcClusterName = ($Env:aksArcClusterName).toLower()
 $aksdrArcClusterName = ($Env:aksdrArcClusterName).toLower()
 
 $clusters = @(
-    [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" }
+    [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; dataController = "$k3sArcDataClusterName-dc" ; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-data" }
     [pscustomobject]@{clusterName = $Env:aksArcClusterName ; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" }
     [pscustomobject]@{clusterName = $Env:aksdrArcClusterName ; dataController = "$aksdrArcClusterName-dc" ; customLocation = "$aksdrArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'DisasterRecovery' ; context = 'aks-dr'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aksdr" }
 )
@@ -149,7 +149,7 @@ Write-Host "`n"
 # Downloading k3s Kubernetes cluster kubeconfig file
 Write-Header "Downloading k3s Kubeconfig"
 $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config"
-azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc"
+azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config-k3s-data"
 azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile "C:\Users\$Env:adminUsername\.kube\config"
 
 $addsDomainNetBiosName = $Env:addsDomainName.Split(".")[0]
@@ -174,7 +174,7 @@ az aks get-credentials --resource-group $Env:resourceGroup --name $Env:aksdrArcC
 
 kubectx aks="$Env:aksArcClusterName-admin"
 kubectx aks-dr="$Env:aksdrArcClusterName-admin"
-kubectx k3s="arcbox-k3s-data"
+kubectx k3s="$namingPrefix-k3s-data"
 
 Start-Sleep -Seconds 10
 
@@ -198,8 +198,7 @@ foreach ($cluster in $clusters) {
                 --resource-group $Env:resourceGroup `
                 --location $Env:azureLocation `
                 --correlation-id "6038cc5b-b814-4d20-bcaa-0f60392416d5" `
-                --kube-config $cluster.kubeConfig `
-                --distribution $cluster.distribution
+                --kube-config $cluster.kubeConfig
         }
         catch {
             <#Do this if a terminating exception happens#>
@@ -400,7 +399,7 @@ Stop-Transcript
 # - Deploying data services on k3s cluster
 ################################################
 
-wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster'
+wt --% --maximized new-tab pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:adminUsername\.kube\config-k3s-data" -clusterName 'k3s Cluster'; split-pane -p "PowerShell" pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aks" -clusterName 'AKS Cluster'; split-pane -H pwsh.exe -NoExit -Command Show-K8sPodStatus -kubeconfig "C:\Users\$Env:USERNAME\.kube\config-aksdr" -clusterName 'AKS-DR Cluster'
 
 Write-Header "Deploying Azure Arc Data Controllers on Kubernetes cluster"
 $clusters | Foreach-Object -ThrottleLimit 5 -Parallel {

azure_jumpstart_arcbox/artifacts/DeploySQLMIADAuth.ps1

@@ -61,7 +61,7 @@ else {
 
 $sqlInstances = @(
 
-    [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-datasvc" }
+    [pscustomobject]@{instanceName = 'k3s-sql'; dataController = "$k3sArcDataClusterName-dc"; customLocation = "$k3sArcDataClusterName-cl" ; storageClassName = 'longhorn' ; licenseType = 'LicenseIncluded' ; context = 'k3s' ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s-data" }
 
     [pscustomobject]@{instanceName = 'aks-sql'; dataController = "$aksArcClusterName-dc" ; customLocation = "$aksArcClusterName-cl" ; storageClassName = 'managed-premium' ; licenseType = 'LicenseIncluded' ; context = 'aks'; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-aks" }
 
@@ -279,10 +279,10 @@ $sqlInstances | Foreach-Object -ThrottleLimit 5 -Parallel {
 
     # Create windows account in SQLMI to support AD authentication and grant sysadmin role
     $podname = "${sqlMIName}-0"
-    kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS"
+    kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "CREATE LOGIN [${domain_netbios_name}\$env:adminUsername] FROM WINDOWS" 2>&1 $null
     Write-Host "Created Windows user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance."
 
-    kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'"
+    kubectl exec $podname -c arc-sqlmi -n arc --kubeconfig $sqlInstance.kubeConfig -- /opt/mssql-tools/bin/sqlcmd -S localhost -U $env:AZDATA_USERNAME -P $AZDATA_PASSWORD -Q "EXEC master..sp_addsrvrolemember @loginame = N'${domain_netbios_name}\$env:adminUsername', @rolename = N'sysadmin'" 2>&1 $null
     Write-Host "Granted sysadmin role to user account ${domain_netbios_name}\$env:AZDATA_USERNAME in SQLMI instance."
 
     # Downloading demo database and restoring onto SQL MI
@@ -376,22 +376,22 @@ az sql instance-failover-group-arc create --shared-name ArcBoxDag --name primary
 Write-Host "`n"
 
 $cnameRecord = $sqlInstances[0].instanceName + ".jumpstart.local"
-Add-DnsServerResourceRecordCName -Name "ArcBoxDag" -ComputerName $dcInfo.HostName -HostNameAlias $cnameRecord -ZoneName jumpstart.local -TimeToLive 00:05:00
+Add-DnsServerResourceRecordCName -Name "${namingPrefix}Dag" -ComputerName $dcInfo.HostName -HostNameAlias $cnameRecord -ZoneName jumpstart.local -TimeToLive 00:05:00
 
 
 Write-Header "Creating Azure Data Studio settings for SQL Managed Instance connection with AD Authentication"
 
 $settingsTemplateFile = "$Env:ArcBoxDir\settingsTemplate.json"
 
 $aks = $sqlInstances[1].instanceName + ".jumpstart.local" + ",$sqlmi_port"
-$arcboxDag = "ArcBoxDag.jumpstart.local" + ",$sqlmi_port"
+$arcboxDag = "${namingPrefix}Dag.jumpstart.local" + ",$sqlmi_port"
 $sa_username = $env:AZDATA_USERNAME
 $sa_password = $AZDATA_PASSWORD
 
 $dagConnection = @"
 {
  "options": {
-      "connectionName": "ArcBoxDAG",
+      "connectionName": "${namingPrefix}DAG",
       "server": "$arcboxDag",
       "database": "",
       "authenticationType": "Integrated",
@@ -433,7 +433,7 @@ $sqlServerConnection = @"
         "database": "",
         "authenticationType": "SqlLogin",
         "user": "sa",
-        "password": "ArcDemo123!!",
+        "password": "JS123!!",
         "applicationName": "azdata",
         "groupId": "C777F06B-202E-4480-B475-FA416154D458",
         "databaseDisplayName": ""

azure_jumpstart_arcbox/artifacts/DevOpsLogonScript.ps1

@@ -18,7 +18,7 @@ $certdns = "arcbox.devops.com"
 $appClonedRepo = "https://github.com/$Env:githubUser/azure-arc-jumpstart-apps"
 
 $clusters = @(
-    [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-k3s-datasvc" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" }
+    [pscustomobject]@{clusterName = $Env:k3sArcDataClusterName; context = "$namingPrefix-k3s-data" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config" }
 
     [pscustomobject]@{clusterName = $Env:k3sArcClusterName; context = "$namingPrefix-k3s" ; kubeConfig = "C:\Users\$Env:adminUsername\.kube\config-k3s" }
 )
@@ -79,24 +79,18 @@ if(-not $($cliDir.Parent.Attributes.HasFlag([System.IO.FileAttributes]::Hidden))
 
 $Env:AZURE_CONFIG_DIR = $cliDir.FullName
 
-$Env:k3sArcDataClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s-Data" | Where-Object { $_ -ne "" }
-$Env:k3sArcDataClusterName=$Env:k3sArcDataClusterName -replace "`n",""
-
-$Env:k3sArcClusterName=(Get-AzResource -ResourceGroupName $Env:resourceGroup -ResourceType microsoft.kubernetes/connectedclusters).Name | Select-String "$namingPrefix-K3s" | Where-Object { $_ -ne "" }
-$Env:k3sArcClusterName=$Env:k3sArcClusterName -replace "`n",""
-
 # Required for CLI commands
 Write-Header "Az CLI Login"
 az login --identity
 az account set -s $env:subscriptionId
 
-# Downloading ArcBox-K3s-DataSvc Kubernetes cluster kubeconfig file
-Write-Header "Downloading $namingPrefix-K3s-DataSvc K8s Kubeconfig"
+# Downloading ArcBox-K3s-data Kubernetes cluster kubeconfig file
+Write-Header "Downloading $namingPrefix-K3s-data K8s Kubeconfig"
 $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/config"
 azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile  "C:\Users\$Env:USERNAME\.kube\config"
 
-# Downloading ArcBox-K3s-DataSvc log file
-Write-Header "Downloading $namingPrefix-K3s-DataSvc Install Logs"
+# Downloading ArcBox-K3s-data log file
+Write-Header "Downloading $namingPrefix-K3s-data Install Logs"
 $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($Env:k3sArcDataClusterName.ToLower())/*"
 $sourceFile = $sourceFile + "?" + $sas
 azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile  "$Env:ArcBoxLogsDir\" --include-pattern "*.log"
@@ -113,18 +107,6 @@ $sourceFile = "https://$Env:stagingStorageAccountName.blob.core.windows.net/$($E
 $sourceFile = $sourceFile + "?" + $sas
 azcopy cp --check-md5 FailIfDifferentOrMissing $sourceFile  "$Env:ArcBoxLogsDir\" --include-pattern "*.log"
 
-# # Merging kubeconfig files from ArcBox-K3s-DataSvc and ArcBox-K3s
-# Write-Header "Merging ArcBox-K3s-DataSvc & ArcBox-K3s Kubeconfigs"
-# Copy-Item -Path "C:\Users\$Env:USERNAME\.kube\config" -Destination "C:\Users\$Env:USERNAME\.kube\config.backup"
-# $Env:KUBECONFIG="C:\Users\$Env:USERNAME\.kube\config;C:\Users\$Env:USERNAME\.kube\config-k3s"
-# kubectl config view --raw > C:\users\$Env:USERNAME\.kube\config_tmp
-# kubectl config get-clusters --kubeconfig=C:\users\$Env:USERNAME\.kube\config_tmp
-# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config"
-# Remove-Item -Path "C:\Users\$Env:USERNAME\.kube\config-k3s"
-# Move-Item -Path "C:\Users\$Env:USERNAME\.kube\config_tmp" -Destination "C:\users\$Env:USERNAME\.kube\config"
-# $Env:KUBECONFIG="C:\users\$Env:USERNAME\.kube\config"
-# kubectx
-
 Write-Header "Adding Tools Folder to PATH"
 [System.Environment]::SetEnvironmentVariable('PATH', $Env:PATH + ";$Env:ToolsDir" ,[System.EnvironmentVariableTarget]::Machine)
 $Env:PATH += ";$Env:ToolsDir"
@@ -305,9 +287,9 @@ $kubeVipDaemonset | kubectl apply -f -
   Write-Host "`n"
 }
 
-# Switch Kubernetes context to ArcBox-K3s-DataSvc cluster
+# Switch Kubernetes context to ArcBox-K3s-data cluster
 foreach ($cluster in $clusters) {
-  if ($cluster.context -like '*-k3s-datasvc') {
+  if ($cluster.context -like '*-k3s-data') {
     $Env:KUBECONFIG=$cluster.kubeConfig
     kubectx
   }
@@ -414,49 +396,16 @@ foreach ($configName in $configs) {
 # # - Install Key Vault Extension / Create Ingress
 # ################################################
 
-# Write-Header "Installing KeyVault Extension"
-
-# Write-Host "Generating a TLS Certificate"
-# $cert = New-SelfSignedCertificate -DnsName $certdns -KeyAlgorithm RSA -KeyLength 2048 -NotAfter (Get-Date).AddYears(1) -CertStoreLocation "Cert:\CurrentUser\My"
-# $certPassword = ConvertTo-SecureString -String "arcbox" -Force -AsPlainText
-# Export-PfxCertificate -Cert "cert:\CurrentUser\My\$($cert.Thumbprint)" -FilePath "$Env:TempDir\$certname.pfx" -Password $certPassword
-# Import-PfxCertificate -FilePath "$Env:TempDir\$certname.pfx" -CertStoreLocation Cert:\LocalMachine\Root -Password $certPassword
-
-# Write-Host "Importing the TLS certificate to Key Vault"
-# az keyvault certificate import `
-#     --vault-name $Env:keyVaultName `
-#     --password "arcbox" `
-#     --name $certname `
-#     --file "$Env:TempDir\$certname.pfx"
-
-# Write-Host "Installing Azure Key Vault Kubernetes extension instance"
-# az k8s-extension create `
-#     --name 'akvsecretsprovider' `
-#     --extension-type Microsoft.AzureKeyVaultSecretsProvider `
-#     --scope cluster `
-#     --cluster-name $Env:k3sArcDataClusterName `
-#     --resource-group $Env:resourceGroup `
-#     --cluster-type connectedClusters `
-#     --release-namespace kube-system `
-#     --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true'
-
 # Replace Variable values
 Get-ChildItem -Path $Env:ArcBoxKVDir |
     ForEach-Object {
-        # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_CERTNAME}', $certname | Set-Content -Path $_.FullName
-        # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_KEYVAULTNAME}', $Env:keyVaultName | Set-Content -Path $_.FullName
         (Get-Content -path $_.FullName -Raw) -Replace '\{JS_HOST}', $certdns | Set-Content -Path $_.FullName
-        # (Get-Content -path $_.FullName -Raw) -Replace '\{JS_TENANTID}', $Env:tenantId | Set-Content -Path $_.FullName
     }
 
 Write-Header "Creating Ingress Controller"
 
 # Deploy Ingress resources for Bookstore and Hello-Arc App
 foreach ($namespace in @('bookstore', 'bookbuyer', 'hello-arc')) {
-    # Create the Kubernetes secret with the service principal credentials
-    # kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=$Env:spnClientID --from-literal clientsecret=$Env:spnClientSecret
-    # kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
-
     # Deploy Key Vault resources and Ingress for Book Store and Hello-Arc App
     kubectl --namespace $namespace apply -f "$Env:ArcBoxKVDir\$namespace.yaml"
 }

azure_jumpstart_arcbox/bicep/main.bicep

@@ -103,7 +103,7 @@ module ubuntuRancherK3sDataSvcDeployment 'kubernetes/ubuntuRancher.bicep' = if (
   }
 }
 
-module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'Full' || flavor == 'DataOps' || flavor == 'DevOps') {
+module ubuntuRancherK3sDataSvcNodesDeployment 'kubernetes/ubuntuRancherNodes.bicep' = [for i in range(0, k3sClusterNodesCount): if (flavor == 'DataOps' || flavor == 'DevOps') {
   name: 'ubuntuRancherK3sDataSvcNodesDeployment-${i}'
   params: {
     sshRSAPublicKey: sshRSAPublicKey
@@ -171,6 +171,8 @@ module clientVmDeployment 'clientVm/clientVm.bicep' = {
   }
   dependsOn: [
     updateVNetDNSServers
+    ubuntuRancherK3sDataSvcDeployment
+    ubuntuRancherK3sDeployment
   ]
 }