diff --git a/builder/azure/arm/builder.go b/builder/azure/arm/builder.go
index 79266c6a..24a07463 100644
--- a/builder/azure/arm/builder.go
+++ b/builder/azure/arm/builder.go
@@ -188,16 +188,6 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 	if b.config.DiskEncryptionSetId != "" {
 		b.stateBag.Put(constants.ArmBuildDiskEncryptionSetId, b.config.DiskEncryptionSetId)
 	}
-	sourceImageSpecialized := false
-	if b.config.SharedGallery.GalleryName != "" {
-		galleryImage, err := azureClient.GalleryImagesClient.Get(ctx, b.config.SharedGallery.ResourceGroup, b.config.SharedGallery.GalleryName, b.config.SharedGallery.ImageName)
-		if err != nil {
-			return nil, fmt.Errorf("the parent Shared Gallery Image '%s' from which to source the managed image version to does not exist in the resource group '%s' or does not contain managed image '%s'", b.config.SharedGallery.GalleryName, b.config.SharedGallery.ResourceGroup, b.config.SharedGallery.ImageName)
-		}
-		if galleryImage.OsState == compute.OperatingSystemStateTypesSpecialized {
-			sourceImageSpecialized = true
-		}
-	}
 	// Validate that Shared Gallery Image exists before publishing to SIG
 	if b.config.isPublishToSIG() {
 		_, err = azureClient.GalleryImagesClient.Get(ctx, b.config.SharedGalleryDestination.SigDestinationResourceGroup, b.config.SharedGalleryDestination.SigDestinationGalleryName, b.config.SharedGalleryDestination.SigDestinationImageName)
@@ -222,11 +212,22 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		}
 		b.stateBag.Put(constants.ArmManagedImageSharedGalleryReplicationRegions, b.config.SharedGalleryDestination.SigDestinationReplicationRegions)
 	}
-	generatedData := &packerbuilderdata.GeneratedData{State: b.stateBag}
+	sourceImageSpecialized := false
+	if b.config.SharedGallery.GalleryName != "" {
+		galleryImage, err := azureClient.GalleryImagesClient.Get(ctx, b.config.SharedGallery.ResourceGroup, b.config.SharedGallery.GalleryName, b.config.SharedGallery.ImageName)
+		if err != nil {
+			return nil, fmt.Errorf("the parent Shared Gallery Image '%s' from which to source the managed image version to does not exist in the resource group '%s' or does not contain managed image '%s'", b.config.SharedGallery.GalleryName, b.config.SharedGallery.ResourceGroup, b.config.SharedGallery.ImageName)
+		}
+		if galleryImage.OsState == compute.OperatingSystemStateTypesSpecialized {
+			sourceImageSpecialized = true
+		}
+	}
+
 	getVirtualMachineDeploymentFunction := GetVirtualMachineDeployment
 	if sourceImageSpecialized {
 		getVirtualMachineDeploymentFunction = GetSpecializedVirtualMachineDeployment
 	}
+	generatedData := &packerbuilderdata.GeneratedData{State: b.stateBag}
 	var steps []multistep.Step
 	if b.config.OSType == constants.Target_Linux {
 		steps = []multistep.Step{
diff --git a/builder/azure/arm/builder_acc_test.go b/builder/azure/arm/builder_acc_test.go
index 2ad1c645..c9208e47 100644
--- a/builder/azure/arm/builder_acc_test.go
+++ b/builder/azure/arm/builder_acc_test.go
@@ -10,7 +10,6 @@ package arm
 // * ARM_CLIENT_SECRET
 // * ARM_SUBSCRIPTION_ID
 // * ARM_STORAGE_ACCOUNT
-// * AZURE_SSH_PASS - In our Linux SIG test we build two specialized images, this requires setting a shared password for both builds
 // * ARM_SSH_PRIVATE_KEY_FILE - the file location of a PEM encoded SSH private key,
 //
 // The subscription in question should have a resource group
@@ -52,13 +51,7 @@ const DeviceLoginAcceptanceTest = "DEVICELOGIN_TEST"
 func TestBuilderAcc_SharedImageGallery_ARM64SpecializedLinuxSIG_WithChildImage(t *testing.T) {
 	t.Parallel()
 	if os.Getenv("AZURE_CLI_AUTH") == "" {
-		t.Skip("Azure CLI Acceptance tests skipped unless env 'AZURE_CLI_AUTH' is set, and an active `az login` session has been established")
-		return
-	}
-
-	// A password is required to be shared between builds as Specialized Images do not have their main user profile removed
-	if os.Getenv("ARM_SSH_PASS") == "" {
-		t.Fatalf("To run this test set a valid ssh password in the env variable ARM_SSH_PASS")
+		t.Fatalf("Azure CLI Acceptance tests require 'AZURE_CLI_AUTH' is set, and an active `az login` session has been established")
 		return
 	}
 
@@ -117,7 +110,7 @@ func TestBuilderAcc_SharedImageGallery_ARM64SpecializedLinuxSIG_WithChildImage(t
 func TestBuilderAcc_SharedImageGallery_WindowsSIG(t *testing.T) {
 	t.Parallel()
 	if os.Getenv("AZURE_CLI_AUTH") == "" {
-		t.Skip("Azure CLI Acceptance tests skipped unless env 'AZURE_CLI_AUTH' is set, and an active `az login` session has been established")
+		t.Fatalf("Azure CLI Acceptance tests require 'AZURE_CLI_AUTH' is set, and an active `az login` session has been established")
 		return
 	}
 
@@ -262,7 +255,7 @@ func TestBuilderAcc_ManagedDisk_Linux_DeviceLogin(t *testing.T) {
 func TestBuilderAcc_ManagedDisk_Linux_AzureCLI(t *testing.T) {
 	t.Parallel()
 	if os.Getenv("AZURE_CLI_AUTH") == "" {
-		t.Skip("Azure CLI Acceptance tests skipped unless env 'AZURE_CLI_AUTH' is set, and an active `az login` session has been established")
+		t.Fatalf("Azure CLI Acceptance tests require 'AZURE_CLI_AUTH' is set, and an active `az login` session has been established")
 		return
 	}
 
diff --git a/builder/azure/arm/testdata/arm_linux_specialized.pkr.hcl b/builder/azure/arm/testdata/arm_linux_specialized.pkr.hcl
index 6cf6bdee..3be91c29 100644
--- a/builder/azure/arm/testdata/arm_linux_specialized.pkr.hcl
+++ b/builder/azure/arm/testdata/arm_linux_specialized.pkr.hcl
@@ -2,12 +2,6 @@
 # SPDX-License-Identifier: MPL-2.0
 locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
 
-variable "ssh_password" {
-  type = string
-  sensitive = true
-  default = "${env("ARM_SSH_PASS")}"
-}
-
 variable "ssh_private_key_location" {
   default = "${env("ARM_SSH_PRIVATE_KEY_FILE")}"
   type = string
@@ -21,7 +15,6 @@ source "azure-arm" "linux-sig" {
   location           = "South Central US"
   vm_size            = "Standard_D4ps_v5"
   ssh_username       = "packer"
-  ssh_password       = var.ssh_password
   ssh_private_key_file = var.ssh_private_key_location
   communicator       = "ssh"
   shared_image_gallery_destination {
diff --git a/builder/azure/arm/testdata/child_from_specialized_parent.pkr.hcl b/builder/azure/arm/testdata/child_from_specialized_parent.pkr.hcl
index 7f0fe4d7..9d8c742b 100644
--- a/builder/azure/arm/testdata/child_from_specialized_parent.pkr.hcl
+++ b/builder/azure/arm/testdata/child_from_specialized_parent.pkr.hcl
@@ -2,12 +2,6 @@
 # SPDX-License-Identifier: MPL-2.0
 locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
 
-variable "ssh_password" {
-  type = string
-  default = "${env("ARM_SSH_PASS")}"
-  sensitive = true
-}
-
 variable "subscription" {
   default = "${env("ARM_SUBSCRIPTION_ID")}"
   type = string
@@ -24,7 +18,6 @@ source "azure-arm" "linux-sig" {
   location           = "South Central US"
   vm_size            = "Standard_D4ps_v5"
   ssh_username       = "packer"
-  ssh_password       = var.ssh_password
   ssh_private_key_file = var.ssh_private_key_location
   communicator       = "ssh"
   shared_image_gallery{