From 4475d655dd7d6afe3bec8a131dc356f48d2fa017 Mon Sep 17 00:00:00 2001 From: Jenna Goldstrich Date: Fri, 19 Apr 2024 11:51:53 -0700 Subject: [PATCH] refactor step deploy template for easier unit testing --- builder/azure/arm/step_deploy_template.go | 340 +++++++++++++++------- 1 file changed, 228 insertions(+), 112 deletions(-) diff --git a/builder/azure/arm/step_deploy_template.go b/builder/azure/arm/step_deploy_template.go index 93ec31fb..5867c3cb 100644 --- a/builder/azure/arm/step_deploy_template.go +++ b/builder/azure/arm/step_deploy_template.go @@ -31,21 +31,27 @@ type DeploymentTemplateType int const ( VirtualMachineTemplate DeploymentTemplateType = iota KeyVaultTemplate + VMResourceType = "Microsoft.Compute/virtualMachines" + NetworkInterfaceResourceType = "Microsoft.Network/networkInterfaces" + KeyVaultResourceType = "Microsoft.KeyVault/vaults" ) type StepDeployTemplate struct { - client *AzureClient - deploy func(ctx context.Context, subscriptionId string, resourceGroupName string, deploymentName string) error - delete func(ctx context.Context, subscriptionId, deploymentName, resourceGroupName string) error - disk func(ctx context.Context, subscriptionId string, resourceGroupName string, computeName string) (string, string, error) - deleteDisk func(ctx context.Context, imageName string, resourceGroupName string, isManagedDisk bool, subscriptionId string, storageAccountName string) error - deleteDeployment func(ctx context.Context, state multistep.StateBag) error - say func(message string) - error func(e error) - config *Config - factory templateFactoryFunc - name string - templateType DeploymentTemplateType + client *AzureClient + deploy func(ctx context.Context, subscriptionId string, resourceGroupName string, deploymentName string) error + deleteNetworkResources func(ctx context.Context, subscriptionId string, resourceGroupName string, resources map[string]string) error + getDisk func(ctx context.Context, subscriptionId string, resourceGroupName string, computeName string) (string, string, error) + deleteDisk func(ctx context.Context, imageName string, resourceGroupName string, isManagedDisk bool, subscriptionId string, storageAccountName string) error + deleteVM func(ctx context.Context, virtualMachineId virtualmachines.VirtualMachineId) error + deleteNic func(ctx context.Context, networkInterfacesId commonids.NetworkInterfaceId) error + deleteDeployment func(ctx context.Context, state multistep.StateBag) error + deleteKV func(ctx context.Context, id commonids.KeyVaultId) error + say func(message string) + error func(e error) + config *Config + factory templateFactoryFunc + name string + templateType DeploymentTemplateType } func NewStepDeployTemplate(client *AzureClient, ui packersdk.Ui, config *Config, deploymentName string, factory templateFactoryFunc, templateType DeploymentTemplateType) *StepDeployTemplate { @@ -60,10 +66,13 @@ func NewStepDeployTemplate(client *AzureClient, ui packersdk.Ui, config *Config, } step.deploy = step.deployTemplate - step.delete = step.deleteDeploymentResources - step.disk = step.getImageDetails + step.getDisk = step.getImageDetails step.deleteDisk = step.deleteImage + step.deleteNetworkResources = step.deleteNetworkResourcesWithQueue step.deleteDeployment = step.deleteDeploymentObject + step.deleteNic = step.deleteNetworkInterface + step.deleteVM = step.deleteVirtualMachine + step.deleteKV = step.deleteKeyVault return step } @@ -94,11 +103,38 @@ func (s *StepDeployTemplate) Cleanup(state multistep.StateBag) { deploymentName := s.name resourceGroupName := state.Get(constants.ArmResourceGroupName).(string) subscriptionId := state.Get(constants.ArmSubscription).(string) + deploymentOpsId := deploymentoperations.ResourceGroupDeploymentId{ + DeploymentName: deploymentName, + ResourceGroupName: resourceGroupName, + SubscriptionId: subscriptionId, + } if s.templateType == KeyVaultTemplate { ui.Say("\nDeleting KeyVault created during build") - err := s.delete(ctx, subscriptionId, deploymentName, resourceGroupName) + deploymentOperations, err := s.listDeploymentOperations(ctx, deploymentOpsId) if err != nil { - s.reportIfError(err, resourceGroupName) + ui.Error(fmt.Sprintf("Could not retrieve deployment operations: %s\n to get the KeyVault, please manually delete it ", err)) + return + } + for _, deploymentOperation := range deploymentOperations { + // Sometimes an empty operation is added to the list by Azure + if deploymentOperation.Properties.TargetResource == nil { + continue + } + resourceName := *deploymentOperation.Properties.TargetResource.ResourceName + resourceType := *deploymentOperation.Properties.TargetResource.ResourceType + + if resourceType == KeyVaultResourceType { + kvID := commonids.KeyVaultId{ + VaultName: resourceName, + ResourceGroupName: resourceGroupName, + SubscriptionId: subscriptionId, + } + s.deleteKV(ctx, kvID) + if err != nil { + s.reportResourceDeletionFailure(err, resourceGroupName) + } + return + } } } else { @@ -109,13 +145,55 @@ func (s *StepDeployTemplate) Cleanup(state multistep.StateBag) { isManagedDisk := state.Get(constants.ArmIsManagedImage).(bool) isSIGImage := state.Get(constants.ArmIsSIGImage).(bool) armStorageAccountName := state.Get(constants.ArmStorageAccountName).(string) - imageType, imageName, err := s.disk(ctx, subscriptionId, resourceGroupName, computeName) + imageType, imageName, err := s.getDisk(ctx, subscriptionId, resourceGroupName, computeName) if err != nil { ui.Error(fmt.Sprintf("Could not retrieve OS Image details: %s", err)) } - err = s.delete(ctx, subscriptionId, deploymentName, resourceGroupName) + + deploymentOperations, err := s.listDeploymentOperations(ctx, deploymentOpsId) + if err != nil { + ui.Error(fmt.Sprintf("Could not retrieve deployment operations: %s\n Virtual Machine %s, and its please manually delete it and its ascoiated resources", err, computeName)) + return + } + resources := map[string]string{} + var vmID *virtualmachines.VirtualMachineId + var networkInterfaceID *commonids.NetworkInterfaceId + for _, deploymentOperation := range deploymentOperations { + // Sometimes an empty operation is added to the list by Azure + if deploymentOperation.Properties.TargetResource == nil { + continue + } + resourceName := *deploymentOperation.Properties.TargetResource.ResourceName + resourceType := *deploymentOperation.Properties.TargetResource.ResourceType + + // Create vm and NIC ID to use them to delete later + switch resourceType { + case "Microsoft.Compute/virtualMachines": + vmIDDeref := virtualmachines.NewVirtualMachineID(subscriptionId, resourceGroupName, resourceName) + vmID = &vmIDDeref + case "Microsoft.Network/networkInterfaces": + networkInterfaceIDDeref := commonids.NewNetworkInterfaceID(subscriptionId, resourceGroupName, resourceName) + networkInterfaceID = &networkInterfaceIDDeref + default: + resources[resourceType] = resourceName + } + } + + if vmID != nil { + err := s.deleteVirtualMachine(ctx, *vmID) + if err != nil { + return + } + } + if networkInterfaceID != nil { + err := s.deleteNetworkInterface(ctx, *networkInterfaceID) + if err != nil { + return + } + } + err = s.deleteNetworkResources(ctx, subscriptionId, resourceGroupName, resources) if err != nil { - s.reportIfError(err, resourceGroupName) + s.reportResourceDeletionFailure(err, resourceGroupName) } // The disk was not found on the VM, this is an error. if imageType == "" && imageName == "" { @@ -125,12 +203,12 @@ func (s *StepDeployTemplate) Cleanup(state multistep.StateBag) { return } if !state.Get(constants.ArmKeepOSDisk).(bool) { - ui.Say(fmt.Sprintf(" Deleting -> %s : '%s'", imageType, imageName)) err = s.deleteDisk(ctx, imageName, resourceGroupName, (isManagedDisk || isSIGImage), subscriptionId, armStorageAccountName) if err != nil { - ui.Error(fmt.Sprintf("Error deleting resource. Please delete manually.\n\n"+ - "Name: %s\n"+ - "Error: %s", imageName, err)) + s.reportResourceDeletionFailure(err, imageName) + + } else { + ui.Say(fmt.Sprintf("Deleted -> %s : '%s'", imageType, imageName)) } } else { ui.Say(fmt.Sprintf("Skipping deletion -> %s : '%s' since 'keep_os_disk' is set to true", imageType, imageName)) @@ -140,11 +218,13 @@ func (s *StepDeployTemplate) Cleanup(state multistep.StateBag) { dataDisks = disks.([]string) } for i, additionaldisk := range dataDisks { - s.say(fmt.Sprintf(" Deleting Additional Disk -> %d: '%s'", i+1, additionaldisk)) err := s.deleteImage(ctx, additionaldisk, resourceGroupName, (isManagedDisk || isSIGImage), subscriptionId, armStorageAccountName) if err != nil { - s.say(fmt.Sprintf("Failed to delete the managed Additional Disk! %s", err)) + s.say(fmt.Sprintf(" Deleted Additional Disk -> %d: '%s'", i+1, additionaldisk)) + } + if err != nil { + s.reportResourceDeletionFailure(err, additionaldisk) } } @@ -192,9 +272,6 @@ func (s *StepDeployTemplate) getImageDetails(ctx context.Context, subscriptionId defer cancel() vmID := virtualmachines.NewVirtualMachineID(subscriptionId, resourceGroupName, computeName) vm, err := s.client.VirtualMachinesClient.Get(pollingContext, vmID, virtualmachines.DefaultGetOperationOptions()) - if err != nil { - return imageName, imageType, err - } if err != nil { s.say(s.client.LastError.Error()) return "", "", err @@ -218,29 +295,6 @@ func (s *StepDeployTemplate) getImageDetails(ctx context.Context, subscriptionId return imageType, imageName, nil } -func deleteResource(ctx context.Context, client *AzureClient, subscriptionId string, resourceType string, resourceName string, resourceGroupName string) error { - - pollingContext, cancel := context.WithTimeout(ctx, client.PollingDuration) - defer cancel() - - var err error - switch resourceType { - case "Microsoft.KeyVault/vaults": - id := commonids.NewKeyVaultID(subscriptionId, resourceGroupName, resourceName) - _, err = client.VaultsClient.Delete(pollingContext, id) - case "Microsoft.Network/virtualNetworks": - vnetID := commonids.NewVirtualNetworkID(subscriptionId, resourceGroupName, resourceName) - err = client.NetworkMetaClient.VirtualNetworks.DeleteThenPoll(pollingContext, vnetID) - case "Microsoft.Network/networkSecurityGroups": - secGroupId := networksecuritygroups.NewNetworkSecurityGroupID(subscriptionId, resourceGroupName, resourceName) - err = client.NetworkMetaClient.NetworkSecurityGroups.DeleteThenPoll(pollingContext, secGroupId) - case "Microsoft.Network/publicIPAddresses": - ipID := commonids.NewPublicIPAddressID(subscriptionId, resourceGroupName, resourceName) - err = client.NetworkMetaClient.PublicIPAddresses.DeleteThenPoll(pollingContext, ipID) - } - return err -} - // TODO Let's split this into two separate methods // deleteVHD and deleteManagedDisk, and then just check in Cleanup which function to call func (s *StepDeployTemplate) deleteImage(ctx context.Context, imageName string, resourceGroupName string, isManagedDisk bool, subscriptionId string, storageAccountName string) error { @@ -272,62 +326,111 @@ func (s *StepDeployTemplate) deleteImage(ctx context.Context, imageName string, return err } -func (s *StepDeployTemplate) deleteDeploymentResources(ctx context.Context, subscriptionId, deploymentName, resourceGroupName string) error { +func (s *StepDeployTemplate) deleteVirtualMachine(ctx context.Context, vmID virtualmachines.VirtualMachineId) error { + retryConfig := retry.Config{ + Tries: 5, + RetryDelay: (&retry.Backoff{ + InitialBackoff: 3 * time.Second, + MaxBackoff: 15 * time.Second, + Multiplier: 1.5, + }).Linear, + } + err := retryConfig.Run(ctx, func(ctx context.Context) error { + log.Printf("[INFO] Attempting deletion -> %s : %s", VMResourceType, vmID.VirtualMachineName) + pollingContext, cancel := context.WithTimeout(ctx, s.client.PollingDuration) + defer cancel() + err := s.client.VirtualMachinesClient.DeleteThenPoll(pollingContext, vmID, virtualmachines.DefaultDeleteOperationOptions()) + if err != nil { + log.Printf("[INFO] Couldn't delete resource %s.%s, will retry", VMResourceType, vmID.VirtualMachineName) + } else { + s.say(fmt.Sprintf("Deleted -> %s : '%s'", VMResourceType, vmID.VirtualMachineName)) + } + return err + }) + if err != nil { + s.reportResourceDeletionFailure(err, vmID.VirtualMachineName) + } + return err +} + +func (s *StepDeployTemplate) deleteKeyVault(ctx context.Context, id commonids.KeyVaultId) error { + retryConfig := retry.Config{ + Tries: 5, + RetryDelay: (&retry.Backoff{ + InitialBackoff: 3 * time.Second, + MaxBackoff: 15 * time.Second, + Multiplier: 1.5, + }).Linear, + } + + err := retryConfig.Run(ctx, func(ctx context.Context) error { + pollingContext, cancel := context.WithTimeout(ctx, s.client.PollingDuration) + defer cancel() + log.Printf("[INFO] Attempting deletion -> %s : %s", KeyVaultResourceType, id.VaultName) + _, err := s.client.VaultsClient.Delete(pollingContext, id) + if err != nil { + log.Printf("[INFO] Couldn't delete resource %s.%s, will retry", KeyVaultResourceType, id.VaultName) + } else { + s.say(fmt.Sprintf("Deleted -> %s : '%s'", KeyVaultResourceType, id.VaultName)) + } + return err + }) + if err != nil { + s.reportResourceDeletionFailure(err, id.VaultName) + } + + return err +} + +func (s *StepDeployTemplate) deleteNetworkInterface(ctx context.Context, networkInterfaceID commonids.NetworkInterfaceId) error { + retryConfig := retry.Config{ + Tries: 5, + RetryDelay: (&retry.Backoff{ + InitialBackoff: 3 * time.Second, + MaxBackoff: 15 * time.Second, + Multiplier: 1.5, + }).Linear, + } + err := retryConfig.Run(ctx, func(ctx context.Context) error { + log.Printf("[INFO] Attempting deletion -> %s : %s", NetworkInterfaceResourceType, networkInterfaceID.NetworkInterfaceName) + pollingContext, cancel := context.WithTimeout(ctx, s.client.PollingDuration) + defer cancel() + err := s.client.NetworkMetaClient.NetworkInterfaces.DeleteThenPoll(pollingContext, networkInterfaceID) + if err != nil { + log.Printf("[WARN] Couldn't delete resource %s.%s, will retry", NetworkInterfaceResourceType, networkInterfaceID.NetworkInterfaceName) + } else { + s.say(fmt.Sprintf("Deleted -> %s : '%s'", NetworkInterfaceResourceType, networkInterfaceID.NetworkInterfaceName)) + } + return err + }) + if err != nil { + s.reportResourceDeletionFailure(err, networkInterfaceID.NetworkInterfaceName) + } + return err +} + +func (s *StepDeployTemplate) listDeploymentOperations(ctx context.Context, id deploymentoperations.ResourceGroupDeploymentId) ([]deploymentoperations.DeploymentOperation, error) { var maxResources int64 = 50 options := deploymentoperations.DefaultListOperationOptions() options.Top = &maxResources - id := deploymentoperations.NewResourceGroupDeploymentID(subscriptionId, resourceGroupName, deploymentName) pollingContext, cancel := context.WithTimeout(ctx, s.client.PollingDuration) defer cancel() deploymentOperations, err := s.client.DeploymentOperationsClient.ListComplete(pollingContext, id, options) if err != nil { - s.reportIfError(err, resourceGroupName) - return err - } - - resources := map[string]string{} - var vmID *virtualmachines.VirtualMachineId - var nicID *commonids.NetworkInterfaceId - for _, deploymentOperation := range deploymentOperations.Items { - // Sometimes an empty operation is added to the list by Azure - if deploymentOperation.Properties.TargetResource == nil { - continue - } - resourceName := *deploymentOperation.Properties.TargetResource.ResourceName - resourceType := *deploymentOperation.Properties.TargetResource.ResourceType - - // Create vm and NIC ID to use them to delete later - switch resourceType { - case "Microsoft.Compute/virtualMachines": - vmIDDeref := virtualmachines.NewVirtualMachineID(subscriptionId, resourceGroupName, resourceName) - vmID = &vmIDDeref - case "Microsoft.Network/networkInterfaces": - nicIDDeref := commonids.NewNetworkInterfaceID(subscriptionId, resourceGroupName, resourceName) - nicID = &nicIDDeref - default: - s.say(fmt.Sprintf("Adding to deletion queue -> %s : '%s'", resourceType, resourceName)) - resources[resourceType] = resourceName - } + return nil, err } + return deploymentOperations.Items, nil +} - // Always delete VM first, then delete NIC, this way we avoid conflicts on deletion - if vmID != nil { - s.say(fmt.Sprintf("Attempting deletion -> %s : '%s'", "Microsoft.Compute/virtualMachines", vmID.VirtualMachineName)) - if err := s.client.VirtualMachinesClient.DeleteThenPoll(pollingContext, *vmID, virtualmachines.DefaultDeleteOperationOptions()); err != nil { - s.reportIfError(err, vmID.VirtualMachineName) - return err - } - s.say(fmt.Sprintf("Finished deleting -> %s : '%s'", "Microsoft.Compute/virtualMachines", vmID.VirtualMachineName)) - } - if nicID != nil { - s.say(fmt.Sprintf("Attempting deletion -> %s : '%s'", "Microsoft.Network/networkInterfaces", nicID.NetworkInterfaceName)) - err := s.client.NetworkMetaClient.NetworkInterfaces.DeleteThenPoll(pollingContext, *nicID) - if err != nil { - s.reportIfError(err, nicID.NetworkInterfaceName) - return err - } - s.say(fmt.Sprintf("Finished deleting -> %s : '%s'", "Microsoft.Network/networkInterfaces", nicID.NetworkInterfaceName)) +func (s *StepDeployTemplate) deleteNetworkResourcesWithQueue(ctx context.Context, subscriptionId string, resourceGroupName string, resources map[string]string) error { + retryConfig := retry.Config{ + Tries: 5, + RetryDelay: (&retry.Backoff{ + InitialBackoff: 3 * time.Second, + MaxBackoff: 15 * time.Second, + Multiplier: 1.5, + }).Linear, } var wg sync.WaitGroup wg.Add(len(resources)) @@ -335,13 +438,9 @@ func (s *StepDeployTemplate) deleteDeploymentResources(ctx context.Context, subs for resourceType, resourceName := range resources { go func(resourceType, resourceName string) { defer wg.Done() - retryConfig := retry.Config{ - Tries: 10, - RetryDelay: (&retry.Backoff{InitialBackoff: 5 * time.Second, MaxBackoff: 60 * time.Second, Multiplier: 1.5}).Linear, - } - err = retryConfig.Run(ctx, func(ctx context.Context) error { - s.say(fmt.Sprintf("Attempting deletion -> %s : '%s'", resourceType, resourceName)) + err := retryConfig.Run(ctx, func(ctx context.Context) error { + log.Printf("[INFO} Attempting deletion -> %s : '%s'", resourceType, resourceName) err := deleteResource(ctx, s.client, subscriptionId, resourceType, @@ -350,27 +449,44 @@ func (s *StepDeployTemplate) deleteDeploymentResources(ctx context.Context, subs if err != nil { log.Printf("[INFO] Couldn't delete resource %s.%s, will retry", resourceType, resourceName) } else { - s.say(fmt.Sprintf("Finished deleting -> %s : '%s'", resourceType, resourceName)) + s.say(fmt.Sprintf("Deleted -> %s : '%s'", resourceType, resourceName)) } return err }) if err != nil { - s.reportIfError(err, resourceName) + s.reportResourceDeletionFailure(err, resourceName) } }(resourceType, resourceName) } - s.say("Waiting for deletion of all resources...") wg.Wait() return nil } -func (s *StepDeployTemplate) reportIfError(err error, resourceName string) { - if err != nil { - s.say(fmt.Sprintf("Error deleting resource. Please delete manually.\n\n"+ - "Name: %s\n"+ - "Error: %s", resourceName, err.Error())) - s.error(err) +func deleteResource(ctx context.Context, client *AzureClient, subscriptionId string, resourceType string, resourceName string, resourceGroupName string) error { + + pollingContext, cancel := context.WithTimeout(ctx, client.PollingDuration) + defer cancel() + + var err error + switch resourceType { + case "Microsoft.Network/virtualNetworks": + vnetID := commonids.NewVirtualNetworkID(subscriptionId, resourceGroupName, resourceName) + err = client.NetworkMetaClient.VirtualNetworks.DeleteThenPoll(pollingContext, vnetID) + case "Microsoft.Network/networkSecurityGroups": + secGroupId := networksecuritygroups.NewNetworkSecurityGroupID(subscriptionId, resourceGroupName, resourceName) + err = client.NetworkMetaClient.NetworkSecurityGroups.DeleteThenPoll(pollingContext, secGroupId) + case "Microsoft.Network/publicIPAddresses": + ipID := commonids.NewPublicIPAddressID(subscriptionId, resourceGroupName, resourceName) + err = client.NetworkMetaClient.PublicIPAddresses.DeleteThenPoll(pollingContext, ipID) } + return err +} + +func (s *StepDeployTemplate) reportResourceDeletionFailure(err error, resourceName string) { + s.say(fmt.Sprintf("Error deleting resource. Please delete manually.\n\n"+ + "Name: %s\n"+ + "Error: %s", resourceName, err.Error())) + s.error(err) }