Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix BackupVaults/BackupInstance extension to exit early on terminal states #4180

Merged
merged 5 commits into from
Aug 15, 2024
Merged

Fix BackupVaults/BackupInstance extension to exit early on terminal states #4180

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b2c1a18
Fix BackupVaults/BackupInstance extension to exit early on terminal s…
super-harsh Aug 6, 2024
153b792
Update test recording and minor changes
super-harsh Aug 8, 2024
9fe0fda
Update recording
super-harsh Aug 9, 2024
b8d3e9a
Update comment with swagger link
super-harsh Aug 14, 2024
084676c
Merge branch 'main' into fix/backup-vault-extension
super-harsh Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 64 additions & 46 deletions v2/api/dataprotection/customizations/backup_vaults_backup_instance_extensions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,28 @@ import (
"github.com/Azure/azure-service-operator/v2/internal/genericarmclient"
. "github.com/Azure/azure-service-operator/v2/internal/logging"
"github.com/Azure/azure-service-operator/v2/internal/resolver"
"github.com/Azure/azure-service-operator/v2/internal/set"
"github.com/Azure/azure-service-operator/v2/internal/util/to"
"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
"github.com/Azure/azure-service-operator/v2/pkg/genruntime/extensions"
)

var _ extensions.PostReconciliationChecker = &BackupVaultsBackupInstanceExtension{}

var protectionError = "ProtectionError"
// These are the states on which there's no future change possible, hence best to return PostReconcileCheck success on them.
// Then further we let Operator decide what condition to put on the resource.
var terminalStates = set.Make(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add some documentation about how we determined this list, and what exactly it means to be terminal here?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are the states listed and we do no foresee any frequent changed here as these states are not usually changed.

Swagger: https://github.com/Azure/azure-rest-api-specs/blob/a651ba25cda4eec698a3a4e35f867ecc2681d126/specification/dataprotection/resource-manager/Microsoft.DataProtection/stable/2023-11-01/dataprotection.json#L5014

Public Doc: (might not be updated with latest swagger)
https://learn.microsoft.com/en-us/rest/api/dataprotection/backup-instances/get?view=rest-dataprotection-2024-04-01&tabs=HTTP#status

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding these URLs as comments would be useful, so that a future maintainer can go verify if things change in the future.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

"configuringprotectionfailed",
"invalid",
"notprotected",
"protectionconfigured",
"softdeleted",
"protectionstopped",
"backupschedulessuspended",
"retentionschedulessuspended",
)

var protectionError = "protectionerror"

const (
BackupInstancePollerResumeTokenAnnotation = "serviceoperator.azure.com/bi-poller-resume-token"
Expand Down Expand Up @@ -80,69 +94,73 @@ func (extension *BackupVaultsBackupInstanceExtension) PostReconcileCheck(
}

protectionStatus := *backupInstance.Status.Properties.ProtectionStatus.Status
protectionStatus = strings.ToLower(protectionStatus)
log.V(Debug).Info(fmt.Sprintf("Protection Status is %q", protectionStatus))

// We only want to continue if protectionStatus == ProtectionError.
if !strings.EqualFold(protectionStatus, protectionError) {
// Return success if the status is in a terminal state
if terminalStates.Contains(protectionStatus) {
log.V(Debug).Info("Returning PostReconcileCheckResultSuccess")
matthchr marked this conversation as resolved.
Show resolved Hide resolved
return next(ctx, obj, owner, resolver, armClient, log)
}

// call sync api only when protection status is ProtectionError and error code is usererror
var protectionStatusErrorCode string
protectionStatusErrorCode = strings.ToLower(*backupInstance.Status.Properties.ProtectionStatus.ErrorDetails.Code)
log.V(Debug).Info(fmt.Sprintf("Protection Error code is %q", protectionStatusErrorCode))
if protectionStatus == protectionError {
// call sync api only when protection status is ProtectionError and error code is usererror
var protectionStatusErrorCode string
protectionStatusErrorCode = strings.ToLower(*backupInstance.Status.Properties.ProtectionStatus.ErrorDetails.Code)
log.V(Debug).Info(fmt.Sprintf("Protection Error code is %q", protectionStatusErrorCode))

if protectionStatusErrorCode != "" && strings.Contains(protectionStatusErrorCode, "usererror") {
id, _ := genruntime.GetAndParseResourceID(backupInstance)
subscription := id.SubscriptionID
rg := id.ResourceGroupName
vaultName := id.Parent.Name
if strings.Contains(protectionStatusErrorCode, "usererror") {
id, _ := genruntime.GetAndParseResourceID(backupInstance)
subscription := id.SubscriptionID
rg := id.ResourceGroupName
vaultName := id.Parent.Name

clientFactory, err := armdataprotection.NewClientFactory(subscription, armClient.Creds(), armClient.ClientOptions())
if err != nil {
return extensions.PostReconcileCheckResultFailure("failed to create armdataprotection client"), err
}
clientFactory, err := armdataprotection.NewClientFactory(subscription, armClient.Creds(), armClient.ClientOptions())
if err != nil {
return extensions.PostReconcileCheckResultFailure("failed to create armdataprotection client"), err
}

var parameters armdataprotection.SyncBackupInstanceRequest
parameters.SyncType = to.Ptr(armdataprotection.SyncTypeDefault)
var parameters armdataprotection.SyncBackupInstanceRequest
parameters.SyncType = to.Ptr(armdataprotection.SyncTypeDefault)

// get the resume token from the resource
pollerResumeToken, _ := GetPollerResumeToken(obj, log)
// get the resume token from the resource
pollerResumeToken, _ := GetPollerResumeToken(obj, log)

// BeginSyncBackupInstance is in-progress - poller resume token is available
log.V(Debug).Info("Starting BeginSyncBackupInstance")
// BeginSyncBackupInstance is in-progress - poller resume token is available
log.V(Debug).Info("Starting BeginSyncBackupInstance")

poller, err := clientFactory.NewBackupInstancesClient().BeginSyncBackupInstance(ctx, rg, vaultName, backupInstance.AzureName(), parameters, &armdataprotection.BackupInstancesClientBeginSyncBackupInstanceOptions{
ResumeToken: pollerResumeToken,
})
if err != nil {
return extensions.PostReconcileCheckResultFailure("Failed Polling for BeginSyncBackupInstance to get the result"), err
}
poller, err := clientFactory.NewBackupInstancesClient().BeginSyncBackupInstance(ctx, rg, vaultName, backupInstance.AzureName(), parameters, &armdataprotection.BackupInstancesClientBeginSyncBackupInstanceOptions{
ResumeToken: pollerResumeToken,
})
if err != nil {
return extensions.PostReconcileCheckResultFailure("Failed Polling for BeginSyncBackupInstance to get the result"), err
}

if pollerResumeToken == "" {
resumeToken, resumeTokenErr := poller.ResumeToken()
if resumeTokenErr != nil {
return extensions.PostReconcileCheckResultFailure("couldn't create PUT resume token for resource"), resumeTokenErr
} else {
SetPollerResumeToken(obj, resumeToken, log)
if pollerResumeToken == "" {
resumeToken, resumeTokenErr := poller.ResumeToken()
if resumeTokenErr != nil {
return extensions.PostReconcileCheckResultFailure("couldn't create PUT resume token for resource"), resumeTokenErr
} else {
SetPollerResumeToken(obj, resumeToken, log)
}
}
}

_, pollErr := poller.Poll(ctx)
if pollErr != nil {
return extensions.PostReconcileCheckResultFailure("couldn't create PUT resume token for resource"), pollErr
}
_, pollErr := poller.Poll(ctx)
if pollErr != nil {
return extensions.PostReconcileCheckResultFailure("couldn't create PUT resume token for resource"), pollErr
}

if poller.Done() {
log.V(Debug).Info("Polling is completed")
ClearPollerResumeToken(obj, log)
_, err := poller.Result(ctx)
if err != nil {
return extensions.PostReconcileCheckResultFailure("couldn't create PUT resume token for resource"), err
if poller.Done() {
log.V(Debug).Info("Polling is completed")
ClearPollerResumeToken(obj, log)
_, err := poller.Result(ctx)
if err != nil {
return extensions.PostReconcileCheckResultFailure("couldn't create PUT resume token for resource"), err
}
}
log.V(Debug).Info("Polling is in-progress")
}
log.V(Debug).Info("Polling is in-progress")
}

return extensions.PostReconcileCheckResultFailure("Backup Instance is in non terminal state"), nil
}
6 changes: 4 additions & 2 deletions v2/internal/controllers/crd_dataprotection_backupinstance_20231101_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func Test_DataProtection_BackupInstance_20231101_CRUD(t *testing.T) {
// Create a test resource group and wait until the operation is completed, where the globalTestContext is a global object that provides the necessary context and utilities for testing.
tc := globalTestContext.ForTest(t)
rg := tc.CreateTestResourceGroupAndWait()
contributorRoleId := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/17d1049b-9a84-46fb-8f53-869881c3d3ab", tc.AzureSubscription)
contributorRoleId := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c", tc.AzureSubscription)
super-harsh marked this conversation as resolved.
Show resolved Hide resolved
readerRoleId := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7", tc.AzureSubscription)
saContributorRoleId := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/17d1049b-9a84-46fb-8f53-869881c3d3ab", tc.AzureSubscription)

Expand All @@ -56,11 +56,13 @@ func Test_DataProtection_BackupInstance_20231101_CRUD(t *testing.T) {
},
}

backupPolicy := newBackupPolicy20231101(tc, backupVault, "asotestbackuppolicy")
backupPolicy := newBackupPolicy20231101(tc, backupVault, "asotestbp")

// create storage account and blob container

acct := newStorageAccount(tc, rg)
// Not allowed by the policy anymore. Will need to update the storage account and respective tests
acct.Spec.AllowBlobPublicAccess = to.Ptr(false)

blobService := &storage.StorageAccountsBlobService{
ObjectMeta: tc.MakeObjectMeta("blobservice"),
Expand Down
Loading