diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 71c55f0446c..955bf0b6e11 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -305,7 +305,6 @@ jobs: # Set tags to not available for the selected cluster so it doesn't get used in another run az resource tag --tags 'freeforpipeline=false' -g $(AKS_CLUSTER_RG) -n $clustername --resource-type Microsoft.ContainerService/managedClusters workingDirectory: '$(System.DefaultWorkingDirectory)' - failOnStandardError: true displayName: Deploy to AKS - Find available AKS cluster and connect to it condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) @@ -384,7 +383,6 @@ jobs: # Turn off this check until our aad-pod-identity dep is updated # so that it's not trying to install v1beta1 # ClusterRoleBindings. - failOnStandardError: false - task: Docker@2 diff --git a/charts/azure-service-operator/templates/secret.yaml b/charts/azure-service-operator/templates/secret.yaml index 76a6f04ee5b..6e4366efc58 100644 --- a/charts/azure-service-operator/templates/secret.yaml +++ b/charts/azure-service-operator/templates/secret.yaml @@ -25,4 +25,10 @@ data: {{- if .Values.azureSecretNamingVersion }} AZURE_SECRET_NAMING_VERSION: {{ .Values.azureSecretNamingVersion | b64enc | quote }} {{- end }} + {{- if .Values.purgeDeletedKeyVaultSecrets }} + PURGE_DELETED_KEYVAULT_SECRETS: {{ .Values.purgeDeletedKeyVaultSecrets | b64enc | quote }} + {{- end }} + {{- if .Values.recoverSoftDeletedKeyVaultSecrets }} + RECOVER_SOFT_DELETED_KEYVAULT_SECRETS: {{ .Values.recoverSoftDeletedKeyVaultSecrets | b64enc | quote }} + {{- end }} {{- end }} diff --git a/charts/azure-service-operator/values.yaml b/charts/azure-service-operator/values.yaml index 19d821f4994..0d401f76b1b 100644 --- a/charts/azure-service-operator/values.yaml +++ b/charts/azure-service-operator/values.yaml @@ -25,6 +25,18 @@ azureUseMI: False # azureSecretNamingVersion allows choosing the algorithm used to derive secret names. Version 2 is recommended. azureSecretNamingVersion: "2" +# purgeDeletedKeyVaultSecrets determines if the operator should issue a secret Purge request in addition +# to Delete when deleting secrets in Azure Key Vault. This only applies to secrets that are stored in Azure Key Vault. +# It does nothing if the secret is stored in Kubernetes. +purgeDeletedKeyVaultSecrets: False + +# recoverSoftDeletedKeyVaultSecrets determines if the operator should issue a secret Recover request when it +# encounters an "ObjectIsDeletedButRecoverable" error from Azure Key Vault during secret creation. This error +# can occur when a Key Vault has soft delete enabled and an ASO resource was deleted and recreated with the same name. +# This only applies to secrets that are stored in Azure Key Vault. +# It does nothing if the secret is stored in Kubernetes. +recoverSoftDeletedKeyVaultSecrets: True + # image defines the container image the ASO pod should run # Note: This should use the latest released tag number explicitly. If # it's ':latest' and someone deploys the chart after a new version has diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 4748c7d8c7e..c23315c2bd6 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -61,6 +61,18 @@ spec: name: azureoperatorsettings key: AZURE_SECRET_NAMING_VERSION optional: true + - name: PURGE_DELETED_KEYVAULT_SECRETS + valueFrom: + secretKeyRef: + name: azureoperatorsettings + key: PURGE_DELETED_KEYVAULT_SECRETS + optional: true + - name: RECOVER_SOFT_DELETED_KEYVAULT_SECRETS + valueFrom: + secretKeyRef: + name: azureoperatorsettings + key: RECOVER_SOFT_DELETED_KEYVAULT_SECRETS + optional: true - name: AZURE_TARGET_NAMESPACES valueFrom: secretKeyRef: diff --git a/controllers/async_controller.go b/controllers/async_controller.go index 0ccc7158ebd..591ed4cbe57 100644 --- a/controllers/async_controller.go +++ b/controllers/async_controller.go @@ -78,7 +78,12 @@ func (r *AsyncReconciler) Reconcile(ctx context.Context, req ctrl.Request, obj c keyVaultName := keyvaultsecretlib.GetKeyVaultName(obj) if len(keyVaultName) != 0 { // Instantiate the KeyVault Secret Client - keyvaultSecretClient = keyvaultsecretlib.New(keyVaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyvaultSecretClient = keyvaultsecretlib.New( + keyVaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) } // Check to see if the skipreconcile annotation is on diff --git a/controllers/azuresql_combined_test.go b/controllers/azuresql_combined_test.go index 41cd123bf96..3a9e5feadb8 100644 --- a/controllers/azuresql_combined_test.go +++ b/controllers/azuresql_combined_test.go @@ -11,7 +11,7 @@ import ( "strings" "testing" - sql "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/v3.0/sql" + "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/v3.0/sql" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,7 +21,9 @@ import ( "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" + resourcemanagerkeyvaults "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" "github.com/Azure/azure-service-operator/pkg/secrets" + testcommon "github.com/Azure/azure-service-operator/test/common" ) func TestAzureSqlServerCombinedHappyPath(t *testing.T) { @@ -338,3 +340,47 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) { }) } + +func TestAzureSqlServer_KeyVaultSoftDelete_CreateDeleteCreateAgain(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + require := require.New(t) + + rgLocation := "westus2" + + // Create a KeyVault with soft delete enabled that we can use to perform our tests + keyVaultName := GenerateAlphaNumTestResourceNameWithRandom("kvsoftdel", 5) + objID, err := resourcemanagerkeyvaults.GetObjectID( + context.Background(), + config.GlobalCredentials(), + config.GlobalCredentials().TenantID(), + config.GlobalCredentials().ClientID()) + require.NoError(err) + + err = testcommon.CreateKeyVaultSoftDeleteEnabled( + context.Background(), + config.GlobalCredentials(), + tc.resourceGroupName, + keyVaultName, + rgLocation, + objID) + require.NoError(err) + + sqlServerName := GenerateTestResourceNameWithRandom("sqlserver", 10) + sqlServerNamespacedName := types.NamespacedName{Name: sqlServerName, Namespace: "default"} + sqlServerInstance := v1beta1.NewAzureSQLServer(sqlServerNamespacedName, tc.resourceGroupName, rgLocation) + sqlServerInstance.Spec.KeyVaultToStoreSecrets = keyVaultName + + // create and wait + RequireInstance(ctx, t, tc, sqlServerInstance) + + EnsureDelete(ctx, t, tc, sqlServerInstance) + + // Recreate with the same name + sqlServerInstance = v1beta1.NewAzureSQLServer(sqlServerNamespacedName, tc.resourceGroupName, rgLocation) + sqlServerInstance.Spec.KeyVaultToStoreSecrets = keyVaultName + RequireInstance(ctx, t, tc, sqlServerInstance) + + EnsureDelete(ctx, t, tc, sqlServerInstance) +} diff --git a/controllers/eventhub_storageaccount_controller_test.go b/controllers/eventhub_storageaccount_controller_test.go index c075290ceb8..1d08e8bb443 100644 --- a/controllers/eventhub_storageaccount_controller_test.go +++ b/controllers/eventhub_storageaccount_controller_test.go @@ -10,6 +10,7 @@ import ( "testing" s "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/api/v1alpha2" "github.com/Azure/azure-service-operator/pkg/secrets" @@ -173,7 +174,12 @@ func TestEventHubControllerCreateAndDeleteCustomKeyVault(t *testing.T) { EnsureInstance(ctx, t, tc, eventhubInstance) // Check that the secret is added to KeyVault - keyvaultSecretClient := kvsecrets.New(keyVaultNameForSecrets, config.GlobalCredentials(), config.SecretNamingVersion()) + keyvaultSecretClient := kvsecrets.New( + keyVaultNameForSecrets, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) key := secrets.SecretKey{Name: eventhubInstance.Name, Namespace: eventhubInstance.Namespace, Kind: "EventHub"} EnsureSecrets(ctx, t, tc, eventhubInstance, keyvaultSecretClient, key) diff --git a/controllers/helpers.go b/controllers/helpers.go index a32531eeb8c..18e15c81f13 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -13,11 +13,7 @@ import ( "testing" "time" - "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2018-02-14/keyvault" - "github.com/Azure/go-autorest/autorest/to" "github.com/go-logr/logr" - "github.com/gofrs/uuid" - "github.com/pkg/errors" "github.com/stretchr/testify/require" "golang.org/x/crypto/ssh" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -35,7 +31,6 @@ import ( resourcemanagersqlfirewallrule "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlfirewallrule" resourcemanagersqlserver "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlserver" resourcemanagersqluser "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqluser" - "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" resourcemanagerconfig "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" resourcemanagereventhub "github.com/Azure/azure-service-operator/pkg/resourcemanager/eventhubs" resourcemanagerkeyvaults "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" @@ -442,61 +437,3 @@ func GenerateRandomSshPublicKeyString() string { sshPublicKeyData := string(ssh.MarshalAuthorizedKey(publicRsaKey)) return sshPublicKeyData } - -//CreateVaultWithAccessPolicies creates a new key vault and provides access policies to the specified user - used in test -func CreateVaultWithAccessPolicies(ctx context.Context, creds config.Credentials, groupName string, vaultName string, location string, clientID string) error { - vaultsClient, err := resourcemanagerkeyvaults.GetKeyVaultClient(creds) - if err != nil { - return errors.Wrapf(err, "couldn't get vaults client") - } - id, err := uuid.FromString(creds.TenantID()) - if err != nil { - return errors.Wrapf(err, "couldn't convert tenantID to UUID") - } - - apList := []keyvault.AccessPolicyEntry{} - ap := keyvault.AccessPolicyEntry{ - TenantID: &id, - Permissions: &keyvault.Permissions{ - Keys: &[]keyvault.KeyPermissions{ - keyvault.KeyPermissionsCreate, - }, - Secrets: &[]keyvault.SecretPermissions{ - keyvault.SecretPermissionsSet, - keyvault.SecretPermissionsGet, - keyvault.SecretPermissionsDelete, - keyvault.SecretPermissionsList, - }, - }, - } - if clientID != "" { - objID, err := resourcemanagerkeyvaults.GetObjectID(ctx, creds, creds.TenantID(), clientID) - if err != nil { - return err - } - if objID != nil { - ap.ObjectID = objID - apList = append(apList, ap) - } - - } - - params := keyvault.VaultCreateOrUpdateParameters{ - Properties: &keyvault.VaultProperties{ - TenantID: &id, - AccessPolicies: &apList, - Sku: &keyvault.Sku{ - Family: to.StringPtr("A"), - Name: keyvault.Standard, - }, - }, - Location: to.StringPtr(location), - } - - future, err := vaultsClient.CreateOrUpdate(ctx, groupName, vaultName, params) - if err != nil { - return err - } - - return future.WaitForCompletionRef(ctx, vaultsClient.Client) -} diff --git a/controllers/keyvault_controller_test.go b/controllers/keyvault_controller_test.go index 5795208b47b..5cde77bfdfe 100644 --- a/controllers/keyvault_controller_test.go +++ b/controllers/keyvault_controller_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - uuid "github.com/gofrs/uuid" + "github.com/gofrs/uuid" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" @@ -126,7 +126,12 @@ func TestKeyvaultControllerWithAccessPolicies(t *testing.T) { //Add code to set secret and get secret from this keyvault using secretclient - keyvaultSecretClient := kvsecrets.New(keyVaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyvaultSecretClient := kvsecrets.New( + keyVaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) secretName := "test-key" key := secrets.SecretKey{Name: secretName, Namespace: "default", Kind: "test"} datanew := map[string][]byte{ @@ -186,7 +191,12 @@ func TestKeyvaultControllerWithLimitedAccessPoliciesAndUpdate(t *testing.T) { }, tc.timeout, tc.retry, "wait for keyVaultInstance to be ready in azure") //Add code to set secret and get secret from this keyvault using secretclient - keyvaultSecretClient := kvsecrets.New(keyVaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyvaultSecretClient := kvsecrets.New( + keyVaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) key := secrets.SecretKey{Name: "test-key", Namespace: "default", Kind: "test"} datanew := map[string][]byte{ "test1": []byte("test2"), @@ -333,7 +343,12 @@ func TestKeyvaultControllerWithVirtualNetworkRulesAndUpdate(t *testing.T) { return result.Response.StatusCode == http.StatusOK }, tc.timeout, tc.retry, "wait for keyVaultInstance to be ready in azure") - keyvaultSecretClient := kvsecrets.New(keyVaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyvaultSecretClient := kvsecrets.New( + keyVaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) secretName := "test-key" key := secrets.SecretKey{Name: secretName, Namespace: "default", Kind: "test"} datanew := map[string][]byte{ diff --git a/controllers/secret_naming_version_test.go b/controllers/secret_naming_version_test.go index cf830f16938..27c6225c5a0 100644 --- a/controllers/secret_naming_version_test.go +++ b/controllers/secret_naming_version_test.go @@ -155,7 +155,12 @@ func assertSQLServerAdminSecretCreated(ctx context.Context, t *testing.T, sqlSer }, tc.timeoutFast, tc.retry, "wait for server to have secret") } else { // Check that the user's secret is in the keyvault - keyVaultSecretClient := kvsecrets.New(sqlServerInstance.Spec.KeyVaultToStoreSecrets, config.GlobalCredentials(), config.SecretNamingVersion()) + keyVaultSecretClient := kvsecrets.New( + sqlServerInstance.Spec.KeyVaultToStoreSecrets, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) assert.Eventually(t, func() bool { expectedSecretName := makeSQLServerKeyVaultSecretName(sqlServerInstance) @@ -294,7 +299,12 @@ func TestAzureSqlServerAndUser_SecretNamedCorrectly(t *testing.T) { EnsureInstance(ctx, t, tc, kvSqlUser1) // Check that the user's secret is in the keyvault - keyVaultSecretClient := kvsecrets.New(tc.keyvaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyVaultSecretClient := kvsecrets.New( + tc.keyvaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) assert.Eventually(func() bool { key := makeSQLUserSecretKey(keyVaultSecretClient, kvSqlUser1) @@ -333,7 +343,12 @@ func TestAzureSqlServerAndUser_SecretNamedCorrectly(t *testing.T) { EnsureInstance(ctx, t, tc, kvSqlUser2) // Check that the user's secret is in the keyvault - keyVaultSecretClient := kvsecrets.New(tc.keyvaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyVaultSecretClient := kvsecrets.New( + tc.keyvaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) assert.Eventually(func() bool { key := makeSQLUserSecretKey(keyVaultSecretClient, kvSqlUser2) @@ -350,7 +365,12 @@ func TestAzureSqlServerAndUser_SecretNamedCorrectly(t *testing.T) { t.Run("deploy sql action and roll user credentials", func(t *testing.T) { keyVaultName := tc.keyvaultName - keyVaultSecretClient := kvsecrets.New(keyVaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyVaultSecretClient := kvsecrets.New( + keyVaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) key := makeSQLUserSecretKey(keyVaultSecretClient, kvSqlUser1) oldSecret, err := keyVaultSecretClient.Get(ctx, key) @@ -395,7 +415,12 @@ func TestAzureSqlServerAndUser_SecretNamedCorrectly(t *testing.T) { EnsureDelete(ctx, t, tc, kvSqlUser2) // Check that the user's secret is in the keyvault - keyVaultSecretClient := kvsecrets.New(tc.keyvaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyVaultSecretClient := kvsecrets.New( + tc.keyvaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) assert.Eventually(func() bool { key := secrets.SecretKey{Name: sqlUser.ObjectMeta.Name, Namespace: sqlUser.ObjectMeta.Namespace, Kind: "azuresqluser"} @@ -519,7 +544,12 @@ func TestAzureSqlServerKVSecretAndUser_SecretNamedCorrectly(t *testing.T) { EnsureInstance(ctx, t, tc, kvSqlUser1) // Check that the user's secret is in the keyvault - keyVaultSecretClient := kvsecrets.New(tc.keyvaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + keyVaultSecretClient := kvsecrets.New( + tc.keyvaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) assert.Eventually(func() bool { key := makeSQLUserSecretKey(keyVaultSecretClient, kvSqlUser1) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 7eb4ddaf26e..d5e20e60fb4 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -35,6 +35,7 @@ import ( resourcemanagerkeyvaults "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" resourcegroupsresourcemanager "github.com/Azure/azure-service-operator/pkg/resourcemanager/resourcegroups" k8sSecrets "github.com/Azure/azure-service-operator/pkg/secrets/kube" + "github.com/Azure/azure-service-operator/test/common" // +kubebuilder:scaffold:imports ) @@ -201,13 +202,22 @@ func setup() error { } log.Println("Creating KV:", keyvaultName) - err = CreateVaultWithAccessPolicies( + objID, err := resourcemanagerkeyvaults.GetObjectID( + context.Background(), + config.GlobalCredentials(), + config.GlobalCredentials().TenantID(), + config.GlobalCredentials().ClientID()) + if err != nil { + return err + } + + err = common.CreateVaultWithAccessPolicies( context.Background(), config.GlobalCredentials(), resourceGroupName, keyvaultName, resourceGroupLocation, - config.GlobalCredentials().ClientID(), + objID, ) if err != nil { return err diff --git a/main.go b/main.go index c9c1e6fbbd3..573c46472ce 100644 --- a/main.go +++ b/main.go @@ -116,7 +116,12 @@ func main() { secretClient = k8sSecrets.New(mgr.GetClient(), config.SecretNamingVersion()) } else { setupLog.Info("Instantiating secrets client for keyvault " + keyvaultName) - secretClient = keyvaultSecrets.New(keyvaultName, config.GlobalCredentials(), config.SecretNamingVersion()) + secretClient = keyvaultSecrets.New( + keyvaultName, + config.GlobalCredentials(), + config.SecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) } if config.SelectedMode().IncludesWatchers() { diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index ebe96f6551e..2710ac37382 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -68,6 +68,8 @@ const ( LongTermRetentionPolicyInvalid = "LongTermRetentionPolicyInvalid" BackupRetentionPolicyInvalid = "InvalidBackupRetentionPeriod" OperationIdNotFound = "OperationIdNotFound" + ObjectIsBeingDeleted = "ObjectIsBeingDeleted" + ObjectIsDeletedButRecoverable = "ObjectIsDeletedButRecoverable" ) func NewAzureError(err error) *AzureError { diff --git a/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go index a2ba5fe4320..1d193a07f73 100644 --- a/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqluser" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" keyvaultsecretlib "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" ) @@ -47,7 +48,12 @@ func (s *AzureSqlActionManager) Ensure(ctx context.Context, obj runtime.Object, if len(instance.Spec.ServerSecretKeyVault) == 0 { adminSecretClient = s.SecretClient } else { - adminSecretClient = keyvaultsecretlib.New(instance.Spec.ServerSecretKeyVault, s.Creds, s.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultsecretlib.New( + instance.Spec.ServerSecretKeyVault, + s.Creds, + s.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) err = keyvaultsecretlib.CheckKeyVaultAccessibility(ctx, adminSecretClient) if err != nil { instance.Status.Message = "InvalidKeyVaultAccess: Keyvault not accessible yet: " + err.Error() @@ -96,7 +102,12 @@ func (s *AzureSqlActionManager) Ensure(ctx context.Context, obj runtime.Object, if len(instance.Spec.ServerSecretKeyVault) == 0 { adminSecretClient = s.SecretClient } else { - adminSecretClient = keyvaultsecretlib.New(instance.Spec.ServerSecretKeyVault, s.Creds, s.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultsecretlib.New( + instance.Spec.ServerSecretKeyVault, + s.Creds, + s.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) err = keyvaultsecretlib.CheckKeyVaultAccessibility(ctx, adminSecretClient) if err != nil { instance.Status.Message = "InvalidKeyVaultAccess: Keyvault not accessible yet: " + err.Error() @@ -109,7 +120,12 @@ func (s *AzureSqlActionManager) Ensure(ctx context.Context, obj runtime.Object, if len(instance.Spec.UserSecretKeyVault) == 0 { userSecretClient = s.SecretClient } else { - userSecretClient = keyvaultsecretlib.New(instance.Spec.UserSecretKeyVault, s.Creds, s.SecretClient.GetSecretNamingVersion()) + userSecretClient = keyvaultsecretlib.New( + instance.Spec.UserSecretKeyVault, + s.Creds, + s.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) err = keyvaultsecretlib.CheckKeyVaultAccessibility(ctx, adminSecretClient) if err != nil { instance.Status.Message = "InvalidKeyVaultAccess: Keyvault not accessible yet: " + err.Error() diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 381b0499943..cdc5baef3bf 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -39,7 +39,12 @@ func (s *AzureSqlUserManager) getAdminSecret(ctx context.Context, instance *v1al // if the admin secret keyvault is not specified, fall back to global secretclient if len(instance.Spec.AdminSecretKeyVault) != 0 { - adminSecretClient = keyvaultSecrets.New(instance.Spec.AdminSecretKeyVault, s.Creds, s.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultSecrets.New( + instance.Spec.AdminSecretKeyVault, + s.Creds, + s.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) // This is here for legacy reasons if len(instance.Spec.AdminSecret) != 0 && s.SecretClient.GetSecretNamingVersion() == secrets.SecretNamingV1 { diff --git a/pkg/resourcemanager/config/config.go b/pkg/resourcemanager/config/config.go index 99c291f000c..fb4034c0ef4 100644 --- a/pkg/resourcemanager/config/config.go +++ b/pkg/resourcemanager/config/config.go @@ -20,22 +20,23 @@ var ( // shouldn't be set here, because mutable vars shouldn't be global. // TODO: eliminate this! - creds credentials - locationDefault string - authorizationServerURL string - cloudName string - useDeviceFlow bool - buildID string - keepResources bool - userAgent string - baseURI string - environment *azure.Environment - podNamespace string - targetNamespaces []string - secretNamingVersion secrets.SecretNamingVersion - operatorMode OperatorMode - - testResourcePrefix string // used to generate resource names in tests, should probably exist in a test only package + creds credentials + locationDefault string + authorizationServerURL string + cloudName string + useDeviceFlow bool + buildID string + keepResources bool + userAgent string + baseURI string + environment *azure.Environment + podNamespace string + targetNamespaces []string + secretNamingVersion secrets.SecretNamingVersion + operatorMode OperatorMode + purgeDeletedKeyVaultSecrets bool + recoverSoftDeletedKeyVaultSecrets bool + testResourcePrefix string // used to generate resource names in tests, should probably exist in a test only package ) // GlobalCredentials returns the configured credentials. @@ -50,7 +51,7 @@ func Location() string { return locationDefault } -// DefaultLocation() returns the default location wherein to create new resources. +// DefaultLocation returns the default location wherein to create new resources. // Some resource types are not available in all locations so another location might need // to be chosen. func DefaultLocation() string { @@ -63,18 +64,18 @@ func AuthorizationServerURL() string { return authorizationServerURL } -// UseDeviceFlow() specifies if interactive auth should be used. Interactive +// UseDeviceFlow specifies if interactive auth should be used. Interactive // auth uses the OAuth Device Flow grant type. func UseDeviceFlow() bool { return useDeviceFlow } -// KeepResources() specifies whether to keep resources created by samples. +// KeepResources specifies whether to keep resources created by samples. func KeepResources() bool { return keepResources } -// UserAgent() specifies a string to append to the agent identifier. +// UserAgent specifies a string to append to the agent identifier. func UserAgent() string { if len(userAgent) > 0 { return userAgent @@ -133,11 +134,28 @@ func SecretNamingVersion() secrets.SecretNamingVersion { return secretNamingVersion } +// PurgeDeletedKeyVaultSecrets determines if the operator should issue a secret Purge request in addition +// to Delete when deleting secrets in Azure Key Vault. This only applies to secrets that are stored in Azure Key Vault. +// It does nothing if the secret is stored in Kubernetes. +func PurgeDeletedKeyVaultSecrets() bool { + return purgeDeletedKeyVaultSecrets +} + +// RecoverSoftDeletedKeyVaultSecrets determines if the operator should issue a secret Recover request when it +// encounters an "ObjectIsDeletedButRecoverable" error from Azure Key Vault during secret creation. This error +// can occur when a Key Vault has soft delete enabled and an ASO resource was deleted and recreated with the same name. +// This only applies to secrets that are stored in Azure Key Vault. +// It does nothing if the secret is stored in Kubernetes. +func RecoverSoftDeletedKeyVaultSecrets() bool { + return recoverSoftDeletedKeyVaultSecrets +} + // ConfigString returns the parts of the configuration file with are not secrets as a string for easy logging func ConfigString() string { creds := GlobalCredentials() return fmt.Sprintf( - "clientID: %q, tenantID: %q, subscriptionID: %q, cloudName: %q, useDeviceFlow: %t, useManagedIdentity: %t, operatorMode: %s, targetNamespaces: %s, podNamespace: %q, secretNamingVersion: %q", + "clientID: %q, tenantID: %q, subscriptionID: %q, cloudName: %q, useDeviceFlow: %t, useManagedIdentity: %t, operatorMode: %s, targetNamespaces: %s,"+ + " podNamespace: %q, secretNamingVersion: %q, purgeDeletedKeyVaultSecrets: %t, recoverSoftDeletedkeyVaultSecrets: %t", creds.ClientID(), creds.TenantID(), creds.SubscriptionID(), @@ -147,5 +165,7 @@ func ConfigString() string { operatorMode, targetNamespaces, podNamespace, - SecretNamingVersion()) + SecretNamingVersion(), + PurgeDeletedKeyVaultSecrets(), + RecoverSoftDeletedKeyVaultSecrets()) } diff --git a/pkg/resourcemanager/config/env.go b/pkg/resourcemanager/config/env.go index 01c5e7ff2e2..f101568b41f 100644 --- a/pkg/resourcemanager/config/env.go +++ b/pkg/resourcemanager/config/env.go @@ -56,17 +56,19 @@ func ParseEnvironment() error { authorizationServerURL = azureEnv.ActiveDirectoryEndpoint baseURI = azureEnv.ResourceManagerEndpoint // BaseURI() - locationDefault = envy.Get("AZURE_LOCATION_DEFAULT", "westus2") // DefaultLocation() - useDeviceFlow = ParseBoolFromEnvironment("AZURE_USE_DEVICEFLOW") // UseDeviceFlow() - creds.useManagedIdentity = ParseBoolFromEnvironment("AZURE_USE_MI") // UseManagedIdentity() - keepResources = ParseBoolFromEnvironment("AZURE_SAMPLES_KEEP_RESOURCES") // KeepResources() - creds.operatorKeyvault = envy.Get("AZURE_OPERATOR_KEYVAULT", "") // operatorKeyvault() + locationDefault = envy.Get("AZURE_LOCATION_DEFAULT", "westus2") // DefaultLocation() + useDeviceFlow = ParseBoolFromEnvironment("AZURE_USE_DEVICEFLOW", false) // UseDeviceFlow() + creds.useManagedIdentity = ParseBoolFromEnvironment("AZURE_USE_MI", false) // UseManagedIdentity() + keepResources = ParseBoolFromEnvironment("AZURE_SAMPLES_KEEP_RESOURCES", false) // KeepResources() + creds.operatorKeyvault = envy.Get("AZURE_OPERATOR_KEYVAULT", "") // operatorKeyvault() testResourcePrefix = envy.Get("TEST_RESOURCE_PREFIX", "t-"+helpers.RandomString(6)) podNamespace, err = envy.MustGet("POD_NAMESPACE") if err != nil { return errors.Wrapf(err, "couldn't get POD_NAMESPACE env variable") } targetNamespaces = ParseStringListFromEnvironment("AZURE_TARGET_NAMESPACES") + purgeDeletedKeyVaultSecrets = ParseBoolFromEnvironment("PURGE_DELETED_KEYVAULT_SECRETS", false) + recoverSoftDeletedKeyVaultSecrets = ParseBoolFromEnvironment("RECOVER_SOFT_DELETED_KEYVAULT_SECRETS", true) operatorMode, err = ParseOperatorMode(envy.Get("AZURE_OPERATOR_MODE", OperatorModeBoth.String())) if err != nil { @@ -135,8 +137,9 @@ func GetExpectedConfigurationVariables() []ConfigRequirementType { return []ConfigRequirementType{RequireClientID, RequireClientSecret, RequireTenantID, RequireSubscriptionID} } -func ParseBoolFromEnvironment(variable string) bool { - env := envy.Get(variable, "0") +func ParseBoolFromEnvironment(variable string, defaultValue bool) bool { + defaultValueStr := fmt.Sprintf("%t", defaultValue) + env := envy.Get(variable, defaultValueStr) value, err := strconv.ParseBool(env) if err != nil { log.Printf("WARNING: invalid input value specified for %q, expected bool, actual: %q. Disabling\n", variable, env) diff --git a/pkg/resourcemanager/mysql/mysqluser/mysqluser_reconcile.go b/pkg/resourcemanager/mysql/mysqluser/mysqluser_reconcile.go index be78d964243..ed393f95004 100644 --- a/pkg/resourcemanager/mysql/mysqluser/mysqluser_reconcile.go +++ b/pkg/resourcemanager/mysql/mysqluser/mysqluser_reconcile.go @@ -9,12 +9,14 @@ import ( "reflect" "strings" - mysqlserver "github.com/Azure/azure-service-operator/pkg/resourcemanager/mysql/server" _ "github.com/go-sql-driver/mysql" //sql drive link "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" + mysqlserver "github.com/Azure/azure-service-operator/pkg/resourcemanager/mysql/server" + "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/api/v1alpha2" "github.com/Azure/azure-service-operator/pkg/errhelp" @@ -43,7 +45,12 @@ func (s *MySqlUserManager) Ensure(ctx context.Context, obj runtime.Object, opts adminSecretClient := s.SecretClient if len(instance.Spec.AdminSecretKeyVault) != 0 { - adminSecretClient = keyvaultSecrets.New(instance.Spec.AdminSecretKeyVault, s.Creds, s.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultSecrets.New( + instance.Spec.AdminSecretKeyVault, + s.Creds, + s.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) } adminSecretKey := secrets.SecretKey{Name: instance.Spec.GetAdminSecretName(), Namespace: instance.Namespace, Kind: reflect.TypeOf(v1alpha2.MySQLServer{}).Name()} @@ -188,7 +195,12 @@ func (s *MySqlUserManager) Delete(ctx context.Context, obj runtime.Object, opts // if the admin secret keyvault is not specified, fall back to configured secretclient if len(instance.Spec.AdminSecretKeyVault) != 0 { - adminSecretClient = keyvaultSecrets.New(instance.Spec.AdminSecretKeyVault, s.Creds, s.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultSecrets.New( + instance.Spec.AdminSecretKeyVault, + s.Creds, + s.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) } adminSecret, err := adminSecretClient.Get(ctx, adminSecretKey) diff --git a/pkg/resourcemanager/psql/psqluser/psqluser_reconcile.go b/pkg/resourcemanager/psql/psqluser/psqluser_reconcile.go index af2f9afbef6..2563ee309c4 100644 --- a/pkg/resourcemanager/psql/psqluser/psqluser_reconcile.go +++ b/pkg/resourcemanager/psql/psqluser/psqluser_reconcile.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/Azure/azure-service-operator/pkg/helpers" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -62,7 +63,12 @@ func (m *PostgreSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, // if the admin secret keyvault is not specified, fall back to configured secretclient if len(instance.Spec.AdminSecretKeyVault) != 0 { - adminSecretClient = keyvaultSecrets.New(instance.Spec.AdminSecretKeyVault, m.Creds, m.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultSecrets.New( + instance.Spec.AdminSecretKeyVault, + m.Creds, + m.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) } // get admin creds for server @@ -209,7 +215,12 @@ func (m *PostgreSqlUserManager) Delete(ctx context.Context, obj runtime.Object, // if the admin secret keyvault is not specified, fall back to configured secretclient if len(instance.Spec.AdminSecretKeyVault) != 0 { - adminSecretClient = keyvaultSecrets.New(instance.Spec.AdminSecretKeyVault, m.Creds, m.SecretClient.GetSecretNamingVersion()) + adminSecretClient = keyvaultSecrets.New( + instance.Spec.AdminSecretKeyVault, + m.Creds, + m.SecretClient.GetSecretNamingVersion(), + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) } adminSecret, err := adminSecretClient.Get(ctx, adminSecretKey) diff --git a/pkg/secrets/keyvault/client.go b/pkg/secrets/keyvault/client.go index bd3741539ba..0e104ebdb2f 100644 --- a/pkg/secrets/keyvault/client.go +++ b/pkg/secrets/keyvault/client.go @@ -10,6 +10,7 @@ import ( "time" keyvaults "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault" + "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/date" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" @@ -26,6 +27,9 @@ type SecretClient struct { KeyVaultClient keyvaults.BaseClient KeyVaultName string SecretNamingVersion secrets.SecretNamingVersion + + PurgeDeletedSecrets bool + RecoverSoftDeletedSecrets bool } var _ secrets.SecretClient = &SecretClient{} @@ -60,15 +64,24 @@ func GetVaultsURL(vaultName string) string { // redundant since that's in the credentials, but it's used to // override the one specified in credentials so it might be right to // keep it. Confirm this. -func New(keyVaultName string, creds config.Credentials, secretNamingVersion secrets.SecretNamingVersion) *SecretClient { +func New( + keyVaultName string, + creds config.Credentials, + secretNamingVersion secrets.SecretNamingVersion, + purgeDeletedSecrets bool, + recoverSoftDeletedSecrets bool) *SecretClient { + keyvaultClient := keyvaults.New() a, _ := iam.GetKeyvaultAuthorizer(creds) keyvaultClient.Authorizer = a keyvaultClient.AddToUserAgent(config.UserAgent()) + return &SecretClient{ - KeyVaultClient: keyvaultClient, - KeyVaultName: keyVaultName, - SecretNamingVersion: secretNamingVersion, + KeyVaultClient: keyvaultClient, + KeyVaultName: keyVaultName, + SecretNamingVersion: secretNamingVersion, + PurgeDeletedSecrets: purgeDeletedSecrets, + RecoverSoftDeletedSecrets: recoverSoftDeletedSecrets, } } @@ -113,7 +126,6 @@ func (k *SecretClient) Upsert(ctx context.Context, key secrets.SecretKey, data m opt(options) } - vaultBaseURL := GetVaultsURL(k.KeyVaultName) secretBaseName, err := k.makeSecretName(key) if err != nil { return err @@ -141,8 +153,6 @@ func (k *SecretClient) Upsert(ctx context.Context, key secrets.SecretKey, data m // if the caller is looking for flat secrets iterate over the array and individually persist each string if options.Flatten { - var err error - for formatName, formatValue := range data { secretName := secretBaseName + "-" + formatName stringSecret := string(formatValue) @@ -163,14 +173,15 @@ func (k *SecretClient) Upsert(ctx context.Context, key secrets.SecretKey, data m } }*/ - _, err = k.KeyVaultClient.SetSecret(ctx, vaultBaseURL, secretName, secretParams) + err = k.setSecret(ctx, secretName, secretParams) if err != nil { - return errors.Wrapf(err, "error setting secret %q in %q", secretBaseName, vaultBaseURL) + return err } } // If flatten has not been declared, convert the map into a json string for persistence } else { - jsonData, err := json.Marshal(data) + var jsonData []byte + jsonData, err = json.Marshal(data) if err != nil { return errors.Wrapf(err, "unable to marshal secret") } @@ -183,7 +194,57 @@ func (k *SecretClient) Upsert(ctx context.Context, key secrets.SecretKey, data m SecretAttributes: &secretAttributes, } - _, err = k.KeyVaultClient.SetSecret(ctx, vaultBaseURL, secretBaseName, secretParams) + err = k.setSecret(ctx, secretBaseName, secretParams) + if err != nil { + return err + } + } + + return nil +} + +func isErrSecretWasSoftDeleted(err error) bool { + var azureErr *azure.RequestError + if errors.As(err, &azureErr) { + if azureErr.ServiceError == nil { + return false + } + if len(azureErr.ServiceError.InnerError) == 0 { + return false + } + + code, ok := azureErr.ServiceError.InnerError["code"] + if !ok { + return false + } + + codeString, ok := code.(string) + if !ok { + return false + } + + return codeString == errhelp.ObjectIsDeletedButRecoverable + } + return false +} + +func (k *SecretClient) setSecret(ctx context.Context, secretBaseName string, secret keyvaults.SecretSetParameters) error { + vaultBaseURL := GetVaultsURL(k.KeyVaultName) + + _, err := k.KeyVaultClient.SetSecret(ctx, vaultBaseURL, secretBaseName, secret) + if err != nil { + if !isErrSecretWasSoftDeleted(err) || !k.RecoverSoftDeletedSecrets { + return errors.Wrapf(err, "error setting secret %q in %q", secretBaseName, vaultBaseURL) + } + + // The secret was soft deleted and we can recover it + _, err := k.KeyVaultClient.RecoverDeletedSecret(ctx, vaultBaseURL, secretBaseName) + if err != nil { + return errors.Wrapf(err, "failed recovering deleted secret %q in %q", secretBaseName, vaultBaseURL) + } + + // Now it's recovered? + _, err = k.KeyVaultClient.SetSecret(ctx, vaultBaseURL, secretBaseName, secret) if err != nil { return errors.Wrapf(err, "error setting secret %q in %q", secretBaseName, vaultBaseURL) } @@ -204,7 +265,20 @@ func (k *SecretClient) deleteKeyVaultSecret(ctx context.Context, secretName stri } // If Keyvault has softdelete enabled, we will need to purge the secret in addition to deleting it - _, err = k.KeyVaultClient.PurgeDeletedSecret(ctx, vaultBaseURL, secretName) + if k.PurgeDeletedSecrets { + err = k.purgeKeyVaultSecret(ctx, secretName) + if err != nil { + return errors.Wrapf(err, "error purging secret %q in %q", secretName, vaultBaseURL) + } + } + + return nil +} + +func (k *SecretClient) purgeKeyVaultSecret(ctx context.Context, secretName string) error { + vaultBaseURL := GetVaultsURL(k.KeyVaultName) + + _, err := k.KeyVaultClient.PurgeDeletedSecret(ctx, vaultBaseURL, secretName) for err != nil { azerr := errhelp.NewAzureError(err) if azerr.Type == errhelp.NotSupported { // Keyvault not softdelete enabled; ignore error @@ -218,6 +292,7 @@ func (k *SecretClient) deleteKeyVaultSecret(ctx context.Context, secretName stri return err } } + return err } diff --git a/pkg/secrets/keyvault/client_test.go b/pkg/secrets/keyvault/client_test.go index abfc334ba60..560be154f16 100644 --- a/pkg/secrets/keyvault/client_test.go +++ b/pkg/secrets/keyvault/client_test.go @@ -16,7 +16,8 @@ import ( "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" "github.com/Azure/azure-service-operator/pkg/secrets" - "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" + keyvaultsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" + testcommon "github.com/Azure/azure-service-operator/test/common" ) func getExpectedSecretName(secretKey secrets.SecretKey, namingScheme secrets.SecretNamingVersion) string { @@ -38,6 +39,7 @@ var _ = Describe("Keyvault Secrets Client", func() { var keyVaultName string var kvManager *kvhelper.AzureKeyVaultManager var vaultBaseUrl string + var objID *string BeforeEach(func() { // Add any setup steps that needs to be executed before each test @@ -46,14 +48,22 @@ var _ = Describe("Keyvault Secrets Client", func() { ctx = context.Background() // Initialize service principal ID to give access to the keyvault - userID := config.GlobalCredentials().ClientID() - kvManager = kvhelper.NewAzureKeyVaultManager(config.GlobalCredentials(), nil) keyVaultName = controllers.GenerateTestResourceNameWithRandom("kv", 5) - vaultBaseUrl = keyvault.GetVaultsURL(keyVaultName) + vaultBaseUrl = keyvaultsecrets.GetVaultsURL(keyVaultName) // Create a keyvault - err := controllers.CreateVaultWithAccessPolicies(ctx, config.GlobalCredentials(), resourceGroupName, keyVaultName, config.DefaultLocation(), userID) + var err error + objID, err = kvhelper.GetObjectID(ctx, config.GlobalCredentials(), config.GlobalCredentials().TenantID(), config.GlobalCredentials().ClientID()) + Expect(err).NotTo(HaveOccurred()) + + err = testcommon.CreateVaultWithAccessPolicies( + ctx, + config.GlobalCredentials(), + resourceGroupName, + keyVaultName, + config.DefaultLocation(), + objID) Expect(err).NotTo(HaveOccurred()) }) @@ -88,7 +98,12 @@ var _ = Describe("Keyvault Secrets Client", func() { "sweet": []byte("potato"), } - client := keyvault.New(keyVaultName, config.GlobalCredentials(), secretNamingScheme) + client := keyvaultsecrets.New( + keyVaultName, + config.GlobalCredentials(), + secretNamingScheme, + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) key := secrets.SecretKey{Name: secretName, Namespace: "default", Kind: "Test"} Context("creating secret with KeyVault client", func() { @@ -151,7 +166,12 @@ var _ = Describe("Keyvault Secrets Client", func() { "sweet": []byte("potato"), } - client := keyvault.New(keyVaultName, config.GlobalCredentials(), secretNamingScheme) + client := keyvaultsecrets.New( + keyVaultName, + config.GlobalCredentials(), + secretNamingScheme, + config.PurgeDeletedKeyVaultSecrets(), + config.RecoverSoftDeletedKeyVaultSecrets()) key := secrets.SecretKey{Name: secretName, Namespace: "default", Kind: "Test"} Context("creating flattened secret with KeyVault client", func() { @@ -224,6 +244,79 @@ var _ = Describe("Keyvault Secrets Client", func() { } }) }) + } }) }) + +var _ = Describe("Keyvault Secrets soft delete", func() { + // Create a context to use in the tests + ctx := context.Background() + var softDeleteKVName string + var kvManager *kvhelper.AzureKeyVaultManager + var objID *string + + BeforeEach(func() { + var err error + // Initialize service principal ID to give access to the keyvault + kvManager = kvhelper.NewAzureKeyVaultManager(config.GlobalCredentials(), nil) + softDeleteKVName = controllers.GenerateTestResourceNameWithRandom("kv", 5) + + objID, err = kvhelper.GetObjectID(ctx, config.GlobalCredentials(), config.GlobalCredentials().TenantID(), config.GlobalCredentials().ClientID()) + Expect(err).NotTo(HaveOccurred()) + + err = testcommon.CreateKeyVaultSoftDeleteEnabled( + ctx, + config.GlobalCredentials(), + resourceGroupName, + softDeleteKVName, + config.DefaultLocation(), + objID) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + // Add any teardown steps that needs to be executed after each test + // Delete the keyvault + _, err := kvManager.DeleteVault(ctx, resourceGroupName, softDeleteKVName) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should delete and recreate soft-delete key when recreate soft-deleted key option is enabled", func() { + Context("create, delete, and recreate soft deleted key", func() { + secretName := "kvsecret" + strconv.FormatInt(GinkgoRandomSeed(), 10) + client := keyvaultsecrets.New( + softDeleteKVName, + config.GlobalCredentials(), + secrets.SecretNamingV2, + false, + true) + + data := map[string][]byte{ + "test": []byte("data"), + "sweet": []byte("potato"), + } + + secretKey := secrets.SecretKey{Name: secretName, Namespace: "default", Kind: "Test"} + + // Create the key + err := client.Upsert(ctx, secretKey, data) + Expect(err).NotTo(HaveOccurred()) + + // Delete the key + err = client.Delete(ctx, secretKey) + Expect(err).NotTo(HaveOccurred()) + + // Wait for the key to be deleted. + Eventually(func() bool { + _, err = client.Get(ctx, secretKey) + return err == nil + }, time.Second*60).Should(BeFalse()) + + // Create the key again + Eventually(func() error { + return client.Upsert(ctx, secretKey, data) + }, time.Second*60).Should(Succeed()) + }) + }) +}) diff --git a/test/common/keyvault.go b/test/common/keyvault.go new file mode 100644 index 00000000000..5d0dcbe04bf --- /dev/null +++ b/test/common/keyvault.go @@ -0,0 +1,119 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +package common + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2018-02-14/keyvault" + "github.com/Azure/go-autorest/autorest/to" + "github.com/gofrs/uuid" + "github.com/pkg/errors" + + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" + kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" +) + +func CreateKeyVaultSoftDeleteEnabled(ctx context.Context, creds config.Credentials, resourceGroupName string, vaultName string, location string, objectID *string) error { + vaultsClient, err := kvhelper.GetKeyVaultClient(creds) + if err != nil { + return errors.Wrapf(err, "couldn't get vaults client") + } + + id, err := uuid.FromString(creds.TenantID()) + if err != nil { + return errors.Wrapf(err, "couldn't convert tenantID to UUID") + } + + accessPolicies, err := CreateKeyVaultTestAccessPolicies(creds, objectID) + if err != nil { + return nil + } + + params := keyvault.VaultCreateOrUpdateParameters{ + Properties: &keyvault.VaultProperties{ + TenantID: &id, + AccessPolicies: &accessPolicies, + EnableSoftDelete: to.BoolPtr(true), + Sku: &keyvault.Sku{ + Family: to.StringPtr("A"), + Name: keyvault.Standard, + }, + }, + Location: to.StringPtr(location), + } + + future, err := vaultsClient.CreateOrUpdate(ctx, resourceGroupName, vaultName, params) + if err != nil { + return err + } + + return future.WaitForCompletionRef(ctx, vaultsClient.Client) +} + +//CreateVaultWithAccessPolicies creates a new key vault and provides access policies to the specified user - used in test +func CreateVaultWithAccessPolicies(ctx context.Context, creds config.Credentials, groupName string, vaultName string, location string, objectID *string) error { + vaultsClient, err := kvhelper.GetKeyVaultClient(creds) + if err != nil { + return errors.Wrapf(err, "couldn't get vaults client") + } + id, err := uuid.FromString(creds.TenantID()) + if err != nil { + return errors.Wrapf(err, "couldn't convert tenantID to UUID") + } + + apList, err := CreateKeyVaultTestAccessPolicies(creds, objectID) + if err != nil { + return nil + } + + params := keyvault.VaultCreateOrUpdateParameters{ + Properties: &keyvault.VaultProperties{ + TenantID: &id, + AccessPolicies: &apList, + Sku: &keyvault.Sku{ + Family: to.StringPtr("A"), + Name: keyvault.Standard, + }, + }, + Location: to.StringPtr(location), + } + + future, err := vaultsClient.CreateOrUpdate(ctx, groupName, vaultName, params) + if err != nil { + return err + } + + return future.WaitForCompletionRef(ctx, vaultsClient.Client) +} + +func CreateKeyVaultTestAccessPolicies(creds config.Credentials, objectID *string) ([]keyvault.AccessPolicyEntry, error) { + id, err := uuid.FromString(creds.TenantID()) + if err != nil { + return nil, errors.Wrapf(err, "couldn't convert tenantID to UUID") + } + + apList := []keyvault.AccessPolicyEntry{ + { + TenantID: &id, + ObjectID: objectID, + Permissions: &keyvault.Permissions{ + Keys: &[]keyvault.KeyPermissions{ + keyvault.KeyPermissionsCreate, + }, + Secrets: &[]keyvault.SecretPermissions{ + keyvault.SecretPermissionsSet, + keyvault.SecretPermissionsGet, + keyvault.SecretPermissionsDelete, + keyvault.SecretPermissionsList, + keyvault.SecretPermissionsRecover, + }, + }, + }, + } + + return apList, nil +}