diff --git a/config/samples/azure_v1alpha1_blobcontainer.yaml b/config/samples/azure_v1alpha1_blobcontainer.yaml index b3dadf1baf3..a7491ee1fd9 100644 --- a/config/samples/azure_v1alpha1_blobcontainer.yaml +++ b/config/samples/azure_v1alpha1_blobcontainer.yaml @@ -4,8 +4,8 @@ metadata: name: blobcontainer-sample spec: location: westus - resourcegroup: resourcegroup-sample - accountname: storageaccount-sample + resourcegroup: resourcegroup-azure-operators + accountname: storageaccountsample777 # accessLevel - Specifies whether data in the container may be accessed publicly and the level of access. # Possible values include: 'Container', 'Blob', 'None' accesslevel: Container \ No newline at end of file diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index f9ca7f45df0..e68912ae8f4 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -28,7 +28,7 @@ func TestCosmosDBHappyPath(t *testing.T) { Namespace: cosmosDBNamespace, }, Spec: v1alpha1.CosmosDBSpec{ - Location: "westus", + Location: tc.resourceGroupLocation, ResourceGroup: tc.resourceGroupName, Kind: v1alpha1.CosmosDBKindGlobalDocumentDB, Properties: v1alpha1.CosmosDBProperties{ @@ -42,3 +42,70 @@ func TestCosmosDBHappyPath(t *testing.T) { EnsureDelete(ctx, t, tc, dbInstance) } + +func TestCosmosDBControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + rgLocation := tc.resourceGroupLocation + //wrong resource group name + resourceGroupName := "gone" + + cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmosdb", 8) + cosmosDBNamespace := "default" + + dbInstance1 := &v1alpha1.CosmosDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: cosmosDBAccountName, + Namespace: cosmosDBNamespace, + }, + Spec: v1alpha1.CosmosDBSpec{ + Location: rgLocation, + ResourceGroup: resourceGroupName, + Kind: v1alpha1.CosmosDBKindGlobalDocumentDB, + Properties: v1alpha1.CosmosDBProperties{ + DatabaseAccountOfferType: v1alpha1.CosmosDBDatabaseAccountOfferTypeStandard, + }, + }, + } + //the expected error meessage to be shown + errMessage := "Waiting for resource group '" + resourceGroupName + "' to be available" + + EnsureInstanceWithResult(ctx, t, tc, dbInstance1, errMessage, false) + EnsureDelete(ctx, t, tc, dbInstance1) +} + +func TestCosmosDBControllerInvalidLocation(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + resourceGroupName := tc.resourceGroupName + //rglocation doesnot exist + rgLocation := GenerateTestResourceNameWithRandom("cosmos-lo", 10) + + cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmos-db", 8) + cosmosDBNamespace := "default" + + dbInstance2 := &v1alpha1.CosmosDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: cosmosDBAccountName, + Namespace: cosmosDBNamespace, + }, + Spec: v1alpha1.CosmosDBSpec{ + Location: rgLocation, + ResourceGroup: resourceGroupName, + Kind: v1alpha1.CosmosDBKindGlobalDocumentDB, + Properties: v1alpha1.CosmosDBProperties{ + DatabaseAccountOfferType: v1alpha1.CosmosDBDatabaseAccountOfferTypeStandard, + }, + }, + } + + //error meessage to be expected + errMessage := "The specified location '" + rgLocation + "' is invalid" + + EnsureInstanceWithResult(ctx, t, tc, dbInstance2, errMessage, false) + EnsureDelete(ctx, t, tc, dbInstance2) +} diff --git a/controllers/helpers.go b/controllers/helpers.go index 29970bfe123..22153b36a2a 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -247,6 +247,15 @@ func EnsureInstanceWithResult(ctx context.Context, t *testing.T, tc TestContext, statused := ConvertToStatus(instance) // if we expect this resource to end up with provisioned == true then failedProvisioning == true is unrecoverable if provisioned && statused.Status.FailedProvisioning { + if strings.Contains(statused.Status.Message, "already exists") || strings.Contains(statused.Status.Message, "AlreadyExists") { + t.Log("") + t.Log("-------") + t.Log("unexpected failed provisioning encountered") + t.Logf("%+v\n", statused.Status) + t.Logf("current time %v\n", time.Now()) + t.Log("-------") + t.Log("") + } return helpers.NewStop(fmt.Errorf("Failed provisioning: %s", statused.Status.Message)) } if !strings.Contains(statused.Status.Message, message) || statused.Status.Provisioned != provisioned { diff --git a/controllers/postgresql_combined_controller_test.go b/controllers/postgresql_combined_controller_test.go index 9d4e1bb567f..befc6787633 100644 --- a/controllers/postgresql_combined_controller_test.go +++ b/controllers/postgresql_combined_controller_test.go @@ -10,24 +10,18 @@ import ( "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestPSQLDatabaseController(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) var rgName string var rgLocation string var postgreSQLServerName string var postgreSQLServerInstance *azurev1alpha1.PostgreSQLServer - var postgreSQLServerNamespacedName types.NamespacedName - var err error // Add any setup steps that needs to be executed before each test rgName = tc.resourceGroupName @@ -56,20 +50,7 @@ func TestPSQLDatabaseController(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, postgreSQLServerInstance) - assert.Equal(nil, err, "create postgreSQLServerInstance in k8s") - - postgreSQLServerNamespacedName = types.NamespacedName{Name: postgreSQLServerName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance) - return HasFinalizer(postgreSQLServerInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for postgreSQLserver to have finlizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance) - return postgreSQLServerInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for postgreSQLserver to be provisioned") + EnsureInstance(ctx, t, tc, postgreSQLServerInstance) postgreSQLDatabaseName := GenerateTestResourceNameWithRandom("psql-db", 10) @@ -85,28 +66,9 @@ func TestPSQLDatabaseController(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, postgreSQLDatabaseInstance) - assert.Equal(nil, err, "create postgreSQLDatabaseInstance in k8s") - - postgreSQLDatabaseNamespacedName := types.NamespacedName{Name: postgreSQLDatabaseName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance) - return HasFinalizer(postgreSQLDatabaseInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to have finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance) - return postgreSQLDatabaseInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to be provisioned") + EnsureInstance(ctx, t, tc, postgreSQLDatabaseInstance) - err = tc.k8sClient.Delete(ctx, postgreSQLDatabaseInstance) - assert.Equal(nil, err, "delete postgreSQLDatabaseInstance in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, postgreSQLDatabaseInstance) // Test firewall rule ------------------------------- @@ -126,36 +88,11 @@ func TestPSQLDatabaseController(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, postgreSQLFirewallRuleInstance) - assert.Equal(nil, err, "create postgreSQLFirewallRuleInstance in k8s") - - postgreSQLFirewallRuleNamespacedName := types.NamespacedName{Name: postgreSQLFirewallRuleName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance) - return HasFinalizer(postgreSQLFirewallRuleInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to have finalizer") + EnsureInstance(ctx, t, tc, postgreSQLFirewallRuleInstance) - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance) - return postgreSQLFirewallRuleInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to be provisioned") - - err = tc.k8sClient.Delete(ctx, postgreSQLFirewallRuleInstance) - assert.Equal(nil, err, "delete postgreSQLFirewallRuleInstance in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, postgreSQLFirewallRuleInstance) // Add any teardown steps that needs to be executed after each test - err = tc.k8sClient.Delete(ctx, postgreSQLServerInstance) - assert.Equal(nil, err, "delete postgreSQLServerInstance in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for postgreSQLServerInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, postgreSQLServerInstance) } diff --git a/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go b/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go index db505b1806b..e6947b01aa1 100644 --- a/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go @@ -6,6 +6,7 @@ package azuresqldb import ( "context" "fmt" + "strings" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" @@ -65,18 +66,16 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *dbGet.ID return true, nil - } else { - azerr := errhelp.NewAzureErrorAzureError(err) - ignore := []string{ - errhelp.NotFoundErrorCode, - errhelp.ResourceNotFound, - errhelp.ResourceGroupNotFoundErrorCode, - } - if !helpers.ContainsString(ignore, azerr.Type) { - instance.Status.Message = err.Error() - instance.Status.Provisioning = false - return false, fmt.Errorf("AzureSqlDb GetDB error %v", err) - } + } + instance.Status.Message = fmt.Sprintf("AzureSqlDb Get error %s", err.Error()) + azerr := errhelp.NewAzureErrorAzureError(err) + requeuErrors := []string{ + errhelp.ParentNotFoundErrorCode, + errhelp.ResourceGroupNotFoundErrorCode, + } + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil } resp, err := db.CreateOrUpdateDB(ctx, groupName, location, server, labels, azureSQLDatabaseProperties) @@ -100,6 +99,13 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt return false, nil } + // if the database is busy, requeue + errorString := err.Error() + if strings.Contains(errorString, "Try again later") { + instance.Status.Provisioning = false + return false, nil + } + // assertion that a 404 error implies that the Azure SQL server hasn't been provisioned yet if resp != nil && resp.StatusCode == 404 { instance.Status.Message = fmt.Sprintf("Waiting for SQL Server %s to provision", server) diff --git a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go index 72f57c28171..4a32fe22bff 100644 --- a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go @@ -58,6 +58,15 @@ func (fg *AzureSqlFailoverGroupManager) Ensure(ctx context.Context, obj runtime. return true, nil } instance.Status.Message = fmt.Sprintf("AzureSqlFailoverGroup Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } _, err = fg.CreateOrUpdateFailoverGroup(ctx, groupName, serverName, failoverGroupName, sqlFailoverGroupProperties) if err != nil { diff --git a/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go index f5503939ea8..e6b1002fb9b 100644 --- a/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go @@ -37,6 +37,15 @@ func (fw *AzureSqlFirewallRuleManager) Ensure(ctx context.Context, obj runtime.O return true, nil } instance.Status.Message = fmt.Sprintf("AzureSqlFirewallRule Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } _, err = fw.CreateOrUpdateSQLFirewallRule(ctx, groupName, server, ruleName, startIP, endIP) if err != nil { diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 28036683751..be2a5471d8d 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -78,28 +78,44 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op _, err = s.GetDB(ctx, instance.Spec.ResourceGroup, instance.Spec.Server, instance.Spec.DbName) if err != nil { - instance.Status.Message = err.Error() + instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false - catch := []string{ + requeuErrors := []string{ errhelp.ResourceNotFound, errhelp.ParentNotFoundErrorCode, errhelp.ResourceGroupNotFoundErrorCode, } azerr := errhelp.NewAzureErrorAzureError(err) - if helpers.ContainsString(catch, azerr.Type) { + if helpers.ContainsString(requeuErrors, azerr.Type) { return false, nil } - return false, err + + // if the database is busy, requeue + errorString := err.Error() + if strings.Contains(errorString, "Please retry the connection later") { + return false, nil + } + + // if this is an unmarshall error - igmore and continue, otherwise report error and requeue + if !strings.Contains(errorString, "cannot unmarshal array into Go struct field serviceError2.details") { + return false, err + } } db, err := s.ConnectToSqlDb(ctx, DriverName, instance.Spec.Server, instance.Spec.DbName, SqlServerPort, adminUser, adminPassword) if err != nil { instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false // catch firewall issue - keep cycling until it clears up if strings.Contains(err.Error(), "create a firewall rule for this IP address") { - instance.Status.Provisioned = false - instance.Status.Provisioning = false + return false, nil + } + + // if the database is busy, requeue + errorString := err.Error() + if strings.Contains(errorString, "Please retry the connection later") { return false, nil } diff --git a/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go index dc64bfb88df..2d43032eac8 100644 --- a/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go @@ -6,6 +6,7 @@ package azuresqlvnetrule import ( "context" "fmt" + "strings" "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/2015-05-01-preview/sql" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -43,6 +44,15 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec return false, nil } instance.Status.Message = fmt.Sprintf("AzureSqlVNetRule Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } instance.Status.Provisioning = true _, err = vr.CreateOrUpdateSQLVNetRule(ctx, groupName, server, ruleName, virtualNetworkRG, virtualnetworkname, subnetName, ignoreendpoint) @@ -66,6 +76,13 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec return false, nil } + // this happens when we try to create the VNet rule and the server doesnt exist yet + errorString := err.Error() + if strings.Contains(errorString, "does not have the server") { + instance.Status.Provisioning = false + return false, nil + } + return false, err } diff --git a/pkg/resourcemanager/config/env.go b/pkg/resourcemanager/config/env.go index 0359d6b64f7..553dd7393aa 100644 --- a/pkg/resourcemanager/config/env.go +++ b/pkg/resourcemanager/config/env.go @@ -50,7 +50,7 @@ func ParseEnvironment() error { authorizationServerURL = azureEnv.ActiveDirectoryEndpoint baseURI = azureEnv.ResourceManagerEndpoint // BaseURI() - locationDefault = envy.Get("AZURE_LOCATION_DEFAULT", "southcentralus") // DefaultLocation() + locationDefault = envy.Get("AZURE_LOCATION_DEFAULT", "westus2") // DefaultLocation() useDeviceFlow = ParseBoolFromEnvironment("AZURE_USE_DEVICEFLOW") // UseDeviceFlow() useMI = ParseBoolFromEnvironment("AZURE_USE_MI") // UseMI() keepResources = ParseBoolFromEnvironment("AZURE_SAMPLES_KEEP_RESOURCES") // KeepResources() diff --git a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go index bf3d5b82b9a..ea7c46f44a6 100644 --- a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go +++ b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go @@ -125,6 +125,13 @@ func (bc *AzureBlobContainerManager) GetParents(obj runtime.Object) ([]resourcem } return []resourcemanager.KubeParent{ + { + Key: types.NamespacedName{ + Name: instance.Spec.AccountName, + Namespace: instance.Namespace, + }, + Target: &azurev1alpha1.StorageAccount{}, + }, { Key: types.NamespacedName{ Name: instance.Spec.ResourceGroup, @@ -133,6 +140,7 @@ func (bc *AzureBlobContainerManager) GetParents(obj runtime.Object) ([]resourcem Target: &azurev1alpha1.ResourceGroup{}, }, }, nil + } func (bc *AzureBlobContainerManager) GetStatus(obj runtime.Object) (*azurev1alpha1.ASOStatus, error) {