Bug: SQL User error on GetDB in transitional state

pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go

@@ -6,6 +6,7 @@ package azuresqldb
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
 	"github.com/Azure/azure-service-operator/pkg/errhelp"
@@ -65,18 +66,16 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt
 		instance.Status.Message = resourcemanager.SuccessMsg
 		instance.Status.ResourceId = *dbGet.ID
 		return true, nil
-	} else {
-		azerr := errhelp.NewAzureErrorAzureError(err)
-		ignore := []string{
-			errhelp.NotFoundErrorCode,
-			errhelp.ResourceNotFound,
-			errhelp.ResourceGroupNotFoundErrorCode,
-		}
-		if !helpers.ContainsString(ignore, azerr.Type) {
-			instance.Status.Message = err.Error()
-			instance.Status.Provisioning = false
-			return false, fmt.Errorf("AzureSqlDb GetDB error %v", err)
-		}
+	}
+	instance.Status.Message = fmt.Sprintf("AzureSqlDb Get error %s", err.Error())
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	requeuErrors := []string{
+		errhelp.ParentNotFoundErrorCode,
+		errhelp.ResourceGroupNotFoundErrorCode,
+	}
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
 	}
 
 	resp, err := db.CreateOrUpdateDB(ctx, groupName, location, server, labels, azureSQLDatabaseProperties)
@@ -100,6 +99,13 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt
 			return false, nil
 		}
 
+		// if the database is busy, requeue
+		errorString := err.Error()
+		if strings.Contains(errorString, "Try again later") {
+			instance.Status.Provisioning = false
+			return false, nil
+		}
+
 		// assertion that a 404 error implies that the Azure SQL server hasn't been provisioned yet
 		if resp != nil && resp.StatusCode == 404 {
 			instance.Status.Message = fmt.Sprintf("Waiting for SQL Server %s to provision", server)

pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go

@@ -58,6 +58,15 @@ func (fg *AzureSqlFailoverGroupManager) Ensure(ctx context.Context, obj runtime.
 		return true, nil
 	}
 	instance.Status.Message = fmt.Sprintf("AzureSqlFailoverGroup Get error %s", err.Error())
+	requeuErrors := []string{
+		errhelp.ResourceGroupNotFoundErrorCode,
+		errhelp.ParentNotFoundErrorCode,
+	}
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
+	}
 
 	_, err = fg.CreateOrUpdateFailoverGroup(ctx, groupName, serverName, failoverGroupName, sqlFailoverGroupProperties)
 	if err != nil {

pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go

@@ -37,6 +37,15 @@ func (fw *AzureSqlFirewallRuleManager) Ensure(ctx context.Context, obj runtime.O
 		return true, nil
 	}
 	instance.Status.Message = fmt.Sprintf("AzureSqlFirewallRule Get error %s", err.Error())
+	requeuErrors := []string{
+		errhelp.ResourceGroupNotFoundErrorCode,
+		errhelp.ParentNotFoundErrorCode,
+	}
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
+	}
 
 	_, err = fw.CreateOrUpdateSQLFirewallRule(ctx, groupName, server, ruleName, startIP, endIP)
 	if err != nil {

pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go

@@ -77,28 +77,44 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op
 
 	_, err = s.GetDB(ctx, instance.Spec.ResourceGroup, instance.Spec.Server, instance.Spec.DbName)
 	if err != nil {
-		instance.Status.Message = err.Error()
+		instance.Status.Message = fmt.Sprintf("AzureSqlUser Get error %s", err.Error())
+		instance.Status.Provisioning = false
 
-		catch := []string{
+		requeuErrors := []string{
 			errhelp.ResourceNotFound,
 			errhelp.ParentNotFoundErrorCode,
 			errhelp.ResourceGroupNotFoundErrorCode,
 		}
 		azerr := errhelp.NewAzureErrorAzureError(err)
-		if helpers.ContainsString(catch, azerr.Type) {
+		if helpers.ContainsString(requeuErrors, azerr.Type) {
 			return false, nil
 		}
-		return false, err
+
+		// if the database is busy, requeue
+		errorString := err.Error()
+		if strings.Contains(errorString, "Please retry the connection later") {
+			return false, nil
+		}
+
+		// if this is an unmarshall error - igmore and continue, otherwise report error and requeue
+		if !strings.Contains(errorString, "cannot unmarshal array into Go struct field serviceError2.details") {
+			return false, err
+		}
 	}
 
 	db, err := s.ConnectToSqlDb(ctx, DriverName, instance.Spec.Server, instance.Spec.DbName, SqlServerPort, adminUser, adminPassword)
 	if err != nil {
 		instance.Status.Message = errhelp.StripErrorIDs(err)
+		instance.Status.Provisioning = false
 
 		// catch firewall issue - keep cycling until it clears up
 		if strings.Contains(err.Error(), "create a firewall rule for this IP address") {
-			instance.Status.Provisioned = false
-			instance.Status.Provisioning = false
+			return false, nil
+		}
+
+		// if the database is busy, requeue
+		errorString := err.Error()
+		if strings.Contains(errorString, "Please retry the connection later") {
 			return false, nil
 		}
 

pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go

@@ -43,6 +43,15 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec
 		return false, nil
 	}
 	instance.Status.Message = fmt.Sprintf("AzureSqlVNetRule Get error %s", err.Error())
+	requeuErrors := []string{
+		errhelp.ResourceGroupNotFoundErrorCode,
+		errhelp.ParentNotFoundErrorCode,
+	}
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
+	}
 
 	instance.Status.Provisioning = true
 	_, err = vr.CreateOrUpdateSQLVNetRule(ctx, groupName, server, ruleName, virtualNetworkRG, virtualnetworkname, subnetName, ignoreendpoint)