Skip to content

Commit

Permalink
Merge pull request #873 from frodopwns/async-sql-errs2
Browse files Browse the repository at this point in the history
Option 2 for catching async errors in sql server creation
  • Loading branch information
frodopwns authored Apr 3, 2020
2 parents 8bc085b + b90de36 commit 2583d4d
Show file tree
Hide file tree
Showing 8 changed files with 164 additions and 24 deletions.
1 change: 1 addition & 0 deletions api/v1alpha1/aso_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type ASOStatus struct {
State string `json:"state,omitempty"`
Message string `json:"message,omitempty"`
ResourceId string `json:"resourceId,omitempty"`
PollingURL string `json:"pollingUrl,omitempty"`
SpecHash string `json:"specHash,omitempty"`
ContainsUpdate bool `json:"containsUpdate,omitempty"`
RequestedAt *metav1.Time `json:"requested,omitempty"`
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.1.0
github.com/Azure/go-autorest/autorest/to v0.2.0
github.com/Azure/go-autorest/autorest/validation v0.1.0
github.com/Azure/go-autorest/tracing v0.1.0
github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e
github.com/go-logr/logr v0.1.0
github.com/gobuffalo/envy v1.7.0
Expand Down
2 changes: 2 additions & 0 deletions pkg/errhelp/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ const (
NotFoundErrorCode = "NotFound"
NoSuchHost = "no such host"
ParentNotFoundErrorCode = "ParentResourceNotFound"
QuotaExceeded = "QuotaExceeded"
ResourceGroupNotFoundErrorCode = "ResourceGroupNotFound"
RegionDoesNotAllowProvisioning = "RegionDoesNotAllowProvisioning"
ResourceNotFound = "ResourceNotFound"
RequestConflictError = "Conflict"
ValidationError = "ValidationError"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func (s *AzureSqlActionManager) UpdateAdminPassword(ctx context.Context, groupNa
azureSqlServerProperties.AdministratorLoginPassword = to.StringPtr(newPassword)

// Update the SQL server with the newly generated password
_, err = azuresqlserverManager.CreateOrUpdateSQLServer(ctx, groupName, *server.Location, serverName, server.Tags, azureSqlServerProperties, true)
_, _, err = azuresqlserverManager.CreateOrUpdateSQLServer(ctx, groupName, *server.Location, serverName, server.Tags, azureSqlServerProperties, true)

if err != nil {
azerr := errhelp.NewAzureErrorAzureError(err)
Expand Down
14 changes: 9 additions & 5 deletions pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,16 +69,18 @@ func (_ *AzureSqlServerManager) GetServer(ctx context.Context, resourceGroupName
}

// CreateOrUpdateSQLServer creates a SQL server in Azure
func (_ *AzureSqlServerManager) CreateOrUpdateSQLServer(ctx context.Context, resourceGroupName string, location string, serverName string, tags map[string]*string, properties azuresqlshared.SQLServerProperties, forceUpdate bool) (result sql.Server, err error) {
func (_ *AzureSqlServerManager) CreateOrUpdateSQLServer(ctx context.Context, resourceGroupName string, location string, serverName string, tags map[string]*string, properties azuresqlshared.SQLServerProperties, forceUpdate bool) (pollingURL string, result sql.Server, err error) {
serversClient := azuresqlshared.GetGoServersClient()
serverProp := azuresqlshared.SQLServerPropertiesToServer(properties)

if forceUpdate == false {
checkNameResult, _ := CheckNameAvailability(ctx, serverName)
if checkNameResult.Reason == sql.AlreadyExists {
return result, errors.New("AlreadyExists")
err = errors.New("AlreadyExists")
return
} else if checkNameResult.Reason == sql.Invalid {
return result, errors.New("InvalidServerName")
err = errors.New("InvalidServerName")
return
}
}

Expand All @@ -94,10 +96,12 @@ func (_ *AzureSqlServerManager) CreateOrUpdateSQLServer(ctx context.Context, res
})

if err != nil {
return result, err
return "", result, err
}

return future.Result(serversClient)
result, err = future.Result(serversClient)

return future.PollingURL(), result, err
}

func CheckNameAvailability(ctx context.Context, serverName string) (result sql.CheckNameAvailabilityResponse, err error) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
)

type SqlServerManager interface {
CreateOrUpdateSQLServer(ctx context.Context, resourceGroupName string, location string, serverName string, tags map[string]*string, properties azuresqlshared.SQLServerProperties, forceUpdate bool) (result sql.Server, err error)
CreateOrUpdateSQLServer(ctx context.Context, resourceGroupName string, location string, serverName string, tags map[string]*string, properties azuresqlshared.SQLServerProperties, forceUpdate bool) (pollingURL string, result sql.Server, err error)
DeleteSQLServer(ctx context.Context, resourceGroupName string, serverName string) (result autorest.Response, err error)
GetServer(ctx context.Context, resourceGroupName string, serverName string) (result sql.Server, err error)
resourcemanager.ARMClient
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/Azure/azure-service-operator/pkg/helpers"
"github.com/Azure/azure-service-operator/pkg/resourcemanager"
azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared"
"github.com/Azure/azure-service-operator/pkg/resourcemanager/pollclient"
"github.com/Azure/azure-service-operator/pkg/secrets"
"github.com/Azure/go-autorest/autorest/to"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -91,16 +92,9 @@ func (s *AzureSqlServerManager) Ensure(ctx context.Context, obj runtime.Object,
AdministratorLoginPassword: to.StringPtr(string(secret["password"])),
}

// convert kube labels to expected tag format
labels := map[string]*string{}
for k, v := range instance.GetLabels() {
value := v
labels[k] = &value
}

// set a spec hash if one hasn't been set
hash := helpers.Hash256(instance.Spec)
if instance.Status.SpecHash == hash && instance.Status.Provisioned {
if instance.Status.SpecHash == hash && (instance.Status.Provisioned || instance.Status.FailedProvisioning) {
instance.Status.RequestedAt = nil
return true, nil
}
Expand All @@ -114,6 +108,22 @@ func (s *AzureSqlServerManager) Ensure(ctx context.Context, obj runtime.Object,
serv, err := s.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name)
if err != nil {
azerr := errhelp.NewAzureErrorAzureError(err)

// handle failures in the async operation
if instance.Status.PollingURL != "" {
pClient := pollclient.NewPollClient()
res, err := pClient.Get(ctx, instance.Status.PollingURL)
if err != nil {
return false, err
}

if res.Status == "Failed" {
instance.Status.Message = res.Error.Error()
instance.Status.Provisioning = false
return true, nil
}
}

// @Todo: ResourceNotFound should be handled if the time since the last PUT is unreasonable
if azerr.Type != errhelp.ResourceNotFound {
return false, err
Expand All @@ -130,6 +140,8 @@ func (s *AzureSqlServerManager) Ensure(ctx context.Context, obj runtime.Object,
instance.Status.Provisioned = true
instance.Status.Provisioning = false
instance.Status.ResourceId = *serv.ID
instance.Status.SpecHash = hash
instance.Status.PollingURL = ""
return true, nil
}

Expand All @@ -139,7 +151,7 @@ func (s *AzureSqlServerManager) Ensure(ctx context.Context, obj runtime.Object,

// create the sql server
instance.Status.Provisioning = true
if _, err := s.CreateOrUpdateSQLServer(ctx, instance.Spec.ResourceGroup, instance.Spec.Location, instance.Name, tags, azureSQLServerProperties, false); err != nil {
if pollURL, _, err := s.CreateOrUpdateSQLServer(ctx, instance.Spec.ResourceGroup, instance.Spec.Location, instance.Name, tags, azureSQLServerProperties, false); err != nil {

instance.Status.Message = err.Error()

Expand All @@ -151,6 +163,7 @@ func (s *AzureSqlServerManager) Ensure(ctx context.Context, obj runtime.Object,
// the first successful call to create the server should result in this type of error
// we save the credentials here
instance.Status.Message = "Resource request successfully submitted to Azure"
instance.Status.PollingURL = pollURL
return false, nil
case errhelp.AlreadyExists:
// SQL Server names are globally unique so if a server with this name exists we
Expand All @@ -172,14 +185,12 @@ func (s *AzureSqlServerManager) Ensure(ctx context.Context, obj runtime.Object,
instance.Status.Provisioning = false
instance.Status.RequestedAt = nil
return true, nil
case errhelp.LocationNotAvailableForResourceType:
// Subscription does not support the requested service in the requested region
instance.Status.Message = fmt.Sprintf("%s is an invalid location for an Azure SQL Server based on your subscription", instance.Spec.Location)
instance.Status.Provisioning = false
instance.Status.Provisioned = false
return true, nil
case errhelp.RequestDisallowedByPolicy:
instance.Status.Message = "Unable to provision Azure SQL Server due to Azure Policy restrictions contact your policy administrators for further assistance"
case errhelp.LocationNotAvailableForResourceType,
errhelp.RequestDisallowedByPolicy,
errhelp.RegionDoesNotAllowProvisioning,
errhelp.QuotaExceeded:

instance.Status.Message = "Unable to provision Azure SQL Server due to error: " + errhelp.StripErrorIDs(err)
instance.Status.Provisioning = false
instance.Status.Provisioned = false
return true, nil
Expand Down
121 changes: 121 additions & 0 deletions pkg/resourcemanager/pollclient/pollclient.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

package pollclient

import (
"context"
"net/http"

"github.com/Azure/azure-service-operator/pkg/resourcemanager/config"
"github.com/Azure/azure-service-operator/pkg/resourcemanager/iam"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/tracing"
)

const fqdn = "github.com/Azure/azure-service-operator/pollingclient"

// BaseClient was modeled off some of the other Baseclients in the go sdk and contains an autorest client
type BaseClient struct {
autorest.Client
BaseURI string
SubscriptionID string
}

// PollClient inherits from the autorest client and has the methods needed to handle GETs to the polling url
type PollClient struct {
BaseClient
}

// NewPollClient returns a client using hte env values from config
func NewPollClient() PollClient {
return NewPollClientWithBaseURI(config.BaseURI(), config.SubscriptionID())
}

// NewPollClientWithBaseURI returns a paramterized client
func NewPollClientWithBaseURI(baseURI string, subscriptionID string) PollClient {
c := PollClient{NewWithBaseURI(baseURI, subscriptionID)}
a, _ := iam.GetResourceManagementAuthorizer()
c.Authorizer = a
c.AddToUserAgent(config.UserAgent())
return c
}

// NewWithBaseURI creates an instance of the BaseClient client.
func NewWithBaseURI(baseURI string, subscriptionID string) BaseClient {
return BaseClient{
Client: autorest.NewClientWithUserAgent(config.UserAgent()),
BaseURI: baseURI,
SubscriptionID: subscriptionID,
}
}

// PollRespons models the expected response from the poll url
type PollRespons struct {
autorest.Response `json:"-"`
Name string `json:"name,omitempty"`
Status string `json:"status,omitempty"`
Error azure.ServiceError `json:"error,omitempty"`
}

// Get takes a context and a polling url and performs a Get request on the url
func (client PollClient) Get(ctx context.Context, pollURL string) (result PollRespons, err error) {
if tracing.IsEnabled() {
ctx = tracing.StartSpan(ctx, fqdn+"/PollClient.Get")
defer func() {
sc := -1
if result.Response.Response != nil {
sc = result.Response.Response.StatusCode
}
tracing.EndSpan(ctx, sc, err)
}()
}
req, err := client.GetPreparer(ctx, pollURL)
if err != nil {
err = autorest.NewErrorWithError(err, "sql.PollClient", "Get", nil, "Failure preparing request")
return
}

resp, err := client.GetSender(req)
if err != nil {
result.Response = autorest.Response{Response: resp}
err = autorest.NewErrorWithError(err, "sql.PollClient", "Get", resp, "Failure sending request")
return
}

result, err = client.GetResponder(resp)
if err != nil {
err = autorest.NewErrorWithError(err, "sql.PollClient", "Get", resp, "Failure responding to request")
}

return
}

// GetPreparer prepares the Get request.
func (client PollClient) GetPreparer(ctx context.Context, pollURL string) (*http.Request, error) {
preparer := autorest.CreatePreparer(
autorest.AsGet(),
autorest.WithBaseURL(pollURL))
return preparer.Prepare((&http.Request{}).WithContext(ctx))
}

// GetSender sends the Get request. The method will close the
// http.Response Body if it receives an error.
func (client PollClient) GetSender(req *http.Request) (*http.Response, error) {
sd := autorest.GetSendDecorators(req.Context(), azure.DoRetryWithRegistration(client.Client))
return autorest.SendWithSender(client, req, sd...)
}

// GetResponder handles the response to the Get request. The method always
// closes the http.Response Body.
func (client PollClient) GetResponder(resp *http.Response) (result PollRespons, err error) {
err = autorest.Respond(
resp,
client.ByInspecting(),
azure.WithErrorUnlessStatusCode(http.StatusOK),
autorest.ByUnmarshallingJSON(&result),
autorest.ByClosing())
result.Response = autorest.Response{Response: resp}
return
}

0 comments on commit 2583d4d

Please sign in to comment.