From 4ab6d2043a225413e13505d2e4afaee992602e7c Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Wed, 8 Apr 2020 06:42:59 +0800 Subject: [PATCH 01/99] issue#858#WIPaddVnetStorageAccount --- api/v1alpha1/storage_types.go | 42 +++++++++++ config/samples/azure_v1alpha1_storage.yaml | 18 ++++- .../storages/storageaccount/storage.go | 69 ++++++++++++++++++- .../storageaccount/storage_manager.go | 4 +- .../storageaccount/storage_reconcile.go | 3 +- 5 files changed, 130 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/storage_types.go b/api/v1alpha1/storage_types.go index 682c81ffc95..457582ab32e 100644 --- a/api/v1alpha1/storage_types.go +++ b/api/v1alpha1/storage_types.go @@ -29,6 +29,8 @@ type StorageSpec struct { EnableHTTPSTrafficOnly *bool `json:"supportsHttpsTrafficOnly,omitempty"` DataLakeEnabled *bool `json:"dataLakeEnabled,omitempty"` + + NetworkRule *StorageNetworkRuleSet `json:"networkRule,omitempty"` } // Sku the SKU of the storage account. @@ -96,6 +98,46 @@ type StorageList struct { Items []Storage `json:"items"` } +type Bypass string +type Action string + +type StorageNetworkRuleSet struct { + // Bypass - Specifies whether traffic is bypassed for Logging/Metrics/AzureServices. Possible values are any combination of Logging|Metrics|AzureServices (For example, "Logging, Metrics"), or None to bypass none of those traffics. Possible values include: 'None', 'Logging', 'Metrics', 'AzureServices' + Bypass Bypass `json:"bypass,omitempty"` + // VirtualNetworkRules - Sets the virtual network rules + VirtualNetworkRules *[]VirtualNetworkRule `json:"virtualNetworkRules,omitempty"` + // IPRules - Sets the IP ACL rules + IPRule *[]IPRule `json:"ipRules,omitempty"` + // DefaultAction - Specifies the default action of allow or deny when no other rules match. Possible values include: 'DefaultActionAllow', 'DefaultActionDeny' + DefaultAction DefaultAction `json:"defaultAction,omitempty"` +} + +// DefaultAction defined as a string + +const ( + + // AzureServices ... + AzureServices Bypass = "AzureServices" + // Logging ... + Logging Bypass = "Logging" + // Metrics ... + Metrics Bypass = "Metrics" + // None ... + None Bypass = "None" +) + +type VirtualNetworkRule struct { + // VirtualNetworkResourceID - Resource ID of a subnet, for example: /subscriptions/{subscriptionId}/resourceGroups/{groupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}. + VirtualNetworkResourceID *string `json:"id,omitempty"` + Action Action `json:"action,omitempty"` +} + +type IPRule struct { + // IPAddressOrRange - Specifies the IP or IP range in CIDR format. Only IPV4 address is allowed. + IPAddressOrRange *string `json:"value,omitempty"` + Action Action `json:"action,omitempty"` +} + func init() { SchemeBuilder.Register(&Storage{}, &StorageList{}) } diff --git a/config/samples/azure_v1alpha1_storage.yaml b/config/samples/azure_v1alpha1_storage.yaml index fd2f6690787..a3eeb567636 100644 --- a/config/samples/azure_v1alpha1_storage.yaml +++ b/config/samples/azure_v1alpha1_storage.yaml @@ -1,12 +1,24 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: Storage metadata: - name: storagesample1908ayzkj + name: storagesamplehong222 spec: location: westus resourceGroup: resourcegroup-azure-operators sku: name: Standard_RAGRS kind: StorageV2 - accessTier: Hot - supportsHttpsTrafficOnly: true + properties: + accessTier: Hot + supportsHttpsTrafficOnly: true + +# Optional: networkRule is only supported on "Standard" tier namespace + networkRule: + bypass: AzureServices + defaultAction: Allow + virtualNetworkRules: + - subnetID: /subscriptions/08daa385-27fa-477a-b556-a9ead8b270d9/resourceGroups/resourcegroup-azure-operator/providers/Microsoft.Network/virtualNetworks/virtualnetwork-sample/subnets/test1 +# ipRules: +# - ipMask: 10.0.0.0/24 +# - ipMask: 2.2.0.0/24 + diff --git a/pkg/resourcemanager/storages/storageaccount/storage.go b/pkg/resourcemanager/storages/storageaccount/storage.go index b43cdd1475b..ec74ca48720 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage.go +++ b/pkg/resourcemanager/storages/storageaccount/storage.go @@ -9,6 +9,7 @@ import ( "log" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + "github.com/Azure/azure-service-operator/api/v1alpha1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" @@ -18,6 +19,63 @@ import ( type azureStorageManager struct{} +// ParseNetworkPolicy - helper function to parse network policies from Kubernetes spec +func ParseNetworkPolicy(ruleSet *v1alpha1.StorageNetworkRuleSet) storage.NetworkRuleSet { + var bypass storage.Bypass + + switch ruleSet.Bypass { + case "AzureServices": + bypass = storage.AzureServices + case "None": + bypass = storage.None + case "Logging": + bypass = storage.Logging + case "Metrics": + bypass = storage.Metrics + default: + bypass = storage.AzureServices + } + + var defaultAction storage.DefaultAction + switch ruleSet.DefaultAction { + case "Allow": + defaultAction = storage.DefaultActionAllow + case "Deny": + defaultAction = storage.DefaultActionDeny + default: + defaultAction = storage.DefaultActionDeny + } + + var ipInstances []storage.IPRule + if ruleSet.IPRule != nil { + for _, i := range *ruleSet.IPRule { + subnetID := i.IPAddressOrRange + ipInstances = append(ipInstances, storage.IPRule{ + IPAddressOrRange: subnetID, + Action: storage.Allow, + }) + } + } + + var vnetInstances []storage.VirtualNetworkRule + if ruleSet.VirtualNetworkRules != nil { + for _, i := range *ruleSet.VirtualNetworkRules { + ventID := i.VirtualNetworkResourceID + vnetInstances = append(vnetInstances, storage.VirtualNetworkRule{ + VirtualNetworkResourceID: ventID, + Action: storage.Allow, + }) + } + } + + return storage.NetworkRuleSet{ + Bypass: bypass, + DefaultAction: defaultAction, + IPRules: &ipInstances, + VirtualNetworkRules: &vnetInstances, + } +} + func getStoragesClient() storage.AccountsClient { storagesClient := storage.NewAccountsClientWithBaseURI(config.BaseURI(), config.SubscriptionID()) a, err := iam.GetResourceManagementAuthorizer() @@ -30,7 +88,8 @@ func getStoragesClient() storage.AccountsClient { } // CreateStorage creates a new storage account -func (_ *azureStorageManager) CreateStorage(ctx context.Context, groupName string, +func (_ *azureStorageManager) CreateStorage(ctx context.Context, instance *v1alpha1.Storage, + groupName string, storageAccountName string, location string, sku azurev1alpha1.StorageSku, @@ -63,6 +122,13 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, groupName strin sKind := storage.Kind(kind) sAccessTier := storage.AccessTier(accessTier) + var networkAcls storage.NetworkRuleSet + if instance.Spec.NetworkRule != nil { + networkAcls = ParseNetworkPolicy(instance.Spec.NetworkRule) + } else { + networkAcls = storage.NetworkRuleSet{} + } + params := storage.AccountCreateParameters{ Location: to.StringPtr(location), Sku: &sSku, @@ -73,6 +139,7 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, groupName strin AccessTier: sAccessTier, EnableHTTPSTrafficOnly: enableHTTPsTrafficOnly, IsHnsEnabled: dataLakeEnabled, + NetworkRuleSet: &networkAcls, }, } diff --git a/pkg/resourcemanager/storages/storageaccount/storage_manager.go b/pkg/resourcemanager/storages/storageaccount/storage_manager.go index 80437e7b729..08b7c6dba4e 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_manager.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_manager.go @@ -7,6 +7,7 @@ import ( "context" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + "github.com/Azure/azure-service-operator/api/v1alpha1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/resourcemanager" "github.com/Azure/go-autorest/autorest" @@ -18,7 +19,8 @@ func New() *azureStorageManager { } type StorageManager interface { - CreateStorage(ctx context.Context, groupName string, + CreateStorage(ctx context.Context, instance *v1alpha1.Storage, + groupName string, storageAccountName string, location string, sku azurev1alpha1.StorageSku, diff --git a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go index 55ebed67f06..aff0bccadc8 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -66,7 +67,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o instance.Status.Provisioning = true instance.Status.Provisioned = false - _, err = sa.CreateStorage(ctx, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled) + _, err = sa.CreateStorage(ctx, instance, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled) if err != nil { instance.Status.Message = err.Error() azerr := errhelp.NewAzureErrorAzureError(err) From d9c1291c38c110e88b648988e75fc95cd7d4712e Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Tue, 7 Apr 2020 17:58:47 -0600 Subject: [PATCH 02/99] Eventhub storage account struct switch --- api/v1alpha1/eventhub_types.go | 7 ++++--- api/v1alpha1/eventhub_types_test.go | 2 +- config/samples/azure_v1alpha1_eventhub_capture.yaml | 2 +- controllers/eventhub_controller_test.go | 2 +- pkg/resourcemanager/eventhubs/hub.go | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/v1alpha1/eventhub_types.go b/api/v1alpha1/eventhub_types.go index addfdfbbbdd..9ec9a14472d 100644 --- a/api/v1alpha1/eventhub_types.go +++ b/api/v1alpha1/eventhub_types.go @@ -33,7 +33,8 @@ type EventhubAuthorizationRule struct { Rights []string `json:"rights,omitempty"` } -type StorageAccount struct { +//EventHubStorageAccount contains details of the eventhub storage account +type EventHubStorageAccount struct { // ResourceGroup - Name of the storage account resource group // +kubebuilder:validation:Pattern=^[-\w\._\(\)]+$ ResourceGroup string `json:"resourceGroup,omitempty"` @@ -53,8 +54,8 @@ type Destination struct { // Name - Name for capture destination // +kubebuilder:validation:Enum=EventHubArchive.AzureBlockBlob;EventHubArchive.AzureDataLake Name string `json:"name,omitempty"` - // StorageAccount - Details of the storage account - StorageAccount StorageAccount `json:"storageAccount,omitempty"` + // EventHubStorageAccount - Details of the storage account + EventHubStorageAccount EventHubStorageAccount `json:"eventHubStorageAccount,omitempty"` } //CaptureDescription defines the properties required for eventhub capture diff --git a/api/v1alpha1/eventhub_types_test.go b/api/v1alpha1/eventhub_types_test.go index 8d84e82658f..af06cf0329a 100644 --- a/api/v1alpha1/eventhub_types_test.go +++ b/api/v1alpha1/eventhub_types_test.go @@ -86,7 +86,7 @@ var _ = Describe("Eventhub", func() { ArchiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}", BlobContainer: "foo-blob-container", Name: "EventHubArchive.AzureBlockBlob", - StorageAccount: StorageAccount{ + EventHubStorageAccount: EventHubStorageAccount{ ResourceGroup: "foo-resource-group", AccountName: "fooaccountname", }, diff --git a/config/samples/azure_v1alpha1_eventhub_capture.yaml b/config/samples/azure_v1alpha1_eventhub_capture.yaml index e8c38564849..b28af3ade28 100644 --- a/config/samples/azure_v1alpha1_eventhub_capture.yaml +++ b/config/samples/azure_v1alpha1_eventhub_capture.yaml @@ -15,7 +15,7 @@ spec: archiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}" blobContainer: "capturecontainer" name: "EventHubArchive.AzureBlockBlob" - storageAccount: + eventHubStorageAccount: resourceGroup: "my-resource-group" accountName: "storageaccountauv1" enabled: true diff --git a/controllers/eventhub_controller_test.go b/controllers/eventhub_controller_test.go index 3faa18626bd..a68109680f4 100644 --- a/controllers/eventhub_controller_test.go +++ b/controllers/eventhub_controller_test.go @@ -267,7 +267,7 @@ func TestEventHubControllerCreateAndDeleteCapture(t *testing.T) { ArchiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}", BlobContainer: bcName, Name: "EventHubArchive.AzureBlockBlob", - StorageAccount: azurev1alpha1.StorageAccount{ + EventHubStorageAccount: azurev1alpha1.EventHubStorageAccount{ ResourceGroup: rgName, AccountName: saName, }, diff --git a/pkg/resourcemanager/eventhubs/hub.go b/pkg/resourcemanager/eventhubs/hub.go index a789f1f91cf..723376595c4 100644 --- a/pkg/resourcemanager/eventhubs/hub.go +++ b/pkg/resourcemanager/eventhubs/hub.go @@ -383,7 +383,7 @@ func getCaptureDescriptionPtr(captureDescription azurev1alpha1.CaptureDescriptio // add capture details var capturePtr *model.CaptureDescription - storage := captureDescription.Destination.StorageAccount + storage := captureDescription.Destination.EventHubStorageAccount storageAccountResourceID := fmt.Sprintf(storageAccountResourceFmt, config.SubscriptionID(), storage.ResourceGroup, storage.AccountName) if captureDescription.Enabled { From 3681ba69e71c363a7fe2d4424fe837a345331d8d Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Tue, 7 Apr 2020 18:31:34 -0600 Subject: [PATCH 03/99] File renames to storageaccount --- ...alpha1_storage.yaml => azure_v1alpha1_storageaccount.yaml} | 2 +- .../{storage_controller.go => storageaccount_controller.go} | 0 main.go | 4 ++-- pkg/resourcemanager/mock/eventhubs/hub.go | 2 +- .../storages/storageaccount/{storage.go => storageaccount.go} | 0 .../{storage_manager.go => storageaccount_manager.go} | 0 .../{storage_reconcile.go => storageaccount_reconcile.go} | 0 7 files changed, 4 insertions(+), 4 deletions(-) rename config/samples/{azure_v1alpha1_storage.yaml => azure_v1alpha1_storageaccount.yaml} (88%) rename controllers/{storage_controller.go => storageaccount_controller.go} (100%) rename pkg/resourcemanager/storages/storageaccount/{storage.go => storageaccount.go} (100%) rename pkg/resourcemanager/storages/storageaccount/{storage_manager.go => storageaccount_manager.go} (100%) rename pkg/resourcemanager/storages/storageaccount/{storage_reconcile.go => storageaccount_reconcile.go} (100%) diff --git a/config/samples/azure_v1alpha1_storage.yaml b/config/samples/azure_v1alpha1_storageaccount.yaml similarity index 88% rename from config/samples/azure_v1alpha1_storage.yaml rename to config/samples/azure_v1alpha1_storageaccount.yaml index fd2f6690787..11015855e91 100644 --- a/config/samples/azure_v1alpha1_storage.yaml +++ b/config/samples/azure_v1alpha1_storageaccount.yaml @@ -1,7 +1,7 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: Storage metadata: - name: storagesample1908ayzkj + name: storageaccount-sample-1 spec: location: westus resourceGroup: resourcegroup-azure-operators diff --git a/controllers/storage_controller.go b/controllers/storageaccount_controller.go similarity index 100% rename from controllers/storage_controller.go rename to controllers/storageaccount_controller.go diff --git a/main.go b/main.go index ddf3ab67ccb..0485f0e48cc 100644 --- a/main.go +++ b/main.go @@ -35,7 +35,7 @@ import ( resourcemanagerresourcegroup "github.com/Azure/azure-service-operator/pkg/resourcemanager/resourcegroups" resourcemanagerstorage "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages" blobContainerManager "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/blobcontainer" - stoageaccountManager "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/storageaccount" + storageaccountManager "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/storageaccount" vnet "github.com/Azure/azure-service-operator/pkg/resourcemanager/vnet" "github.com/Azure/azure-service-operator/pkg/secrets" keyvaultSecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" @@ -153,7 +153,7 @@ func main() { err = (&controllers.StorageReconciler{ Reconciler: &controllers.AsyncReconciler{ Client: mgr.GetClient(), - AzureClient: stoageaccountManager.New(), + AzureClient: storageaccountManager.New(), Telemetry: telemetry.InitializeTelemetryDefault( "Storage", ctrl.Log.WithName("controllers").WithName("Storage"), diff --git a/pkg/resourcemanager/mock/eventhubs/hub.go b/pkg/resourcemanager/mock/eventhubs/hub.go index 75b711679d1..71f236e0132 100644 --- a/pkg/resourcemanager/mock/eventhubs/hub.go +++ b/pkg/resourcemanager/mock/eventhubs/hub.go @@ -260,7 +260,7 @@ func getCaptureDescriptionPtr(captureDescription azurev1alpha1.CaptureDescriptio // add capture details var capturePtr *eventhub.CaptureDescription - storage := captureDescription.Destination.StorageAccount + storage := captureDescription.Destination.EventHubStorageAccount storageAccountResourceID := fmt.Sprintf(storageAccountResourceFmt, config.SubscriptionID(), storage.ResourceGroup, storage.AccountName) if captureDescription.Enabled { diff --git a/pkg/resourcemanager/storages/storageaccount/storage.go b/pkg/resourcemanager/storages/storageaccount/storageaccount.go similarity index 100% rename from pkg/resourcemanager/storages/storageaccount/storage.go rename to pkg/resourcemanager/storages/storageaccount/storageaccount.go diff --git a/pkg/resourcemanager/storages/storageaccount/storage_manager.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go similarity index 100% rename from pkg/resourcemanager/storages/storageaccount/storage_manager.go rename to pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go diff --git a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go similarity index 100% rename from pkg/resourcemanager/storages/storageaccount/storage_reconcile.go rename to pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go From 6cc931fcacb6c2b12b88eef828744189d4760158 Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Tue, 7 Apr 2020 22:34:44 -0600 Subject: [PATCH 04/99] var changes to storageaccount across files --- .../{storage_types.go => storageaccount_types.go} | 12 ++++++------ controllers/storageaccount_controller.go | 12 ++++++------ ...ler_test.go => storageaccount_controller_test.go} | 2 +- controllers/suite_test.go | 8 ++++---- main.go | 8 ++++---- ...ainer_reconlie.go => blob_container_reconcile.go} | 6 +++--- .../storageaccount/storageaccount_reconcile.go | 10 +++++----- 7 files changed, 29 insertions(+), 29 deletions(-) rename api/v1alpha1/{storage_types.go => storageaccount_types.go} (93%) rename controllers/{storage_controller_test.go => storageaccount_controller_test.go} (95%) rename pkg/resourcemanager/storages/blobcontainer/{blob_container_reconlie.go => blob_container_reconcile.go} (97%) diff --git a/api/v1alpha1/storage_types.go b/api/v1alpha1/storageaccount_types.go similarity index 93% rename from api/v1alpha1/storage_types.go rename to api/v1alpha1/storageaccount_types.go index 682c81ffc95..f4544a56d18 100644 --- a/api/v1alpha1/storage_types.go +++ b/api/v1alpha1/storageaccount_types.go @@ -62,8 +62,8 @@ type StorageAccessTier string // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// Storage is the Schema for the storages API -type Storage struct { +// StorageAccount is the Schema for the storages API +type StorageAccount struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -93,13 +93,13 @@ type StorageAdditionalResources struct { type StorageList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []Storage `json:"items"` + Items []StorageAccount `json:"items"` } func init() { - SchemeBuilder.Register(&Storage{}, &StorageList{}) + SchemeBuilder.Register(&StorageAccount{}, &StorageList{}) } - -func (storage *Storage) IsSubmitted() bool { +i +func (storage *StorageAccount) IsSubmitted() bool { return storage.Status.Provisioning || storage.Status.Provisioned } diff --git a/controllers/storageaccount_controller.go b/controllers/storageaccount_controller.go index 6285f11cdbb..78460c2f273 100644 --- a/controllers/storageaccount_controller.go +++ b/controllers/storageaccount_controller.go @@ -11,8 +11,8 @@ import ( const storageFinalizerName = "storage.finalizers.azure.com" -// StorageReconciler reconciles a Storage object -type StorageReconciler struct { +// StorageAccountReconciler reconciles a Storage Account object +type StorageAccountReconciler struct { Reconciler *AsyncReconciler } @@ -20,13 +20,13 @@ type StorageReconciler struct { // +kubebuilder:rbac:groups=azure.microsoft.com,resources=storages/status,verbs=get;update;patch // Reconcile function does the main reconciliation loop of the operator -func (r *StorageReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - return r.Reconciler.Reconcile(req, &azurev1alpha1.Storage{}) +func (r *StorageAccountReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + return r.Reconciler.Reconcile(req, &azurev1alpha1.StorageAccount{}) } // SetupWithManager sets up the controller functions -func (r *StorageReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *StorageAccountReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&azurev1alpha1.Storage{}). + For(&azurev1alpha1.StorageAccount{}). Complete(r) } diff --git a/controllers/storage_controller_test.go b/controllers/storageaccount_controller_test.go similarity index 95% rename from controllers/storage_controller_test.go rename to controllers/storageaccount_controller_test.go index f4a6c00f9a7..ed165c0642a 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storageaccount_controller_test.go @@ -23,7 +23,7 @@ func TestStorageControllerHappyPath(t *testing.T) { StorageAccountName := GenerateAlphaNumTestResourceName("sadev") // Create the ResourceGroup object and expect the Reconcile to be created - saInstance := &azurev1alpha1.Storage{ + saInstance := &azurev1alpha1.StorageAccount{ ObjectMeta: metav1.ObjectMeta{ Name: StorageAccountName, Namespace: "default", diff --git a/controllers/suite_test.go b/controllers/suite_test.go index b15a55042e8..1446d267005 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -609,15 +609,15 @@ func setup() error { return err } - err = (&StorageReconciler{ + err = (&StorageAccountReconciler{ Reconciler: &AsyncReconciler{ Client: k8sManager.GetClient(), AzureClient: storageAccountManager, Telemetry: telemetry.InitializeTelemetryDefault( - "Storage", - ctrl.Log.WithName("controllers").WithName("Storage"), + "StorageAccount", + ctrl.Log.WithName("controllers").WithName("StorageAccount"), ), - Recorder: k8sManager.GetEventRecorderFor("Storage-controller"), + Recorder: k8sManager.GetEventRecorderFor("StorageAccount-controller"), Scheme: scheme.Scheme, }, }).SetupWithManager(k8sManager) diff --git a/main.go b/main.go index 0485f0e48cc..ba5a10dd8d2 100644 --- a/main.go +++ b/main.go @@ -150,15 +150,15 @@ func main() { ) sqlActionManager := resourcemanagersqlaction.NewAzureSqlActionManager(secretClient, scheme) - err = (&controllers.StorageReconciler{ + err = (&controllers.StorageAccountReconciler{ Reconciler: &controllers.AsyncReconciler{ Client: mgr.GetClient(), AzureClient: storageaccountManager.New(), Telemetry: telemetry.InitializeTelemetryDefault( - "Storage", - ctrl.Log.WithName("controllers").WithName("Storage"), + "StorageAccount", + ctrl.Log.WithName("controllers").WithName("StorageAccount"), ), - Recorder: mgr.GetEventRecorderFor("Storage-controller"), + Recorder: mgr.GetEventRecorderFor("StorageAccount-controller"), Scheme: scheme, }, }).SetupWithManager(mgr) diff --git a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconlie.go b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go similarity index 97% rename from pkg/resourcemanager/storages/blobcontainer/blob_container_reconlie.go rename to pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go index 605a0572584..bf3d5b82b9a 100644 --- a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconlie.go +++ b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go @@ -16,7 +16,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// Ensure creates an AzureSqlDb +// Ensure creates a blob container func (bc *AzureBlobContainerManager) Ensure(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { instance, err := bc.convert(obj) @@ -81,7 +81,7 @@ func (bc *AzureBlobContainerManager) Ensure(ctx context.Context, obj runtime.Obj return true, nil } -// Delete drops a AzureSqlDb +// Delete drops a blob container func (bc *AzureBlobContainerManager) Delete(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { instance, err := bc.convert(obj) if err != nil { @@ -117,7 +117,7 @@ func (bc *AzureBlobContainerManager) Delete(ctx context.Context, obj runtime.Obj return false, nil } -// GetParents returns the parents of AzureSqlDatabase +// GetParents returns the parents of a blob container func (bc *AzureBlobContainerManager) GetParents(obj runtime.Object) ([]resourcemanager.KubeParent, error) { instance, err := bc.convert(obj) if err != nil { diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go index 55ebed67f06..488a3b73d01 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go @@ -15,7 +15,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// Ensure creates an AzureSqlDb +// Ensure creates a storage account func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { instance, err := sa.convert(obj) @@ -122,7 +122,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o return false, nil } -// Delete drops a AzureSqlDb +// Delete drops a storage account func (sa *azureStorageManager) Delete(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { instance, err := sa.convert(obj) if err != nil { @@ -154,7 +154,7 @@ func (sa *azureStorageManager) Delete(ctx context.Context, obj runtime.Object, o return true, nil } -// GetParents returns the parents of AzureSqlDatabase +// GetParents returns the parents of a storage account func (sa *azureStorageManager) GetParents(obj runtime.Object) ([]resourcemanager.KubeParent, error) { instance, err := sa.convert(obj) if err != nil { @@ -180,8 +180,8 @@ func (g *azureStorageManager) GetStatus(obj runtime.Object) (*azurev1alpha1.ASOS return &instance.Status, nil } -func (sa *azureStorageManager) convert(obj runtime.Object) (*azurev1alpha1.Storage, error) { - local, ok := obj.(*azurev1alpha1.Storage) +func (sa *azureStorageManager) convert(obj runtime.Object) (*azurev1alpha1.StorageAccount, error) { + local, ok := obj.(*azurev1alpha1.StorageAccount) if !ok { return nil, fmt.Errorf("failed type assertion on kind: %s", obj.GetObjectKind().GroupVersionKind().String()) } From 24b8092ad671823b0b71e71b45e2ea3480b67871 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Wed, 8 Apr 2020 21:42:54 +0800 Subject: [PATCH 05/99] issue#858AddVentStorage0.2 --- api/v1alpha1/storage_types.go | 19 +++--- config/samples/azure_v1alpha1_storage.yaml | 18 +++--- controllers/storage_controller_test.go | 61 +++++++++++++++++++ controllers/suite_test.go | 2 +- pkg/resourcemanager/mock/storages/storage.go | 2 +- .../storageaccount/storage_manager.go | 5 +- .../storageaccount/storage_reconcile.go | 8 ++- 7 files changed, 89 insertions(+), 26 deletions(-) diff --git a/api/v1alpha1/storage_types.go b/api/v1alpha1/storage_types.go index 457582ab32e..a61c370f1d9 100644 --- a/api/v1alpha1/storage_types.go +++ b/api/v1alpha1/storage_types.go @@ -99,21 +99,20 @@ type StorageList struct { } type Bypass string -type Action string type StorageNetworkRuleSet struct { - // Bypass - Specifies whether traffic is bypassed for Logging/Metrics/AzureServices. Possible values are any combination of Logging|Metrics|AzureServices (For example, "Logging, Metrics"), or None to bypass none of those traffics. Possible values include: 'None', 'Logging', 'Metrics', 'AzureServices' + // Bypass - Specifies whether traffic is bypassed for Logging/Metrics/AzureServices. + //Possible values are any combination of Logging|Metrics|AzureServices (For example, "Logging, Metrics"), or None to bypass none of those traffics. + //Possible values include: 'None', 'Logging', 'Metrics', 'AzureServices' Bypass Bypass `json:"bypass,omitempty"` // VirtualNetworkRules - Sets the virtual network rules VirtualNetworkRules *[]VirtualNetworkRule `json:"virtualNetworkRules,omitempty"` // IPRules - Sets the IP ACL rules - IPRule *[]IPRule `json:"ipRules,omitempty"` + IPRules *[]IPRule `json:"ipRules,omitempty"` // DefaultAction - Specifies the default action of allow or deny when no other rules match. Possible values include: 'DefaultActionAllow', 'DefaultActionDeny' - DefaultAction DefaultAction `json:"defaultAction,omitempty"` + DefaultAction string `json:"defaultAction,omitempty"` } -// DefaultAction defined as a string - const ( // AzureServices ... @@ -127,15 +126,13 @@ const ( ) type VirtualNetworkRule struct { - // VirtualNetworkResourceID - Resource ID of a subnet, for example: /subscriptions/{subscriptionId}/resourceGroups/{groupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}. - VirtualNetworkResourceID *string `json:"id,omitempty"` - Action Action `json:"action,omitempty"` + // SubnetId - Resource ID of a subnet, for example: /subscriptions/{subscriptionId}/resourceGroups/{groupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}. + SubnetId *string `json:"subnetId,omitempty"` } type IPRule struct { // IPAddressOrRange - Specifies the IP or IP range in CIDR format. Only IPV4 address is allowed. - IPAddressOrRange *string `json:"value,omitempty"` - Action Action `json:"action,omitempty"` + IPAddressOrRange *string `json:"ipAddressOrRange,omitempty"` } func init() { diff --git a/config/samples/azure_v1alpha1_storage.yaml b/config/samples/azure_v1alpha1_storage.yaml index a3eeb567636..02b25809087 100644 --- a/config/samples/azure_v1alpha1_storage.yaml +++ b/config/samples/azure_v1alpha1_storage.yaml @@ -1,7 +1,7 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: Storage metadata: - name: storagesamplehong222 + name: storagesamplegfdsa spec: location: westus resourceGroup: resourcegroup-azure-operators @@ -11,14 +11,14 @@ spec: properties: accessTier: Hot supportsHttpsTrafficOnly: true - -# Optional: networkRule is only supported on "Standard" tier namespace +# Optional: networkRule networkRule: - bypass: AzureServices - defaultAction: Allow + bypass: AzureServices # Possible values are AzureServices, Metrics, None, Logging + defaultAction: Allow # Possible values are Allow, Deny virtualNetworkRules: - - subnetID: /subscriptions/08daa385-27fa-477a-b556-a9ead8b270d9/resourceGroups/resourcegroup-azure-operator/providers/Microsoft.Network/virtualNetworks/virtualnetwork-sample/subnets/test1 -# ipRules: -# - ipMask: 10.0.0.0/24 -# - ipMask: 2.2.0.0/24 + - subnetId: /subscriptions/08daa385-27fa-477a-b556-a9ead8b270d9/resourceGroups/resourcegroup-azure-operator/providers/Microsoft.Network/virtualNetworks/virtualnetwork-sample/subnets/test1 + ipRules: + - ipAddressOrRange: 2.2.0.0/24 + - ipAddressOrRange: 2.2.2.1 + diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index f4a6c00f9a7..b8ce7f1fbaf 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -10,6 +10,7 @@ import ( "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + config "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/go-autorest/autorest/to" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,3 +47,63 @@ func TestStorageControllerHappyPath(t *testing.T) { // delete rg EnsureDelete(ctx, t, tc, saInstance) } + +func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + StorageAccountName := GenerateAlphaNumTestResourceName("cndev") + + rgName := tc.resourceGroupName + rgLocation := tc.resourceGroupLocation + VNetName := GenerateTestResourceNameWithRandom("vnet", 10) + subnetName := "subnet-test" + + subnetID := "/subscriptions/" + config.SubscriptionID() + "/resourceGroups/" + rgName + "/providers/Microsoft.Network/virtualNetworks/" + VNetName + "/subnets/" + subnetName + vnetRules := []azurev1alpha1.VirtualNetworkRule{ + { + SubnetId: &subnetID, + }, + } + ipAddress := "1.1.1.1" + ipRange := "2.2.2.2/24" + ipRules := []azurev1alpha1.IPRule{ + { + IPAddressOrRange: &ipAddress, + }, + { + IPAddressOrRange: &ipRange, + }, + } + + // Create the ResourceGroup object and expect the Reconcile to be created + cnInstance := &azurev1alpha1.Storage{ + ObjectMeta: metav1.ObjectMeta{ + Name: StorageAccountName, + Namespace: "default", + }, + Spec: azurev1alpha1.StorageSpec{ + Location: rgLocation, + ResourceGroup: rgName, + Sku: azurev1alpha1.StorageSku{ + Name: "Standard_RAGRS", + }, + Kind: "StorageV2", + AccessTier: "Hot", + EnableHTTPSTrafficOnly: to.BoolPtr(true), + NetworkRule: &azurev1alpha1.StorageNetworkRuleSet{ + Bypass: "AzureServices", + VirtualNetworkRules: &vnetRules, + IPRules: &ipRules, + DefaultAction: "Deny", + }, + }, + } + + // create rg + EnsureInstance(ctx, t, tc, cnInstance) + + // delete rg + EnsureDelete(ctx, t, tc, cnInstance) +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 63f50e48416..048ad379567 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -706,7 +706,7 @@ func setup() error { // Create the Storage Account and Container _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageSku{ Name: "Standard_LRS", - }, "Storage", map[string]*string{}, "", nil, nil) + }, "Storage", map[string]*string{}, "", nil, nil, nil) // Storage account needs to be in "Suceeded" state // for container create to succeed diff --git a/pkg/resourcemanager/mock/storages/storage.go b/pkg/resourcemanager/mock/storages/storage.go index 92cc51d207f..7c4d0be5b12 100644 --- a/pkg/resourcemanager/mock/storages/storage.go +++ b/pkg/resourcemanager/mock/storages/storage.go @@ -50,7 +50,7 @@ func (manager *mockStorageManager) CreateStorage(ctx context.Context, groupName kind azurev1alpha1.StorageKind, tags map[string]*string, accessTier azurev1alpha1.StorageAccessTier, - enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool) (result storage.Account, err error) { + enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *azurev1alpha1.NetworkRuleSet) (result storage.Account, err error) { s := storageResource{ resourceGroupName: groupName, storageAccountName: storageAccountName, diff --git a/pkg/resourcemanager/storages/storageaccount/storage_manager.go b/pkg/resourcemanager/storages/storageaccount/storage_manager.go index 08b7c6dba4e..3fdd74fcd1d 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_manager.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_manager.go @@ -7,7 +7,6 @@ import ( "context" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" - "github.com/Azure/azure-service-operator/api/v1alpha1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/resourcemanager" "github.com/Azure/go-autorest/autorest" @@ -19,7 +18,7 @@ func New() *azureStorageManager { } type StorageManager interface { - CreateStorage(ctx context.Context, instance *v1alpha1.Storage, + CreateStorage(ctx context.Context, groupName string, storageAccountName string, location string, @@ -27,7 +26,7 @@ type StorageManager interface { kind azurev1alpha1.StorageKind, tags map[string]*string, accessTier azurev1alpha1.StorageAccessTier, - enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool) (result storage.Account, err error) + enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (result storage.Account, err error) // Get gets the description of the specified storage account. // Parameters: diff --git a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go index aff0bccadc8..74929a12ce1 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/helpers" @@ -33,6 +34,11 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o enableHTTPSTrafficOnly := instance.Spec.EnableHTTPSTrafficOnly dataLakeEnabled := instance.Spec.DataLakeEnabled + networkAcls := storage.NetworkRuleSet{} + + if instance.Spec.NetworkRule != nil { + networkAcls = ParseNetworkPolicy(instance.Spec.NetworkRule) + } // convert kube labels to expected tag format labels := map[string]*string{} for k, v := range instance.GetLabels() { @@ -67,7 +73,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o instance.Status.Provisioning = true instance.Status.Provisioned = false - _, err = sa.CreateStorage(ctx, instance, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled) + _, err = sa.CreateStorage(ctx, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled, &networkAcls) if err != nil { instance.Status.Message = err.Error() azerr := errhelp.NewAzureErrorAzureError(err) From 105c75095f8b04e41ef3eeb44d6d5f8d7b7ec635 Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Wed, 8 Apr 2020 08:53:59 -0600 Subject: [PATCH 06/99] accidental char --- api/v1alpha1/storageaccount_types.go | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/storageaccount_types.go b/api/v1alpha1/storageaccount_types.go index f4544a56d18..73f39e38e54 100644 --- a/api/v1alpha1/storageaccount_types.go +++ b/api/v1alpha1/storageaccount_types.go @@ -99,7 +99,7 @@ type StorageList struct { func init() { SchemeBuilder.Register(&StorageAccount{}, &StorageList{}) } -i + func (storage *StorageAccount) IsSubmitted() bool { return storage.Status.Provisioning || storage.Status.Provisioned } diff --git a/main.go b/main.go index ba5a10dd8d2..03e03f51513 100644 --- a/main.go +++ b/main.go @@ -163,7 +163,7 @@ func main() { }, }).SetupWithManager(mgr) if err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Storage") + setupLog.Error(err, "unable to create controller", "controller", "StorageAccount") os.Exit(1) } err = (&controllers.CosmosDBReconciler{ From 12c04f3034777e5db199a7b325af9ab4834ed7ca Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Thu, 9 Apr 2020 07:24:57 +0800 Subject: [PATCH 07/99] issue#858#fixYamlfile --- api/v1alpha1/storage_types.go | 2 + config/samples/azure_v1alpha1_storage.yaml | 24 ++++++------ .../storages/storageaccount/storage.go | 39 +++++++------------ 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/api/v1alpha1/storage_types.go b/api/v1alpha1/storage_types.go index a61c370f1d9..1c7b83ceaff 100644 --- a/api/v1alpha1/storage_types.go +++ b/api/v1alpha1/storage_types.go @@ -31,6 +31,8 @@ type StorageSpec struct { DataLakeEnabled *bool `json:"dataLakeEnabled,omitempty"` NetworkRule *StorageNetworkRuleSet `json:"networkRule,omitempty"` + + //Properties StorageAccountProperties `json:"properties,omitempty"` } // Sku the SKU of the storage account. diff --git a/config/samples/azure_v1alpha1_storage.yaml b/config/samples/azure_v1alpha1_storage.yaml index 02b25809087..30f34270af0 100644 --- a/config/samples/azure_v1alpha1_storage.yaml +++ b/config/samples/azure_v1alpha1_storage.yaml @@ -7,18 +7,16 @@ spec: resourceGroup: resourcegroup-azure-operators sku: name: Standard_RAGRS - kind: StorageV2 - properties: - accessTier: Hot - supportsHttpsTrafficOnly: true + kind: BlobStorage + accessTier: Hot + supportsHttpsTrafficOnly: true # Optional: networkRule - networkRule: - bypass: AzureServices # Possible values are AzureServices, Metrics, None, Logging - defaultAction: Allow # Possible values are Allow, Deny - virtualNetworkRules: - - subnetId: /subscriptions/08daa385-27fa-477a-b556-a9ead8b270d9/resourceGroups/resourcegroup-azure-operator/providers/Microsoft.Network/virtualNetworks/virtualnetwork-sample/subnets/test1 - ipRules: - - ipAddressOrRange: 2.2.0.0/24 - - ipAddressOrRange: 2.2.2.1 - + networkRule: + bypass: AzureServices # Possible values are AzureServices, Metrics, None, Logging + defaultAction: Deny # Possible values are Allow, Deny + virtualNetworkRules: + - subnetId: /subscriptions/{subscription}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet}/subnets/{subnet} + ipRules: #could be an ip range or a ip address + - ipAddressOrRange: 2.2.0.0/24 + - ipAddressOrRange: 2.2.2.1 diff --git a/pkg/resourcemanager/storages/storageaccount/storage.go b/pkg/resourcemanager/storages/storageaccount/storage.go index ec74ca48720..0b45d1d69bd 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage.go +++ b/pkg/resourcemanager/storages/storageaccount/storage.go @@ -7,6 +7,7 @@ import ( "context" "errors" "log" + "strings" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -21,8 +22,8 @@ type azureStorageManager struct{} // ParseNetworkPolicy - helper function to parse network policies from Kubernetes spec func ParseNetworkPolicy(ruleSet *v1alpha1.StorageNetworkRuleSet) storage.NetworkRuleSet { - var bypass storage.Bypass + bypass := storage.AzureServices switch ruleSet.Bypass { case "AzureServices": bypass = storage.AzureServices @@ -32,26 +33,19 @@ func ParseNetworkPolicy(ruleSet *v1alpha1.StorageNetworkRuleSet) storage.Network bypass = storage.Logging case "Metrics": bypass = storage.Metrics - default: - bypass = storage.AzureServices } - var defaultAction storage.DefaultAction - switch ruleSet.DefaultAction { - case "Allow": + defaultAction := storage.DefaultActionDeny + if strings.ToLower(ruleSet.DefaultAction) == "allow" { defaultAction = storage.DefaultActionAllow - case "Deny": - defaultAction = storage.DefaultActionDeny - default: - defaultAction = storage.DefaultActionDeny } var ipInstances []storage.IPRule - if ruleSet.IPRule != nil { - for _, i := range *ruleSet.IPRule { - subnetID := i.IPAddressOrRange + if ruleSet.IPRules != nil { + for _, i := range *ruleSet.IPRules { + ipmask := i.IPAddressOrRange ipInstances = append(ipInstances, storage.IPRule{ - IPAddressOrRange: subnetID, + IPAddressOrRange: ipmask, Action: storage.Allow, }) } @@ -60,9 +54,9 @@ func ParseNetworkPolicy(ruleSet *v1alpha1.StorageNetworkRuleSet) storage.Network var vnetInstances []storage.VirtualNetworkRule if ruleSet.VirtualNetworkRules != nil { for _, i := range *ruleSet.VirtualNetworkRules { - ventID := i.VirtualNetworkResourceID + vnetID := i.SubnetId vnetInstances = append(vnetInstances, storage.VirtualNetworkRule{ - VirtualNetworkResourceID: ventID, + VirtualNetworkResourceID: vnetID, Action: storage.Allow, }) } @@ -88,7 +82,7 @@ func getStoragesClient() storage.AccountsClient { } // CreateStorage creates a new storage account -func (_ *azureStorageManager) CreateStorage(ctx context.Context, instance *v1alpha1.Storage, +func (_ *azureStorageManager) CreateStorage(ctx context.Context, groupName string, storageAccountName string, location string, @@ -96,7 +90,7 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, instance *v1alp kind azurev1alpha1.StorageKind, tags map[string]*string, accessTier azurev1alpha1.StorageAccessTier, - enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool) (result storage.Account, err error) { + enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (result storage.Account, err error) { storagesClient := getStoragesClient() @@ -122,13 +116,6 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, instance *v1alp sKind := storage.Kind(kind) sAccessTier := storage.AccessTier(accessTier) - var networkAcls storage.NetworkRuleSet - if instance.Spec.NetworkRule != nil { - networkAcls = ParseNetworkPolicy(instance.Spec.NetworkRule) - } else { - networkAcls = storage.NetworkRuleSet{} - } - params := storage.AccountCreateParameters{ Location: to.StringPtr(location), Sku: &sSku, @@ -139,7 +126,7 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, instance *v1alp AccessTier: sAccessTier, EnableHTTPSTrafficOnly: enableHTTPsTrafficOnly, IsHnsEnabled: dataLakeEnabled, - NetworkRuleSet: &networkAcls, + NetworkRuleSet: networkRule, }, } From 400a0dd6af9c1aff528e6814868cd5ae1b6fa607 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Thu, 9 Apr 2020 20:14:55 +0800 Subject: [PATCH 08/99] #issue858#Controllertest --- controllers/storage_controller_test.go | 57 +++++++++++++++++++------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index b8ce7f1fbaf..03bd01edc57 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -7,22 +7,23 @@ package controllers import ( "context" + "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" config "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/go-autorest/autorest/to" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) -func TestStorageControllerHappyPath(t *testing.T) { +func TestStorageControllerHappyPathWithoutNetworkRule(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - StorageAccountName := GenerateAlphaNumTestResourceName("sadev") - // Create the ResourceGroup object and expect the Reconcile to be created saInstance := &azurev1alpha1.Storage{ ObjectMeta: metav1.ObjectMeta{ @@ -40,25 +41,47 @@ func TestStorageControllerHappyPath(t *testing.T) { EnableHTTPSTrafficOnly: to.BoolPtr(true), }, } - // create rg EnsureInstance(ctx, t, tc, saInstance) - // delete rg EnsureDelete(ctx, t, tc, saInstance) } - func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() + assert := assert.New(t) + var err error - StorageAccountName := GenerateAlphaNumTestResourceName("cndev") + StorageAccountName := GenerateTestResourceNameWithRandom("storage", 10) rgName := tc.resourceGroupName rgLocation := tc.resourceGroupLocation - VNetName := GenerateTestResourceNameWithRandom("vnet", 10) - subnetName := "subnet-test" + VNetName := GenerateTestResourceNameWithRandom("svnet", 10) + subnetName := "subnet-storage-test" + + // Create a VNET as prereq for the test + + VNetSubNetInstance := azurev1alpha1.VNetSubnets{ + SubnetName: subnetName, + SubnetAddressPrefix: "110.1.0.0/16", + } + + // Create a VNET + VNetInstance := &azurev1alpha1.VirtualNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: VNetName, + Namespace: "default", + }, + Spec: azurev1alpha1.VirtualNetworkSpec{ + Location: rgLocation, + ResourceGroup: rgName, + AddressSpace: "110.0.0.0/8", + Subnets: []azurev1alpha1.VNetSubnets{VNetSubNetInstance}, + }, + } + + EnsureInstance(ctx, t, tc, VNetInstance) subnetID := "/subscriptions/" + config.SubscriptionID() + "/resourceGroups/" + rgName + "/providers/Microsoft.Network/virtualNetworks/" + VNetName + "/subnets/" + subnetName vnetRules := []azurev1alpha1.VirtualNetworkRule{ @@ -77,7 +100,7 @@ func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { }, } - // Create the ResourceGroup object and expect the Reconcile to be created + // Create the storage account object and expect the Reconcile to be created cnInstance := &azurev1alpha1.Storage{ ObjectMeta: metav1.ObjectMeta{ Name: StorageAccountName, @@ -101,9 +124,15 @@ func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { }, } - // create rg - EnsureInstance(ctx, t, tc, cnInstance) + err = tc.k8sClient.Create(ctx, cnInstance) + assert.Equal(nil, err, "create StorageAccount in k8s") + + storageAccountNamespacedName := types.NamespacedName{Name: StorageAccountName, Namespace: "default"} + + // Wait for the APIMgmtAPI instance to be written to k8s + assert.Eventually(func() bool { + err = tc.k8sClient.Get(ctx, storageAccountNamespacedName, cnInstance) + return strings.Contains(cnInstance.Status.Message, successMsg) + }, tc.timeout, tc.retry, "awaiting storageAccount instance creation") - // delete rg - EnsureDelete(ctx, t, tc, cnInstance) } From 47850efa862fd8e18faa10d82d909ac58a49b571 Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 9 Apr 2020 09:34:54 -0600 Subject: [PATCH 09/99] more variable name changes to storageaccount --- PROJECT | 3 ++ api/v1alpha1/storageaccount_types.go | 53 ++++++++++--------- .../azure_v1alpha1_storageaccount.yaml | 4 +- controllers/storageaccount_controller_test.go | 4 +- controllers/suite_test.go | 4 +- .../{storage.go => storageaccount.go} | 6 +-- pkg/resourcemanager/storages/managers.go | 12 ++--- .../storages/storageaccount/storageaccount.go | 6 +-- .../storageaccount/storageaccount_manager.go | 6 +-- 9 files changed, 51 insertions(+), 47 deletions(-) rename pkg/resourcemanager/mock/storages/{storage.go => storageaccount.go} (95%) diff --git a/PROJECT b/PROJECT index 0b9e8a8e412..1de76180841 100644 --- a/PROJECT +++ b/PROJECT @@ -5,6 +5,9 @@ resources: - group: azure version: v1alpha1 kind: Storage +- group: azure + version: v1alpha1 + kind: StorageAccount - group: azure version: v1alpha1 kind: CosmosDB diff --git a/api/v1alpha1/storageaccount_types.go b/api/v1alpha1/storageaccount_types.go index 73f39e38e54..32bf8bdf876 100644 --- a/api/v1alpha1/storageaccount_types.go +++ b/api/v1alpha1/storageaccount_types.go @@ -10,8 +10,8 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// StorageSpec defines the desired state of Storage -type StorageSpec struct { +// StorageAccountSpec defines the desired state of Storage +type StorageAccountSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file @@ -20,44 +20,44 @@ type StorageSpec struct { Location string `json:"location,omitempty"` ResourceGroup string `json:"resourceGroup"` - Sku StorageSku `json:"sku,omitempty"` + Sku StorageAccountSku `json:"sku,omitempty"` - Kind StorageKind `json:"kind,omitempty"` + Kind StorageAccountKind `json:"kind,omitempty"` - AccessTier StorageAccessTier `json:"accessTier,omitempty"` + AccessTier StorageAccountAccessTier `json:"accessTier,omitempty"` EnableHTTPSTrafficOnly *bool `json:"supportsHttpsTrafficOnly,omitempty"` DataLakeEnabled *bool `json:"dataLakeEnabled,omitempty"` } -// Sku the SKU of the storage account. -type StorageSku struct { +// StorageAccountSku the SKU of the storage account. +type StorageAccountSku struct { // Name - The SKU name. Required for account creation; optional for update. // Possible values include: 'StandardLRS', 'StandardGRS', 'StandardRAGRS', 'StandardZRS', 'PremiumLRS', 'PremiumZRS', 'StandardGZRS', 'StandardRAGZRS' - Name StorageSkuName `json:"name,omitempty"` + Name StorageAccountSkuName `json:"name,omitempty"` } -// StorageSkuName enumerates the values for sku name. +// StorageAccountSkuName enumerates the values for sku name. // Only one of the following sku names may be specified. // If none of the following sku names is specified, the default one // is StorageV2. // +kubebuilder:validation:Enum=Premium_LRS;Premium_ZRS;Standard_GRS;Standard_GZRS;Standard_LRS;Standard_RAGRS;Standard_RAGZRS;Standard_ZRS -type StorageSkuName string +type StorageAccountSkuName string -// StorageKind enumerates the values for kind. +// StorageAccountKind enumerates the values for kind. // Only one of the following kinds may be specified. // If none of the following kinds is specified, the default one // is StorageV2. // +kubebuilder:validation:Enum=BlobStorage;BlockBlobStorage;FileStorage;Storage;StorageV2 -type StorageKind string +type StorageAccountKind string -// AccessTier enumerates the values for access tier. +// StorageAccountAccessTier enumerates the values for access tier. // Only one of the following access tiers may be specified. // If none of the following access tiers is specified, the default one // is Hot. // +kubebuilder:validation:Enum=Cool;Hot -type StorageAccessTier string +type StorageAccountAccessTier string // +kubebuilder:object:root=true // +kubebuilder:subresource:status @@ -67,13 +67,14 @@ type StorageAccount struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec StorageSpec `json:"spec,omitempty"` - Status ASOStatus `json:"status,omitempty"` - Output StorageOutput `json:"output,omitempty"` - AdditionalResources StorageAdditionalResources `json:"additionalResources,omitempty"` + Spec StorageAccountSpec `json:"spec,omitempty"` + Status ASOStatus `json:"status,omitempty"` + Output StorageAccountOutput `json:"output,omitempty"` + AdditionalResources StorageAccountAdditionalResources `json:"additionalResources,omitempty"` } -type StorageOutput struct { +// StorageAccountOutput is the object that contains the output from creating a Storage Account object +type StorageAccountOutput struct { StorageAccountName string `json:"storageAccountName,omitempty"` Key1 string `json:"key1,omitempty"` Key2 string `json:"key2,omitempty"` @@ -81,25 +82,25 @@ type StorageOutput struct { ConnectionString2 string `json:"connectionString2,omitempty"` } -// StorageAdditionalResources holds the additional resources -type StorageAdditionalResources struct { +// StorageAccountAdditionalResources holds the additional resources +type StorageAccountAdditionalResources struct { Secrets []string `json:"secrets,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// StorageList contains a list of Storage -type StorageList struct { +// StorageAccountList contains a list of Storage +type StorageAccountList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` Items []StorageAccount `json:"items"` } func init() { - SchemeBuilder.Register(&StorageAccount{}, &StorageList{}) + SchemeBuilder.Register(&StorageAccount{}, &StorageAccountList{}) } -func (storage *StorageAccount) IsSubmitted() bool { - return storage.Status.Provisioning || storage.Status.Provisioned +func (storageAccount *StorageAccount) IsSubmitted() bool { + return storageAccount.Status.Provisioning || storageAccount.Status.Provisioned } diff --git a/config/samples/azure_v1alpha1_storageaccount.yaml b/config/samples/azure_v1alpha1_storageaccount.yaml index 11015855e91..12a1dd0a229 100644 --- a/config/samples/azure_v1alpha1_storageaccount.yaml +++ b/config/samples/azure_v1alpha1_storageaccount.yaml @@ -1,7 +1,7 @@ apiVersion: azure.microsoft.com/v1alpha1 -kind: Storage +kind: StorageAccount metadata: - name: storageaccount-sample-1 + name: storageaccountsample777 spec: location: westus resourceGroup: resourcegroup-azure-operators diff --git a/controllers/storageaccount_controller_test.go b/controllers/storageaccount_controller_test.go index ed165c0642a..fca6e8f4ea0 100644 --- a/controllers/storageaccount_controller_test.go +++ b/controllers/storageaccount_controller_test.go @@ -28,10 +28,10 @@ func TestStorageControllerHappyPath(t *testing.T) { Name: StorageAccountName, Namespace: "default", }, - Spec: azurev1alpha1.StorageSpec{ + Spec: azurev1alpha1.StorageAccountSpec{ Location: tc.resourceGroupLocation, ResourceGroup: tc.resourceGroupName, - Sku: azurev1alpha1.StorageSku{ + Sku: azurev1alpha1.StorageAccountSku{ Name: "Standard_RAGRS", }, Kind: "StorageV2", diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 1446d267005..ec679818c5f 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -705,9 +705,9 @@ func setup() error { log.Println("Creating SA:", storageAccountName) // Create the Storage Account and Container - _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageSku{ + _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageAccountSku{ Name: "Standard_LRS", - }, "Storage", map[string]*string{}, "", nil, nil) + }, "StorageAccount", map[string]*string{}, "", nil, nil) // Storage account needs to be in "Suceeded" state // for container create to succeed diff --git a/pkg/resourcemanager/mock/storages/storage.go b/pkg/resourcemanager/mock/storages/storageaccount.go similarity index 95% rename from pkg/resourcemanager/mock/storages/storage.go rename to pkg/resourcemanager/mock/storages/storageaccount.go index 92cc51d207f..ec7c86ed64e 100644 --- a/pkg/resourcemanager/mock/storages/storage.go +++ b/pkg/resourcemanager/mock/storages/storageaccount.go @@ -46,10 +46,10 @@ func (srs *StorageResources) Find(predicate func(storageResource) bool) { func (manager *mockStorageManager) CreateStorage(ctx context.Context, groupName string, storageAccountName string, location string, - sku azurev1alpha1.StorageSku, - kind azurev1alpha1.StorageKind, + sku azurev1alpha1.StorageAccountSku, + kind azurev1alpha1.StorageAccountKind, tags map[string]*string, - accessTier azurev1alpha1.StorageAccessTier, + accessTier azurev1alpha1.StorageAccountAccessTier, enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool) (result storage.Account, err error) { s := storageResource{ resourceGroupName: groupName, diff --git a/pkg/resourcemanager/storages/managers.go b/pkg/resourcemanager/storages/managers.go index ac5bdd184f2..3fdd5a6f62e 100644 --- a/pkg/resourcemanager/storages/managers.go +++ b/pkg/resourcemanager/storages/managers.go @@ -9,13 +9,13 @@ import ( ) type StorageManagers struct { - Storage storageaccount.StorageManager - BlobContainer blobcontainer.BlobContainerManager - FileSystem FileSystemManager + StorageAccount storageaccount.StorageManager + BlobContainer blobcontainer.BlobContainerManager + FileSystem FileSystemManager } var AzureStorageManagers = StorageManagers{ - Storage: storageaccount.New(), - BlobContainer: blobcontainer.New(), - FileSystem: &azureFileSystemManager{}, + StorageAccount: storageaccount.New(), + BlobContainer: blobcontainer.New(), + FileSystem: &azureFileSystemManager{}, } diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount.go b/pkg/resourcemanager/storages/storageaccount/storageaccount.go index b43cdd1475b..f14c92e3db6 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount.go @@ -33,10 +33,10 @@ func getStoragesClient() storage.AccountsClient { func (_ *azureStorageManager) CreateStorage(ctx context.Context, groupName string, storageAccountName string, location string, - sku azurev1alpha1.StorageSku, - kind azurev1alpha1.StorageKind, + sku azurev1alpha1.StorageAccountSku, + kind azurev1alpha1.StorageAccountKind, tags map[string]*string, - accessTier azurev1alpha1.StorageAccessTier, + accessTier azurev1alpha1.StorageAccountAccessTier, enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool) (result storage.Account, err error) { storagesClient := getStoragesClient() diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go index 80437e7b729..e06f1577d53 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go @@ -21,10 +21,10 @@ type StorageManager interface { CreateStorage(ctx context.Context, groupName string, storageAccountName string, location string, - sku azurev1alpha1.StorageSku, - kind azurev1alpha1.StorageKind, + sku azurev1alpha1.StorageAccountSku, + kind azurev1alpha1.StorageAccountKind, tags map[string]*string, - accessTier azurev1alpha1.StorageAccessTier, + accessTier azurev1alpha1.StorageAccountAccessTier, enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool) (result storage.Account, err error) // Get gets the description of the specified storage account. From d6acb2bc5cdfd4063aecafdfeec66ad50a7d1bf1 Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 9 Apr 2020 09:42:39 -0600 Subject: [PATCH 10/99] updating storages to storageaccounts for yamls --- api/v1alpha1/storageaccount_types.go | 2 +- config/crd/kustomization.yaml | 2 +- config/default/manager_role_patch.yaml | 4 ++-- controllers/storageaccount_controller.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/storageaccount_types.go b/api/v1alpha1/storageaccount_types.go index 32bf8bdf876..01530effb97 100644 --- a/api/v1alpha1/storageaccount_types.go +++ b/api/v1alpha1/storageaccount_types.go @@ -49,7 +49,7 @@ type StorageAccountSkuName string // Only one of the following kinds may be specified. // If none of the following kinds is specified, the default one // is StorageV2. -// +kubebuilder:validation:Enum=BlobStorage;BlockBlobStorage;FileStorage;Storage;StorageV2 +// +kubebuilder:validation:Enum=BlobStorage;BlockBlobStorage;FileStorage;StorageAccount;StorageV2 type StorageAccountKind string // StorageAccountAccessTier enumerates the values for access tier. diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 6325b47d8df..bf4d1bd98f4 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,7 +2,7 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- bases/azure.microsoft.com_storages.yaml +- bases/azure.microsoft.com_storageaccounts.yaml - bases/azure.microsoft.com_cosmosdbs.yaml - bases/azure.microsoft.com_rediscaches.yaml - bases/azure.microsoft.com_eventhubs.yaml diff --git a/config/default/manager_role_patch.yaml b/config/default/manager_role_patch.yaml index 5dcb6a58b08..9cca51f71e7 100644 --- a/config/default/manager_role_patch.yaml +++ b/config/default/manager_role_patch.yaml @@ -48,7 +48,7 @@ rules: - apiGroups: - azure.microsoft.com resources: - - storages + - storageaccounts verbs: - create - delete @@ -60,7 +60,7 @@ rules: - apiGroups: - azure.microsoft.com resources: - - storages/status + - storageaccounts/status verbs: - get - patch diff --git a/controllers/storageaccount_controller.go b/controllers/storageaccount_controller.go index 78460c2f273..27c7be5317a 100644 --- a/controllers/storageaccount_controller.go +++ b/controllers/storageaccount_controller.go @@ -16,8 +16,8 @@ type StorageAccountReconciler struct { Reconciler *AsyncReconciler } -// +kubebuilder:rbac:groups=azure.microsoft.com,resources=storages,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=azure.microsoft.com,resources=storages/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=azure.microsoft.com,resources=storageaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=azure.microsoft.com,resources=storageaccounts/status,verbs=get;update;patch // Reconcile function does the main reconciliation loop of the operator func (r *StorageAccountReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { From 985eafe5fb2dd14f150941d90e8fd2c631160a68 Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 9 Apr 2020 12:00:07 -0600 Subject: [PATCH 11/99] cleaning kustomize --- PROJECT | 3 --- ...ebhook_in_storages.yaml => webhook_in_storageaccounts.yaml} | 2 +- config/samples/azure_v1alpha1_azuredatalakegen2storage.yaml | 2 +- controllers/storageaccount_controller.go | 2 -- 4 files changed, 2 insertions(+), 7 deletions(-) rename config/crd/patches/{webhook_in_storages.yaml => webhook_in_storageaccounts.yaml} (93%) diff --git a/PROJECT b/PROJECT index 1de76180841..6ac7e0c9520 100644 --- a/PROJECT +++ b/PROJECT @@ -2,9 +2,6 @@ version: "2" domain: microsoft.com repo: github.com/Azure/azure-service-operator resources: -- group: azure - version: v1alpha1 - kind: Storage - group: azure version: v1alpha1 kind: StorageAccount diff --git a/config/crd/patches/webhook_in_storages.yaml b/config/crd/patches/webhook_in_storageaccounts.yaml similarity index 93% rename from config/crd/patches/webhook_in_storages.yaml rename to config/crd/patches/webhook_in_storageaccounts.yaml index 51d315b7a6e..b129451e721 100644 --- a/config/crd/patches/webhook_in_storages.yaml +++ b/config/crd/patches/webhook_in_storageaccounts.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: storages.azure.microsoft.com + name: storageaccounts.azure.microsoft.com spec: conversion: strategy: Webhook diff --git a/config/samples/azure_v1alpha1_azuredatalakegen2storage.yaml b/config/samples/azure_v1alpha1_azuredatalakegen2storage.yaml index 96459b87cb1..2a3fd7e6881 100644 --- a/config/samples/azure_v1alpha1_azuredatalakegen2storage.yaml +++ b/config/samples/azure_v1alpha1_azuredatalakegen2storage.yaml @@ -1,5 +1,5 @@ apiVersion: azure.microsoft.com/v1alpha1 -kind: Storage +kind: StorageAccount metadata: name: adlsaccountsample spec: diff --git a/controllers/storageaccount_controller.go b/controllers/storageaccount_controller.go index 27c7be5317a..f67ac32610a 100644 --- a/controllers/storageaccount_controller.go +++ b/controllers/storageaccount_controller.go @@ -9,8 +9,6 @@ import ( azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" ) -const storageFinalizerName = "storage.finalizers.azure.com" - // StorageAccountReconciler reconciles a Storage Account object type StorageAccountReconciler struct { Reconciler *AsyncReconciler From 91e6ebfa4e7a3d6bb55342e650f39b40a84ea3c0 Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 9 Apr 2020 12:43:38 -0600 Subject: [PATCH 12/99] Storage account name switch --- pkg/resourcemanager/mock/storages/storageaccount.go | 8 ++++---- .../storages/storageaccount/storageaccount.go | 6 +++--- .../storages/storageaccount/storageaccount_manager.go | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/resourcemanager/mock/storages/storageaccount.go b/pkg/resourcemanager/mock/storages/storageaccount.go index ec7c86ed64e..83f8d88a115 100644 --- a/pkg/resourcemanager/mock/storages/storageaccount.go +++ b/pkg/resourcemanager/mock/storages/storageaccount.go @@ -70,12 +70,12 @@ func (manager *mockStorageManager) CreateStorage(ctx context.Context, groupName } // Get gets the description of the specified storage account. -func (manager *mockStorageManager) GetStorage(ctx context.Context, resourceGroupName string, accountName string) (storage.Account, error) { +func (manager *mockStorageManager) GetStorage(ctx context.Context, resourceGroupName string, storageAccountName string) (storage.Account, error) { groups := manager.storageResource index, group := findStorage(groups, func(g storageResource) bool { return g.resourceGroupName == resourceGroupName && - g.storageAccountName == accountName + g.storageAccountName == storageAccountName }) if index == -1 { @@ -86,12 +86,12 @@ func (manager *mockStorageManager) GetStorage(ctx context.Context, resourceGroup } // removes the storage account -func (manager *mockStorageManager) DeleteStorage(ctx context.Context, resourceGroupName string, accountName string) (autorest.Response, error) { +func (manager *mockStorageManager) DeleteStorage(ctx context.Context, resourceGroupName string, storageAccountName string) (autorest.Response, error) { groups := manager.storageResource index, _ := findStorage(groups, func(g storageResource) bool { return g.resourceGroupName == resourceGroupName && - g.storageAccountName == accountName + g.storageAccountName == storageAccountName }) if index == -1 { diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount.go b/pkg/resourcemanager/storages/storageaccount/storageaccount.go index f14c92e3db6..f8357e57817 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount.go @@ -89,10 +89,10 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, groupName strin // Get gets the description of the specified storage account. // Parameters: // resourceGroupName - name of the resource group within the azure subscription. -// accountName - the name of the storage account -func (_ *azureStorageManager) GetStorage(ctx context.Context, resourceGroupName string, accountName string) (result storage.Account, err error) { +// storageAccountName - the name of the storage account +func (_ *azureStorageManager) GetStorage(ctx context.Context, resourceGroupName string, storageAccountName string) (result storage.Account, err error) { storagesClient := getStoragesClient() - return storagesClient.GetProperties(ctx, resourceGroupName, accountName, "") + return storagesClient.GetProperties(ctx, resourceGroupName, storageAccountName, "") } // DeleteStorage removes the resource group named by env var diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go index e06f1577d53..00e5a6577f1 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go @@ -30,13 +30,13 @@ type StorageManager interface { // Get gets the description of the specified storage account. // Parameters: // resourceGroupName - name of the resource group within the azure subscription. - // accountName - the name of the storage account - GetStorage(ctx context.Context, resourceGroupName string, accountName string) (result storage.Account, err error) + // storageAccountName - the name of the storage account + GetStorage(ctx context.Context, resourceGroupName string, storageAccountName string) (result storage.Account, err error) // DeleteStorage removes the storage account // Parameters: // resourceGroupName - name of the resource group within the azure subscription. - // accountName - the name of the storage account + // storageAccountName - the name of the storage account DeleteStorage(ctx context.Context, groupName string, storageAccountName string) (result autorest.Response, err error) ListKeys(ctx context.Context, groupName string, storageAccountName string) (result storage.AccountListKeysResult, err error) From d06ba1b06c94e9e1c897f977370d5384d1d291bd Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 9 Apr 2020 14:15:30 -0600 Subject: [PATCH 13/99] fixing createstorage 404 --- controllers/suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index ec679818c5f..5c5981d613e 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -707,7 +707,7 @@ func setup() error { // Create the Storage Account and Container _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageAccountSku{ Name: "Standard_LRS", - }, "StorageAccount", map[string]*string{}, "", nil, nil) + }, "Storage", map[string]*string{}, "", nil, nil) // Storage account needs to be in "Suceeded" state // for container create to succeed From 824bc706f03682d021aabc50d9f8606ed5bfb961 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Thu, 9 Apr 2020 15:37:55 -0600 Subject: [PATCH 14/99] Types file for replica support --- api/v1alpha1/mysqlserver_types.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 9fb8df5b540..0892729e51c 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -12,12 +12,16 @@ import ( // MySQLServerSpec defines the desired state of MySQLServer type MySQLServerSpec struct { - Location string `json:"location"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Sku AzureDBsSQLSku `json:"sku,omitempty"` - ServerVersion ServerVersion `json:"serverVersion,omitempty"` - SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` - KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` + Location string `json:"location"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Sku AzureDBsSQLSku `json:"sku,omitempty"` + ServerVersion ServerVersion `json:"serverVersion,omitempty"` + SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` + MinimalTLSVersion string `json:"minimalTLSVersion,omitempty"` + InfrastructureEncryption string `json:"infrastructureEncryption,omitempty"` + CreateMode string `json:"createMode,omitempty"` + ReplicaProperties ReplicaProperties `json:"replicaProperties, omitempty"` + KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` } // +kubebuilder:object:root=true @@ -41,6 +45,10 @@ type MySQLServerList struct { Items []MySQLServer `json:"items"` } +type ReplicaProperties struct { + SourceServerId string `json:"sourceServerId,omitempty"` +} + func init() { SchemeBuilder.Register(&MySQLServer{}, &MySQLServerList{}) } From 7e4c7ae80e6f06645e5401f1c94bf21952cc633a Mon Sep 17 00:00:00 2001 From: jananivMS Date: Thu, 9 Apr 2020 15:49:53 -0600 Subject: [PATCH 15/99] yamls --- config/samples/azure_v1alpha1_mysqlserver.yaml | 4 ++++ config/samples/azure_v1alpha1_mysqlserver_replica.yaml | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 config/samples/azure_v1alpha1_mysqlserver_replica.yaml diff --git a/config/samples/azure_v1alpha1_mysqlserver.yaml b/config/samples/azure_v1alpha1_mysqlserver.yaml index 3b2ff695ab5..c73f0e3b4f1 100644 --- a/config/samples/azure_v1alpha1_mysqlserver.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver.yaml @@ -7,9 +7,13 @@ spec: resourceGroup: resourcegroup-azure-operators serverVersion: "8.0" sslEnforcement: Enabled + minimalTLSVersion: TLS10 # Possible values include: 'TLS10', 'TLS11', 'TLS12', 'Disabled' + infrastructureEncryption: Enabled # Possible values include: Enabled, Disabled + createMode: Default # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) sku: name: B_Gen5_2 tier: Basic family: Gen5 size: "51200" capacity: 2 + diff --git a/config/samples/azure_v1alpha1_mysqlserver_replica.yaml b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml new file mode 100644 index 00000000000..b256cec145d --- /dev/null +++ b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml @@ -0,0 +1,10 @@ +apiVersion: azure.microsoft.com/v1alpha1 +kind: MySQLServer +metadata: + name: mysqlserver-replica +spec: + location: eastus2 + createMode: Replica # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) + replicaProperties: + sourceServerId: /subscriptions/{SUBID}/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.DBforMySQL/servers/mysqlserver-sample + From 182a66a623374a4ee8ba2405b743f77421a90936 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Thu, 9 Apr 2020 17:45:38 -0600 Subject: [PATCH 16/99] updates --- controllers/storage_controller_test.go | 18 ++++-------------- .../storages/storageaccount/storage.go | 3 +++ .../storageaccount/storage_reconcile.go | 3 +++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index 03bd01edc57..c45e1d557af 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -7,16 +7,13 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" config "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/go-autorest/autorest/to" - "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestStorageControllerHappyPathWithoutNetworkRule(t *testing.T) { @@ -50,10 +47,8 @@ func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) - var err error - StorageAccountName := GenerateTestResourceNameWithRandom("storage", 10) + StorageAccountName := GenerateAlphaNumTestResourceName("sanet") rgName := tc.resourceGroupName rgLocation := tc.resourceGroupLocation @@ -124,15 +119,10 @@ func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, cnInstance) - assert.Equal(nil, err, "create StorageAccount in k8s") + EnsureInstance(ctx, t, tc, cnInstance) - storageAccountNamespacedName := types.NamespacedName{Name: StorageAccountName, Namespace: "default"} + // Delete instance - // Wait for the APIMgmtAPI instance to be written to k8s - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, storageAccountNamespacedName, cnInstance) - return strings.Contains(cnInstance.Status.Message, successMsg) - }, tc.timeout, tc.retry, "awaiting storageAccount instance creation") + EnsureDelete(ctx, t, tc, cnInstance) } diff --git a/pkg/resourcemanager/storages/storageaccount/storage.go b/pkg/resourcemanager/storages/storageaccount/storage.go index 0b45d1d69bd..5cb2a5eb29b 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage.go +++ b/pkg/resourcemanager/storages/storageaccount/storage.go @@ -8,6 +8,7 @@ import ( "errors" "log" "strings" + "time" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -136,6 +137,8 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, return result, err } + time.Sleep(2 * time.Second) + return future.Result(storagesClient) } diff --git a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go index 74929a12ce1..3a91ad05ceb 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go @@ -76,6 +76,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o _, err = sa.CreateStorage(ctx, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled, &networkAcls) if err != nil { instance.Status.Message = err.Error() + fmt.Println(err.Error()) azerr := errhelp.NewAzureErrorAzureError(err) instance.Status.Provisioning = false @@ -118,8 +119,10 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o stop := []string{ errhelp.AccountNameInvalid, + errhelp.NetworkAclsValidationFailure, } if helpers.ContainsString(stop, azerr.Type) { + instance.Status.Provisioning = false return true, nil } From 4a89de5b0f76eef42087a7c235b004d400eec21c Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Fri, 10 Apr 2020 13:49:11 +0800 Subject: [PATCH 17/99] #issue858#StorageControllertest#HandleAsyncError --- controllers/storage_controller_test.go | 9 ++++--- controllers/suite_test.go | 2 +- pkg/errhelp/errors.go | 1 + .../storages/storageaccount/storage.go | 20 ++++++++------- .../storageaccount/storage_manager.go | 2 +- .../storageaccount/storage_reconcile.go | 25 +++++++++++++++++-- 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index c45e1d557af..8fc67df0a91 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -12,6 +12,7 @@ import ( azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" config "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" + "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/go-autorest/autorest/to" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -43,7 +44,7 @@ func TestStorageControllerHappyPathWithoutNetworkRule(t *testing.T) { // delete rg EnsureDelete(ctx, t, tc, saInstance) } -func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { +func TestStorageControllerFailurePathWithNetworkRule(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() @@ -75,7 +76,7 @@ func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { Subnets: []azurev1alpha1.VNetSubnets{VNetSubNetInstance}, }, } - + //Create VNet EnsureInstance(ctx, t, tc, VNetInstance) subnetID := "/subscriptions/" + config.SubscriptionID() + "/resourceGroups/" + rgName + "/providers/Microsoft.Network/virtualNetworks/" + VNetName + "/subnets/" + subnetName @@ -119,10 +120,10 @@ func TestStorageControllerHappyPathWithNetworkRule(t *testing.T) { }, } - EnsureInstance(ctx, t, tc, cnInstance) + //Due to the service endpoint cannot be automatically created, can only verify the failure result + EnsureInstanceWithResult(ctx, t, tc, cnInstance, errhelp.NetworkAclsValidationFailure, false) // Delete instance - EnsureDelete(ctx, t, tc, cnInstance) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index a9d3ac01976..36db6ea89ae 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -705,7 +705,7 @@ func setup() error { log.Println("Creating SA:", storageAccountName) // Create the Storage Account and Container - _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageSku{ + _, _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageSku{ Name: "Standard_LRS", }, "Storage", map[string]*string{}, "", nil, nil, nil) diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index f459b077182..b8ffd15f4dd 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -53,6 +53,7 @@ const ( RequestDisallowedByPolicy = "RequestDisallowedByPolicy" ServiceBusy = "ServiceBusy" NameNotAvailable = "NameNotAvailable" + NetworkAclsValidationFailure = "NetworkAclsValidationFailure" ) func NewAzureError(err error) error { diff --git a/pkg/resourcemanager/storages/storageaccount/storage.go b/pkg/resourcemanager/storages/storageaccount/storage.go index 5cb2a5eb29b..24e0aa06445 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage.go +++ b/pkg/resourcemanager/storages/storageaccount/storage.go @@ -8,7 +8,6 @@ import ( "errors" "log" "strings" - "time" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -91,7 +90,7 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, kind azurev1alpha1.StorageKind, tags map[string]*string, accessTier azurev1alpha1.StorageAccessTier, - enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (result storage.Account, err error) { + enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (pollingURL string, result storage.Account, err error) { storagesClient := getStoragesClient() @@ -100,16 +99,19 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, checkAccountParams := storage.AccountCheckNameAvailabilityParameters{Name: &storageAccountName, Type: &storageType} checkNameResult, err := storagesClient.CheckNameAvailability(ctx, checkAccountParams) if err != nil { - return result, err + return "", result, err } if dataLakeEnabled == to.BoolPtr(true) && kind != "StorageV2" { - return result, errors.New("unable to create datalake enabled storage account") + err = errors.New("unable to create datalake enabled storage account") + return } if *checkNameResult.NameAvailable == false { if checkNameResult.Reason == storage.AccountNameInvalid { - return result, errors.New("AccountNameInvalid") + err = errors.New("AccountNameInvalid") + return } else if checkNameResult.Reason == storage.AlreadyExists { - return result, errors.New("AlreadyExists") + err = errors.New("AlreadyExists") + return } } @@ -134,12 +136,12 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, //log.Println(fmt.Sprintf("creating storage '%s' in resource group '%s' and location: %v", storageAccountName, groupName, location)) future, err := storagesClient.Create(ctx, groupName, storageAccountName, params) if err != nil { - return result, err + return "", result, err } - time.Sleep(2 * time.Second) + result, err = future.Result(storagesClient) - return future.Result(storagesClient) + return future.PollingURL(), result, err } diff --git a/pkg/resourcemanager/storages/storageaccount/storage_manager.go b/pkg/resourcemanager/storages/storageaccount/storage_manager.go index 3fdd74fcd1d..6bd272928a6 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_manager.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_manager.go @@ -26,7 +26,7 @@ type StorageManager interface { kind azurev1alpha1.StorageKind, tags map[string]*string, accessTier azurev1alpha1.StorageAccessTier, - enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (result storage.Account, err error) + enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (pollingURL string, result storage.Account, err error) // Get gets the description of the specified storage account. // Parameters: diff --git a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go index 3a91ad05ceb..a46a1e0d0d5 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storage_reconcile.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/pollclient" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -33,6 +34,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o accessTier := instance.Spec.AccessTier enableHTTPSTrafficOnly := instance.Spec.EnableHTTPSTrafficOnly dataLakeEnabled := instance.Spec.DataLakeEnabled + pollURL := instance.Status.PollingURL networkAcls := storage.NetworkRuleSet{} @@ -51,11 +53,26 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o if err != nil { instance.Status.Message = err.Error() instance.Status.State = "NotReady" + + // handle failures in the async operation + if pollURL != "" { + pClient := pollclient.NewPollClient() + res, err := pClient.Get(ctx, pollURL) + if err != nil { + return false, err + } + + if res.Status == "Failed" { + instance.Status.Message = res.Error.Error() + instance.Status.Provisioning = false + return true, nil + } + } } else { instance.Status.State = string(stor.ProvisioningState) hash = helpers.Hash256(instance.Spec) - if instance.Status.SpecHash == hash && instance.Status.Provisioned { + if instance.Status.SpecHash == hash && (instance.Status.Provisioned || instance.Status.FailedProvisioning) { instance.Status.RequestedAt = nil return true, nil } @@ -67,13 +84,14 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o instance.Status.Provisioning = false instance.Status.SpecHash = hash instance.Status.ResourceId = *stor.ID + instance.Status.PollingURL = "" return true, nil } instance.Status.Provisioning = true instance.Status.Provisioned = false - _, err = sa.CreateStorage(ctx, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled, &networkAcls) + pollURL, _, err = sa.CreateStorage(ctx, groupName, name, location, sku, kind, labels, accessTier, enableHTTPSTrafficOnly, dataLakeEnabled, &networkAcls) if err != nil { instance.Status.Message = err.Error() fmt.Println(err.Error()) @@ -113,6 +131,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o if azerr.Type == errhelp.AsyncOpIncompleteError { instance.Status.Provisioning = true + instance.Status.PollingURL = pollURL } return false, nil } @@ -122,7 +141,9 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o errhelp.NetworkAclsValidationFailure, } if helpers.ContainsString(stop, azerr.Type) { + instance.Status.Message = "Unable to provision Azure Storage Account due to error: " + errhelp.StripErrorIDs(err) instance.Status.Provisioning = false + instance.Status.Provisioned = false return true, nil } From 9c0587b421835708e734a4fc4daba18255ee09ad Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Fri, 10 Apr 2020 20:05:14 +0800 Subject: [PATCH 18/99] #issue858#conflict resolve --- controllers/storage_controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index 8fc67df0a91..cca3e843209 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -124,6 +124,7 @@ func TestStorageControllerFailurePathWithNetworkRule(t *testing.T) { EnsureInstanceWithResult(ctx, t, tc, cnInstance, errhelp.NetworkAclsValidationFailure, false) // Delete instance + EnsureDelete(ctx, t, tc, cnInstance) } From 68614704e26dc1f4806e3ee0ebc13b047694f30b Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Fri, 10 Apr 2020 20:41:46 +0800 Subject: [PATCH 19/99] #issue858#updatecontroller test --- controllers/storage_controller_test.go | 84 -------------------------- 1 file changed, 84 deletions(-) diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index cca3e843209..d12b2b04cdf 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -44,87 +44,3 @@ func TestStorageControllerHappyPathWithoutNetworkRule(t *testing.T) { // delete rg EnsureDelete(ctx, t, tc, saInstance) } -func TestStorageControllerFailurePathWithNetworkRule(t *testing.T) { - t.Parallel() - defer PanicRecover(t) - ctx := context.Background() - - StorageAccountName := GenerateAlphaNumTestResourceName("sanet") - - rgName := tc.resourceGroupName - rgLocation := tc.resourceGroupLocation - VNetName := GenerateTestResourceNameWithRandom("svnet", 10) - subnetName := "subnet-storage-test" - - // Create a VNET as prereq for the test - - VNetSubNetInstance := azurev1alpha1.VNetSubnets{ - SubnetName: subnetName, - SubnetAddressPrefix: "110.1.0.0/16", - } - - // Create a VNET - VNetInstance := &azurev1alpha1.VirtualNetwork{ - ObjectMeta: metav1.ObjectMeta{ - Name: VNetName, - Namespace: "default", - }, - Spec: azurev1alpha1.VirtualNetworkSpec{ - Location: rgLocation, - ResourceGroup: rgName, - AddressSpace: "110.0.0.0/8", - Subnets: []azurev1alpha1.VNetSubnets{VNetSubNetInstance}, - }, - } - //Create VNet - EnsureInstance(ctx, t, tc, VNetInstance) - - subnetID := "/subscriptions/" + config.SubscriptionID() + "/resourceGroups/" + rgName + "/providers/Microsoft.Network/virtualNetworks/" + VNetName + "/subnets/" + subnetName - vnetRules := []azurev1alpha1.VirtualNetworkRule{ - { - SubnetId: &subnetID, - }, - } - ipAddress := "1.1.1.1" - ipRange := "2.2.2.2/24" - ipRules := []azurev1alpha1.IPRule{ - { - IPAddressOrRange: &ipAddress, - }, - { - IPAddressOrRange: &ipRange, - }, - } - - // Create the storage account object and expect the Reconcile to be created - cnInstance := &azurev1alpha1.Storage{ - ObjectMeta: metav1.ObjectMeta{ - Name: StorageAccountName, - Namespace: "default", - }, - Spec: azurev1alpha1.StorageSpec{ - Location: rgLocation, - ResourceGroup: rgName, - Sku: azurev1alpha1.StorageSku{ - Name: "Standard_RAGRS", - }, - Kind: "StorageV2", - AccessTier: "Hot", - EnableHTTPSTrafficOnly: to.BoolPtr(true), - NetworkRule: &azurev1alpha1.StorageNetworkRuleSet{ - Bypass: "AzureServices", - VirtualNetworkRules: &vnetRules, - IPRules: &ipRules, - DefaultAction: "Deny", - }, - }, - } - - //Due to the service endpoint cannot be automatically created, can only verify the failure result - EnsureInstanceWithResult(ctx, t, tc, cnInstance, errhelp.NetworkAclsValidationFailure, false) - - // Delete instance - - EnsureDelete(ctx, t, tc, cnInstance) - -} From c0fdecbe389c2f1b95fc569f9ea6a058a4bfc401 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Fri, 10 Apr 2020 20:58:43 +0800 Subject: [PATCH 20/99] #issue858#remove unused code --- api/v1alpha1/storage_types.go | 2 -- controllers/storage_controller_test.go | 3 --- 2 files changed, 5 deletions(-) diff --git a/api/v1alpha1/storage_types.go b/api/v1alpha1/storage_types.go index 1c7b83ceaff..a61c370f1d9 100644 --- a/api/v1alpha1/storage_types.go +++ b/api/v1alpha1/storage_types.go @@ -31,8 +31,6 @@ type StorageSpec struct { DataLakeEnabled *bool `json:"dataLakeEnabled,omitempty"` NetworkRule *StorageNetworkRuleSet `json:"networkRule,omitempty"` - - //Properties StorageAccountProperties `json:"properties,omitempty"` } // Sku the SKU of the storage account. diff --git a/controllers/storage_controller_test.go b/controllers/storage_controller_test.go index d12b2b04cdf..4b853ae7210 100644 --- a/controllers/storage_controller_test.go +++ b/controllers/storage_controller_test.go @@ -10,9 +10,6 @@ import ( "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - config "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" - - "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/go-autorest/autorest/to" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) From 90ba2c4de9ffc8d1318436bbd852a77cd22f7553 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 13 Apr 2020 13:29:56 -0600 Subject: [PATCH 21/99] additional files changes --- config/crd/kustomization.yaml | 2 ++ config/crd/patches/cainjection_in_storages.yaml | 8 -------- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 config/crd/patches/cainjection_in_storages.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index bf4d1bd98f4..7f45832ddec 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -60,6 +60,7 @@ resources: #- patches/webhook_in_mysqlservers.yaml #- patches/webhook_in_mysqldatabases.yaml #- patches/webhook_in_mysqlfirewallrules.yaml +#- patches/webhook_in_storageaccounts.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CAINJECTION] patches here are for enabling the CA injection for each CRD @@ -88,6 +89,7 @@ resources: #- patches/cainjection_in_mysqlservers.yaml #- patches/cainjection_in_mysqldatabases.yaml #- patches/cainjection_in_mysqlfirewallrules.yaml +#- patches/cainjection_in_storageaccounts.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_storages.yaml b/config/crd/patches/cainjection_in_storages.yaml deleted file mode 100644 index 5a817835efb..00000000000 --- a/config/crd/patches/cainjection_in_storages.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -# CRD conversion requires k8s 1.13 or later. -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(NAMESPACE)/$(CERTIFICATENAME) - name: storages.azure.microsoft.com From 3cffd0ccc15cc969b9555287d9144f18f5171bac Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 13 Apr 2020 13:31:33 -0600 Subject: [PATCH 22/99] missed one file --- config/crd/patches/cainjection_in_storageaccounts.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 config/crd/patches/cainjection_in_storageaccounts.yaml diff --git a/config/crd/patches/cainjection_in_storageaccounts.yaml b/config/crd/patches/cainjection_in_storageaccounts.yaml new file mode 100644 index 00000000000..a30904d6799 --- /dev/null +++ b/config/crd/patches/cainjection_in_storageaccounts.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(NAMESPACE)/$(CERTIFICATENAME) + name: storageaccounts.azure.microsoft.com From 00c1cad2e5823805840b6ce136173101b19611b7 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 13 Apr 2020 14:38:39 -0600 Subject: [PATCH 23/99] eventhub types change --- api/v1alpha1/eventhub_types.go | 4 ++-- api/v1alpha1/eventhub_types_test.go | 2 +- controllers/eventhub_controller_test.go | 2 +- pkg/resourcemanager/eventhubs/hub.go | 2 +- pkg/resourcemanager/mock/eventhubs/hub.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/eventhub_types.go b/api/v1alpha1/eventhub_types.go index 9ec9a14472d..d2c07a7175a 100644 --- a/api/v1alpha1/eventhub_types.go +++ b/api/v1alpha1/eventhub_types.go @@ -54,8 +54,8 @@ type Destination struct { // Name - Name for capture destination // +kubebuilder:validation:Enum=EventHubArchive.AzureBlockBlob;EventHubArchive.AzureDataLake Name string `json:"name,omitempty"` - // EventHubStorageAccount - Details of the storage account - EventHubStorageAccount EventHubStorageAccount `json:"eventHubStorageAccount,omitempty"` + // StorageAccount - Details of the storage account + StorageAccount EventHubStorageAccount `json:"storageAccount,omitempty"` } //CaptureDescription defines the properties required for eventhub capture diff --git a/api/v1alpha1/eventhub_types_test.go b/api/v1alpha1/eventhub_types_test.go index af06cf0329a..aec8dcb9fb8 100644 --- a/api/v1alpha1/eventhub_types_test.go +++ b/api/v1alpha1/eventhub_types_test.go @@ -86,7 +86,7 @@ var _ = Describe("Eventhub", func() { ArchiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}", BlobContainer: "foo-blob-container", Name: "EventHubArchive.AzureBlockBlob", - EventHubStorageAccount: EventHubStorageAccount{ + StorageAccount: EventHubStorageAccount{ ResourceGroup: "foo-resource-group", AccountName: "fooaccountname", }, diff --git a/controllers/eventhub_controller_test.go b/controllers/eventhub_controller_test.go index a68109680f4..3e2597eee1b 100644 --- a/controllers/eventhub_controller_test.go +++ b/controllers/eventhub_controller_test.go @@ -267,7 +267,7 @@ func TestEventHubControllerCreateAndDeleteCapture(t *testing.T) { ArchiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}", BlobContainer: bcName, Name: "EventHubArchive.AzureBlockBlob", - EventHubStorageAccount: azurev1alpha1.EventHubStorageAccount{ + StorageAccount: azurev1alpha1.EventHubStorageAccount{ ResourceGroup: rgName, AccountName: saName, }, diff --git a/pkg/resourcemanager/eventhubs/hub.go b/pkg/resourcemanager/eventhubs/hub.go index 723376595c4..a789f1f91cf 100644 --- a/pkg/resourcemanager/eventhubs/hub.go +++ b/pkg/resourcemanager/eventhubs/hub.go @@ -383,7 +383,7 @@ func getCaptureDescriptionPtr(captureDescription azurev1alpha1.CaptureDescriptio // add capture details var capturePtr *model.CaptureDescription - storage := captureDescription.Destination.EventHubStorageAccount + storage := captureDescription.Destination.StorageAccount storageAccountResourceID := fmt.Sprintf(storageAccountResourceFmt, config.SubscriptionID(), storage.ResourceGroup, storage.AccountName) if captureDescription.Enabled { diff --git a/pkg/resourcemanager/mock/eventhubs/hub.go b/pkg/resourcemanager/mock/eventhubs/hub.go index 71f236e0132..75b711679d1 100644 --- a/pkg/resourcemanager/mock/eventhubs/hub.go +++ b/pkg/resourcemanager/mock/eventhubs/hub.go @@ -260,7 +260,7 @@ func getCaptureDescriptionPtr(captureDescription azurev1alpha1.CaptureDescriptio // add capture details var capturePtr *eventhub.CaptureDescription - storage := captureDescription.Destination.EventHubStorageAccount + storage := captureDescription.Destination.StorageAccount storageAccountResourceID := fmt.Sprintf(storageAccountResourceFmt, config.SubscriptionID(), storage.ResourceGroup, storage.AccountName) if captureDescription.Enabled { From a3eb0cfaea0c5d1c269861266ddf36fa7702ca9e Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Mon, 13 Apr 2020 13:43:47 -0700 Subject: [PATCH 24/99] Added Application Insights Docs (#909) * added app insights docs * Update docs/appinsights/appinsights.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> * added secrets section * Update docs/appinsights/appinsights.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> Co-authored-by: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Co-authored-by: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/appinsights/appinsights.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/appinsights/appinsights.md b/docs/appinsights/appinsights.md index e69de29bb2d..ec24f5d76d1 100644 --- a/docs/appinsights/appinsights.md +++ b/docs/appinsights/appinsights.md @@ -0,0 +1,25 @@ +# App Insights Operator + +This operator deploys an Application Insights instance into a specified resource group at the specified location. + +Learn more about Application Insights [here](https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_appinsights.yaml) to provision an Application Insights instance. + +### Required Fields + +An Application Insights instance needs the following fields to deploy, along with a location and resource group. + +* `Kind` specify the kind of application this component refers to, possible values are: 'web', 'ios', 'other', 'store', 'java', 'phone' +* `ApplicationType` specify the types of application being monitored, possible values are: 'web', 'other' + +### Secrets + +After creating an Application Insights instance, the operator stores a JSON formatted secret with the following fields. This secret is stored as a Kubernetes Secret or in an Azure Keyvault, with the same name as your Application Insights instance. +For more details on where the secrets are stored, look [here](/docs/secrets.md) + +* `instrumentationKey` + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. From cb77158bdf821d559ba74ebe6b8df32aa5308380 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Mon, 13 Apr 2020 14:29:20 -0700 Subject: [PATCH 25/99] Added Storage Documentation (#920) * added storage docs * forgot storage docs * forgot a section * removed secrets section Co-authored-by: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/storage/blobcontainer.md | 18 +++++++++++++++++ docs/storage/storageaccount.md | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/docs/storage/blobcontainer.md b/docs/storage/blobcontainer.md index e69de29bb2d..66ebc4abcad 100644 --- a/docs/storage/blobcontainer.md +++ b/docs/storage/blobcontainer.md @@ -0,0 +1,18 @@ +# Blob Container Operator + +This operator deploys a Blob Container into a specified resource group at the specified location. A Storage Account must first be created prior to creating a Blob Container. + +Learn more about Azure Blob Storage [here](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_blobcontainer.yaml) to provision a Blob Container. + +### Required Fields + +A Blob Container needs the following fields to deploy, along with a location and resource group. + +* `AccountName` specify the account name of the Storage Account to add the Blob Container to +* `AccessLevel` specify the access level of the data in the container. Possible values include: 'Container', 'Blob', 'None' + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. \ No newline at end of file diff --git a/docs/storage/storageaccount.md b/docs/storage/storageaccount.md index e69de29bb2d..5809ecea01f 100644 --- a/docs/storage/storageaccount.md +++ b/docs/storage/storageaccount.md @@ -0,0 +1,35 @@ +# Storage Account Operator + +This operator deploys an Azure Storage Account into a specified resource group at the specified location. + +Learn more about Azure Storage Accounts [here](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_storageaccount.yaml) to provision an Azure Storage Account. + +The spec is comprised of the following fields: + +* Location +* ResourceGroup +* Sku + * Name +* Kind +* AccessTier +* SupportsHttpsTrafficOnly +* DataLakeEnabled + +### Required Fields + +A Storage Account needs the following fields to deploy, along with a location and resource group. + +* `Sku.Name` SKU of the storage account, possible values include: 'StandardLRS', 'StandardGRS', 'StandardRAGRS', 'StandardZRS', 'PremiumLRS', 'PremiumZRS', 'StandardGZRS', 'StandardRAGZRS' +* `Kind` specify the kind of storage account, defaults to 'StorageV2'. Possible values include: 'BlobStorage', 'BlockBlobStorage', 'FileStorage', 'Storage', 'StorageV2' +* `AccessTier` specify the access tier, defaults to 'Hot'. Possible values include: 'Hot', 'Cold' +* `SupportsHttpTrafficOnly` allows only HTTPS traffic to storage account if set to true. Default value is true. + +### Optional Fields + +* `DataLakeEnabled` Enable a hierarchical namespace in your storage account, which requires the Storage Account `Kind` to be set to 'StorageV2'. Read more about Data Lake Storage hierarchical namespaces [here](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-namespace). + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. \ No newline at end of file From 85d018934078b2f7d03cfad6e38b39029cf37195 Mon Sep 17 00:00:00 2001 From: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> Date: Mon, 13 Apr 2020 15:30:44 -0600 Subject: [PATCH 26/99] Update config/samples/azure_v1alpha1_eventhub_capture.yaml --- config/samples/azure_v1alpha1_eventhub_capture.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/samples/azure_v1alpha1_eventhub_capture.yaml b/config/samples/azure_v1alpha1_eventhub_capture.yaml index b28af3ade28..e8c38564849 100644 --- a/config/samples/azure_v1alpha1_eventhub_capture.yaml +++ b/config/samples/azure_v1alpha1_eventhub_capture.yaml @@ -15,7 +15,7 @@ spec: archiveNameFormat: "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}" blobContainer: "capturecontainer" name: "EventHubArchive.AzureBlockBlob" - eventHubStorageAccount: + storageAccount: resourceGroup: "my-resource-group" accountName: "storageaccountauv1" enabled: true From 609890326dfe86b8d496d3eb104d48c3d741ebcc Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Mon, 13 Apr 2020 15:01:36 -0700 Subject: [PATCH 27/99] Virtual Network documentation (#901) * added vnet docs * formatting * updates * Update docs/virtualnetwork/virtualnetwork.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> Co-authored-by: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Co-authored-by: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/virtualnetwork/virtualnetwork.md | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/virtualnetwork/virtualnetwork.md b/docs/virtualnetwork/virtualnetwork.md index e69de29bb2d..c9e328bfece 100644 --- a/docs/virtualnetwork/virtualnetwork.md +++ b/docs/virtualnetwork/virtualnetwork.md @@ -0,0 +1,33 @@ +# Virtual Network Operator + +This operator deploys an Azure Virtual Network into a specified resource group at the specified location, in the address space provided. Users are also able to add subnets to their virtual network through the operator. + +Learn more about Azure Virtual Networks [here](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_virtualnetwork.yaml) to provision a Virtual Network. + +The spec is comprised of the following fields: + +* Location +* ResourceGroup +* AddressSpace +* Subnets + * SubnetName + * SubnetAddressPrefix + +### Required Fields + +A Virtual Network needs the following fields to deploy, along with a location and resource group. + +* `AddressSpace` specify an address space for your virtual network + +### Optional Fields + +You are able to specify a single or multiple subnets for your virtual network. + +* `Subnets.SubnetName` specify a name for your subnet +* `Subnets.SubnetAddressPrefix` specify an address space for your subnet + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. From 8038529c3fa16d7e26d8f32f2ea1bc7698241857 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Mon, 13 Apr 2020 17:36:16 -0600 Subject: [PATCH 28/99] saving cosmosdb keys as secrets --- api/v1alpha1/cosmosdb_types.go | 9 +-- config/samples/azure_v1alpha1_cosmosdb.yaml | 4 ++ controllers/cosmosdb_controller_test.go | 25 ++++++-- controllers/suite_test.go | 2 +- main.go | 5 +- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 23 ++++++- .../cosmosdbs/cosmosdb_manager.go | 8 ++- .../cosmosdbs/cosmosdb_reconcile.go | 60 ++++++++++++++++++- 8 files changed, 122 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 34fea0935fc..36796e27ea0 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -17,10 +17,11 @@ type CosmosDBSpec struct { // +kubebuilder:validation:MinLength=0 - Location string `json:"location,omitempty"` - ResourceGroup string `json:"resourceGroup"` - Kind CosmosDBKind `json:"kind,omitempty"` - Properties CosmosDBProperties `json:"properties,omitempty"` + Location string `json:"location,omitempty"` + ResourceGroup string `json:"resourceGroup"` + Kind CosmosDBKind `json:"kind,omitempty"` + Properties CosmosDBProperties `json:"properties,omitempty"` + KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` } // CosmosDBKind enumerates the values for kind. diff --git a/config/samples/azure_v1alpha1_cosmosdb.yaml b/config/samples/azure_v1alpha1_cosmosdb.yaml index d3efa498467..3d84aab4dea 100644 --- a/config/samples/azure_v1alpha1_cosmosdb.yaml +++ b/config/samples/azure_v1alpha1_cosmosdb.yaml @@ -8,3 +8,7 @@ spec: resourceGroup: resourcegroup-azure-operators properties: databaseAccountOfferType: Standard + + # Use the field below to optionally specify a different keyvault + # to store the connectiong string secrets in + #keyVaultToStoreSecrets: asoSecretKeyVault \ No newline at end of file diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index f9ca7f45df0..2fd0ccdb935 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -10,22 +10,26 @@ import ( "testing" "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) func TestCosmosDBHappyPath(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() + assert := assert.New(t) - cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmosdb", 8) - cosmosDBNamespace := "default" + name := GenerateTestResourceNameWithRandom("cosmosdb", 8) + namespace := "default" dbInstance := &v1alpha1.CosmosDB{ ObjectMeta: metav1.ObjectMeta{ - Name: cosmosDBAccountName, - Namespace: cosmosDBNamespace, + Name: name, + Namespace: namespace, }, Spec: v1alpha1.CosmosDBSpec{ Location: "westus", @@ -37,8 +41,21 @@ func TestCosmosDBHappyPath(t *testing.T) { }, } + key := types.NamespacedName{Name: name, Namespace: namespace} + secret := &v1.Secret{} + EnsureInstance(ctx, t, tc, dbInstance) + assert.Eventually(func() bool { + err := tc.k8sClient.Get(ctx, key, secret) + return err == nil && secret.ObjectMeta.Name == name && secret.ObjectMeta.Namespace == namespace + }, tc.timeoutFast, tc.retry, "wait for cosmosdb to have secret") + EnsureDelete(ctx, t, tc, dbInstance) + assert.Eventually(func() bool { + err := tc.k8sClient.Get(ctx, key, secret) + return err != nil + }, tc.timeoutFast, tc.retry, "wait for cosmosdb to delete secret") + } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 3eb278353f0..f617cd3fa26 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -154,7 +154,7 @@ func setup() error { secretClient, scheme.Scheme, ) - cosmosDbManager = resourcemanagercosmosdb.NewAzureCosmosDBManager() + cosmosDbManager = resourcemanagercosmosdb.NewAzureCosmosDBManager(secretClient) apiMgmtManager = resourcemanagerapimgmt.NewManager() resourceGroupManager = resourcegroupsresourcemanager.NewAzureResourceGroupManager() eventHubManagers = resourcemanagereventhub.AzureEventHubManagers diff --git a/main.go b/main.go index 101141afdce..da81dbe5bcb 100644 --- a/main.go +++ b/main.go @@ -126,6 +126,9 @@ func main() { ) eventhubNamespaceClient := resourcemanagereventhub.NewEventHubNamespaceClient() consumerGroupClient := resourcemanagereventhub.NewConsumerGroupClient() + cosmosDBClient := resourcemanagercosmosdb.NewAzureCosmosDBManager( + secretClient, + ) storageManagers := resourcemanagerstorage.AzureStorageManagers keyVaultManager := resourcemanagerkeyvault.NewAzureKeyVaultManager(mgr.GetScheme()) keyVaultKeyManager := &resourcemanagerkeyvault.KeyvaultKeyClient{ @@ -171,7 +174,7 @@ func main() { err = (&controllers.CosmosDBReconciler{ Reconciler: &controllers.AsyncReconciler{ Client: mgr.GetClient(), - AzureClient: resourcemanagercosmosdb.NewAzureCosmosDBManager(), + AzureClient: cosmosDBClient, Telemetry: telemetry.InitializeTelemetryDefault( "CosmosDB", ctrl.Log.WithName("controllers").WithName("CosmosDB"), diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 0d22bb85e85..8f7f8ada4d1 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -13,12 +13,15 @@ import ( "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" + "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" ) // AzureCosmosDBManager is the struct which contains helper functions for resource groups -type AzureCosmosDBManager struct{} +type AzureCosmosDBManager struct { + SecretClient secrets.SecretClient +} func getCosmosDBClient() (documentdb.DatabaseAccountsClient, error) { cosmosDBClient := documentdb.NewDatabaseAccountsClientWithBaseURI(config.BaseURI(), config.SubscriptionID()) @@ -164,3 +167,21 @@ func (*AzureCosmosDBManager) DeleteCosmosDB( } return &ar, nil } + +// ListKeys lists the read & write keys for a database account +func (*AzureCosmosDBManager) ListKeys( + ctx context.Context, + groupName string, + accountName string) (*documentdb.DatabaseAccountListKeysResult, error) { + client, err := getCosmosDBClient() + if err != nil { + return nil, errhelp.NewAzureErrorAzureError(err) + } + + result, err := client.ListKeys(ctx, groupName, accountName) + if err != nil { + return nil, errhelp.NewAzureErrorAzureError(err) + } + + return &result, nil +} diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index f916e70fd54..e67ec2c5471 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -10,12 +10,13 @@ import ( "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest" ) // NewAzureCosmosDBManager creates a new cosmos db client -func NewAzureCosmosDBManager() *AzureCosmosDBManager { - return &AzureCosmosDBManager{} +func NewAzureCosmosDBManager(secretClient secrets.SecretClient) *AzureCosmosDBManager { + return &AzureCosmosDBManager{secretClient} } // CosmosDBManager client functions @@ -32,5 +33,8 @@ type CosmosDBManager interface { // CheckNameExistsCosmosDB check if the account name already exists globally CheckNameExistsCosmosDB(ctx context.Context, accountName string) (bool, *errhelp.AzureError) + // ListKeys lists the read & write keys for a database account + ListKeys(ctx context.Context, groupName string, accountName string) (*documentdb.DatabaseAccountListKeysResult, error) + resourcemanager.ARMClient } diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 4eb8f6b0a0f..680612c3a48 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -18,6 +18,15 @@ import ( // Ensure ensures that cosmosdb is provisioned as specified func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { + options := &resourcemanager.Options{} + for _, opt := range opts { + opt(options) + } + + if options.SecretClient != nil { + m.SecretClient = options.SecretClient + } + instance, err := m.convert(obj) if err != nil { return false, err @@ -44,6 +53,12 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o } if instance.Status.State == "Succeeded" { + // provisioning is complete, update the secrets + if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil { + instance.Status.Message = err.Error() + return false, err + } + instance.Status.Message = resourcemanager.SuccessMsg instance.Status.Provisioning = false instance.Status.Provisioned = true @@ -120,6 +135,15 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o // Delete drops cosmosdb func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { + options := &resourcemanager.Options{} + for _, opt := range opts { + opt(options) + } + + if options.SecretClient != nil { + m.SecretClient = options.SecretClient + } + instance, err := m.convert(obj) if err != nil { return false, err @@ -160,8 +184,14 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o return true, nil } - // try to delete the cosmosdb instance + // create must succeed before delete succeeds + if instance.Status.State == "Creating" { + return true, nil + } + + // try to delete the cosmosdb instance & secrets _, azerr = m.DeleteCosmosDB(ctx, groupName, accountName) + m.deleteAccountKeysSecret(ctx, instance) if azerr != nil { // this is likely to happen on first try due to not waiting for the future to complete if azerr.Type == errhelp.AsyncOpIncompleteError { @@ -217,3 +247,31 @@ func (m *AzureCosmosDBManager) convert(obj runtime.Object) (*v1alpha1.CosmosDB, } return db, nil } + +func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error { + result, err := m.ListKeys(ctx, instance.Spec.ResourceGroup, instance.ObjectMeta.Name) + if err != nil { + return err + } + + secretKey := types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + } + secretData := map[string][]byte{ + "primaryConnectionString": []byte(*result.PrimaryMasterKey), + "secondaryConnectionString": []byte(*result.SecondaryMasterKey), + "primaryReadonlyMasterKey": []byte(*result.PrimaryReadonlyMasterKey), + "secondaryReadonlyMasterKey": []byte(*result.SecondaryReadonlyMasterKey), + } + + return m.SecretClient.Upsert(ctx, secretKey, secretData) +} + +func (m *AzureCosmosDBManager) deleteAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error { + secretKey := types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + } + return m.SecretClient.Delete(ctx, secretKey) +} From b5ff61c5c47ccc203b2a49095848ce357b753452 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Tue, 14 Apr 2020 10:49:51 +0800 Subject: [PATCH 29/99] successMsg --- controllers/cosmosdb_controller_test.go | 69 ++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index f9ca7f45df0..e68912ae8f4 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -28,7 +28,7 @@ func TestCosmosDBHappyPath(t *testing.T) { Namespace: cosmosDBNamespace, }, Spec: v1alpha1.CosmosDBSpec{ - Location: "westus", + Location: tc.resourceGroupLocation, ResourceGroup: tc.resourceGroupName, Kind: v1alpha1.CosmosDBKindGlobalDocumentDB, Properties: v1alpha1.CosmosDBProperties{ @@ -42,3 +42,70 @@ func TestCosmosDBHappyPath(t *testing.T) { EnsureDelete(ctx, t, tc, dbInstance) } + +func TestCosmosDBControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + rgLocation := tc.resourceGroupLocation + //wrong resource group name + resourceGroupName := "gone" + + cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmosdb", 8) + cosmosDBNamespace := "default" + + dbInstance1 := &v1alpha1.CosmosDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: cosmosDBAccountName, + Namespace: cosmosDBNamespace, + }, + Spec: v1alpha1.CosmosDBSpec{ + Location: rgLocation, + ResourceGroup: resourceGroupName, + Kind: v1alpha1.CosmosDBKindGlobalDocumentDB, + Properties: v1alpha1.CosmosDBProperties{ + DatabaseAccountOfferType: v1alpha1.CosmosDBDatabaseAccountOfferTypeStandard, + }, + }, + } + //the expected error meessage to be shown + errMessage := "Waiting for resource group '" + resourceGroupName + "' to be available" + + EnsureInstanceWithResult(ctx, t, tc, dbInstance1, errMessage, false) + EnsureDelete(ctx, t, tc, dbInstance1) +} + +func TestCosmosDBControllerInvalidLocation(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + resourceGroupName := tc.resourceGroupName + //rglocation doesnot exist + rgLocation := GenerateTestResourceNameWithRandom("cosmos-lo", 10) + + cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmos-db", 8) + cosmosDBNamespace := "default" + + dbInstance2 := &v1alpha1.CosmosDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: cosmosDBAccountName, + Namespace: cosmosDBNamespace, + }, + Spec: v1alpha1.CosmosDBSpec{ + Location: rgLocation, + ResourceGroup: resourceGroupName, + Kind: v1alpha1.CosmosDBKindGlobalDocumentDB, + Properties: v1alpha1.CosmosDBProperties{ + DatabaseAccountOfferType: v1alpha1.CosmosDBDatabaseAccountOfferTypeStandard, + }, + }, + } + + //error meessage to be expected + errMessage := "The specified location '" + rgLocation + "' is invalid" + + EnsureInstanceWithResult(ctx, t, tc, dbInstance2, errMessage, false) + EnsureDelete(ctx, t, tc, dbInstance2) +} From 3584cc44822209af3461bd9f96efea39618ceb53 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Tue, 14 Apr 2020 16:12:34 +0800 Subject: [PATCH 30/99] task#475#ControllertestPgsqlUnhappyPath --- .../postgresqldatabase_controller_test.go | 73 ++++++++++++++++++ .../postgresqlfirewallrule_controller_test.go | 75 +++++++++++++++++++ .../postgresqlserver_controller_test.go | 52 +++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 controllers/postgresqldatabase_controller_test.go create mode 100644 controllers/postgresqlfirewallrule_controller_test.go create mode 100644 controllers/postgresqlserver_controller_test.go diff --git a/controllers/postgresqldatabase_controller_test.go b/controllers/postgresqldatabase_controller_test.go new file mode 100644 index 00000000000..2254fee4212 --- /dev/null +++ b/controllers/postgresqldatabase_controller_test.go @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// +build all psqldatabase + +package controllers + +import ( + "context" + "testing" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//Postgresql database controller unhappy test cases + +func TestPSQLDatabaseControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + // Add any setup steps that needs to be executed before each test + rgName := GenerateTestResourceNameWithRandom("psqlsrv-rg", 10) + + postgreSQLServerName := GenerateTestResourceNameWithRandom("psql-srv", 10) + postgreSQLDatabaseName := GenerateTestResourceNameWithRandom("psql-db", 10) + + // Create the PostgreSQLDatabase object and expect the Reconcile to be created + postgreSQLDatabaseInstance1 := &azurev1alpha1.PostgreSQLDatabase{ + ObjectMeta: metav1.ObjectMeta{ + Name: postgreSQLDatabaseName, + Namespace: "default", + }, + Spec: azurev1alpha1.PostgreSQLDatabaseSpec{ + ResourceGroup: rgName, + Server: postgreSQLServerName, + }, + } + + EnsureInstanceWithResult(ctx, t, tc, postgreSQLDatabaseInstance1, errhelp.ResourceGroupNotFoundErrorCode, false) + EnsureDelete(ctx, t, tc, postgreSQLDatabaseInstance1) + +} + +func TestPSQLDatabaseControllerNoSever(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + // Add any setup steps that needs to be executed before each test + rgName := tc.resourceGroupName + + postgreSQLServerName := GenerateTestResourceNameWithRandom("psql-srv", 10) + postgreSQLDatabaseName := GenerateTestResourceNameWithRandom("psql-db", 10) + + // Create the PostgreSQLDatabase object and expect the Reconcile to be created + postgreSQLDatabaseInstance2 := &azurev1alpha1.PostgreSQLDatabase{ + ObjectMeta: metav1.ObjectMeta{ + Name: postgreSQLDatabaseName, + Namespace: "default", + }, + Spec: azurev1alpha1.PostgreSQLDatabaseSpec{ + ResourceGroup: rgName, + Server: postgreSQLServerName, + }, + } + + EnsureInstanceWithResult(ctx, t, tc, postgreSQLDatabaseInstance2, errhelp.ResourceNotFound, false) + EnsureDelete(ctx, t, tc, postgreSQLDatabaseInstance2) + +} diff --git a/controllers/postgresqlfirewallrule_controller_test.go b/controllers/postgresqlfirewallrule_controller_test.go new file mode 100644 index 00000000000..4805e8bb26e --- /dev/null +++ b/controllers/postgresqlfirewallrule_controller_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// +build all psqlfirewallrule + +package controllers + +import ( + "context" + "testing" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestPSQLFirewallRuleControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + // Add any setup steps that needs to be executed before each test + rgName := GenerateTestResourceNameWithRandom("psqlsrv-rg", 10) + + postgreSQLServerName := GenerateTestResourceNameWithRandom("psql-srv", 10) + postgreSQLFirewallRuleName := GenerateTestResourceNameWithRandom("psql-fwrule", 10) + + // Create the PostgreSQLFirewallRule object and expect the Reconcile to be created + postgreSQLFirewallRuleInstance := &azurev1alpha1.PostgreSQLFirewallRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: postgreSQLFirewallRuleName, + Namespace: "default", + }, + Spec: azurev1alpha1.PostgreSQLFirewallRuleSpec{ + ResourceGroup: rgName, + Server: postgreSQLServerName, + StartIPAddress: "0.0.0.0", + EndIPAddress: "0.0.0.0", + }, + } + + EnsureInstanceWithResult(ctx, t, tc, postgreSQLFirewallRuleInstance, errhelp.ResourceGroupNotFoundErrorCode, false) + EnsureDelete(ctx, t, tc, postgreSQLFirewallRuleInstance) + +} + +func TestPSQLFirewallRuleControllerNoServer(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + // Add any setup steps that needs to be executed before each test + rgName := tc.resourceGroupName + + postgreSQLServerName := GenerateTestResourceNameWithRandom("psql-srv", 10) + postgreSQLFirewallRuleName := GenerateTestResourceNameWithRandom("psql-fwrule", 10) + + // Create the PostgreSQLFirewallRule object and expect the Reconcile to be created + postgreSQLFirewallRuleInstance := &azurev1alpha1.PostgreSQLFirewallRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: postgreSQLFirewallRuleName, + Namespace: "default", + }, + Spec: azurev1alpha1.PostgreSQLFirewallRuleSpec{ + ResourceGroup: rgName, + Server: postgreSQLServerName, + StartIPAddress: "0.0.0.0", + EndIPAddress: "0.0.0.0", + }, + } + + EnsureInstanceWithResult(ctx, t, tc, postgreSQLFirewallRuleInstance, errhelp.ResourceNotFound, false) + EnsureDelete(ctx, t, tc, postgreSQLFirewallRuleInstance) + +} diff --git a/controllers/postgresqlserver_controller_test.go b/controllers/postgresqlserver_controller_test.go new file mode 100644 index 00000000000..c97948d06d6 --- /dev/null +++ b/controllers/postgresqlserver_controller_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// +build all psqlserver + +package controllers + +import ( + "context" + "testing" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//test Postgre SQL server unhappy path +func TestPSQLServerControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + // Add any setup steps that needs to be executed before each test + rgName := GenerateTestResourceNameWithRandom("psqlsrv-rg", 10) + rgLocation := tc.resourceGroupLocation + + postgreSQLServerName := GenerateTestResourceNameWithRandom("psql-srv", 10) + + // Create the PostgreSQLServer object and expect the Reconcile to be created + postgreSQLServerInstance := &azurev1alpha1.PostgreSQLServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: postgreSQLServerName, + Namespace: "default", + }, + Spec: azurev1alpha1.PostgreSQLServerSpec{ + Location: rgLocation, + ResourceGroup: rgName, + Sku: azurev1alpha1.AzureDBsSQLSku{ + Name: "B_Gen5_2", + Tier: azurev1alpha1.SkuTier("Basic"), + Family: "Gen5", + Size: "51200", + Capacity: 2, + }, + ServerVersion: azurev1alpha1.ServerVersion("10"), + SSLEnforcement: azurev1alpha1.SslEnforcementEnumEnabled, + }, + } + EnsureInstanceWithResult(ctx, t, tc, postgreSQLServerInstance, errhelp.ResourceGroupNotFoundErrorCode, false) + EnsureDelete(ctx, t, tc, postgreSQLServerInstance) + +} From ebca5a5101fca1e63ac742c83391fa89f947c5f5 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Tue, 14 Apr 2020 16:22:52 +0800 Subject: [PATCH 31/99] #task475#modifytheBuildTagForpsql --- controllers/postgresql_combined_controller_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/postgresql_combined_controller_test.go b/controllers/postgresql_combined_controller_test.go index 31d2c6a12aa..9d4e1bb567f 100644 --- a/controllers/postgresql_combined_controller_test.go +++ b/controllers/postgresql_combined_controller_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// +build all psql psqldatabase +// +build all psql package controllers From 09c8772faed0173b58b87fde1bf360c20c7ad3b0 Mon Sep 17 00:00:00 2001 From: Hong Bu <37413937+buhongw7583c@users.noreply.github.com> Date: Tue, 14 Apr 2020 17:10:22 +0800 Subject: [PATCH 32/99] Update config/samples/azure_v1alpha1_storage.yaml Co-Authored-By: Erin Corson --- config/samples/azure_v1alpha1_storage.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/samples/azure_v1alpha1_storage.yaml b/config/samples/azure_v1alpha1_storage.yaml index 30f34270af0..52241e9ee0b 100644 --- a/config/samples/azure_v1alpha1_storage.yaml +++ b/config/samples/azure_v1alpha1_storage.yaml @@ -1,7 +1,7 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: Storage metadata: - name: storagesamplegfdsa + name: storagesample spec: location: westus resourceGroup: resourcegroup-azure-operators @@ -19,4 +19,3 @@ spec: ipRules: #could be an ip range or a ip address - ipAddressOrRange: 2.2.0.0/24 - ipAddressOrRange: 2.2.2.1 - From 5e9f8e9d879d02a53778f5a9f9bb3a9e3103f8b4 Mon Sep 17 00:00:00 2001 From: Hong Bu <37413937+buhongw7583c@users.noreply.github.com> Date: Tue, 14 Apr 2020 17:11:12 +0800 Subject: [PATCH 33/99] Update pkg/resourcemanager/storages/storageaccount/storage.go Co-Authored-By: Erin Corson --- pkg/resourcemanager/storages/storageaccount/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resourcemanager/storages/storageaccount/storage.go b/pkg/resourcemanager/storages/storageaccount/storage.go index 24e0aa06445..e09fda9c314 100644 --- a/pkg/resourcemanager/storages/storageaccount/storage.go +++ b/pkg/resourcemanager/storages/storageaccount/storage.go @@ -36,7 +36,7 @@ func ParseNetworkPolicy(ruleSet *v1alpha1.StorageNetworkRuleSet) storage.Network } defaultAction := storage.DefaultActionDeny - if strings.ToLower(ruleSet.DefaultAction) == "allow" { + if strings.EqualFold(ruleSet.DefaultAction, "allow") { defaultAction = storage.DefaultActionAllow } From d874813927569e809b2145eeb17760a0c81a4225 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 08:35:25 -0600 Subject: [PATCH 34/99] changes for mysql replica --- api/v1alpha1/mysqlserver_types.go | 18 ++++++ pkg/resourcemanager/mysql/server/reconcile.go | 62 ++++++++++++++----- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 0892729e51c..8643937dc1e 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -71,6 +71,24 @@ func NewDefaultMySQLServer(name, resourceGroup, location string) *MySQLServer { }, ServerVersion: ServerVersion("8.0"), SSLEnforcement: SslEnforcementEnumEnabled, + CreateMode: "Default", + }, + } +} + +func NewReplicaMySQLServer(name, resourceGroup, location string, sourceserverid string) *MySQLServer { + return &MySQLServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: MySQLServerSpec{ + Location: location, + ResourceGroup: resourceGroup, + CreateMode: "Replica", + ReplicaProperties: ReplicaProperties{ + SourceServerId: sourceserverid, + }, }, } } diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 94155a7c50b..e7ae2b34ec3 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -6,6 +6,7 @@ package server import ( "context" "fmt" + "strings" mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -35,6 +36,19 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return true, err } + createmode := "Default" + if len(instance.Spec.CreateMode) != 0 { + createmode = instance.Spec.CreateMode + } + + // If a replica is requested, return error if source server is not specified + if strings.EqualFold(createmode, "replica") { + if len(instance.Spec.ReplicaProperties.SourceServerId) == 0 { + instance.Status.Message = "Replica requested but source server unspecified" + return true, nil + } + } + // Check to see if secret exists and if yes retrieve the admin login and password secret, err := m.GetOrPrepareSecret(ctx, instance) if err != nil { @@ -219,23 +233,43 @@ func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretN // GetOrPrepareSecret gets tje admin credentials if they are stored or generates some if not func (m *MySQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.MySQLServer) (map[string][]byte, error) { name := instance.Name + createmode := instance.Spec.CreateMode - secret := map[string][]byte{} + // If createmode == default, then this is a new server creation, so generate username/password + // If createmode == replica, then get the credentials from the source server secret and use that - key := types.NamespacedName{Name: name, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil - } + secret := map[string][]byte{} + var key types.NamespacedName + var err error + if strings.EqualFold(createmode, "default") { // new Mysql server creation - randomUsername := helpers.GenerateRandomUsername(10) - randomPassword := helpers.NewPassword() + key = types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } - secret["username"] = []byte(randomUsername) - secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) - secret["password"] = []byte(randomPassword) - secret["postgreSqlServerName"] = []byte(name) - // TODO: The below may not be right for non Azure public cloud. - secret["fullyQualifiedServerName"] = []byte(name + ".mysql.database.azure.com") + randomUsername := helpers.GenerateRandomUsername(10) + randomPassword := helpers.NewPassword() - return secret, nil + secret["username"] = []byte(randomUsername) + secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) + secret["password"] = []byte(randomPassword) + secret["mySqlServerName"] = []byte(name) + // TODO: The below may not be right for non Azure public cloud. + secret["fullyQualifiedServerName"] = []byte(name + ".mysql.database.azure.com") + return secret, nil + } + // replica creation + sourceServerId := instance.Spec.ReplicaProperties.SourceServerId + if len(sourceServerId) != 0 { + //Parse to get source server name + sourceServerIdSplit := strings.Split(sourceServerId, "/") + sourceserver := sourceServerIdSplit[len(sourceServerIdSplit)-1] + key = types.NamespacedName{Name: sourceserver, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } + // secret for source server does not exist, return error + } + return secret, err } From 12dc1a85b6f4d53bbc87a12aafd17d7f67e790fa Mon Sep 17 00:00:00 2001 From: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Date: Tue, 14 Apr 2020 09:45:09 -0700 Subject: [PATCH 35/99] Update the ResourceGroup operator to set Provisioning -> False (#932) * first * Oops Co-authored-by: William Mortl --- pkg/resourcemanager/resourcegroups/reconcile.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/resourcemanager/resourcegroups/reconcile.go b/pkg/resourcemanager/resourcegroups/reconcile.go index 77a8cbc74ab..49455d0449e 100644 --- a/pkg/resourcemanager/resourcegroups/reconcile.go +++ b/pkg/resourcemanager/resourcegroups/reconcile.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net/http" + "strings" "github.com/Azure/azure-service-operator/api/v1alpha1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -28,21 +29,28 @@ func (g *AzureResourceGroupManager) Ensure(ctx context.Context, obj runtime.Obje group, err := g.CreateGroup(ctx, resourcegroupName, resourcegroupLocation) if err != nil { - instance.Status.Provisioned = false instance.Status.Message = err.Error() + azerr := errhelp.NewAzureErrorAzureError(err) + + // this happens when op isnt complete, just requeue + if strings.Contains(azerr.Type, errhelp.AsyncOpIncompleteError) { + return false, nil + } + + instance.Status.Provisioning = false // handle special cases that won't work without a change to spec if group.StatusCode == http.StatusBadRequest { - instance.Status.Provisioning = false + instance.Status.FailedProvisioning = true return true, nil } return false, fmt.Errorf("ResourceGroup create error %v", err) - } instance.Status.Provisioned = true instance.Status.Provisioning = false + instance.Status.FailedProvisioning = false instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *group.ID From 387479b4e245252be605beddd19ff999f2a8e8a4 Mon Sep 17 00:00:00 2001 From: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Date: Tue, 14 Apr 2020 10:22:57 -0700 Subject: [PATCH 36/99] Update the EventHub operators to set Provisioning -> False (#931) * first * forgot return because im dumb Co-authored-by: William Mortl --- .../eventhubs/consumergroup.go | 11 +++++-- pkg/resourcemanager/eventhubs/hub.go | 29 +++++++------------ pkg/resourcemanager/eventhubs/namespace.go | 12 +++----- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/pkg/resourcemanager/eventhubs/consumergroup.go b/pkg/resourcemanager/eventhubs/consumergroup.go index d11f0375592..4e999f58290 100644 --- a/pkg/resourcemanager/eventhubs/consumergroup.go +++ b/pkg/resourcemanager/eventhubs/consumergroup.go @@ -6,6 +6,7 @@ package eventhubs import ( "context" "fmt" + "strings" "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" @@ -105,15 +106,19 @@ func (cg *azureConsumerGroupManager) Ensure(ctx context.Context, obj runtime.Obj newCg, err := cg.CreateConsumerGroup(ctx, resourcegroup, namespaceName, eventhubName, azureConsumerGroupName) if err != nil { instance.Status.Message = err.Error() - instance.Status.Provisioning = false + azerr := errhelp.NewAzureErrorAzureError(err) + + // this happens when op isnt complete, just requeue + if strings.Contains(azerr.Type, errhelp.AsyncOpIncompleteError) { + return false, nil + } + instance.Status.Provisioning = false catch := []string{ errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode, errhelp.NotFoundErrorCode, } - - azerr := errhelp.NewAzureErrorAzureError(err) if helpers.ContainsString(catch, azerr.Type) { // reconciliation is not done but error is acceptable return false, nil diff --git a/pkg/resourcemanager/eventhubs/hub.go b/pkg/resourcemanager/eventhubs/hub.go index a789f1f91cf..a216702a724 100644 --- a/pkg/resourcemanager/eventhubs/hub.go +++ b/pkg/resourcemanager/eventhubs/hub.go @@ -228,61 +228,54 @@ func (e *azureEventHubManager) Ensure(ctx context.Context, obj runtime.Object, o hub, err := e.CreateHub(ctx, resourcegroup, eventhubNamespace, eventhubName, messageRetentionInDays, partitionCount, capturePtr) if err != nil { - // let the user know what happened - instance.Status.Message = err.Error() - instance.Status.Provisioning = false + azerr := errhelp.NewAzureErrorAzureError(err) + + // this happens when op isnt complete, just requeue + if azerr.Type == errhelp.AsyncOpIncompleteError { + return false, nil + } + // errors we expect might happen that we are ok with waiting for + instance.Status.Provisioning = false catch := []string{ errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode, errhelp.NotFoundErrorCode, - errhelp.AsyncOpIncompleteError, errhelp.BadRequest, } - - azerr := errhelp.NewAzureErrorAzureError(err) if helpers.ContainsString(catch, azerr.Type) { - // most of these error technically mean the resource is actually not provisioning - - switch azerr.Type { - case errhelp.AsyncOpIncompleteError: - instance.Status.Provisioning = true - case errhelp.BadRequest: + if strings.Contains(azerr.Type, errhelp.BadRequest) { // can't put the error for this one in Message as the tracking id changes every time caussing extra reconciles if strings.Contains(azerr.Reason, "Storage Account") && strings.Contains(azerr.Reason, "was not found") { instance.Status.Message = "Storage Account was not found" } } - // reconciliation is not done but error is acceptable return false, nil } + // reconciliation not done and we don't know what happened return false, err - } err = e.createOrUpdateAccessPolicyEventHub(resourcegroup, eventhubNamespace, eventhubName, instance) if err != nil { instance.Status.Message = err.Error() - return false, err } err = e.listAccessKeysAndCreateSecrets(resourcegroup, eventhubNamespace, eventhubName, secretName, instance.Spec.AuthorizationRule.Name, instance) if err != nil { instance.Status.Message = err.Error() - return false, err } - // write information back to instance + // reconciliation done and everything looks ok instance.Status.State = string(hub.Status) instance.Status.Message = resourcemanager.SuccessMsg instance.Status.Provisioning = false instance.Status.Provisioned = true instance.Status.ResourceId = *hub.ID - // reconciliation done and everything looks ok return true, nil } diff --git a/pkg/resourcemanager/eventhubs/namespace.go b/pkg/resourcemanager/eventhubs/namespace.go index c84ee35dfa5..262d5892a3c 100644 --- a/pkg/resourcemanager/eventhubs/namespace.go +++ b/pkg/resourcemanager/eventhubs/namespace.go @@ -326,24 +326,20 @@ func (ns *azureEventHubNamespaceManager) Ensure(ctx context.Context, obj runtime newNs, err := ns.CreateNamespace(ctx, resourcegroup, namespaceName, namespaceLocation, eventHubNSSku, eventHubNSProperties) if err != nil { instance.Status.Message = err.Error() - catch := []string{ - errhelp.ResourceGroupNotFoundErrorCode, - errhelp.AsyncOpIncompleteError, - } azerr := errhelp.NewAzureErrorAzureError(err) + if strings.Contains(azerr.Type, errhelp.AsyncOpIncompleteError) { // resource creation request sent to Azure instance.Status.SpecHash = hash - } - - if helpers.ContainsString(catch, azerr.Type) || strings.Contains(err.Error(), "validation failed") { return false, nil } instance.Status.Provisioning = false + if strings.Contains(azerr.Type, errhelp.ResourceGroupNotFoundErrorCode) || strings.Contains(err.Error(), "validation failed") { + return false, nil + } return false, fmt.Errorf("EventhubNamespace create error %v", err) - } // write information back to instance From 1987aa1ac5d434da20b74cff4e226f58a4de3bcd Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Tue, 14 Apr 2020 12:30:08 -0700 Subject: [PATCH 37/99] added apim docs --- docs/apimgmt/apimgmt.md | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index e69de29bb2d..b91cf25212b 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -0,0 +1,48 @@ +# API Management Operator + +The API Management suite is made up of the following operators: +* API Management Service +* API Management API + +Learn more about Azure API Management [here](https://docs.microsoft.com/en-us/azure/api-management/api-management-key-concepts). + +### API Management Service + +The API Management Service deploys an API Management instance into a specified resource group at the specified location. + +The spec consists of the following fields: + +#### Required Fields +* `Tier` Specify the tier of the service. Options include: 'basic', 'standard', and 'premium'. +* `PublisherName` Specify the name of the publisher. +* `PublisherEmail` Specify the email of the publisher. + +#### Optional Fields +* `VnetType` Specify the type of vnet for the service. Options include: 'none', 'internal', and 'external'. If selecting either 'internal' or 'external', make sure to set the `Tier` to 'premium'. +* `VnetResourceGroup` Resource group of the Virtual Network +* `VnetName` Name of the Virtual Network +* `VnetSubnetName` Name of the Virtual Network Subnet +* `AppInsightsResourceGroup` Resource group of the Application Insights instance +* `AppInsightsName` Name of the Application Insights instance + +### API Management API + +The API Management API Operator manages your API Management service, and deploys an instance into a specified resource group at the specified location. + +The spec consists of the following fields: + +* `apiService` The name of the API Management service to managed +* `apiId` Specify an ID for the API +* `properties` + `isCurrent` Indicate if this API revision is the current revision + `isOnline` Indicate if this API is accessible via the gateway + `displayName` Display name for the API + `description` Description of the API + `apiVersionDescription` Description of the API Version + `path` Path for the API + `protocols` Protocols for the service. Possible values are: 'http' or 'https' + `subscriptionRequired` Specify whether an API or Product subscription is required for accessing the API + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. \ No newline at end of file From dcac8df10b2d8afe50b1ef5a65fb18c0e1cb1662 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Tue, 14 Apr 2020 12:32:28 -0700 Subject: [PATCH 38/99] formatting --- docs/apimgmt/apimgmt.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index b91cf25212b..8fa95af31a7 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -34,14 +34,14 @@ The spec consists of the following fields: * `apiService` The name of the API Management service to managed * `apiId` Specify an ID for the API * `properties` - `isCurrent` Indicate if this API revision is the current revision - `isOnline` Indicate if this API is accessible via the gateway - `displayName` Display name for the API - `description` Description of the API - `apiVersionDescription` Description of the API Version - `path` Path for the API - `protocols` Protocols for the service. Possible values are: 'http' or 'https' - `subscriptionRequired` Specify whether an API or Product subscription is required for accessing the API + * `isCurrent` Indicate if this API revision is the current revision + * `isOnline` Indicate if this API is accessible via the gateway + * `displayName` Display name for the API + * `description` Description of the API + * `apiVersionDescription` Description of the API Version + * `path` Path for the API + * `protocols` Protocols for the service. Possible values are: 'http' or 'https' + * `subscriptionRequired` Specify whether an API or Product subscription is required for accessing the API ## Deploy, view and delete resources From 78dbf84cd5af8f9df140d1ccc0bf6df82d588db8 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Tue, 7 Apr 2020 19:55:00 -0600 Subject: [PATCH 39/99] refactor AzureError creation for cleanliness --- pkg/errhelp/errors.go | 57 ++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 4318fde13e5..38f676ef77e 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -66,10 +66,12 @@ func NewAzureError(err error) error { Original: err, } - if det, ok := err.(autorest.DetailedError); ok { + switch v := err.(type) { + case autorest.DetailedError: + ae.Code = v.StatusCode.(int) - ae.Code = det.StatusCode.(int) - if e, ok := det.Original.(*azure.RequestError); ok { + switch e := v.Original.(type) { + case *azure.RequestError: if e.ServiceError != nil { kind = e.ServiceError.Code reason = e.ServiceError.Message @@ -77,7 +79,7 @@ func NewAzureError(err error) error { kind = CannotParseError reason = CannotParseError } - } else if e, ok := det.Original.(azure.RequestError); ok { + case azure.RequestError: if e.ServiceError != nil { kind = e.ServiceError.Code reason = e.ServiceError.Message @@ -85,7 +87,7 @@ func NewAzureError(err error) error { kind = CannotParseError reason = CannotParseError } - } else if e, ok := det.Original.(*azure.ServiceError); ok { + case *azure.ServiceError: kind = e.Code reason = e.Message if e.Code == "Failed" && len(e.AdditionalInfo) == 1 { @@ -93,36 +95,41 @@ func NewAzureError(err error) error { kind = v.(string) } } - } else if _, ok := det.Original.(*errors.StatusError); ok { + case *errors.StatusError: kind = "StatusError" reason = "StatusError" - } else if _, ok := det.Original.(*json.UnmarshalTypeError); ok { + case *json.UnmarshalTypeError: kind = NotFoundErrorCode reason = NotFoundErrorCode } - } else if _, ok := err.(azure.AsyncOpIncompleteError); ok { + case azure.AsyncOpIncompleteError: kind = "AsyncOpIncomplete" reason = "AsyncOpIncomplete" - } else if verr, ok := err.(validation.Error); ok { + case validation.Error: kind = "ValidationError" - reason = verr.Message - } else if err.Error() == InvalidServerName { - kind = InvalidServerName - reason = InvalidServerName - } else if err.Error() == AlreadyExists { - kind = AlreadyExists - reason = AlreadyExists - } else if err.Error() == AccountNameInvalid { - kind = AccountNameInvalid - reason = AccountNameInvalid - } else if strings.Contains(err.Error(), InvalidAccessPolicy) { - kind = InvalidAccessPolicy - reason = InvalidAccessPolicy - } else if strings.Contains(err.Error(), LocationNotAvailableForResourceType) { - kind = LocationNotAvailableForResourceType - reason = LocationNotAvailableForResourceType + reason = v.Message + default: + switch e := err.Error(); e { + case InvalidServerName, AlreadyExists, AccountNameInvalid: + kind = e + reason = e + default: + contains := []string{ + InvalidAccessPolicy, + LocationNotAvailableForResourceType, + } + for _, val := range contains { + if strings.Contains(e, val) { + kind = val + reason = val + break + } + } + } + } + ae.Reason = reason ae.Type = kind From 49560ce4a0f6560a0e4f0aa76c11e7ee403b5e1b Mon Sep 17 00:00:00 2001 From: jpflueger Date: Tue, 14 Apr 2020 15:22:24 -0600 Subject: [PATCH 40/99] testing with secret client --- controllers/cosmosdb_controller_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index 2fd0ccdb935..c0a47d398a0 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -42,19 +41,18 @@ func TestCosmosDBHappyPath(t *testing.T) { } key := types.NamespacedName{Name: name, Namespace: namespace} - secret := &v1.Secret{} EnsureInstance(ctx, t, tc, dbInstance) assert.Eventually(func() bool { - err := tc.k8sClient.Get(ctx, key, secret) - return err == nil && secret.ObjectMeta.Name == name && secret.ObjectMeta.Namespace == namespace + secret, err := tc.secretClient.Get(ctx, key) + return err == nil && len(secret) > 0 }, tc.timeoutFast, tc.retry, "wait for cosmosdb to have secret") EnsureDelete(ctx, t, tc, dbInstance) assert.Eventually(func() bool { - err := tc.k8sClient.Get(ctx, key, secret) + _, err := tc.secretClient.Get(ctx, key) return err != nil }, tc.timeoutFast, tc.retry, "wait for cosmosdb to delete secret") From 2ccd809aa02c6752129fb0ce4c6fc843f82d8828 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Tue, 14 Apr 2020 15:30:10 -0600 Subject: [PATCH 41/99] returning error from cosmosdb resource manager --- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 35 +++++++++---------- .../cosmosdbs/cosmosdb_manager.go | 9 +++-- .../cosmosdbs/cosmosdb_reconcile.go | 21 ++++++----- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 8f7f8ada4d1..a79573e4976 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -10,7 +10,6 @@ import ( "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" "github.com/Azure/azure-service-operator/pkg/secrets" @@ -45,10 +44,10 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, - tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) { + tags map[string]*string) (*documentdb.DatabaseAccount, error) { cosmosDBClient, err := getCosmosDBClient() if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } dbKind := documentdb.DatabaseAccountKind(kind) @@ -94,13 +93,13 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( if err != nil { // initial create request failed, wrap error - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } result, err := createUpdateFuture.Result(cosmosDBClient) if err != nil { // there is no immediate result, wrap error - return &result, errhelp.NewAzureErrorAzureError(err) + return &result, err } return &result, nil } @@ -109,15 +108,15 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( func (*AzureCosmosDBManager) GetCosmosDB( ctx context.Context, groupName string, - cosmosDBName string) (*documentdb.DatabaseAccount, *errhelp.AzureError) { + cosmosDBName string) (*documentdb.DatabaseAccount, error) { cosmosDBClient, err := getCosmosDBClient() if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } result, err := cosmosDBClient.Get(ctx, groupName, cosmosDBName) if err != nil { - return &result, errhelp.NewAzureErrorAzureError(err) + return &result, err } return &result, nil } @@ -125,15 +124,15 @@ func (*AzureCosmosDBManager) GetCosmosDB( // CheckNameExistsCosmosDB checks if the global account name already exists func (*AzureCosmosDBManager) CheckNameExistsCosmosDB( ctx context.Context, - accountName string) (bool, *errhelp.AzureError) { + accountName string) (bool, error) { cosmosDBClient, err := getCosmosDBClient() if err != nil { - return false, errhelp.NewAzureErrorAzureError(err) + return false, err } response, err := cosmosDBClient.CheckNameExists(ctx, accountName) if err != nil { - return false, errhelp.NewAzureErrorAzureError(err) + return false, err } switch response.StatusCode { @@ -142,7 +141,7 @@ func (*AzureCosmosDBManager) CheckNameExistsCosmosDB( case http.StatusOK: return true, nil default: - return false, errhelp.NewAzureErrorAzureError(fmt.Errorf("unhandled status code for CheckNameExists")) + return false, fmt.Errorf("unhandled status code for CheckNameExists") } } @@ -150,20 +149,20 @@ func (*AzureCosmosDBManager) CheckNameExistsCosmosDB( func (*AzureCosmosDBManager) DeleteCosmosDB( ctx context.Context, groupName string, - cosmosDBName string) (*autorest.Response, *errhelp.AzureError) { + cosmosDBName string) (*autorest.Response, error) { cosmosDBClient, err := getCosmosDBClient() if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } deleteFuture, err := cosmosDBClient.Delete(ctx, groupName, cosmosDBName) if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } ar, err := deleteFuture.Result(cosmosDBClient) if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } return &ar, nil } @@ -175,12 +174,12 @@ func (*AzureCosmosDBManager) ListKeys( accountName string) (*documentdb.DatabaseAccountListKeysResult, error) { client, err := getCosmosDBClient() if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } result, err := client.ListKeys(ctx, groupName, accountName) if err != nil { - return nil, errhelp.NewAzureErrorAzureError(err) + return nil, err } return &result, nil diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index e67ec2c5471..9ce839c076d 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -8,7 +8,6 @@ import ( "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest" @@ -22,16 +21,16 @@ func NewAzureCosmosDBManager(secretClient secrets.SecretClient) *AzureCosmosDBMa // CosmosDBManager client functions type CosmosDBManager interface { // CreateOrUpdateCosmosDB creates a new cosmos database account - CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) + CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, tags map[string]*string) (*documentdb.DatabaseAccount, error) // GetCosmosDB gets a cosmos database account - GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, *errhelp.AzureError) + GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, error) // DeleteCosmosDB removes the cosmos database account - DeleteCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*autorest.Response, *errhelp.AzureError) + DeleteCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*autorest.Response, error) // CheckNameExistsCosmosDB check if the account name already exists globally - CheckNameExistsCosmosDB(ctx context.Context, accountName string) (bool, *errhelp.AzureError) + CheckNameExistsCosmosDB(ctx context.Context, accountName string) (bool, error) // ListKeys lists the read & write keys for a database account ListKeys(ctx context.Context, groupName string, accountName string) (*documentdb.DatabaseAccountListKeysResult, error) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 680612c3a48..e746619f4ff 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -42,7 +42,8 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return true, nil } else if instance.Status.Provisioning { // get the instance and update status - db, azerr := m.GetCosmosDB(ctx, instance.Spec.ResourceGroup, instance.Name) + db, err := m.GetCosmosDB(ctx, instance.Spec.ResourceGroup, instance.Name) + azerr := errhelp.NewAzureErrorAzureError(err) if azerr == nil { instance.Status.ResourceId = *db.ID instance.Status.State = *db.ProvisioningState @@ -106,10 +107,10 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o kind := instance.Spec.Kind dbType := instance.Spec.Properties.DatabaseAccountOfferType - db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) + db, err := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) // everything is in a created/updated state - if azerr == nil { + if err == nil { instance.Status.Provisioned = true instance.Status.Provisioning = false instance.Status.Message = resourcemanager.SuccessMsg @@ -118,7 +119,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return true, nil } - switch azerr.Type { + switch azerr := errhelp.NewAzureErrorAzureError(err); azerr.Type { case errhelp.AsyncOpIncompleteError: instance.Status.Message = "Resource request successfully submitted to Azure" @@ -165,8 +166,10 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o } // fetch the latest to inspect provisioning state - cosmosDB, azerr := m.GetCosmosDB(ctx, groupName, accountName) - if azerr != nil { + cosmosDB, err := m.GetCosmosDB(ctx, groupName, accountName) + if err != nil { + azerr := errhelp.NewAzureErrorAzureError(err) + // deletion finished if helpers.ContainsString(notFoundErrors, azerr.Type) { return false, nil @@ -190,9 +193,11 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o } // try to delete the cosmosdb instance & secrets - _, azerr = m.DeleteCosmosDB(ctx, groupName, accountName) + _, err = m.DeleteCosmosDB(ctx, groupName, accountName) m.deleteAccountKeysSecret(ctx, instance) - if azerr != nil { + if err != nil { + azerr := errhelp.NewAzureErrorAzureError(err) + // this is likely to happen on first try due to not waiting for the future to complete if azerr.Type == errhelp.AsyncOpIncompleteError { instance.Status.Message = "Deletion request submitted successfully" From 25735b17c6babb87e3605a5d6b912521498979a5 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 16:53:01 -0600 Subject: [PATCH 42/99] Mysql replica support --- api/v1alpha1/mysqlserver_types.go | 18 ++-- .../samples/azure_v1alpha1_mysqlserver.yaml | 6 +- .../azure_v1alpha1_mysqlserver_replica.yaml | 2 + pkg/errhelp/errhelp.go | 1 + pkg/errhelp/errors.go | 2 + pkg/resourcemanager/mysql/server/client.go | 65 +++++++---- pkg/resourcemanager/mysql/server/manager.go | 2 +- pkg/resourcemanager/mysql/server/reconcile.go | 101 ++++++++++-------- 8 files changed, 117 insertions(+), 80 deletions(-) diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 8643937dc1e..ae04a831b13 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -12,16 +12,14 @@ import ( // MySQLServerSpec defines the desired state of MySQLServer type MySQLServerSpec struct { - Location string `json:"location"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Sku AzureDBsSQLSku `json:"sku,omitempty"` - ServerVersion ServerVersion `json:"serverVersion,omitempty"` - SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` - MinimalTLSVersion string `json:"minimalTLSVersion,omitempty"` - InfrastructureEncryption string `json:"infrastructureEncryption,omitempty"` - CreateMode string `json:"createMode,omitempty"` - ReplicaProperties ReplicaProperties `json:"replicaProperties, omitempty"` - KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` + Location string `json:"location"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Sku AzureDBsSQLSku `json:"sku,omitempty"` + ServerVersion ServerVersion `json:"serverVersion,omitempty"` + SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` + CreateMode string `json:"createMode,omitempty"` + ReplicaProperties ReplicaProperties `json:"replicaProperties,omitempty"` + KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` } // +kubebuilder:object:root=true diff --git a/config/samples/azure_v1alpha1_mysqlserver.yaml b/config/samples/azure_v1alpha1_mysqlserver.yaml index c73f0e3b4f1..d8f43cd3c75 100644 --- a/config/samples/azure_v1alpha1_mysqlserver.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver.yaml @@ -11,9 +11,9 @@ spec: infrastructureEncryption: Enabled # Possible values include: Enabled, Disabled createMode: Default # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) sku: - name: B_Gen5_2 - tier: Basic - family: Gen5 + name: B_Gen5_2 # tier + family + cores eg. - B_Gen4_1, GP_Gen5_4 + tier: Basic # possible values - 'Basic', 'GeneralPurpose', 'MemoryOptimized' + family: Gen5 size: "51200" capacity: 2 diff --git a/config/samples/azure_v1alpha1_mysqlserver_replica.yaml b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml index b256cec145d..515e40589b4 100644 --- a/config/samples/azure_v1alpha1_mysqlserver_replica.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml @@ -4,7 +4,9 @@ metadata: name: mysqlserver-replica spec: location: eastus2 + resourceGroup: resourcegroup-azure-operators createMode: Replica # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) replicaProperties: + # sourceServer tier should be "GeneralPurpose" or higher for replica support sourceServerId: /subscriptions/{SUBID}/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.DBforMySQL/servers/mysqlserver-sample diff --git a/pkg/errhelp/errhelp.go b/pkg/errhelp/errhelp.go index 58008895018..d2614f2e823 100644 --- a/pkg/errhelp/errhelp.go +++ b/pkg/errhelp/errhelp.go @@ -49,6 +49,7 @@ func StripErrorIDs(err error) string { patterns := []string{ "RequestID=", "CorrelationId:\\s", + "Tracking ID: ", } reg := regexp.MustCompile(fmt.Sprintf(`(%s)\S+`, strings.Join(patterns, "|"))) return reg.ReplaceAllString(err.Error(), "") diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 4318fde13e5..ec0fa6c7fe2 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -55,6 +55,8 @@ const ( ServiceBusy = "ServiceBusy" NameNotAvailable = "NameNotAvailable" PublicIPIdleTimeoutIsOutOfRange = "PublicIPIdleTimeoutIsOutOfRange" + InvalidRequestContent = "InvalidRequestContent" + InternalServerError = "InternalServerError" ) func NewAzureError(err error) error { diff --git a/pkg/resourcemanager/mysql/server/client.go b/pkg/resourcemanager/mysql/server/client.go index c6a8804989f..8e26294dc5c 100644 --- a/pkg/resourcemanager/mysql/server/client.go +++ b/pkg/resourcemanager/mysql/server/client.go @@ -5,11 +5,13 @@ package server import ( "context" + "strings" mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" "github.com/Azure/azure-service-operator/pkg/secrets" + "github.com/Azure/go-autorest/autorest/to" "k8s.io/apimachinery/pkg/runtime" ) @@ -59,34 +61,55 @@ func (m *MySQLServerClient) CheckServerNameAvailability(ctx context.Context, ser } -func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string) (future mysql.ServersCreateFuture, err error) { +func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (server mysql.Server, err error) { client := getMySQLServersClient() // Check if name is valid if this is the first create call valid, err := m.CheckServerNameAvailability(ctx, servername) - if valid == false { - return future, err + if !valid { + return server, err } - - return client.Create( - ctx, - resourcegroup, - servername, - mysql.ServerForCreate{ - Location: &location, - Tags: tags, - Properties: &mysql.ServerPropertiesForDefaultCreate{ - AdministratorLogin: &adminlogin, - AdministratorLoginPassword: &adminpassword, - Version: serverversion, - SslEnforcement: sslenforcement, - //StorageProfile: &mysql.StorageProfile{}, - CreateMode: mysql.CreateModeServerPropertiesForCreate, + var result mysql.ServersCreateFuture + if strings.EqualFold(createmode, "replica") { + result, _ = client.Create( + ctx, + resourcegroup, + servername, + mysql.ServerForCreate{ + Location: &location, + Tags: tags, + Properties: &mysql.ServerPropertiesForReplica{ + SourceServerID: to.StringPtr(sourceserver), + CreateMode: mysql.CreateModeReplica, + }, + }, + ) + + } else { + + result, _ = client.Create( + ctx, + resourcegroup, + servername, + mysql.ServerForCreate{ + Location: &location, + Tags: tags, + Properties: &mysql.ServerPropertiesForDefaultCreate{ + AdministratorLogin: &adminlogin, + AdministratorLoginPassword: &adminpassword, + Version: serverversion, + SslEnforcement: sslenforcement, + //StorageProfile: &mysql.StorageProfile{}, + CreateMode: mysql.CreateModeServerPropertiesForCreate, + }, + Sku: &skuInfo, }, - Sku: &skuInfo, - }, - ) + ) + } + + return result.Result(client) + } func (m *MySQLServerClient) DeleteServer(ctx context.Context, resourcegroup string, servername string) (status string, err error) { diff --git a/pkg/resourcemanager/mysql/server/manager.go b/pkg/resourcemanager/mysql/server/manager.go index cc4b620f8d0..350a6450476 100644 --- a/pkg/resourcemanager/mysql/server/manager.go +++ b/pkg/resourcemanager/mysql/server/manager.go @@ -11,7 +11,7 @@ import ( ) type MySQLServerManager interface { - CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string) (mysql.ServersCreateFuture, error) + CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (mysql.Server, error) DeleteServer(ctx context.Context, resourcegroup string, servername string) (string, error) GetServer(ctx context.Context, resourcegroup string, servername string) (mysql.Server, error) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index e7ae2b34ec3..e0f07a5de6f 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -41,7 +41,7 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts createmode = instance.Spec.CreateMode } - // If a replica is requested, return error if source server is not specified + // If a replica is requested, ensure that source server is specified if strings.EqualFold(createmode, "replica") { if len(instance.Spec.ReplicaProperties.SourceServerId) == 0 { instance.Status.Message = "Replica requested but source server unspecified" @@ -55,17 +55,8 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } - // Update secret - err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) - if err != nil { - return false, err - } - // convert kube labels to expected tag format - labels := map[string]*string{} - for k, v := range instance.GetLabels() { - labels[k] = &v - } + labels := helpers.LabelsToTags(instance.GetLabels()) // Check if this server already exists and its state if it does. This is required // to overcome the issue with the lack of idempotence of the Create call @@ -77,6 +68,14 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.Provisioning = false instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *server.ID + instance.Status.State = string(server.UserVisibleState) + + // Update secret - we do this on success as we need the FQ name of the server + err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) + if err != nil { + return false, err + } + return true, nil } return false, nil @@ -98,7 +97,7 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts Family: to.StringPtr(instance.Spec.Sku.Family), } - _, err = m.CreateServerIfValid( + server, err = m.CreateServerIfValid( ctx, instance.Name, instance.Spec.ResourceGroup, @@ -109,10 +108,12 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts skuInfo, adminlogin, adminpassword, + createmode, + instance.Spec.ReplicaProperties.SourceServerId, ) if err != nil { // let the user know what happened - instance.Status.Message = err.Error() + instance.Status.Message = errhelp.StripErrorIDs(err) instance.Status.Provisioning = false azerr := errhelp.NewAzureErrorAzureError(err) @@ -120,12 +121,16 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: // errors we expect might happen that we are ok with waiting for return false, nil - case errhelp.ProvisioningDisabled, errhelp.LocationNotAvailableForResourceType: + case errhelp.ProvisioningDisabled, errhelp.LocationNotAvailableForResourceType, errhelp.InvalidRequestContent, errhelp.InternalServerError: // Unrecoverable error, so stop reconcilation - instance.Status.Message = "Reconcilation hit unrecoverable error: " + err.Error() + instance.Status.Message = "Reconcilation hit unrecoverable error: " + errhelp.StripErrorIDs(err) return true, nil + case errhelp.AsyncOpIncompleteError: + // Creation in progress + instance.Status.Provisioning = true + instance.Status.Message = "Server request submitted to Azure" + return false, nil } - // reconciliation not done and we don't know what happened return false, err } @@ -211,12 +216,15 @@ func (m *MySQLServerClient) convert(obj runtime.Object) (*v1alpha1.MySQLServer, } // AddServerCredsToSecrets saves the server's admin credentials in the secret store -func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer) error { +func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer, fullservername string) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, } + // Update fullyQualifiedServerName from the created server + data["fullyQualifiedServerName"] = []byte(fullservername) + err := m.SecretClient.Upsert(ctx, key, data, @@ -240,36 +248,39 @@ func (m *MySQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *az secret := map[string][]byte{} var key types.NamespacedName - var err error - if strings.EqualFold(createmode, "default") { // new Mysql server creation - - key = types.NamespacedName{Name: name, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil - } + var Username string + var Password string - randomUsername := helpers.GenerateRandomUsername(10) - randomPassword := helpers.NewPassword() - - secret["username"] = []byte(randomUsername) - secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) - secret["password"] = []byte(randomPassword) - secret["mySqlServerName"] = []byte(name) - // TODO: The below may not be right for non Azure public cloud. - secret["fullyQualifiedServerName"] = []byte(name + ".mysql.database.azure.com") - return secret, nil + // See if secret already exists and return if it does + key = types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil } - // replica creation - sourceServerId := instance.Spec.ReplicaProperties.SourceServerId - if len(sourceServerId) != 0 { - //Parse to get source server name - sourceServerIdSplit := strings.Split(sourceServerId, "/") - sourceserver := sourceServerIdSplit[len(sourceServerIdSplit)-1] - key = types.NamespacedName{Name: sourceserver, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil + + if strings.EqualFold(createmode, "default") { // new Mysql server creation + // Generate random username password if secret does not exist already + Username = helpers.GenerateRandomUsername(10) + Password = helpers.NewPassword() + } else { // replica + sourceServerId := instance.Spec.ReplicaProperties.SourceServerId + if len(sourceServerId) != 0 { + // Parse to get source server name + sourceServerIdSplit := strings.Split(sourceServerId, "/") + sourceserver := sourceServerIdSplit[len(sourceServerIdSplit)-1] + + // Get the username and password from the source server's secret + key = types.NamespacedName{Name: sourceserver, Namespace: instance.Namespace} + if sourcesecret, err := m.SecretClient.Get(ctx, key); err == nil { + Username = string(sourcesecret["username"]) + Password = string(sourcesecret["password"]) + } } - // secret for source server does not exist, return error } - return secret, err + + // Populate secret fields + secret["username"] = []byte(Username) + secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", Username, name)) + secret["password"] = []byte(Password) + secret["mySqlServerName"] = []byte(name) + return secret, nil } From 0ee4ce4cfef793a241ca59cbe2b3b1629e6f605e Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Mon, 13 Apr 2020 16:46:45 -0600 Subject: [PATCH 43/99] small cleanup from working through the tasks in issue 703 --- .../psql/database/database_reconcile.go | 20 ++++++++++--------- .../firewallrule/firewallrule_reconcile.go | 20 ++++++++++--------- .../psql/server/server_reconcile.go | 20 ++++++++++--------- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/pkg/resourcemanager/psql/database/database_reconcile.go b/pkg/resourcemanager/psql/database/database_reconcile.go index f93d4f6d100..70e2665dc99 100644 --- a/pkg/resourcemanager/psql/database/database_reconcile.go +++ b/pkg/resourcemanager/psql/database/database_reconcile.go @@ -37,6 +37,7 @@ func (p *PSQLDatabaseClient) Ensure(ctx context.Context, obj runtime.Object, opt } instance.Status.Provisioning = true + instance.Status.FailedProvisioning = false resp, err := p.CreateDatabaseIfValid( ctx, instance.Name, @@ -45,6 +46,8 @@ func (p *PSQLDatabaseClient) Ensure(ctx context.Context, obj runtime.Object, opt ) if err != nil { instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false + azerr := errhelp.NewAzureErrorAzureError(err) catchInProgress := []string{ @@ -68,19 +71,18 @@ func (p *PSQLDatabaseClient) Ensure(ctx context.Context, obj runtime.Object, opt // handle the errors if helpers.ContainsString(catchInProgress, azerr.Type) { instance.Status.Message = "Postgres database exists but may not be ready" + instance.Status.Provisioning = true return false, nil } else if helpers.ContainsString(catchKnownError, azerr.Type) { - instance.Status.Provisioning = false return false, nil - } else { - - // serious error occured, end reconcilliation and mark it as failed - instance.Status.Message = fmt.Sprintf("Error occurred creating the Postgres server: %s", errhelp.StripErrorIDs(err)) - instance.Status.Provisioned = false - instance.Status.Provisioning = false - instance.Status.FailedProvisioning = true - return true, nil } + + // serious error occured, end reconcilliation and mark it as failed + instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioned = false + instance.Status.FailedProvisioning = true + return true, nil + } return false, nil diff --git a/pkg/resourcemanager/psql/firewallrule/firewallrule_reconcile.go b/pkg/resourcemanager/psql/firewallrule/firewallrule_reconcile.go index 7d4a7c005c1..75cde160be4 100644 --- a/pkg/resourcemanager/psql/firewallrule/firewallrule_reconcile.go +++ b/pkg/resourcemanager/psql/firewallrule/firewallrule_reconcile.go @@ -34,6 +34,7 @@ func (p *PSQLFirewallRuleClient) Ensure(ctx context.Context, obj runtime.Object, } instance.Status.Provisioning = true + instance.Status.FailedProvisioning = false resp, err := p.CreateFirewallRule( ctx, instance.Spec.ResourceGroup, @@ -44,6 +45,8 @@ func (p *PSQLFirewallRuleClient) Ensure(ctx context.Context, obj runtime.Object, ) if err != nil { instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false + azerr := errhelp.NewAzureErrorAzureError(err) catchInProgress := []string{ @@ -67,19 +70,18 @@ func (p *PSQLFirewallRuleClient) Ensure(ctx context.Context, obj runtime.Object, // handle the errors if helpers.ContainsString(catchInProgress, azerr.Type) { instance.Status.Message = "Postgres database exists but may not be ready" + instance.Status.Provisioning = true return false, nil } else if helpers.ContainsString(catchKnownError, azerr.Type) { - instance.Status.Provisioning = false return false, nil - } else { - - // serious error occured, end reconcilliation and mark it as failed - instance.Status.Message = fmt.Sprintf("Error occurred creating the Postgres server: %s", errhelp.StripErrorIDs(err)) - instance.Status.Provisioned = false - instance.Status.Provisioning = false - instance.Status.FailedProvisioning = true - return true, nil } + + // serious error occured, end reconcilliation and mark it as failed + instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioned = false + instance.Status.FailedProvisioning = true + return true, nil + } return false, nil diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index 94ed8b64d92..f00497a8ed1 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -79,6 +79,7 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // create the server instance.Status.Provisioning = true + instance.Status.FailedProvisioning = false _, err = p.CreateServerIfValid( ctx, instance.Name, @@ -93,6 +94,8 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts ) if err != nil { instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false + azerr := errhelp.NewAzureErrorAzureError(err) catchInProgress := []string{ @@ -109,19 +112,18 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // handle the errors if helpers.ContainsString(catchInProgress, azerr.Type) { instance.Status.Message = "Postgres server exists but may not be ready" + instance.Status.Provisioning = true return false, nil } else if helpers.ContainsString(catchKnownError, azerr.Type) { - instance.Status.Provisioning = false return false, nil - } else { - - // serious error occured, end reconcilliation and mark it as failed - instance.Status.Message = fmt.Sprintf("Error occurred creating the Postgres server: %s", errhelp.StripErrorIDs(err)) - instance.Status.Provisioned = false - instance.Status.Provisioning = false - instance.Status.FailedProvisioning = true - return true, nil } + + // serious error occured, end reconcilliation and mark it as failed + instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioned = false + instance.Status.FailedProvisioning = true + return true, nil + } return false, nil From 4e1b7940cac904fecdec613abf0de75468fb28f3 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 17:21:29 -0600 Subject: [PATCH 44/99] add tests and docs --- api/v1alpha1/mysqlserver_types.go | 6 +-- controllers/mysql_combined_test.go | 9 +++++ docs/mysql/mysql.md | 60 ++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 docs/mysql/mysql.md diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index ae04a831b13..3a3c8386512 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -61,11 +61,11 @@ func NewDefaultMySQLServer(name, resourceGroup, location string) *MySQLServer { Location: location, ResourceGroup: resourceGroup, Sku: AzureDBsSQLSku{ - Name: "B_Gen5_2", - Tier: SkuTier("Basic"), + Name: "GP_Gen5_4", + Tier: SkuTier("GeneralPurpose"), Family: "Gen5", Size: "51200", - Capacity: 2, + Capacity: 4, }, ServerVersion: ServerVersion("8.0"), SSLEnforcement: SslEnforcementEnumEnabled, diff --git a/controllers/mysql_combined_test.go b/controllers/mysql_combined_test.go index 59ad4976ec9..1ee87db8a1b 100644 --- a/controllers/mysql_combined_test.go +++ b/controllers/mysql_combined_test.go @@ -7,8 +7,10 @@ package controllers import ( "context" + "fmt" "testing" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -29,6 +31,12 @@ func TestMySQLHappyPath(t *testing.T) { RequireInstance(ctx, t, tc, mySQLServerInstance) + // Create a mySQL replica + fullSourceServerId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.DBforMySQL/servers/%s", config.Subscription(), rgName, mySQLServerName) + mySQLReplicaInstance := azurev1alpha1.NewReplicaMySQLServer(mySQLServerName, rgName, rgLocation, fullSourceServerId) + + EnsureInstance(ctx, t, tc, mySQLReplicaInstance) + mySQLDBName := GenerateTestResourceNameWithRandom("mysql-db", 10) // Create the mySQLDB object and expect the Reconcile to be created @@ -65,4 +73,5 @@ func TestMySQLHappyPath(t *testing.T) { EnsureDelete(ctx, t, tc, ruleInstance) EnsureDelete(ctx, t, tc, mySQLDBInstance) EnsureDelete(ctx, t, tc, mySQLServerInstance) + EnsureDelete(ctx, t, tc, mySQLReplicaInstance) } diff --git a/docs/mysql/mysql.md b/docs/mysql/mysql.md new file mode 100644 index 00000000000..4185aac9197 --- /dev/null +++ b/docs/mysql/mysql.md @@ -0,0 +1,60 @@ +# MySQL Operator + +## Resources Supported + +The MySQL operator suite consists of the following operators. + +1. MySQL server - Deploys an `Azure Database for MySQL server` given the Location, Resource group and other properties. This operator also helps creating read replicas for MySQL server. +2. MySQL database - Deploys a database under the given `Azure Database for MySQL server` +3. MySQL firewall rule - Deploys a firewall rule to allow access to the `Azure Database for MySQL server` from the specified IP range + +### MySQL server + +Here is a [sample YAML](/config/samples/azure_v1alpha1_mysqlserver.yaml) for the MySQL server. + +The value for kind, `MySQLServer` is the Custom Resource Definition (CRD) name. +`mysqlserver-sample` is the name of the MySQL server resource that will be created. + +The values under `spec` provide the values for the location where you want to create the server at and the Resource group in which you want to create it under. It also contains other values that are required to create the server like the `serverVersion`, `sslEnforcement` and the `sku` information. + +Along with creating the MySQL server, this operator also generates the admin username and password for the MySQL server and stores it in a kube secret or keyvault (based on what is specified) with the same name as the MySQL server. + +This secret contains the following fields. + +- `fullyqualifiedservername` : Fully qualified name of the MySQL server such as mysqlserver.mysql.database.azure.com +- `mysqlservername` : MySQL server name +- `username` : Server admin +- `password` : Password for the server admin +- `fullyqualifiedusername` : Fully qualified user name that is required by some apps such as @ + +For more information on where and how secrets are stored, look [here](/docs/secrets.md) + +#### Read Replicas in Azure Database for MySQL + +The MySQL server operator can also be used to create Read Replicas given the `sourceserverid` and the `location`. + +The replica inherits all other properties including the admin username and password from the source server. + +The operator reads the admin username and password for the source server from its secret (if available) and creates a secret with the same fields as described above for the replica. + +For more information on read replicas, refer [here](https://docs.microsoft.com/en-us/azure/mysql/concepts-read-replicas) + +### MySQL Database + +Here is a [sample YAML](/config/samples/azure_v1alpha1_mysqldatabase.yaml) for MySQL database + +Update the `resourcegroup` to where you want to provision the MySQL database. `server` is the name of the MySQL server where you want to create the database in. + +### MySQL firewall rule + +The MySQL firewall rule operator allows you to add a firewall rule to the MySQL server. + +Here is a [sample YAML](/config/samples/azure_v1alpha1_mysqlfirewallrule.yaml) for MySQL firewall rule + +The `server` indicates the MySQL server on which you want to configure the new MySQL firewall rule on and `resourceGroup` is the resource group of the MySQL server. The `startIpAddress` and `endIpAddress` indicate the IP range of sources to allow access to the server. + +*Note*: When the `startIpAddress` and `endIpAddress` are 0.0.0.0, it denotes a special case that adds a firewall rule to allow all Azure services to access the server. + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. From 5f371caff3409ce4086844b5529353fbec483c40 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 20:19:14 -0600 Subject: [PATCH 45/99] test fixes --- controllers/mysql_combined_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/controllers/mysql_combined_test.go b/controllers/mysql_combined_test.go index 1ee87db8a1b..a0cc952fbbe 100644 --- a/controllers/mysql_combined_test.go +++ b/controllers/mysql_combined_test.go @@ -7,10 +7,8 @@ package controllers import ( "context" - "fmt" "testing" - "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -25,6 +23,7 @@ func TestMySQLHappyPath(t *testing.T) { rgLocation := "eastus2" rgName := tc.resourceGroupName mySQLServerName := GenerateTestResourceNameWithRandom("mysql-srv", 10) + mySQLReplicaName := GenerateTestResourceNameWithRandom("mysql-rep", 10) // Create the mySQLServer object and expect the Reconcile to be created mySQLServerInstance := azurev1alpha1.NewDefaultMySQLServer(mySQLServerName, rgName, rgLocation) @@ -32,8 +31,7 @@ func TestMySQLHappyPath(t *testing.T) { RequireInstance(ctx, t, tc, mySQLServerInstance) // Create a mySQL replica - fullSourceServerId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.DBforMySQL/servers/%s", config.Subscription(), rgName, mySQLServerName) - mySQLReplicaInstance := azurev1alpha1.NewReplicaMySQLServer(mySQLServerName, rgName, rgLocation, fullSourceServerId) + mySQLReplicaInstance := azurev1alpha1.NewReplicaMySQLServer(mySQLReplicaName, rgName, rgLocation, mySQLServerInstance.Status.ResourceId) EnsureInstance(ctx, t, tc, mySQLReplicaInstance) From 37b29e25cbd17dba84f2b4519d4bc4107d0790d9 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 20:39:30 -0600 Subject: [PATCH 46/99] use suffix based on cloud env --- .../azuresqlfailovergroup/azuresqlfailovergroup.go | 5 +++-- .../azuresqlserver/azuresqlserver_reconcile.go | 3 ++- .../azuresql/azuresqluser/azuresqluser.go | 5 +++-- .../azuresql/azuresqluser/azuresqluser_reconcile.go | 5 +++-- pkg/resourcemanager/psql/server/server.go | 6 +++--- pkg/resourcemanager/psql/server/server_reconcile.go | 12 +++++++----- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go index ba36e73e454..45e4cf92fdd 100644 --- a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go +++ b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go @@ -9,6 +9,7 @@ import ( azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -167,9 +168,9 @@ func (f *AzureSqlFailoverGroupManager) GetOrPrepareSecret(ctx context.Context, i } secret["azureSqlPrimaryServer"] = []byte(azuresqlprimaryserver) - secret["readWriteListenerEndpoint"] = []byte(failovergroupname + ".database.windows.net") + secret["readWriteListenerEndpoint"] = []byte(failovergroupname + "." + config.Environment().SQLDatabaseDNSSuffix) secret["azureSqlSecondaryServer"] = []byte(azuresqlsecondaryserver) - secret["readOnlyListenerEndpoint"] = []byte(failovergroupname + ".secondary.database.windows.net") + secret["readOnlyListenerEndpoint"] = []byte(failovergroupname + ".secondary." + config.Environment().SQLDatabaseDNSSuffix) return secret, nil } diff --git a/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go index ed0bfaebbfb..8e76fb24ab9 100644 --- a/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/pollclient" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest/to" @@ -335,7 +336,7 @@ func NewSecret(serverName string) (map[string][]byte, error) { secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, serverName)) secret["password"] = []byte(randomPassword) secret["azureSqlServerName"] = []byte(serverName) - secret["fullyQualifiedServerName"] = []byte(serverName + ".database.windows.net") + secret["fullyQualifiedServerName"] = []byte(serverName + "." + config.Environment().SQLDatabaseDNSSuffix) return secret, nil } diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go index 007d93d14bc..14d71ff6890 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go @@ -13,6 +13,7 @@ import ( azuresql "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/2015-05-01-preview/sql" "github.com/Azure/azure-service-operator/pkg/helpers" azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -65,7 +66,7 @@ func (s *AzureSqlUserManager) GetDB(ctx context.Context, resourceGroupName strin // ConnectToSqlDb connects to the SQL db using the given credentials func (s *AzureSqlUserManager) ConnectToSqlDb(ctx context.Context, drivername string, server string, database string, port int, user string, password string) (*sql.DB, error) { - fullServerAddress := fmt.Sprintf("%s.database.windows.net", server) + fullServerAddress := fmt.Sprintf("%s."+config.Environment().SQLDatabaseDNSSuffix, server) connString := fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;database=%s;Persist Security Info=False;Pooling=False;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30", fullServerAddress, user, password, port, database) db, err := sql.Open(drivername, connString) @@ -231,7 +232,7 @@ func (s *AzureSqlUserManager) GetOrPrepareSecret(ctx context.Context, instance * "password": []byte(pw), "azureSqlServerNamespace": []byte(instance.Namespace), "azureSqlServerName": []byte(instance.Spec.Server), - "fullyQualifiedServerName": []byte(instance.Spec.Server + ".database.windows.net"), + "fullyQualifiedServerName": []byte(instance.Spec.Server + "." + config.Environment().SQLDatabaseDNSSuffix), "azureSqlDatabaseName": []byte(instance.Spec.DbName), } } diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 235838a30a0..28036683751 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Azure/azure-service-operator/pkg/helpers" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -178,7 +179,7 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op case "jdbc": formattedSecrets["jdbc"] = []byte(fmt.Sprintf( - "jdbc:sqlserver://%v:1433;database=%v;user=%v@%v;password=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;", + "jdbc:sqlserver://%v:1433;database=%v;user=%v@%v;password=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*."+config.Environment().SQLDatabaseDNSSuffix+";loginTimeout=30;", string(DBSecret["fullyQualifiedServerName"]), instance.Spec.DbName, user, @@ -187,7 +188,7 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op )) case "jdbc-urlonly": formattedSecrets["jdbc-urlonly"] = []byte(fmt.Sprintf( - "jdbc:sqlserver://%v:1433;database=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;", + "jdbc:sqlserver://%v:1433;database=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*."+config.Environment().SQLDatabaseDNSSuffix+";loginTimeout=30;", string(DBSecret["fullyQualifiedServerName"]), instance.Spec.DbName, )) diff --git a/pkg/resourcemanager/psql/server/server.go b/pkg/resourcemanager/psql/server/server.go index 965fd7c8454..71fa650f10e 100644 --- a/pkg/resourcemanager/psql/server/server.go +++ b/pkg/resourcemanager/psql/server/server.go @@ -137,12 +137,14 @@ func (p *PSQLServerClient) GetServer(ctx context.Context, resourcegroup string, return client.Get(ctx, resourcegroup, servername) } -func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer) error { +func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer, fullservername string) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, } + data["fullyQualifiedServerName"] = []byte(fullservername) + err := p.SecretClient.Upsert(ctx, key, data, @@ -175,8 +177,6 @@ func (p *PSQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *azu secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) secret["password"] = []byte(randomPassword) secret["postgreSqlServerName"] = []byte(name) - // TODO: The below may not be right for non Azure public cloud. - secret["fullyQualifiedServerName"] = []byte(name + ".postgres.database.azure.com") return secret, nil } diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index f00497a8ed1..cb39ca8d3df 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -39,11 +39,6 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts if err != nil { return false, err } - // Update secret - err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) - if err != nil { - return false, err - } // if an error occurs thats ok as it means that it doesn't exist yet getServer, err := p.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name) @@ -52,6 +47,13 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // succeeded! so end reconcilliation successfully if getServer.UserVisibleState == "Ready" { + + // Update secret with the fully qualified server name + err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) + if err != nil { + return false, err + } + instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *getServer.ID instance.Status.Provisioned = true From f187f6955716331753a6a7eb727626ef6ec5cf74 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 21:03:52 -0600 Subject: [PATCH 47/99] fix build error --- pkg/resourcemanager/psql/server/server_manager.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/psql/server/server_manager.go b/pkg/resourcemanager/psql/server/server_manager.go index 3fc3ad42f8c..73948d3af05 100644 --- a/pkg/resourcemanager/psql/server/server_manager.go +++ b/pkg/resourcemanager/psql/server/server_manager.go @@ -39,7 +39,8 @@ type PostgreSQLServerManager interface { AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, - instance *azurev1alpha1.PostgreSQLServer) error + instance *azurev1alpha1.PostgreSQLServer, + fullservername string) error GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.PostgreSQLServer) (map[string][]byte, error) From 37a47e2c29236ba61b33344f68eb98931dedbf4b Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 21:05:30 -0600 Subject: [PATCH 48/99] fixed edge casee --- pkg/resourcemanager/mysql/server/reconcile.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index e0f07a5de6f..7c429dfc2a0 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -64,11 +64,6 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts if err == nil { instance.Status.State = string(server.UserVisibleState) if server.UserVisibleState == mysql.ServerStateReady { - instance.Status.Provisioned = true - instance.Status.Provisioning = false - instance.Status.Message = resourcemanager.SuccessMsg - instance.Status.ResourceId = *server.ID - instance.Status.State = string(server.UserVisibleState) // Update secret - we do this on success as we need the FQ name of the server err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) @@ -76,6 +71,11 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } + instance.Status.Provisioned = true + instance.Status.Provisioning = false + instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.ResourceId = *server.ID + instance.Status.State = string(server.UserVisibleState) return true, nil } return false, nil From 45c69890e48c002e796cc47f0e579c221340af1f Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Wed, 15 Apr 2020 11:06:47 +0800 Subject: [PATCH 49/99] #issue858#changePerComments --- controllers/suite_test.go | 2 +- pkg/resourcemanager/mock/storages/storageaccount.go | 2 +- .../storages/storageaccount/storageaccount.go | 2 +- .../storages/storageaccount/storageaccount_manager.go | 2 +- .../storages/storageaccount/storageaccount_reconcile.go | 7 +++++++ 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 10524dd8241..f3fb0123e48 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -745,7 +745,7 @@ func setup() error { log.Println("Creating SA:", storageAccountName) // Create the Storage Account and Container - _, _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageSku{ + _, _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageAccountSku{ Name: "Standard_LRS", }, "Storage", map[string]*string{}, "", nil, nil, nil) diff --git a/pkg/resourcemanager/mock/storages/storageaccount.go b/pkg/resourcemanager/mock/storages/storageaccount.go index d3962b954f9..48a939f73a4 100644 --- a/pkg/resourcemanager/mock/storages/storageaccount.go +++ b/pkg/resourcemanager/mock/storages/storageaccount.go @@ -49,7 +49,7 @@ func (manager *mockStorageManager) CreateStorage(ctx context.Context, groupName sku azurev1alpha1.StorageAccountSku, kind azurev1alpha1.StorageAccountKind, tags map[string]*string, - accessTier azurev1alpha1.StorageAccessTier, + accessTier azurev1alpha1.StorageAccountAccessTier, enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *azurev1alpha1.NetworkRuleSet) (result storage.Account, err error) { s := storageResource{ resourceGroupName: groupName, diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount.go b/pkg/resourcemanager/storages/storageaccount/storageaccount.go index 3b80ca5d7dc..e2e75b353d6 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount.go @@ -89,7 +89,7 @@ func (_ *azureStorageManager) CreateStorage(ctx context.Context, sku azurev1alpha1.StorageAccountSku, kind azurev1alpha1.StorageAccountKind, tags map[string]*string, - accessTier azurev1alpha1.StorageAccessTier, + accessTier azurev1alpha1.StorageAccountAccessTier, enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (pollingURL string, result storage.Account, err error) { storagesClient := getStoragesClient() diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go index a1d19888d04..5ef05d8e7d2 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_manager.go @@ -25,7 +25,7 @@ type StorageManager interface { sku azurev1alpha1.StorageAccountSku, kind azurev1alpha1.StorageAccountKind, tags map[string]*string, - accessTier azurev1alpha1.StorageAccessTier, + accessTier azurev1alpha1.StorageAccountAccessTier, enableHTTPsTrafficOnly *bool, dataLakeEnabled *bool, networkRule *storage.NetworkRuleSet) (pollingURL string, result storage.Account, err error) // Get gets the description of the specified storage account. diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go index 445d967958e..45efda9e48c 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go @@ -58,7 +58,14 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o if pollURL != "" { pClient := pollclient.NewPollClient() res, err := pClient.Get(ctx, pollURL) + azerr := errhelp.NewAzureErrorAzureError(err) if err != nil { + if azerr.Type == errhelp.NetworkAclsValidationFailure { + instance.Status.Message = "Unable to provision Azure Storage Account due to error: " + errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false + instance.Status.Provisioned = false + return true, nil + } return false, err } From 03459caf249b1bbebb3cff1721622e5795ae6369 Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Wed, 15 Apr 2020 11:45:56 +0800 Subject: [PATCH 50/99] #updateyaml --- config/samples/azure_v1alpha1_storageaccount.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/samples/azure_v1alpha1_storageaccount.yaml b/config/samples/azure_v1alpha1_storageaccount.yaml index 9be18126eda..77deb93e411 100644 --- a/config/samples/azure_v1alpha1_storageaccount.yaml +++ b/config/samples/azure_v1alpha1_storageaccount.yaml @@ -1,7 +1,7 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: StorageAccount metadata: - name: storagesample + name: storagesample777 spec: location: westus resourceGroup: resourcegroup-azure-operators From f050ac36a8460b04039a48f852bc9d5c6a78180e Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Wed, 15 Apr 2020 11:46:57 +0800 Subject: [PATCH 51/99] #updateyamle --- config/samples/azure_v1alpha1_storageaccount.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/samples/azure_v1alpha1_storageaccount.yaml b/config/samples/azure_v1alpha1_storageaccount.yaml index 77deb93e411..39b590fe276 100644 --- a/config/samples/azure_v1alpha1_storageaccount.yaml +++ b/config/samples/azure_v1alpha1_storageaccount.yaml @@ -1,7 +1,7 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: StorageAccount metadata: - name: storagesample777 + name: storageaccountsample777 spec: location: westus resourceGroup: resourcegroup-azure-operators From 3727b0a2498ae289b4532963152dcb57586dcfcc Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Wed, 15 Apr 2020 17:39:04 +0800 Subject: [PATCH 52/99] issue#933#resolvemergeissue --- .../storages/blobcontainer/blob_container_reconcile.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go index bf3d5b82b9a..ea7c46f44a6 100644 --- a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go +++ b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go @@ -125,6 +125,13 @@ func (bc *AzureBlobContainerManager) GetParents(obj runtime.Object) ([]resourcem } return []resourcemanager.KubeParent{ + { + Key: types.NamespacedName{ + Name: instance.Spec.AccountName, + Namespace: instance.Namespace, + }, + Target: &azurev1alpha1.StorageAccount{}, + }, { Key: types.NamespacedName{ Name: instance.Spec.ResourceGroup, @@ -133,6 +140,7 @@ func (bc *AzureBlobContainerManager) GetParents(obj runtime.Object) ([]resourcem Target: &azurev1alpha1.ResourceGroup{}, }, }, nil + } func (bc *AzureBlobContainerManager) GetStatus(obj runtime.Object) (*azurev1alpha1.ASOStatus, error) { From 03be5198c7c3692975fb4fd46b3ccb5157d6226c Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 09:51:26 -0600 Subject: [PATCH 53/99] fix error return --- pkg/resourcemanager/mysql/server/reconcile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 7c429dfc2a0..100cfeec7a3 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -68,7 +68,8 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // Update secret - we do this on success as we need the FQ name of the server err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) if err != nil { - return false, err + instance.Status.Message = "Could not save secrets" + return true, nil } instance.Status.Provisioned = true From aaea1d9571bc04816c53ea2b5eb2be6d25e74e20 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 09:52:30 -0600 Subject: [PATCH 54/99] fix error return --- pkg/resourcemanager/psql/server/server_reconcile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index cb39ca8d3df..bff682f536f 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -51,7 +51,8 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // Update secret with the fully qualified server name err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) if err != nil { - return false, err + instance.Status.Message = "Could not save secrets" + return true, nil } instance.Status.Message = resourcemanager.SuccessMsg From ba70604777389a66bf1ceb48d1c4e75114f7c7e2 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Wed, 15 Apr 2020 11:17:39 -0600 Subject: [PATCH 55/99] simplifying ensure --- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 2 +- .../cosmosdbs/cosmosdb_reconcile.go | 208 ++++++++---------- 2 files changed, 90 insertions(+), 120 deletions(-) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index a79573e4976..ad83a5577a7 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -30,9 +30,9 @@ func getCosmosDBClient() (documentdb.DatabaseAccountsClient, error) { cosmosDBClient = documentdb.DatabaseAccountsClient{} } else { cosmosDBClient.Authorizer = a - cosmosDBClient.AddToUserAgent(config.UserAgent()) } + err = cosmosDBClient.AddToUserAgent(config.UserAgent()) return cosmosDBClient, err } diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index e746619f4ff..24ff5838558 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -33,72 +33,59 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o } hash := helpers.Hash256(instance.Spec) - if instance.Status.SpecHash != hash { - // need to push a create or update - instance.Status.SpecHash = hash - } else if instance.Status.Provisioned { - // provisioned and no changes needed + + if instance.Status.SpecHash == hash && instance.Status.Provisioned { instance.Status.RequestedAt = nil return true, nil - } else if instance.Status.Provisioning { - // get the instance and update status - db, err := m.GetCosmosDB(ctx, instance.Spec.ResourceGroup, instance.Name) - azerr := errhelp.NewAzureErrorAzureError(err) - if azerr == nil { - instance.Status.ResourceId = *db.ID - instance.Status.State = *db.ProvisioningState - - if instance.Status.State == "Creating" { - instance.Status.Message = "Waiting for resource to finish creation" - return false, nil - } - - if instance.Status.State == "Succeeded" { - // provisioning is complete, update the secrets - if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil { - instance.Status.Message = err.Error() - return false, err - } + } + instance.Status.Provisioned = false - instance.Status.Message = resourcemanager.SuccessMsg - instance.Status.Provisioning = false - instance.Status.Provisioned = true - return true, nil - } + // get the instance and update status + db, err := m.GetCosmosDB(ctx, instance.Spec.ResourceGroup, instance.Name) + if err != nil { + azerr := errhelp.NewAzureErrorAzureError(err) - if instance.Status.State == "Failed" { - instance.Status.Message = "Failed to provision CosmosDB" - instance.Status.Provisioning = false - return true, nil - } - } else if azerr.Type == errhelp.ResourceGroupNotFoundErrorCode { + switch azerr.Type { + case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: instance.Status.Provisioning = false - instance.Status.Message = fmt.Sprintf("Waiting for resource group '%s' to be available", instance.Spec.ResourceGroup) + instance.Status.Message = azerr.Reason instance.Status.State = "Waiting" return false, nil - } else if azerr.Type == errhelp.ResourceNotFound { - exists, azerr := m.CheckNameExistsCosmosDB(ctx, instance.Name) - if azerr != nil { - instance.Status.Provisioning = false - instance.Status.Message = "Unexpected error occurred during resource request" - instance.Status.State = "Failed" - return false, err - } else if exists { - // get request returned resource not found and the name already exists - // so it must exist in a different resource group, user must fix it - instance.Status.Provisioning = false - instance.Status.Message = "CosmosDB name already exists" - instance.Status.State = "Failed" - return true, nil - } - } else { - instance.Status.Provisioning = false - instance.Status.Message = azerr.Error() - return false, azerr.Original + case errhelp.ResourceNotFound: + //NO-OP, try to create + default: + instance.Status.Message = fmt.Sprintf("Unhandled error after Get %v", azerr.Reason) } + + } else { + instance.Status.ResourceId = *db.ID + instance.Status.State = *db.ProvisioningState } - instance.Status.Provisioning = true + if instance.Status.State == "Creating" { + // avoid multiple CreateOrUpdate requests while resource is already creating + return false, nil + } + + if instance.Status.State == "Succeeded" && instance.Status.SpecHash == hash { + // provisioning is complete, update the secrets + if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil { + instance.Status.Message = err.Error() + return false, err + } + + instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.Provisioning = false + instance.Status.Provisioned = true + return true, nil + } + + if instance.Status.State == "Failed" { + instance.Status.Message = "Failed to provision CosmosDB" + instance.Status.Provisioning = false + instance.Status.Provisioned = false + return true, nil + } tags := helpers.LabelsToTags(instance.GetLabels()) accountName := instance.ObjectMeta.Name @@ -107,30 +94,44 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o kind := instance.Spec.Kind dbType := instance.Spec.Properties.DatabaseAccountOfferType - db, err := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) + db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) + if err != nil { + azerr := errhelp.NewAzureErrorAzureError(err) - // everything is in a created/updated state - if err == nil { - instance.Status.Provisioned = true - instance.Status.Provisioning = false - instance.Status.Message = resourcemanager.SuccessMsg - instance.Status.State = "Succeeded" - instance.Status.ResourceId = *db.ID - return true, nil + switch azerr.Type { + case errhelp.AsyncOpIncompleteError: + instance.Status.State = "Creating" + instance.Status.Message = "Resource request successfully submitted to Azure" + return false, nil + case errhelp.InvalidResourceLocation, errhelp.LocationNotAvailableForResourceType: + instance.Status.Provisioning = false + instance.Status.Message = azerr.Reason + return true, nil + case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: + instance.Status.Provisioning = false + instance.Status.Message = azerr.Reason + return false, nil + case errhelp.NotFoundErrorCode: + if nameExists, err := m.CheckNameExistsCosmosDB(ctx, accountName); err != nil { + instance.Status.Message = err.Error() + return false, err + } else if nameExists { + instance.Status.Provisioning = false + instance.Status.Message = "CosmosDB Account name already exists" + return true, nil + } + default: + instance.Status.Message = azerr.Reason + return false, nil + } } - switch azerr := errhelp.NewAzureErrorAzureError(err); azerr.Type { - - case errhelp.AsyncOpIncompleteError: - instance.Status.Message = "Resource request successfully submitted to Azure" - instance.Status.State = "Creating" - - case errhelp.InvalidResourceLocation: - instance.Status.Provisioning = false - instance.Status.Message = azerr.Reason - return true, nil - - } + instance.Status.SpecHash = hash + instance.Status.ResourceId = *db.ID + instance.Status.State = *db.ProvisioningState + instance.Status.Provisioned = true + instance.Status.Provisioning = false + instance.Status.Message = resourcemanager.SuccessMsg return false, nil } @@ -150,51 +151,17 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o return false, err } - accountName := instance.ObjectMeta.Name - groupName := instance.Spec.ResourceGroup - // if the resource is in a failed state it was never created or could never be verified // so we skip attempting to delete the resrouce from Azure if instance.Status.FailedProvisioning { return false, nil } - notFoundErrors := []string{ - errhelp.NotFoundErrorCode, // happens on first request after deletion succeeds - errhelp.ResourceNotFound, // happens on subsequent requests after deletion succeeds - errhelp.ResourceGroupNotFoundErrorCode, // database doesn't exist in this resource group but the name exists globally - } - - // fetch the latest to inspect provisioning state - cosmosDB, err := m.GetCosmosDB(ctx, groupName, accountName) - if err != nil { - azerr := errhelp.NewAzureErrorAzureError(err) - - // deletion finished - if helpers.ContainsString(notFoundErrors, azerr.Type) { - return false, nil - } - - //TODO: are there other errors that need handling here? - instance.Status.Message = azerr.Error() - return true, azerr.Original - } - - instance.Status.State = *cosmosDB.ProvisioningState - - // already deleting the resource, try again later - if instance.Status.State == "Deleting" { - return true, nil - } - - // create must succeed before delete succeeds - if instance.Status.State == "Creating" { - return true, nil - } + groupName := instance.Spec.ResourceGroup + accountName := instance.ObjectMeta.Name // try to delete the cosmosdb instance & secrets _, err = m.DeleteCosmosDB(ctx, groupName, accountName) - m.deleteAccountKeysSecret(ctx, instance) if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) @@ -204,18 +171,21 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o return true, nil } - // already deleted + notFoundErrors := []string{ + errhelp.NotFoundErrorCode, // happens on first request after deletion succeeds + errhelp.ResourceNotFound, // happens on subsequent requests after deletion succeeds + errhelp.ResourceGroupNotFoundErrorCode, // database doesn't exist in this resource group but the name exists globally + } if helpers.ContainsString(notFoundErrors, azerr.Type) { - return false, nil + return false, m.deleteAccountKeysSecret(ctx, instance) } // unhandled error - instance.Status.Message = azerr.Error() - return false, azerr.Original + instance.Status.Message = azerr.Reason + return false, err } - // second delete calls succeed immediately - return false, nil + return false, m.deleteAccountKeysSecret(ctx, instance) } // GetParents returns the parents of cosmosdb From 7a53105f6bfcae4e89e0cec845c2a8bd1da6e1e7 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Wed, 15 Apr 2020 11:24:19 -0600 Subject: [PATCH 56/99] using Error() instead of Reason --- pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 24ff5838558..8125fd8d079 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -48,13 +48,13 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o switch azerr.Type { case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: instance.Status.Provisioning = false - instance.Status.Message = azerr.Reason + instance.Status.Message = azerr.Error() instance.Status.State = "Waiting" return false, nil case errhelp.ResourceNotFound: //NO-OP, try to create default: - instance.Status.Message = fmt.Sprintf("Unhandled error after Get %v", azerr.Reason) + instance.Status.Message = fmt.Sprintf("Unhandled error after Get %v", azerr.Error()) } } else { @@ -105,11 +105,11 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return false, nil case errhelp.InvalidResourceLocation, errhelp.LocationNotAvailableForResourceType: instance.Status.Provisioning = false - instance.Status.Message = azerr.Reason + instance.Status.Message = azerr.Error() return true, nil case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: instance.Status.Provisioning = false - instance.Status.Message = azerr.Reason + instance.Status.Message = azerr.Error() return false, nil case errhelp.NotFoundErrorCode: if nameExists, err := m.CheckNameExistsCosmosDB(ctx, accountName); err != nil { @@ -121,7 +121,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return true, nil } default: - instance.Status.Message = azerr.Reason + instance.Status.Message = azerr.Error() return false, nil } } @@ -181,7 +181,7 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o } // unhandled error - instance.Status.Message = azerr.Reason + instance.Status.Message = azerr.Error() return false, err } From 9bc040750e047d36bf7165de022e0cd6606ff13b Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 11:38:52 -0600 Subject: [PATCH 57/99] fixed secret update --- pkg/resourcemanager/mysql/server/reconcile.go | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 100cfeec7a3..7b36c556346 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -55,6 +55,11 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } + err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) + if err != nil { + return false, err + } + // convert kube labels to expected tag format labels := helpers.LabelsToTags(instance.GetLabels()) @@ -65,12 +70,8 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.State = string(server.UserVisibleState) if server.UserVisibleState == mysql.ServerStateReady { - // Update secret - we do this on success as we need the FQ name of the server - err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) - if err != nil { - instance.Status.Message = "Could not save secrets" - return true, nil - } + // Update secret with FQ name of the server. We ignore the error. + m.UpdateServerNameInSecret(ctx, instance.Name, secret, *server.FullyQualifiedDomainName, instance) instance.Status.Provisioned = true instance.Status.Provisioning = false @@ -217,13 +218,32 @@ func (m *MySQLServerClient) convert(obj runtime.Object) (*v1alpha1.MySQLServer, } // AddServerCredsToSecrets saves the server's admin credentials in the secret store -func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer, fullservername string) error { +func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer) error { + key := types.NamespacedName{ + Name: secretName, + Namespace: instance.Namespace, + } + + err := m.SecretClient.Upsert(ctx, + key, + data, + secrets.WithOwner(instance), + secrets.WithScheme(m.Scheme), + ) + if err != nil { + return err + } + + return nil +} + +// UpdateSecretWithFullServerName updates the secret with the fully qualified server name +func (m *MySQLServerClient) UpdateServerNameInSecret(ctx context.Context, secretName string, data map[string][]byte, fullservername string, instance *azurev1alpha1.MySQLServer) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, } - // Update fullyQualifiedServerName from the created server data["fullyQualifiedServerName"] = []byte(fullservername) err := m.SecretClient.Upsert(ctx, From 7e53ee4f0398e2d10843a8afa583b4712bb47b8c Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 11:44:11 -0600 Subject: [PATCH 58/99] updated secret updation --- pkg/resourcemanager/psql/server/server.go | 23 ++++++++++++++++++- .../psql/server/server_reconcile.go | 15 +++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/pkg/resourcemanager/psql/server/server.go b/pkg/resourcemanager/psql/server/server.go index 71fa650f10e..a456a9cb290 100644 --- a/pkg/resourcemanager/psql/server/server.go +++ b/pkg/resourcemanager/psql/server/server.go @@ -137,7 +137,28 @@ func (p *PSQLServerClient) GetServer(ctx context.Context, resourcegroup string, return client.Get(ctx, resourcegroup, servername) } -func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer, fullservername string) error { +func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer) error { + key := types.NamespacedName{ + Name: secretName, + Namespace: instance.Namespace, + } + + data["fullyQualifiedServerName"] = []byte(fullservername) + + err := p.SecretClient.Upsert(ctx, + key, + data, + secrets.WithOwner(instance), + secrets.WithScheme(p.Scheme), + ) + if err != nil { + return err + } + + return nil +} + +func (p *PSQLServerClient) UpdateSecretWithFullServerName(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer, fullservername string) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index bff682f536f..b879adac85a 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -40,6 +40,12 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } + // Update secret with the fully qualified server name + err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) + if err != nil { + return false, err + } + // if an error occurs thats ok as it means that it doesn't exist yet getServer, err := p.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name) if err == nil { @@ -48,17 +54,14 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // succeeded! so end reconcilliation successfully if getServer.UserVisibleState == "Ready" { - // Update secret with the fully qualified server name - err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) - if err != nil { - instance.Status.Message = "Could not save secrets" - return true, nil - } + // Update the secret with fully qualified server name. Ignore error as we have the admin creds which is critical. + p.UpdateSecretWithFullServerName(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *getServer.ID instance.Status.Provisioned = true instance.Status.Provisioning = false + instance.Status.State = string(getServer.UserVisibleState) return true, nil } From fb8ce7b2977a30f220c8e0d85ce42c9293984369 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 11:49:24 -0600 Subject: [PATCH 59/99] updated errors --- pkg/resourcemanager/psql/server/server.go | 2 -- pkg/resourcemanager/psql/server/server_manager.go | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/resourcemanager/psql/server/server.go b/pkg/resourcemanager/psql/server/server.go index a456a9cb290..21c2b1c4a73 100644 --- a/pkg/resourcemanager/psql/server/server.go +++ b/pkg/resourcemanager/psql/server/server.go @@ -143,8 +143,6 @@ func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretNa Namespace: instance.Namespace, } - data["fullyQualifiedServerName"] = []byte(fullservername) - err := p.SecretClient.Upsert(ctx, key, data, diff --git a/pkg/resourcemanager/psql/server/server_manager.go b/pkg/resourcemanager/psql/server/server_manager.go index 73948d3af05..3fc3ad42f8c 100644 --- a/pkg/resourcemanager/psql/server/server_manager.go +++ b/pkg/resourcemanager/psql/server/server_manager.go @@ -39,8 +39,7 @@ type PostgreSQLServerManager interface { AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, - instance *azurev1alpha1.PostgreSQLServer, - fullservername string) error + instance *azurev1alpha1.PostgreSQLServer) error GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.PostgreSQLServer) (map[string][]byte, error) From 954b323bfc75f3ba5d57ffddfca82a0f45b69dbe Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Wed, 15 Apr 2020 11:18:05 -0700 Subject: [PATCH 60/99] Update docs/apimgmt/apimgmt.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/apimgmt/apimgmt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index 8fa95af31a7..bea831dcabe 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -31,7 +31,7 @@ The API Management API Operator manages your API Management service, and deploys The spec consists of the following fields: -* `apiService` The name of the API Management service to managed +* `apiService` The name of the API Management service to manage * `apiId` Specify an ID for the API * `properties` * `isCurrent` Indicate if this API revision is the current revision @@ -45,4 +45,4 @@ The spec consists of the following fields: ## Deploy, view and delete resources -You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. \ No newline at end of file +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. From 4e003d6d9af3400269125f5400aba859b6c6a4a2 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Wed, 15 Apr 2020 11:18:24 -0700 Subject: [PATCH 61/99] Update docs/apimgmt/apimgmt.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/apimgmt/apimgmt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index bea831dcabe..86867cb2ed4 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -8,7 +8,7 @@ Learn more about Azure API Management [here](https://docs.microsoft.com/en-us/az ### API Management Service -The API Management Service deploys an API Management instance into a specified resource group at the specified location. +The API Management Service deploys an API Management instance into a specified resource group at the specified location. It also provides the option to link to an Application Insights instance for logging, and to place the API Management instance in a specified Virtual Network. The spec consists of the following fields: From d3c57ddae43a41103ec79dc4de0f716b8e47e673 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Wed, 15 Apr 2020 11:18:40 -0700 Subject: [PATCH 62/99] Update docs/apimgmt/apimgmt.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/apimgmt/apimgmt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index 86867cb2ed4..255af62a885 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -27,7 +27,7 @@ The spec consists of the following fields: ### API Management API -The API Management API Operator manages your API Management service, and deploys an instance into a specified resource group at the specified location. +The API Management API Operator creates an API with the specified properties in the specified API Management service. The spec consists of the following fields: From 3eac7127e5702ded666ad3a66ef2e72f29610845 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Wed, 15 Apr 2020 12:36:42 -0700 Subject: [PATCH 63/99] removed pkg/util (#943) Co-authored-by: Erin Corson --- pkg/util/util.go | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 pkg/util/util.go diff --git a/pkg/util/util.go b/pkg/util/util.go deleted file mode 100644 index e85ed10f8e8..00000000000 --- a/pkg/util/util.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package util - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" -) - -// PrintAndLog writes to stdout and to a logger. -func PrintAndLog(message string) { - log.Println(message) - fmt.Println(message) -} - -func Contains(array []string, element string) bool { - for _, e := range array { - if e == element { - return true - } - } - return false -} - -// ReadJSON reads a json file, and unmashals it. -// Very useful for template deployments. -func ReadJSON(path string) (*map[string]interface{}, error) { - data, err := ioutil.ReadFile(path) - if err != nil { - log.Fatalf("failed to read template file: %v\n", err) - } - contents := make(map[string]interface{}) - json.Unmarshal(data, &contents) - return &contents, nil -} From c843020fa8491b94416857b9b3f93acbcfe9dcfd Mon Sep 17 00:00:00 2001 From: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Date: Wed, 15 Apr 2020 14:22:05 -0700 Subject: [PATCH 64/99] Bug: SQL User error on GetDB in transitional state (#928) * refactor * small update: * parent error code * additional error trap for db * janani comment * janani vnet fix Co-authored-by: William Mortl Co-authored-by: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- .../azuresqldb/azuresqldb_reconcile.go | 30 +++++++++++-------- .../azuresqlfailovergroup_reconcile.go | 9 ++++++ .../azuresqlfirewallrule_reconcile.go | 9 ++++++ .../azuresqluser/azuresqluser_reconcile.go | 28 +++++++++++++---- .../azuresqlvnetrule_reconcile.go | 17 +++++++++++ 5 files changed, 75 insertions(+), 18 deletions(-) diff --git a/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go b/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go index db505b1806b..e6947b01aa1 100644 --- a/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go @@ -6,6 +6,7 @@ package azuresqldb import ( "context" "fmt" + "strings" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" @@ -65,18 +66,16 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *dbGet.ID return true, nil - } else { - azerr := errhelp.NewAzureErrorAzureError(err) - ignore := []string{ - errhelp.NotFoundErrorCode, - errhelp.ResourceNotFound, - errhelp.ResourceGroupNotFoundErrorCode, - } - if !helpers.ContainsString(ignore, azerr.Type) { - instance.Status.Message = err.Error() - instance.Status.Provisioning = false - return false, fmt.Errorf("AzureSqlDb GetDB error %v", err) - } + } + instance.Status.Message = fmt.Sprintf("AzureSqlDb Get error %s", err.Error()) + azerr := errhelp.NewAzureErrorAzureError(err) + requeuErrors := []string{ + errhelp.ParentNotFoundErrorCode, + errhelp.ResourceGroupNotFoundErrorCode, + } + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil } resp, err := db.CreateOrUpdateDB(ctx, groupName, location, server, labels, azureSQLDatabaseProperties) @@ -100,6 +99,13 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt return false, nil } + // if the database is busy, requeue + errorString := err.Error() + if strings.Contains(errorString, "Try again later") { + instance.Status.Provisioning = false + return false, nil + } + // assertion that a 404 error implies that the Azure SQL server hasn't been provisioned yet if resp != nil && resp.StatusCode == 404 { instance.Status.Message = fmt.Sprintf("Waiting for SQL Server %s to provision", server) diff --git a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go index 72f57c28171..4a32fe22bff 100644 --- a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go @@ -58,6 +58,15 @@ func (fg *AzureSqlFailoverGroupManager) Ensure(ctx context.Context, obj runtime. return true, nil } instance.Status.Message = fmt.Sprintf("AzureSqlFailoverGroup Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } _, err = fg.CreateOrUpdateFailoverGroup(ctx, groupName, serverName, failoverGroupName, sqlFailoverGroupProperties) if err != nil { diff --git a/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go index f5503939ea8..e6b1002fb9b 100644 --- a/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go @@ -37,6 +37,15 @@ func (fw *AzureSqlFirewallRuleManager) Ensure(ctx context.Context, obj runtime.O return true, nil } instance.Status.Message = fmt.Sprintf("AzureSqlFirewallRule Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } _, err = fw.CreateOrUpdateSQLFirewallRule(ctx, groupName, server, ruleName, startIP, endIP) if err != nil { diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 235838a30a0..649752a4d96 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -77,28 +77,44 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op _, err = s.GetDB(ctx, instance.Spec.ResourceGroup, instance.Spec.Server, instance.Spec.DbName) if err != nil { - instance.Status.Message = err.Error() + instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false - catch := []string{ + requeuErrors := []string{ errhelp.ResourceNotFound, errhelp.ParentNotFoundErrorCode, errhelp.ResourceGroupNotFoundErrorCode, } azerr := errhelp.NewAzureErrorAzureError(err) - if helpers.ContainsString(catch, azerr.Type) { + if helpers.ContainsString(requeuErrors, azerr.Type) { return false, nil } - return false, err + + // if the database is busy, requeue + errorString := err.Error() + if strings.Contains(errorString, "Please retry the connection later") { + return false, nil + } + + // if this is an unmarshall error - igmore and continue, otherwise report error and requeue + if !strings.Contains(errorString, "cannot unmarshal array into Go struct field serviceError2.details") { + return false, err + } } db, err := s.ConnectToSqlDb(ctx, DriverName, instance.Spec.Server, instance.Spec.DbName, SqlServerPort, adminUser, adminPassword) if err != nil { instance.Status.Message = errhelp.StripErrorIDs(err) + instance.Status.Provisioning = false // catch firewall issue - keep cycling until it clears up if strings.Contains(err.Error(), "create a firewall rule for this IP address") { - instance.Status.Provisioned = false - instance.Status.Provisioning = false + return false, nil + } + + // if the database is busy, requeue + errorString := err.Error() + if strings.Contains(errorString, "Please retry the connection later") { return false, nil } diff --git a/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go index dc64bfb88df..2d43032eac8 100644 --- a/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go @@ -6,6 +6,7 @@ package azuresqlvnetrule import ( "context" "fmt" + "strings" "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/2015-05-01-preview/sql" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -43,6 +44,15 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec return false, nil } instance.Status.Message = fmt.Sprintf("AzureSqlVNetRule Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } instance.Status.Provisioning = true _, err = vr.CreateOrUpdateSQLVNetRule(ctx, groupName, server, ruleName, virtualNetworkRG, virtualnetworkname, subnetName, ignoreendpoint) @@ -66,6 +76,13 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec return false, nil } + // this happens when we try to create the VNet rule and the server doesnt exist yet + errorString := err.Error() + if strings.Contains(errorString, "does not have the server") { + instance.Status.Provisioning = false + return false, nil + } + return false, err } From 64e8cb18b1bc4023e14a24444cedb01edaf957f6 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 15:45:17 -0600 Subject: [PATCH 65/99] add mysql doc to readme and fix formatting --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 79c97963b85..b0b69389fb2 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,16 @@ This project maintains [releases of the Azure Service Operator](https://github.c 1. [Resource Group](/docs/resourcegroup/resourcegroup.md) 2. [EventHub](/docs/eventhub/eventhub.md) 3. [Azure SQL](/docs/azuresql/azuresql.md) -4. [Azure Keyvault](/docs/keyvault/keyvault.md) -5. [Azure Rediscache](/docs/rediscache/rediscache.md) -6. [Storage Account](/docs/storage/storageaccount.md) -7. [Blob container](/docs/storage/blobcontainer.md) -8. [Azure Database for PostgreSQL](/docs/postgresql/postgresql.md) -9. [Virtual Network](/docs/virtualnetwork/virtualnetwork.md) -10.[Application Insights](/docs/appinsights/appinsights.md) -11.[API Management](/docs/apimgmt/apimgmt.md) -12.[Cosmos DB](/docs/cosmosdb/cosmosdb.md) +4. [Azure Database for PostgreSQL](/docs/postgresql/postgresql.md) +5. [Azure Database for MySQL](/docs/mysql/mysql.md) +6. [Azure Keyvault](/docs/keyvault/keyvault.md) +7. [Azure Rediscache](/docs/rediscache/rediscache.md) +8. [Storage Account](/docs/storage/storageaccount.md) +9. [Blob container](/docs/storage/blobcontainer.md) +10. [Virtual Network](/docs/virtualnetwork/virtualnetwork.md) +11. [Application Insights](/docs/appinsights/appinsights.md) +12. [API Management](/docs/apimgmt/apimgmt.md) +13. [Cosmos DB](/docs/cosmosdb/cosmosdb.md) For more information on deploying, troubleshooting & deleting resources, refer to [this](/docs/customresource.md) link From c8156b9cb54d9775c930a88b52b3cfe6d7699403 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Wed, 15 Apr 2020 16:03:11 -0600 Subject: [PATCH 66/99] fixing several issues from feedback --- pkg/errhelp/errors.go | 1 + .../cosmosdbs/cosmosdb_reconcile.go | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 4318fde13e5..f27f90f41f2 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -42,6 +42,7 @@ const ( NotFoundErrorCode = "NotFound" NoSuchHost = "no such host" ParentNotFoundErrorCode = "ParentResourceNotFound" + PreconditionFailed = "PreconditionFailed" QuotaExceeded = "QuotaExceeded" ResourceGroupNotFoundErrorCode = "ResourceGroupNotFound" RegionDoesNotAllowProvisioning = "RegionDoesNotAllowProvisioning" diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 8125fd8d079..659bc37f079 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -6,6 +6,7 @@ package cosmosdbs import ( "context" "fmt" + "strings" "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" @@ -45,16 +46,13 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) + instance.Status.Message = err.Error() + switch azerr.Type { case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: instance.Status.Provisioning = false - instance.Status.Message = azerr.Error() instance.Status.State = "Waiting" return false, nil - case errhelp.ResourceNotFound: - //NO-OP, try to create - default: - instance.Status.Message = fmt.Sprintf("Unhandled error after Get %v", azerr.Error()) } } else { @@ -87,6 +85,8 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return true, nil } + instance.Status.Provisioning = true + tags := helpers.LabelsToTags(instance.GetLabels()) accountName := instance.ObjectMeta.Name groupName := instance.Spec.ResourceGroup @@ -165,18 +165,19 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) - // this is likely to happen on first try due to not waiting for the future to complete - if azerr.Type == errhelp.AsyncOpIncompleteError { + // request submitted or already in progress + if azerr.Type == errhelp.AsyncOpIncompleteError || (azerr.Type == errhelp.PreconditionFailed && strings.Contains(azerr.Reason, "operation in progress")) { + instance.Status.State = "Deleting" instance.Status.Message = "Deletion request submitted successfully" return true, nil } - notFoundErrors := []string{ - errhelp.NotFoundErrorCode, // happens on first request after deletion succeeds - errhelp.ResourceNotFound, // happens on subsequent requests after deletion succeeds - errhelp.ResourceGroupNotFoundErrorCode, // database doesn't exist in this resource group but the name exists globally + notFound := []string{ + errhelp.NotFoundErrorCode, + errhelp.ResourceNotFound, + errhelp.ResourceGroupNotFoundErrorCode, } - if helpers.ContainsString(notFoundErrors, azerr.Type) { + if helpers.ContainsString(notFound, azerr.Type) { return false, m.deleteAccountKeysSecret(ctx, instance) } From 799e654c499dfec2519debf57c1ee9e7c42d8a3c Mon Sep 17 00:00:00 2001 From: jpflueger Date: Wed, 15 Apr 2020 16:15:50 -0600 Subject: [PATCH 67/99] setting hash for op incomplete --- .../cosmosdbs/cosmosdb_reconcile.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 659bc37f079..9533331f459 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -97,33 +97,32 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) + instance.Status.Message = err.Error() switch azerr.Type { case errhelp.AsyncOpIncompleteError: instance.Status.State = "Creating" instance.Status.Message = "Resource request successfully submitted to Azure" - return false, nil + instance.Status.SpecHash = hash case errhelp.InvalidResourceLocation, errhelp.LocationNotAvailableForResourceType: instance.Status.Provisioning = false instance.Status.Message = azerr.Error() return true, nil case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: instance.Status.Provisioning = false - instance.Status.Message = azerr.Error() - return false, nil case errhelp.NotFoundErrorCode: - if nameExists, err := m.CheckNameExistsCosmosDB(ctx, accountName); err != nil { + nameExists, err := m.CheckNameExistsCosmosDB(ctx, accountName) + if err != nil { instance.Status.Message = err.Error() - return false, err - } else if nameExists { + } + if nameExists { instance.Status.Provisioning = false instance.Status.Message = "CosmosDB Account name already exists" return true, nil } - default: - instance.Status.Message = azerr.Error() - return false, nil } + + return false, nil } instance.Status.SpecHash = hash From beceea191d62e61ed0ffa2e98b941c6009f1c6f1 Mon Sep 17 00:00:00 2001 From: Mel Rush Date: Wed, 15 Apr 2020 16:34:22 -0600 Subject: [PATCH 68/99] yaml name fix (#951) --- config/samples/azure_v1alpha1_blobcontainer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/samples/azure_v1alpha1_blobcontainer.yaml b/config/samples/azure_v1alpha1_blobcontainer.yaml index b3dadf1baf3..a7491ee1fd9 100644 --- a/config/samples/azure_v1alpha1_blobcontainer.yaml +++ b/config/samples/azure_v1alpha1_blobcontainer.yaml @@ -4,8 +4,8 @@ metadata: name: blobcontainer-sample spec: location: westus - resourcegroup: resourcegroup-sample - accountname: storageaccount-sample + resourcegroup: resourcegroup-azure-operators + accountname: storageaccountsample777 # accessLevel - Specifies whether data in the container may be accessed publicly and the level of access. # Possible values include: 'Container', 'Blob', 'None' accesslevel: Container \ No newline at end of file From 1e62a4a2ec8e5f8c473571c742d52ca5c78e9a8b Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Thu, 16 Apr 2020 08:49:23 +0800 Subject: [PATCH 69/99] #issue858#removeWrongFile --- config/default/manager_image_patch.yaml-e | 50 ----------------------- 1 file changed, 50 deletions(-) delete mode 100644 config/default/manager_image_patch.yaml-e diff --git a/config/default/manager_image_patch.yaml-e b/config/default/manager_image_patch.yaml-e deleted file mode 100644 index 9c62eb69973..00000000000 --- a/config/default/manager_image_patch.yaml-e +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - # Change the value of image field below to your controller image URL - - image: controller:latest - name: manager - env: - - name: AZURE_CLIENT_ID - valueFrom: - secretKeyRef: - name: azureoperatorsettings - key: AZURE_CLIENT_ID - optional: true - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: azureoperatorsettings - key: AZURE_CLIENT_SECRET - optional: true - - name: AZURE_TENANT_ID - valueFrom: - secretKeyRef: - name: azureoperatorsettings - key: AZURE_TENANT_ID - - name: AZURE_SUBSCRIPTION_ID - valueFrom: - secretKeyRef: - name: azureoperatorsettings - key: AZURE_SUBSCRIPTION_ID - - name: AZURE_USE_MI - valueFrom: - secretKeyRef: - name: azureoperatorsettings - key: AZURE_USE_MI - optional: true - - name: AZURE_OPERATOR_KEYVAULT - valueFrom: - secretKeyRef: - name: azureoperatorsettings - key: AZURE_OPERATOR_KEYVAULT - optional: true - #requeue after time in seconds" - - name: REQUEUE_AFTER - value: "30" From 65576bcf0f8f02564c7df8bdc06c64a1ed1d44ff Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 16 Apr 2020 14:21:57 -0600 Subject: [PATCH 70/99] location to create --- api/v1alpha1/cosmosdb_types.go | 4 +++- config/samples/azure_v1alpha1_cosmosdb.yaml | 3 ++- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 5 +++-- pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go | 2 +- pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 3 ++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 34fea0935fc..4b328a531b8 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -40,12 +40,14 @@ const ( // CosmosDBProperties the CosmosDBProperties of CosmosDB. type CosmosDBProperties struct { // CosmosDBDatabaseAccountOfferType - The offer type for the Cosmos DB database account. - DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` + DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` + EnableMultipleWriteLocations CosmosDBEnableMultipleWriteLocations `json:"enableMultipleWriteLocations,omitempty"` //Locations []CosmosDBLocation `json:"locations,omitempty"` } // +kubebuilder:validation:Enum=Standard type CosmosDBDatabaseAccountOfferType string +type CosmosDBEnableMultipleWriteLocations bool const ( // CosmosDBDatabaseAccountOfferTypeStandard string constant describing standard account offer type diff --git a/config/samples/azure_v1alpha1_cosmosdb.yaml b/config/samples/azure_v1alpha1_cosmosdb.yaml index d3efa498467..0dceae2222d 100644 --- a/config/samples/azure_v1alpha1_cosmosdb.yaml +++ b/config/samples/azure_v1alpha1_cosmosdb.yaml @@ -1,10 +1,11 @@ apiVersion: azure.microsoft.com/v1alpha1 kind: CosmosDB metadata: - name: cosmosdb-sample1908xyzkj + name: cosmosdb-sample-1 spec: kind: GlobalDocumentDB location: westus resourceGroup: resourcegroup-azure-operators properties: databaseAccountOfferType: Standard + enableMultipleWriteLocations: false diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 0d22bb85e85..a2cbf1744b5 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -42,6 +42,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, + enableWriteLocations v1alpha1.CosmosDBEnableMultipleWriteLocations, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) { cosmosDBClient, err := getCosmosDBClient() if err != nil { @@ -50,7 +51,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( dbKind := documentdb.DatabaseAccountKind(kind) sDBType := string(dbType) - + bWriteLocal := bool(enableWriteLocations) /* * Current state of Locations and CosmosDB properties: * Creating a Database account with CosmosDB requires @@ -81,7 +82,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( ID: &cosmosDBName, DatabaseAccountCreateUpdateProperties: &documentdb.DatabaseAccountCreateUpdateProperties{ DatabaseAccountOfferType: &sDBType, - EnableMultipleWriteLocations: to.BoolPtr(false), + EnableMultipleWriteLocations: &bWriteLocal, IsVirtualNetworkFilterEnabled: to.BoolPtr(false), Locations: &locationsArray, }, diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index f916e70fd54..09d4134f72f 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -21,7 +21,7 @@ func NewAzureCosmosDBManager() *AzureCosmosDBManager { // CosmosDBManager client functions type CosmosDBManager interface { // CreateOrUpdateCosmosDB creates a new cosmos database account - CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) + CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, enableWriteLocations v1alpha1.CosmosDBEnableMultipleWriteLocations, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) // GetCosmosDB gets a cosmos database account GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, *errhelp.AzureError) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 4eb8f6b0a0f..4f5ff0d4568 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -90,8 +90,9 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o location := instance.Spec.Location kind := instance.Spec.Kind dbType := instance.Spec.Properties.DatabaseAccountOfferType + enableWriteLocations := instance.Spec.Properties.EnableMultipleWriteLocations - db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) + db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, enableWriteLocations, tags) // everything is in a created/updated state if azerr == nil { From 9519962f6c9b6fddfc9346fb8446c72fc73aa8db Mon Sep 17 00:00:00 2001 From: Melanie Rush Date: Thu, 16 Apr 2020 16:38:29 -0600 Subject: [PATCH 71/99] adjusting method to properties --- api/v1alpha1/cosmosdb_types.go | 6 ++---- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 8 ++++---- pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go | 3 ++- pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 10 +++++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 4b328a531b8..0a291437f63 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -40,14 +40,12 @@ const ( // CosmosDBProperties the CosmosDBProperties of CosmosDB. type CosmosDBProperties struct { // CosmosDBDatabaseAccountOfferType - The offer type for the Cosmos DB database account. - DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` - EnableMultipleWriteLocations CosmosDBEnableMultipleWriteLocations `json:"enableMultipleWriteLocations,omitempty"` - //Locations []CosmosDBLocation `json:"locations,omitempty"` + DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` + EnableMultipleWriteLocations bool `json:"enableMultipleWriteLocations,omitempty"` } // +kubebuilder:validation:Enum=Standard type CosmosDBDatabaseAccountOfferType string -type CosmosDBEnableMultipleWriteLocations bool const ( // CosmosDBDatabaseAccountOfferTypeStandard string constant describing standard account offer type diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index a2cbf1744b5..8952ed049c6 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" @@ -41,8 +42,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, - dbType v1alpha1.CosmosDBDatabaseAccountOfferType, - enableWriteLocations v1alpha1.CosmosDBEnableMultipleWriteLocations, + properties azurev1alpha1.CosmosDBProperties, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) { cosmosDBClient, err := getCosmosDBClient() if err != nil { @@ -50,8 +50,8 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( } dbKind := documentdb.DatabaseAccountKind(kind) - sDBType := string(dbType) - bWriteLocal := bool(enableWriteLocations) + sDBType := string(properties.DatabaseAccountOfferType) + bWriteLocal := bool(properties.EnableMultipleWriteLocations) /* * Current state of Locations and CosmosDB properties: * Creating a Database account with CosmosDB requires diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index 09d4134f72f..5f117fc6554 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager" "github.com/Azure/go-autorest/autorest" @@ -21,7 +22,7 @@ func NewAzureCosmosDBManager() *AzureCosmosDBManager { // CosmosDBManager client functions type CosmosDBManager interface { // CreateOrUpdateCosmosDB creates a new cosmos database account - CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, enableWriteLocations v1alpha1.CosmosDBEnableMultipleWriteLocations, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) + CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, properties azurev1alpha1.CosmosDBProperties, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) // GetCosmosDB gets a cosmos database account GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, *errhelp.AzureError) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 4f5ff0d4568..c4ad59cad98 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/Azure/azure-service-operator/api/v1alpha1" + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" @@ -89,10 +90,13 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o groupName := instance.Spec.ResourceGroup location := instance.Spec.Location kind := instance.Spec.Kind - dbType := instance.Spec.Properties.DatabaseAccountOfferType - enableWriteLocations := instance.Spec.Properties.EnableMultipleWriteLocations - db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, enableWriteLocations, tags) + cosmosDBProperties := azurev1alpha1.CosmosDBProperties{ + DatabaseAccountOfferType: instance.Spec.Properties.DatabaseAccountOfferType, + EnableMultipleWriteLocations: instance.Spec.Properties.EnableMultipleWriteLocations, + } + + db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, cosmosDBProperties, tags) // everything is in a created/updated state if azerr == nil { From 5308d24f38f4c39f9cbe7bf39278d17fed0c1ce8 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Wed, 15 Apr 2020 16:54:30 -0600 Subject: [PATCH 72/99] using this branch for test failure investigation --- controllers/helpers.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/controllers/helpers.go b/controllers/helpers.go index 29970bfe123..815c0470ad4 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -247,6 +247,14 @@ func EnsureInstanceWithResult(ctx context.Context, t *testing.T, tc TestContext, statused := ConvertToStatus(instance) // if we expect this resource to end up with provisioned == true then failedProvisioning == true is unrecoverable if provisioned && statused.Status.FailedProvisioning { + if strings.Contains(instance.Status.Message, "already exists") { + t.Log("") + t.Log("-------") + t.Log("unexpected failed provisioning encountered") + t.Logf("%+v\n", statused.Status) + t.Log("-------") + t.Log("") + } return helpers.NewStop(fmt.Errorf("Failed provisioning: %s", statused.Status.Message)) } if !strings.Contains(statused.Status.Message, message) || statused.Status.Provisioned != provisioned { From 4989bf2fa4737b64e8f56537b0ae7226ba68fad8 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Wed, 15 Apr 2020 17:18:52 -0600 Subject: [PATCH 73/99] bad var --- controllers/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/helpers.go b/controllers/helpers.go index 815c0470ad4..05def440496 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -247,7 +247,7 @@ func EnsureInstanceWithResult(ctx context.Context, t *testing.T, tc TestContext, statused := ConvertToStatus(instance) // if we expect this resource to end up with provisioned == true then failedProvisioning == true is unrecoverable if provisioned && statused.Status.FailedProvisioning { - if strings.Contains(instance.Status.Message, "already exists") { + if strings.Contains(statused.Status.Message, "already exists") { t.Log("") t.Log("-------") t.Log("unexpected failed provisioning encountered") From 8c1fd77edbd15f53872d4de4d1276eb4ffb9a685 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Thu, 16 Apr 2020 09:55:00 -0600 Subject: [PATCH 74/99] print timestamp with data to diff time since requested --- controllers/helpers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/helpers.go b/controllers/helpers.go index 05def440496..22153b36a2a 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -247,11 +247,12 @@ func EnsureInstanceWithResult(ctx context.Context, t *testing.T, tc TestContext, statused := ConvertToStatus(instance) // if we expect this resource to end up with provisioned == true then failedProvisioning == true is unrecoverable if provisioned && statused.Status.FailedProvisioning { - if strings.Contains(statused.Status.Message, "already exists") { + if strings.Contains(statused.Status.Message, "already exists") || strings.Contains(statused.Status.Message, "AlreadyExists") { t.Log("") t.Log("-------") t.Log("unexpected failed provisioning encountered") t.Logf("%+v\n", statused.Status) + t.Logf("current time %v\n", time.Now()) t.Log("-------") t.Log("") } From e60a672e78c56e5e9ef6af5bbc287e94bcaa4d4b Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Thu, 16 Apr 2020 13:23:58 -0600 Subject: [PATCH 75/99] experimenting with another region --- controllers/suite_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index fdb064b4c00..f3738b6578e 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -72,11 +72,11 @@ func setup() error { } resourceGroupName := GenerateTestResourceName("rg-prime") - resourcegroupLocation := resourcemanagerconfig.DefaultLocation() + resourcegroupLocation := "westus2" eventhubNamespaceName := GenerateTestResourceName("evns-prime") eventhubName := GenerateTestResourceName("ev-prime") - namespaceLocation := resourcemanagerconfig.DefaultLocation() + namespaceLocation := "westus2" storageAccountName := GenerateAlphaNumTestResourceName("saprime") blobContainerName := GenerateTestResourceName("blob-prime") From fabc83b9e3cbbb8f5c7fd8eecf0550be12c51a4d Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Thu, 16 Apr 2020 14:00:11 -0600 Subject: [PATCH 76/99] test tweaks to unblock pipeline --- .../postgresql_combined_controller_test.go | 75 ++----------------- controllers/suite_test.go | 4 +- pkg/resourcemanager/config/env.go | 2 +- 3 files changed, 9 insertions(+), 72 deletions(-) diff --git a/controllers/postgresql_combined_controller_test.go b/controllers/postgresql_combined_controller_test.go index 9d4e1bb567f..befc6787633 100644 --- a/controllers/postgresql_combined_controller_test.go +++ b/controllers/postgresql_combined_controller_test.go @@ -10,24 +10,18 @@ import ( "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestPSQLDatabaseController(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) var rgName string var rgLocation string var postgreSQLServerName string var postgreSQLServerInstance *azurev1alpha1.PostgreSQLServer - var postgreSQLServerNamespacedName types.NamespacedName - var err error // Add any setup steps that needs to be executed before each test rgName = tc.resourceGroupName @@ -56,20 +50,7 @@ func TestPSQLDatabaseController(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, postgreSQLServerInstance) - assert.Equal(nil, err, "create postgreSQLServerInstance in k8s") - - postgreSQLServerNamespacedName = types.NamespacedName{Name: postgreSQLServerName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance) - return HasFinalizer(postgreSQLServerInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for postgreSQLserver to have finlizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance) - return postgreSQLServerInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for postgreSQLserver to be provisioned") + EnsureInstance(ctx, t, tc, postgreSQLServerInstance) postgreSQLDatabaseName := GenerateTestResourceNameWithRandom("psql-db", 10) @@ -85,28 +66,9 @@ func TestPSQLDatabaseController(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, postgreSQLDatabaseInstance) - assert.Equal(nil, err, "create postgreSQLDatabaseInstance in k8s") - - postgreSQLDatabaseNamespacedName := types.NamespacedName{Name: postgreSQLDatabaseName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance) - return HasFinalizer(postgreSQLDatabaseInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to have finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance) - return postgreSQLDatabaseInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to be provisioned") + EnsureInstance(ctx, t, tc, postgreSQLDatabaseInstance) - err = tc.k8sClient.Delete(ctx, postgreSQLDatabaseInstance) - assert.Equal(nil, err, "delete postgreSQLDatabaseInstance in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, postgreSQLDatabaseInstance) // Test firewall rule ------------------------------- @@ -126,36 +88,11 @@ func TestPSQLDatabaseController(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, postgreSQLFirewallRuleInstance) - assert.Equal(nil, err, "create postgreSQLFirewallRuleInstance in k8s") - - postgreSQLFirewallRuleNamespacedName := types.NamespacedName{Name: postgreSQLFirewallRuleName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance) - return HasFinalizer(postgreSQLFirewallRuleInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to have finalizer") + EnsureInstance(ctx, t, tc, postgreSQLFirewallRuleInstance) - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance) - return postgreSQLFirewallRuleInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to be provisioned") - - err = tc.k8sClient.Delete(ctx, postgreSQLFirewallRuleInstance) - assert.Equal(nil, err, "delete postgreSQLFirewallRuleInstance in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, postgreSQLFirewallRuleInstance) // Add any teardown steps that needs to be executed after each test - err = tc.k8sClient.Delete(ctx, postgreSQLServerInstance) - assert.Equal(nil, err, "delete postgreSQLServerInstance in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for postgreSQLServerInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, postgreSQLServerInstance) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index f3738b6578e..fdb064b4c00 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -72,11 +72,11 @@ func setup() error { } resourceGroupName := GenerateTestResourceName("rg-prime") - resourcegroupLocation := "westus2" + resourcegroupLocation := resourcemanagerconfig.DefaultLocation() eventhubNamespaceName := GenerateTestResourceName("evns-prime") eventhubName := GenerateTestResourceName("ev-prime") - namespaceLocation := "westus2" + namespaceLocation := resourcemanagerconfig.DefaultLocation() storageAccountName := GenerateAlphaNumTestResourceName("saprime") blobContainerName := GenerateTestResourceName("blob-prime") diff --git a/pkg/resourcemanager/config/env.go b/pkg/resourcemanager/config/env.go index 0359d6b64f7..553dd7393aa 100644 --- a/pkg/resourcemanager/config/env.go +++ b/pkg/resourcemanager/config/env.go @@ -50,7 +50,7 @@ func ParseEnvironment() error { authorizationServerURL = azureEnv.ActiveDirectoryEndpoint baseURI = azureEnv.ResourceManagerEndpoint // BaseURI() - locationDefault = envy.Get("AZURE_LOCATION_DEFAULT", "southcentralus") // DefaultLocation() + locationDefault = envy.Get("AZURE_LOCATION_DEFAULT", "westus2") // DefaultLocation() useDeviceFlow = ParseBoolFromEnvironment("AZURE_USE_DEVICEFLOW") // UseDeviceFlow() useMI = ParseBoolFromEnvironment("AZURE_USE_MI") // UseMI() keepResources = ParseBoolFromEnvironment("AZURE_SAMPLES_KEEP_RESOURCES") // KeepResources() From 4c83c1ef17625db8bfd4ca08697426892904e9a4 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Thu, 16 Apr 2020 17:01:31 -0600 Subject: [PATCH 77/99] adjusting no resource group test --- controllers/cosmosdb_controller_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index 7c0b4ff255e..31ad06cb5e7 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -84,10 +86,8 @@ func TestCosmosDBControllerNoResourceGroup(t *testing.T) { }, }, } - //the expected error meessage to be shown - errMessage := "Waiting for resource group '" + resourceGroupName + "' to be available" - EnsureInstanceWithResult(ctx, t, tc, dbInstance1, errMessage, false) + EnsureInstanceWithResult(ctx, t, tc, dbInstance1, errhelp.ResourceGroupNotFoundErrorCode, false) EnsureDelete(ctx, t, tc, dbInstance1) } From 6179c653c60f50aa9c1ef40e22121eb668bbe78e Mon Sep 17 00:00:00 2001 From: huangpf <3858524+huangpf@users.noreply.github.com> Date: Fri, 17 Apr 2020 09:55:04 -0700 Subject: [PATCH 78/99] Virtual Machine Operator (#947) --- PROJECT | 4 +- api/v1alpha1/azurevirtualmachine_types.go | 59 +++++ .../azurevirtualmachine_types_test.go | 74 +++++++ config/crd/kustomization.yaml | 3 + .../cainjection_in_azurevirtualmachines.yaml | 8 + .../webhook_in_azurevirtualmachines.yaml | 17 ++ .../azure_v1alpha1_azurevirtualmachine.yaml | 14 ++ controllers/azurevirtualmachine_controller.go | 28 +++ .../azurevirtualmachine_controller_test.go | 167 ++++++++++++++ docs/virtualmachine/virtualmachine.md | 37 ++++ docs/virtualnetwork/networkinterface.md | 31 +++ docs/virtualnetwork/publicipaddress.md | 33 +++ go.mod | 2 +- go.sum | 1 + main.go | 21 ++ pkg/helpers/helpers_strings_test.go | 50 +++++ pkg/helpers/stringhelper.go | 36 ++++ pkg/resourcemanager/nic/client.go | 27 +-- pkg/resourcemanager/nic/manager.go | 2 +- pkg/resourcemanager/vm/client.go | 191 ++++++++++++++++ pkg/resourcemanager/vm/manager.go | 36 ++++ pkg/resourcemanager/vm/reconcile.go | 203 ++++++++++++++++++ 22 files changed, 1017 insertions(+), 27 deletions(-) create mode 100644 api/v1alpha1/azurevirtualmachine_types.go create mode 100644 api/v1alpha1/azurevirtualmachine_types_test.go create mode 100644 config/crd/patches/cainjection_in_azurevirtualmachines.yaml create mode 100644 config/crd/patches/webhook_in_azurevirtualmachines.yaml create mode 100644 config/samples/azure_v1alpha1_azurevirtualmachine.yaml create mode 100644 controllers/azurevirtualmachine_controller.go create mode 100644 controllers/azurevirtualmachine_controller_test.go create mode 100644 docs/virtualmachine/virtualmachine.md create mode 100644 docs/virtualnetwork/networkinterface.md create mode 100644 docs/virtualnetwork/publicipaddress.md create mode 100644 pkg/resourcemanager/vm/client.go create mode 100644 pkg/resourcemanager/vm/manager.go create mode 100644 pkg/resourcemanager/vm/reconcile.go diff --git a/PROJECT b/PROJECT index a5a71f1f7c0..00905ec53b7 100644 --- a/PROJECT +++ b/PROJECT @@ -86,4 +86,6 @@ resources: - group: azure version: v1alpha1 kind: MySQLFirewallRule - +- group: azure + version: v1alpha1 + kind: AzureVirtualMachine diff --git a/api/v1alpha1/azurevirtualmachine_types.go b/api/v1alpha1/azurevirtualmachine_types.go new file mode 100644 index 00000000000..05555d355a0 --- /dev/null +++ b/api/v1alpha1/azurevirtualmachine_types.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// AzureVirtualMachineSpec defines the desired state of AzureVirtualMachine +type AzureVirtualMachineSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + Location string `json:"location"` + ResourceGroup string `json:"resourceGroup"` + VMSize string `json:"vmSize"` + OSType OSType `json:"osType"` + AdminUserName string `json:"adminUserName"` + SSHPublicKeyData string `json:"sshPublicKeyData,omitempty"` + NetworkInterfaceName string `json:"networkInterfaceName"` + PlatformImageURN string `json:"platformImageURN"` +} + +type OSType string + +const ( + // Windows ... + Windows OSType = "Windows" + // Linux ... + Linux OSType = "Linux" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// AzureVirtualMachine is the Schema for the azurevirtualmachines API +type AzureVirtualMachine struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureVirtualMachineSpec `json:"spec,omitempty"` + Status ASOStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureVirtualMachineList contains a list of AzureVirtualMachine +type AzureVirtualMachineList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureVirtualMachine `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AzureVirtualMachine{}, &AzureVirtualMachineList{}) +} diff --git a/api/v1alpha1/azurevirtualmachine_types_test.go b/api/v1alpha1/azurevirtualmachine_types_test.go new file mode 100644 index 00000000000..ec7b74a8961 --- /dev/null +++ b/api/v1alpha1/azurevirtualmachine_types_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha1 + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// These tests are written in BDD-style using Ginkgo framework. Refer to +// http://onsi.github.io/ginkgo to learn more. + +var _ = Describe("AzureVirtualMachine", func() { + var ( + key types.NamespacedName + created, fetched *AzureVirtualMachine + ) + + BeforeEach(func() { + // Add any setup steps that needs to be executed before each test + }) + + AfterEach(func() { + // Add any teardown steps that needs to be executed after each test + }) + + // Add Tests for OpenAPI validation (or additonal CRD features) specified in + // your API definition. + // Avoid adding tests for vanilla CRUD operations because they would + // test Kubernetes API server, which isn't the goal here. + Context("Create API", func() { + + It("should create an object successfully", func() { + + key = types.NamespacedName{ + Name: "foo", + Namespace: "default", + } + created = &AzureVirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }, + Spec: AzureVirtualMachineSpec{ + Location: "westus", + ResourceGroup: "foo-vm", + VMSize: "test", + OSType: OSType("Linux"), + AdminUserName: "test", + SSHPublicKeyData: "test", + NetworkInterfaceName: "test", + PlatformImageURN: "w:x:y:z", + }} + + By("creating an API obj") + Expect(k8sClient.Create(context.TODO(), created)).To(Succeed()) + + fetched = &AzureVirtualMachine{} + Expect(k8sClient.Get(context.TODO(), key, fetched)).To(Succeed()) + Expect(fetched).To(Equal(created)) + + By("deleting the created object") + Expect(k8sClient.Delete(context.TODO(), created)).To(Succeed()) + Expect(k8sClient.Get(context.TODO(), key, created)).ToNot(Succeed()) + }) + + }) + +}) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index a4978b866b5..d32a5f49f90 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -33,6 +33,7 @@ resources: - bases/azure.microsoft.com_mysqlfirewallrules.yaml - bases/azure.microsoft.com_azurepublicipaddresses.yaml - bases/azure.microsoft.com_azurenetworkinterfaces.yaml +- bases/azure.microsoft.com_azurevirtualmachines.yaml # +kubebuilder:scaffold:crdkustomizeresource #patches: @@ -65,6 +66,7 @@ resources: #- patches/webhook_in_storageaccounts.yaml #- patches/webhook_in_azurepublicipaddresses.yaml #- patches/webhook_in_azurenetworkinterfaces.yaml +#- patches/webhook_in_azurevirtualmachines.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CAINJECTION] patches here are for enabling the CA injection for each CRD @@ -96,6 +98,7 @@ resources: #- patches/cainjection_in_storageaccounts.yaml #- patches/cainjection_in_azurepublicipaddresses.yaml #- patches/cainjection_in_azurenetworkinterfaces.yaml +#- patches/cainjection_in_azurevirtualmachines.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_azurevirtualmachines.yaml b/config/crd/patches/cainjection_in_azurevirtualmachines.yaml new file mode 100644 index 00000000000..bee5073111d --- /dev/null +++ b/config/crd/patches/cainjection_in_azurevirtualmachines.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: azurevirtualmachines.azure.microsoft.com diff --git a/config/crd/patches/webhook_in_azurevirtualmachines.yaml b/config/crd/patches/webhook_in_azurevirtualmachines.yaml new file mode 100644 index 00000000000..fb9cf38eef8 --- /dev/null +++ b/config/crd/patches/webhook_in_azurevirtualmachines.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: azurevirtualmachines.azure.microsoft.com +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/samples/azure_v1alpha1_azurevirtualmachine.yaml b/config/samples/azure_v1alpha1_azurevirtualmachine.yaml new file mode 100644 index 00000000000..c4a54fec2c4 --- /dev/null +++ b/config/samples/azure_v1alpha1_azurevirtualmachine.yaml @@ -0,0 +1,14 @@ +apiVersion: azure.microsoft.com/v1alpha1 +kind: AzureVirtualMachine +metadata: + name: hpfvm20 +spec: + location: SouthCentralUS + resourceGroup: resourcegroup-azure-operators + vmSize: Standard_DS1_v2 + osType: Linux + adminUserName: azureuser + # SSH public key to be used with VM (eg cat ~/.ssh/id_rsa.pub) + sshPublicKeyData: "{ssh public key}" + networkInterfaceName: hpfnic20 + platformImageURN: Canonical:UbuntuServer:16.04-LTS:latest diff --git a/controllers/azurevirtualmachine_controller.go b/controllers/azurevirtualmachine_controller.go new file mode 100644 index 00000000000..a10447c180b --- /dev/null +++ b/controllers/azurevirtualmachine_controller.go @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package controllers + +import ( + ctrl "sigs.k8s.io/controller-runtime" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" +) + +// AzureVirtualMachineReconciler reconciles a AzureVirtualMachine object +type AzureVirtualMachineReconciler struct { + Reconciler *AsyncReconciler +} + +// +kubebuilder:rbac:groups=azure.microsoft.com,resources=azurevirtualmachines,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=azure.microsoft.com,resources=azurevirtualmachines/status,verbs=get;update;patch + +func (r *AzureVirtualMachineReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + return r.Reconciler.Reconcile(req, &azurev1alpha1.AzureVirtualMachine{}) +} + +func (r *AzureVirtualMachineReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&azurev1alpha1.AzureVirtualMachine{}). + Complete(r) +} diff --git a/controllers/azurevirtualmachine_controller_test.go b/controllers/azurevirtualmachine_controller_test.go new file mode 100644 index 00000000000..f9e38720f71 --- /dev/null +++ b/controllers/azurevirtualmachine_controller_test.go @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// +build all azurevirtualmachine + +package controllers + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "testing" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + "golang.org/x/crypto/ssh" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func generateRandomSshPublicKeyString() string { + privateKey, _ := rsa.GenerateKey(rand.Reader, 2048) + publicRsaKey, _ := ssh.NewPublicKey(&privateKey.PublicKey) + sshPublicKeyData := string(ssh.MarshalAuthorizedKey(publicRsaKey)) + return sshPublicKeyData +} + +func TestVirtualMachineControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + rgName := GenerateTestResourceNameWithRandom("rg", 10) + vmName := GenerateTestResourceNameWithRandom("vm", 10) + vmSize := "Standard_DS1_v2" + osType := azurev1alpha1.OSType("Linux") + adminUserName := GenerateTestResourceNameWithRandom("u", 10) + sshPublicKeyData := GenerateTestResourceNameWithRandom("ssh", 10) + nicName := GenerateTestResourceNameWithRandom("nic", 10) + platformImageUrn := "Canonical:UbuntuServer:16.04-LTS:latest" + + // Create a VM + vmInstance := &azurev1alpha1.AzureVirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmName, + Namespace: "default", + }, + Spec: azurev1alpha1.AzureVirtualMachineSpec{ + Location: tc.resourceGroupLocation, + ResourceGroup: rgName, + VMSize: vmSize, + OSType: osType, + AdminUserName: adminUserName, + SSHPublicKeyData: sshPublicKeyData, + NetworkInterfaceName: nicName, + PlatformImageURN: platformImageUrn, + }, + } + + EnsureInstanceWithResult(ctx, t, tc, vmInstance, errhelp.ResourceGroupNotFoundErrorCode, false) + + EnsureDelete(ctx, t, tc, vmInstance) +} + +func TestVirtualMachineHappyPathWithNicPipVNetAndSubnet(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + + // Create a vnet with a subnet + vnetName := GenerateTestResourceNameWithRandom("vnt", 10) + subnetName := GenerateTestResourceNameWithRandom("snt", 10) + vnetSubNetInstance := azurev1alpha1.VNetSubnets{ + SubnetName: subnetName, + SubnetAddressPrefix: "110.1.0.0/16", + } + pipName := GenerateTestResourceNameWithRandom("pip3", 10) + vnetInstance := &azurev1alpha1.VirtualNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: vnetName, + Namespace: "default", + }, + Spec: azurev1alpha1.VirtualNetworkSpec{ + Location: tc.resourceGroupLocation, + ResourceGroup: tc.resourceGroupName, + AddressSpace: "110.0.0.0/8", + Subnets: []azurev1alpha1.VNetSubnets{vnetSubNetInstance}, + }, + } + + EnsureInstance(ctx, t, tc, vnetInstance) + + // Create a Public IP Address + publicIPAllocationMethod := "Static" + idleTimeoutInMinutes := 10 + publicIPAddressVersion := "IPv4" + skuName := "Basic" + pipInstance := &azurev1alpha1.AzurePublicIPAddress{ + ObjectMeta: metav1.ObjectMeta{ + Name: pipName, + Namespace: "default", + }, + Spec: azurev1alpha1.AzurePublicIPAddressSpec{ + Location: tc.resourceGroupLocation, + ResourceGroup: tc.resourceGroupName, + PublicIPAllocationMethod: publicIPAllocationMethod, + IdleTimeoutInMinutes: idleTimeoutInMinutes, + PublicIPAddressVersion: publicIPAddressVersion, + SkuName: skuName, + }, + } + + EnsureInstance(ctx, t, tc, pipInstance) + + // Create a NIC + nicName := GenerateTestResourceNameWithRandom("nic", 10) + nicInstance := &azurev1alpha1.AzureNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: nicName, + Namespace: "default", + }, + Spec: azurev1alpha1.AzureNetworkInterfaceSpec{ + Location: tc.resourceGroupLocation, + ResourceGroup: tc.resourceGroupName, + VNetName: vnetName, + SubnetName: subnetName, + PublicIPAddressName: pipName, + }, + } + + EnsureInstance(ctx, t, tc, nicInstance) + + // Create a VM + vmName := GenerateTestResourceNameWithRandom("vm", 10) + vmSize := "Standard_DS1_v2" + osType := azurev1alpha1.OSType("Linux") + vmImageUrn := "Canonical:UbuntuServer:16.04-LTS:latest" + userName := "azureuser" + + sshPublicKeyData := generateRandomSshPublicKeyString() + + vmInstance := &azurev1alpha1.AzureVirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmName, + Namespace: "default", + }, + Spec: azurev1alpha1.AzureVirtualMachineSpec{ + Location: tc.resourceGroupLocation, + ResourceGroup: tc.resourceGroupName, + VMSize: vmSize, + OSType: osType, + AdminUserName: userName, + SSHPublicKeyData: sshPublicKeyData, + NetworkInterfaceName: nicName, + PlatformImageURN: vmImageUrn, + }, + } + + EnsureInstance(ctx, t, tc, vmInstance) + + EnsureDelete(ctx, t, tc, vmInstance) + + EnsureDelete(ctx, t, tc, nicInstance) + + EnsureDelete(ctx, t, tc, pipInstance) + + EnsureDelete(ctx, t, tc, vnetInstance) +} diff --git a/docs/virtualmachine/virtualmachine.md b/docs/virtualmachine/virtualmachine.md new file mode 100644 index 00000000000..1bd1854f60f --- /dev/null +++ b/docs/virtualmachine/virtualmachine.md @@ -0,0 +1,37 @@ +# Virtual Machine Operator + +This operator deploys an Azure Virtual Machine (VM) into a specified resource group at the specified location. Users can specify platform image, size, user name and public SSH key, etc. for the VM. + +Note that in the current version you can only create VM using Linux platform images. + +Learn more about Azure Virtual Machine [here](https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_azurevirtualmachine.yaml) to provision a Virtual Machine. + +The spec is comprised of the following fields: + +* Location +* ResourceGroup +* VMSize +* AdminUserName +* SshPublicKeyData +* NetworkInterfaceName +* PlatformImageURN + +### Required Fields + +A Virtual Machine needs the following fields to deploy, along with a location and resource group. + +* `VMSize` specify the VM size for the virtual machine +* `AdminUserName` specify the user name for the virtual machine +* `SshPublicKeyData` specify the SSH public key data for loging into the virtual machine +* `NetworkInterfaceName` specify the network interface that the VM will use +* `PlatformImageURN` specify the platform image's uniform resource name (URN) in the 'publisher:offer:sku:version' format. + +### Optional Fields + +Not available. + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. diff --git a/docs/virtualnetwork/networkinterface.md b/docs/virtualnetwork/networkinterface.md new file mode 100644 index 00000000000..ec090426548 --- /dev/null +++ b/docs/virtualnetwork/networkinterface.md @@ -0,0 +1,31 @@ +# Network Interface Operator + +This operator deploys an Azure Network Interface (NIC) into a specified resource group at the specified location. Users can specify underlying public IP address and virtual network configurations in their NIC setup. + +Learn more about Azure Network Interface [here](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_azurenetworkinterface.yaml) to provision a Network Interface. + +The spec is comprised of the following fields: + +* Location +* ResourceGroup +* VNetName +* SubnetName +* PublicIPAddressName + +### Required Fields + +A Network Interface needs the following fields to deploy, along with a location and resource group. + +* `VNetName` specify the name for the virtual network that the network interface belongs to +* `SubnetName` specify the name for the subnet that the network interface belongs to +* `PublicIPAddressName` specify the name for the public IP address that the network interface uses + +### Optional Fields + +Not available. + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. diff --git a/docs/virtualnetwork/publicipaddress.md b/docs/virtualnetwork/publicipaddress.md new file mode 100644 index 00000000000..adc1ea63e2d --- /dev/null +++ b/docs/virtualnetwork/publicipaddress.md @@ -0,0 +1,33 @@ +# Public IP Address Operator + +This operator deploys an Azure Public IP Address (PIP) into a specified resource group at the specified location. Users can specify IP allocation method, idle timeout, IP address version, and SKU. + +Learn more about Azure Public IP Address [here](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-ip-addresses-overview-arm). + +Here is a [sample YAML](/config/samples/azure_v1alpha1_azurepublicipaddress.yaml) to provision a Public IP Address. + +The spec is comprised of the following fields: + +* Location +* ResourceGroup +* PublicIPAllocationMethod +* IdleTimeoutInMinutes +* PublicIPAddressVersion +* SkuName + +### Required Fields + +A Public IP Address needs the following fields to deploy, along with a location and resource group. + +* `PublicIPAllocationMethod` specify the allocation method for the public IP address, either 'Static' or 'Dynamic' +* `IdleTimeoutInMinutes` specify the idle timeout value (in minutes) for the public IP address +* `PublicIPAddressVersion` specify the version for the public IP address, either 'IPv4' or 'IPv6' +* `SkuName` specify the SKU name for the public IP address, either 'Basic' or 'Standard' + +### Optional Fields + +Not available. + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. diff --git a/go.mod b/go.mod index aeb2caafcb5..eadaa38c31b 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/sethvargo/go-password v0.1.2 github.com/stretchr/testify v1.5.1 - golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 // indirect + golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/sys v0.0.0-20190621203818-d432491b9138 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect diff --git a/go.sum b/go.sum index 7ea6274b6b7..50237bd6893 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-sdk-for-go v40.6.0+incompatible h1:ULjp/a/UsBfnZcl45jjywhcBKex/k/A1cG9s9NapLFw= github.com/Azure/azure-sdk-for-go v41.0.0+incompatible h1:nQc4CAuBSr8rO0aZ90NvHoKyWYodhtzSAS4DPDrCtqo= github.com/Azure/azure-sdk-for-go v41.1.0+incompatible h1:AkS9XaeC8TDd0W0UiJRnEcYEHBO6xzILqqswHiNlSZQ= +github.com/Azure/azure-sdk-for-go v41.2.0+incompatible h1:JOlv1wDuxcJi1ExJpQLNfEj6znsTFt2TiwQMow2YaXI= github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= github.com/Azure/go-autorest/autorest v0.5.0 h1:Mlm9qy2fpQ9MvfyI41G2Zf5B4CsgjjNbLOWszfK6KrY= github.com/Azure/go-autorest/autorest v0.5.0/go.mod h1:9HLKlQjVBH6U3oDfsXOeVc56THsLPw1L03yban4xThw= diff --git a/main.go b/main.go index 995bd302a06..2b1fb8b788c 100644 --- a/main.go +++ b/main.go @@ -38,6 +38,7 @@ import ( resourcemanagerstorage "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages" blobContainerManager "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/blobcontainer" storageaccountManager "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/storageaccount" + vm "github.com/Azure/azure-service-operator/pkg/resourcemanager/vm" vnet "github.com/Azure/azure-service-operator/pkg/resourcemanager/vnet" "github.com/Azure/azure-service-operator/pkg/secrets" keyvaultSecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" @@ -640,6 +641,26 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NetworkInterface") os.Exit(1) } + + if err = (&controllers.AzureVirtualMachineReconciler{ + Reconciler: &controllers.AsyncReconciler{ + Client: mgr.GetClient(), + AzureClient: vm.NewAzureVirtualMachineClient( + secretClient, + mgr.GetScheme(), + ), + Telemetry: telemetry.InitializeTelemetryDefault( + "VirtualMachine", + ctrl.Log.WithName("controllers").WithName("VirtualMachine"), + ), + Recorder: mgr.GetEventRecorderFor("VirtualMachine-controller"), + Scheme: scheme, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "VirtualMachine") + os.Exit(1) + } + // +kubebuilder:scaffold:builder setupLog.Info("starting manager") diff --git a/pkg/helpers/helpers_strings_test.go b/pkg/helpers/helpers_strings_test.go index 4bdf31dc9cc..6ac1dee4744 100644 --- a/pkg/helpers/helpers_strings_test.go +++ b/pkg/helpers/helpers_strings_test.go @@ -17,3 +17,53 @@ func TestGenerateRandomUsername(t *testing.T) { } } } + +func TestMakeResourceIDWithSubResource(t *testing.T) { + testOutput := MakeResourceID( + "00000000-0000-0000-0000-000000000000", + "test", + "Microsoft.Network", + "networkInterfaces", + "test", + "subnets", + "test", + ) + expectedOutput := "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test/providers/Microsoft.Network/networkInterfaces/test/subnets/test" + + if testOutput != expectedOutput { + t.Errorf("Test output string '%s' is not as expected: '%s'.", testOutput, expectedOutput) + } +} + +func TestMakeResourceIDWithNoSubResource(t *testing.T) { + testOutput := MakeResourceID( + "00000000-0000-0000-0000-000000000000", + "test", + "Microsoft.Network", + "networkInterfaces", + "test", + "", + "", + ) + expectedOutput := "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test/providers/Microsoft.Network/networkInterfaces/test" + + if testOutput != expectedOutput { + t.Errorf("Test output string '%s' is not as expected: '%s'.", testOutput, expectedOutput) + } +} + +func TestDecodingFromBase64EncodedString(t *testing.T) { + testOutput1 := FromBase64EncodedString("dGVzdA==") + expectedOutput1 := "test" + + if testOutput1 != expectedOutput1 { + t.Errorf("Test output string '%s' is not as expected: '%s'.", testOutput1, expectedOutput1) + } + + testOutput2 := FromBase64EncodedString("") + expectedOutput2 := "" + + if testOutput2 != expectedOutput2 { + t.Errorf("Test output string '%s' is not as expected: '%s'.", testOutput2, expectedOutput2) + } +} diff --git a/pkg/helpers/stringhelper.go b/pkg/helpers/stringhelper.go index 7f150a24993..6c269e260c6 100644 --- a/pkg/helpers/stringhelper.go +++ b/pkg/helpers/stringhelper.go @@ -5,6 +5,7 @@ package helpers import ( "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "math/rand" @@ -159,3 +160,38 @@ func ReplaceAny(s string, chars []string) string { reg := regexp.MustCompile(fmt.Sprintf(`(%s)`, strings.Join(chars, "|"))) return reg.ReplaceAllString(s, ".") } + +// MakeResourceID can be used to construct a resource ID using the input segments +// Sample 1: /subscriptions/88fd8cb2-8248-499e-9a2d-4929a4b0133c/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.Network/publicIPAddresses/azurepublicipaddress-sample-3 +// Sample 2: /subscriptions/88fd8cb2-8248-499e-9a2d-4929a4b0133c/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.Network/virtualNetworks/vnet-sample-hpf-1/subnets/test2 +func MakeResourceID(subscriptionID string, resourceGroupName string, provider string, resourceType string, resourceName string, subResourceType string, subResourceName string) string { + segments := []string{ + "subscriptions", + subscriptionID, + "resourceGroups", + resourceGroupName, + "providers", + provider, + resourceType, + resourceName, + } + + if subResourceType != "" && subResourceName != "" { + segments = append(segments, subResourceType, subResourceName) + } + + result := "/" + strings.Join(segments, "/") + + return result +} + +// FromBase64EncodedString can be used to decode a base64 encoded string into another string in the return. +func FromBase64EncodedString(input string) string { + output, err := base64.StdEncoding.DecodeString(input) + if err != nil { + _ = fmt.Errorf("cannot decode input string '%s' with error '%s'", input, err) + } + + decodedString := string(output) + return decodedString +} diff --git a/pkg/resourcemanager/nic/client.go b/pkg/resourcemanager/nic/client.go index 91bf3f4747c..e83bf91a63d 100644 --- a/pkg/resourcemanager/nic/client.go +++ b/pkg/resourcemanager/nic/client.go @@ -5,9 +5,9 @@ package nic import ( "context" - "strings" vnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-09-01/network" + "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" "github.com/Azure/azure-service-operator/pkg/secrets" @@ -34,32 +34,11 @@ func getNetworkInterfaceClient() vnetwork.InterfacesClient { return nicClient } -func MakeResourceId(subscriptionId string, resourceGroupName string, provider string, resourceType string, resourceName string, subResourceType string, subResourceName string) string { - // Sample 1: /subscriptions/88fd8cb2-8248-499e-9a2d-4929a4b0133c/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.Network/publicIPAddresses/azurepublicipaddress-sample-3 - // Sample 2: /subscriptions/88fd8cb2-8248-499e-9a2d-4929a4b0133c/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.Network/virtualNetworks/vnet-sample-hpf-1/subnets/test2 - segments := []string{ - "subscriptions", - subscriptionId, - "resourceGroups", - resourceGroupName, - "providers", - provider, - resourceType, - resourceName, - subResourceType, - subResourceName, - } - - result := "/" + strings.Join(segments, "/") - - return result -} - func (m *AzureNetworkInterfaceClient) CreateNetworkInterface(ctx context.Context, location string, resourceGroupName string, resourceName string, vnetName string, subnetName string, publicIPAddressName string) (future vnetwork.InterfacesCreateOrUpdateFuture, err error) { client := getNetworkInterfaceClient() - subnetIDInput := MakeResourceId( + subnetIDInput := helpers.MakeResourceID( client.SubscriptionID, resourceGroupName, "Microsoft.Network", @@ -69,7 +48,7 @@ func (m *AzureNetworkInterfaceClient) CreateNetworkInterface(ctx context.Context subnetName, ) - publicIPAddressIDInput := MakeResourceId( + publicIPAddressIDInput := helpers.MakeResourceID( client.SubscriptionID, resourceGroupName, "Microsoft.Network", diff --git a/pkg/resourcemanager/nic/manager.go b/pkg/resourcemanager/nic/manager.go index da8a7754b79..32a9f649bcf 100644 --- a/pkg/resourcemanager/nic/manager.go +++ b/pkg/resourcemanager/nic/manager.go @@ -23,7 +23,7 @@ type NetworkInterfaceManager interface { resourceName string, resourceGroupName string) (string, error) - GetNetwork(ctx context.Context, + GetNetworkInterface(ctx context.Context, resourceGroupName string, resourceName string) (network.Interface, error) diff --git a/pkg/resourcemanager/vm/client.go b/pkg/resourcemanager/vm/client.go new file mode 100644 index 00000000000..da6b737c393 --- /dev/null +++ b/pkg/resourcemanager/vm/client.go @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package vm + +import ( + "context" + "fmt" + "strings" + + compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/helpers" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" + "github.com/Azure/azure-service-operator/pkg/secrets" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +type AzureVirtualMachineClient struct { + SecretClient secrets.SecretClient + Scheme *runtime.Scheme +} + +func NewAzureVirtualMachineClient(secretclient secrets.SecretClient, scheme *runtime.Scheme) *AzureVirtualMachineClient { + return &AzureVirtualMachineClient{ + SecretClient: secretclient, + Scheme: scheme, + } +} + +func getVirtualMachineClient() compute.VirtualMachinesClient { + computeClient := compute.NewVirtualMachinesClientWithBaseURI(config.BaseURI(), config.SubscriptionID()) + a, _ := iam.GetResourceManagementAuthorizer() + computeClient.Authorizer = a + computeClient.AddToUserAgent(config.UserAgent()) + return computeClient +} + +func (m *AzureVirtualMachineClient) CreateVirtualMachine(ctx context.Context, location string, resourceGroupName string, resourceName string, vmSize string, osType string, adminUserName string, adminPassword string, sshPublicKeyData string, networkInterfaceName string, platformImageURN string) (future compute.VirtualMachinesCreateOrUpdateFuture, err error) { + + client := getVirtualMachineClient() + + vmSizeInput := compute.VirtualMachineSizeTypes(vmSize) + provisionVMAgent := true + platformImageUrnTokens := strings.Split(platformImageURN, ":") + + adminPasswordInput := "" + adminPasswordBase64Decoded := helpers.FromBase64EncodedString(adminPassword) + if adminPasswordBase64Decoded != "" { + adminPasswordInput = adminPasswordBase64Decoded + } + + addAsPrimaryNic := true + nicIDInput := helpers.MakeResourceID( + client.SubscriptionID, + resourceGroupName, + "Microsoft.Network", + "networkInterfaces", + networkInterfaceName, + "", + "", + ) + + nicsToAdd := []compute.NetworkInterfaceReference{ + compute.NetworkInterfaceReference{ + ID: &nicIDInput, + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: &addAsPrimaryNic, + }, + }, + } + + sshKeyPath := fmt.Sprintf("/home/%s/.ssh/authorized_keys", adminUserName) + sshKeysToAdd := []compute.SSHPublicKey{ + compute.SSHPublicKey{ + Path: &sshKeyPath, + KeyData: &sshPublicKeyData, + }, + } + linuxProfile := compute.OSProfile{ + ComputerName: &resourceName, + AdminUsername: &adminUserName, + AdminPassword: &adminPasswordInput, + LinuxConfiguration: &compute.LinuxConfiguration{ + SSH: &compute.SSHConfiguration{ + PublicKeys: &sshKeysToAdd, + }, + ProvisionVMAgent: &provisionVMAgent, + }, + } + + windowsProfile := compute.OSProfile{ + ComputerName: &resourceName, + AdminUsername: &adminUserName, + AdminPassword: &adminPasswordInput, + WindowsConfiguration: &compute.WindowsConfiguration{ + ProvisionVMAgent: &provisionVMAgent, + }, + } + + osProfile := linuxProfile + if osType == "Windows" { + osProfile = windowsProfile + } + + future, err = client.CreateOrUpdate( + ctx, + resourceGroupName, + resourceName, + compute.VirtualMachine{ + Location: &location, + VirtualMachineProperties: &compute.VirtualMachineProperties{ + HardwareProfile: &compute.HardwareProfile{ + VMSize: vmSizeInput, + }, + StorageProfile: &compute.StorageProfile{ + ImageReference: &compute.ImageReference{ + Publisher: &platformImageUrnTokens[0], + Offer: &platformImageUrnTokens[1], + Sku: &platformImageUrnTokens[2], + Version: &platformImageUrnTokens[3], + }, + }, + OsProfile: &osProfile, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &nicsToAdd, + }, + }, + }, + ) + + return future, err +} + +func (m *AzureVirtualMachineClient) DeleteVirtualMachine(ctx context.Context, vmName string, resourcegroup string) (status string, err error) { + + client := getVirtualMachineClient() + + _, err = client.Get(ctx, resourcegroup, vmName, "") + if err == nil { // vm present, so go ahead and delete + future, err := client.Delete(ctx, resourcegroup, vmName) + return future.Status(), err + } + // VM not present so return success anyway + return "VM not present", nil + +} + +func (m *AzureVirtualMachineClient) GetVirtualMachine(ctx context.Context, resourcegroup string, vmName string) (vm compute.VirtualMachine, err error) { + + client := getVirtualMachineClient() + + return client.Get(ctx, resourcegroup, vmName, "") +} + +func (p *AzureVirtualMachineClient) AddVirtualMachineCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.AzureVirtualMachine) error { + key := types.NamespacedName{ + Name: secretName, + Namespace: instance.Namespace, + } + + err := p.SecretClient.Upsert(ctx, + key, + data, + secrets.WithOwner(instance), + secrets.WithScheme(p.Scheme), + ) + if err != nil { + return err + } + + return nil +} + +func (p *AzureVirtualMachineClient) GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.AzureVirtualMachine) (map[string][]byte, error) { + name := instance.Name + + secret := map[string][]byte{} + + key := types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := p.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } + + randomPassword := helpers.NewPassword() + secret["password"] = []byte(randomPassword) + + return secret, nil +} diff --git a/pkg/resourcemanager/vm/manager.go b/pkg/resourcemanager/vm/manager.go new file mode 100644 index 00000000000..d078d3dd60e --- /dev/null +++ b/pkg/resourcemanager/vm/manager.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package vm + +import ( + "context" + + compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute" + "github.com/Azure/azure-service-operator/pkg/resourcemanager" +) + +type VirtualMachineManager interface { + CreateVirtualMachine(ctx context.Context, + location string, + resourceGroupName string, + resourceName string, + vmSize string, + osType string, + adminUserName string, + adminPassword string, + sshPublicKeyData string, + networkInterfaceName string, + platformImageURN string) (compute.VirtualMachine, error) + + DeleteVirtualMachine(ctx context.Context, + resourceName string, + resourceGroupName string) (string, error) + + GetVirtualMachine(ctx context.Context, + resourceGroupName string, + resourceName string) (compute.VirtualMachine, error) + + // also embed async client methods + resourcemanager.ARMClient +} diff --git a/pkg/resourcemanager/vm/reconcile.go b/pkg/resourcemanager/vm/reconcile.go new file mode 100644 index 00000000000..ec9e5153e76 --- /dev/null +++ b/pkg/resourcemanager/vm/reconcile.go @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package vm + +import ( + "context" + "fmt" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + "github.com/Azure/azure-service-operator/pkg/helpers" + "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +func (g *AzureVirtualMachineClient) Ensure(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { + + instance, err := g.convert(obj) + if err != nil { + return true, err + } + + client := getVirtualMachineClient() + + location := instance.Spec.Location + resourceGroup := instance.Spec.ResourceGroup + resourceName := instance.Name + vmSize := instance.Spec.VMSize + osType := instance.Spec.OSType + adminUserName := instance.Spec.AdminUserName + sshPublicKeyData := instance.Spec.SSHPublicKeyData + nicName := instance.Spec.NetworkInterfaceName + imageURN := instance.Spec.PlatformImageURN + + // Check to see if secret exists and if yes retrieve the admin login and password + secret, err := g.GetOrPrepareSecret(ctx, instance) + if err != nil { + return false, err + } + // Update secret + err = g.AddVirtualMachineCredsToSecrets(ctx, instance.Name, secret, instance) + if err != nil { + return false, err + } + + adminPassword := string(secret["password"]) + + instance.Status.Provisioning = true + // Check if this item already exists. This is required + // to overcome the issue with the lack of idempotence of the Create call + item, err := g.GetVirtualMachine(ctx, resourceGroup, resourceName) + if err == nil { + instance.Status.Provisioned = true + instance.Status.Provisioning = false + instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.ResourceId = *item.ID + return true, nil + } + future, err := g.CreateVirtualMachine( + ctx, + location, + resourceGroup, + resourceName, + vmSize, + string(osType), + adminUserName, + adminPassword, + sshPublicKeyData, + nicName, + imageURN, + ) + if err != nil { + // let the user know what happened + instance.Status.Message = err.Error() + instance.Status.Provisioning = false + // errors we expect might happen that we are ok with waiting for + catch := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + errhelp.NotFoundErrorCode, + errhelp.AsyncOpIncompleteError, + errhelp.ResourceNotFound, + errhelp.InvalidResourceReference, + } + + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(catch, azerr.Type) { + // most of these error technically mean the resource is actually not provisioning + switch azerr.Type { + case errhelp.AsyncOpIncompleteError: + instance.Status.Provisioning = true + } + // reconciliation is not done but error is acceptable + return false, nil + } + // reconciliation not done and we don't know what happened + return false, err + } + + _, err = future.Result(client) + if err != nil { + // let the user know what happened + instance.Status.Message = err.Error() + instance.Status.Provisioning = false + // errors we expect might happen that we are ok with waiting for + catch := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + errhelp.NotFoundErrorCode, + errhelp.AsyncOpIncompleteError, + errhelp.SubscriptionDoesNotHaveServer, + } + + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(catch, azerr.Type) { + // most of these error technically mean the resource is actually not provisioning + switch azerr.Type { + case errhelp.AsyncOpIncompleteError: + instance.Status.Provisioning = true + } + // reconciliation is not done but error is acceptable + return false, nil + } + // reconciliation not done and we don't know what happened + return false, err + } + + if instance.Status.Provisioning { + instance.Status.Provisioned = true + instance.Status.Provisioning = false + instance.Status.Message = resourcemanager.SuccessMsg + } else { + instance.Status.Provisioned = false + instance.Status.Provisioning = true + } + return true, nil +} + +func (g *AzureVirtualMachineClient) Delete(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { + + instance, err := g.convert(obj) + if err != nil { + return true, err + } + + resourceGroup := instance.Spec.ResourceGroup + resourceName := instance.Name + + status, err := g.DeleteVirtualMachine( + ctx, + resourceName, + resourceGroup, + ) + if err != nil { + if !errhelp.IsAsynchronousOperationNotComplete(err) { + return true, err + } + } + + if err == nil { + if status != "InProgress" { + return false, nil + } + } + + return true, nil +} +func (g *AzureVirtualMachineClient) GetParents(obj runtime.Object) ([]resourcemanager.KubeParent, error) { + + instance, err := g.convert(obj) + if err != nil { + return nil, err + } + + return []resourcemanager.KubeParent{ + { + Key: types.NamespacedName{ + Namespace: instance.Namespace, + Name: instance.Spec.ResourceGroup, + }, + Target: &azurev1alpha1.ResourceGroup{}, + }, + }, nil +} + +func (g *AzureVirtualMachineClient) GetStatus(obj runtime.Object) (*azurev1alpha1.ASOStatus, error) { + + instance, err := g.convert(obj) + if err != nil { + return nil, err + } + return &instance.Status, nil +} + +func (g *AzureVirtualMachineClient) convert(obj runtime.Object) (*azurev1alpha1.AzureVirtualMachine, error) { + local, ok := obj.(*azurev1alpha1.AzureVirtualMachine) + if !ok { + return nil, fmt.Errorf("failed type assertion on kind: %s", obj.GetObjectKind().GroupVersionKind().String()) + } + return local, nil +} From bf4decd5c929dfa610f9102e646f3ebbd6e438f1 Mon Sep 17 00:00:00 2001 From: Mel Rush Date: Fri, 17 Apr 2020 12:18:20 -0600 Subject: [PATCH 79/99] Add ResourceID to VirtualNetwork (#952) * fixing get return values for resourceid * removing test vars * false nil at end of ensure * get name change * method header comment fix Co-authored-by: Erin Corson --- pkg/resourcemanager/vnet/reconcile.go | 19 ++++++++----------- pkg/resourcemanager/vnet/vnet.go | 19 ++++--------------- pkg/resourcemanager/vnet/vnet_manager.go | 4 ++-- pkg/resourcemanager/vnet/vnet_test.go | 12 ++++++------ 4 files changed, 20 insertions(+), 34 deletions(-) diff --git a/pkg/resourcemanager/vnet/reconcile.go b/pkg/resourcemanager/vnet/reconcile.go index 81355dd3079..931147dbfd7 100644 --- a/pkg/resourcemanager/vnet/reconcile.go +++ b/pkg/resourcemanager/vnet/reconcile.go @@ -29,18 +29,21 @@ func (g *AzureVNetManager) Ensure(ctx context.Context, obj runtime.Object, opts addressSpace := instance.Spec.AddressSpace subnets := instance.Spec.Subnets - instance.Status.Provisioning = true - instance.Status.Provisioned = false - // check first to see if the VNet exists, if it does, dont create it and // consider the reconcilliation successful - if exists, _ := g.VNetExists(ctx, resourceGroup, resourceName); exists { + vNet, err := g.GetVNet(ctx, resourceGroup, resourceName) + if err == nil { + // succeeded! end reconcilliation successfully instance.Status.Provisioning = false instance.Status.Provisioned = true instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.ResourceId = *vNet.ID return true, nil } + instance.Status.Provisioning = true + instance.Status.Provisioned = false + _, err = g.CreateVNet( ctx, location, @@ -84,13 +87,7 @@ func (g *AzureVNetManager) Ensure(ctx context.Context, obj runtime.Object, opts return false, fmt.Errorf("Error creating VNet: %s, %s - %v", resourceGroup, resourceName, err) } - // success - instance.Status.Message = resourcemanager.SuccessMsg - instance.Status.Provisioning = false - instance.Status.Provisioned = true - instance.Status.Message = resourcemanager.SuccessMsg - - return true, nil + return false, nil } // Delete makes sure that the VNet has been deleted diff --git a/pkg/resourcemanager/vnet/vnet.go b/pkg/resourcemanager/vnet/vnet.go index 7dd91ac934c..00e43f907d4 100644 --- a/pkg/resourcemanager/vnet/vnet.go +++ b/pkg/resourcemanager/vnet/vnet.go @@ -92,23 +92,12 @@ func (_ *AzureVNetManager) DeleteVNet(ctx context.Context, resourceGroupName str return future.Result(client) } -// VNetExists checks to see if a VNet exists -func (_ *AzureVNetManager) VNetExists(ctx context.Context, resourceGroupName string, resourceName string) (bool, error) { +// GetVNet gets a VNet +func (v *AzureVNetManager) GetVNet(ctx context.Context, resourceGroupName string, resourceName string) (vNet vnetwork.VirtualNetwork, err error) { client, err := getVNetClient() if err != nil { - return false, err - } - - result, err := client.Get( - ctx, - resourceGroupName, - resourceName, - "") - if err != nil { - return false, err - } else if result.Name == nil { - return false, nil + return vnetwork.VirtualNetwork{}, err } - return true, nil + return client.Get(ctx, resourceGroupName, resourceName, "") } diff --git a/pkg/resourcemanager/vnet/vnet_manager.go b/pkg/resourcemanager/vnet/vnet_manager.go index dd9bd6b8444..246d4986bfe 100644 --- a/pkg/resourcemanager/vnet/vnet_manager.go +++ b/pkg/resourcemanager/vnet/vnet_manager.go @@ -30,9 +30,9 @@ type VNetManager interface { resourceGroupName string, resourceName string) (autorest.Response, error) - VNetExists(ctx context.Context, + GetVNet(ctx context.Context, resourceGroupName string, - resourceName string) (bool, error) + resourceName string) (vnetwork.VirtualNetwork, error) // also embed async client methods resourcemanager.ARMClient diff --git a/pkg/resourcemanager/vnet/vnet_test.go b/pkg/resourcemanager/vnet/vnet_test.go index df442c02917..320cff05984 100644 --- a/pkg/resourcemanager/vnet/vnet_test.go +++ b/pkg/resourcemanager/vnet/vnet_test.go @@ -42,11 +42,11 @@ var _ = Describe("VNet", func() { // Create vnet instance Eventually(func() bool { time.Sleep(3 * time.Second) - exists, _ := vnetManager.VNetExists(ctx, rgName, vnetName) - if exists { + _, err := vnetManager.GetVNet(ctx, rgName, vnetName) + if err == nil { return true } - _, err := vnetManager.CreateVNet(ctx, location, rgName, vnetName, addressSpace, []azurev1alpha1.VNetSubnets{ + _, err = vnetManager.CreateVNet(ctx, location, rgName, vnetName, addressSpace, []azurev1alpha1.VNetSubnets{ azurev1alpha1.VNetSubnets{ SubnetName: subnetName, SubnetAddressPrefix: subnetPrefix, @@ -68,11 +68,11 @@ var _ = Describe("VNet", func() { // Delete vnet instance Eventually(func() bool { time.Sleep(3 * time.Second) - exists, _ := vnetManager.VNetExists(ctx, rgName, vnetName) - if !exists { + _, err := vnetManager.GetVNet(ctx, rgName, vnetName) + if err != nil { return true } - _, err := vnetManager.DeleteVNet(ctx, rgName, vnetName) + _, err = vnetManager.DeleteVNet(ctx, rgName, vnetName) if err != nil { fmt.Println(err.Error()) if !errhelp.IsAsynchronousOperationNotComplete(err) { From 7ae0ac45cf505691cc9c8f0787ce6efb02635aa8 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Fri, 17 Apr 2020 12:14:18 -0700 Subject: [PATCH 80/99] Update docs/apimgmt/apimgmt.md Co-Authored-By: Janani Vasudevan <49576785+jananivMS@users.noreply.github.com> --- docs/apimgmt/apimgmt.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index e76c8fd4875..305ae941ba6 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -36,7 +36,8 @@ The spec consists of the following fields: * `properties` * `apiRevision` Describes the Revision of the Api. If no value is provided, default revision 1 is created * `apiRevisionDescription` Description of the Api Revision. - * `apiVersionSet` APIVersionSetContractDetails an API Version Set contains the common configuration for a set of API versions. + * `apiVersionSet` API Version Set contains the common configuration for a set of API versions. + * `apiVersion` Indicates the Version identifier of the API if the API is versioned * `id` Identifier for existing API Version Set. Omit this value to create a new Version Set. * `name` The display Name of the API Version Set. * `description` Description of API Version Set. From 00681af9f5c6d331efa6fc7b592d4d0917c3bfe4 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Fri, 17 Apr 2020 12:48:02 -0700 Subject: [PATCH 81/99] Update apimgmt.md --- docs/apimgmt/apimgmt.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index 305ae941ba6..e22447b2104 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -34,14 +34,14 @@ The spec consists of the following fields: * `apiService` The name of the API Management service to manage * `apiId` Specify an ID for the API * `properties` - * `apiRevision` Describes the Revision of the Api. If no value is provided, default revision 1 is created - * `apiRevisionDescription` Description of the Api Revision. - * `apiVersionSet` API Version Set contains the common configuration for a set of API versions. + * `apiRevision` Describes the Revision of the API. If no value is provided, default revision 1 is created + * `apiRevisionDescription` Description of the API Revision * `apiVersion` Indicates the Version identifier of the API if the API is versioned + * `apiVersionDescription` Description of the API Version + * `apiVersionSet` API Version Set contains the common configuration for a set of API versions * `id` Identifier for existing API Version Set. Omit this value to create a new Version Set. * `name` The display Name of the API Version Set. * `description` Description of API Version Set. - * `apiVersionDescription` Description of the API Version * `apiVersionSetID` A resource identifier for the related ApiVersionSet * `description` Description of the API * `displayName` Display name for the API. Must be 1 to 300 characters long From ea4f61d8fca7e9c56efc0574f2d2665828d46035 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Fri, 17 Apr 2020 12:59:15 -0700 Subject: [PATCH 82/99] Update apimgmt.md --- docs/apimgmt/apimgmt.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/apimgmt/apimgmt.md b/docs/apimgmt/apimgmt.md index e22447b2104..bcf9a46cf90 100644 --- a/docs/apimgmt/apimgmt.md +++ b/docs/apimgmt/apimgmt.md @@ -10,6 +10,8 @@ Learn more about Azure API Management [here](https://docs.microsoft.com/en-us/az The API Management Service deploys an API Management instance into a specified resource group at the specified location. It also provides the option to link to an Application Insights instance for logging, and to place the API Management instance in a specified Virtual Network. +Here is a [sample YAML](/config/samples/azure_v1alpha1_apimservice.yaml) to provision an API Management Service. + The spec consists of the following fields: #### Required Fields @@ -29,6 +31,8 @@ The spec consists of the following fields: The API Management API Operator creates an API with the specified properties in the specified API Management service. +Here is a [sample YAML](/config/samples/azure_v1alpha1_apimgmtapi.yaml) to provision an API Management API. + The spec consists of the following fields: * `apiService` The name of the API Management service to manage From 4fc8814f19901d8c5d51622ee9330dd940c253c4 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Fri, 17 Apr 2020 16:09:54 -0600 Subject: [PATCH 83/99] adding temporary logging to diagnose CI test failures --- .../cosmosdbs/cosmosdb_reconcile.go | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 9533331f459..13efe1dd368 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -6,6 +6,7 @@ package cosmosdbs import ( "context" "fmt" + "log" "strings" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -226,6 +227,15 @@ func (m *AzureCosmosDBManager) convert(obj runtime.Object) (*v1alpha1.CosmosDB, func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error { result, err := m.ListKeys(ctx, instance.Spec.ResourceGroup, instance.ObjectMeta.Name) if err != nil { + //TODO: remove before completing pull request + log.Printf( + `Failed to list CosmosDB Account Keys: + Status: %v + Error: %v + `, + instance.Status, + err, + ) return err } @@ -240,7 +250,21 @@ func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Conte "secondaryReadonlyMasterKey": []byte(*result.SecondaryReadonlyMasterKey), } - return m.SecretClient.Upsert(ctx, secretKey, secretData) + err = m.SecretClient.Upsert(ctx, secretKey, secretData) + if err != nil { + //TODO: remove before completing pull request + log.Printf( + `Failed to upsert CosmosDB Account Keys: + Status: %v + Error: %v + `, + instance.Status, + err, + ) + return err + } + + return nil } func (m *AzureCosmosDBManager) deleteAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error { From a83bff64aac046e44456595b8c31b672c8f32938 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Fri, 17 Apr 2020 16:56:47 -0600 Subject: [PATCH 84/99] adding more temporary logging --- controllers/cosmosdb_controller_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index 31ad06cb5e7..2e43c40123e 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -7,6 +7,7 @@ package controllers import ( "context" + "log" "testing" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -48,6 +49,12 @@ func TestCosmosDBHappyPath(t *testing.T) { assert.Eventually(func() bool { secret, err := tc.secretClient.Get(ctx, key) + if err != nil { + log.Printf("CosmosDB Get Secret Failed: %v\n", err) + } + if len(secret) <= 0 { + log.Printf("CosmosDB Length Secret Failed: %v\n", secret) + } return err == nil && len(secret) > 0 }, tc.timeoutFast, tc.retry, "wait for cosmosdb to have secret") From 1b96561aadb3deca8965d9c80bf1bbdf001393f5 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Fri, 17 Apr 2020 17:02:40 -0600 Subject: [PATCH 85/99] updated to address PR comments --- .../azuresqluser/azuresqluser_reconcile.go | 1 - pkg/resourcemanager/mysql/server/client.go | 7 +- pkg/resourcemanager/mysql/server/manager.go | 2 +- pkg/resourcemanager/mysql/server/reconcile.go | 82 ++++++++++++++----- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 649752a4d96..3540721b6e0 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -277,7 +277,6 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op err = s.GrantUserRoles(ctx, user, instance.Spec.Roles, db) if err != nil { - fmt.Println(err) instance.Status.Message = "GrantUserRoles failed" return false, fmt.Errorf("GrantUserRoles failed") } diff --git a/pkg/resourcemanager/mysql/server/client.go b/pkg/resourcemanager/mysql/server/client.go index 8e26294dc5c..22f78de705d 100644 --- a/pkg/resourcemanager/mysql/server/client.go +++ b/pkg/resourcemanager/mysql/server/client.go @@ -61,14 +61,14 @@ func (m *MySQLServerClient) CheckServerNameAvailability(ctx context.Context, ser } -func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (server mysql.Server, err error) { +func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (pollingURL string, server mysql.Server, err error) { client := getMySQLServersClient() // Check if name is valid if this is the first create call valid, err := m.CheckServerNameAvailability(ctx, servername) if !valid { - return server, err + return "", server, err } var result mysql.ServersCreateFuture if strings.EqualFold(createmode, "replica") { @@ -108,7 +108,8 @@ func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername ) } - return result.Result(client) + res, err := result.Result(client) + return result.PollingURL(), res, err } diff --git a/pkg/resourcemanager/mysql/server/manager.go b/pkg/resourcemanager/mysql/server/manager.go index 350a6450476..f90b58e5c70 100644 --- a/pkg/resourcemanager/mysql/server/manager.go +++ b/pkg/resourcemanager/mysql/server/manager.go @@ -11,7 +11,7 @@ import ( ) type MySQLServerManager interface { - CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (mysql.Server, error) + CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (pollingURL string, server mysql.Server, err error) DeleteServer(ctx context.Context, resourcegroup string, servername string) (string, error) GetServer(ctx context.Context, resourcegroup string, servername string) (mysql.Server, error) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 7b36c556346..c8a75490add 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/pollclient" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest/to" "k8s.io/apimachinery/pkg/runtime" @@ -65,11 +66,40 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // Check if this server already exists and its state if it does. This is required // to overcome the issue with the lack of idempotence of the Create call + hash := "" server, err := m.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name) - if err == nil { + if err != nil { + // handle failures in the async operation + if instance.Status.PollingURL != "" { + pClient := pollclient.NewPollClient() + res, err := pClient.Get(ctx, instance.Status.PollingURL) + if err != nil { + instance.Status.Provisioning = false + return false, err + } + + if res.Status == "Failed" { + instance.Status.Provisioning = false + instance.Status.RequestedAt = nil + ignore := []string{ + errhelp.SubscriptionDoesNotHaveServer, + errhelp.ServiceBusy, + } + if !helpers.ContainsString(ignore, res.Error.Code) { + instance.Status.Message = res.Error.Error() + return true, nil + } + } + } + } else { instance.Status.State = string(server.UserVisibleState) - if server.UserVisibleState == mysql.ServerStateReady { + hash = helpers.Hash256(instance.Spec) + if instance.Status.SpecHash == hash && (instance.Status.Provisioned || instance.Status.FailedProvisioning) { + instance.Status.RequestedAt = nil + return true, nil + } + if server.UserVisibleState == mysql.ServerStateReady { // Update secret with FQ name of the server. We ignore the error. m.UpdateServerNameInSecret(ctx, instance.Name, secret, *server.FullyQualifiedDomainName, instance) @@ -78,9 +108,9 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *server.ID instance.Status.State = string(server.UserVisibleState) + instance.Status.SpecHash = hash return true, nil } - return false, nil } // if the create has been sent with no error we need to wait before calling it again @@ -99,7 +129,7 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts Family: to.StringPtr(instance.Spec.Sku.Family), } - server, err = m.CreateServerIfValid( + pollURL, server, err := m.CreateServerIfValid( ctx, instance.Name, instance.Spec.ResourceGroup, @@ -119,20 +149,35 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.Provisioning = false azerr := errhelp.NewAzureErrorAzureError(err) - switch azerr.Type { - case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: - // errors we expect might happen that we are ok with waiting for + catchRequeue := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + errhelp.AsyncOpIncompleteError, + errhelp.SubscriptionDoesNotHaveServer, + errhelp.ServiceBusy, + } + catchUnrecoverable := []string{ + errhelp.ProvisioningDisabled, + errhelp.LocationNotAvailableForResourceType, + errhelp.InvalidRequestContent, + errhelp.InternalServerError, + } + + // handle the errors + if helpers.ContainsString(catchRequeue, azerr.Type) { + if azerr.Type == errhelp.AsyncOpIncompleteError { + instance.Status.Provisioning = true + instance.Status.PollingURL = pollURL + } return false, nil - case errhelp.ProvisioningDisabled, errhelp.LocationNotAvailableForResourceType, errhelp.InvalidRequestContent, errhelp.InternalServerError: + } + + if helpers.ContainsString(catchUnrecoverable, azerr.Type) { // Unrecoverable error, so stop reconcilation instance.Status.Message = "Reconcilation hit unrecoverable error: " + errhelp.StripErrorIDs(err) return true, nil - case errhelp.AsyncOpIncompleteError: - // Creation in progress - instance.Status.Provisioning = true - instance.Status.Message = "Server request submitted to Azure" - return false, nil } + // reconciliation not done and we don't know what happened return false, err } @@ -272,13 +317,12 @@ func (m *MySQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *az var Username string var Password string - // See if secret already exists and return if it does - key = types.NamespacedName{Name: name, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil - } - if strings.EqualFold(createmode, "default") { // new Mysql server creation + // See if secret already exists and return if it does + key = types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } // Generate random username password if secret does not exist already Username = helpers.GenerateRandomUsername(10) Password = helpers.NewPassword() From b7b151623074ade5cfc16318a03b72820d9e633c Mon Sep 17 00:00:00 2001 From: jpflueger Date: Fri, 17 Apr 2020 17:07:37 -0600 Subject: [PATCH 86/99] adding more diagnostic logging --- controllers/cosmosdb_controller_test.go | 23 +++++++++++++++---- .../cosmosdbs/cosmosdb_reconcile.go | 2 ++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index 2e43c40123e..7b5aaa31c7a 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -49,11 +49,24 @@ func TestCosmosDBHappyPath(t *testing.T) { assert.Eventually(func() bool { secret, err := tc.secretClient.Get(ctx, key) - if err != nil { - log.Printf("CosmosDB Get Secret Failed: %v\n", err) - } - if len(secret) <= 0 { - log.Printf("CosmosDB Length Secret Failed: %v\n", secret) + cond := err == nil && len(secret) > 0 + if !cond { + //TODO: remove before completing pull request + tc.k8sClient.Get(ctx, key, dbInstance) + log.Printf( + `----- COSMOSDB ----- + Secret: %v + Assertion: %v + Error: %v + Secret: %v + Status: %v + `, + key, + cond, + err, + secret, + dbInstance.Status, + ) } return err == nil && len(secret) > 0 }, tc.timeoutFast, tc.retry, "wait for cosmosdb to have secret") diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 13efe1dd368..26d6c4f280d 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -264,6 +264,8 @@ func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Conte return err } + //TODO: remove before completing pull request + log.Printf("Successfully upserted CosmosDB Account Keys: %v\n", secretKey) return nil } From 348f49f295989549ce22503fdd3b18c9e9fee1ec Mon Sep 17 00:00:00 2001 From: jpflueger Date: Fri, 17 Apr 2020 19:49:03 -0600 Subject: [PATCH 87/99] adding more logging and moving secret creation --- .../cosmosdbs/cosmosdb_reconcile.go | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 26d6c4f280d..18ec817e172 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -66,17 +66,22 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return false, nil } - if instance.Status.State == "Succeeded" && instance.Status.SpecHash == hash { + if instance.Status.State == "Succeeded" { + log.Println("Hit Get success path") + // provisioning is complete, update the secrets if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil { instance.Status.Message = err.Error() return false, err } - instance.Status.Message = resourcemanager.SuccessMsg - instance.Status.Provisioning = false - instance.Status.Provisioned = true - return true, nil + if instance.Status.SpecHash == hash { + log.Println("Hit Get + SpecHash success path") + instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.Provisioning = false + instance.Status.Provisioned = true + return true, nil + } } if instance.Status.State == "Failed" { @@ -105,6 +110,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o instance.Status.State = "Creating" instance.Status.Message = "Resource request successfully submitted to Azure" instance.Status.SpecHash = hash + return false, nil case errhelp.InvalidResourceLocation, errhelp.LocationNotAvailableForResourceType: instance.Status.Provisioning = false instance.Status.Message = azerr.Error() @@ -123,9 +129,16 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o } } - return false, nil + log.Printf("Hit unhandled error case: %v\n", err) + return false, err + } + + if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil { + instance.Status.Message = err.Error() + return false, err } + log.Println("Hit terminal success case.") instance.Status.SpecHash = hash instance.Status.ResourceId = *db.ID instance.Status.State = *db.ProvisioningState From dac4cb242b2df79dfd7eca9902fbde6d552ee209 Mon Sep 17 00:00:00 2001 From: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Date: Sat, 18 Apr 2020 06:49:20 -0700 Subject: [PATCH 88/99] first (#944) Co-authored-by: William Mortl --- .../azuresqlaction_reconcile.go | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go index 0dbc30db6d1..7cc5a5615cd 100644 --- a/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlaction/azuresqlaction_reconcile.go @@ -67,11 +67,17 @@ func (s *AzureSqlActionManager) Ensure(ctx context.Context, obj runtime.Object, if helpers.ContainsString(catch, azerr.Type) { return false, nil //requeue until server/RG ready } - return true, nil // unrecoverable error + + // unrecoverable error + instance.Status.Provisioned = false + instance.Status.Provisioning = false + instance.Status.FailedProvisioning = true + return true, nil } instance.Status.Provisioned = true instance.Status.Provisioning = false + instance.Status.FailedProvisioning = false instance.Status.Message = resourcemanager.SuccessMsg } @@ -111,16 +117,32 @@ func (s *AzureSqlActionManager) Ensure(ctx context.Context, obj runtime.Object, err := s.UpdateUserPassword(ctx, groupName, serverName, instance.Spec.DbUser, instance.Spec.DbName, adminKey, adminSecretClient, userSecretClient) if err != nil { - instance.Status.Message = err.Error() - return true, nil // unrecoverable error + instance.Status.Message = errhelp.StripErrorIDs(err) + + // catch firewall issue - keep cycling until it clears up + if strings.Contains(err.Error(), "create a firewall rule for this IP address") { + instance.Status.Provisioned = false + instance.Status.Provisioning = false + return false, nil + } + + // unrecoverable error + instance.Status.Provisioned = false + instance.Status.Provisioning = false + instance.Status.FailedProvisioning = true + return true, nil } instance.Status.Provisioned = true instance.Status.Provisioning = false + instance.Status.FailedProvisioning = false instance.Status.Message = resourcemanager.SuccessMsg } } else { instance.Status.Message = "Unrecognized action" + instance.Status.Provisioned = false + instance.Status.Provisioning = false + instance.Status.FailedProvisioning = true } return true, nil From 421942eab99055b2c1da3efc62368d4f654bc3c1 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Sun, 19 Apr 2020 23:52:27 -0600 Subject: [PATCH 89/99] removing diagnostic logging --- controllers/cosmosdb_controller_test.go | 20 -------------- .../cosmosdbs/cosmosdb_reconcile.go | 26 ------------------- 2 files changed, 46 deletions(-) diff --git a/controllers/cosmosdb_controller_test.go b/controllers/cosmosdb_controller_test.go index 7b5aaa31c7a..31ad06cb5e7 100644 --- a/controllers/cosmosdb_controller_test.go +++ b/controllers/cosmosdb_controller_test.go @@ -7,7 +7,6 @@ package controllers import ( "context" - "log" "testing" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -49,25 +48,6 @@ func TestCosmosDBHappyPath(t *testing.T) { assert.Eventually(func() bool { secret, err := tc.secretClient.Get(ctx, key) - cond := err == nil && len(secret) > 0 - if !cond { - //TODO: remove before completing pull request - tc.k8sClient.Get(ctx, key, dbInstance) - log.Printf( - `----- COSMOSDB ----- - Secret: %v - Assertion: %v - Error: %v - Secret: %v - Status: %v - `, - key, - cond, - err, - secret, - dbInstance.Status, - ) - } return err == nil && len(secret) > 0 }, tc.timeoutFast, tc.retry, "wait for cosmosdb to have secret") diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 18ec817e172..bd44b45a448 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -6,7 +6,6 @@ package cosmosdbs import ( "context" "fmt" - "log" "strings" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -67,8 +66,6 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o } if instance.Status.State == "Succeeded" { - log.Println("Hit Get success path") - // provisioning is complete, update the secrets if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil { instance.Status.Message = err.Error() @@ -76,7 +73,6 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o } if instance.Status.SpecHash == hash { - log.Println("Hit Get + SpecHash success path") instance.Status.Message = resourcemanager.SuccessMsg instance.Status.Provisioning = false instance.Status.Provisioned = true @@ -129,7 +125,6 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o } } - log.Printf("Hit unhandled error case: %v\n", err) return false, err } @@ -138,7 +133,6 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o return false, err } - log.Println("Hit terminal success case.") instance.Status.SpecHash = hash instance.Status.ResourceId = *db.ID instance.Status.State = *db.ProvisioningState @@ -240,15 +234,6 @@ func (m *AzureCosmosDBManager) convert(obj runtime.Object) (*v1alpha1.CosmosDB, func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error { result, err := m.ListKeys(ctx, instance.Spec.ResourceGroup, instance.ObjectMeta.Name) if err != nil { - //TODO: remove before completing pull request - log.Printf( - `Failed to list CosmosDB Account Keys: - Status: %v - Error: %v - `, - instance.Status, - err, - ) return err } @@ -265,20 +250,9 @@ func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Conte err = m.SecretClient.Upsert(ctx, secretKey, secretData) if err != nil { - //TODO: remove before completing pull request - log.Printf( - `Failed to upsert CosmosDB Account Keys: - Status: %v - Error: %v - `, - instance.Status, - err, - ) return err } - //TODO: remove before completing pull request - log.Printf("Successfully upserted CosmosDB Account Keys: %v\n", secretKey) return nil } From fd96971edfb6fda4b733762e7d115401bc80d69e Mon Sep 17 00:00:00 2001 From: hobu <37413937+buhongw7583c@users.noreply.github.com> Date: Mon, 20 Apr 2020 16:43:10 +0800 Subject: [PATCH 90/99] #issue918#addVnetToCosmosDB --- api/v1alpha1/cosmosdb_types.go | 19 +++++++++++++++---- config/samples/azure_v1alpha1_cosmosdb.yaml | 7 +++++++ pkg/resourcemanager/cosmosdbs/cosmosdb.go | 17 ++++++++++++++++- .../cosmosdbs/cosmosdb_manager.go | 2 +- .../cosmosdbs/cosmosdb_reconcile.go | 4 +++- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 34fea0935fc..c36d23e6618 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -17,10 +17,11 @@ type CosmosDBSpec struct { // +kubebuilder:validation:MinLength=0 - Location string `json:"location,omitempty"` - ResourceGroup string `json:"resourceGroup"` - Kind CosmosDBKind `json:"kind,omitempty"` - Properties CosmosDBProperties `json:"properties,omitempty"` + Location string `json:"location,omitempty"` + ResourceGroup string `json:"resourceGroup"` + Kind CosmosDBKind `json:"kind,omitempty"` + Properties CosmosDBProperties `json:"properties,omitempty"` + VirtualNetworkRules *[]CosmosDBVirtualNetworkRule `json:"virtualNetworkRules,omitempty"` } // CosmosDBKind enumerates the values for kind. @@ -41,6 +42,8 @@ const ( type CosmosDBProperties struct { // CosmosDBDatabaseAccountOfferType - The offer type for the Cosmos DB database account. DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` + //IsVirtualNetworkFilterEnabled - Flag to indicate whether to enable/disable Virtual Network ACL rules. + IsVirtualNetworkFilterEnabled bool `json:"isVirtualNetworkFilterEnabled,omitempty"` //Locations []CosmosDBLocation `json:"locations,omitempty"` } @@ -82,6 +85,14 @@ type CosmosDBList struct { Items []CosmosDB `json:"items"` } +//CosmosDBVirtualNetworkRule virtual Network ACL Rule object +type CosmosDBVirtualNetworkRule struct { + // ID - Resource ID of a subnet, for example: /subscriptions/{subscriptionId}/resourceGroups/{groupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. + SubnetID *string `json:"subnetID,omitempty"` + // IgnoreMissingVNetServiceEndpoint - Create firewall rule before the virtual network has vnet service endpoint enabled. + IgnoreMissingVNetServiceEndpoint *bool `json:"ignoreMissingVNetServiceEndpoint,omitempty"` +} + func init() { SchemeBuilder.Register(&CosmosDB{}, &CosmosDBList{}) } diff --git a/config/samples/azure_v1alpha1_cosmosdb.yaml b/config/samples/azure_v1alpha1_cosmosdb.yaml index d3efa498467..9c5c8e22349 100644 --- a/config/samples/azure_v1alpha1_cosmosdb.yaml +++ b/config/samples/azure_v1alpha1_cosmosdb.yaml @@ -8,3 +8,10 @@ spec: resourceGroup: resourcegroup-azure-operators properties: databaseAccountOfferType: Standard +#optional for network rule set +# isVirtualNetworkFilterEnabled: true +# virtualNetworkRules: +# - subnetId: /subscriptions/{subscription_id}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet_name}/subnets/{subnet_name} +# ignoreMissingServiceEndpoint: false + + diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 0d22bb85e85..d838a97dd73 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -42,6 +42,8 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, + vnetEnabled bool, + networkRule *[]v1alpha1.CosmosDBVirtualNetworkRule, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) { cosmosDBClient, err := getCosmosDBClient() if err != nil { @@ -72,6 +74,18 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( locationsArray := []documentdb.Location{ locationObj, } + + vNetRulesSet := []documentdb.VirtualNetworkRule{} + if networkRule != nil { + for _, i := range *networkRule { + subnetID := i.SubnetID + ignoreEndpoint := i.IgnoreMissingVNetServiceEndpoint + vNetRulesSet = append(vNetRulesSet, documentdb.VirtualNetworkRule{ + ID: subnetID, + IgnoreMissingVNetServiceEndpoint: ignoreEndpoint, + }) + } + } createUpdateParams := documentdb.DatabaseAccountCreateUpdateParameters{ Location: to.StringPtr(location), Tags: tags, @@ -82,8 +96,9 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( DatabaseAccountCreateUpdateProperties: &documentdb.DatabaseAccountCreateUpdateProperties{ DatabaseAccountOfferType: &sDBType, EnableMultipleWriteLocations: to.BoolPtr(false), - IsVirtualNetworkFilterEnabled: to.BoolPtr(false), + IsVirtualNetworkFilterEnabled: &vnetEnabled, Locations: &locationsArray, + VirtualNetworkRules: &vNetRulesSet, }, } createUpdateFuture, err := cosmosDBClient.CreateOrUpdate( diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index f916e70fd54..006281fa985 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -21,7 +21,7 @@ func NewAzureCosmosDBManager() *AzureCosmosDBManager { // CosmosDBManager client functions type CosmosDBManager interface { // CreateOrUpdateCosmosDB creates a new cosmos database account - CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) + CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, vnetEnabled bool, networkRule *[]v1alpha1.CosmosDBVirtualNetworkRule, tags map[string]*string) (*documentdb.DatabaseAccount, *errhelp.AzureError) // GetCosmosDB gets a cosmos database account GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, *errhelp.AzureError) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 4eb8f6b0a0f..58831efaf05 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -90,8 +90,10 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o location := instance.Spec.Location kind := instance.Spec.Kind dbType := instance.Spec.Properties.DatabaseAccountOfferType + networkRule := instance.Spec.VirtualNetworkRules + vnetEnabled := instance.Spec.Properties.IsVirtualNetworkFilterEnabled - db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) + db, azerr := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, vnetEnabled, networkRule, tags) // everything is in a created/updated state if azerr == nil { From 631352c6fdd3b08cd510a5a342c86881fe25d6e9 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 20 Apr 2020 11:10:48 -0600 Subject: [PATCH 91/99] updated yaml --- config/samples/azure_v1alpha1_mysqlserver.yaml | 6 +++--- pkg/errhelp/errors.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/samples/azure_v1alpha1_mysqlserver.yaml b/config/samples/azure_v1alpha1_mysqlserver.yaml index d8f43cd3c75..05d543e51f2 100644 --- a/config/samples/azure_v1alpha1_mysqlserver.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver.yaml @@ -11,9 +11,9 @@ spec: infrastructureEncryption: Enabled # Possible values include: Enabled, Disabled createMode: Default # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) sku: - name: B_Gen5_2 # tier + family + cores eg. - B_Gen4_1, GP_Gen5_4 - tier: Basic # possible values - 'Basic', 'GeneralPurpose', 'MemoryOptimized' + name: GP_Gen5_4 # tier + family + cores eg. - B_Gen4_1, GP_Gen5_4 + tier: GeneralPurpose # possible values - 'Basic', 'GeneralPurpose', 'MemoryOptimized' family: Gen5 size: "51200" - capacity: 2 + capacity: 4 diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 548427f06e5..4ff6b8d5eb5 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -57,7 +57,7 @@ const ( PublicIPIdleTimeoutIsOutOfRange = "PublicIPIdleTimeoutIsOutOfRange" InvalidRequestContent = "InvalidRequestContent" InternalServerError = "InternalServerError" - NetworkAclsValidationFailure = "NetworkAclsValidationFailure" + NetworkAclsValidationFailure = "NetworkAclsValidationFailure" ) func NewAzureError(err error) error { From d53a8323225f79c72fffc1516223f966eb2d1e7d Mon Sep 17 00:00:00 2001 From: jpflueger Date: Mon, 20 Apr 2020 12:59:26 -0600 Subject: [PATCH 92/99] adding mongodb version to cosmosdb --- api/v1alpha1/cosmosdb_types.go | 1 + config/samples/azure_v1alpha1_cosmosdb.yaml | 5 ++++- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 9 +++++++++ pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go | 2 +- pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 3 ++- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 36796e27ea0..a037f4e02f1 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -43,6 +43,7 @@ type CosmosDBProperties struct { // CosmosDBDatabaseAccountOfferType - The offer type for the Cosmos DB database account. DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` //Locations []CosmosDBLocation `json:"locations,omitempty"` + MongoDBVersion string `json:"mongoDBVersion,omitempty"` } // +kubebuilder:validation:Enum=Standard diff --git a/config/samples/azure_v1alpha1_cosmosdb.yaml b/config/samples/azure_v1alpha1_cosmosdb.yaml index 3d84aab4dea..8580c570468 100644 --- a/config/samples/azure_v1alpha1_cosmosdb.yaml +++ b/config/samples/azure_v1alpha1_cosmosdb.yaml @@ -8,7 +8,10 @@ spec: resourceGroup: resourcegroup-azure-operators properties: databaseAccountOfferType: Standard + # optionally set the mongoDBVersion to "3.2" or "3.6", if omitted the default is "3.2" + # NOTE: kind must be set to MongoDB for this to take effect + #mongoDBVersion: "3.6" - # Use the field below to optionally specify a different keyvault + # Use the field below to optionally specify a different keyvault # to store the connectiong string secrets in #keyVaultToStoreSecrets: asoSecretKeyVault \ No newline at end of file diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index ad83a5577a7..1813aab22ce 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -44,6 +44,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, + dbVersion string, tags map[string]*string) (*documentdb.DatabaseAccount, error) { cosmosDBClient, err := getCosmosDBClient() if err != nil { @@ -53,6 +54,13 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( dbKind := documentdb.DatabaseAccountKind(kind) sDBType := string(dbType) + var capabilities []documentdb.Capability + if dbKind == documentdb.MongoDB && dbVersion == "3.6" { + capabilities = []documentdb.Capability{ + {Name: to.StringPtr("EnableMongo")}, + } + } + /* * Current state of Locations and CosmosDB properties: * Creating a Database account with CosmosDB requires @@ -86,6 +94,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( EnableMultipleWriteLocations: to.BoolPtr(false), IsVirtualNetworkFilterEnabled: to.BoolPtr(false), Locations: &locationsArray, + Capabilities: &capabilities, }, } createUpdateFuture, err := cosmosDBClient.CreateOrUpdate( diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index 9ce839c076d..88e42e8dc31 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -21,7 +21,7 @@ func NewAzureCosmosDBManager(secretClient secrets.SecretClient) *AzureCosmosDBMa // CosmosDBManager client functions type CosmosDBManager interface { // CreateOrUpdateCosmosDB creates a new cosmos database account - CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, tags map[string]*string) (*documentdb.DatabaseAccount, error) + CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, dbType v1alpha1.CosmosDBDatabaseAccountOfferType, dbVersion string, tags map[string]*string) (*documentdb.DatabaseAccount, error) // GetCosmosDB gets a cosmos database account GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, error) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index bd44b45a448..7885ff5958a 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -95,8 +95,9 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o location := instance.Spec.Location kind := instance.Spec.Kind dbType := instance.Spec.Properties.DatabaseAccountOfferType + dbVersion := instance.Spec.Properties.MongoDBVersion - db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, tags) + db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, dbType, dbVersion, tags) if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) instance.Status.Message = err.Error() From 89963d0a7aee4481f1f66129990a43cb75e3c4b4 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Mon, 20 Apr 2020 13:48:21 -0600 Subject: [PATCH 93/99] add empty capabilities array --- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 1813aab22ce..aa405dac93a 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -59,6 +59,8 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( capabilities = []documentdb.Capability{ {Name: to.StringPtr("EnableMongo")}, } + } else { + capabilities = make([]documentdb.Capability, 0) } /* From 67fabde80751679dbed3378453c28c3270c417fb Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Fri, 17 Apr 2020 13:58:57 -0600 Subject: [PATCH 94/99] improve message output for storage account and comment optional vals in sample --- .../samples/azure_v1alpha1_storageaccount.yaml | 16 ++++++++-------- .../storageaccount/storageaccount_reconcile.go | 8 +++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/config/samples/azure_v1alpha1_storageaccount.yaml b/config/samples/azure_v1alpha1_storageaccount.yaml index 39b590fe276..29cd0fe5cdf 100644 --- a/config/samples/azure_v1alpha1_storageaccount.yaml +++ b/config/samples/azure_v1alpha1_storageaccount.yaml @@ -11,11 +11,11 @@ spec: accessTier: Hot supportsHttpsTrafficOnly: true # Optional: networkRule - networkRule: - bypass: AzureServices # Possible values are AzureServices, Metrics, None, Logging - defaultAction: Deny # Possible values are Allow, Deny - virtualNetworkRules: - - subnetId: /subscriptions/{subscription}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet}/subnets/{subnet} - ipRules: #could be an ip range or a ip address - - ipAddressOrRange: 2.2.0.0/24 - - ipAddressOrRange: 2.2.2.1 + # networkRule: + # bypass: AzureServices # Possible values are AzureServices, Metrics, None, Logging + # defaultAction: Deny # Possible values are Allow, Deny + # virtualNetworkRules: + # - subnetId: /subscriptions/{subscription}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet}/subnets/{subnet} + # ipRules: #could be an ip range or a ip address + # - ipAddressOrRange: 2.2.0.0/24 + # - ipAddressOrRange: 2.2.2.1 diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go index 45efda9e48c..b21e2e2b682 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go @@ -42,11 +42,7 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o networkAcls = ParseNetworkPolicy(instance.Spec.NetworkRule) } // convert kube labels to expected tag format - labels := map[string]*string{} - for k, v := range instance.GetLabels() { - value := v - labels[k] = &value - } + labels := helpers.LabelsToTags(instance.GetLabels()) hash := "" stor, err := sa.GetStorage(ctx, groupName, name) @@ -132,6 +128,8 @@ func (sa *azureStorageManager) Ensure(ctx context.Context, obj runtime.Object, o instance.Status.Message = "Storage Account Already exists somewhere else" return true, nil } + + instance.Status.Message = "Storage Account already exists and should be available shortly" instance.Status.Provisioning = true } From a212e9c4fbfc169b76e2c74cd666389f18c8d8b5 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Fri, 17 Apr 2020 17:12:54 -0600 Subject: [PATCH 95/99] attempt to prevent failedProvisioning due to race conditions in keyvault --- pkg/resourcemanager/keyvaults/keyvault.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/resourcemanager/keyvaults/keyvault.go b/pkg/resourcemanager/keyvaults/keyvault.go index 2522fe0f3af..c50eefc9818 100644 --- a/pkg/resourcemanager/keyvaults/keyvault.go +++ b/pkg/resourcemanager/keyvaults/keyvault.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strings" + "time" auth "github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac" "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2018-02-14/keyvault" @@ -21,6 +22,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" uuid "github.com/satori/go.uuid" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -447,6 +449,14 @@ func (k *azureKeyVaultManager) Ensure(ctx context.Context, obj runtime.Object, o } if helpers.ContainsString(catchUnrecoverableErrors, azerr.Type) { // Unrecoverable error, so stop reconcilation + switch azerr.Type { + case errhelp.AlreadyExists: + timeNow := metav1.NewTime(time.Now()) + if timeNow.Sub(instance.Status.RequestedAt.Time) < (30 * time.Second) { + return false, nil + } + + } instance.Status.Message = "Reconcilation hit unrecoverable error " + err.Error() return true, nil } From 4b0b17e35d7187e4c7e15df625af8bbf72610be6 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Fri, 17 Apr 2020 17:17:55 -0600 Subject: [PATCH 96/99] don't reset RequestedAt until after the 30 seconds --- pkg/resourcemanager/keyvaults/keyvault.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/resourcemanager/keyvaults/keyvault.go b/pkg/resourcemanager/keyvaults/keyvault.go index c50eefc9818..17fc2e07edd 100644 --- a/pkg/resourcemanager/keyvaults/keyvault.go +++ b/pkg/resourcemanager/keyvaults/keyvault.go @@ -453,6 +453,7 @@ func (k *azureKeyVaultManager) Ensure(ctx context.Context, obj runtime.Object, o case errhelp.AlreadyExists: timeNow := metav1.NewTime(time.Now()) if timeNow.Sub(instance.Status.RequestedAt.Time) < (30 * time.Second) { + instance.Status.Provisioning = true return false, nil } From 95002d726126f92fb95470e2a0cc609dd7eff0ee Mon Sep 17 00:00:00 2001 From: jpflueger Date: Mon, 20 Apr 2020 16:31:54 -0600 Subject: [PATCH 97/99] fixing merge branch 'master' into cosmoslocation vet --- api/v1alpha1/cosmosdb_types.go | 2 +- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 1 - pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go | 1 - pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 4 ++-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index adeab99d032..2481a4d1a2f 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -43,7 +43,7 @@ type CosmosDBProperties struct { // CosmosDBDatabaseAccountOfferType - The offer type for the Cosmos DB database account. DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` EnableMultipleWriteLocations bool `json:"enableMultipleWriteLocations,omitempty"` - MongoDBVersion string `json:"mongoDBVersion,omitempty"` + MongoDBVersion string `json:"mongoDBVersion,omitempty"` } // +kubebuilder:validation:Enum=Standard diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 497ec177c48..1e76e65b73f 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -10,7 +10,6 @@ import ( "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" "github.com/Azure/azure-service-operator/pkg/secrets" diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index f2656bedfa7..87f6bfd8345 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -8,7 +8,6 @@ import ( "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/resourcemanager" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest" diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 83cf6e1ebb1..4173f42fdc8 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -98,10 +98,10 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o cosmosDBProperties := v1alpha1.CosmosDBProperties{ DatabaseAccountOfferType: instance.Spec.Properties.DatabaseAccountOfferType, EnableMultipleWriteLocations: instance.Spec.Properties.EnableMultipleWriteLocations, - MongoDBVersion: instance.Spec.Properties.MongoDBVersion, + MongoDBVersion: instance.Spec.Properties.MongoDBVersion, } - db, err := m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, cosmosDBProperties, tags) + db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, cosmosDBProperties, tags) if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) instance.Status.Message = err.Error() From e91604587eb51e90cc75ed5b2106949379f3217b Mon Sep 17 00:00:00 2001 From: jpflueger Date: Mon, 20 Apr 2020 18:37:27 -0600 Subject: [PATCH 98/99] fixing vet after merging secrets & multiple write locations --- api/v1alpha1/cosmosdb_types.go | 6 +++--- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 3 ++- pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 294f504e66e..1a0c32794fa 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -44,9 +44,9 @@ type CosmosDBProperties struct { // DatabaseAccountOfferType - The offer type for the Cosmos DB database account. DatabaseAccountOfferType CosmosDBDatabaseAccountOfferType `json:"databaseAccountOfferType,omitempty"` // IsVirtualNetworkFilterEnabled - Flag to indicate whether to enable/disable Virtual Network ACL rules. - IsVirtualNetworkFilterEnabled bool `json:"isVirtualNetworkFilterEnabled,omitempty"` - EnableMultipleWriteLocations bool `json:"enableMultipleWriteLocations,omitempty"` - MongoDBVersion string `json:"mongoDBVersion,omitempty"` + IsVirtualNetworkFilterEnabled bool `json:"isVirtualNetworkFilterEnabled,omitempty"` + EnableMultipleWriteLocations bool `json:"enableMultipleWriteLocations,omitempty"` + MongoDBVersion string `json:"mongoDBVersion,omitempty"` } // +kubebuilder:validation:Enum=Standard diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 5a2458cce89..8c00b59f7ee 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -54,7 +54,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( dbKind := documentdb.DatabaseAccountKind(kind) sDBType := string(properties.DatabaseAccountOfferType) bWriteLocal := bool(properties.EnableMultipleWriteLocations) - vnetEnabled := bool(properties.IsVirtualNetworkFilterEnabled) + vnetEnabled := bool(properties.IsVirtualNetworkFilterEnabled) var capabilities []documentdb.Capability if dbKind == documentdb.MongoDB && properties.MongoDBVersion == "3.6" { @@ -110,6 +110,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( IsVirtualNetworkFilterEnabled: &vnetEnabled, VirtualNetworkRules: &vNetRulesSet, EnableMultipleWriteLocations: &bWriteLocal, + Locations: &locationsArray, Capabilities: &capabilities, }, } diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 76a9cc96b30..88120e15809 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -100,7 +100,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o DatabaseAccountOfferType: instance.Spec.Properties.DatabaseAccountOfferType, EnableMultipleWriteLocations: instance.Spec.Properties.EnableMultipleWriteLocations, MongoDBVersion: instance.Spec.Properties.MongoDBVersion, - IsVirtualNetworkFilterEnabled: instance.Spec.Properties.IsVirtualNetworkFilterEnabled, + IsVirtualNetworkFilterEnabled: instance.Spec.Properties.IsVirtualNetworkFilterEnabled, } db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, networkRule, cosmosDBProperties, tags) From d259e00459cad94b81da3e5cfa16b55570cffb87 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 20 Apr 2020 23:33:58 -0600 Subject: [PATCH 99/99] PR comments --- pkg/resourcemanager/psql/server/server_reconcile.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index b879adac85a..68f4bc235b9 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -61,13 +61,11 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.ResourceId = *getServer.ID instance.Status.Provisioned = true instance.Status.Provisioning = false - instance.Status.State = string(getServer.UserVisibleState) return true, nil } // the database exists but has not provisioned yet - so keep waiting instance.Status.Message = "Postgres server exists but may not be ready" - instance.Status.State = string(getServer.UserVisibleState) return false, nil }