From 6df186aeac93c4dc6bdde4874e58079766b34191 Mon Sep 17 00:00:00 2001 From: Marcin Wielgus Date: Thu, 17 Aug 2017 22:25:37 +0200 Subject: [PATCH] Remove Azure support --- README.md | 2 +- .../cloudprovider/azure/README.md | 1 - .../azure/azure_cloud_provider.go | 285 ------------ .../azure/azure_cloud_provider_test.go | 354 --------------- .../cloudprovider/azure/azure_manager.go | 416 ------------------ .../cloudprovider/azure/azure_manager_test.go | 56 --- .../builder/cloud_provider_builder.go | 26 -- cluster-autoscaler/main.go | 2 +- 8 files changed, 2 insertions(+), 1140 deletions(-) delete mode 100644 cluster-autoscaler/cloudprovider/azure/README.md delete mode 100644 cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go delete mode 100644 cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go delete mode 100644 cluster-autoscaler/cloudprovider/azure/azure_manager.go delete mode 100644 cluster-autoscaler/cloudprovider/azure/azure_manager_test.go diff --git a/README.md b/README.md index 41b4a0cdfde3..b663f11e5f0c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This repository contains autoscaling-related components for Kubernetes. ## What's inside [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) - a component that automagically adjusts the size of a Kubernetes -Cluster so that all pods have a place to run and there are no unneeded nodes. Works with GCP, AWS and Azure. Current state - late beta, moving towards GA. +Cluster so that all pods have a place to run and there are no unneeded nodes. Works with GCP and AWS. Current state - late beta, moving towards GA. [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler) - a set of components to automagically adjust the amount of CPU and memory requested by pods running in the Kubernetes Cluster. Current state - under development. diff --git a/cluster-autoscaler/cloudprovider/azure/README.md b/cluster-autoscaler/cloudprovider/azure/README.md deleted file mode 100644 index b6bcd453a6af..000000000000 --- a/cluster-autoscaler/cloudprovider/azure/README.md +++ /dev/null @@ -1 +0,0 @@ -Implementation of cloud provider for Azure. diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go deleted file mode 100644 index 0a9cd610e39f..000000000000 --- a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go +++ /dev/null @@ -1,285 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "fmt" - "strconv" - "strings" - - "github.com/golang/glog" - - apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" - "k8s.io/autoscaler/cluster-autoscaler/utils/errors" - "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" -) - -// AzureCloudProvider provides implementation of CloudProvider interface for Azure. -type AzureCloudProvider struct { - azureManager *AzureManager - scaleSets []*ScaleSet -} - -// BuildAzureCloudProvider creates new AzureCloudProvider -func BuildAzureCloudProvider(azureManager *AzureManager, specs []string) (*AzureCloudProvider, error) { - azure := &AzureCloudProvider{ - azureManager: azureManager, - } - for _, spec := range specs { - if err := azure.addNodeGroup(spec); err != nil { - return nil, err - } - } - - return azure, nil -} - -// addNodeGroup adds node group defined in string spec. Format: -// minNodes:maxNodes:scaleSetName -func (azure *AzureCloudProvider) addNodeGroup(spec string) error { - scaleSet, err := buildScaleSet(spec, azure.azureManager) - if err != nil { - return err - } - azure.scaleSets = append(azure.scaleSets, scaleSet) - azure.azureManager.RegisterScaleSet(scaleSet) - return nil -} - -// Name returns name of the cloud provider. -func (azure *AzureCloudProvider) Name() string { - return "azure" -} - -// NodeGroups returns all node groups configured for this cloud provider. -func (azure *AzureCloudProvider) NodeGroups() []cloudprovider.NodeGroup { - result := make([]cloudprovider.NodeGroup, 0, len(azure.scaleSets)) - for _, scaleSet := range azure.scaleSets { - result = append(result, scaleSet) - } - return result -} - -// NodeGroupForNode returns the node group for the given node. -func (azure *AzureCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.NodeGroup, error) { - glog.V(6).Infof("Searching for node group for the node: %s, %s\n", node.Spec.ExternalID, node.Spec.ProviderID) - ref := &AzureRef{ - Name: node.Spec.ProviderID, - } - - scaleSet, err := azure.azureManager.GetScaleSetForInstance(ref) - - return scaleSet, err -} - -// Pricing returns pricing model for this cloud provider or error if not available. -func (azure *AzureCloudProvider) Pricing() (cloudprovider.PricingModel, errors.AutoscalerError) { - return nil, cloudprovider.ErrNotImplemented -} - -// AzureRef contains a reference to some entity in Azure world. -type AzureRef struct { - Name string -} - -// GetKey returns key of the given azure reference. -func (m *AzureRef) GetKey() string { - return m.Name -} - -// AzureRefFromProviderId creates InstanceConfig object from provider id which -// must be in format: azure:///resourceGroupName/name -func AzureRefFromProviderId(id string) (*AzureRef, error) { - splitted := strings.Split(id[9:], "/") - if len(splitted) != 2 { - return nil, fmt.Errorf("Wrong id: expected format azure:////, got %v", id) - } - return &AzureRef{ - Name: splitted[len(splitted)-1], - }, nil -} - -// ScaleSet implements NodeGroup interface. -type ScaleSet struct { - AzureRef - - azureManager *AzureManager - minSize int - maxSize int -} - -// MinSize returns minimum size of the node group. -func (scaleSet *ScaleSet) MinSize() int { - return scaleSet.minSize -} - -// MaxSize returns maximum size of the node group. -func (scaleSet *ScaleSet) MaxSize() int { - return scaleSet.maxSize -} - -// TargetSize returns the current TARGET size of the node group. It is possible that the -// number is different from the number of nodes registered in Kubernetes. -func (scaleSet *ScaleSet) TargetSize() (int, error) { - size, err := scaleSet.azureManager.GetScaleSetSize(scaleSet) - return int(size), err -} - -// IncreaseSize increases Scale Set size -func (scaleSet *ScaleSet) IncreaseSize(delta int) error { - if delta <= 0 { - return fmt.Errorf("size increase must be positive") - } - size, err := scaleSet.azureManager.GetScaleSetSize(scaleSet) - if err != nil { - return err - } - if int(size)+delta > scaleSet.MaxSize() { - return fmt.Errorf("size increase too large - desired:%d max:%d", int(size)+delta, scaleSet.MaxSize()) - } - return scaleSet.azureManager.SetScaleSetSize(scaleSet, size+int64(delta)) -} - -// DecreaseTargetSize decreases the target size of the node group. This function -// doesn't permit to delete any existing node and can be used only to reduce the -// request for new nodes that have not been yet fulfilled. Delta should be negative. -// It is assumed that cloud provider will not delete the existing nodes if the size -// when there is an option to just decrease the target. -func (scaleSet *ScaleSet) DecreaseTargetSize(delta int) error { - if delta >= 0 { - return fmt.Errorf("size decrease size must be negative") - } - size, err := scaleSet.azureManager.GetScaleSetSize(scaleSet) - if err != nil { - return err - } - nodes, err := scaleSet.azureManager.GetScaleSetVms(scaleSet) - if err != nil { - return err - } - if int(size)+delta < len(nodes) { - return fmt.Errorf("attempt to delete existing nodes targetSize:%d delta:%d existingNodes: %d", - size, delta, len(nodes)) - } - return scaleSet.azureManager.SetScaleSetSize(scaleSet, size+int64(delta)) -} - -// Belongs returns true if the given node belongs to the NodeGroup. -func (scaleSet *ScaleSet) Belongs(node *apiv1.Node) (bool, error) { - glog.V(6).Infof("Check if node belongs to this scale set: scaleset:%v, node:%v\n", scaleSet, node) - - ref := &AzureRef{ - Name: node.Spec.ProviderID, - } - - targetAsg, err := scaleSet.azureManager.GetScaleSetForInstance(ref) - if err != nil { - return false, err - } - if targetAsg == nil { - return false, fmt.Errorf("%s doesn't belong to a known scale set", node.Name) - } - if targetAsg.Id() != scaleSet.Id() { - return false, nil - } - return true, nil -} - -// DeleteNodes deletes the nodes from the group. -func (scaleSet *ScaleSet) DeleteNodes(nodes []*apiv1.Node) error { - glog.V(8).Infof("Delete nodes requested: %v\n", nodes) - size, err := scaleSet.azureManager.GetScaleSetSize(scaleSet) - if err != nil { - return err - } - if int(size) <= scaleSet.MinSize() { - return fmt.Errorf("min size reached, nodes will not be deleted") - } - refs := make([]*AzureRef, 0, len(nodes)) - for _, node := range nodes { - belongs, err := scaleSet.Belongs(node) - if err != nil { - return err - } - if belongs != true { - return fmt.Errorf("%s belongs to a different asg than %s", node.Name, scaleSet.Id()) - } - azureRef := &AzureRef{ - Name: node.Spec.ProviderID, - } - refs = append(refs, azureRef) - } - return scaleSet.azureManager.DeleteInstances(refs) -} - -// Id returns ScaleSet id. -func (scaleSet *ScaleSet) Id() string { - return scaleSet.Name -} - -// Debug returns a debug string for the Scale Set. -func (scaleSet *ScaleSet) Debug() string { - return fmt.Sprintf("%s (%d:%d)", scaleSet.Id(), scaleSet.MinSize(), scaleSet.MaxSize()) -} - -// TemplateNodeInfo returns a node template for this scale set. -func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulercache.NodeInfo, error) { - return nil, cloudprovider.ErrNotImplemented -} - -// Create ScaleSet from provided spec. -// spec is in the following format: min-size:max-size:scale-set-name. -func buildScaleSet(spec string, azureManager *AzureManager) (*ScaleSet, error) { - tokens := strings.SplitN(spec, ":", 3) - if len(tokens) != 3 { - return nil, fmt.Errorf("wrong nodes configuration: %s", spec) - } - - scaleSet := ScaleSet{ - azureManager: azureManager, - } - if size, err := strconv.Atoi(tokens[0]); err == nil { - if size <= 0 { - return nil, fmt.Errorf("min size must be >= 1, got: %d", size) - } - scaleSet.minSize = size - } else { - return nil, fmt.Errorf("failed to set min size: %s, expected integer", tokens[0]) - } - - if size, err := strconv.Atoi(tokens[1]); err == nil { - if size < scaleSet.minSize { - return nil, fmt.Errorf("max size must be greater or equal to min size") - } - scaleSet.maxSize = size - } else { - return nil, fmt.Errorf("failed to set max size: %s, expected integer", tokens[1]) - } - - if tokens[2] == "" { - return nil, fmt.Errorf("scale set name must not be blank, got spec: %s", spec) - } - - scaleSet.Name = tokens[2] - return &scaleSet, nil -} - -// Nodes returns a list of all nodes that belong to this node group. -func (scaleSet *ScaleSet) Nodes() ([]string, error) { - return scaleSet.azureManager.GetScaleSetVms(scaleSet) -} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go deleted file mode 100644 index 71527b53ad2b..000000000000 --- a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go +++ /dev/null @@ -1,354 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "fmt" - "testing" - - "github.com/Azure/azure-sdk-for-go/arm/compute" - "github.com/Azure/go-autorest/autorest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - apiv1 "k8s.io/api/core/v1" - "net/http" -) - -// Mock for VirtualMachineScaleSetsClient -type VirtualMachineScaleSetsClientMock struct { - mock.Mock -} - -func (client *VirtualMachineScaleSetsClientMock) Get(resourceGroupName string, - vmScaleSetName string) (result compute.VirtualMachineScaleSet, err error) { - fmt.Printf("Called VirtualMachineScaleSetsClientMock.Get(%s,%s)\n", resourceGroupName, vmScaleSetName) - capacity := int64(2) - properties := compute.VirtualMachineScaleSetProperties{} - return compute.VirtualMachineScaleSet{ - - Name: &vmScaleSetName, - Sku: &compute.Sku{ - Capacity: &capacity, - }, - VirtualMachineScaleSetProperties: &properties, - }, nil -} - -func (client *VirtualMachineScaleSetsClientMock) CreateOrUpdate( - resourceGroupName string, name string, parameters compute.VirtualMachineScaleSet, cancel <-chan struct{}) (result autorest.Response, err error) { - fmt.Printf("Called VirtualMachineScaleSetsClientMock.CreateOrUpdate(%s,%s)\n", resourceGroupName, name) - return autorest.Response{}, nil -} - -func (client *VirtualMachineScaleSetsClientMock) DeleteInstances(resourceGroupName string, vmScaleSetName string, - vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs, cancel <-chan struct{}) (result autorest.Response, err error) { - - args := client.Called(resourceGroupName, vmScaleSetName, vmInstanceIDs, cancel) - return args.Get(0).(autorest.Response), args.Error(1) -} - -// Mock for VirtualMachineScaleSetVMsClient -type VirtualMachineScaleSetVMsClientMock struct { - mock.Mock -} - -func (m *VirtualMachineScaleSetVMsClientMock) List(resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result compute.VirtualMachineScaleSetVMListResult, err error) { - - value := make([]compute.VirtualMachineScaleSetVM, 1) - vmInstanceId := "test-instance-id" - properties := compute.VirtualMachineScaleSetVMProperties{} - vmId := "67453E12-9BE8-D312-A456-426655440000" - properties.VMID = &vmId - value[0] = compute.VirtualMachineScaleSetVM{ - InstanceID: &vmInstanceId, - VirtualMachineScaleSetVMProperties: &properties, - } - - return compute.VirtualMachineScaleSetVMListResult{ - Value: &value, - }, nil - -} - -var testAzureManager = &AzureManager{ - scaleSets: make([]*scaleSetInformation, 0), - scaleSetClient: &VirtualMachineScaleSetsClientMock{}, - scaleSetVmClient: &VirtualMachineScaleSetVMsClientMock{}, - scaleSetCache: make(map[AzureRef]*ScaleSet), -} - -func testProvider(t *testing.T, m *AzureManager) *AzureCloudProvider { - provider, err := BuildAzureCloudProvider(m, nil) - assert.NoError(t, err) - return provider -} - -func TestBuildAwsCloudProvider(t *testing.T) { - m := testAzureManager - _, err := BuildAzureCloudProvider(m, []string{"bad spec"}) - assert.Error(t, err) - - _, err = BuildAzureCloudProvider(m, nil) - assert.NoError(t, err) -} - -func TestAddNodeGroup(t *testing.T) { - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("bad spec") - assert.Error(t, err) - assert.Equal(t, len(provider.scaleSets), 0) - - err = provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - assert.Equal(t, len(provider.scaleSets), 1) -} - -func TestName(t *testing.T) { - provider := testProvider(t, testAzureManager) - assert.Equal(t, provider.Name(), "azure") -} - -func TestNodeGroups(t *testing.T) { - provider := testProvider(t, testAzureManager) - assert.Equal(t, len(provider.NodeGroups()), 0) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - assert.Equal(t, len(provider.NodeGroups()), 1) -} - -func TestNodeGroupForNode(t *testing.T) { - node := &apiv1.Node{ - Spec: apiv1.NodeSpec{ - ProviderID: "azure:////123E4567-E89B-12D3-A456-426655440000", - }, - } - - scaleSetVmClient := VirtualMachineScaleSetVMsClientMock{} - - var testAzureManager = &AzureManager{ - scaleSets: make([]*scaleSetInformation, 0), - scaleSetClient: &VirtualMachineScaleSetsClientMock{}, - scaleSetVmClient: &scaleSetVmClient, - scaleSetCache: make(map[AzureRef]*ScaleSet), - } - - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - - assert.Equal(t, len(provider.scaleSets), 1) - - group, err := provider.NodeGroupForNode(node) - - assert.NoError(t, err) - assert.NotNil(t, group, "Group should not be nil") - - assert.Equal(t, group.Id(), "test-asg") - assert.Equal(t, group.MinSize(), 1) - assert.Equal(t, group.MaxSize(), 5) - - // test node in cluster that is not in a group managed by cluster autoscaler - nodeNotInGroup := &apiv1.Node{ - Spec: apiv1.NodeSpec{ - ProviderID: "azure:///subscriptions/subscripion/resourceGroups/test-resource-group/providers/Microsoft.Compute/virtualMachines/test-instance-id-not-in-group", - }, - } - - group, err = provider.NodeGroupForNode(nodeNotInGroup) - assert.NoError(t, err) - assert.Nil(t, group) - -} - -func TestAzureRefFromProviderId(t *testing.T) { - _, err := AzureRefFromProviderId("azure:///123") - assert.Error(t, err) - _, err = AzureRefFromProviderId("azure://test/rg/test-instance-id") - assert.Error(t, err) - - // Example id: "azure:///subscriptions/subscriptionId/resourceGroups/kubernetes/providers/Microsoft.Compute/virtualMachines/kubernetes-master" - azureRef, err := AzureRefFromProviderId("azure:////kubernetes-master") - assert.NoError(t, err) - assert.Equal(t, &AzureRef{ - Name: "kubernetes-master", - }, azureRef) -} - -func TestMaxSize(t *testing.T) { - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - assert.Equal(t, len(provider.scaleSets), 1) - assert.Equal(t, provider.scaleSets[0].MaxSize(), 5) -} - -func TestMinSize(t *testing.T) { - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - assert.Equal(t, len(provider.scaleSets), 1) - assert.Equal(t, provider.scaleSets[0].MinSize(), 1) -} - -func TestTargetSize(t *testing.T) { - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - targetSize, err := provider.scaleSets[0].TargetSize() - assert.Equal(t, targetSize, 2) - assert.NoError(t, err) -} - -func TestIncreaseSize(t *testing.T) { - - var testAzureManager = &AzureManager{ - - scaleSets: make([]*scaleSetInformation, 0), - scaleSetClient: &VirtualMachineScaleSetsClientMock{}, - scaleSetVmClient: &VirtualMachineScaleSetVMsClientMock{}, - scaleSetCache: make(map[AzureRef]*ScaleSet), - } - - provider := testProvider(t, testAzureManager) - - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - assert.Equal(t, len(provider.scaleSets), 1) - - err = provider.scaleSets[0].IncreaseSize(1) - assert.NoError(t, err) - -} - -func TestBelongs(t *testing.T) { - - var testAzureManager = &AzureManager{ - - scaleSets: make([]*scaleSetInformation, 0), - scaleSetClient: &VirtualMachineScaleSetsClientMock{}, - scaleSetVmClient: &VirtualMachineScaleSetVMsClientMock{}, - scaleSetCache: make(map[AzureRef]*ScaleSet), - } - - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - - invalidNode := &apiv1.Node{ - Spec: apiv1.NodeSpec{ - ProviderID: "azure:///subscriptions/subscriptionId/resourceGroups/kubernetes/providers/Microsoft.Compute/virtualMachines/invalid-instance-id", - }, - } - - _, err = provider.scaleSets[0].Belongs(invalidNode) - assert.Error(t, err) - - validNode := &apiv1.Node{ - Spec: apiv1.NodeSpec{ - ProviderID: "azure:////123E4567-E89B-12D3-A456-426655440000", - }, - } - belongs, err := provider.scaleSets[0].Belongs(validNode) - assert.Equal(t, belongs, true) - assert.NoError(t, err) - -} - -func TestDeleteNodes(t *testing.T) { - scaleSetClient := &VirtualMachineScaleSetsClientMock{} - m := &AzureManager{ - scaleSets: make([]*scaleSetInformation, 0), - scaleSetClient: scaleSetClient, - scaleSetVmClient: &VirtualMachineScaleSetVMsClientMock{}, - scaleSetCache: make(map[AzureRef]*ScaleSet), - } - - //(resourceGroupName string, vmScaleSetName string, - // vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs, cancel <-chan struct{}) - // (result autorest.Response, err error) - //cancel := make(<-chan struct{}) - instanceIds := make([]string, 1) - instanceIds[0] = "test-instance-id" - - //requiredIds := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ - // InstanceIds: &instanceIds, - //} - response := autorest.Response{ - Response: &http.Response{ - Status: "OK", - }, - } - scaleSetClient.On("DeleteInstances", mock.Anything, "test-asg", mock.Anything, mock.Anything).Return(response, nil) - - provider := testProvider(t, m) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - - node := &apiv1.Node{ - Spec: apiv1.NodeSpec{ - ProviderID: "azure:////123E4567-E89B-12D3-A456-426655440000", - }, - } - err = provider.scaleSets[0].DeleteNodes([]*apiv1.Node{node}) - assert.NoError(t, err) - scaleSetClient.AssertNumberOfCalls(t, "DeleteInstances", 1) -} - -func TestId(t *testing.T) { - provider := testProvider(t, testAzureManager) - err := provider.addNodeGroup("1:5:test-asg") - assert.NoError(t, err) - assert.Equal(t, len(provider.scaleSets), 1) - assert.Equal(t, provider.scaleSets[0].Id(), "test-asg") -} - -func TestDebug(t *testing.T) { - asg := ScaleSet{ - azureManager: testAzureManager, - minSize: 5, - maxSize: 55, - } - asg.Name = "test-scale-set" - assert.Equal(t, asg.Debug(), "test-scale-set (5:55)") -} - -func TestBuildAsg(t *testing.T) { - _, err := buildScaleSet("a", nil) - assert.Error(t, err) - _, err = buildScaleSet("a:b:c", nil) - assert.Error(t, err) - _, err = buildScaleSet("1:", nil) - assert.Error(t, err) - _, err = buildScaleSet("1:2:", nil) - assert.Error(t, err) - - _, err = buildScaleSet("-1:2:", nil) - assert.Error(t, err) - - _, err = buildScaleSet("5:3:", nil) - assert.Error(t, err) - - _, err = buildScaleSet("5:ddd:test-name", nil) - assert.Error(t, err) - - asg, err := buildScaleSet("111:222:test-name", nil) - assert.NoError(t, err) - assert.Equal(t, 111, asg.MinSize()) - assert.Equal(t, 222, asg.MaxSize()) - assert.Equal(t, "test-name", asg.Name) -} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager.go b/cluster-autoscaler/cloudprovider/azure/azure_manager.go deleted file mode 100644 index 5e64d54ceec9..000000000000 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager.go +++ /dev/null @@ -1,416 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "io" - "sync" - "time" - - "github.com/Azure/azure-sdk-for-go/arm/compute" - "github.com/golang/glog" - "gopkg.in/gcfg.v1" - - "bytes" - "fmt" - "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/azure" - "k8s.io/apimachinery/pkg/util/wait" - "net/http" - "os" - "strings" -) - -type scaleSetInformation struct { - config *ScaleSet - basename string -} - -type scaleSetClient interface { - Get(resourceGroupName string, vmScaleSetName string) (result compute.VirtualMachineScaleSet, err error) - CreateOrUpdate(resourceGroupName string, name string, parameters compute.VirtualMachineScaleSet, cancel <-chan struct{}) (result autorest.Response, err error) - DeleteInstances(resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs, cancel <-chan struct{}) (result autorest.Response, err error) -} - -type scaleSetVMClient interface { - List(resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result compute.VirtualMachineScaleSetVMListResult, err error) -} - -// AzureManager handles Azure communication and data caching. -type AzureManager struct { - resourceGroupName string - subscription string - scaleSetClient scaleSetClient - scaleSetVmClient scaleSetVMClient - - scaleSets []*scaleSetInformation - scaleSetCache map[AzureRef]*ScaleSet - - // cache of mapping from instance id to the scale set id - scaleSetIdCache map[string]string - - cacheMutex sync.Mutex -} - -// Config holds the configuration parsed from the --cloud-config flag -type Config struct { - Cloud string `json:"cloud" yaml:"cloud"` - TenantID string `json:"tenantId" yaml:"tenantId"` - SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` - ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"` - Location string `json:"location" yaml:"location"` - VnetName string `json:"vnetName" yaml:"vnetName"` - SubnetName string `json:"subnetName" yaml:"subnetName"` - SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"` - RouteTableName string `json:"routeTableName" yaml:"routeTableName"` - PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName" yaml:"primaryAvailabilitySetName"` - - AADClientID string `json:"aadClientId" yaml:"aadClientId"` - AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` - AADTenantID string `json:"aadTenantId" yaml:"aadTenantId"` -} - -// CreateAzureManager creates Azure Manager object to work with Azure. -func CreateAzureManager(configReader io.Reader) (*AzureManager, error) { - subscriptionId := string("") - resourceGroup := string("") - tenantId := string("") - clientId := string("") - clientSecret := string("") - var scaleSetAPI scaleSetClient - var scaleSetVmAPI scaleSetVMClient - if configReader != nil { - var cfg Config - if err := gcfg.ReadInto(&cfg, configReader); err != nil { - glog.Errorf("Couldn't read config: %v", err) - return nil, err - } - subscriptionId = cfg.SubscriptionID - resourceGroup = cfg.ResourceGroup - tenantId = cfg.AADTenantID - clientId = cfg.AADClientID - clientSecret = cfg.AADClientSecret - - } else { - subscriptionId = os.Getenv("ARM_SUBSCRIPTION_ID") - resourceGroup = os.Getenv("ARM_RESOURCE_GROUP") - tenantId = os.Getenv("ARM_TENANT_ID") - clientId = os.Getenv("ARM_CLIENT_ID") - clientSecret = os.Getenv("ARM_CLIENT_SECRET") - } - - if resourceGroup == "" { - panic("Resource group not found") - } - - if subscriptionId == "" { - panic("Subscription ID not found") - } - - if tenantId == "" { - panic("Tenant ID not found.") - } - - if clientId == "" { - panic("ARM Client ID not found") - } - - if clientSecret == "" { - panic("ARM Client Secret not found.") - } - - glog.Infof("read configuration: %v", subscriptionId) - - spt, err := NewServicePrincipalTokenFromCredentials(tenantId, clientId, clientSecret, azure.PublicCloud.ServiceManagementEndpoint) - if err != nil { - panic(err) - } - - scaleSetAPI = compute.NewVirtualMachineScaleSetsClient(subscriptionId) - scaleSetsClient := scaleSetAPI.(compute.VirtualMachineScaleSetsClient) - scaleSetsClient.Authorizer = spt - - scaleSetsClient.Sender = autorest.CreateSender( - //autorest.WithLogging(log.New(os.Stdout, "sdk-example: ", log.LstdFlags)), - ) - - //scaleSetsClient.RequestInspector = withInspection() - //scaleSetsClient.ResponseInspector = byInspecting() - - glog.Infof("Created scale set client with authorizer: %v", scaleSetsClient) - - scaleSetVmAPI = compute.NewVirtualMachineScaleSetVMsClient(subscriptionId) - scaleSetVMsClient := scaleSetVmAPI.(compute.VirtualMachineScaleSetVMsClient) - scaleSetVMsClient.Authorizer = spt - scaleSetVMsClient.RequestInspector = withInspection() - scaleSetVMsClient.ResponseInspector = byInspecting() - - glog.Infof("Created scale set vm client with authorizer: %v", scaleSetVMsClient) - - // Create Availability Sets Azure Client. - manager := &AzureManager{ - subscription: subscriptionId, - resourceGroupName: resourceGroup, - scaleSetClient: scaleSetsClient, - scaleSetVmClient: scaleSetVMsClient, - scaleSets: make([]*scaleSetInformation, 0), - scaleSetCache: make(map[AzureRef]*ScaleSet), - } - - go wait.Forever(func() { - manager.cacheMutex.Lock() - defer manager.cacheMutex.Unlock() - if err := manager.regenerateCache(); err != nil { - glog.Errorf("Error while regenerating AS cache: %v", err) - } - }, time.Hour) - - return manager, nil -} - -// NewServicePrincipalTokenFromCredentials creates a new ServicePrincipalToken using values of the -// passed credentials map. -func NewServicePrincipalTokenFromCredentials(tenantId string, clientId string, clientSecret string, scope string) (*azure.ServicePrincipalToken, error) { - oauthConfig, err := azure.PublicCloud.OAuthConfigForTenant(tenantId) - if err != nil { - panic(err) - } - return azure.NewServicePrincipalToken(*oauthConfig, clientId, clientSecret, scope) -} - -func withInspection() autorest.PrepareDecorator { - return func(p autorest.Preparer) autorest.Preparer { - return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) { - glog.Infof("Inspecting Request: %s %s\n", r.Method, r.URL) - return p.Prepare(r) - }) - } -} - -func byInspecting() autorest.RespondDecorator { - return func(r autorest.Responder) autorest.Responder { - return autorest.ResponderFunc(func(resp *http.Response) error { - glog.Infof("Inspecting Response: %s for %s %s\n", resp.Status, resp.Request.Method, resp.Request.URL) - return r.Respond(resp) - }) - } -} - -// RegisterScaleSet registers scale set in Azure Manager. -func (m *AzureManager) RegisterScaleSet(scaleSet *ScaleSet) { - m.cacheMutex.Lock() - defer m.cacheMutex.Unlock() - - m.scaleSets = append(m.scaleSets, - &scaleSetInformation{ - config: scaleSet, - basename: scaleSet.Name, - }) - -} - -// GetScaleSetSize gets Scale Set size. -func (m *AzureManager) GetScaleSetSize(asConfig *ScaleSet) (int64, error) { - glog.V(5).Infof("Get scale set size: %v\n", asConfig) - set, err := m.scaleSetClient.Get(m.resourceGroupName, asConfig.Name) - if err != nil { - return -1, err - } - glog.V(5).Infof("Returning scale set capacity: %d\n", *set.Sku.Capacity) - return *set.Sku.Capacity, nil -} - -// SetScaleSetSize sets ScaleSet size. -func (m *AzureManager) SetScaleSetSize(asConfig *ScaleSet, size int64) error { - op, err := m.scaleSetClient.Get(m.resourceGroupName, asConfig.Name) - if err != nil { - return err - } - op.Sku.Capacity = &size - op.VirtualMachineScaleSetProperties.ProvisioningState = nil - cancel := make(chan struct{}) - _, err = m.scaleSetClient.CreateOrUpdate(m.resourceGroupName, asConfig.Name, op, cancel) - - if err != nil { - return err - } - return nil -} - -// GetScaleSetForInstance returns ScaleSetConfig of the given Instance -func (m *AzureManager) GetScaleSetForInstance(instance *AzureRef) (*ScaleSet, error) { - glog.V(5).Infof("Looking for scale set for instance: %v\n", instance) - //if m.resourceGroupName == "" { - // m.resourceGroupName = instance.ResourceGroup - //} - - glog.V(8).Infof("Cache BEFORE: %v\n", m.scaleSetCache) - - m.cacheMutex.Lock() - defer m.cacheMutex.Unlock() - if config, found := m.scaleSetCache[*instance]; found { - return config, nil - } - if err := m.regenerateCache(); err != nil { - return nil, fmt.Errorf("Error while looking for ScaleSet for instance %+v, error: %v", *instance, err) - } - - glog.V(8).Infof("Cache AFTER: %v\n", m.scaleSetCache) - - if config, found := m.scaleSetCache[*instance]; found { - return config, nil - } - // instance does not belong to any configured Scale Set - return nil, nil -} - -// DeleteInstances deletes the given instances. All instances must be controlled by the same ASG. -func (m *AzureManager) DeleteInstances(instances []*AzureRef) error { - if len(instances) == 0 { - return nil - } - commonAsg, err := m.GetScaleSetForInstance(instances[0]) - if err != nil { - return err - } - for _, instance := range instances { - asg, err := m.GetScaleSetForInstance(instance) - if err != nil { - return err - } - if asg != commonAsg { - return fmt.Errorf("Cannot delete instances which don't belong to the same Scale Set.") - } - } - - instanceIds := make([]string, len(instances)) - for i, instance := range instances { - instanceIds[i] = m.scaleSetIdCache[instance.Name] - } - requiredIds := &compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ - InstanceIds: &instanceIds, - } - cancel := make(chan struct{}) - resp, err := m.scaleSetClient.DeleteInstances(m.resourceGroupName, commonAsg.Name, *requiredIds, cancel) - if err != nil { - return err - } - - glog.V(4).Infof(resp.Status) - - return nil -} - -func (m *AzureManager) regenerateCache() error { - - newCache := make(map[AzureRef]*ScaleSet) - newScaleSetIdCache := make(map[string]string) - for _, sset := range m.scaleSets { - - glog.V(4).Infof("Regenerating Scale Set information for %s", sset.config.Name) - - scaleSet, err := m.scaleSetClient.Get(m.resourceGroupName, sset.config.Name) - if err != nil { - glog.V(4).Infof("Failed AS info request for %s: %v", sset.config.Name, err) - return err - } - sset.basename = *scaleSet.Name - - result, err := m.scaleSetVmClient.List(m.resourceGroupName, sset.basename, "", "", "") - - if err != nil { - glog.V(4).Infof("Failed AS info request for %s: %v", sset.config.Name, err) - return err - } - - for _, instance := range *result.Value { - var name = "azure:////" + fixEndiannessUUID(string(strings.ToUpper(*instance.VirtualMachineScaleSetVMProperties.VMID))) - ref := AzureRef{ - Name: name, - } - newCache[ref] = sset.config - - newScaleSetIdCache[name] = *instance.InstanceID - } - } - - m.scaleSetCache = newCache - m.scaleSetIdCache = newScaleSetIdCache - return nil -} - -// GetScaleSetVms returns list of nodes for the given scale set. -func (m *AzureManager) GetScaleSetVms(scaleSet *ScaleSet) ([]string, error) { - instances, err := m.scaleSetVmClient.List(m.resourceGroupName, scaleSet.Name, "", "", "") - - if err != nil { - glog.V(4).Infof("Failed AS info request for %s: %v", scaleSet.Name, err) - return []string{}, err - } - result := make([]string, 0) - for _, instance := range *instances.Value { - var name = "azure:////" + fixEndiannessUUID(string(strings.ToUpper(*instance.VirtualMachineScaleSetVMProperties.VMID))) - - result = append(result, name) - } - return result, nil - -} - -// fixEndiannessUUID fixes UUID representation broken because of the bug in linux kernel. -// According to RFC 4122 (http://tools.ietf.org/html/rfc4122), Section 4.1.2 first three fields have Big Endian encoding. -// There is a bug in DMI code in Linux kernel (https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1551419) which -// prevents proper reading of UUID, so, there is a situation, when VMID read by kubernetes on the host is different from -// VMID reported by Azure REST API. To fix it, we need manually fix Big Endianness on first three fields of UUID. -func fixEndiannessUUID(uuid string) string { - if len(uuid) != 36 { - panic("Passed string is not an UUID: " + uuid) - } - sections := strings.Split(uuid, "-") - if len(sections) != 5 { - panic("Passed string is not an UUID: " + uuid) - } - - var buffer bytes.Buffer - buffer.WriteString(reverseBytes(sections[0])) - buffer.WriteString("-") - buffer.WriteString(reverseBytes(sections[1])) - buffer.WriteString("-") - buffer.WriteString(reverseBytes(sections[2])) - buffer.WriteString("-") - buffer.WriteString(sections[3]) - buffer.WriteString("-") - buffer.WriteString(sections[4]) - return buffer.String() -} - -// reverseBytes is a helper function used by fixEndiannessUUID. -// it reverses order of pairs of bytes in string. i.e. passing ABCD will produce CDAB. -func reverseBytes(s string) string { - // string length should be even. - if len(s)%2 != 0 { - panic("Passed string should have even length: " + s) - } - var buffer bytes.Buffer - - var l int = len(s) / 2 - for i := l; i > 0; i-- { - var startIndex int = (i - 1) * 2 - buffer.WriteString(s[startIndex : startIndex+2]) - } - return buffer.String() -} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go deleted file mode 100644 index 946e03c44e1f..000000000000 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package azure - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestFixEndiannessUUID(t *testing.T) { - var toFix = "60D7F925-4C67-DF44-A144-A3FE111ECDE3" - var expected = ("25F9D760-674C-44DF-A144-A3FE111ECDE3") - var result = fixEndiannessUUID(toFix) - assert.Equal(t, result, expected) -} - -func TestDoubleFixShouldProduceSameString(t *testing.T) { - var toFix = "60D7F925-4C67-DF44-A144-A3FE111ECDE3" - var tmp = fixEndiannessUUID(toFix) - var result = fixEndiannessUUID(tmp) - assert.Equal(t, result, toFix) -} - -func TestFixEndiannessUUIDFailsOnInvalidUUID(t *testing.T) { - assert.Panics(t, func() { - var toFix = "60D7F925-4C67-DF44-A144-A3FE111ECDE3XXXX" - _ = fixEndiannessUUID(toFix) - }, "Calling with invalid UUID should panic") - -} - -func TestFixEndiannessUUIDFailsOnInvalidUUID2(t *testing.T) { - assert.Panics(t, func() { - var toFix = "60D7-F925-4C67-DF44-A144-A3FE-111E-CDE3-XXXX" - _ = fixEndiannessUUID(toFix) - }, "Calling with invalid UUID should panic") - -} - -func TestReverseBytes(t *testing.T) { - assert.Equal(t, "CDAB", reverseBytes("ABCD")) -} diff --git a/cluster-autoscaler/cloudprovider/builder/cloud_provider_builder.go b/cluster-autoscaler/cloudprovider/builder/cloud_provider_builder.go index 4c60ffe9ae56..74956f7b159f 100644 --- a/cluster-autoscaler/cloudprovider/builder/cloud_provider_builder.go +++ b/cluster-autoscaler/cloudprovider/builder/cloud_provider_builder.go @@ -20,7 +20,6 @@ import ( "github.com/golang/glog" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/azure" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/gce" "os" ) @@ -91,30 +90,5 @@ func (b CloudProviderBuilder) Build(discoveryOpts cloudprovider.NodeGroupDiscove glog.Fatalf("Failed to create AWS cloud provider: %v", err) } } - - if b.cloudProviderFlag == "azure" { - var azureManager *azure.AzureManager - var azureError error - if b.cloudConfig != "" { - glog.Info("Creating Azure Manager using cloud-config file: %v", b.cloudConfig) - config, fileErr := os.Open(b.cloudConfig) - if fileErr != nil { - glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err) - } - defer config.Close() - azureManager, azureError = azure.CreateAzureManager(config) - } else { - glog.Info("Creating Azure Manager with default configuration.") - azureManager, azureError = azure.CreateAzureManager(nil) - } - if azureError != nil { - glog.Fatalf("Failed to create Azure Manager: %v", err) - } - cloudProvider, err = azure.BuildAzureCloudProvider(azureManager, nodeGroupsFlag) - if err != nil { - glog.Fatalf("Failed to create Azure cloud provider: %v", err) - } - } - return cloudProvider } diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index 63ec7cb352f9..72732a4a160d 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -82,7 +82,7 @@ var ( "How often scale down possiblity is check") scanInterval = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down") maxNodesTotal = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.") - cloudProviderFlag = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce, aws, azure") + cloudProviderFlag = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce, aws") maxEmptyBulkDeleteFlag = flag.Int("max-empty-bulk-delete", 10, "Maximum number of empty nodes that can be deleted at the same time.") maxGracefulTerminationFlag = flag.Int("max-graceful-termination-sec", 60, "Maximum number of seconds CA waints for pod termination when trying to scale down a node.") maxTotalUnreadyPercentage = flag.Float64("max-total-unready-percentage", 33, "Maximum percentage of unready nodes after which CA halts operations")