feat: Support existing load balancer backend address pool for agent nodes

docs/topics/clusterdefinitions.md

@@ -569,7 +569,8 @@ A cluster can have 0 to 12 agent pool profiles. Agent Pool Profiles are used for
 | acceleratedNetworkingEnabledWindows | no                                                                   | Use [Azure Accelerated Networking](https://azure.microsoft.com/en-us/blog/maximize-your-vm-s-performance-with-accelerated-networking-now-generally-available-for-both-windows-and-linux/) feature for Windows agents (You must select a VM SKU that supports Accelerated Networking). Defaults to `false`                                                                                                                                                                                                                                                      |
 | vmssOverProvisioningEnabled | no                                                                   | Use [Overprovisioning](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-design-overview#overprovisioning) with VMSS. This configuration is only valid on an agent pool with an `"availabilityProfile"` value of `"VirtualMachineScaleSets"`. Defaults to `false`                                                                                                                                                                                                                                                      |
 | enableVMSSNodePublicIP | no                                                                   | Enable creation of public IP on VMSS nodes. This configuration is only valid on an agent pool with an `"availabilityProfile"` value of `"VirtualMachineScaleSets"`. Defaults to `false`                                                                                                                                                                                                                                                      |
-
+| LoadBalancerBackendAddressPoolIDs | no                                                                   | Enables automatic placement of the agent pool nodes into existing load balancer's backend address pools. Each element value of this string array is the corresponding load balancer backend address pool's Azure Resource Manager(ARM) resource ID. By default this property is not included in the api model, which is equivalent to an empty string array.
+|
 ### linuxProfile
 
 `linuxProfile` provides the linux configuration for each linux node in the cluster

pkg/api/converterfromapi.go

@@ -1017,6 +1017,7 @@ func convertAgentPoolProfileToVLabs(api *AgentPoolProfile, p *vlabs.AgentPoolPro
 	p.AvailabilityZones = api.AvailabilityZones
 	p.SinglePlacementGroup = api.SinglePlacementGroup
 	p.EnableVMSSNodePublicIP = api.EnableVMSSNodePublicIP
+	p.LoadBalancerBackendAddressPoolIDs = api.LoadBalancerBackendAddressPoolIDs
 
 	for k, v := range api.CustomNodeLabels {
 		p.CustomNodeLabels[k] = v

pkg/api/convertertoapi.go

@@ -1033,6 +1033,7 @@ func convertVLabsAgentPoolProfile(vlabs *vlabs.AgentPoolProfile, api *AgentPoolP
 	api.AvailabilityZones = vlabs.AvailabilityZones
 	api.SinglePlacementGroup = vlabs.SinglePlacementGroup
 	api.EnableVMSSNodePublicIP = vlabs.EnableVMSSNodePublicIP
+	api.LoadBalancerBackendAddressPoolIDs = vlabs.LoadBalancerBackendAddressPoolIDs
 
 	api.CustomNodeLabels = map[string]string{}
 	for k, v := range vlabs.CustomNodeLabels {

pkg/api/types.go

@@ -533,6 +533,7 @@ type AgentPoolProfile struct {
 	PreserveNodesProperties             *bool                `json:"preserveNodesProperties,omitempty"`
 	WindowsNameVersion                  string               `json:"windowsNameVersion,omitempty"`
 	EnableVMSSNodePublicIP              *bool                `json:"enableVMSSNodePublicIP,omitempty"`
+	LoadBalancerBackendAddressPoolIDs   []string             `json:"loadBalancerBackendAddressPoolIDs,omitempty"`
 }
 
 // AgentPoolProfileRole represents an agent role

pkg/api/vlabs/types.go

@@ -457,13 +457,14 @@ type AgentPoolProfile struct {
 	// subnet is internal
 	subnet string
 
-	FQDN                   string            `json:"fqdn"`
-	CustomNodeLabels       map[string]string `json:"customNodeLabels,omitempty"`
-	PreProvisionExtension  *Extension        `json:"preProvisionExtension"`
-	Extensions             []Extension       `json:"extensions"`
-	SinglePlacementGroup   *bool             `json:"singlePlacementGroup,omitempty"`
-	AvailabilityZones      []string          `json:"availabilityZones,omitempty"`
-	EnableVMSSNodePublicIP *bool             `json:"enableVMSSNodePublicIP,omitempty"`
+	FQDN                              string            `json:"fqdn"`
+	CustomNodeLabels                  map[string]string `json:"customNodeLabels,omitempty"`
+	PreProvisionExtension             *Extension        `json:"preProvisionExtension"`
+	Extensions                        []Extension       `json:"extensions"`
+	SinglePlacementGroup              *bool             `json:"singlePlacementGroup,omitempty"`
+	AvailabilityZones                 []string          `json:"availabilityZones,omitempty"`
+	EnableVMSSNodePublicIP            *bool             `json:"enableVMSSNodePublicIP,omitempty"`
+	LoadBalancerBackendAddressPoolIDs []string          `json:"loadBalancerBackendAddressPoolIDs,omitempty"`
 }
 
 // AgentPoolProfileRole represents an agent role

pkg/api/vlabs/validate.go

@@ -465,6 +465,10 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error {
 		if e := agentPoolProfile.validateWindows(a.OrchestratorProfile, a.WindowsProfile, isUpdate); agentPoolProfile.OSType == Windows && e != nil {
 			return e
 		}
+
+		if e := agentPoolProfile.validateLoadBalancerBackendAddressPoolIDs(); e != nil {
+			return e
+		}
 	}
 
 	return nil
@@ -918,6 +922,19 @@ func (a *AgentPoolProfile) validateOrchestratorSpecificProperties(orchestratorTy
 	return nil
 }
 
+func (a *AgentPoolProfile) validateLoadBalancerBackendAddressPoolIDs() error {
+
+	if a.LoadBalancerBackendAddressPoolIDs != nil {
+		for _, backendPoolID := range a.LoadBalancerBackendAddressPoolIDs {
+			if len(backendPoolID) == 0 {
+				return errors.Errorf("AgentPoolProfile.LoadBalancerBackendAddressPoolIDs can not contain empty string. Agent pool name: %s", a.Name)
+			}
+		}
+	}
+
+	return nil
+}
+
 func validateKeyVaultSecrets(secrets []KeyVaultSecrets, requireCertificateStore bool) error {
 	for _, s := range secrets {
 		if len(s.VaultCertificates) == 0 {

pkg/api/vlabs/validate_test.go

@@ -2486,6 +2486,18 @@ func TestAgentPoolProfile_ValidateAvailabilityProfile(t *testing.T) {
 			t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
 		}
 	})
+
+	t.Run("Should fail for AvailabilitySet + invalid LoadBalancerBackendAddressPoolIDs", func(t *testing.T) {
+		t.Parallel()
+		cs := getK8sDefaultContainerService(false)
+		agentPoolProfiles := cs.Properties.AgentPoolProfiles
+		agentPoolProfiles[0].AvailabilityProfile = AvailabilitySet
+		agentPoolProfiles[0].LoadBalancerBackendAddressPoolIDs = []string{"/subscriptions/123/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/myVMSSSLB/backendAddressPools/myVMSSSLBBEPool", ""}
+		expectedMsg := fmt.Sprintf("AgentPoolProfile.LoadBalancerBackendAddressPoolIDs can not contain empty string. Agent pool name: %s", agentPoolProfiles[0].Name)
+		if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
+			t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
+		}
+	})
 }
 
 func TestAgentPoolProfile_ValidateVirtualMachineScaleSet(t *testing.T) {
@@ -2579,6 +2591,18 @@ func TestAgentPoolProfile_ValidateVirtualMachineScaleSet(t *testing.T) {
 			t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
 		}
 	})
+
+	t.Run("Should fail for VMSS + invalid LoadBalancerBackendAddressPoolIDs", func(t *testing.T) {
+		t.Parallel()
+		cs := getK8sDefaultContainerService(false)
+		agentPoolProfiles := cs.Properties.AgentPoolProfiles
+		agentPoolProfiles[0].AvailabilityProfile = VirtualMachineScaleSets
+		agentPoolProfiles[0].LoadBalancerBackendAddressPoolIDs = []string{"/subscriptions/123/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/myVMSSSLB/backendAddressPools/myVMSSSLBBEPool", ""}
+		expectedMsg := fmt.Sprintf("AgentPoolProfile.LoadBalancerBackendAddressPoolIDs can not contain empty string. Agent pool name: %s", agentPoolProfiles[0].Name)
+		if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
+			t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
+		}
+	})
 }
 
 func TestValidateCustomCloudProfile(t *testing.T) {

pkg/engine/networkinterfaces.go

@@ -320,6 +320,17 @@ func createAgentVMASNetworkInterface(cs *api.ContainerService, profile *api.Agen
 		}
 		if i == 1 {
 			ipConfig.Primary = to.BoolPtr(true)
+			if profile.LoadBalancerBackendAddressPoolIDs != nil {
+				backendPools := make([]network.BackendAddressPool, 0)
+				for _, lbBackendPoolID := range profile.LoadBalancerBackendAddressPoolIDs {
+					backendPools = append(backendPools,
+						network.BackendAddressPool{
+							ID: to.StringPtr(lbBackendPoolID),
+						},
+					)
+				}
+				ipConfig.LoadBalancerBackendAddressPools = &backendPools
+			}
 		}
 		ipConfig.PrivateIPAllocationMethod = network.Dynamic
 		ipConfig.Subnet = &network.Subnet{

pkg/engine/networkinterfaces_test.go

@@ -642,9 +642,10 @@ func TestCreateAgentVMASNIC(t *testing.T) {
 	}
 
 	profile := &api.AgentPoolProfile{
-		Name:   "fooAgent",
-		OSType: "Linux",
-		Role:   "Infra",
+		Name:                              "fooAgent",
+		OSType:                            "Linux",
+		Role:                              "Infra",
+		LoadBalancerBackendAddressPoolIDs: []string{"/subscriptions/123/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/mySLB/backendAddressPools/mySLBBEPool"},
 	}
 
 	actual := createAgentVMASNetworkInterface(cs, profile)

pkg/engine/virtualmachinescalesets.go

@@ -446,6 +446,18 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
 		if i == 1 {
 			ipconfig.Primary = to.BoolPtr(true)
 
+			if profile.LoadBalancerBackendAddressPoolIDs != nil {
+				backendPools := make([]compute.SubResource, 0)
+				for _, lbBackendPoolID := range profile.LoadBalancerBackendAddressPoolIDs {
+					backendPools = append(backendPools,
+						compute.SubResource{
+							ID: to.StringPtr(lbBackendPoolID),
+						},
+					)
+				}
+				ipconfig.LoadBalancerBackendAddressPools = &backendPools
+			}
+
 			// Set VMSS node public IP if requested
 			if to.Bool(profile.EnableVMSSNodePublicIP) {
 				publicIPAddressConfiguration := &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{