bastion.go

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsmodel

import (
	"fmt"
	"sort"
	"strings"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
	"k8s.io/kops/upup/pkg/fi/utils"
)

// BastionModelBuilder adds model objects to support bastions
//
// Bastion instances live in the utility subnets created in the private topology.
// All traffic goes through an ELB, and the ELB has port 22 open to SSHAccess.
// Bastion instances have access to all internal master and node instances.

type BastionModelBuilder struct {
	*AWSModelContext
	Lifecycle         fi.Lifecycle
	SecurityLifecycle fi.Lifecycle
}

var _ fi.CloudupModelBuilder = &BastionModelBuilder{}

func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
	var bastionInstanceGroups []*kops.InstanceGroup
	for _, ig := range b.InstanceGroups {
		if ig.Spec.Role == kops.InstanceGroupRoleBastion {
			bastionInstanceGroups = append(bastionInstanceGroups, ig)
		}
	}

	if len(bastionInstanceGroups) == 0 {
		return nil
	}

	bastionGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleBastion)
	if err != nil {
		return err
	}
	nodeGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleNode)
	if err != nil {
		return err
	}
	masterGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleControlPlane)
	if err != nil {
		return err
	}

	// Create security group for bastion instances
	for _, bastionGroup := range bastionGroups {
		bastionGroup.Task.Lifecycle = b.SecurityLifecycle
		c.AddTask(bastionGroup.Task)
	}

	for _, src := range bastionGroups {
		// Allow traffic from bastion instances to egress freely
		{
			t := &awstasks.SecurityGroupRule{
				Name:          fi.PtrTo("ipv4-bastion-egress" + src.Suffix),
				Lifecycle:     b.SecurityLifecycle,
				SecurityGroup: src.Task,
				Egress:        fi.PtrTo(true),
				CIDR:          fi.PtrTo("0.0.0.0/0"),
			}
			AddDirectionalGroupRule(c, t)
		}
		{
			t := &awstasks.SecurityGroupRule{
				Name:          fi.PtrTo("ipv6-bastion-egress" + src.Suffix),
				Lifecycle:     b.SecurityLifecycle,
				SecurityGroup: src.Task,
				Egress:        fi.PtrTo(true),
				IPv6CIDR:      fi.PtrTo("::/0"),
			}
			AddDirectionalGroupRule(c, t)
		}
	}

	var bastionLoadBalancerType kops.LoadBalancerType
	{
		// Check if we requested a public or internal NLB
		if b.Cluster.Spec.Networking.Topology != nil && b.Cluster.Spec.Networking.Topology.Bastion != nil && b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer != nil {
			if b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type != "" {
				switch b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type {
				case kops.LoadBalancerTypeInternal:
					bastionLoadBalancerType = "Internal"
				case kops.LoadBalancerTypePublic:
					bastionLoadBalancerType = "Public"
				default:
					return fmt.Errorf("unhandled bastion LoadBalancer type %q", b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type)
				}
			} else {
				// Default to Public
				b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type = kops.LoadBalancerTypePublic
				bastionLoadBalancerType = "Public"
			}
		} else {
			// Default to Public
			bastionLoadBalancerType = "Public"
		}
	}

	// Allow bastion nodes to SSH to masters
	for _, src := range bastionGroups {
		for _, dest := range masterGroups {
			t := &awstasks.SecurityGroupRule{
				Name:          fi.PtrTo("bastion-to-master-ssh" + JoinSuffixes(src, dest)),
				Lifecycle:     b.SecurityLifecycle,
				SecurityGroup: dest.Task,
				SourceGroup:   src.Task,
				Protocol:      fi.PtrTo("tcp"),
				FromPort:      fi.PtrTo(int64(22)),
				ToPort:        fi.PtrTo(int64(22)),
			}
			AddDirectionalGroupRule(c, t)
		}
	}

	// Allow bastion nodes to SSH to nodes
	for _, src := range bastionGroups {
		for _, dest := range nodeGroups {
			t := &awstasks.SecurityGroupRule{
				Name:          fi.PtrTo("bastion-to-node-ssh" + JoinSuffixes(src, dest)),
				Lifecycle:     b.SecurityLifecycle,
				SecurityGroup: dest.Task,
				SourceGroup:   src.Task,
				Protocol:      fi.PtrTo("tcp"),
				FromPort:      fi.PtrTo(int64(22)),
				ToPort:        fi.PtrTo(int64(22)),
			}
			AddDirectionalGroupRule(c, t)
		}
	}

	var sshAllowedCIDRs []string
	var nlbSubnetMappings []*awstasks.SubnetMapping
	{
		// Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForNLB
		subnetsByZone := make(map[string][]*kops.ClusterSubnetSpec)
		for i := range b.Cluster.Spec.Networking.Subnets {
			subnet := &b.Cluster.Spec.Networking.Subnets[i]

			switch subnet.Type {
			case kops.SubnetTypePublic, kops.SubnetTypeUtility:
				if bastionLoadBalancerType != kops.LoadBalancerTypePublic {
					continue
				}

			case kops.SubnetTypeDualStack, kops.SubnetTypePrivate:
				if bastionLoadBalancerType != kops.LoadBalancerTypeInternal {
					continue
				}

			default:
				return fmt.Errorf("subnet %q had unknown type %q", subnet.Name, subnet.Type)
			}

			subnetsByZone[subnet.Zone] = append(subnetsByZone[subnet.Zone], subnet)
		}

		for zone, subnets := range subnetsByZone {
			for _, subnet := range subnets {
				sshAllowedCIDRs = append(sshAllowedCIDRs, subnet.CIDR)
			}
			subnet := b.chooseBestSubnetForNLB(zone, subnets)
			nlbSubnetMappings = append(nlbSubnetMappings, &awstasks.SubnetMapping{Subnet: b.LinkToSubnet(subnet)})
		}
	}

	sshAllowedCIDRs = append(sshAllowedCIDRs, b.Cluster.Spec.SSHAccess...)
	for _, cidr := range sshAllowedCIDRs {
		// Allow incoming SSH traffic to bastions, through the NLB
		// TODO: Could we get away without an NLB here?  Tricky to fix if dns-controller breaks though...
		for _, bastionGroup := range bastionGroups {
			{
				t := &awstasks.SecurityGroupRule{
					Name:          fi.PtrTo(fmt.Sprintf("ssh-nlb-%s", cidr)),
					Lifecycle:     b.SecurityLifecycle,
					SecurityGroup: bastionGroup.Task,
					Protocol:      fi.PtrTo("tcp"),
					FromPort:      fi.PtrTo(int64(22)),
					ToPort:        fi.PtrTo(int64(22)),
				}
				t.SetCidrOrPrefix(cidr)
				AddDirectionalGroupRule(c, t)
			}

			if strings.HasPrefix(cidr, "pl-") {
				// In case of a prefix list we do not add a rule for ICMP traffic for PMTU discovery.
				// This would require calling out to AWS to check whether the prefix list is IPv4 or IPv6.
			} else if utils.IsIPv6CIDR(cidr) {
				// Allow ICMP traffic required for PMTU discovery
				t := &awstasks.SecurityGroupRule{
					Name:          fi.PtrTo("icmpv6-pmtu-ssh-nlb-" + cidr),
					Lifecycle:     b.SecurityLifecycle,
					FromPort:      fi.PtrTo(int64(-1)),
					Protocol:      fi.PtrTo("icmpv6"),
					SecurityGroup: bastionGroup.Task,
					ToPort:        fi.PtrTo(int64(-1)),
				}
				t.SetCidrOrPrefix(cidr)
				c.AddTask(t)
			} else {
				t := &awstasks.SecurityGroupRule{
					Name:          fi.PtrTo("icmp-pmtu-ssh-nlb-" + cidr),
					Lifecycle:     b.SecurityLifecycle,
					FromPort:      fi.PtrTo(int64(3)),
					Protocol:      fi.PtrTo("icmp"),
					SecurityGroup: bastionGroup.Task,
					ToPort:        fi.PtrTo(int64(4)),
				}
				t.SetCidrOrPrefix(cidr)
				c.AddTask(t)
			}
		}
	}

	// Create NLB itself
	var nlb *awstasks.NetworkLoadBalancer
	{
		loadBalancerName := b.LBName32("bastion")

		tags := b.CloudTags(loadBalancerName, false)
		for k, v := range b.Cluster.Spec.CloudLabels {
			tags[k] = v
		}
		// Override the returned name to be the expected ELB name
		tags["Name"] = "bastion." + b.ClusterName()

		nlbListeners := []*awstasks.NetworkLoadBalancerListener{
			{
				Port:            22,
				TargetGroupName: b.NLBTargetGroupName("bastion"),
			},
		}
		nlb = &awstasks.NetworkLoadBalancer{
			Name:      fi.PtrTo(b.NLBName("bastion")),
			Lifecycle: b.Lifecycle,

			LoadBalancerName: fi.PtrTo(loadBalancerName),
			CLBName:          fi.PtrTo("bastion." + b.ClusterName()),
			SubnetMappings:   nlbSubnetMappings,
			Listeners:        nlbListeners,
			TargetGroups:     make([]*awstasks.TargetGroup, 0),

			Tags:          tags,
			VPC:           b.LinkToVPC(),
			Type:          fi.PtrTo("network"),
			IpAddressType: fi.PtrTo("ipv4"),
		}
		if useIPv6ForBastion(b) {
			nlb.IpAddressType = fi.PtrTo("dualstack")
		}
		// Set the NLB Scheme according to load balancer Type
		switch bastionLoadBalancerType {
		case kops.LoadBalancerTypeInternal:
			nlb.Scheme = fi.PtrTo("internal")
		case kops.LoadBalancerTypePublic:
			nlb.Scheme = nil
		default:
			return fmt.Errorf("unhandled bastion LoadBalancer type %q", bastionLoadBalancerType)
		}

		sshGroupName := b.NLBTargetGroupName("bastion")
		sshGroupTags := b.CloudTags(sshGroupName, false)

		// Override the returned name to be the expected NLB TG name
		sshGroupTags["Name"] = sshGroupName

		groupAttrs := map[string]string{
			awstasks.TargetGroupAttributeDeregistrationDelayConnectionTerminationEnabled: "true",
			awstasks.TargetGroupAttributeDeregistrationDelayTimeoutSeconds:               "30",
		}

		tg := &awstasks.TargetGroup{
			Name:               fi.PtrTo(sshGroupName),
			Lifecycle:          b.Lifecycle,
			VPC:                b.LinkToVPC(),
			Tags:               sshGroupTags,
			Protocol:           fi.PtrTo("TCP"),
			Port:               fi.PtrTo(int64(22)),
			Attributes:         groupAttrs,
			Interval:           fi.PtrTo(int64(10)),
			HealthyThreshold:   fi.PtrTo(int64(2)),
			UnhealthyThreshold: fi.PtrTo(int64(2)),
			Shared:             fi.PtrTo(false),
		}

		c.AddTask(tg)

		nlb.TargetGroups = append(nlb.TargetGroups, tg)

		sort.Stable(awstasks.OrderTargetGroupsByName(nlb.TargetGroups))
		c.AddTask(nlb)
	}

	publicName := ""
	if b.Cluster.Spec.Networking.Topology != nil && b.Cluster.Spec.Networking.Topology.Bastion != nil {
		publicName = b.Cluster.Spec.Networking.Topology.Bastion.PublicName
	}
	if publicName != "" {
		// Here we implement the bastion CNAME logic
		// By default bastions will create a CNAME that follows the `bastion-$clustername` formula
		t := &awstasks.DNSName{
			Name:      fi.PtrTo(publicName),
			Lifecycle: b.Lifecycle,

			Zone:               b.LinkToDNSZone(),
			ResourceName:       fi.PtrTo(publicName),
			ResourceType:       fi.PtrTo("A"),
			TargetLoadBalancer: b.LinkToNLB("bastion"),
		}
		c.AddTask(t)
		t = &awstasks.DNSName{
			Name:      fi.PtrTo(publicName + "-AAAA"),
			Lifecycle: b.Lifecycle,

			Zone:               b.LinkToDNSZone(),
			ResourceName:       fi.PtrTo(publicName),
			ResourceType:       fi.PtrTo("AAAA"),
			TargetLoadBalancer: b.LinkToNLB("bastion"),
		}
		c.AddTask(t)

	}
	return nil
}

func useIPv6ForBastion(b *BastionModelBuilder) bool {
	for _, ig := range b.InstanceGroups {
		for _, igSubnetName := range ig.Spec.Subnets {
			for _, clusterSubnet := range b.Cluster.Spec.Networking.Subnets {
				if igSubnetName != clusterSubnet.Name {
					continue
				}
				if clusterSubnet.IPv6CIDR != "" {
					return true
				}
			}
		}
	}
	return false
}

// Choose between subnets in a zone.
// We have already applied the rules to match internal subnets to internal NLBs and vice-versa for public-facing NLBs.
// For internal NLBs: we prefer the master subnets
// For public facing NLBs: we prefer the utility subnets
func (b *BastionModelBuilder) chooseBestSubnetForNLB(zone string, subnets []*kops.ClusterSubnetSpec) *kops.ClusterSubnetSpec {
	if len(subnets) == 0 {
		return nil
	}
	if len(subnets) == 1 {
		return subnets[0]
	}

	migSubnets := sets.NewString()
	for _, ig := range b.MasterInstanceGroups() {
		for _, subnet := range ig.Spec.Subnets {
			migSubnets.Insert(subnet)
		}
	}

	var scoredSubnets []*scoredSubnet
	for _, subnet := range subnets {
		score := 0

		if migSubnets.Has(subnet.Name) {
			score += 1
		}

		if subnet.Type == kops.SubnetTypeDualStack {
			score += 2
		}

		if subnet.Type == kops.SubnetTypeUtility {
			score += 3
		}

		scoredSubnets = append(scoredSubnets, &scoredSubnet{
			score:  score,
			subnet: subnet,
		})
	}

	sort.Sort(ByScoreDescending(scoredSubnets))

	if scoredSubnets[0].score == scoredSubnets[1].score {
		klog.V(2).Infof("Making arbitrary choice between subnets in zone %q to attach to NLB (%q vs %q)", zone, scoredSubnets[0].subnet.Name, scoredSubnets[1].subnet.Name)
	}

	return scoredSubnets[0].subnet
}