Skip to content
This repository has been archived by the owner on Jun 15, 2021. It is now read-only.

Commit

Permalink
Merge pull request kubernetes-retired#284 from mumoshu/flexible-netwo…
Browse files Browse the repository at this point in the history 
…rk-topology

Add support for customization of network topologies
  • Loading branch information
@mumoshu
mumoshu authored Feb 1, 2017
2 parents ad81ee6 + 4bdcb88 commit 45394f0
Show file tree
Hide file tree
Showing 17 changed files with 1,609 additions and 449 deletions.
246 changes: 209 additions & 37 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func NewDefaultCluster() *Cluster {
AWSCliImageRepo: "quay.io/coreos/awscli",
AWSCliTag: "master",
ContainerRuntime: "docker",
Subnets: []*model.Subnet{},
Subnets: []model.Subnet{},
EIPAllocationIDs: []string{},
MapPublicIPs: true,
Experimental: experimental,
Expand Down Expand Up @@ -168,25 +168,106 @@ func ClusterFromBytes(data []byte) (*Cluster, error) {
return nil, fmt.Errorf("invalid cluster: %v", err)
}

c.SetDefaults()

return c, nil
}

func (c *Cluster) SetDefaults() {
// For backward-compatibility
if len(c.Subnets) == 0 {
c.Subnets = []*model.Subnet{
{
AvailabilityZone: c.AvailabilityZone,
InstanceCIDR: c.InstanceCIDR,
},
c.Subnets = []model.Subnet{
model.NewPublicSubnet(c.AvailabilityZone, c.InstanceCIDR),
}
}

privateTopologyImplied := c.RouteTableID != "" && !c.MapPublicIPs
publicTopologyImplied := c.RouteTableID != "" && c.MapPublicIPs

for i, s := range c.Subnets {
if s.CustomName == "" {
s.CustomName = fmt.Sprintf("Subnet%d", i)
if s.Name == "" {
c.Subnets[i].Name = fmt.Sprintf("Subnet%d", i)
}

// DEPRECATED AND REMOVED IN THE FUTURE
// See https://github.com/coreos/kube-aws/pull/284#issuecomment-275998862
//
// This implies a deployment to an existing VPC with a route table with a preconfigured Internet Gateway
// and all the subnets created by kube-aws are public
if publicTopologyImplied {
c.Subnets[i].RouteTable.ID = c.RouteTableID
if s.Private {
panic(fmt.Sprintf("mapPublicIPs(=%v) and subnets[%d].private(=%v) conflicts: %+v", c.MapPublicIPs, i, s.Private, s))
}
c.Subnets[i].Private = false
}

// DEPRECATED AND REMOVED IN THE FUTURE
// See https://github.com/coreos/kube-aws/pull/284#issuecomment-275998862
//
// This implies a deployment to an existing VPC with a route table with a preconfigured NAT Gateway
// and all the subnets created by kube-aws are private
if privateTopologyImplied {
c.Subnets[i].RouteTable.ID = c.RouteTableID
if s.Private {
panic(fmt.Sprintf("mapPublicIPs(=%v) and subnets[%d].private(=%v) conflicts. You don't need to set true to both of them. If you want to make all the subnets private, make mapPublicIPs false. If you want to make only part of subnets private, make subnets[].private true accordingly: %+v", c.MapPublicIPs, i, s.Private, s))
}
c.Subnets[i].Private = true
}
// Mark top-level subnets appropriately
s.TopLevel = true
}

return c, nil
for i, s := range c.Worker.Subnets {
linkedSubnet := c.FindSubnetMatching(s)
c.Worker.Subnets[i] = linkedSubnet
}

for i, s := range c.Controller.Subnets {
linkedSubnet := c.FindSubnetMatching(s)
c.Controller.Subnets[i] = linkedSubnet
}

for i, s := range c.Controller.LoadBalancer.Subnets {
linkedSubnet := c.FindSubnetMatching(s)
c.Controller.LoadBalancer.Subnets[i] = linkedSubnet
}

for i, s := range c.Etcd.Subnets {
linkedSubnet := c.FindSubnetMatching(s)
c.Etcd.Subnets[i] = linkedSubnet
}

if len(c.Worker.Subnets) == 0 {
if privateTopologyImplied {
c.Worker.Subnets = c.PrivateSubnets()
} else {
c.Worker.Subnets = c.PublicSubnets()
}
}

if len(c.Controller.Subnets) == 0 {
if privateTopologyImplied {
c.Controller.Subnets = c.PrivateSubnets()
} else {
c.Controller.Subnets = c.PublicSubnets()
}
}

if len(c.Controller.LoadBalancer.Subnets) == 0 {
if c.Controller.LoadBalancer.Private || privateTopologyImplied {
c.Controller.LoadBalancer.Subnets = c.PrivateSubnets()
c.Controller.LoadBalancer.Private = true
} else {
c.Controller.LoadBalancer.Subnets = c.PublicSubnets()
}
}

if len(c.Etcd.Subnets) == 0 {
if privateTopologyImplied {
c.Etcd.Subnets = c.PrivateSubnets()
} else {
c.Etcd.Subnets = c.PublicSubnets()
}
}
}

func ClusterFromBytesWithEncryptService(data []byte, encryptService EncryptService) (*Cluster, error) {
Expand Down Expand Up @@ -245,7 +326,7 @@ type DeploymentSettings struct {
ContainerRuntime string `yaml:"containerRuntime,omitempty"`
KMSKeyARN string `yaml:"kmsKeyArn,omitempty"`
StackTags map[string]string `yaml:"stackTags,omitempty"`
Subnets []*model.Subnet `yaml:"subnets,omitempty"`
Subnets []model.Subnet `yaml:"subnets,omitempty"`
EIPAllocationIDs []string `yaml:"eipAllocationIDs,omitempty"`
MapPublicIPs bool `yaml:"mapPublicIPs,omitempty"`
ElasticFileSystemID string `yaml:"elasticFileSystemId,omitempty"`
Expand All @@ -271,15 +352,14 @@ type WorkerSettings struct {

// Part of configuration which is specific to controller nodes
type ControllerSettings struct {
model.Controller `yaml:"controller,omitempty"`
ControllerCount int `yaml:"controllerCount,omitempty"`
ControllerCreateTimeout string `yaml:"controllerCreateTimeout,omitempty"`
ControllerInstanceType string `yaml:"controllerInstanceType,omitempty"`
ControllerLoadBalancerPrivate bool `yaml:"controllerLoadBalancerPrivate,omitempty"`
ControllerRootVolumeType string `yaml:"controllerRootVolumeType,omitempty"`
ControllerRootVolumeIOPS int `yaml:"controllerRootVolumeIOPS,omitempty"`
ControllerRootVolumeSize int `yaml:"controllerRootVolumeSize,omitempty"`
ControllerTenancy string `yaml:"controllerTenancy,omitempty"`
model.Controller `yaml:"controller,omitempty"`
ControllerCount int `yaml:"controllerCount,omitempty"`
ControllerCreateTimeout string `yaml:"controllerCreateTimeout,omitempty"`
ControllerInstanceType string `yaml:"controllerInstanceType,omitempty"`
ControllerRootVolumeType string `yaml:"controllerRootVolumeType,omitempty"`
ControllerRootVolumeIOPS int `yaml:"controllerRootVolumeIOPS,omitempty"`
ControllerRootVolumeSize int `yaml:"controllerRootVolumeSize,omitempty"`
ControllerTenancy string `yaml:"controllerTenancy,omitempty"`
}

// Part of configuration which is specific to etcd nodes
Expand Down Expand Up @@ -498,14 +578,21 @@ func (c Cluster) Config() (*Config, error) {
for etcdIndex := 0; etcdIndex < config.EtcdCount; etcdIndex++ {

//Round-robbin etcd instances across all available subnets
subnetIndex := etcdIndex % len(config.Subnets)
subnet := *config.Subnets[subnetIndex]
if config.Etcd.TopologyPrivate() {
subnet = *config.Etcd.Subnets[subnetIndex]
}
subnetIndex := etcdIndex % len(config.Etcd.Subnets)
subnet := config.Etcd.Subnets[subnetIndex]

var instance model.EtcdInstance

if subnet.ManageNATGateway() {
ngw, err := c.FindNATGatewayForPrivateSubnet(subnet)

if err != nil {
return nil, fmt.Errorf("failed getting a NAT gateway for the subnet %s in %v: %v", subnet.LogicalName(), c.NATGateways(), err)
}

instance := model.EtcdInstance{
Subnet: subnet,
instance = model.NewEtcdInstanceDependsOnNewlyCreatedNGW(subnet, *ngw)
} else {
instance = model.NewEtcdInstance(subnet)
}

config.EtcdInstances[etcdIndex] = instance
Expand All @@ -516,18 +603,13 @@ func (c Cluster) Config() (*Config, error) {

// Populate top-level subnets to model
if len(config.Subnets) > 0 {
if config.WorkerSettings.MinWorkerCount() > 0 && config.WorkerSettings.TopologyPrivate() == false {
if config.WorkerSettings.MinWorkerCount() > 0 && len(config.WorkerSettings.Subnets) == 0 {
config.WorkerSettings.Subnets = config.Subnets
}
if config.ControllerSettings.MinControllerCount() > 0 && config.ControllerSettings.TopologyPrivate() == false {
if config.ControllerSettings.MinControllerCount() > 0 && len(config.ControllerSettings.Subnets) == 0 {
config.ControllerSettings.Subnets = config.Subnets
}
}
config.ControllerElb.Private = config.ControllerSettings.ControllerLoadBalancerPrivate
config.ControllerElb.Subnets = config.Subnets
if config.ControllerElb.Private == true {
config.ControllerElb.Subnets = config.ControllerSettings.Subnets
}

config.IsChinaRegion = strings.HasPrefix(config.Region, "cn")

Expand Down Expand Up @@ -612,7 +694,6 @@ func (c Cluster) StackConfig(opts StackTemplateOptions) (*StackConfig, error) {
type Config struct {
Cluster

ControllerElb model.ControllerElb
EtcdInstances []model.EtcdInstance

// Encoded TLS assets
Expand Down Expand Up @@ -824,8 +905,11 @@ func (c DeploymentSettings) Valid() (*DeploymentValidationResult, error) {

var instanceCIDRs = make([]*net.IPNet, 0)

allPrivate := true
allPublic := true

for i, subnet := range c.Subnets {
if subnet.ID != "" {
if subnet.ID != "" || subnet.IDFromStackOutput != "" {
continue
}
if subnet.AvailabilityZone == "" {
Expand All @@ -843,6 +927,17 @@ func (c DeploymentSettings) Valid() (*DeploymentValidationResult, error) {
i,
)
}

if subnet.RouteTableID() != "" && c.RouteTableID != "" {
return nil, fmt.Errorf("either subnets[].routeTable.id(%s) or routeTableId(%s) but not both can be specified", subnet.RouteTableID(), c.RouteTableID)
}

allPrivate = allPrivate && subnet.Private
allPublic = allPublic && subnet.Public()
}

if c.RouteTableID != "" && !allPublic && !allPrivate {
return nil, fmt.Errorf("network topology including both private and public subnets specified while the single route table(%s) is also specified. You must differentiate the route table at least between private and public subnets. Use subets[].routeTable.id instead of routeTableId for that.", c.RouteTableID)
}

for i, a := range instanceCIDRs {
Expand All @@ -858,9 +953,86 @@ func (c DeploymentSettings) Valid() (*DeploymentValidationResult, error) {
return nil, err
}

for i, ngw := range c.NATGateways() {
if err := ngw.Validate(); err != nil {
return nil, fmt.Errorf("NGW %d is not valid: %v", i, err)
}
}

return &DeploymentValidationResult{vpcNet: vpcNet}, nil
}

func (s DeploymentSettings) AllSubnets() []model.Subnet {
subnets := s.Subnets
return subnets
}

func (c DeploymentSettings) FindSubnetMatching(condition model.Subnet) model.Subnet {
for _, s := range c.Subnets {
if s.Name == condition.Name {
return s
}
}
out := ""
for _, s := range c.Subnets {
out = fmt.Sprintf("%s%+v ", out, s)
}
panic(fmt.Errorf("No subnet matching %v found in %s", condition, out))
}

func (c DeploymentSettings) PrivateSubnets() []model.Subnet {
result := []model.Subnet{}
for _, s := range c.Subnets {
if s.Private {
result = append(result, s)
}
}
return result
}

func (c DeploymentSettings) PublicSubnets() []model.Subnet {
result := []model.Subnet{}
for _, s := range c.Subnets {
if !s.Private {
result = append(result, s)
}
}
return result
}

func (c DeploymentSettings) FindNATGatewayForPrivateSubnet(s model.Subnet) (*model.NATGateway, error) {
for _, ngw := range c.NATGateways() {
if ngw.IsConnectedToPrivateSubnet(s) {
return &ngw, nil
}
}
return nil, fmt.Errorf("No NATGateway found for the subnet %v", s)
}

func (c DeploymentSettings) NATGateways() []model.NATGateway {
ngws := []model.NATGateway{}
for _, privateSubnet := range c.PrivateSubnets() {
var publicSubnet model.Subnet
ngwConfig := privateSubnet.NATGateway
if privateSubnet.ManageNATGateway() {
publicSubnetFound := false
for _, s := range c.PublicSubnets() {
if s.AvailabilityZone == privateSubnet.AvailabilityZone {
publicSubnet = s
publicSubnetFound = true
break
}
}
if !publicSubnetFound {
panic(fmt.Sprintf("No appropriate public subnet found for a non-preconfigured NAT gateway associated to private subnet %s", privateSubnet.LogicalName()))
}
ngw := model.NewNATGateway(ngwConfig, privateSubnet, publicSubnet)
ngws = append(ngws, ngw)
}
}
return ngws
}

func (c WorkerSettings) Valid() error {
if c.WorkerRootVolumeType == "io1" {
if c.WorkerRootVolumeIOPS < 100 || c.WorkerRootVolumeIOPS > 2000 {
Expand Down
Loading

0 comments on commit 45394f0

Please sign in to comment.