From 2cbd855e6325d71328061c5e27920b6a1856b32f Mon Sep 17 00:00:00 2001 From: Tom Taylor Date: Mon, 8 May 2017 17:48:06 +0100 Subject: [PATCH] aws-vpc: add support for multiple route tables Resolving conflicts for pull request #561 and adding documentation. --- Documentation/aws-vpc-backend.md | 11 ++++- backend/awsvpc/awsvpc.go | 80 +++++++++++++++++++++++--------- main.go | 1 + subnet/kube/kube.go | 1 + 4 files changed, 70 insertions(+), 23 deletions(-) diff --git a/Documentation/aws-vpc-backend.md b/Documentation/aws-vpc-backend.md index 021c028b2c..10c117da85 100644 --- a/Documentation/aws-vpc-backend.md +++ b/Documentation/aws-vpc-backend.md @@ -114,11 +114,20 @@ First, SSH into `demo-instance-1`: ``` $ etcd --advertise-client-urls http://$INTERNAL_IP:2379 --listen-client-urls http://0.0.0.0:2379 ``` -- Publish configuration in etcd (ensure that the network range does not overlap with the one configured for the VPC) +- Publish configuration in etcd (ensure that the network range does not overlap with the one configured for the VPC). This will + attempt to automatically determine your route table. ``` $ etcdctl set /coreos.com/network/config '{"Network":"10.20.0.0/16", "Backend": {"Type": "aws-vpc"}}' ``` + +- If you want to manually specify your route table ID or if you want to update multiple route tables, e.g. for a deployment across multiple availability zones, use either a string for one or an array for one or more route tables like this. + +``` +$ etcdctl set /coreos.com/network/config '{"Network":"10.20.0.0/16", "Backend": {"Type": "aws-vpc", "RouteTableID": ["rtb-abc00001","rtb-abc00002","rtb-abc00003"]} }'}}' +``` + + - Fetch the latest release using wget from [here](https://github.com/coreos/flannel/releases/download/v0.7.0/flannel-v0.7.0-linux-amd64.tar.gz) - Run flannel daemon: diff --git a/backend/awsvpc/awsvpc.go b/backend/awsvpc/awsvpc.go index 10d5801f07..0d3df4d6ba 100644 --- a/backend/awsvpc/awsvpc.go +++ b/backend/awsvpc/awsvpc.go @@ -53,13 +53,42 @@ func (be *AwsVpcBackend) Run(ctx context.Context) { <-ctx.Done() } +type backendConfig struct { + RouteTableID interface{} `json:"RouteTableID"` +} + +func (conf *backendConfig) routeTables() ([]string, error) { + if table, ok := conf.RouteTableID.(string); ok { + log.Info("RouteTableID configured as string: %s", table) + return []string{table}, nil + } + if rawTables, ok := conf.RouteTableID.([]interface{}); ok { + log.Info("RouteTableID configured as slice: %+v", rawTables) + tables := make([]string, len(rawTables)) + for idx, t := range rawTables { + table, ok := t.(string) + if !ok { + return nil, fmt.Errorf("Unexpected type in RouteTableID slice. Must be strings.") + } + tables[idx] = table + } + return tables, nil + } + return nil, fmt.Errorf("Unexpected RouteTableID type. Must be string or array of strings.") +} + +func (conf *backendConfig) routeTableConfigured() bool { + configured := conf.RouteTableID != nil + log.Infof("Route table configured: %t", configured) + return configured +} + func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, config *subnet.Config) (backend.Network, error) { // Parse our configuration - cfg := struct { - RouteTableID string - }{} + var cfg backendConfig if len(config.Backend) > 0 { + log.Info("Backend configured as: %s", string(config.Backend)) if err := json.Unmarshal(config.Backend, &cfg); err != nil { return nil, fmt.Errorf("error decoding VPC backend config: %v", err) } @@ -108,7 +137,7 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, config *subnet.Con log.Warningf("failed to disable SourceDestCheck on %s: %s.\n", *eni.NetworkInterfaceId, err) } - if cfg.RouteTableID == "" { + if !cfg.routeTableConfigured() { if cfg.RouteTableID, err = be.detectRouteTableID(eni, ec2c); err != nil { return nil, err } @@ -120,29 +149,36 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, config *subnet.Con log.Errorf("Error fetching network config: %v", err) } - err = be.cleanupBlackholeRoutes(cfg.RouteTableID, networkConfig.Network, ec2c) + tables, err := cfg.routeTables() if err != nil { - log.Errorf("Error cleaning up blackhole routes: %v", err) + return nil, err } - matchingRouteFound, err := be.checkMatchingRoutes(cfg.RouteTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c) - if err != nil { - log.Errorf("Error describing route tables: %v", err) - } + for _, routeTableID := range tables { + err = be.cleanupBlackholeRoutes(routeTableID, networkConfig.Network, ec2c) + if err != nil { + log.Errorf("Error cleaning up blackhole routes: %v", err) + } - if !matchingRouteFound { - cidrBlock := l.Subnet.String() - deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &cfg.RouteTableID, DestinationCidrBlock: &cidrBlock} - if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil { - if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" { - // an error other than the route not already existing occurred - return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err) - } + matchingRouteFound, err := be.checkMatchingRoutes(routeTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c) + if err != nil { + log.Errorf("Error describing route tables: %v", err) } - // Add the route for this machine's subnet - if err := be.createRoute(cfg.RouteTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c); err != nil { - return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err) + if !matchingRouteFound { + cidrBlock := l.Subnet.String() + deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &routeTableID, DestinationCidrBlock: &cidrBlock} + if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil { + if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" { + // an error other than the route not already existing occurred + return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err) + } + } + + // Add the route for this machine's subnet + if err := be.createRoute(routeTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c); err != nil { + return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err) + } } } @@ -220,7 +256,7 @@ func (be *AwsVpcBackend) createRoute(routeTableID, subnet string, eniID *string, if _, err := ec2c.CreateRoute(route); err != nil { return err } - log.Infof("Route added %s - %s.\n", subnet, *eniID) + log.Infof("Route added to table %s: %s - %s.\n", routeTableID, subnet, *eniID) return nil } diff --git a/main.go b/main.go index 56ed85e872..ec24403625 100644 --- a/main.go +++ b/main.go @@ -144,6 +144,7 @@ func main() { log.Error("Failed to create SubnetManager: ", err) os.Exit(1) } + log.Infof("Created subnet manager: %+v", sm) // Register for SIGINT and SIGTERM log.Info("Installing signal handlers") diff --git a/subnet/kube/kube.go b/subnet/kube/kube.go index 9b14354cee..f388dd8dbf 100644 --- a/subnet/kube/kube.go +++ b/subnet/kube/kube.go @@ -102,6 +102,7 @@ func NewSubnetManager() (subnet.Manager, error) { if err != nil { return nil, fmt.Errorf("error parsing subnet config: %s", err) } + sm, err := newKubeSubnetManager(c, sc, nodeName) if err != nil { return nil, fmt.Errorf("error creating network manager: %s", err)