backend/aws-vpc: add support for multiple route tables

backend/awsvpc/awsvpc.go

@@ -53,13 +53,42 @@ func (be *AwsVpcBackend) Run(ctx context.Context) {
 	<-ctx.Done()
 }
 
+type backendConfig struct {
+	RouteTableID interface{} `json:"RouteTableID"`
+}
+
+func (conf *backendConfig) routeTables() ([]string, error) {
+	if table, ok := conf.RouteTableID.(string); ok {
+		log.Info(fmt.Sprintf("RouteTableID configured as string: %s", table))
+		return []string{table}, nil
+	}
+	if rawTables, ok := conf.RouteTableID.([]interface{}); ok {
+		log.Info(fmt.Sprintf("RouteTableID configured as slice: %+v", rawTables))
+		tables := make([]string, len(rawTables))
+		for idx, t := range rawTables {
+			table, ok := t.(string)
+			if !ok {
+				return nil, fmt.Errorf("Unexpected type in RouteTableID slice. Must be strings.")
+			}
+			tables[idx] = table
+		}
+		return tables, nil
+	}
+	return nil, fmt.Errorf("Unexpected RouteTableID type. Must be string or array of strings.")
+}
+
+func (conf *backendConfig) routeTableConfigured() bool {
+	configured := conf.RouteTableID != nil
+	log.Info(fmt.Sprintf("Route table configured: %t", configured))
+	return configured
+}
+
 func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (backend.Network, error) {
 	// Parse our configuration
-	cfg := struct {
-		RouteTableID string
-	}{}
+	var cfg backendConfig
 
 	if len(config.Backend) > 0 {
+		log.Info("Backend configured as: %s", string(config.Backend))
 		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
 			return nil, fmt.Errorf("error decoding VPC backend config: %v", err)
 		}
@@ -108,7 +137,7 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, co
 		log.Warningf("failed to disable SourceDestCheck on %s: %s.\n", *eni.NetworkInterfaceId, err)
 	}
 
-	if cfg.RouteTableID == "" {
+	if !cfg.routeTableConfigured() {
 		if cfg.RouteTableID, err = be.detectRouteTableID(eni, ec2c); err != nil {
 			return nil, err
 		}
@@ -117,29 +146,36 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, co
 
 	networkConfig, err := be.sm.GetNetworkConfig(ctx, network)
 
-	err = be.cleanupBlackholeRoutes(cfg.RouteTableID, networkConfig.Network, ec2c)
+	tables, err := cfg.routeTables()
 	if err != nil {
-		log.Errorf("Error cleaning up blackhole routes: %v", err)
+		return nil, err
 	}
 
-	matchingRouteFound, err := be.checkMatchingRoutes(cfg.RouteTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c)
-	if err != nil {
-		log.Errorf("Error describing route tables: %v", err)
-	}
+	for _, routeTableID := range tables {
+		err = be.cleanupBlackholeRoutes(routeTableID, networkConfig.Network, ec2c)
+		if err != nil {
+			log.Errorf("Error cleaning up blackhole routes: %v", err)
+		}
 
-	if !matchingRouteFound {
-		cidrBlock := l.Subnet.String()
-		deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &cfg.RouteTableID, DestinationCidrBlock: &cidrBlock}
-		if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil {
-			if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" {
-				// an error other than the route not already existing occurred
-				return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err)
-			}
+		matchingRouteFound, err := be.checkMatchingRoutes(routeTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c)
+		if err != nil {
+			log.Errorf("Error describing route tables: %v", err)
 		}
 
-		// Add the route for this machine's subnet
-		if err := be.createRoute(cfg.RouteTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c); err != nil {
-			return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err)
+		if !matchingRouteFound {
+			cidrBlock := l.Subnet.String()
+			deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &routeTableID, DestinationCidrBlock: &cidrBlock}
+			if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil {
+				if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" {
+					// an error other than the route not already existing occurred
+					return nil, fmt.Errorf("error deleting existing route for %s: %v", l.Subnet.String(), err)
+				}
+			}
+
+			// Add the route for this machine's subnet
+			if err := be.createRoute(routeTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c); err != nil {
+				return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err)
+			}
 		}
 	}
 
@@ -217,7 +253,7 @@ func (be *AwsVpcBackend) createRoute(routeTableID, subnet string, eniID *string,
 	if _, err := ec2c.CreateRoute(route); err != nil {
 		return err
 	}
-	log.Infof("Route added %s - %s.\n", subnet, *eniID)
+	log.Infof("Route added to table %s: %s - %s.\n", routeTableID, subnet, *eniID)
 	return nil
 }
 

main.go

@@ -127,6 +127,7 @@ func main() {
 		log.Error("Failed to create SubnetManager: ", err)
 		os.Exit(1)
 	}
+	log.Info(fmt.Sprintf("Created subnet manager: %+v", sm))
 
 	// Register for SIGINT and SIGTERM
 	log.Info("Installing signal handlers")

subnet/kube/kube.go

@@ -96,6 +96,7 @@ func NewSubnetManager() (subnet.Manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error parsing subnet config: %s", err)
 	}
+
 	sm, err := newKubeSubnetManager(c, sc, nodeName)
 	if err != nil {
 		return nil, fmt.Errorf("error creating network manager: %s", err)