Skip to content

Commit

Permalink
backend: fixes and cleanups in awsvpc backend
Browse files Browse the repository at this point in the history
  • Loading branch information
@vaijab
vaijab committed Nov 7, 2016
1 parent 536bc5c commit 921e7a9
Showing 1 changed file with 65 additions and 60 deletions.
125 changes: 65 additions & 60 deletions backend/awsvpc/awsvpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
log "github.com/golang/glog"
"golang.org/x/net/context"
Expand Down Expand Up @@ -80,47 +81,50 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, co
return nil, fmt.Errorf("failed to acquire lease: %v", err)
}

sess, _ := session.NewSession(aws.NewConfig().WithMaxRetries(5))

// Figure out this machine's EC2 instance ID and region
metadataClient := ec2metadata.New(nil)
metadataClient := ec2metadata.New(sess)
region, err := metadataClient.Region()
if err != nil {
return nil, fmt.Errorf("error getting EC2 region name: %v", err)
}
sess.Config.Region = aws.String(region)
instanceID, err := metadataClient.GetMetadata("instance-id")
if err != nil {
return nil, fmt.Errorf("error getting EC2 instance ID: %v", err)
}

ec2c := ec2.New(&aws.Config{Region: aws.String(region)})
ec2c := ec2.New(sess)

// Find ENI which contains the external network interface IP address
eni, err := be.findENI(instanceID, ec2c)
if err != nil || eni == nil {
return nil, fmt.Errorf("unable to find ENI that matches the %s IP address. %s\n", be.extIface.IfaceAddr, err)
}

if _, err = be.disableSrcDestCheck(instanceID, ec2c); err != nil {
log.Infof("Warning- disabling source destination check failed: %v", err)
// Try to disable SourceDestCheck on the main network interface
if err := be.disableSrcDestCheck(eni.NetworkInterfaceId, ec2c); err != nil {
log.Warningf("failed to disable SourceDestCheck on %s: %s.\n", *eni.NetworkInterfaceId, err)
}

if cfg.RouteTableID == "" {
log.Infof("RouteTableID not passed as config parameter, detecting ...")
if cfg.RouteTableID, err = be.detectRouteTableID(instanceID, ec2c); err != nil {
if cfg.RouteTableID, err = be.detectRouteTableID(eni, ec2c); err != nil {
return nil, err
}
log.Infof("Found route table %s.\n", cfg.RouteTableID)
}

log.Info("RouteRouteTableID: ", cfg.RouteTableID)
networkConfig, err := be.sm.GetNetworkConfig(ctx, network)

err = be.cleanupInvalidRoutes(cfg.RouteTableID, networkConfig.Network, ec2c)
err = be.cleanupBlackholeRoutes(cfg.RouteTableID, networkConfig.Network, ec2c)
if err != nil {
log.Errorf("Error cleaning up route table: %v", err)
log.Errorf("Error cleaning up blackhole routes: %v", err)
}

matchingRouteFound, err := be.checkMatchingRoutes(cfg.RouteTableID, instanceID, l.Subnet.String(), ec2c)
matchingRouteFound, err := be.checkMatchingRoutes(cfg.RouteTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c)
if err != nil {
log.Errorf("Error describing route tables: %v", err)

if ec2Err, ok := err.(awserr.Error); ok {
if ec2Err.Code() == "UnauthorizedOperation" {
log.Errorf("Note: DescribeRouteTables permission cannot be bound to any resource")
}
}
}

if !matchingRouteFound {
Expand All @@ -134,7 +138,7 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, co
}

// Add the route for this machine's subnet
if _, err := be.createRoute(cfg.RouteTableID, instanceID, l.Subnet.String(), ec2c); err != nil {
if err := be.createRoute(cfg.RouteTableID, l.Subnet.String(), eni.NetworkInterfaceId, ec2c); err != nil {
return nil, fmt.Errorf("unable to add route %s: %v", l.Subnet.String(), err)
}
}
Expand All @@ -145,7 +149,7 @@ func (be *AwsVpcBackend) RegisterNetwork(ctx context.Context, network string, co
}, nil
}

func (be *AwsVpcBackend) cleanupInvalidRoutes(routeTableID string, network ip.IP4Net, ec2c *ec2.EC2) error {
func (be *AwsVpcBackend) cleanupBlackholeRoutes(routeTableID string, network ip.IP4Net, ec2c *ec2.EC2) error {
filter := newFilter()
filter.Add("route.state", "blackhole")

Expand All @@ -160,7 +164,7 @@ func (be *AwsVpcBackend) cleanupInvalidRoutes(routeTableID string, network ip.IP
if *route.State == "blackhole" && route.DestinationCidrBlock != nil {
_, subnet, err := net.ParseCIDR(*route.DestinationCidrBlock)
if err == nil && network.Contains(ip.FromIP(subnet.IP)) {
log.Info("Removing route: ", *route.DestinationCidrBlock)
log.Info("Removing blackhole route: ", *route.DestinationCidrBlock)
deleteRouteInput := &ec2.DeleteRouteInput{RouteTableId: &routeTableID, DestinationCidrBlock: route.DestinationCidrBlock}
if _, err := ec2c.DeleteRoute(deleteRouteInput); err != nil {
if ec2err, ok := err.(awserr.Error); !ok || ec2err.Code() != "InvalidRoute.NotFound" {
Expand All @@ -176,7 +180,7 @@ func (be *AwsVpcBackend) cleanupInvalidRoutes(routeTableID string, network ip.IP
return nil
}

func (be *AwsVpcBackend) checkMatchingRoutes(routeTableID, instanceID, subnet string, ec2c *ec2.EC2) (bool, error) {
func (be *AwsVpcBackend) checkMatchingRoutes(routeTableID, subnet string, eniID *string, ec2c *ec2.EC2) (bool, error) {
matchingRouteFound := false

filter := newFilter()
Expand All @@ -192,63 +196,45 @@ func (be *AwsVpcBackend) checkMatchingRoutes(routeTableID, instanceID, subnet st

for _, routeTable := range resp.RouteTables {
for _, route := range routeTable.Routes {
if route.DestinationCidrBlock != nil && subnet == *route.DestinationCidrBlock && *route.State == "active" {

if *route.InstanceId == instanceID {
matchingRouteFound = true
break
}

log.Errorf("Deleting invalid *active* matching route: %s, %s \n", *route.DestinationCidrBlock, *route.InstanceId)
if route.DestinationCidrBlock != nil && subnet == *route.DestinationCidrBlock &&
*route.State == "active" && route.NetworkInterfaceId == eniID {
matchingRouteFound = true
break
}
}
}

return matchingRouteFound, nil
}

func (be *AwsVpcBackend) createRoute(routeTableID, instanceID, subnet string, ec2c *ec2.EC2) (*ec2.CreateRouteOutput, error) {
func (be *AwsVpcBackend) createRoute(routeTableID, subnet string, eniID *string, ec2c *ec2.EC2) error {
route := &ec2.CreateRouteInput{
RouteTableId: &routeTableID,
InstanceId: &instanceID,
NetworkInterfaceId: eniID,
DestinationCidrBlock: &subnet,
}

return ec2c.CreateRoute(route)
}

func (be *AwsVpcBackend) disableSrcDestCheck(instanceID string, ec2c *ec2.EC2) (*ec2.ModifyInstanceAttributeOutput, error) {
modifyAttributes := &ec2.ModifyInstanceAttributeInput{
InstanceId: aws.String(instanceID),
SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
if _, err := ec2c.CreateRoute(route); err != nil {
return err
}

return ec2c.ModifyInstanceAttribute(modifyAttributes)
log.Infof("Route added %s - %s.\n", subnet, *eniID)
return nil
}

func (be *AwsVpcBackend) detectRouteTableID(instanceID string, ec2c *ec2.EC2) (string, error) {
instancesInput := &ec2.DescribeInstancesInput{
InstanceIds: []*string{&instanceID},
}

resp, err := ec2c.DescribeInstances(instancesInput)
if err != nil {
return "", fmt.Errorf("error getting instance info: %v", err)
}

if len(resp.Reservations) == 0 {
return "", fmt.Errorf("no reservations found")
}

if len(resp.Reservations[0].Instances) == 0 {
return "", fmt.Errorf("no matching instance found with id: %v", instanceID)
func (be *AwsVpcBackend) disableSrcDestCheck(eniID *string, ec2c *ec2.EC2) error {
attr := &ec2.ModifyNetworkInterfaceAttributeInput{
NetworkInterfaceId: eniID,
SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
}
_, err := ec2c.ModifyNetworkInterfaceAttribute(attr)
return err
}

subnetID := resp.Reservations[0].Instances[0].SubnetId
vpcID := resp.Reservations[0].Instances[0].VpcId

log.Info("Subnet-ID: ", *subnetID)
log.Info("VPC-ID: ", *vpcID)
// detectRouteTableID detect the routing table that is associated with the ENI,
// subnet can be implicitly associated with the main routing table
func (be *AwsVpcBackend) detectRouteTableID(eni *ec2.InstanceNetworkInterface, ec2c *ec2.EC2) (string, error) {
subnetID := eni.SubnetId
vpcID := eni.VpcId

filter := newFilter()
filter.Add("association.subnet-id", *subnetID)
Expand Down Expand Up @@ -285,3 +271,22 @@ func (be *AwsVpcBackend) detectRouteTableID(instanceID string, ec2c *ec2.EC2) (s

return *res.RouteTables[0].RouteTableId, nil
}

func (be *AwsVpcBackend) findENI(instanceID string, ec2c *ec2.EC2) (*ec2.InstanceNetworkInterface, error) {
instance, err := ec2c.DescribeInstances(&ec2.DescribeInstancesInput{
InstanceIds: []*string{aws.String(instanceID)}},
)
if err != nil {
return nil, err
}

for _, n := range instance.Reservations[0].Instances[0].NetworkInterfaces {
for _, a := range n.PrivateIpAddresses {
if *a.PrivateIpAddress == be.extIface.IfaceAddr.String() {
log.Infof("Found %s that has %s IP address.\n", *n.NetworkInterfaceId, be.extIface.IfaceAddr)
return n, nil
}
}
}
return nil, err
}

0 comments on commit 921e7a9

Please sign in to comment.