diff --git a/api/router.go b/api/router.go new file mode 100644 index 0000000000..555ee5f849 --- /dev/null +++ b/api/router.go @@ -0,0 +1,12 @@ +package api + +import ( + "fmt" + "net" +) + +// Expose calls the router to assign the given IP addr to the weave bridge. +func (client *Client) Expose(ipAddr *net.IPNet) error { + _, err := client.httpVerb("POST", fmt.Sprintf("/expose/%s", ipAddr), nil) + return err +} diff --git a/net/bridge.go b/net/bridge.go index 61c9fc2c07..88b7cd6b4b 100644 --- a/net/bridge.go +++ b/net/bridge.go @@ -8,7 +8,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/coreos/go-iptables/iptables" - "github.com/j-keck/arping" "github.com/pkg/errors" "github.com/vishvananda/netlink" @@ -487,35 +486,3 @@ func linkSetUpByName(linkName string) error { } return netlink.LinkSetUp(link) } - -func AddBridgeAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error { - link, err := netlink.LinkByName(bridgeName) - if err != nil { - return errors.Wrapf(err, "AddBridgeAddr finding bridge %q", bridgeName) - } - newAddresses, err := AddAddresses(link, []*net.IPNet{addr}) - if err != nil { - return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName) - } - for _, ipnet := range newAddresses { - arping.GratuitousArpOverIfaceByName(ipnet.IP, bridgeName) - } - if removeDefaultRoute { - routeFilter := &netlink.Route{ - LinkIndex: link.Attrs().Index, - Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask}, - Protocol: 2, // RTPROT_KERNEL - } - filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL - routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask) - if err != nil { - return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName) - } - for _, r := range routes { - if err = netlink.RouteDel(&r); err != nil { - return errors.Wrapf(err, "failed to delete default route %+v", r) - } - } - } - return nil -} diff --git a/net/expose.go b/net/expose.go new file mode 100644 index 0000000000..a6afbf84f4 --- /dev/null +++ b/net/expose.go @@ -0,0 +1,109 @@ +package net + +import ( + "net" + "syscall" + + "github.com/coreos/go-iptables/iptables" + "github.com/j-keck/arping" + "github.com/pkg/errors" + "github.com/vishvananda/netlink" +) + +// Expose makes the network accessible from a host by assigning a given IP address +// to the weave bridge. +// +// List of params: +// * "bridgeName" - a name of the weave bridge. +// * "ipAddr" - IP addr to be assigned to the bridge. +// * "removeDefaultRoute" - whether to remove a default route installed by the kernel (used only in the AWSVPC mode). +// * "npc" - whether is Weave NPC running. +func Expose(bridgeName string, ipAddr *net.IPNet, removeDefaultRoute, npc bool) error { + if err := addBridgeIPAddr(bridgeName, ipAddr, removeDefaultRoute); err != nil { + return errors.Wrap(err, "addBridgeIPAddr") + } + + if err := exposeNAT(ipAddr); err != nil { + return errors.Wrap(err, "exposeNAT") + } + + if !npc { + // TODO comment why not in npc mode && add filter rules and docs + } + + return nil +} + +func addBridgeIPAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + return errors.Wrapf(err, "addBridgeIPAddr finding bridge %q", bridgeName) + } + err = netlink.AddrAdd(link, &netlink.Addr{IPNet: addr}) + // The IP addr might have been already set by a concurrent request, just ignore then + if err != nil && err != syscall.Errno(syscall.EEXIST) { + return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName) + } + + // Sending multiple ARP REQUESTs in the case of EEXIST above does not hurt + arping.GratuitousArpOverIfaceByName(addr.IP, bridgeName) + + // Remove a default route installed by the kernel. Required by the AWSVPC mode. + if removeDefaultRoute { + routeFilter := &netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask}, + Protocol: 2, // RTPROT_KERNEL + } + filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL + routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask) + if err != nil { + return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName) + } + for _, r := range routes { + err = netlink.RouteDel(&r) + // Again, there might be a concurrent request for removing routes + if err != nil && err != syscall.Errno(syscall.ESRCH) { + return errors.Wrapf(err, "failed to delete default route %+v", r) + } + } + } + return nil +} + +func exposeNAT(ipnet *net.IPNet) error { + ipt, err := iptables.New() + if err != nil { + return err + } + cidr := ipnet.String() + + if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil { + return err + } + if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil { + return err + } + if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil { + return err + } + return nil +} + +func addNatRule(ipt *iptables.IPTables, rulespec ...string) error { + // Loop until we get an exit code other than "temporarily unavailable" + for { + if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil { + if ierr, ok := err.(*iptables.Error); ok { + if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok { + // (magic exit code 4 found in iptables source code; undocumented) + if status.ExitStatus() == 4 { + continue + } + } + } + return err + } + return nil + } +} diff --git a/net/veth.go b/net/veth.go index c42c27132c..27945183c9 100644 --- a/net/veth.go +++ b/net/veth.go @@ -4,7 +4,6 @@ import ( "fmt" "net" "strings" - "syscall" "time" "github.com/coreos/go-iptables/iptables" @@ -301,39 +300,3 @@ func subnets(addrs []netlink.Addr) map[string]struct{} { } return subnets } - -func addNatRule(ipt *iptables.IPTables, rulespec ...string) error { - // Loop until we get an exit code other than "temporarily unavailable" - for { - if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil { - if ierr, ok := err.(*iptables.Error); ok { - if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok { - // (magic exit code 4 found in iptables source code; undocumented) - if status.ExitStatus() == 4 { - continue - } - } - } - return err - } - return nil - } -} - -func ExposeNAT(ipnet net.IPNet) error { - ipt, err := iptables.New() - if err != nil { - return err - } - cidr := ipnet.String() - if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil { - return err - } - if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil { - return err - } - if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil { - return err - } - return nil -} diff --git a/plugin/net/cni.go b/plugin/net/cni.go index bfc4f45e01..06df7e3a88 100644 --- a/plugin/net/cni.go +++ b/plugin/net/cni.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "os" - "syscall" "github.com/containernetworking/cni/pkg/ipam" "github.com/containernetworking/cni/pkg/skel" @@ -100,12 +99,11 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { return fmt.Errorf("unable to allocate IP address for bridge: %s", err) } bridgeCIDR := bridgeIPResult.IPs[0].Address - if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil { - return fmt.Errorf("unable to assign IP address to bridge: %s", err) - } - if err := weavenet.ExposeNAT(bridgeCIDR); err != nil { - return fmt.Errorf("unable to create NAT rules: %s", err) + + if err := c.weave.Expose(&bridgeCIDR); err != nil { + return fmt.Errorf("unable to expose bridge %q: %s", conf.BrName, err) } + bridgeIP = bridgeCIDR.IP } else if err != nil { return err @@ -167,22 +165,6 @@ func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, rou return nil } -func assignBridgeIP(bridgeName string, ipnet net.IPNet) error { - link, err := netlink.LinkByName(bridgeName) - if err != nil { - return err - } - if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil { - // Treat as non-error if this address is already there - // - maybe another copy of this program just added it - if err == syscall.Errno(syscall.EEXIST) { - return nil - } - return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err) - } - return nil -} - // As of CNI 0.5 spec: // "Plugins should generally complete a DEL action without error even if some resources are missing" // this method should therefore return nil in most, if not all, cases. diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 0bdb6c3aa6..60c0d20d55 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -249,6 +249,7 @@ func main() { } config.ProtocolMinVersion = byte(protocolMinVersion) + // BUG(mp) *all* WaitGroup.Add should be called before checking WaitGroup.IsDone() var waitReady common.WaitGroup var proxy *weaveproxy.Proxy @@ -322,7 +323,7 @@ func main() { checkFatal(err) defer db.Close() - router, err := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db) + router, err := weave.NewNetworkRouter(config, networkConfig, bridgeConfig, name, nickName, overlay, db) checkFatal(err) Log.Println("Our name is", router.Ourself) @@ -469,17 +470,17 @@ func main() { // Run this on its own goroutine because the allocator can block // We remove the default route installed by the kernel, // because awsvpc has installed it as well - go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.AWSVPC, waitReady.Add()) + go exposeForAWSVPC(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, waitReady.Add()) } signals.SignalHandlerLoop(common.Log, router) } -func expose(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, removeDefaultRoute bool, ready func()) { +func exposeForAWSVPC(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, ready func()) { addr, err := alloc.Allocate("weave:expose", subnet, false, func() bool { return false }) checkFatal(err) cidr := address.MakeCIDR(subnet, addr) - err = weavenet.AddBridgeAddr(bridgeName, cidr.IPNet(), removeDefaultRoute) + err = weavenet.Expose(bridgeName, cidr.IPNet(), true, false) checkFatal(err) Log.Printf("Bridge %q exposed on address %v", bridgeName, cidr) ready() diff --git a/prog/weaveutil/expose.go b/prog/weaveutil/expose.go deleted file mode 100644 index 758285a740..0000000000 --- a/prog/weaveutil/expose.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - weavenet "github.com/weaveworks/weave/net" -) - -func exposeNAT(args []string) error { - if len(args) < 1 { - cmdUsage("expose-nat", "...") - } - - cidrs, err := parseCIDRs(args) - if err != nil { - return err - } - - for _, cidr := range cidrs { - if err := weavenet.ExposeNAT(*cidr); err != nil { - return err - } - } - return nil -} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index 525516ea81..c8d859e226 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -36,7 +36,6 @@ func init() { "list-netdevs": listNetDevs, "cni-net": cniNet, "cni-ipam": cniIPAM, - "expose-nat": exposeNAT, "bridge-ip": bridgeIP, "unique-id": uniqueID, "swarm-manager-peers": swarmManagerPeers, diff --git a/router/http.go b/router/http.go index 90079ba70d..88e66ca276 100644 --- a/router/http.go +++ b/router/http.go @@ -5,7 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/net" + "github.com/weaveworks/weave/net/address" ) func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) { @@ -26,4 +29,20 @@ func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) { router.ForgetConnections(r.Form["peer"]) }) + muxRouter.Methods("POST").Path("/expose/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + cidr, err := address.ParseCIDR(vars["ip"] + "/" + vars["prefixlen"]) + if err != nil { + http.Error(w, fmt.Sprint("unable to parse ip addr: ", err.Error()), http.StatusBadRequest) + return + } + + if err = net.Expose(router.BridgeConfig.WeaveBridgeName, cidr.IPNet(), router.BridgeConfig.AWSVPC, router.BridgeConfig.NPC); err != nil { + http.Error(w, fmt.Sprint("unable to expose: ", err.Error()), http.StatusInternalServerError) + return + } + + w.WriteHeader(204) + }) + } diff --git a/router/network_router.go b/router/network_router.go index 6825e58402..cfb3b94848 100644 --- a/router/network_router.go +++ b/router/network_router.go @@ -11,6 +11,7 @@ import ( "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/db" + weavenet "github.com/weaveworks/weave/net" ) const ( @@ -47,11 +48,12 @@ type PacketLogging interface { type NetworkRouter struct { *mesh.Router NetworkConfig + weavenet.BridgeConfig Macs *MacCache db db.DB } -func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) { +func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, bridgeConfig weavenet.BridgeConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) { if overlay == nil { overlay = NullNetworkOverlay{} } @@ -63,7 +65,7 @@ func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh if err != nil { return nil, err } - router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, db: db} + router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, BridgeConfig: bridgeConfig, db: db} router.Peers.OnInvalidateShortIDs(overlay.InvalidateShortIDs) router.Routes.OnChange(overlay.InvalidateRoutes) router.Macs = NewMacCache(macMaxAge, diff --git a/test/130_expose_test.sh b/test/130_expose_test.sh index ccb0079166..b2d1c6fe5c 100755 --- a/test/130_expose_test.sh +++ b/test/130_expose_test.sh @@ -33,13 +33,6 @@ check_container_connectivity() { start_suite "exposing weave network to host" -# expose/hide with a CIDR work prior to launching weave -run_on1 "! $PING $EXP" -weave_on1 "expose $EXP/24" -run_on1 " $PING $EXP" -weave_on1 "hide $EXP/24" -run_on1 "! $PING $EXP" - weave_on $HOST1 launch --ipalloc-range $UNIVERSE start_container $HOST1 $C1/24 --name=c1 diff --git a/weave b/weave index 166386ffd9..7cfb067a8e 100755 --- a/weave +++ b/weave @@ -460,10 +460,8 @@ EOF expose_ip() { ipam_cidrs allocate_no_check_alive weave:expose $CIDR_ARGS for CIDR in $ALL_CIDRS ; do - if ! ip addr show dev $BRIDGE | grep -qF $CIDR ; then - ip addr add dev $BRIDGE $CIDR - arp_update $BRIDGE $CIDR || true - fi + call_weave "POST" "/expose/$CIDR" + [ -z "$FQDN" ] || when_weave_running put_dns_fqdn_no_check_alive weave:expose $FQDN $CIDR done } @@ -1526,7 +1524,6 @@ case "$COMMAND" in fi create_bridge expose_ip - util_op expose-nat $ALL_CIDRS show_addrs $ALL_CIDRS ;; hide)