Skip to content
This repository has been archived by the owner on Jun 20, 2024. It is now read-only.

Commit

Permalink
Handle "weave expose" in weaver
Browse files Browse the repository at this point in the history
The change makes "weave expose" to call the "/expose" HTTP
endpoint implemented in router which exposes the bridge.

Such architectural change required as we need to know whether NPC is
enabled when exposing the bridge and weaver knows this fact the best.

Unfortunately, this change makes impossible to "weave expose"
when weaver is not running.
  • Loading branch information
brb committed Aug 26, 2017
1 parent ff1cd7c commit f5959cd
Show file tree
Hide file tree
Showing 12 changed files with 155 additions and 134 deletions.
12 changes: 12 additions & 0 deletions api/router.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package api

import (
"fmt"
"net"
)

// Expose calls the router to assign the given IP addr to the weave bridge.
func (client *Client) Expose(ipAddr *net.IPNet) error {
_, err := client.httpVerb("POST", fmt.Sprintf("/expose/%s", ipAddr), nil)
return err
}
33 changes: 0 additions & 33 deletions net/bridge.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

"github.com/Sirupsen/logrus"
"github.com/coreos/go-iptables/iptables"
"github.com/j-keck/arping"
"github.com/pkg/errors"
"github.com/vishvananda/netlink"

Expand Down Expand Up @@ -487,35 +486,3 @@ func linkSetUpByName(linkName string) error {
}
return netlink.LinkSetUp(link)
}

func AddBridgeAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error {
link, err := netlink.LinkByName(bridgeName)
if err != nil {
return errors.Wrapf(err, "AddBridgeAddr finding bridge %q", bridgeName)
}
newAddresses, err := AddAddresses(link, []*net.IPNet{addr})
if err != nil {
return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName)
}
for _, ipnet := range newAddresses {
arping.GratuitousArpOverIfaceByName(ipnet.IP, bridgeName)
}
if removeDefaultRoute {
routeFilter := &netlink.Route{
LinkIndex: link.Attrs().Index,
Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask},
Protocol: 2, // RTPROT_KERNEL
}
filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask)
if err != nil {
return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName)
}
for _, r := range routes {
if err = netlink.RouteDel(&r); err != nil {
return errors.Wrapf(err, "failed to delete default route %+v", r)
}
}
}
return nil
}
109 changes: 109 additions & 0 deletions net/expose.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package net

import (
"net"
"syscall"

"github.com/coreos/go-iptables/iptables"
"github.com/j-keck/arping"
"github.com/pkg/errors"
"github.com/vishvananda/netlink"
)

// Expose makes the network accessible from a host by assigning a given IP address
// to the weave bridge.
//
// List of params:
// * "bridgeName" - a name of the weave bridge.
// * "ipAddr" - IP addr to be assigned to the bridge.
// * "removeDefaultRoute" - whether to remove a default route installed by the kernel (used only in the AWSVPC mode).
// * "npc" - whether is Weave NPC running.
func Expose(bridgeName string, ipAddr *net.IPNet, removeDefaultRoute, npc bool) error {
if err := addBridgeIPAddr(bridgeName, ipAddr, removeDefaultRoute); err != nil {
return errors.Wrap(err, "addBridgeIPAddr")
}

if err := exposeNAT(ipAddr); err != nil {
return errors.Wrap(err, "exposeNAT")
}

if !npc {
// TODO comment why not in npc mode && add filter rules and docs
}

return nil
}

func addBridgeIPAddr(bridgeName string, addr *net.IPNet, removeDefaultRoute bool) error {
link, err := netlink.LinkByName(bridgeName)
if err != nil {
return errors.Wrapf(err, "addBridgeIPAddr finding bridge %q", bridgeName)
}
err = netlink.AddrAdd(link, &netlink.Addr{IPNet: addr})
// The IP addr might have been already set by a concurrent request, just ignore then
if err != nil && err != syscall.Errno(syscall.EEXIST) {
return errors.Wrapf(err, "adding address %v to %q", addr, bridgeName)
}

// Sending multiple ARP REQUESTs in the case of EEXIST above does not hurt
arping.GratuitousArpOverIfaceByName(addr.IP, bridgeName)

// Remove a default route installed by the kernel. Required by the AWSVPC mode.
if removeDefaultRoute {
routeFilter := &netlink.Route{
LinkIndex: link.Attrs().Index,
Dst: &net.IPNet{IP: addr.IP.Mask(addr.Mask), Mask: addr.Mask},
Protocol: 2, // RTPROT_KERNEL
}
filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_DST | netlink.RT_FILTER_PROTOCOL
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, routeFilter, filterMask)
if err != nil {
return errors.Wrapf(err, "failed to get route list for bridge %q", bridgeName)
}
for _, r := range routes {
err = netlink.RouteDel(&r)
// Again, there might be a concurrent request for removing routes
if err != nil && err != syscall.Errno(syscall.ESRCH) {
return errors.Wrapf(err, "failed to delete default route %+v", r)
}
}
}
return nil
}

func exposeNAT(ipnet *net.IPNet) error {
ipt, err := iptables.New()
if err != nil {
return err
}
cidr := ipnet.String()

if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
return err
}
if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
return err
}
if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
return err
}
return nil
}

func addNatRule(ipt *iptables.IPTables, rulespec ...string) error {
// Loop until we get an exit code other than "temporarily unavailable"
for {
if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil {
if ierr, ok := err.(*iptables.Error); ok {
if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
// (magic exit code 4 found in iptables source code; undocumented)
if status.ExitStatus() == 4 {
continue
}
}
}
return err
}
return nil
}
}
37 changes: 0 additions & 37 deletions net/veth.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"fmt"
"net"
"strings"
"syscall"
"time"

"github.com/coreos/go-iptables/iptables"
Expand Down Expand Up @@ -301,39 +300,3 @@ func subnets(addrs []netlink.Addr) map[string]struct{} {
}
return subnets
}

func addNatRule(ipt *iptables.IPTables, rulespec ...string) error {
// Loop until we get an exit code other than "temporarily unavailable"
for {
if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil {
if ierr, ok := err.(*iptables.Error); ok {
if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
// (magic exit code 4 found in iptables source code; undocumented)
if status.ExitStatus() == 4 {
continue
}
}
}
return err
}
return nil
}
}

func ExposeNAT(ipnet net.IPNet) error {
ipt, err := iptables.New()
if err != nil {
return err
}
cidr := ipnet.String()
if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
return err
}
if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
return err
}
if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
return err
}
return nil
}
26 changes: 4 additions & 22 deletions plugin/net/cni.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"net"
"os"
"syscall"

"github.com/containernetworking/cni/pkg/ipam"
"github.com/containernetworking/cni/pkg/skel"
Expand Down Expand Up @@ -100,12 +99,11 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
return fmt.Errorf("unable to allocate IP address for bridge: %s", err)
}
bridgeCIDR := bridgeIPResult.IPs[0].Address
if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil {
return fmt.Errorf("unable to assign IP address to bridge: %s", err)
}
if err := weavenet.ExposeNAT(bridgeCIDR); err != nil {
return fmt.Errorf("unable to create NAT rules: %s", err)

if err := c.weave.Expose(&bridgeCIDR); err != nil {
return fmt.Errorf("unable to expose bridge %q: %s", conf.BrName, err)
}

bridgeIP = bridgeCIDR.IP
} else if err != nil {
return err
Expand Down Expand Up @@ -167,22 +165,6 @@ func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, rou
return nil
}

func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
link, err := netlink.LinkByName(bridgeName)
if err != nil {
return err
}
if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil {
// Treat as non-error if this address is already there
// - maybe another copy of this program just added it
if err == syscall.Errno(syscall.EEXIST) {
return nil
}
return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err)
}
return nil
}

// As of CNI 0.5 spec:
// "Plugins should generally complete a DEL action without error even if some resources are missing"
// this method should therefore return nil in most, if not all, cases.
Expand Down
9 changes: 5 additions & 4 deletions prog/weaver/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ func main() {
}
config.ProtocolMinVersion = byte(protocolMinVersion)

// BUG(mp) *all* WaitGroup.Add should be called before checking WaitGroup.IsDone()
var waitReady common.WaitGroup

var proxy *weaveproxy.Proxy
Expand Down Expand Up @@ -322,7 +323,7 @@ func main() {
checkFatal(err)
defer db.Close()

router, err := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
router, err := weave.NewNetworkRouter(config, networkConfig, bridgeConfig, name, nickName, overlay, db)
checkFatal(err)
Log.Println("Our name is", router.Ourself)

Expand Down Expand Up @@ -469,17 +470,17 @@ func main() {
// Run this on its own goroutine because the allocator can block
// We remove the default route installed by the kernel,
// because awsvpc has installed it as well
go expose(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, bridgeConfig.AWSVPC, waitReady.Add())
go exposeForAWSVPC(allocator, defaultSubnet, bridgeConfig.WeaveBridgeName, waitReady.Add())
}

signals.SignalHandlerLoop(common.Log, router)
}

func expose(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, removeDefaultRoute bool, ready func()) {
func exposeForAWSVPC(alloc *ipam.Allocator, subnet address.CIDR, bridgeName string, ready func()) {
addr, err := alloc.Allocate("weave:expose", subnet, false, func() bool { return false })
checkFatal(err)
cidr := address.MakeCIDR(subnet, addr)
err = weavenet.AddBridgeAddr(bridgeName, cidr.IPNet(), removeDefaultRoute)
err = weavenet.Expose(bridgeName, cidr.IPNet(), true, false)
checkFatal(err)
Log.Printf("Bridge %q exposed on address %v", bridgeName, cidr)
ready()
Expand Down
23 changes: 0 additions & 23 deletions prog/weaveutil/expose.go

This file was deleted.

1 change: 0 additions & 1 deletion prog/weaveutil/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ func init() {
"list-netdevs": listNetDevs,
"cni-net": cniNet,
"cni-ipam": cniIPAM,
"expose-nat": exposeNAT,
"bridge-ip": bridgeIP,
"unique-id": uniqueID,
"swarm-manager-peers": swarmManagerPeers,
Expand Down
19 changes: 19 additions & 0 deletions router/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ import (
"net/http"

"github.com/gorilla/mux"

"github.com/weaveworks/weave/common"
"github.com/weaveworks/weave/net"
"github.com/weaveworks/weave/net/address"
)

func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) {
Expand All @@ -26,4 +29,20 @@ func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) {
router.ForgetConnections(r.Form["peer"])
})

muxRouter.Methods("POST").Path("/expose/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
cidr, err := address.ParseCIDR(vars["ip"] + "/" + vars["prefixlen"])
if err != nil {
http.Error(w, fmt.Sprint("unable to parse ip addr: ", err.Error()), http.StatusBadRequest)
return
}

if err = net.Expose(router.BridgeConfig.WeaveBridgeName, cidr.IPNet(), router.BridgeConfig.AWSVPC, router.BridgeConfig.NPC); err != nil {
http.Error(w, fmt.Sprint("unable to expose: ", err.Error()), http.StatusInternalServerError)
return
}

w.WriteHeader(204)
})

}
6 changes: 4 additions & 2 deletions router/network_router.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/weaveworks/weave/common"
"github.com/weaveworks/weave/db"
weavenet "github.com/weaveworks/weave/net"
)

const (
Expand Down Expand Up @@ -47,11 +48,12 @@ type PacketLogging interface {
type NetworkRouter struct {
*mesh.Router
NetworkConfig
weavenet.BridgeConfig
Macs *MacCache
db db.DB
}

func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) {
func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, bridgeConfig weavenet.BridgeConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) {
if overlay == nil {
overlay = NullNetworkOverlay{}
}
Expand All @@ -63,7 +65,7 @@ func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh
if err != nil {
return nil, err
}
router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, db: db}
router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, BridgeConfig: bridgeConfig, db: db}
router.Peers.OnInvalidateShortIDs(overlay.InvalidateShortIDs)
router.Routes.OnChange(overlay.InvalidateRoutes)
router.Macs = NewMacCache(macMaxAge,
Expand Down
Loading

0 comments on commit f5959cd

Please sign in to comment.