diff --git a/Makefile b/Makefile index e232792053e..4738fae3b62 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ build-go: CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn -ldflags "-w -s" -v ./cmd/cni CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-controller -ldflags "-w -s" -v ./cmd/controller CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-daemon -ldflags "-w -s" -v ./cmd/daemon + CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-gateway -ldflags "-w -s" -v ./cmd/gateway release: build-go docker build -t index.alauda.cn/alaudak8s/kube-ovn-node:`cat VERSION` -f dist/images/Dockerfile.node dist/images/ diff --git a/cmd/gateway/gateway.go b/cmd/gateway/gateway.go new file mode 100644 index 00000000000..d62c0dcc20e --- /dev/null +++ b/cmd/gateway/gateway.go @@ -0,0 +1,59 @@ +package main + +import ( + "bitbucket.org/mathildetech/kube-ovn/pkg/gateway" + "bitbucket.org/mathildetech/kube-ovn/pkg/ovs" + "bitbucket.org/mathildetech/kube-ovn/pkg/util" + "github.com/vishvananda/netlink" + "k8s.io/klog" + "os" +) + +func main() { + klog.SetOutput(os.Stdout) + defer klog.Flush() + + config, err := gateway.ParseFlags() + if err != nil { + klog.Errorf("parse config failed %v", err) + os.Exit(1) + } + + bridge, err := util.NicToBridge(config.Interface) + if err != nil { + klog.Errorf("failed to move nic to bridge %v", err) + os.Exit(1) + } + klog.Infof("create bridge %v for gw", bridge) + + ovnClient := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.OvnSbHost, config.OvnSbPort, "", "", "") + + err = ovnClient.CreateGatewayRouter(config.EdgeRouterName, config.Chassis) + if err != nil { + klog.Errorf("failed to crate gateway router %v", err) + os.Exit(1) + } + + err = ovnClient.CreateTransitLogicalSwitch(config.TransitSwitchName, config.ClusterRouterName, config.EdgeRouterName, config.ClusterRouterIP, config.EdgeRouterIP) + if err != nil { + klog.Errorf("failed to connect edge and cluster by transit %v", err) + os.Exit(1) + } + + addrList, err := netlink.AddrList(bridge, netlink.FAMILY_V4) + if err != nil { + klog.Errorf("failed to list %s addr %v", bridge, err) + os.Exit(1) + } + if len(addrList) == 0 { + klog.Errorf("nic %s has no ip address", bridge) + os.Exit(1) + } + + err = ovnClient.CreateOutsideLogicalSwitch(config.OutsideSwitchName, config.EdgeRouterName, addrList[0].IPNet.String(), bridge.Attrs().HardwareAddr.String()) + if err != nil { + klog.Errorf("failed to connect edge to outside %v", err) + os.Exit(1) + } + return +} diff --git a/dist/images/Dockerfile.cni b/dist/images/Dockerfile.cni index 464896fe179..506ad20f906 100644 --- a/dist/images/Dockerfile.cni +++ b/dist/images/Dockerfile.cni @@ -11,7 +11,9 @@ RUN yum install -y \ unbound unbound-devel && \ yum clean all -RUN rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-2.10.1-1.el7.centos.x86_64.rpm +RUN rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-2.10.1-1.el7.centos.x86_64.rpm && \ + rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-common-2.10.1-1.el7.centos.x86_64.rpm && \ + rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-host-2.10.1-1.el7.centos.x86_64.rpm COPY start-cniserver.sh /kube-ovn/start-cniserver.sh COPY kube-ovn.conflist /kube-ovn/kube-ovn.conflist @@ -20,4 +22,5 @@ WORKDIR /kube-ovn CMD ["sh", "start-cniserver.sh"] COPY kube-ovn /kube-ovn/kube-ovn -COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon \ No newline at end of file +COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon +COPY kube-ovn-gateway /kube-ovn/kube-ovn-gateway \ No newline at end of file diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 46f35376318..3c0d2022e0a 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -83,7 +83,7 @@ func NewController( controller := &Controller{ config: config, - ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer), + ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer), kubeclientset: config.KubeClient, podsLister: podInformer.Lister(), diff --git a/pkg/controller/init.go b/pkg/controller/init.go index 5977a75d041..99a446adc87 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -50,7 +50,7 @@ func InitDefaultLogicalSwitch(config *Configuration) error { } func InitNodeSwitch(config *Configuration) error { - client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) + client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) ss, err := client.ListLogicalSwitch() if err != nil { return err @@ -70,7 +70,7 @@ func InitNodeSwitch(config *Configuration) error { } func InitClusterRouter(config *Configuration) error { - client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) + client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) lrs, err := client.ListLogicalRouter() if err != nil { return err @@ -85,7 +85,7 @@ func InitClusterRouter(config *Configuration) error { } func InitLoadBalancer(config *Configuration) error { - client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) + client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) tcpLb, err := client.FindLoadbalancer(config.ClusterTcpLoadBalancer) if err != nil { return fmt.Errorf("failed to find tcp lb %v", err) @@ -121,7 +121,7 @@ func InitLoadBalancer(config *Configuration) error { } func InitDnsTable(config *Configuration) error { - client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) + client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer) uuid, err := client.CreateDnsTable() if err != nil { return err diff --git a/pkg/gateway/config.go b/pkg/gateway/config.go new file mode 100644 index 00000000000..26750bd331f --- /dev/null +++ b/pkg/gateway/config.go @@ -0,0 +1,61 @@ +package gateway + +import ( + "flag" + "github.com/spf13/pflag" +) + +type Configuration struct { + Interface string + SnatIP string + ClusterRouterIP string + EdgeRouterName string + EdgeRouterIP string + Chassis string + TransitSwitchName string + OutsideSwitchName string + ClusterRouterName string + OvnNbHost string + OvnNbPort int + OvnSbHost string + OvnSbPort int +} + +func ParseFlags() (*Configuration, error) { + var ( + argInterface = pflag.String("interface", "eth0", "The gateway interface") + argSnatIP = pflag.String("snat-ip", "", "The snat ip") + argClusterRouterIP = pflag.String("cluster-router-ip", "172.16.255.2/30", "The cluster route to transit switch ip") + argEdgeRouterIP = pflag.String("edge-router-ip", "172.16.255.1/30", "The edge router to transit switch ip") + argChassis = pflag.String("chassis", "", "chassis id found in ovn-sb") + argTransitSwitchName = pflag.String("transit-switch-name", "transit", "The name of switch between cluster router and edge router.") + argOutsideSwitchName = pflag.String("outside-switch-name", "outside", "The name of switch between edge router and physic interface.") + argEdgeRouterName = pflag.String("edge-router-name", "edge", "The edge router name") + argClusterRouterName = pflag.String("cluster-router-name", "ovn-cluster", "The cluster router name") + argOvnNbHost = pflag.String("ovn-nb-host", "", "") + argOvnNbPort = pflag.Int("ovn-nb-port", 6641, "") + argOvnSbHost = pflag.String("ovn-sb-host", "", "") + argOvnSbPort = pflag.Int("ovn-sb-port", 6642, "") + ) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + flag.CommandLine.Parse(make([]string, 0)) + + config := &Configuration{ + Interface: *argInterface, + SnatIP: *argSnatIP, + ClusterRouterIP: *argClusterRouterIP, + EdgeRouterIP: *argEdgeRouterIP, + EdgeRouterName: *argEdgeRouterName, + Chassis: *argChassis, + TransitSwitchName: *argTransitSwitchName, + ClusterRouterName: *argClusterRouterName, + OutsideSwitchName: *argOutsideSwitchName, + OvnNbHost: *argOvnNbHost, + OvnNbPort: *argOvnNbPort, + OvnSbHost: *argOvnSbHost, + OvnSbPort: *argOvnSbPort, + } + + return config, nil +} diff --git a/pkg/ovs/client.go b/pkg/ovs/client.go index 1f1fc66b5e8..198e9c3cc9b 100644 --- a/pkg/ovs/client.go +++ b/pkg/ovs/client.go @@ -11,6 +11,7 @@ import ( type Client struct { OvnNbAddress string + OvnSbAddress string ClusterRouter string ClusterTcpLoadBalancer string ClusterUdpLoadBalancer string @@ -32,7 +33,7 @@ func (c Client) ovnCommand(arg ...string) (string, error) { cmdArgs = append(cmdArgs, arg...) raw, err := exec.Command(OvnNbCtl, cmdArgs...).CombinedOutput() if err != nil { - return "", fmt.Errorf(string(raw)) + return "", fmt.Errorf("%s, %v", string(raw), err) } return trimCommandOutput(raw), nil } @@ -42,10 +43,11 @@ func trimCommandOutput(raw []byte) string { return strings.Trim(output, "\"") } -func NewClient(ovnNbHost string, ovnNbPort int, clusterRouter, ClusterTcpLoadBalancer, ClusterUdpLoadBalancer string) *Client { +func NewClient(ovnNbHost string, ovnNbPort int, ovnSbHost string, ovnSbPort int, ClusterRouter, ClusterTcpLoadBalancer, ClusterUdpLoadBalancer string) *Client { return &Client{ OvnNbAddress: fmt.Sprintf("tcp:%s:%d", ovnNbHost, ovnNbPort), - ClusterRouter: clusterRouter, + OvnSbAddress: fmt.Sprintf("tcp:%s:%d", ovnSbHost, ovnSbPort), + ClusterRouter: ClusterRouter, ClusterTcpLoadBalancer: ClusterTcpLoadBalancer, ClusterUdpLoadBalancer: ClusterUdpLoadBalancer, } @@ -114,6 +116,75 @@ type Nic struct { Gateway string } +func (c Client) CreateTransitLogicalSwitch(ls, clusterLr, edgeLr, toClusterIP, toEdgeIP string) error { + mac := util.GenerateMac() + edgeToTransit := fmt.Sprintf("%s-%s", edgeLr, ls) + transitToEdge := fmt.Sprintf("%s-%s", ls, edgeLr) + _, err := c.ovnCommand(WaitSb, MayExist, "ls-add", ls, "--", + "lrp-add", edgeLr, edgeToTransit, mac, toEdgeIP, "--", + "lsp-add", ls, transitToEdge, "--", + "lsp-set-type", transitToEdge, "router", "--", + "lsp-set-addresses", transitToEdge, mac, "--", + "lsp-set-options", transitToEdge, fmt.Sprintf("router-port=%s", edgeToTransit)) + if err != nil { + klog.Errorf("connect edge to transit failed %v", err) + return err + } + + mac = util.GenerateMac() + clusterToTransit := fmt.Sprintf("%s-%s", clusterLr, ls) + transitToCluster := fmt.Sprintf("%s-%s", ls, clusterLr) + _, err = c.ovnCommand("lrp-add", clusterLr, clusterToTransit, mac, toClusterIP, "--", + "lsp-add", ls, transitToCluster, "--", + "lsp-set-type", transitToCluster, "router", "--", + "lsp-set-addresses", transitToCluster, mac, "--", + "lsp-set-options", transitToCluster, fmt.Sprintf("router-port=%s", clusterToTransit)) + if err != nil { + klog.Errorf("connect cluster to transit failed %v", err) + return err + } + return nil +} + +func (c Client) CreateOutsideLogicalSwitch(ls, edgeLr, ip, mac string) error { + // 1. create outside logical switch + _, err := c.ovnCommand(WaitSb, MayExist, "ls-add", ls) + if err != nil { + klog.Errorf("create outside ls %s failed, %v", ls, err) + return err + } + + // 2. connect outside ls with edge lr + outsideToEdge := fmt.Sprintf("%s-%s", ls, edgeLr) + edgeToOutside := fmt.Sprintf("%s-%s", edgeLr, ls) + _, err = c.ovnCommand("lrp-add", edgeLr, edgeToOutside, mac, ip) + if err != nil { + klog.Errorf("create lsp on edge failed, %v", err) + return err + } + + _, err = c.ovnCommand(WaitSb, MayExist, "lsp-add", ls, outsideToEdge, "--", + "lsp-set-type", outsideToEdge, "router", "--", + "lsp-set-addresses", outsideToEdge, mac, "--", + "lsp-set-options", outsideToEdge, fmt.Sprintf("router-port=%s", edgeToOutside)) + if err != nil { + klog.Errorf("failed to connect outside to edge, %v", err) + return err + } + + // 3. create localnet port to connect outside to physic net + outsideToLocal := fmt.Sprintf("%s-localnet", ls) + _, err = c.ovnCommand(WaitSb, MayExist, "lsp-add", ls, outsideToLocal, "--", + "lsp-set-addresses", outsideToLocal, "unknown", "--", + "lsp-set-type", outsideToLocal, "localnet", "--", + "lsp-set-options", outsideToLocal, "network_name=dataNet") + if err != nil { + klog.Errorf("failed to create localnet port %v", err) + return err + } + return nil +} + func (c Client) CreateLogicalSwitch(ls, subnet, gateway, excludeIps string) error { _, err := c.ovnCommand(WaitSb, MayExist, "ls-add", ls, "--", "set", "logical_switch", ls, fmt.Sprintf("other_config:subnet=%s", subnet), "--", @@ -207,6 +278,12 @@ func (c Client) DeleteLogicalSwitch(ls string) error { return nil } +func (c Client) CreateGatewayRouter(lr, chassis string) error { + _, err := c.ovnCommand(WaitSb, MayExist, "lr-add", lr, "--", + "set", "logical_router", lr, fmt.Sprintf("options:chassis=%s", chassis)) + return err +} + func (c Client) CreateLogicalRouter(lr string) error { _, err := c.ovnCommand(WaitSb, MayExist, "lr-add", lr) return err diff --git a/pkg/util/nicstobridge.go b/pkg/util/nicstobridge.go new file mode 100644 index 00000000000..e78b8aba1ee --- /dev/null +++ b/pkg/util/nicstobridge.go @@ -0,0 +1,212 @@ +package util + +import ( + "fmt" + "os" + "os/exec" + "strings" + "syscall" + + "github.com/vishvananda/netlink" + "k8s.io/klog" +) + +func getBridgeName(iface string) string { + return fmt.Sprintf("gw%s", iface) +} + +// GetNicName returns the physical NIC name, given an OVS bridge name +// configured by NicToBridge() +func GetNicName(brName string) string { + stdout, err := exec.Command("ovs-vsctl", + "br-get-external-id", brName, "bridge-uplink").CombinedOutput() + if err != nil { + klog.Errorf("Failed to get the bridge-uplink for the bridge %q:, stderr: %q, error: %v", + brName, string(stdout), err) + return "" + } + if string(stdout) == "" && strings.HasPrefix(brName, "br") { + // This would happen if the bridge was created before the bridge-uplink + // changes got integrated. + return fmt.Sprintf("%s", brName[len("br"):]) + } + return string(stdout) +} + +func saveIPAddress(oldLink, newLink netlink.Link, addrs []netlink.Addr) error { + for i := range addrs { + addr := addrs[i] + + // Remove from oldLink + if err := netlink.AddrDel(oldLink, &addr); err != nil { + klog.Errorf("Remove addr from %q failed: %v", oldLink.Attrs().Name, err) + return err + } + + // Add to newLink + addr.Label = newLink.Attrs().Name + if err := netlink.AddrAdd(newLink, &addr); err != nil { + klog.Errorf("Add addr to newLink %q failed: %v", newLink.Attrs().Name, err) + return err + } + klog.Infof("Successfully saved addr %q to newLink %q", addr.String(), newLink.Attrs().Name) + } + + return netlink.LinkSetUp(newLink) +} + +// delAddRoute removes 'route' from 'oldLink' and moves to 'newLink' +func delAddRoute(oldLink, newLink netlink.Link, route netlink.Route) error { + // Remove route from old interface + if err := netlink.RouteDel(&route); err != nil && !strings.Contains(err.Error(), "no such process") { + klog.Errorf("Remove route from %q failed: %v", oldLink.Attrs().Name, err) + return err + } + + // Add route to newLink + route.LinkIndex = newLink.Attrs().Index + if err := netlink.RouteAdd(&route); err != nil && !os.IsExist(err) { + klog.Errorf("Add route to newLink %q failed: %v", newLink.Attrs().Name, err) + return err + } + + klog.Infof("Successfully saved route %q", route.String()) + return nil +} + +func saveRoute(oldLink, newLink netlink.Link, routes []netlink.Route) error { + for i := range routes { + route := routes[i] + + // Handle routes for default gateway later. This is a special case for + // GCE where we have /32 IP addresses and we can't add the default + // gateway before the route to the gateway. + if route.Dst == nil && route.Gw != nil && route.LinkIndex > 0 { + continue + } + + err := delAddRoute(oldLink, newLink, route) + if err != nil { + return err + } + } + + // Now add the default gateway (if any) via this interface. + for i := range routes { + route := routes[i] + if route.Dst == nil && route.Gw != nil && route.LinkIndex > 0 { + // Remove route from 'oldLink' and move it to 'newLink' + err := delAddRoute(oldLink, newLink, route) + if err != nil { + return err + } + } + } + + return nil +} + +// NicToBridge creates a OVS bridge for the 'iface' and also moves the IP +// address and routes of 'iface' to OVS bridge. +func NicToBridge(iface string) (netlink.Link, error) { + ifaceLink, err := netlink.LinkByName(iface) + if err != nil { + return nil, err + } + + bridge := getBridgeName(iface) + stdout, err := exec.Command("ovs-vsctl", + "--", "--may-exist", "add-br", bridge, + "--", "br-set-external-id", bridge, "bridge-id", bridge, + "--", "br-set-external-id", bridge, "bridge-uplink", iface, + "--", "set", "bridge", bridge, "fail-mode=standalone", + fmt.Sprintf("other_config:hwaddr=%s", ifaceLink.Attrs().HardwareAddr), + "--", "--may-exist", "add-port", bridge, iface, + "--", "set", "port", iface, "other-config:transient=true", + "--", "set", "Open_vSwitch", ".", fmt.Sprintf("external-ids:ovn-bridge-mappings=dataNet:%s", bridge)).CombinedOutput() + if err != nil { + klog.Errorf("Failed to create OVS bridge, %v", string(stdout)) + return nil, err + } + klog.Infof("Successfully created OVS bridge %q", bridge) + + // Get ip addresses and routes before any real operations. + addrs, err := netlink.AddrList(ifaceLink, syscall.AF_INET) + if err != nil { + return nil, err + } + routes, err := netlink.RouteList(ifaceLink, syscall.AF_INET) + if err != nil { + return nil, err + } + + bridgeLink, err := netlink.LinkByName(bridge) + if err != nil { + return nil, err + } + + // save ip addresses to bridge. + if err = saveIPAddress(ifaceLink, bridgeLink, addrs); err != nil { + return nil, err + } + + // save routes to bridge. + if err = saveRoute(ifaceLink, bridgeLink, routes); err != nil { + return nil, err + } + + return bridgeLink, nil +} + +// BridgeToNic moves the IP address and routes of internal port of the bridge to +// underlying NIC interface and deletes the OVS bridge. +func BridgeToNic(bridge string) error { + // Internal port is named same as the bridge + bridgeLink, err := netlink.LinkByName(bridge) + if err != nil { + return err + } + + // Get ip addresses and routes before any real operations. + addrs, err := netlink.AddrList(bridgeLink, syscall.AF_INET) + if err != nil { + return err + } + routes, err := netlink.RouteList(bridgeLink, syscall.AF_INET) + if err != nil { + return err + } + + ifaceLink, err := netlink.LinkByName(GetNicName(bridge)) + if err != nil { + return err + } + + // save ip addresses to iface. + if err = saveIPAddress(bridgeLink, ifaceLink, addrs); err != nil { + return err + } + + // save routes to iface. + if err = saveRoute(bridgeLink, ifaceLink, routes); err != nil { + return err + } + + // Now delete the bridge + stdout, err := exec.Command("ovs-vsctl", "--", "--if-exists", "del-br", bridge).CombinedOutput() + if err != nil { + klog.Errorf("Failed to delete OVS bridge, %v", string(stdout)) + return err + } + klog.Infof("Successfully deleted OVS bridge %q", bridge) + + // Now delete the patch port on the integration bridge, if present + stdout, err = exec.Command("ovs-vsctl", "--", "--if-exists", "del-port", "br-int", + fmt.Sprintf("k8s-patch-br-int-%s", bridge)).CombinedOutput() + if err != nil { + klog.Errorf("Failed to delete patch port on br-int, %v", string(stdout), err) + return err + } + + return nil +}