Skip to content

Commit

Permalink
check IPv4 gateway by resolving gateway MAC in underlay subnets
Browse files Browse the repository at this point in the history
  • Loading branch information
zhangzujian committed Dec 6, 2021
1 parent 90f62fd commit 3837b0a
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 42 deletions.
2 changes: 1 addition & 1 deletion docs/vlan.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ In the Vlan/Underlay mode, OVS sends origin Pods packets directly to the physica
1. For K8s running on VMs provided by OpenStack, `PortSecurity` of the network ports MUST be `disabled`;
2. For K8s running on VMs provided by VMware, the switch security options `MAC Address Changes`, `Forged Transmits` and `Promiscuous Mode Operation` MUST be `allowed`;
3. The Vlan/Underlay mode can not run on public IaaS providers like AWS/GCE/Alibaba Cloud as their network can not provide the capability to transmit this type packets;
4. When Kube-OVN creates network it checks the connectivity to the subnet gateway through ICMP, so the gateway MUST respond the ICMP messages;
4. In versions prior to v1.9.0, Kube-OVN checks the connectivity to the subnet gateway through ICMP, so the gateway MUST respond the ICMP messages if you are using those versions, or you can turn off the check by setting `disableGatewayCheck` to `true` which is introduced in v1.8.0;
5. For in-cluster service traffic, Pods set the dst mac to gateway mac and then Kube-OVN applies DNAT to transfer the dst ip, the packets will first be sent to the gateway, so the gateway MUST be capable of transmitting the packets back to the subnet.

## Comparison with Macvlan
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ require (
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.0
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc
github.com/moul/http2curl v1.0.0 // indirect
github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9
github.com/oilbeater/go-ping v0.0.0-20200413021620-332b7197c5b5
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,12 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc h1:m7rJJJeXrYCFpsxXYapkDW53wJCDmf9bsIXUg0HoeQY=
github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc/go.mod h1:eOj1DDj3NAZ6yv+WafaKzY37MFZ58TdfIhQ+8nQbiis=
github.com/mdlayher/ethernet v0.0.0-20190313224307-5b5fc417d966 h1:O3p5UmisBhl3V6lgs4Vdfg8HpjzbWJPyOfGLdwVJSmI=
github.com/mdlayher/ethernet v0.0.0-20190313224307-5b5fc417d966/go.mod h1:5s5p/sMJ6sNsFl6uCh85lkFGV8kLuIYJCRJLavVJwvg=
github.com/mdlayher/raw v0.0.0-20190313224157-43dbcdd7739d h1:rjAS0af7FIYCScTtEU5KjIldC6qVaEScUJhABHC+ccM=
github.com/mdlayher/raw v0.0.0-20190313224157-43dbcdd7739d/go.mod h1:r1fbeITl2xL/zLbVnNHFyOzQJTgr/3fpf1lJX/cjzR8=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
Expand Down Expand Up @@ -637,6 +643,7 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
Expand Down
22 changes: 18 additions & 4 deletions pkg/daemon/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ import (
"github.com/kubeovn/kube-ovn/pkg/util"
)

const (
gatewayModeDisabled = iota
gatewayCheckModePing
gatewayCheckModeArping
)

type cniServerHandler struct {
Config *Configuration
KubeClient kubernetes.Interface
Expand Down Expand Up @@ -69,7 +75,8 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
}

klog.Infof("add port request %v", podRequest)
var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, providerNetwork, ifName, nicType, netns, podNicName, priority string
var gatewayCheckMode int
var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, providerNetwork, ifName, nicType, podNicName, priority string
var isDefaultRoute bool
var pod *v1.Pod
var err error
Expand Down Expand Up @@ -113,7 +120,6 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
} else {
nicType = pod.Annotations[util.PodNicAnnotation]
}
netns = podRequest.NetNs

switch pod.Annotations[fmt.Sprintf(util.DefaultRouteAnnotationTemplate, podRequest.Provider)] {
case "true":
Expand Down Expand Up @@ -158,6 +164,14 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
return
}

if !podSubnet.Spec.DisableGatewayCheck {
if podSubnet.Spec.Vlan != "" && !podSubnet.Spec.LogicalGateway {
gatewayCheckMode = gatewayCheckModeArping
} else {
gatewayCheckMode = gatewayCheckModePing
}
}

var mtu int
if providerNetwork != "" {
node, err := csh.Controller.nodesLister.Get(csh.Config.NodeName)
Expand Down Expand Up @@ -186,10 +200,10 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon

klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, podRequest.Routes)
if nicType == util.InternalType {
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, priority, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode)
} else {
podNicName = ifName
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, priority, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode)
}
if err != nil {
errMsg := fmt.Errorf("configure nic failed %v", err)
Expand Down
96 changes: 59 additions & 37 deletions pkg/daemon/ovs.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@ import (
"github.com/kubeovn/kube-ovn/pkg/util"
)

const gatewayCheckMaxRetry = 200

var pciAddrRegexp = regexp.MustCompile(`\b([0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}.\d{1}\S*)`)

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, priority, DeviceID, nicType, podNetns string, checkGw bool) error {
func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) error {
var err error
var hostNicName, containerNicName string
if DeviceID == "" {
Expand All @@ -53,7 +55,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
fmt.Sprintf("external_ids:pod_name=%s", podName),
fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),
fmt.Sprintf("external_ids:ip=%s", ipStr),
fmt.Sprintf("external_ids:pod_netns=%s", podNetns))
fmt.Sprintf("external_ids:pod_netns=%s", netns))
if err != nil {
return fmt.Errorf("add nic to ovs failed %v: %q", err, output)
}
Expand All @@ -77,7 +79,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", netns, err)
}
if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, routes, macAddr, podNS, mtu, nicType, gwCheckMode); err != nil {
return err
}
return nil
Expand Down Expand Up @@ -157,7 +159,7 @@ func configureHostNic(nicName string) error {
return nil
}

func configureContainerNic(nicName, ifName string, ipAddr, gateway string, isDefaultRoute bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, checkGw bool) error {
func configureContainerNic(nicName, ifName string, ipAddr, gateway string, isDefaultRoute bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, gwCheckMode int) error {
containerLink, err := netlink.LinkByName(nicName)
if err != nil {
return fmt.Errorf("can not find container nic %s: %v", nicName, err)
Expand All @@ -170,7 +172,7 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, isDef
}

if err = netlink.LinkSetNsFd(containerLink, int(netns.Fd())); err != nil {
return fmt.Errorf("failed to link netns: %v", err)
return fmt.Errorf("failed to move link to netns: %v", err)
}

return ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error {
Expand Down Expand Up @@ -283,40 +285,59 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, isDef
}
}

if checkGw {
return waitNetworkReady(ipAddr, gateway, true)
if gwCheckMode != gatewayModeDisabled {
underlayGateway := gwCheckMode == gatewayCheckModeArping
if nicType != util.InternalType {
return waitNetworkReady(ifName, ipAddr, gateway, underlayGateway, true)
}
return waitNetworkReady(nicName, ipAddr, gateway, underlayGateway, true)
}

return nil
})
}

func waitNetworkReady(src, gateway string, verbose bool) error {
for _, gw := range strings.Split(gateway, ",") {
pinger, err := goping.NewPinger(gw)
if err != nil {
return fmt.Errorf("failed to init pinger: %v", err)
}
pinger.SetPrivileged(true)
// CNITimeoutSec = 220, cannot exceed
count := 200
pinger.Count = count
pinger.Timeout = time.Duration(count) * time.Second
pinger.Interval = 1 * time.Second

success := false
pinger.OnRecv = func(p *goping.Packet) {
success = true
pinger.Stop()
}
pinger.Run()
func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool) error {
ips := strings.Split(ipAddr, ",")
for i, gw := range strings.Split(gateway, ",") {
src := strings.Split(ips[i], "/")[0]
if underlayGateway && util.CheckProtocol(gw) == kubeovnv1.ProtocolIPv4 {
mac, count, err := util.Arping(nic, src, gw, time.Second, gatewayCheckMaxRetry)
cniConnectivityResult.WithLabelValues(nodeName).Add(float64(count))
if err != nil {
err = fmt.Errorf("network %s with gateway %s is not ready for interface %s after %d checks: %v", ips[i], gw, nic, count, err)
klog.Warning(err)
return err
}
if verbose {
klog.Infof("MAC addresses of gateway %s is %s", gw, mac.String())
klog.Infof("network %s with gateway %s is ready for interface %s after %d checks", ips[i], gw, nic, count)
}
} else {
pinger, err := goping.NewPinger(gw)
if err != nil {
return fmt.Errorf("failed to init pinger: %v", err)
}
pinger.SetPrivileged(true)
// CNITimeoutSec = 220, cannot exceed
pinger.Count = gatewayCheckMaxRetry
pinger.Timeout = gatewayCheckMaxRetry * time.Second
pinger.Interval = time.Second

var success bool
pinger.OnRecv = func(p *goping.Packet) {
success = true
pinger.Stop()
}
pinger.Run()

cniConnectivityResult.WithLabelValues(nodeName).Add(float64(pinger.PacketsSent))
if !success {
return fmt.Errorf("%s network not ready after %d ping %s", src, count, gw)
}
if verbose {
klog.Infof("%s network ready after %d ping, gw %s", src, pinger.PacketsSent, gw)
cniConnectivityResult.WithLabelValues(nodeName).Add(float64(pinger.PacketsSent))
if !success {
return fmt.Errorf("%s network not ready after %d ping %s", src, gatewayCheckMaxRetry, gw)
}
if verbose {
klog.Infof("%s network ready after %d ping, gw %s", src, pinger.PacketsSent, gw)
}
}
}
return nil
Expand Down Expand Up @@ -347,7 +368,7 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int
}

// ping ovn0 gw to activate the flow
if err := waitNetworkReady(util.NodeNic, gw, true); err != nil {
if err := waitNetworkReady(util.NodeNic, ip, gw, false, true); err != nil {
klog.Errorf("failed to init ovn0 check: %v", err)
return err
}
Expand All @@ -372,8 +393,9 @@ func (c *Controller) loopOvn0Check() {
klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)
return
}
ip := node.Annotations[util.IpAddressAnnotation]
gw := node.Annotations[util.GatewayAnnotation]
if err := waitNetworkReady(util.NodeNic, gw, false); err != nil {
if err := waitNetworkReady(util.NodeNic, ip, gw, false, false); err != nil {
klog.Fatalf("failed to ping ovn0 gw: %s, %v", gw, err)
}
}
Expand Down Expand Up @@ -849,7 +871,7 @@ func renameLink(curName, newName string) error {
return nil
}

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, priority, DeviceID, nicType, podNetns string, checkGw bool) (string, error) {
func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) (string, error) {
_, containerNicName := generateNicName(containerID, ifName)
ipStr := util.GetIpWithoutMask(ip)
ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)
Expand All @@ -862,7 +884,7 @@ func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace,
fmt.Sprintf("external_ids:pod_name=%s", podName),
fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),
fmt.Sprintf("external_ids:ip=%s", ipStr),
fmt.Sprintf("external_ids:pod_netns=%s", podNetns))
fmt.Sprintf("external_ids:pod_netns=%s", netns))
if err != nil {
return containerNicName, fmt.Errorf("add nic to ovs failed %v: %q", err, output)
}
Expand All @@ -881,7 +903,7 @@ func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace,
if err != nil {
return containerNicName, fmt.Errorf("failed to open netns %q: %v", netns, err)
}
if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, routes, macAddr, podNS, mtu, nicType, gwCheckMode); err != nil {
return containerNicName, err
}
return containerNicName, nil
Expand Down
52 changes: 52 additions & 0 deletions pkg/util/arping.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package util

import (
"fmt"
"net"
"time"

"github.com/mdlayher/arp"
)

func Arping(nic, srcIP, dstIP string, timeout time.Duration, maxRetry int) (net.HardwareAddr, int, error) {
target := net.ParseIP(dstIP)
if target == nil {
return nil, 0, fmt.Errorf("%s is not a valid IP address", dstIP)
}

var count int
var err error
var ifi *net.Interface
for ; count < maxRetry; count++ {
if ifi, err = net.InterfaceByName(nic); err == nil {
break
}
time.Sleep(timeout)
}
if err != nil {
return nil, count, fmt.Errorf("failed to get interface %s: %v", nic, err)
}

var client *arp.Client
for ; count < maxRetry; count++ {
if client, err = arp.Dial(ifi); err == nil {
defer client.Close()
break
}
time.Sleep(timeout)
}
if err != nil {
return nil, count, fmt.Errorf("failed to set up ARP client: %v", err)
}

for ; count < maxRetry; count++ {
if err = client.SetDeadline(time.Now().Add(timeout)); err != nil {
continue
}
if mac, err := client.Resolve(target); err == nil {
return mac, count + 1, nil
}
}

return nil, count, fmt.Errorf("resolve MAC address of %s timeout: %v", dstIP, err)
}

0 comments on commit 3837b0a

Please sign in to comment.