From 7ab439f7d6f6e68fed8328509a30b73042398dae Mon Sep 17 00:00:00 2001 From: Zhenggen Xu Date: Tue, 7 Aug 2018 16:05:17 -0700 Subject: [PATCH 1/4] Fix potential blackholing/looping traffic and refresh ipv6 neighbor to avoid CPU hit In case ipv6 global addresses were configured on L3 interfaces and used for peering, and routing protocol was using link-local addresses on the same interfaces as prefered nexthops, the link-local addresses could be aged out after a while due to no activities towards the link-local addresses themselves. And when we receive new routes with the link-local nexthops, SONiC won't insert them to the HW, and thus cause looping or blackholing traffic. Global ipv6 addresses on L3 interfaces between switches are refreshed by BGP keeplive and other messages. On server facing side, traffic may hit fowarding plane only, and no refresh for the ipv6 neighbor entries regularly. This could age-out the linux kernel ipv6 neighbor entries, and HW neighbor table entries could be removed, and thus traffic going to those neighbors would hit CPU, and cause traffic drop and temperary CPU high load. Also, if link-local addresses were not learned, we may not get them at all later. It is intended to fix all above issues. Changes: Add ndisc6 package in swss docker and use it for ipv6 ndp ping to update the neighbors' state on Vlan interfaces Change the default ipv6 neighbor reachable timer to 30mins Add periodical ipv6 multicast ping to ff02::11 to get/refresh link-local neighbor info. --- build_debian.sh | 1 + dockers/docker-orchagent/Dockerfile.j2 | 1 + files/scripts/arp_update | 30 ++++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index 767dd7b83b79..8c38eca62e47 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -302,6 +302,7 @@ set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_notify 1 set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_ignore 2 set /files/etc/sysctl.conf/net.ipv4.neigh.default.base_reachable_time_ms 1800000 +set /files/etc/sysctl.conf/net.ipv6.neigh.default.base_reachable_time_ms 1800000 set /files/etc/sysctl.conf/net.ipv6.conf.default.forwarding 1 set /files/etc/sysctl.conf/net.ipv6.conf.all.forwarding 1 diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2 index eceb82a06c76..2d204df9078f 100755 --- a/dockers/docker-orchagent/Dockerfile.j2 +++ b/dockers/docker-orchagent/Dockerfile.j2 @@ -10,6 +10,7 @@ RUN apt-get update RUN apt-get install -f -y ifupdown arping libdbus-1-3 libdaemon0 libjansson4 +RUN apt-get install -f -y ndisc6 ## Install redis-tools dependencies ## TODO: implicitly install dependencies RUN apt-get -y install libjemalloc1 diff --git a/files/scripts/arp_update b/files/scripts/arp_update index ab44ca6a4580..749319194841 100755 --- a/files/scripts/arp_update +++ b/files/scripts/arp_update @@ -1,10 +1,24 @@ #!/bin/bash # # usage: -# arp_update: Send gratuitous ARP requests to VLAN member neighbors to refresh -# the neighbors state. +# arp_update: +# Send ipv6 multicast pings to all "UP" L3 interfaces including vlan interfaces to +# refresh link-local addresses from neighbors. +# Send gratuitous ARP/NDP requests to VLAN member neighbors to refresh +# the ipv4/ipv6 neighbors state. while /bin/true; do + # find L3 interfaces which are UP, send ipv6 multicast pings + echo "{% for (name, prefix) in INTERFACE %} {{name}} {% endfor %}" > /tmp/intf_tmp.j2 + INTERFACE=`sonic-cfggen -d -t /tmp/intf_tmp.j2` + for intf in $INTERFACE; do + ping6cmd="ping6 -I $intf -n -q -i 0 -c 1 -W 0 ff02::1 >/dev/null" + intf_up=$(ip link show $intf | grep "state UP") + if [[ -n "$intf_up" ]]; then + eval $ping6cmd + fi + done + VLAN=`sonic-cfggen -d -v 'VLAN.keys() | join(" ") if VLAN'` for vlan in $VLAN; do # generate a list of arping commands: @@ -15,6 +29,18 @@ while /bin/true; do ipcmd="ip -4 neigh show | grep $vlan | cut -d ' ' -f 1,3 | $arpingcmd" eval `eval $ipcmd` + + # send ipv6 multicast pings to Vlan interfaces to get/refresh link-local addrs + ping6cmd="ping6 -I $vlan -n -q -i 0 -c 1 -W 0 ff02::1 >/dev/null" + eval $ping6cmd + + # generate a list of ndisc6 commands (exclude link-local addrs since it is done above): + # ndisc6 -q -w 0 -1 ; + # ndisc6 -q -w 0 -1 ; + # ... + ndisc6cmd="sed -e 's/^/ndisc6 -q -w 0 -1 /' -e 's/$/;/'" + ip6cmd="ip -6 neigh show | grep -v fe80 | grep $vlan | cut -d ' ' -f 1,3 | $ndisc6cmd" + eval `eval $ip6cmd` done sleep 300 done From ca0f9ef3f3f03fac49da24ba1f943a2cd4322dec Mon Sep 17 00:00:00 2001 From: Zhenggen Xu Date: Wed, 8 Aug 2018 18:11:14 -0700 Subject: [PATCH 2/4] Fix review comments: Add PORTCHANNEL_INTERFACE interface for ipv6 multicast ping format issue --- files/scripts/arp_update | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/files/scripts/arp_update b/files/scripts/arp_update index 749319194841..4a4a985cef38 100755 --- a/files/scripts/arp_update +++ b/files/scripts/arp_update @@ -15,7 +15,17 @@ while /bin/true; do ping6cmd="ping6 -I $intf -n -q -i 0 -c 1 -W 0 ff02::1 >/dev/null" intf_up=$(ip link show $intf | grep "state UP") if [[ -n "$intf_up" ]]; then - eval $ping6cmd + eval $ping6cmd + fi + done + + echo "{% for (name, prefix) in PORTCHANNEL_INTERFACE %} {{name}} {% endfor %}" > /tmp/pc_intf_tmp.j2 + PC_INTERFACE=`sonic-cfggen -d -t /tmp/pc_intf_tmp.j2` + for intf in $PC_INTERFACE; do + ping6cmd="ping6 -I $intf -n -q -i 0 -c 1 -W 0 ff02::1 >/dev/null" + intf_up=$(ip link show $intf | grep "state UP") + if [[ -n "$intf_up" ]]; then + eval $ping6cmd fi done From 6f25dfff4dab569369d57df03913d2c9926d0c47 Mon Sep 17 00:00:00 2001 From: Zhenggen Xu Date: Wed, 8 Aug 2018 23:23:21 -0700 Subject: [PATCH 3/4] Combine regular L3 interface and portchannel interface for looping --- files/scripts/arp_update | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/files/scripts/arp_update b/files/scripts/arp_update index 4a4a985cef38..f7c5ec8bb856 100755 --- a/files/scripts/arp_update +++ b/files/scripts/arp_update @@ -11,17 +11,11 @@ while /bin/true; do # find L3 interfaces which are UP, send ipv6 multicast pings echo "{% for (name, prefix) in INTERFACE %} {{name}} {% endfor %}" > /tmp/intf_tmp.j2 INTERFACE=`sonic-cfggen -d -t /tmp/intf_tmp.j2` - for intf in $INTERFACE; do - ping6cmd="ping6 -I $intf -n -q -i 0 -c 1 -W 0 ff02::1 >/dev/null" - intf_up=$(ip link show $intf | grep "state UP") - if [[ -n "$intf_up" ]]; then - eval $ping6cmd - fi - done - echo "{% for (name, prefix) in PORTCHANNEL_INTERFACE %} {{name}} {% endfor %}" > /tmp/pc_intf_tmp.j2 PC_INTERFACE=`sonic-cfggen -d -t /tmp/pc_intf_tmp.j2` - for intf in $PC_INTERFACE; do + + ALL_INTERFACE="$INTERFACE $PC_INTERFACE" + for intf in $ALL_INTERFACE; do ping6cmd="ping6 -I $intf -n -q -i 0 -c 1 -W 0 ff02::1 >/dev/null" intf_up=$(ip link show $intf | grep "state UP") if [[ -n "$intf_up" ]]; then From d7baed2210dd12b1c6e458ff7291dc144a23a32a Mon Sep 17 00:00:00 2001 From: Zhenggen Xu Date: Fri, 10 Aug 2018 14:37:08 -0700 Subject: [PATCH 4/4] Add ndisc6 package to vs docker --- platform/vs/docker-sonic-vs/Dockerfile.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/vs/docker-sonic-vs/Dockerfile.j2 b/platform/vs/docker-sonic-vs/Dockerfile.j2 index 2d9b72aa3298..ded75caff524 100644 --- a/platform/vs/docker-sonic-vs/Dockerfile.j2 +++ b/platform/vs/docker-sonic-vs/Dockerfile.j2 @@ -13,6 +13,7 @@ RUN apt-get update RUN apt-get install -y net-tools \ arping \ + ndisc6 \ ethtool \ tcpdump \ ifupdown \