chassis-packet: resolve the missing static routes (sonic-net#14230)

arp_update should resolve the missing arp/ndp static route
entries. Added code to check for missing entries and try ping to
resolve the missing entry.

Why I did it
Fixes sonic-net#14179

chassis-packet: missing arp entries for static routes causing high orchagent cpu usage

It is observed that some sonic-mgmt test case calls sonic-clear arp, which clears the static arp entries as well. Orchagent or arp_update process does not try to resolve the missing arp entries after clear.

How I did it
arp_update should resolve the missing arp/ndp static route
entries. Added code to check for missing entries and try ping if any
found to resolve it.

How to verify it
After boot or config reload, check ipv4 and ipv4 neigh entries to make sure all static route entries are present
manual validation:
Use sonic-clear arp and sonic-clear ndp to clear all neighbor entries
run arp_update
Check for neigh entries. All entries should be present.

Signed-off-by: anamehra <anamehra@cisco.com>

files/build_templates/arp_update_vars.j2

@@ -4,5 +4,6 @@
     "pc_interface" : "{% for (name, prefix) in PORTCHANNEL_INTERFACE|pfx_filter %}{% if prefix|ipv6 %}{{ name }} {% endif %}{% endfor %}",
     "vlan_sub_interface": "{% for (name, prefix) in VLAN_SUB_INTERFACE|pfx_filter %}{% if prefix|ipv6 %}{{ name }} {% endif %}{% endfor %}",
     "vlan" : "{% if VLAN %}{{ VLAN.keys() | join(' ') }}{% endif %}",
-    "static_route_nexthops": "{% if STATIC_ROUTE %}{% for static_route_prefix, static_route_attr in STATIC_ROUTE.items() %}{%- if static_route_prefix -%}{{ static_route_attr['nexthop'].split(',') | join(' ') | lower + " " }}{%- endif -%}{% endfor %}{% endif %}"
+    "static_route_nexthops": "{% if STATIC_ROUTE %}{% for static_route_prefix, static_route_attr in STATIC_ROUTE.items() %}{%- if static_route_prefix -%}{{ static_route_attr['nexthop'].split(',') | join(' ') | lower + " " }}{%- endif -%}{% endfor %}{% endif %}",
+    "static_route_ifnames": "{% if STATIC_ROUTE %}{% for static_route_prefix, static_route_attr in STATIC_ROUTE.items() %}{%- if static_route_prefix -%}{{ static_route_attr['ifname'].split(',') | join(' ') + " " }}{%- endif -%}{% endfor %}{% endif %}"
 }

files/scripts/arp_update

@@ -14,25 +14,36 @@ while /bin/true; do
   ARP_UPDATE_VARS=$(sonic-cfggen -d -t ${ARP_UPDATE_VARS_FILE})
   SWITCH_TYPE=$(echo $ARP_UPDATE_VARS | jq -r '.switch_type')
   if [[ "$SWITCH_TYPE" == "chassis-packet" ]]; then
-      STATIC_ROUTE_NEXTHOPS=$(echo $ARP_UPDATE_VARS | jq -r '.static_route_nexthops')
+      # Get array of Nexthops and ifnames. Nexthops and ifnames are mapped one to one
+      STATIC_ROUTE_NEXTHOPS=($(echo $ARP_UPDATE_VARS | jq -r '.static_route_nexthops'))
+      STATIC_ROUTE_IFNAMES=($(echo $ARP_UPDATE_VARS | jq -r '.static_route_ifnames'))
       # on supervisor/rp exit the script gracefully
-      if [[ -z "$STATIC_ROUTE_NEXTHOPS" ]]; then
+      if [[ -z "$STATIC_ROUTE_NEXTHOPS" ]] || [[ -z "$STATIC_ROUTE_IFNAMES" ]]; then
           logger "arp_update: exiting as no static route in packet based chassis"
           exit 0
       fi
-      for nexthop in $STATIC_ROUTE_NEXTHOPS; do
+      for i in ${!STATIC_ROUTE_NEXTHOPS[@]}; do
+          nexthop="${STATIC_ROUTE_NEXTHOPS[i]}"
           if [[ $nexthop == *"."* ]]; then
               neigh_state=( $(ip -4 neigh show | grep -w $nexthop | tr -s ' ' | cut -d ' ' -f 3,4) )
               ping_prefix=ping
           elif [[ $nexthop == *":"* ]] ; then
               neigh_state=( $(ip -6 neigh show | grep -w $nexthop | tr -s ' ' | cut -d ' ' -f 3,4) )
               ping_prefix=ping6
           fi
-
-          if [[ "${neigh_state[1]}" == "INCOMPLETE" ]] || [[ "${neigh_state[1]}" == "FAILED" ]]; then
-              pingcmd="timeout 0.2 $ping_prefix -I ${neigh_state[0]} -n -q -i 0 -c 1 -W 1 $nexthop >/dev/null"
-              eval $pingcmd
-              logger "arp_update: sttaic route nexthop not resolved, pinging $nexthop on ${neigh_state[0]}"
+          if [[ -z "${neigh_state}" ]] || [[ "${neigh_state[1]}" == "INCOMPLETE" ]] || [[ "${neigh_state[1]}" == "FAILED" ]]; then
+              interface="${STATIC_ROUTE_IFNAMES[i]}"
+              if [[ -z "$interface" ]]; then
+                  # should never be here, handling just in case
+                  logger "ERR: arp_update: missing interface entry for static route $nexthop"
+                  interface=${neigh_state[0]}
+              fi
+              intf_up=$(ip link show $interface | grep "state UP")
+              if [[ -n "$intf_up" ]]; then
+                  pingcmd="timeout 0.2 $ping_prefix -I ${interface} -n -q -i 0 -c 1 -W 1 $nexthop >/dev/null"
+                  eval $pingcmd
+                  logger "arp_update: static route nexthop not resolved, pinging $nexthop on ${neigh_state[0]}"
+              fi
           fi
       done