Changes to support TSA from supervisor (sonic-net#14691)

Why I did it
Support for SONIC chassis isolation using TSA and un-isolation using TSB from supervisor module

Work item tracking
Microsoft ADO (number only): 17826134
How I did it
When TSA is run on the supervisor, it triggers TSA on each of the linecards using the secure rexec infrastructure introduced in sonic-net/sonic-utilities#2701. User password is requested to allow secure login to linecards through ssh, before execution of TSA/TSB on the linecards

TSA of the chassis withdraws routes from all the external BGP neighbors on each linecard, in order to isolate the entire chassis. No route withdrawal is done from the internal BGP sessions between the linecards to prevent transient drops during internal route deletion. With these changes, complete isolation of a single linecard using TSA will not be possible (a separate CLI/script option will be introduced at a later time to achieve this)

Changes also include no-stats option with TSC for quick retrieval of the current system isolation state

This PR also reverts changes in sonic-net#11403

How to verify it
These changes have a dependency on sonic-net/sonic-utilities#2701 for testing

Run TSA from supervisor module and ensure transition to Maintenance mode on each linecard
Verify that all routes are withdrawn from eBGP neighbors on all linecards
Run TSB from supervisor module and ensure transition to Normal mode on each linecard
Verify that all routes are re-advertised from eBGP neighbors on all linecards
Run TSC no-stats from supervisor and verify that just the system maintenance state is returned from all linecards

dockers/docker-fpm-frr/TS

@@ -4,7 +4,7 @@ switch_type=`sonic-cfggen -d -v "DEVICE_METADATA['localhost']['switch_type']"`
 # Check whether the routemap is for internal BGP sessions.
 function is_internal_route_map()
 {
-    [[ "$1" =~ .*"_INTERNAL_".* && $switch_type != "chassis-packet" ]]
+    [[ "$1" =~ .*"_INTERNAL_".* || "$1" =~ .*"VOQ_".* ]]
 }
 
 function check_not_installed()

dockers/docker-fpm-frr/base_image_files/TSA

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+if [ -f /etc/sonic/chassisdb.conf ]; then
+  rexec all -c "TSA chassis"
+  echo "Please execute \"rexec all -c 'sudo config save -y'\" to preserve System mode in Maintenance after reboot\
+  or config reload on all linecards"
+  exit 0
+fi
 # toggle the mux to standby if dualtor and any mux active
 if
 [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.subtype | tr [:upper:] [:lower:])" == *"dualtor"* ]] &&
@@ -10,4 +16,11 @@ then
 fi
 
 /usr/bin/TS TSA
-echo "Please execute 'config save' to preserve System mode in Maintenance after reboot or config reload"
+if [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.type)" == *"SpineRouter"* ]] ; then
+  if [[ "$1" != "chassis" ]] ; then
+    echo "Please execute 'sudo config save' to preserve System mode in Maintenance after reboot or config reload"
+    echo -e "\nWARNING: Please execute 'TSA' on all other linecards of the chassis to fully isolate this device"    
+  fi
+else
+  echo "Please execute 'sudo config save' to preserve System mode in Maintenance after reboot or config reload"
+fi

dockers/docker-fpm-frr/base_image_files/TSB

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+# If run on supervisor of chassis, trigger remote execution of TSB on all linecards
+if [ -f /etc/sonic/chassisdb.conf ]; then
+  rexec all -c "TSB chassis"
+  echo "Please execute \"rexec all -c 'sudo config save -y'\" to preserve System mode in Normal state after reboot\
+ or config reload on all linecards"
+  exit 0
+fi
+
 # toggle the mux to auto if dualtor
 if [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.subtype | tr [:upper:] [:lower:])" == *"dualtor"* ]];
 then
@@ -8,4 +16,10 @@ then
 fi
 
 /usr/bin/TS TSB
-echo "Please execute 'config save' to preserve System mode in Normal state after reboot or config reload"
+if [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.type)" == *"SpineRouter"* ]] ; then
+  if [[ "$1" != "chassis" ]] ; then
+    echo "Please execute 'sudo config save' to preserve System mode in Normal state after reboot or config reload"    
+  fi
+else
+  echo "Please execute 'sudo config save' to preserve System mode in Normal state after reboot or config reload"
+fi

dockers/docker-fpm-frr/base_image_files/TSC

@@ -1,8 +1,16 @@
 #!/bin/bash
 
-/usr/bin/TS TSC
+if [ -f /etc/sonic/chassisdb.conf ]; then
+  if [[ $1 == "no-stats" ]]; then
+    rexec all -c "TSC no-stats"
+  else
+    rexec all -c "TSC"
+  fi
+  exit 0
+fi
 
-portstat -p 5
+/usr/bin/TS TSC
+[[ $1 != "no-stats" ]] && portstat -p 5
 
 if [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.subtype | tr [:upper:] [:lower:])" == *"dualtor"* ]]
 then

files/image_config/config-setup/config-setup

@@ -109,15 +109,7 @@ run_hookdir() {
 reload_minigraph()
 {
     echo "Reloading minigraph..."
-    if
-    [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.subtype | tr [:upper:] [:lower:])" == *"dualtor"* ]] ||
-    [[ "$(sonic-cfggen -d -v DEVICE_METADATA.localhost.type | tr [:upper:] [:lower:])" == *"leafrouter"* ]];
-    then
-        #Keep device isolated with traffic-shift-away option on LeafRouter and Dualtor
-        config load_minigraph -y -n -t
-    else
-        config load_minigraph -y -n
-    fi
+    config load_minigraph -y -n
     config save -y
 }
 

src/sonic-bgpcfgd/bgpcfgd/managers_device_global.py

@@ -90,7 +90,7 @@ def __generate_routemaps_from_template(self, route_map_names, template):
         for rm in sorted(route_map_names):
             # For packet-based chassis, the bgp session between the linecards are also considered internal sessions 
             # While isolating a single linecard, these sessions should not be skipped
-            if "_INTERNAL_" in rm and self.switch_type != "chassis-packet":
+            if "_INTERNAL_" in rm or "VOQ_" in rm:
                 continue            
             if "V4" in rm:
                 ipv="V4" ; ipp="ip"