[test-gap] Test IPinIP decap after VXLAN setup and teardown (#5834)

Fix test-gap to address a recently discovered issue where ipinip decap functionality failed after CPA teardown (teardown of vxlan tunnel + mirror acls).
A recent warmboot issue highlighted that decap functionality will fail after CPA (control plane assistant) session was teardown. Although CPA is relevant during warmboot, but this issue does not necessarily require warmboot.
Just setting up and teardown CPA session will trigger this.
I this PR, a new param vxlan to test_decap is added. Now all the cases in decap will run with or without a prestep of vxlan set-and-unset.

Changes made:
Before running decap test, if vxlan set_unset knob is enabled, create a CPA session and then tear it down.
Also, do not apply the config from template. This will ensure new tables are not created, and hence the test will run with default config loaded on the device.

tests/common/plugins/conditional_mark/tests_mark_conditions.yaml

@@ -39,23 +39,37 @@ copp/test_copp.py::TestCOPP::test_trap_config_save_after_reboot:
 #######################################
 #####            decap            #####
 #######################################
-decap/test_decap.py::test_decap[ttl=pipe, dscp=pipe]:
+decap/test_decap.py::test_decap[ttl=pipe, dscp=pipe, vxlan=disable]:
   skip:
     reason: "Not supported on broadcom after 201911 release, mellanox all releases and cisco-8000 all releases"
     conditions:
       - "(asic_type in ['broadcom'] and release not in ['201811', '201911']) or (asic_type in ['mellanox']) or (asic_type in ['cisco-8000'])"
-
-decap/test_decap.py::test_decap[ttl=pipe, dscp=uniform]:
+decap/test_decap.py::test_decap[ttl=pipe, dscp=uniform, vxlan=disable]:
   skip:
     reason: "Not supported on backend and broadcom before 202012 release"
     conditions:
       - "(topo_name in ['t1-backend', 't0-backend']) or (asic_type in ['broadcom'] and release in ['201811', '201911'])"
-
-decap/test_decap.py::test_decap[ttl=uniform, dscp=pipe]:
+decap/test_decap.py::test_decap[ttl=uniform, dscp=pipe, vxlan=disable]:
+  skip:
+    reason: "Not supported uniform ttl mode"
+decap/test_decap.py::test_decap[ttl=uniform, dscp=uniform, vxlan=disable]:
   skip:
     reason: "Not supported uniform ttl mode"
 
-decap/test_decap.py::test_decap[ttl=uniform, dscp=uniform]:
+decap/test_decap.py::test_decap[ttl=pipe, dscp=pipe, vxlan=set_unset]:
+  skip:
+    reason: "Not supported on broadcom after 201911 release, mellanox all releases and cisco-8000 all releases"
+    conditions:
+      - "(asic_type in ['broadcom'] and release not in ['201811', '201911']) or (asic_type in ['mellanox']) or (asic_type in ['cisco-8000'])"
+decap/test_decap.py::test_decap[ttl=pipe, dscp=uniform, vxlan=set_unset]:
+  skip:
+    reason: "Not supported on backend and broadcom before 202012 release"
+    conditions:
+      - "(topo_name in ['t1-backend', 't0-backend']) or (asic_type in ['broadcom'] and release in ['201811', '201911'])"
+decap/test_decap.py::test_decap[ttl=uniform, dscp=pipe, vxlan=set_unset]:
+  skip:
+    reason: "Not supported uniform ttl mode"
+decap/test_decap.py::test_decap[ttl=uniform, dscp=uniform, vxlan=set_unset]:
   skip:
     reason: "Not supported uniform ttl mode"
 

tests/common/utilities.py

@@ -548,3 +548,70 @@ def str2bool(str):
     :return: False if value is 0 or false, else True
     """
     return str.lower() not in ["0", "false", "no"]
+
+
+def setup_ferret(duthost, ptfhost, tbinfo):
+    '''
+        Sets Ferret service on PTF host.
+    '''
+    VXLAN_CONFIG_FILE = '/tmp/vxlan_decap.json'
+
+    def prepareVxlanConfigData(duthost, ptfhost, tbinfo):
+        '''
+            Prepares Vxlan Configuration data for Ferret service running on PTF host
+
+            Args:
+                duthost (AnsibleHost): Device Under Test (DUT)
+                ptfhost (AnsibleHost): Packet Test Framework (PTF)
+
+            Returns:
+                None
+        '''
+        mgFacts = duthost.get_extended_minigraph_facts(tbinfo)
+        vxlanConfigData = {
+            'minigraph_port_indices': mgFacts['minigraph_ptf_indices'],
+            'minigraph_portchannel_interfaces': mgFacts['minigraph_portchannel_interfaces'],
+            'minigraph_portchannels': mgFacts['minigraph_portchannels'],
+            'minigraph_lo_interfaces': mgFacts['minigraph_lo_interfaces'],
+            'minigraph_vlans': mgFacts['minigraph_vlans'],
+            'minigraph_vlan_interfaces': mgFacts['minigraph_vlan_interfaces'],
+            'dut_mac': duthost.facts['router_mac']
+        }
+        with open(VXLAN_CONFIG_FILE, 'w') as file:
+            file.write(json.dumps(vxlanConfigData, indent=4))
+
+        logger.info('Copying ferret config file to {0}'.format(ptfhost.hostname))
+        ptfhost.copy(src=VXLAN_CONFIG_FILE, dest='/tmp/')
+
+    ptfhost.copy(src="arp/files/ferret.py", dest="/opt")
+    result = duthost.shell(
+        cmd='''ip route show type unicast |
+        sed -e '/proto 186\|proto zebra\|proto bgp/!d' -e '/default/d' -ne '/0\//p' |
+        head -n 1 |
+        sed -ne 's/0\/.*$/1/p'
+        '''
+    )
+    dip = result['stdout']
+    logger.info('VxLan Sender {0}'.format(dip))
+    vxlan_port_out = duthost.shell('redis-cli -n 0 hget "SWITCH_TABLE:switch" "vxlan_port"')
+    if 'stdout' in vxlan_port_out and vxlan_port_out['stdout'].isdigit():
+        vxlan_port = int(vxlan_port_out['stdout'])
+        ferret_args = '-f /tmp/vxlan_decap.json -s {0} -a {1} -p {2}'.format(
+            dip, duthost.facts["asic_type"], vxlan_port)
+    else:
+        ferret_args = '-f /tmp/vxlan_decap.json -s {0} -a {1}'.format(dip, duthost.facts["asic_type"])
+
+    ptfhost.host.options['variable_manager'].extra_vars.update({'ferret_args': ferret_args})
+    logger.info('Copying ferret config file to {0}'.format(ptfhost.hostname))
+    ptfhost.template(src='arp/files/ferret.conf.j2', dest='/etc/supervisor/conf.d/ferret.conf')
+
+    logger.info('Generate pem and key files for ssl')
+    ptfhost.command(
+        cmd='''openssl req -new -x509 -keyout test.key -out test.pem -days 365 -nodes
+        -subj "/C=10/ST=Test/L=Test/O=Test/OU=Test/CN=test.com"''',
+        chdir='/opt'
+    )
+    prepareVxlanConfigData(duthost, ptfhost, tbinfo)
+    logger.info('Refreshing supervisor control with ferret configuration')
+    ptfhost.shell('supervisorctl reread && supervisorctl update')
+    ptfhost.shell('supervisorctl restart ferret')

tests/decap/conftest.py

@@ -1,20 +1,25 @@
-
 def build_ttl_dscp_params(uniform_support_info):
-    ttl_uni = {'ttl': 'uniform', 'dscp': 'pipe'}
-    dscp_uni = {'ttl': 'pipe', 'dscp': 'uniform'}
-    both_pipe = {'ttl': 'pipe', 'dscp': 'pipe'}
+    ttl_uni_vxlan = {'ttl': 'uniform', 'dscp': 'pipe', 'vxlan': 'set_unset'}
+    dscp_uni_vxlan = {'ttl': 'pipe', 'dscp': 'uniform', 'vxlan': 'set_unset'}
+    both_pipe_vxlan = {'ttl': 'pipe', 'dscp': 'pipe', 'vxlan': 'set_unset'}
+    ttl_uni = {'ttl': 'uniform', 'dscp': 'pipe', 'vxlan': 'disable'}
+    dscp_uni = {'ttl': 'pipe', 'dscp': 'uniform', 'vxlan': 'disable'}
+    both_pipe = {'ttl': 'pipe', 'dscp': 'pipe', 'vxlan': 'disable'}
     params = []
     if uniform_support_info['ttl']:
         params.append(ttl_uni)
+        params.append(ttl_uni_vxlan)
     if uniform_support_info['dscp']:
         params.append(dscp_uni)
-    if len(params) < 2:
+        params.append(dscp_uni_vxlan)
+    if len(params) < 4:
         params.append(both_pipe)
+        params.append(both_pipe_vxlan)
     return params
 
 def pytest_generate_tests(metafunc):
   ttl = metafunc.config.getoption("ttl_uniform")
   dscp = metafunc.config.getoption("dscp_uniform")
   if "supported_ttl_dscp_params" in metafunc.fixturenames:
       params = build_ttl_dscp_params({'ttl': ttl, 'dscp': dscp})
-      metafunc.parametrize("supported_ttl_dscp_params", params, ids=lambda p: "ttl=%s, dscp=%s" % (p['ttl'], p['dscp']), scope="module")
+      metafunc.parametrize("supported_ttl_dscp_params", params, ids=lambda p: "ttl=%s, dscp=%s, vxlan=%s" % (p['ttl'], p['dscp'], p['vxlan']), scope="module")

tests/decap/test_decap.py

@@ -10,6 +10,7 @@
 import json
 import logging
 from datetime import datetime
+import time
 
 import pytest
 import requests
@@ -27,7 +28,7 @@
 from tests.ptf_runner import ptf_runner
 from tests.common.helpers.assertions import pytest_assert as pt_assert
 from tests.common.dualtor.mux_simulator_control import mux_server_url
-from tests.common.utilities import wait
+from tests.common.utilities import wait, setup_ferret
 
 logger = logging.getLogger(__name__)
 
@@ -75,8 +76,6 @@ def loopback_ips(duthosts, duts_running_config_facts):
     lo_ips = []
     lo_ipv6s = []
     for duthost in duthosts:
-        if duthost.is_supervisor_node():
-            continue
         cfg_facts = duts_running_config_facts[duthost.hostname]
         lo_ip = None
         lo_ipv6 = None
@@ -93,8 +92,10 @@ def loopback_ips(duthosts, duts_running_config_facts):
 
 
 @pytest.fixture(scope='module')
-def setup_teardown(request, duthosts, duts_running_config_facts, ip_ver, loopback_ips, fib_info_files, single_fib_for_duts):
+def setup_teardown(request, duthosts, duts_running_config_facts, ip_ver, loopback_ips,
+    fib_info_files, single_fib_for_duts, supported_ttl_dscp_params):
 
+    vxlan = supported_ttl_dscp_params['vxlan']
     is_multi_asic = duthosts[0].sonichost.is_multi_asic
 
     setup_info = {
@@ -108,13 +109,15 @@ def setup_teardown(request, duthosts, duts_running_config_facts, ip_ver, loopbac
     setup_info.update(loopback_ips)
     logger.info(json.dumps(setup_info, indent=2))
 
-    # Remove default tunnel
-    remove_default_decap_cfg(duthosts)
+    if vxlan != "set_unset":
+        # Remove default tunnel
+        remove_default_decap_cfg(duthosts)
 
     yield setup_info
 
-    # Restore default tunnel
-    restore_default_decap_cfg(duthosts)
+    if vxlan != "set_unset":
+        # Restore default tunnel
+        restore_default_decap_cfg(duthosts)
 
 
 def apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mode, op):
@@ -148,29 +151,51 @@ def apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mod
             duthost.shell_cmds(cmds=cmds)
         duthost.shell('rm /tmp/decap_conf_{}.json'.format(op))
 
+
 def set_mux_side(tbinfo, mux_server_url, side):
     if 'dualtor' in tbinfo['topo']['name']:
         res = requests.post(mux_server_url, json={"active_side": side})
         pt_assert(res.status_code==200, 'Failed to set active side: {}'.format(res.text))
         return res.json()   # Response is new mux_status of all mux Y-cables.
     return {}
 
+
+def simulate_vxlan_teardown(duthosts, ptfhost, tbinfo):
+    for duthost in duthosts:
+        setup_ferret(duthost, ptfhost, tbinfo)
+        reboot_script_path = duthost.shell('which {}'.format("neighbor_advertiser"))['stdout']
+        ptf_ip = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host']
+        duthost.shell("{} -s {} -m set".format(reboot_script_path, ptf_ip))
+        time.sleep(10)
+        duthost.shell("{} -s {} -m reset".format(reboot_script_path, ptf_ip))
+        ptfhost.shell('supervisorctl stop ferret')
+
+
 @pytest.fixture
 def set_mux_random(tbinfo, mux_server_url):
     return set_mux_side(tbinfo, mux_server_url, 'random')
 
+
 def test_decap(tbinfo, duthosts, ptfhost, setup_teardown, mux_server_url, set_mux_random, supported_ttl_dscp_params, ip_ver, loopback_ips,
                duts_running_config_facts, duts_minigraph_facts):
     setup_info = setup_teardown
 
     ecn_mode = "copy_from_outer"
     ttl_mode = supported_ttl_dscp_params['ttl']
     dscp_mode = supported_ttl_dscp_params['dscp']
+    vxlan = supported_ttl_dscp_params['vxlan']
     if duthosts[0].facts['asic_type'] in ['mellanox']:
         ecn_mode = 'standard'
 
     try:
-        apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mode, 'SET')
+        if vxlan == "set_unset":
+            # checking decap after vxlan set/unset is to make sure that deletion of vxlan
+            # tunnel and CPA ACLs won't negatively impact ipinip tunnel & decap mechanism
+            # Hence a new decap config is not applied to the device in this case. This is
+            # to avoid creating new tables and test ipinip decap with default loaded config
+            simulate_vxlan_teardown(duthosts, ptfhost, tbinfo)
+        else:
+            apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mode, 'SET')
 
         if 'dualtor' in tbinfo['topo']['name']:
             wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state')
@@ -186,6 +211,7 @@ def test_decap(tbinfo, duthosts, ptfhost, setup_teardown, mux_server_url, set_mu
                             "inner_ipv6": setup_info["inner_ipv6"],
                             "lo_ips": setup_info["lo_ips"],
                             "lo_ipv6s": setup_info["lo_ipv6s"],
+                            "router_macs": setup_info["router_macs"],
                             "ttl_mode": ttl_mode,
                             "dscp_mode": dscp_mode,
                             "ignore_ttl": setup_info["ignore_ttl"],
@@ -200,4 +226,6 @@ def test_decap(tbinfo, duthosts, ptfhost, setup_teardown, mux_server_url, set_mu
         raise Exception(detail)
     finally:
         # Remove test decap configuration
-        apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mode, 'DEL')
+        if vxlan != "set_unset":
+            # in vxlan setunset case the config was not applied, hence DEL is also not required
+            apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mode, 'DEL')