As reported by this issue PR#box/flaky#128

currently pytest will call session/module tear-down before flaky
can retry the test. Session/Module will be setup again to run retry
count.

Because of above behaviour when last test fail for given module
all the logs are lost for all test-case upto that point since when
tear down is called dvs container is destroyed and to run retry instance
DVS container is setup up again and logs only belonging to this instance
of run are captured and overwrite all previous logs.

Workaround to have default dummy pass test case at end always so we
avoid module tear-down and setup again.

Signed-off-by: Abhishek Dosi <abdosi@microsoft.com>

tests/test_acl.py

@@ -492,3 +492,7 @@ def test_AclActionValidation(self, dvs, dvs_acl):
                 dvs.runcmd("supervisorctl restart syncd")
                 dvs.stop_swss()
                 dvs.start_swss()
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_acl_ctrl.py

@@ -69,3 +69,7 @@ def test_AclCtrl(self, dvs):
         self.remove_acl_table(dvs)
         self.remove_acl_rule(dvs)
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_acl_egress_table.py

@@ -340,3 +340,7 @@ def test_EgressAclTableDeletion(self, dvs):
         # Remove Acl table
         self.remove_acl_table("egress_acl_table")
         self.check_asic_table_absent(dvs)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_acl_mclag.py

@@ -197,3 +197,7 @@ def test_AclRuleOutPorts(self, dvs, testlog):
         # check acl in asic db
         acl_table_id = self.get_acl_table_id(dvs)
         assert acl_table_id is None
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_acl_portchannel.py

@@ -215,3 +215,7 @@ def test_AclOnPortChannelMember(self, dvs):
 
         # remove port channel
         self.remove_port_channel(dvs, "PortChannel01")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_admin_status.py

@@ -78,3 +78,7 @@ def test_PortChannelMemberAdminStatus(self, dvs, testlog):
 
         # remove port channel
         self.remove_port_channel(dvs, "PortChannel6")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_crm.py

@@ -904,3 +904,7 @@ def test_Configure_fdb(self, dvs, testlog):
         assert threshold_high == 90
         threshold_type = getCrmConfigStr(dvs, 'Config', 'fdb_entry_threshold_type')
         assert threshold_type == 'percentage'
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_dirbcast.py

@@ -94,3 +94,7 @@ def test_DirectedBroadcast(self, dvs, testlog):
 
             if neigh['ip'] == "192.169.0.30":
                 assert False
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_drop_counters.py

@@ -705,3 +705,7 @@ def test_createAndDeleteMultipleCounters(self, dvs, testlog):
         # Cleanup for the next test.
         self.delete_drop_counter(name1)
         self.remove_drop_reason(name1, reason1)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_dtel.py

@@ -295,3 +295,7 @@ def test_DtelEventAttribs(self, dvs, testlog):
         tbl._del("EVENT_TYPE_QUEUE_REPORT_THRESHOLD_BREACH")
         tbl._del("EVENT_TYPE_QUEUE_REPORT_TAIL_DROP")
         tbl._del("EVENT_TYPE_DROP_REPORT")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_fdb.py

@@ -382,3 +382,7 @@ def test_FdbAddedAfterMemberCreated(self, dvs, testlog):
         dvs.runcmd("sonic-clear fdb all")
         dvs.remove_vlan_member("2", "Ethernet0")
         dvs.remove_vlan("2")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_fdb_update.py

@@ -322,3 +322,7 @@ def test_FDBLearnedAndFlushed(self, dvs, testlog):
 
         # restore the default value of the servers
         dvs.servers[server].runcmd("ifconfig eth0 0 down")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_interface.py

@@ -1962,3 +1962,7 @@ def test_LoopbackInterfaceIpv4AddressWithVrf(self, dvs, testlog):
             route = json.loads(key)
             if route["dest"] == "10.0.0.4/32":
                 assert False
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_intf_mac.py

@@ -252,3 +252,7 @@ def test_InterfaceChangeMac(self, dvs, testlog):
 
         # remove port channel
         self.remove_port_channel(dvs, "PortChannel002")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_mirror.py

@@ -872,3 +872,7 @@ def test_AclBindMirror(self, dvs, testlog):
         self.remove_neighbor("Ethernet32", "20.0.0.1")
         self.remove_ip_address("Ethernet32", "20.0.0.0/31")
         self.set_interface_status(dvs, "Ethernet32", "down")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_mirror_ipv6_combined.py

@@ -556,3 +556,7 @@ def test_AclBindMirrorV6WrongConfig(self, dvs, testlog):
         self.set_interface_status("Ethernet32", "down")
 
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_mirror_ipv6_separate.py

@@ -663,3 +663,7 @@ def test_AclBindMirrorV6WrongConfig(self, dvs, testlog):
         self.set_interface_status("Ethernet32", "down")
 
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_mirror_policer.py

@@ -238,3 +238,7 @@ def test_MirrorPolicerWithAcl(self, dvs, testlog):
 
         # remove policer
         self.remove_policer(policer)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_mirror_port_erspan.py

@@ -542,3 +542,7 @@ def test_LAGMirrorToERSPANLagAddRemove(self, dvs, testlog):
         self.dvs_mirror.remove_mirror_session(session)
         self.dvs_mirror.verify_no_mirror()
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_mirror_port_span.py

@@ -470,3 +470,7 @@ def test_PortLAGMirrorUpdateLAG(self, dvs, testlog):
         self.dvs_lag.get_and_verify_port_channel(0)
         self.dvs_vlan.asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_LAG", 0)
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_nat.py

@@ -318,3 +318,7 @@ def test_VerifyConntrackTimeoutForNatEntry(self, dvs, testlog):
         # delete a static nat entry
         dvs.runcmd("config nat remove static basic 67.66.65.1 18.18.18.2")
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_neighbor.py

@@ -389,4 +389,7 @@ def test_NeighborAddRemoveIpv4WithVrf(self, dvs, testlog):
             tbl = swsscommon.Table(self.adb, "ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY")
             current_neigh_entries_cnt = len(tbl.getKeys())
             dec_neigh_entries_cnt = (old_neigh_entries_cnt - current_neigh_entries_cnt)
-            assert dec_neigh_entries_cnt == 1
+            assert dec_neigh_entries_cnt == 1# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_nhg.py

@@ -319,3 +319,7 @@ def asic_route_nhg_fvs(k):
         assert k is not None
         fvs = asic_route_nhg_fvs(k)
         assert fvs is not None
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_pfc.py

@@ -101,3 +101,7 @@ def test_PfcAsymmetric(self, dvs, testlog):
         pfc = getPortAttr(dvs, port_oid, 'SAI_PORT_ATTR_PRIORITY_FLOW_CONTROL')
         assert pfc == pfc_tx
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_policer.py

@@ -72,3 +72,7 @@ def test_PolicerBasic(self, dvs, testlog):
         tbl = swsscommon.Table(dvs.adb, "ASIC_STATE:SAI_OBJECT_TYPE_POLICER")
         policer_entries = tbl.getKeys()
         assert len(policer_entries) == 0
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port.py

@@ -221,3 +221,7 @@ def test_PortIpredriver(self, dvs, testlog):
         for fv in fvs:
             if fv[0] == "SAI_PORT_ATTR_SERDES_IPREDRIVER":
                 assert fv[1] == ipre_val_asic
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_an.py

@@ -174,3 +174,7 @@ def test_PortAutoNegWarm(self, dvs, testlog):
             dvs.runcmd("config warm_restart disable swss")
             # slow down crm polling
             dvs.runcmd("crm config polling interval 10000")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_buffer_rel.py

@@ -34,3 +34,7 @@ def test_PortsAreUpAfterBuffers(self, dvs, testlog):
                     num_set += 1
         # make sure that state is set for all "num_ports" ports
         assert num_set == num_ports, "Not all ports are up"
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_config.py

@@ -155,3 +155,7 @@ def test_port_breakout(self, dvs, port_config):
             assert hw_lane_value == "1:%s" % (new_lanes[i])
 
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_dpb.py

@@ -104,3 +104,7 @@ def test_port_breakout_all(self, dvs):
             start = i*maxBreakOut
             end = start+maxBreakOut
             dpb.breakin(dvs, child_port_names[start:end])
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_dpb_acl.py

@@ -182,3 +182,7 @@ def test_many_ports_many_acl_tables(self, dvs, dvs_acl):
         for aclTable in aclTableNames:
             dvs_acl.remove_acl_table(aclTable)
         dvs_acl.verify_acl_table_count(0)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_dpb_vlan.py

@@ -237,3 +237,7 @@ def test_all_port_10_vlans(self, dvs):
         for vlan_name in vlan_names:
             self.dvs_vlan.remove_vlan(vlan_name)
         self.dvs_vlan.get_and_verify_vlan_ids(0)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_port_mac_learn.py

@@ -181,3 +181,7 @@ def test_PortchannelMacLearnMode(self, dvs, testlog):
         tbl = swsscommon.Table(self.cdb, "PORTCHANNEL")
         tbl._del("PortChannel001")
         time.sleep(1)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_portchannel.py

@@ -263,3 +263,7 @@ def test_Portchannel_oper_down(self, dvs, testlog):
         tbl._del("PortChannel003")
         tbl._del("PortChannel004")
         time.sleep(1)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_qos_map.py

@@ -104,3 +104,7 @@ def test_port_dot1p(self, dvs):
 
         port_cnt = len(swsscommon.Table(self.config_db, CFG_PORT_TABLE_NAME).getKeys())
         assert port_cnt == cnt
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_route.py

@@ -623,3 +623,7 @@ def test_RouteAndNexthopInDifferentVrf(self, dvs, testlog):
         dvs.servers[2].runcmd("ip address del 20.0.0.2/24 dev eth0")
         dvs.servers[3].runcmd("ip route del default dev eth0")
         dvs.servers[3].runcmd("ip address del 20.0.1.2/24 dev eth0")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_setro.py

@@ -44,3 +44,7 @@ def test_SetReadOnlyAttribute(self, dvs, testlog):
 
         # make action on appdb so orchagent will get RO value
         # read asic db to see if orchagent behaved correctly
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_sflow.py

@@ -136,3 +136,7 @@ def test_Teardown(self, dvs, testlog):
 
         self.cdb.delete_entry("SFLOW", "global")
         self.adb.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_SAMPLEPACKET", 0)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_speed.py

@@ -57,3 +57,7 @@ def test_SpeedAndBufferSet(self, dvs, testlog):
 
                 expected_fields = {"profile": "[BUFFER_PROFILE|{}]".format(expected_new_profile_name)}
                 cdb.wait_for_field_match("BUFFER_PG", expected_pg_table, expected_fields)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_sub_port_intf.py

@@ -424,3 +424,7 @@ def test_sub_port_intf_removal(self, dvs):
 
         self._test_sub_port_intf_removal(dvs, self.SUB_PORT_INTERFACE_UNDER_TEST)
         self._test_sub_port_intf_removal(dvs, self.LAG_SUB_PORT_INTERFACE_UNDER_TEST)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_switch.py

@@ -71,3 +71,7 @@ def test_switch_attribute(self, dvs, testlog):
         vxlan_switch_test(dvs, switch_oid, "12345", "00:01:02:03:04:05")
 
         vxlan_switch_test(dvs, switch_oid, "56789", "00:0A:0B:0C:0D:0E")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_tunnel.py

@@ -229,3 +229,7 @@ def test_TunnelSymmetric_v6(self, dvs, testlog):
                                     ecn_mode="standard", ttl_mode="pipe")
         self.remove_and_test_tunnel(db, asicdb, "IPINIPv6Symmetric")
 
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_vlan.py

@@ -418,3 +418,7 @@ def test_VlanMemberDbData(self, dvs, test_input, expected):
 
         self.dvs_vlan.remove_vlan(vlan)
         self.dvs_vlan.get_and_verify_vlan_ids(0)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_vnet.py

@@ -1352,3 +1352,7 @@ def test_vnet_orch_5(self, dvs, testlog):
 
         vnet_obj.check_default_vnet_entry(dvs, 'Vnet_5')
         vnet_obj.check_vxlan_tunnel_entry(dvs, tunnel_name, 'Vnet_5', '4789')
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_vnet_bitmap.py

@@ -25,3 +25,7 @@ class TestVnetBitmapOrch(vnet.TestVnetOrch):
     '''
     def get_vnet_obj(self):
         return vnet.VnetBitmapVxlanTunnel()
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_vrf.py

@@ -304,3 +304,7 @@ def create_entry_tbl(self, db, table, key, pairs):
         # check linux kernel
         (exitcode, num) = dvs.runcmd(['sh', '-c', "ip link show | grep Vrf | wc -l"])
         assert num.strip() == '0'
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_vxlan_tunnel.py

@@ -309,3 +309,7 @@ def test_vxlan_term_orch(self, dvs, testlog):
 
         create_vxlan_tunnel_entry(dvs, 'tunnel_4', 'entry_2', tunnel_map_map, 'Vlan57', '857',
                                   tunnel_map_ids, tunnel_map_entry_ids, tunnel_ids, tunnel_term_ids)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_warm_reboot.py

@@ -2190,3 +2190,7 @@ def test_VrfMgrdWarmRestart(self, dvs, testlog):
         dvs.servers[0].runcmd("ifconfig eth0 0")
         dvs.servers[1].runcmd("ifconfig eth0 0")
         time.sleep(2)
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass

tests/test_watermark.py

@@ -231,3 +231,7 @@ def test_clear(self, dvs):
         self.verify_value(dvs, self.mc_q, WmTables.user, SaiWmStats.queue_shared, "288")
 
         self.enable_unittests(dvs, "false")
+# Add Dummy always-pass test at end as workaroud
+# for issue when Flaky fail on final test it invokes module tear-down before retrying
+def test_nonflaky_dummy():
+    pass