Merge remote-tracking branch 'upstream/master' into dash (#2663)

*Merge remote-tracking branch 'upstream/master' into dash (#2663)
* Modify coppmgr mergeConfig to support preserving copp tables through reboot. (#2548)
* Avoid aborting orchagent when setting TUNNEL attributes (#2591)
* Handle Mac address 'none' (#2593)
* Increase diff coverage to 80% (#2599)
* Add missing parameter to on_switch_shutdown_request method. (#2567)
* Add ZMQ based ProducerStateTable and CustomerStateTable.
* Revert "[voq][chassis]Add show fabric counters port/queue commands (#2522)" (#2611)
* Added new attributes for Vnet and Vxlan ecmp configurations. (#2584)
* added support for monitoring, primary and adv_prefix and overlay_dmac.
* [routesync] Fix for stale dynamic neighbor (#2553)
* [MuxOrch] Enabling neighbor when adding in active state (#2601)
* Changed the BFD default detect multiplier to 10x (#2614)
* Remove TODO comments that are no longer relevant (#2622)
* Fix issue #13341 ARP entry can be out of sync between kernel and APPL_DB if multiple updates are received from RTNL (#2619)
* [refactor]Refactoring sai handle status (#2621)
* Vxlan tunnel endpoint custom monitoring APPL DB table. (#2589)
* added support for monitoring, primary and adv_prefix. changed filter_mac to overlay_dmac
* Data Structures and code to write APP_DB VNET_MONITOR table entries for custom monitoring of Vxlan tunnel endpoints.
* [bfdorch] add local discriminator to state DB (#2629)
* [acl] Add new ACL key BTH_OPCODE and AETH_SYNDROME  (#2617)
* [voq][chassis] Remove created ports from the default vlan. (#2607)
* [EVPN]Handling race condition when remote VNI arrives before tunnel map entry (#2642)
*Added check in remote VNI add to ensure vxlan tunnel map is created before adding the remote end point.
* [test_mux] add sleep in test_NH (#2648)
* [autoneg]Fixing adv interface types to be set when AN is disabled (#2638)
* [hash]: Add UT infra. (#2660)
*Added UT infra for Generic Hash feature
*Aligned PBH tests with Generic Hash UT infra
* [sai_failure_dump]Invoking dump during SAI failure (#2644)
* [ResponsePublisher] add pipeline support  (#2511)
* [dash] Fix compilation issue caused by missing include.

.azure-pipelines/gcov.yml

@@ -47,7 +47,7 @@ jobs:
       vmImage: 'ubuntu-20.04'
 
   variables:
-    DIFF_COVER_CHECK_THRESHOLD: 50
+    DIFF_COVER_CHECK_THRESHOLD: 80
     DIFF_COVER_ENABLE: 'true'
 
   container:

.gitignore

@@ -74,6 +74,7 @@ swssconfig/swssplayer
 tlm_teamd/tlm_teamd
 teamsyncd/teamsyncd
 tests/tests
+tests/mock_tests/tests_response_publisher
 tests/mock_tests/tests_fpmsyncd
 
 

cfgmgr/buffermgrdyn.cpp

@@ -3141,8 +3141,6 @@ task_process_status BufferMgrDynamic::handleSingleBufferQueueEntry(const string
             return task_process_status::task_failed;
         }
 
-        // TODO: check overlap. Currently, assume there is no overlap
-
         auto &portQueue = m_portQueueLookup[port][queues];
         if (PORT_ADMIN_DOWN == portInfo.state)
         {

cfgmgr/coppmgr.cpp

@@ -9,6 +9,8 @@
 #include "shellcmd.h"
 #include "warm_restart.h"
 #include "json.hpp"
+#include <unordered_map>
+#include <unordered_set>
 
 using json = nlohmann::json;
 
@@ -255,6 +257,42 @@ void CoppMgr::mergeConfig(CoppCfg &init_cfg, CoppCfg &m_cfg, std::vector<std::st
     }
 }
 
+bool CoppMgr::isDupEntry(const std::string &key, std::vector<FieldValueTuple> &fvs)
+{
+    /* Compare with the existing contents of copp tables, in case for a key K preserved fvs are the same
+     * as the fvs in trap_group_fvs it will be ignored as a duplicate continue to next key.
+     * In case one of the fvs differs the preserved entry will be deleted and new entry will be set instead.
+     */
+    std::vector<FieldValueTuple> preserved_fvs;
+    bool key_found = m_coppTable.get(key, preserved_fvs);
+    if (!key_found)
+    {
+        return false;
+    }
+    else
+    {
+        unordered_map<string, string> preserved_copp_entry;
+        for (auto prev_fv : preserved_fvs)
+        {
+            preserved_copp_entry[fvField(prev_fv)] = fvValue(prev_fv);
+        }
+        for (auto fv: fvs)
+        {
+            string field = fvField(fv);
+            string value = fvValue(fv);
+            auto preserved_copp_it = preserved_copp_entry.find(field);
+            bool field_found = (preserved_copp_it != preserved_copp_entry.end());
+            if ((!field_found) || (field_found && preserved_copp_it->second.compare(value)))
+            {
+                // overwrite -> delete preserved entry from copp table and set a new entry instead
+                m_coppTable.del(key);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
 CoppMgr::CoppMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, const vector<string> &tableNames) :
         Orch(cfgDb, tableNames),
         m_cfgCoppTrapTable(cfgDb, CFG_COPP_TRAP_TABLE_NAME),
@@ -270,16 +308,19 @@ CoppMgr::CoppMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, c
     std::vector<string> group_keys;
     std::vector<string> trap_keys;
     std::vector<string> feature_keys;
+    std::vector<string> preserved_copp_keys;
 
     std::vector<string> group_cfg_keys;
     std::vector<string> trap_cfg_keys;
+    unordered_set<string> supported_copp_keys;
 
     CoppCfg group_cfg;
     CoppCfg trap_cfg;
 
     m_cfgCoppGroupTable.getKeys(group_cfg_keys);
     m_cfgCoppTrapTable.getKeys(trap_cfg_keys);
     m_cfgFeatureTable.getKeys(feature_keys);
+    m_coppTable.getKeys(preserved_copp_keys);
 
 
     for (auto i: feature_keys)
@@ -352,15 +393,31 @@ CoppMgr::CoppMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, c
 
         if (!trap_group_fvs.empty())
         {
+            supported_copp_keys.emplace(i.first);
+            if (isDupEntry(i.first, trap_group_fvs))
+            {
+                continue;
+            }
             m_appCoppTable.set(i.first, trap_group_fvs);
         }
+
         setCoppGroupStateOk(i.first);
         auto g_cfg = std::find(group_cfg_keys.begin(), group_cfg_keys.end(), i.first);
         if (g_cfg != group_cfg_keys.end())
         {
             g_copp_init_set.insert(i.first);
         }
     }
+
+    // Delete unsupported keys from preserved copp tables
+    for (auto it : preserved_copp_keys)
+    {
+        auto copp_it = supported_copp_keys.find(it);
+        if (copp_it == supported_copp_keys.end())
+        {
+            m_coppTable.del(it);
+        }
+    }
 }
 
 void CoppMgr::setCoppGroupStateOk(string alias)

cfgmgr/coppmgr.h

@@ -100,6 +100,7 @@ class CoppMgr : public Orch
     bool isTrapGroupInstalled(std::string key);
     bool isFeatureEnabled(std::string feature);
     void mergeConfig(CoppCfg &init_cfg, CoppCfg &m_cfg, std::vector<std::string> &cfg_keys, Table &cfgTable);
+    bool isDupEntry(const std::string &key, std::vector<FieldValueTuple> &fvs);
 
     void removeTrap(std::string key);
     void addTrap(std::string trap_ids, std::string trap_group);

cfgmgr/intfmgrd.cpp

@@ -62,8 +62,6 @@ int main(int argc, char **argv)
         WarmStart::checkWarmStart("intfmgrd", "swss");
 
         IntfMgr intfmgr(&cfgDb, &appDb, &stateDb, cfg_intf_tables);
-
-        // TODO: add tables in stateDB which interface depends on to monitor list
         std::vector<Orch *> cfgOrchList = {&intfmgr};
 
         swss::Select s;

cfgmgr/portmgrd.cpp

@@ -53,8 +53,6 @@ int main(int argc, char **argv)
         DBConnector stateDb("STATE_DB", 0);
 
         PortMgr portmgr(&cfgDb, &appDb, &stateDb, cfg_port_tables);
-
-        // TODO: add tables in stateDB which interface depends on to monitor list
         vector<Orch *> cfgOrchList = {&portmgr};
 
         swss::Select s;

cfgmgr/vrfmgrd.cpp

@@ -64,7 +64,6 @@ int main(int argc, char **argv)
 
         isWarmStart = WarmStart::isWarmStart();
 
-        // TODO: add tables in stateDB which interface depends on to monitor list
         std::vector<Orch *> cfgOrchList = {&vrfmgr};
 
         swss::Select s;

fpmsyncd/routesync.cpp

@@ -733,6 +733,32 @@ void RouteSync::onRouteMsg(int nlmsg_type, struct nl_object *obj, char *vrf)
         {
             SWSS_LOG_DEBUG("Skip routes to eth0 or docker0: %s %s %s",
                     destipprefix, gw_list.c_str(), intf_list.c_str());
+            // If intf_list has only this interface, that means all of the next hops of this route 
+            // have been removed and the next hop on the eth0/docker0 has become the only next hop. 
+            // In this case since we do not want the route with next hop on eth0/docker0, we return. 
+            // But still we need to clear the route from the APPL_DB. Otherwise the APPL_DB and data 
+            // path will be left with stale route entry
+            if(alsv.size() == 1)
+            {
+                if (!warmRestartInProgress)
+                {
+                    SWSS_LOG_NOTICE("RouteTable del msg for route with only one nh on eth0/docker0: %s %s %s %s",
+                            destipprefix, gw_list.c_str(), intf_list.c_str(), mpls_list.c_str());
+
+                    m_routeTable.del(destipprefix);
+                }
+                else
+                {
+                    SWSS_LOG_NOTICE("Warm-Restart mode: Receiving delete msg for route with only nh on eth0/docker0: %s %s %s %s",
+                            destipprefix, gw_list.c_str(), intf_list.c_str(), mpls_list.c_str());
+
+                    vector<FieldValueTuple> fvVector;
+                    const KeyOpFieldsValuesTuple kfv = std::make_tuple(destipprefix,
+                                                                       DEL_COMMAND,
+                                                                       fvVector);
+                    m_warmStartHelper.insertRefreshMap(kfv);
+                }
+            }
             return;
         }
     }

neighsyncd/neighsync.cpp

@@ -143,6 +143,12 @@ void NeighSync::onMsg(int nlmsg_type, struct nl_object *obj)
         nl_addr2str(rtnl_neigh_get_lladdr(neigh), macStr, MAX_ADDR_SIZE);
     }
 
+    if (!delete_key && !strncmp(macStr, "none", MAX_ADDR_SIZE))
+    {
+        SWSS_LOG_NOTICE("Mac address is 'none' for ADD op, ignoring for %s", ipStr);
+        return;
+    }
+
     /* Ignore neighbor entries with Broadcast Mac - Trigger for directed broadcast */
     if (!delete_key && (MacAddress(macStr) == MacAddress("ff:ff:ff:ff:ff:ff")))
     {

orchagent/aclorch.cpp

@@ -70,7 +70,9 @@ acl_rule_attr_lookup_t aclMatchLookup =
     { MATCH_INNER_ETHER_TYPE,  SAI_ACL_ENTRY_ATTR_FIELD_INNER_ETHER_TYPE },
     { MATCH_INNER_IP_PROTOCOL, SAI_ACL_ENTRY_ATTR_FIELD_INNER_IP_PROTOCOL },
     { MATCH_INNER_L4_SRC_PORT, SAI_ACL_ENTRY_ATTR_FIELD_INNER_L4_SRC_PORT },
-    { MATCH_INNER_L4_DST_PORT, SAI_ACL_ENTRY_ATTR_FIELD_INNER_L4_DST_PORT }
+    { MATCH_INNER_L4_DST_PORT, SAI_ACL_ENTRY_ATTR_FIELD_INNER_L4_DST_PORT },
+    { MATCH_BTH_OPCODE,        SAI_ACL_ENTRY_ATTR_FIELD_BTH_OPCODE},
+    { MATCH_AETH_SYNDROME,     SAI_ACL_ENTRY_ATTR_FIELD_AETH_SYNDROME}
 };
 
 static acl_range_type_lookup_t aclRangeTypeLookup =
@@ -971,6 +973,36 @@ bool AclRule::validateAddMatch(string attr_name, string attr_value)
             matchData.data.u8 = to_uint<uint8_t>(attr_value);
             matchData.mask.u8 = 0xFF;
         }
+        else if (attr_name == MATCH_BTH_OPCODE)
+        {
+            auto opcode_data = tokenize(attr_value, '/');
+
+            if (opcode_data.size() == 2)
+            {
+                matchData.data.u8 = to_uint<uint8_t>(opcode_data[0]);
+                matchData.mask.u8 = to_uint<uint8_t>(opcode_data[1]);
+            }
+            else
+            {
+                SWSS_LOG_ERROR("Invalid BTH_OPCODE configuration: %s, expected format <data>/<mask>", attr_value.c_str());
+                return false;
+            }
+        }
+        else if (attr_name == MATCH_AETH_SYNDROME)
+        {
+            auto syndrome_data = tokenize(attr_value, '/');
+
+            if (syndrome_data.size() == 2)
+            {
+                matchData.data.u8 = to_uint<uint8_t>(syndrome_data[0]);
+                matchData.mask.u8 = to_uint<uint8_t>(syndrome_data[1]);
+            }
+            else
+            {
+                SWSS_LOG_ERROR("Invalid AETH_SYNDROME configuration: %s, expected format <data>/<mask>", attr_value.c_str());
+                return false;
+            }
+        }
     }
     catch (exception &e)
     {
@@ -3800,7 +3832,7 @@ bool AclOrch::addAclTable(AclTable &newTable)
     }
     // Update matching field according to ACL stage
     newTable.addStageMandatoryMatchFields();
-    
+
     // Add mandatory ACL action if not present
     // We need to call addMandatoryActions here because addAclTable is directly called in other orchs.
     // The action_list is already added if the ACL table creation is triggered by CONFIGDD, but calling addMandatoryActions

orchagent/aclorch.h

@@ -49,6 +49,8 @@
 #define MATCH_INNER_IP_PROTOCOL "INNER_IP_PROTOCOL"
 #define MATCH_INNER_L4_SRC_PORT "INNER_L4_SRC_PORT"
 #define MATCH_INNER_L4_DST_PORT "INNER_L4_DST_PORT"
+#define MATCH_BTH_OPCODE        "BTH_OPCODE"
+#define MATCH_AETH_SYNDROME     "AETH_SYNDROME"
 
 #define BIND_POINT_TYPE_PORT "PORT"
 #define BIND_POINT_TYPE_PORTCHANNEL "PORTCHANNEL"

orchagent/bfdorch.cpp

@@ -13,7 +13,7 @@ using namespace swss;
 
 #define BFD_SESSION_DEFAULT_TX_INTERVAL 1000
 #define BFD_SESSION_DEFAULT_RX_INTERVAL 1000
-#define BFD_SESSION_DEFAULT_DETECT_MULTIPLIER 3
+#define BFD_SESSION_DEFAULT_DETECT_MULTIPLIER 10
 #define BFD_SESSION_MILLISECOND_TO_MICROSECOND 1000
 #define BFD_SRCPORTINIT 49152
 #define BFD_SRCPORTMAX 65536
@@ -306,9 +306,11 @@ bool BfdOrch::create_bfd_session(const string& key, const vector<FieldValueTuple
     attrs.emplace_back(attr);
     fvVector.emplace_back("type", session_type_lookup.at(bfd_session_type));
 
+    uint32_t local_discriminator = bfd_gen_id();
     attr.id = SAI_BFD_SESSION_ATTR_LOCAL_DISCRIMINATOR;
-    attr.value.u32 = bfd_gen_id();
+    attr.value.u32 = local_discriminator;
     attrs.emplace_back(attr);
+    fvVector.emplace_back("local_discriminator", to_string(local_discriminator));
 
     attr.id = SAI_BFD_SESSION_ATTR_UDP_SRC_PORT;
     attr.value.u32 = bfd_src_port();

orchagent/cbf/cbfnhgorch.cpp

@@ -343,10 +343,10 @@ bool CbfNhg::sync()
         SWSS_LOG_ERROR("Failed to create CBF next hop group %s, rv %d",
                         m_key.c_str(),
                         status);
-        task_process_status handle_status = gCbfNhgOrch->handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status);
+        task_process_status handle_status = handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status);
         if (handle_status != task_success)
         {
-            return gCbfNhgOrch->parseHandleSaiStatusFailure(handle_status);
+            return parseHandleSaiStatusFailure(handle_status);
         }
     }
 

orchagent/crmorch.cpp

@@ -4,6 +4,7 @@
 #include "crmorch.h"
 #include "converter.h"
 #include "timer.h"
+#include "saihelper.h"
 
 #define CRM_POLLING_INTERVAL "polling_interval"
 #define CRM_COUNTERS_TABLE_KEY "STATS"

orchagent/dash/dashorch.h

@@ -14,6 +14,7 @@
 #include "macaddress.h"
 #include "timer.h"
 #include "dashorch.h"
+#include "saihelper.h"
 
 struct ApplianceEntry
 {

orchagent/dash/dashvnetorch.cpp

@@ -17,6 +17,7 @@
 #include "saiextensions.h"
 #include "swssnet.h"
 #include "tokenize.h"
+#include "dashorch.h"
 
 using namespace std;
 using namespace swss;

orchagent/fabricportsorch.cpp

@@ -9,6 +9,7 @@
 #include "schema.h"
 #include "sai_serialize.h"
 #include "timer.h"
+#include "saihelper.h"
 
 #define FABRIC_POLLING_INTERVAL_DEFAULT   (30)
 #define FABRIC_PORT_PREFIX    "PORT"