From 3dd37a49c42cb57eb0254cd58cf7100a191e488e Mon Sep 17 00:00:00 2001 From: Volodymyr Samotiy Date: Tue, 26 Mar 2019 20:41:17 +0200 Subject: [PATCH] [vnetorch]: Add VNET/tunnel/route removal flows for Bitmap VNET implementation (#816) Signed-off-by: Volodymyr Samotiy --- orchagent/vnetorch.cpp | 307 +++++++++++++++++++++++++++++++++++------ orchagent/vnetorch.h | 61 ++++++-- 2 files changed, 316 insertions(+), 52 deletions(-) diff --git a/orchagent/vnetorch.cpp b/orchagent/vnetorch.cpp index 84fe3c13a727..564b87aecd78 100644 --- a/orchagent/vnetorch.cpp +++ b/orchagent/vnetorch.cpp @@ -264,8 +264,7 @@ std::bitset VNetBitmapObject::vnetBitmap_; std::bitset VNetBitmapObject::tunnelOffsets_; map VNetBitmapObject::vnetIds_; map VNetBitmapObject::bridgeInfoMap_; -map, sai_fdb_entry_t> VNetBitmapObject::fdbMap_; -map, sai_neighbor_entry_t> VNetBitmapObject::neighMap_; +map, VnetNeighInfo> VNetBitmapObject::neighInfoMap_; VNetBitmapObject::VNetBitmapObject(const std::string& vnet, const VNetInfo& vnetInfo, vector& attrs) : VNetObject(vnetInfo) @@ -275,6 +274,7 @@ VNetBitmapObject::VNetBitmapObject(const std::string& vnet, const VNetInfo& vnet setVniInfo(vnetInfo.vni); vnet_id_ = getFreeBitmapId(vnet); + vnet_name_ = vnet; } bool VNetBitmapObject::updateObj(vector&) @@ -314,11 +314,16 @@ uint32_t VNetBitmapObject::getBitmapId(const string& vnet) return vnetIds_[vnet]; } -void VNetBitmapObject::recycleBitmapId(uint32_t id) +void VNetBitmapObject::recycleBitmapId(const string& vnet) { SWSS_LOG_ENTER(); - vnetBitmap_ &= ~id; + uint32_t id = getBitmapId(vnet); + if (id) + { + vnetBitmap_ &= ~id; + vnetIds_.erase(vnet); + } } uint32_t VNetBitmapObject::getFreeTunnelRouteTableOffset() @@ -350,6 +355,8 @@ VnetBridgeInfo VNetBitmapObject::getBridgeInfoByVni(uint32_t vni, string tunnelN if (bridgeInfoMap_.find(vni) != bridgeInfoMap_.end()) { + bridgeInfoMap_.at(vni).use_count++; + return std::move(bridgeInfoMap_.at(vni)); } @@ -469,18 +476,115 @@ VnetBridgeInfo VNetBitmapObject::getBridgeInfoByVni(uint32_t vni, string tunnelN throw std::runtime_error("vni creation failed"); } - tunnel->addEncapMapperEntry(info.bridge_id, vni); + // FIXME: Use "createVxlanTunnelMap()" for tunnel mapper creation + auto tunnelEncapMapperEntry = tunnel->addEncapMapperEntry(info.bridge_id, vni); + tunnel->insertMapperEntry(tunnelEncapMapperEntry, SAI_NULL_OBJECT_ID, vni); + info.use_count = 1; bridgeInfoMap_.emplace(vni, info); return std::move(info); } +bool VNetBitmapObject::clearBridgeInfoByVni(uint32_t vni, string tunnelName) +{ + if (bridgeInfoMap_.find(vni) == bridgeInfoMap_.end()) + { + SWSS_LOG_ERROR("Bridge info doesn't exist for VNI %u", vni); + return false; + } + + auto& bridgeInfo = bridgeInfoMap_.at(vni); + if (bridgeInfo.use_count > 1) + { + bridgeInfo.use_count--; + } + else + { + auto vxlan_orch = gDirectory.get(); + if (!vxlan_orch->removeVxlanTunnelMap(tunnelName, vni)) + { + return false; + } + + sai_status_t status; + + status = sai_bridge_api->remove_bridge_port(bridgeInfo.bridge_port_tunnel_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove tunnel bridge port for VNI %u, SAI rc: %d", vni, status); + return false; + } + + status = sai_bridge_api->remove_bridge_port(bridgeInfo.bridge_port_rif_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove RIF bridge port for VNI %u, SAI rc: %d", vni, status); + return false; + } + + status = sai_router_intfs_api->remove_router_interface(bridgeInfo.rif_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove RIF for VNI %u, SAI rc: %d",vni, status); + return false; + } + + status = sai_bridge_api->remove_bridge(bridgeInfo.bridge_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove bridge for VNI %u, SAI rc: %d", vni, status); + return false; + } + + bridgeInfoMap_.erase(vni); + } + + return true; +} + +bool VNetBitmapObject::clearNeighInfo(MacAddress mac, sai_object_id_t bridge) +{ + auto macBridge = make_tuple(mac, bridge); + + if (neighInfoMap_.find(macBridge) == neighInfoMap_.end()) + { + SWSS_LOG_ERROR("VNET neighbor doesn't exist"); + return false; + } + + if (neighInfoMap_.at(macBridge).use_count > 1) + { + neighInfoMap_.at(macBridge).use_count--; + } + else + { + sai_status_t status; + + status = sai_neighbor_api->remove_neighbor_entry(&neighInfoMap_.at(macBridge).neigh_entry); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove neighbor entry, SAI rc: %d", status); + return false; + } + + status = sai_fdb_api->remove_fdb_entry(&neighInfoMap_.at(macBridge).fdb_entry); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove FDB entry, SAI rc: %d", status); + return false; + } + + neighInfoMap_.erase(macBridge); + } + + return true; +} + void VNetBitmapObject::setVniInfo(uint32_t vni) { sai_attribute_t attr; vector vnet_attrs; - sai_object_id_t vnetTableEntryId; auto info = getBridgeInfoByVni(getVni(), getTunnelName()); attr.id = SAI_TABLE_BITMAP_CLASSIFICATION_ENTRY_ATTR_ACTION; @@ -496,7 +600,7 @@ void VNetBitmapObject::setVniInfo(uint32_t vni) vnet_attrs.push_back(attr); sai_status_t status = sai_bmtor_api->create_table_bitmap_classification_entry( - &vnetTableEntryId, + &vnetTableEntryId_, gSwitchId, (uint32_t)vnet_attrs.size(), vnet_attrs.data()); @@ -639,11 +743,11 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) sai_status_t status; sai_attribute_t attr; - sai_object_id_t tunnelRouteTableEntryId; auto& peer_list = getPeerList(); auto bInfo = getBridgeInfoByVni(endp.vni == 0 ? getVni() : endp.vni, getTunnelName()); uint32_t peerBitmap = vnet_id_; MacAddress mac = endp.mac ? endp.mac : gVxlanMacAddress; + TunnelRouteInfo tunnelRouteInfo; VNetOrch* vnet_orch = gDirectory.get(); for (auto peer : peer_list) @@ -657,17 +761,17 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) } auto macBridge = make_tuple(mac, bInfo.bridge_id); - - if (fdbMap_.find(macBridge) == fdbMap_.end()) + if (neighInfoMap_.find(macBridge) == neighInfoMap_.end()) { + VnetNeighInfo neighInfo; + /* FDB entry to the tunnel */ vector fdb_attrs; sai_ip_address_t underlayAddr; copy(underlayAddr, endp.ip); - sai_fdb_entry_t fdbEntry; - fdbEntry.switch_id = gSwitchId; - mac.getMac(fdbEntry.mac_address); - fdbEntry.bv_id = bInfo.bridge_id; + neighInfo.fdb_entry.switch_id = gSwitchId; + mac.getMac(neighInfo.fdb_entry.mac_address); + neighInfo.fdb_entry.bv_id = bInfo.bridge_id; attr.id = SAI_FDB_ENTRY_ATTR_TYPE; attr.value.s32 = SAI_FDB_ENTRY_TYPE_STATIC; @@ -682,7 +786,7 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) fdb_attrs.push_back(attr); status = sai_fdb_api->create_fdb_entry( - &fdbEntry, + &neighInfo.fdb_entry, (uint32_t)fdb_attrs.size(), fdb_attrs.data()); if (status != SAI_STATUS_SUCCESS) @@ -691,25 +795,19 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) throw std::runtime_error("VNet route creation failed"); } - fdbMap_.emplace(macBridge, fdbEntry); - } - - /* Fake neighbor */ - sai_neighbor_entry_t neigh; - if (neighMap_.find(macBridge) == neighMap_.end()) - { + /* Fake neighbor */ + neighInfo.neigh_entry.switch_id = gSwitchId; + neighInfo.neigh_entry.rif_id = bInfo.rif_id; + neighInfo.neigh_entry.ip_address.addr_family = SAI_IP_ADDR_FAMILY_IPV4; + neighInfo.neigh_entry.ip_address.addr.ip4 = htonl(getFreeNeighbor()); + vector n_attrs; - neigh.switch_id = gSwitchId; - neigh.rif_id = bInfo.rif_id; - neigh.ip_address.addr_family = SAI_IP_ADDR_FAMILY_IPV4; - neigh.ip_address.addr.ip4 = htonl(getFreeNeighbor()); - attr.id = SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS; mac.getMac(attr.value.mac); n_attrs.push_back(attr); status = sai_neighbor_api->create_neighbor_entry( - &neigh, + &neighInfo.neigh_entry, (uint32_t)n_attrs.size(), n_attrs.data()); if (status != SAI_STATUS_SUCCESS) @@ -717,24 +815,24 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) SWSS_LOG_ERROR("Failed to create neighbor entry for tunnel, SAI rc: %d", status); throw std::runtime_error("VNet route creation failed"); } - - neighMap_.emplace(macBridge, neigh); + + neighInfo.use_count = 1; + neighInfoMap_.emplace(macBridge, neighInfo); } else { - neigh = neighMap_.at(macBridge); + neighInfoMap_.at(macBridge).use_count++; } /* Nexthop */ vector nh_attrs; - sai_object_id_t nexthopId; attr.id = SAI_NEXT_HOP_ATTR_TYPE; attr.value.s32 = SAI_NEXT_HOP_TYPE_IP; nh_attrs.push_back(attr); attr.id = SAI_NEXT_HOP_ATTR_IP; - attr.value.ipaddr = neigh.ip_address; + attr.value.ipaddr = neighInfoMap_.at(macBridge).neigh_entry.ip_address; nh_attrs.push_back(attr); attr.id = SAI_NEXT_HOP_ATTR_ROUTER_INTERFACE_ID; @@ -742,7 +840,7 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) nh_attrs.push_back(attr); status = sai_next_hop_api->create_next_hop( - &nexthopId, + &tunnelRouteInfo.nexthopId, gSwitchId, (uint32_t)nh_attrs.size(), nh_attrs.data()); @@ -761,8 +859,9 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) attr.value.s32 = SAI_TABLE_BITMAP_ROUTER_ENTRY_ACTION_TO_NEXTHOP; tr_attrs.push_back(attr); + tunnelRouteInfo.offset = getFreeTunnelRouteTableOffset(); attr.id = SAI_TABLE_BITMAP_ROUTER_ENTRY_ATTR_PRIORITY; - attr.value.u32 = getFreeTunnelRouteTableOffset(); + attr.value.u32 = tunnelRouteInfo.offset; tr_attrs.push_back(attr); attr.id = SAI_TABLE_BITMAP_ROUTER_ENTRY_ATTR_IN_RIF_METADATA_KEY; @@ -778,11 +877,11 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) tr_attrs.push_back(attr); attr.id = SAI_TABLE_BITMAP_ROUTER_ENTRY_ATTR_NEXT_HOP; - attr.value.oid = nexthopId; + attr.value.oid = tunnelRouteInfo.nexthopId; tr_attrs.push_back(attr); status = sai_bmtor_api->create_table_bitmap_router_entry( - &tunnelRouteTableEntryId, + &tunnelRouteInfo.tunnelRouteTableEntryId, gSwitchId, (uint32_t)tr_attrs.size(), tr_attrs.data()); @@ -793,6 +892,70 @@ bool VNetBitmapObject::addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp) throw std::runtime_error("VNet route creation failed"); } + tunnelRouteInfo.vni = endp.vni == 0 ? getVni() : endp.vni; + tunnelRouteInfo.mac = mac; + tunnelRouteMap_.emplace(ipPrefix, tunnelRouteInfo); + + return true; +} + +bool VNetBitmapObject::removeTunnelRoute(IpPrefix& ipPrefix) +{ + SWSS_LOG_ENTER(); + + if (tunnelRouteMap_.find(ipPrefix) == tunnelRouteMap_.end()) + { + SWSS_LOG_WARN("VNET tunnel route %s doesn't exist", ipPrefix.to_string().c_str()); + return false; + } + + auto tunnelRouteInfo = tunnelRouteMap_.at(ipPrefix); + + if (bridgeInfoMap_.find(tunnelRouteInfo.vni) == bridgeInfoMap_.end()) + { + SWSS_LOG_ERROR("VNET bridge doesn't exist for tunnel route %s", ipPrefix.to_string().c_str()); + throw std::runtime_error("VNET tunnel route removal failed"); + } + + auto bridgeInfo = bridgeInfoMap_.at(tunnelRouteInfo.vni); + auto macBridge = make_tuple(tunnelRouteInfo.mac, bridgeInfo.bridge_id); + + if (neighInfoMap_.find(macBridge) == neighInfoMap_.end()) + { + SWSS_LOG_ERROR("VNET neighbor doesn't exist for tunnel route %s", ipPrefix.to_string().c_str()); + throw std::runtime_error("VNET tunnel route removal failed"); + } + + sai_status_t status; + + status = sai_bmtor_api->remove_table_bitmap_router_entry(tunnelRouteInfo.tunnelRouteTableEntryId); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove VNET tunnel route entry, SAI rc: %d", status); + throw std::runtime_error("VNET tunnel route removal failed"); + } + + status = sai_next_hop_api->remove_next_hop(tunnelRouteInfo.nexthopId); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove nexthop entry for VNET tunnel route, SAI rc: %d", status); + throw std::runtime_error("VNET tunnel route removal failed"); + } + + if (!clearNeighInfo(tunnelRouteInfo.mac, bridgeInfo.bridge_id)) + { + throw std::runtime_error("VNET tunnel route removal failed"); + } + + if (!clearBridgeInfoByVni(tunnelRouteInfo.vni, getTunnelName())) + { + throw std::runtime_error("VNET tunnel route removal failed"); + } + + recycleTunnelRouteTableOffset(tunnelRouteInfo.offset); + + tunnelRouteMap_.erase(ipPrefix); + return true; } @@ -804,10 +967,10 @@ bool VNetBitmapObject::addRoute(IpPrefix& ipPrefix, nextHop& nh) sai_attribute_t attr; vector attrs; sai_ip_prefix_t pfx; - sai_object_id_t tunnelRouteTableEntryId; sai_object_id_t nh_id = SAI_NULL_OBJECT_ID; uint32_t peerBitmap = vnet_id_; Port port; + RouteInfo routeInfo; bool is_subnet = (!nh.ips.getSize()) ? true : false; @@ -870,8 +1033,9 @@ bool VNetBitmapObject::addRoute(IpPrefix& ipPrefix, nextHop& nh) return true; } + routeInfo.offset = getFreeTunnelRouteTableOffset(); attr.id = SAI_TABLE_BITMAP_ROUTER_ENTRY_ATTR_PRIORITY; - attr.value.u32 = getFreeTunnelRouteTableOffset(); + attr.value.u32 = routeInfo.offset; attrs.push_back(attr); attr.id = SAI_TABLE_BITMAP_ROUTER_ENTRY_ATTR_IN_RIF_METADATA_KEY; @@ -887,7 +1051,7 @@ bool VNetBitmapObject::addRoute(IpPrefix& ipPrefix, nextHop& nh) attrs.push_back(attr); status = sai_bmtor_api->create_table_bitmap_router_entry( - &tunnelRouteTableEntryId, + &routeInfo.routeTableEntryId, gSwitchId, (uint32_t)attrs.size(), attrs.data()); @@ -898,9 +1062,56 @@ bool VNetBitmapObject::addRoute(IpPrefix& ipPrefix, nextHop& nh) throw std::runtime_error("VNet route creation failed"); } + routeMap_.emplace(ipPrefix, routeInfo); + + return true; +} + +bool VNetBitmapObject::removeRoute(IpPrefix& ipPrefix) +{ + SWSS_LOG_ENTER(); + + if (routeMap_.find(ipPrefix) == routeMap_.end()) + { + SWSS_LOG_WARN("VNET route %s doesn't exist", ipPrefix.to_string().c_str()); + return false; + } + + sai_status_t status = sai_bmtor_api->remove_table_bitmap_router_entry(routeMap_.at(ipPrefix).routeTableEntryId); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove VNET route entry, SAI rc: %d", status); + throw std::runtime_error("VNET route removal failed"); + } + + recycleTunnelRouteTableOffset(routeMap_.at(ipPrefix).offset); + + routeMap_.erase(ipPrefix); + return true; } +VNetBitmapObject::~VNetBitmapObject() +{ + sai_status_t status; + + status = sai_bmtor_api->remove_table_bitmap_classification_entry(vnetTableEntryId_); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove VNET '%s' table entry, SAI rc: %d", getVnetName().c_str(), status); + throw std::runtime_error("VNET removal failed"); + } + + if (!clearBridgeInfoByVni(getVni(), getTunnelName())) + { + throw std::runtime_error("VNET removal failed"); + } + + recycleBitmapId(getVnetName()); + + SWSS_LOG_INFO("VNET '%s' deleted ", getVnetName().c_str()); +} + /* * VNet Orch class definitions */ @@ -1108,7 +1319,13 @@ bool VNetOrch::delOperation(const Request& request) } else { - // BRIDGE Handling + auto vnet_obj = dynamic_cast(it->second.get()); + + if (vnet_obj->getRouteCount()) + { + SWSS_LOG_ERROR("VNET '%s': Routes are still present", vnet_name.c_str()); + return false; + } } } catch(std::runtime_error& _) @@ -1393,6 +1610,10 @@ bool VNetRouteOrch::doRouteTask(const string& vnet, IpPrefix& { return vnet_obj->addTunnelRoute(ipPrefix, endp); } + else + { + return vnet_obj->removeTunnelRoute(ipPrefix); + } return true; } @@ -1414,6 +1635,10 @@ bool VNetRouteOrch::doRouteTask(const string& vnet, IpPrefix& { return vnet_obj->addRoute(ipPrefix, nh); } + else + { + return vnet_obj->removeRoute(ipPrefix); + } return true; } diff --git a/orchagent/vnetorch.h b/orchagent/vnetorch.h index 1a2bf371314a..20a23f0bcd6d 100644 --- a/orchagent/vnetorch.h +++ b/orchagent/vnetorch.h @@ -170,6 +170,29 @@ struct VnetBridgeInfo sai_object_id_t bridge_port_rif_id; sai_object_id_t bridge_port_tunnel_id; sai_object_id_t rif_id; + uint32_t use_count; +}; + +struct VnetNeighInfo +{ + sai_fdb_entry_t fdb_entry; + sai_neighbor_entry_t neigh_entry; + uint32_t use_count; +}; + +struct TunnelRouteInfo +{ + sai_object_id_t tunnelRouteTableEntryId; + sai_object_id_t nexthopId; + uint32_t vni; + MacAddress mac; + uint32_t offset; +}; + +struct RouteInfo +{ + sai_object_id_t routeTableEntryId; + uint32_t offset; }; class VNetBitmapObject: public VNetObject @@ -177,30 +200,42 @@ class VNetBitmapObject: public VNetObject public: VNetBitmapObject(const string& vnet, const VNetInfo& vnetInfo, vector& attrs); - virtual bool addIntf(const string& alias, const IpPrefix *prefix); + bool addIntf(const string& alias, const IpPrefix *prefix); - virtual bool addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp); + bool addTunnelRoute(IpPrefix& ipPrefix, tunnelEndpoint& endp); + bool removeTunnelRoute(IpPrefix& ipPrefix); - virtual bool addRoute(IpPrefix& ipPrefix, nextHop& nh); + bool addRoute(IpPrefix& ipPrefix, nextHop& nh); + bool removeRoute(IpPrefix& ipPrefix); void setVniInfo(uint32_t vni); bool updateObj(vector&); - virtual ~VNetBitmapObject() {} + size_t getRouteCount() const + { + return (routeMap_.size() + tunnelRouteMap_.size()); + } + + string getVnetName() const + { + return vnet_name_; + } + + ~VNetBitmapObject(); private: static uint32_t getFreeBitmapId(const string& name); - static uint32_t getBitmapId(const string& name); - - static void recycleBitmapId(uint32_t id); - + static void recycleBitmapId(const string& name); + static uint32_t getFreeTunnelRouteTableOffset(); - static void recycleTunnelRouteTableOffset(uint32_t offset); static VnetBridgeInfo getBridgeInfoByVni(uint32_t vni, string tunnelName); + static bool clearBridgeInfoByVni(uint32_t vni, string tunnelName); + + static bool clearNeighInfo(MacAddress mac, sai_object_id_t bridge); static uint32_t getFreeNeighbor(void); @@ -208,10 +243,14 @@ class VNetBitmapObject: public VNetObject static map vnetIds_; static std::bitset tunnelOffsets_; static map bridgeInfoMap_; - static map, sai_fdb_entry_t> fdbMap_; - static map, sai_neighbor_entry_t> neighMap_; + static map, VnetNeighInfo> neighInfoMap_; + + map routeMap_; + map tunnelRouteMap_; uint32_t vnet_id_; + string vnet_name_; + sai_object_id_t vnetTableEntryId_; }; typedef std::unique_ptr VNetObject_T;