From f931ae497c366be44eb8e757be13907cdf809df9 Mon Sep 17 00:00:00 2001 From: Kamil Cudnik Date: Tue, 1 Jun 2021 10:16:06 +0200 Subject: [PATCH] [VS] Add support for context and multiple switches (#830) So far virtual switch supported only 1 switch to be created and only default hwinfo which was empty string. This change allows to reuse context_config.json to allow multiple switches to be created in virtual switch and each can have different hwinfo. This scenario can be useful when running VS test with multiple swss containers running. --- lib/src/ContextConfigContainer.cpp | 6 ++ lib/src/SwitchConfigContainer.cpp | 2 +- syncd/Syncd.cpp | 6 ++ tests/aspell.en.pws | 5 +- vslib/inc/Context.h | 24 +++++ vslib/inc/ContextConfig.h | 31 ++++++ vslib/inc/ContextConfigContainer.h | 43 ++++++++ vslib/inc/Sai.h | 8 ++ vslib/inc/SwitchConfig.h | 4 +- vslib/inc/SwitchConfigContainer.h | 3 + vslib/inc/saivs.h | 17 +++ vslib/src/Context.cpp | 28 +++++ vslib/src/ContextConfig.cpp | 31 ++++++ vslib/src/ContextConfigContainer.cpp | 148 +++++++++++++++++++++++++++ vslib/src/Makefile.am | 3 + vslib/src/Sai.cpp | 111 ++++++++++++++++---- vslib/src/SwitchConfig.cpp | 9 +- vslib/src/SwitchConfigContainer.cpp | 16 ++- vslib/src/SwitchStateBase.cpp | 3 + 19 files changed, 467 insertions(+), 31 deletions(-) create mode 100644 vslib/inc/Context.h create mode 100644 vslib/inc/ContextConfig.h create mode 100644 vslib/inc/ContextConfigContainer.h create mode 100644 vslib/src/Context.cpp create mode 100644 vslib/src/ContextConfig.cpp create mode 100644 vslib/src/ContextConfigContainer.cpp diff --git a/lib/src/ContextConfigContainer.cpp b/lib/src/ContextConfigContainer.cpp index 551162381307..0d07fcfc75f8 100644 --- a/lib/src/ContextConfigContainer.cpp +++ b/lib/src/ContextConfigContainer.cpp @@ -132,9 +132,13 @@ std::shared_ptr ContextConfigContainer::loadFromFile( auto sc = std::make_shared(switchIndex, hwinfo); cc->insert(sc); + + SWSS_LOG_NOTICE("insert into context '%s' config for hwinfo '%s'", cc->m_name.c_str(), hwinfo.c_str()); } ccc->insert(cc); + + SWSS_LOG_NOTICE("insert context '%s' into container list", cc->m_name.c_str()); } } catch (const std::exception& e) @@ -144,6 +148,8 @@ std::shared_ptr ContextConfigContainer::loadFromFile( return getDefault(); } + SWSS_LOG_NOTICE("loaded %zu context configs", ccc->m_map.size()); + return ccc; } diff --git a/lib/src/SwitchConfigContainer.cpp b/lib/src/SwitchConfigContainer.cpp index 521e2314aee6..5458847dcaf7 100644 --- a/lib/src/SwitchConfigContainer.cpp +++ b/lib/src/SwitchConfigContainer.cpp @@ -24,7 +24,7 @@ void SwitchConfigContainer::insert( config->m_hardwareInfo.c_str()); } - SWSS_LOG_NOTICE("added switch: %u:%s", + SWSS_LOG_NOTICE("added switch: idx %u, hwinfo '%s'", config->m_switchIndex, config->m_hardwareInfo.c_str()); diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 78c6709e0103..af5a68f45b59 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -25,6 +25,8 @@ #include "meta/sai_serialize.h" +#include "vslib/inc/saivs.h" + #include #include @@ -2671,6 +2673,10 @@ void Syncd::loadProfileMap() { SWSS_LOG_ENTER(); + // in case of virtual switch, populate context config + m_profileMap[SAI_KEY_VS_GLOBAL_CONTEXT] = std::to_string(m_commandLineOptions->m_globalContext); + m_profileMap[SAI_KEY_VS_CONTEXT_CONFIG] = m_commandLineOptions->m_contextConfig; + if (m_commandLineOptions->m_profileMapFile.size() == 0) { SWSS_LOG_NOTICE("profile map file not specified"); diff --git a/tests/aspell.en.pws b/tests/aspell.en.pws index 0810854f5e14..4e2c20902dac 100644 --- a/tests/aspell.en.pws +++ b/tests/aspell.en.pws @@ -102,6 +102,7 @@ getVid github GRE gSwitchId +GUID hardcoded hasEqualAttribute hostif @@ -120,10 +121,10 @@ ini init INIT inout -INSEG +inseg Inseg +INSEG insegs -inseg ip IP ipc diff --git a/vslib/inc/Context.h b/vslib/inc/Context.h new file mode 100644 index 000000000000..a4ae55a4f044 --- /dev/null +++ b/vslib/inc/Context.h @@ -0,0 +1,24 @@ +#pragma once + +#include "ContextConfig.h" + +namespace saivs +{ + class Context + { + public: + + Context( + _In_ std::shared_ptr contextConfig); + + virtual ~Context(); + + public: + + std::shared_ptr getContextConfig() const; + + private: + + std::shared_ptr m_contextConfig; + }; +} diff --git a/vslib/inc/ContextConfig.h b/vslib/inc/ContextConfig.h new file mode 100644 index 000000000000..bba85c3cf610 --- /dev/null +++ b/vslib/inc/ContextConfig.h @@ -0,0 +1,31 @@ +#pragma once + +#include "SwitchConfigContainer.h" + +namespace saivs +{ + class ContextConfig + { + public: + + ContextConfig( + _In_ uint32_t guid, + _In_ const std::string& name); + + virtual ~ContextConfig(); + + public: + + void insert( + _In_ std::shared_ptr config); + + + public: // TODO to private + + uint32_t m_guid; + + std::string m_name; + + std::shared_ptr m_scc; + }; +} diff --git a/vslib/inc/ContextConfigContainer.h b/vslib/inc/ContextConfigContainer.h new file mode 100644 index 000000000000..ca0f040ccea6 --- /dev/null +++ b/vslib/inc/ContextConfigContainer.h @@ -0,0 +1,43 @@ +#pragma once + +#include "ContextConfig.h" + +#include "swss/sal.h" + +#include +#include +#include + +namespace saivs +{ + class ContextConfigContainer + { + public: + + ContextConfigContainer(); + + virtual ~ContextConfigContainer(); + + public: + + void insert( + _In_ std::shared_ptr contextConfig); + + std::shared_ptr get( + _In_ uint32_t guid); + + std::set> getAllContextConfigs(); + + public: + + static std::shared_ptr loadFromFile( + _In_ const char* contextConfig); + + static std::shared_ptr getDefault(); + + private: + + std::map> m_map; + + }; +} diff --git a/vslib/inc/Sai.h b/vslib/inc/Sai.h index 515a6e4a2f5a..6560fe4a304c 100644 --- a/vslib/inc/Sai.h +++ b/vslib/inc/Sai.h @@ -6,6 +6,7 @@ #include "EventPayloadNotification.h" #include "ResourceLimiterContainer.h" #include "CorePortIndexMapContainer.h" +#include "Context.h" #include "meta/Meta.h" @@ -401,6 +402,11 @@ namespace saivs std::shared_ptr m_signal; + private: + + std::shared_ptr getContext( + _In_ uint32_t globalContext) const; + private: bool m_apiInitialized; @@ -424,5 +430,7 @@ namespace saivs std::shared_ptr m_resourceLimiterContainer; std::shared_ptr m_corePortIndexMapContainer; + + std::map> m_contextMap; }; } diff --git a/vslib/inc/SwitchConfig.h b/vslib/inc/SwitchConfig.h index e6c336a4d25e..fc53f2bc40fd 100644 --- a/vslib/inc/SwitchConfig.h +++ b/vslib/inc/SwitchConfig.h @@ -40,7 +40,9 @@ namespace saivs { public: - SwitchConfig(); + SwitchConfig( + _In_ uint32_t switchIndex, + _In_ const std::string& hwinfo); virtual ~SwitchConfig() = default; diff --git a/vslib/inc/SwitchConfigContainer.h b/vslib/inc/SwitchConfigContainer.h index 3fb9fb66619e..990283b1a217 100644 --- a/vslib/inc/SwitchConfigContainer.h +++ b/vslib/inc/SwitchConfigContainer.h @@ -3,6 +3,7 @@ #include "SwitchConfig.h" #include +#include #include namespace saivs @@ -26,6 +27,8 @@ namespace saivs std::shared_ptr getConfig( _In_ const std::string& hardwareInfo) const; + std::set> getSwitchConfigs() const; + private: std::map> m_indexToConfig; diff --git a/vslib/inc/saivs.h b/vslib/inc/saivs.h index 924dee5f9a32..634a7d885407 100644 --- a/vslib/inc/saivs.h +++ b/vslib/inc/saivs.h @@ -69,6 +69,23 @@ extern "C" { */ #define SAI_KEY_VS_CORE_PORT_INDEX_MAP_FILE "SAI_VS_CORE_PORT_INDEX_MAP_FILE" +/** + * @brief Context config. + * + * Optional. Should point to a context_config.json which will contain how many + * contexts (syncd) we have in the system globally and each context how many + * switches it manages. Only one of this contexts will be used in VS. + */ +#define SAI_KEY_VS_CONTEXT_CONFIG "SAI_VS_CONTEXT_CONFIG" + +/** + * @brief Global context. + * + * Optional. Should point to GUID value which is provided in context_config.json + * by SAI_KEY_VS_CONTEXT_CONFIG. Default is 0. + */ +#define SAI_KEY_VS_GLOBAL_CONTEXT "SAI_VS_GLOBAL_CONTEXT" + #define SAI_VALUE_VS_SWITCH_TYPE_BCM56850 "SAI_VS_SWITCH_TYPE_BCM56850" #define SAI_VALUE_VS_SWITCH_TYPE_BCM81724 "SAI_VS_SWITCH_TYPE_BCM81724" #define SAI_VALUE_VS_SWITCH_TYPE_MLNX2700 "SAI_VS_SWITCH_TYPE_MLNX2700" diff --git a/vslib/src/Context.cpp b/vslib/src/Context.cpp new file mode 100644 index 000000000000..8d72ff97bb98 --- /dev/null +++ b/vslib/src/Context.cpp @@ -0,0 +1,28 @@ +#include "Context.h" + +#include "swss/logger.h" + +using namespace saivs; + +Context::Context( + _In_ std::shared_ptr contextConfig): + m_contextConfig(contextConfig) +{ + SWSS_LOG_ENTER(); + + // empty +} + +Context::~Context() +{ + SWSS_LOG_ENTER(); + + // empty +} + +std::shared_ptr Context::getContextConfig() const +{ + SWSS_LOG_ENTER(); + + return m_contextConfig; +} diff --git a/vslib/src/ContextConfig.cpp b/vslib/src/ContextConfig.cpp new file mode 100644 index 000000000000..cda04fe0429c --- /dev/null +++ b/vslib/src/ContextConfig.cpp @@ -0,0 +1,31 @@ +#include "ContextConfig.h" + +#include "swss/logger.h" + +using namespace saivs; + +ContextConfig::ContextConfig( + _In_ uint32_t guid, + _In_ const std::string& name): + m_guid(guid), + m_name(name) +{ + SWSS_LOG_ENTER(); + + m_scc = std::make_shared(); +} + +ContextConfig::~ContextConfig() +{ + SWSS_LOG_ENTER(); + + // empty +} + +void ContextConfig::insert( + _In_ std::shared_ptr config) +{ + SWSS_LOG_ENTER(); + + m_scc->insert(config); +} diff --git a/vslib/src/ContextConfigContainer.cpp b/vslib/src/ContextConfigContainer.cpp new file mode 100644 index 000000000000..e5820253bb41 --- /dev/null +++ b/vslib/src/ContextConfigContainer.cpp @@ -0,0 +1,148 @@ +#include "ContextConfigContainer.h" + +#include "swss/logger.h" +#include "swss/json.hpp" + +#include +#include + +using json = nlohmann::json; + +using namespace saivs; + +ContextConfigContainer::ContextConfigContainer() +{ + SWSS_LOG_ENTER(); + + // empty +} + +ContextConfigContainer::~ContextConfigContainer() +{ + SWSS_LOG_ENTER(); + + // empty +} + +std::shared_ptr ContextConfigContainer::getDefault() +{ + SWSS_LOG_ENTER(); + + auto ccc = std::make_shared(); + + auto cc = std::make_shared(0, "VirtualSwitch"); + + auto sc = std::make_shared(0, ""); + + cc->insert(sc); + + ccc->m_map[0] = cc; + + return ccc; +} + +void ContextConfigContainer::insert( + _In_ std::shared_ptr contextConfig) +{ + SWSS_LOG_ENTER(); + + m_map[contextConfig->m_guid] = contextConfig; +} + +std::shared_ptr ContextConfigContainer::get( + _In_ uint32_t guid) +{ + SWSS_LOG_ENTER(); + + auto it = m_map.find(guid); + + if (it == m_map.end()) + return nullptr; + + return it->second; +} + +std::shared_ptr ContextConfigContainer::loadFromFile( + _In_ const char* contextConfig) +{ + SWSS_LOG_ENTER(); + + auto ccc = std::make_shared(); + + if (contextConfig == nullptr || strlen(contextConfig) == 0) + { + SWSS_LOG_NOTICE("no context config specified, will load default context config"); + + return getDefault(); + } + + std::ifstream ifs(contextConfig); + + if (!ifs.good()) + { + SWSS_LOG_ERROR("failed to read '%s', err: %s, returning default", contextConfig, strerror(errno)); + + return getDefault(); + } + + try + { + json j; + ifs >> j; + + for (size_t idx = 0; idx < j["CONTEXTS"].size(); idx++) + { + json& item = j["CONTEXTS"][idx]; + + uint32_t guid = item["guid"]; + + const std::string& name = item["name"]; + + SWSS_LOG_NOTICE("contextConfig: guid: %u, name: %s", guid, name.c_str()); + + auto cc = std::make_shared(guid, name); + + for (size_t k = 0; k < item["switches"].size(); k++) + { + json& sw = item["switches"][k]; + + uint32_t switchIndex = sw["index"]; + const std::string& hwinfo = sw["hwinfo"]; + + auto sc = std::make_shared(switchIndex, hwinfo); + + cc->insert(sc); + + SWSS_LOG_NOTICE("insert into context '%s' config for hwinfo '%s'", cc->m_name.c_str(), hwinfo.c_str()); + } + + ccc->insert(cc); + + SWSS_LOG_NOTICE("insert context '%s' into container list", cc->m_name.c_str()); + } + } + catch (const std::exception& e) + { + SWSS_LOG_ERROR("Failed to load '%s': %s, returning default", contextConfig, e.what()); + + return getDefault(); + } + + SWSS_LOG_NOTICE("loaded %zu context configs", ccc->m_map.size()); + + return ccc; +} + +std::set> ContextConfigContainer::getAllContextConfigs() +{ + SWSS_LOG_ENTER(); + + std::set> set; + + for (auto&item: m_map) + { + set.insert(item.second); + } + + return set; +} diff --git a/vslib/src/Makefile.am b/vslib/src/Makefile.am index de6862aae378..cb96b81b3fb6 100644 --- a/vslib/src/Makefile.am +++ b/vslib/src/Makefile.am @@ -14,6 +14,9 @@ libSaiVS_a_SOURCES = \ ../../lib/src/Notification.cpp \ ../../lib/src/NotificationPortStateChange.cpp \ ../../lib/src/PerformanceIntervalTimer.cpp \ + Context.cpp \ + ContextConfig.cpp \ + ContextConfigContainer.cpp \ ResourceLimiter.cpp \ ResourceLimiterContainer.cpp \ ResourceLimiterParser.cpp \ diff --git a/vslib/src/Sai.cpp b/vslib/src/Sai.cpp index a674f502b3f1..2a0f84fd0e80 100644 --- a/vslib/src/Sai.cpp +++ b/vslib/src/Sai.cpp @@ -8,6 +8,7 @@ #include "SwitchConfigContainer.h" #include "ResourceLimiterParser.h" #include "CorePortIndexMapFileParser.h" +#include "ContextConfigContainer.h" #include "swss/logger.h" @@ -103,7 +104,7 @@ sai_status_t Sai::initialize( { SWSS_LOG_NOTICE("failed to obtain service method table value: %s", SAI_KEY_VS_SAI_SWITCH_TYPE); saiSwitchType = SAI_SWITCH_TYPE_NPU; - } + } else if (!SwitchConfig::parseSaiSwitchType(sai_switch_type, saiSwitchType)) { return SAI_STATUS_FAILURE; @@ -151,36 +152,86 @@ sai_status_t Sai::initialize( SWSS_LOG_NOTICE("hostif use TAP device: %s", (useTapDevice ? "true" : "false")); - m_signal = std::make_shared(); + auto cstrGlobalContext = service_method_table->profile_get_value(0, SAI_KEY_VS_GLOBAL_CONTEXT); - m_eventQueue = std::make_shared(m_signal); + uint32_t globalContext = 0; + + if (cstrGlobalContext != nullptr) + { + if (sscanf(cstrGlobalContext, "%u", &globalContext) != 1) + { + SWSS_LOG_WARN("failed to parse '%s' as uint32 using default globalContext", cstrGlobalContext); - auto sc = std::make_shared(); + globalContext = 0; + } + } + + SWSS_LOG_NOTICE("using globalContext = %u", globalContext); - sc->m_saiSwitchType = saiSwitchType; - sc->m_switchType = switchType; - sc->m_bootType = bootType; - sc->m_switchIndex = 0; - sc->m_useTapDevice = useTapDevice; - sc->m_laneMap = m_laneMapContainer->getLaneMap(sc->m_switchIndex); - if (m_fabricLaneMapContainer) + auto cstrContextConfig = service_method_table->profile_get_value(0, SAI_KEY_VS_CONTEXT_CONFIG); + + auto ccc = ContextConfigContainer::loadFromFile(cstrContextConfig); + + for (auto& cc: ccc->getAllContextConfigs()) { - sc->m_fabricLaneMap = m_fabricLaneMapContainer->getLaneMap(sc->m_switchIndex); + auto context = std::make_shared(cc); + + m_contextMap[cc->m_guid] = context; } - sc->m_eventQueue = m_eventQueue; - sc->m_resourceLimiter = m_resourceLimiterContainer->getResourceLimiter(sc->m_switchIndex); - sc->m_corePortIndexMap = m_corePortIndexMapContainer->getCorePortIndexMap(sc->m_switchIndex); - auto scc = std::make_shared(); + auto context = getContext(globalContext); - // TODO add support for multiple switches, (global context?) and config context will need - // to be passed over SAI_KEY_ service method table, and here we need to load them - // we also need global context value for those switches (VirtualSwitchSaiInterface/RealObjectIdManager) + if (context == nullptr) + { + SWSS_LOG_ERROR("no context defined for global context %u", globalContext); + + return SAI_STATUS_FAILURE; + } + + auto contextConfig = context->getContextConfig(); + + auto scc = contextConfig->m_scc; + + if (scc->getSwitchConfigs().size() == 0) + { + SWSS_LOG_WARN("no switch configs defined, using default switch config"); + + auto sc = std::make_shared(0, ""); + + scc->insert(sc); + } - scc->insert(sc); + // TODO currently switch configuration will share signal and event queue + // but it should be moved to Context class + + m_signal = std::make_shared(); + + m_eventQueue = std::make_shared(m_signal); + + for (auto& sc: scc->getSwitchConfigs()) + { + // NOTE: switch index and hardware info is already populated + + sc->m_saiSwitchType = saiSwitchType; + sc->m_switchType = switchType; + sc->m_bootType = bootType; + sc->m_useTapDevice = useTapDevice; + sc->m_laneMap = m_laneMapContainer->getLaneMap(sc->m_switchIndex); + + if (m_fabricLaneMapContainer) + { + sc->m_fabricLaneMap = m_fabricLaneMapContainer->getLaneMap(sc->m_switchIndex); + } + + sc->m_eventQueue = m_eventQueue; + sc->m_resourceLimiter = m_resourceLimiterContainer->getResourceLimiter(sc->m_switchIndex); + sc->m_corePortIndexMap = m_corePortIndexMapContainer->getCorePortIndexMap(sc->m_switchIndex); + } // most important + // TODO move to Context class + m_vsSai = std::make_shared(scc); m_meta = std::make_shared(m_vsSai); @@ -193,7 +244,10 @@ sai_status_t Sai::initialize( { SWSS_LOG_WARN("failed to read warm boot read file, switching to COLD BOOT"); - sc->m_bootType = SAI_VS_BOOT_TYPE_COLD; + for (auto& sc: scc->getSwitchConfigs()) + { + sc->m_bootType = SAI_VS_BOOT_TYPE_COLD; + } } } @@ -201,7 +255,7 @@ sai_status_t Sai::initialize( startUnittestThread(); - if (saiSwitchType == SAI_SWITCH_TYPE_NPU) + if (saiSwitchType == SAI_SWITCH_TYPE_NPU) { startFdbAgingThread(); } @@ -813,3 +867,16 @@ sai_status_t Sai::logSet( return m_meta->logSet(api, log_level); } + +std::shared_ptr Sai::getContext( + _In_ uint32_t globalContext) const +{ + SWSS_LOG_ENTER(); + + auto it = m_contextMap.find(globalContext); + + if (it == m_contextMap.end()) + return nullptr; + + return it->second; +} diff --git a/vslib/src/SwitchConfig.cpp b/vslib/src/SwitchConfig.cpp index b986b346bfb0..efb30312b37f 100644 --- a/vslib/src/SwitchConfig.cpp +++ b/vslib/src/SwitchConfig.cpp @@ -5,12 +5,14 @@ using namespace saivs; -SwitchConfig::SwitchConfig(): +SwitchConfig::SwitchConfig( + _In_ uint32_t switchIndex, + _In_ const std::string& hwinfo): m_saiSwitchType(SAI_SWITCH_TYPE_NPU), m_switchType(SAI_VS_SWITCH_TYPE_NONE), m_bootType(SAI_VS_BOOT_TYPE_COLD), - m_switchIndex(0), - m_hardwareInfo(""), + m_switchIndex(switchIndex), + m_hardwareInfo(hwinfo), m_useTapDevice(false) { SWSS_LOG_ENTER(); @@ -18,7 +20,6 @@ SwitchConfig::SwitchConfig(): // empty } - bool SwitchConfig::parseSaiSwitchType( _In_ const char* saiSwitchTypeStr, _Out_ sai_switch_type_t& saiSwitchType) diff --git a/vslib/src/SwitchConfigContainer.cpp b/vslib/src/SwitchConfigContainer.cpp index cfaf0cee8c61..1407a5cdd7fb 100644 --- a/vslib/src/SwitchConfigContainer.cpp +++ b/vslib/src/SwitchConfigContainer.cpp @@ -24,7 +24,7 @@ void SwitchConfigContainer::insert( config->m_hardwareInfo.c_str()); } - SWSS_LOG_NOTICE("added switch: %u:%s", + SWSS_LOG_NOTICE("added switch: idx %u, hwinfo '%s'", config->m_switchIndex, config->m_hardwareInfo.c_str()); @@ -62,3 +62,17 @@ std::shared_ptr SwitchConfigContainer::getConfig( return nullptr; } + +std::set> SwitchConfigContainer::getSwitchConfigs() const +{ + SWSS_LOG_ENTER(); + + std::set> set; + + for (auto& kvp: m_hwinfoToConfig) + { + set.insert(kvp.second); + } + + return set; +} diff --git a/vslib/src/SwitchStateBase.cpp b/vslib/src/SwitchStateBase.cpp index 743b84b0af09..e5675574d191 100644 --- a/vslib/src/SwitchStateBase.cpp +++ b/vslib/src/SwitchStateBase.cpp @@ -45,6 +45,9 @@ SwitchStateBase::SwitchStateBase( if (m_switchConfig->m_useTapDevice) { m_fdb_info_set = warmBootState->m_fdbInfoSet; + + // TODO populate m_hostif_info_map - need to be able to remove port after warm boot + // should be auto populated vs_recreate_hostif_tap_interfaces on create_switch } } }