diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index ffd9922bd1a7..9912aa460356 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -33,10 +33,13 @@ class RouteConfigProviderManager { * @param rds supplies the proto configuration of an RDS-configured RouteConfigProvider. * @param factory_context is the context to use for the route config provider. * @param stat_prefix supplies the stat_prefix to use for the provider stats. + * @param init_manager the Init::Manager used to coordinate initialization of a the underlying RDS + * subscription. */ virtual RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) PURE; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) PURE; /** * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index a1a7b02d71b7..4a7499c5abdb 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -1,7 +1,5 @@ #pragma once -#include - #include "envoy/config/config_provider.h" #include "envoy/config/config_provider_manager.h" #include "envoy/init/manager.h" @@ -146,9 +144,7 @@ class MutableConfigProviderCommonBase; * shared ownership of the underlying subscription. * */ -class ConfigSubscriptionCommonBase - : protected Logger::Loggable, - public std::enable_shared_from_this { +class ConfigSubscriptionCommonBase : protected Logger::Loggable { public: // Callback for updating a Config implementation held in each worker thread, the callback is // called in applyConfigUpdate() with the current version Config, and is expected to return the @@ -224,21 +220,14 @@ class ConfigSubscriptionCommonBase */ void applyConfigUpdate( const ConfigUpdateCb& update_fn, const Event::PostCb& complete_cb = []() {}) { - // It is safe to call shared_from_this here as this is in main thread, and destruction of a - // ConfigSubscriptionCommonBase owner (i.e., a provider) happens in main thread as well. - auto shared_this = shared_from_this(); tls_->runOnAllThreads( [this, update_fn]() { - tls_->getTyped().config_ = update_fn(this->getConfig()); + // NOTE: there is a known race condition between *this* subscription being teared down in + // main thread and the posted callback being executed before the destruction. See more + // details in https://github.com/envoyproxy/envoy/issues/7902 + tls_->getTyped().config_ = update_fn(getConfig()); }, - // During the update propagation, a subscription may get teared down in main thread due to - // all owners/providers destructed in a xDS update (e.g. LDS demolishes a - // RouteConfigProvider and its subscription). - // If such a race condition happens, holding a reference to the "*this" subscription - // instance in this cb will ensure the shared "*this" gets posted back to main thread, after - // all the workers finish calling the update_fn, at which point it's safe to destruct - // "*this" instance. - [shared_this, complete_cb]() { complete_cb(); }); + complete_cb); } void setLastUpdated() { last_updated_ = time_source_.systemTime(); } @@ -287,8 +276,8 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { /** * Must be called by the derived class' constructor. - * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with - * the underlying subscription, shared across all providers and workers. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated + * with the underlying subscription, shared across all providers and workers. */ void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) { tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { @@ -302,7 +291,8 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { * @param config_proto supplies the newly received config proto. * @param config_name supplies the name associated with the config. * @param version_info supplies the version associated with the config. - * @return bool false when the config proto has no delta from the previous config, true otherwise. + * @return bool false when the config proto has no delta from the previous config, true + * otherwise. */ bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info); @@ -369,8 +359,8 @@ class MutableConfigProviderCommonBase : public ConfigProvider { }; /** - * Provides generic functionality required by all config provider managers, such as managing shared - * lifetime of subscriptions and dynamic config providers, along with determining which + * Provides generic functionality required by all config provider managers, such as managing + * shared lifetime of subscriptions and dynamic config providers, along with determining which * subscriptions should be associated with newly instantiated providers. * * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase @@ -379,9 +369,9 @@ class MutableConfigProviderCommonBase : public ConfigProvider { * * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care - * must be taken with regards to destruction of these objects, since it must also happen on the main - * thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they were - * created. + * must be taken with regards to destruction of these objects, since it must also happen on the + * main thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they + * were created. * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. @@ -415,8 +405,8 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl const ConfigProviderSet& immutableConfigProviders(ConfigProviderInstanceType type) const; /** - * Returns the subscription associated with the config_source_proto; if none exists, a new one is - * allocated according to the subscription_factory_fn. + * Returns the subscription associated with the config_source_proto; if none exists, a new one + * is allocated according to the subscription_factory_fn. * @param config_source_proto supplies the proto specifying the config subscription parameters. * @param init_manager supplies the init manager. * @param subscription_factory_fn supplies a function to be called when a new subscription needs diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 31d354477d5e..21970908db10 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -132,6 +132,7 @@ envoy_cc_library( "//include/envoy/singleton:instance_interface", "//include/envoy/thread_local:thread_local_interface", "//source/common/common:assert_lib", + "//source/common/common:callback_impl_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:rds_json_lib", "//source/common/config:subscription_factory_lib", @@ -146,22 +147,16 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "scoped_config_manager_lib", - srcs = ["scoped_config_manager.cc"], - hdrs = ["scoped_config_manager.h"], - deps = [ - "@envoy_api//envoy/api/v2:srds_cc", - ], -) - envoy_cc_library( name = "scoped_config_lib", srcs = ["scoped_config_impl.cc"], hdrs = ["scoped_config_impl.h"], + external_deps = [ + "abseil_str_format", + ], deps = [ ":config_lib", - ":scoped_config_manager_lib", + "//include/envoy/router:rds_interface", "//include/envoy/router:scopes_interface", "//include/envoy/thread_local:thread_local_interface", "@envoy_api//envoy/api/v2:srds_cc", @@ -174,13 +169,18 @@ envoy_cc_library( srcs = ["scoped_rds.cc"], hdrs = ["scoped_rds.h"], deps = [ + ":rds_lib", ":scoped_config_lib", "//include/envoy/config:config_provider_interface", "//include/envoy/config:subscription_interface", + "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", + "//source/common/common:cleanup_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:config_provider_lib", + "//source/common/init:manager_lib", + "//source/common/init:watcher_lib", "@envoy_api//envoy/admin/v2alpha:config_dump_cc", "@envoy_api//envoy/api/v2:srds_cc", ], diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index b95554601aef..64a60062a30c 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -30,8 +30,8 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( return route_config_provider_manager.createStaticRouteConfigProvider(config.route_config(), factory_context); case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds: - return route_config_provider_manager.createRdsRouteConfigProvider(config.rds(), factory_context, - stat_prefix); + return route_config_provider_manager.createRdsRouteConfigProvider( + config.rds(), factory_context, stat_prefix, factory_context.initManager()); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -124,6 +124,7 @@ void RdsRouteConfigSubscription::onConfigUpdate( } vhds_subscription_.release(); } + update_callback_manager_.runCallbacks(); } init_target_.ready(); @@ -218,8 +219,8 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& ad Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) { - + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) { // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. const uint64_t manager_identifier = MessageUtil::hash(rds); @@ -232,9 +233,7 @@ Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteCon // of simplicity. subscription.reset(new RdsRouteConfigSubscription(rds, manager_identifier, factory_context, stat_prefix, *this)); - - factory_context.initManager().add(subscription->init_target_); - + init_manager.add(subscription->init_target_); route_config_subscriptions_.insert({manager_identifier, subscription}); } else { // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 21acac47cfa7..418512bb6e50 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -22,6 +22,7 @@ #include "envoy/stats/scope.h" #include "envoy/thread_local/thread_local.h" +#include "common/common/callback_impl.h" #include "common/common/logger.h" #include "common/init/target_impl.h" #include "common/protobuf/utility.h" @@ -31,6 +32,9 @@ namespace Envoy { namespace Router { +// For friend class declaration in RdsRouteConfigSubscription. +class ScopedRdsConfigSubscription; + /** * Route configuration provider utilities. */ @@ -121,6 +125,10 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, return MessageUtil::anyConvert(resource).name(); } + Common::CallbackHandle* addUpdateCallback(std::function callback) { + return update_callback_manager_.add(callback); + } + RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, @@ -142,8 +150,11 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, VhdsSubscriptionPtr vhds_subscription_; RouteConfigUpdatePtr config_update_info_; ProtobufMessage::ValidationVisitor& validation_visitor_; + Common::CallbackManager<> update_callback_manager_; friend class RouteConfigProviderManagerImpl; + // Access to addUpdateCallback + friend class ScopedRdsConfigSubscription; }; using RdsRouteConfigSubscriptionSharedPtr = std::shared_ptr; @@ -195,8 +206,8 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, // RouteConfigProviderManager RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) override; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::api::v2::RouteConfiguration& route_config, diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index f5c8007f9e74..c6f00f58ea1e 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -10,13 +10,7 @@ bool ScopeKey::operator==(const ScopeKey& other) const { // An empty key equals to nothing, "NULL" != "NULL". return false; } - return std::equal(fragments_.begin(), fragments_.end(), other.fragments_.begin(), - other.fragments_.end(), - [](const std::unique_ptr& left, - const std::unique_ptr& right) -> bool { - // Both should be non-NULL now. - return *left == *right; - }); + return this->hash() == other.hash(); } HeaderValueExtractorImpl::HeaderValueExtractorImpl( @@ -76,6 +70,22 @@ HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const return nullptr; } +ScopedRouteInfo::ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, + ConfigConstSharedPtr&& route_config) + : config_proto_(std::move(config_proto)), route_config_(std::move(route_config)) { + // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of + // Fragment. + for (const auto& fragment : config_proto_.key().fragments()) { + switch (fragment.type_case()) { + case envoy::api::v2::ScopedRouteConfiguration::Key::Fragment::kStringKey: + scope_key_.addFragment(std::make_unique(fragment.string_key())); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + } +} + ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config) : ScopeKeyBuilderBase(std::move(config)) { for (const auto& fragment_builder : config_.fragments()) { @@ -104,12 +114,37 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { return std::make_unique(std::move(key)); } -void ScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} +void ScopedConfigImpl::addOrUpdateRoutingScope( + const ScopedRouteInfoConstSharedPtr& scoped_route_info) { + const auto iter = scoped_route_info_by_name_.find(scoped_route_info->scopeName()); + if (iter != scoped_route_info_by_name_.end()) { + ASSERT(scoped_route_info_by_key_.contains(iter->second->scopeKey().hash())); + scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); + } + scoped_route_info_by_name_[scoped_route_info->scopeName()] = scoped_route_info; + scoped_route_info_by_key_[scoped_route_info->scopeKey().hash()] = scoped_route_info; +} -void ScopedConfigImpl::removeRoutingScope(const std::string&) {} +void ScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { + const auto iter = scoped_route_info_by_name_.find(scope_name); + if (iter != scoped_route_info_by_name_.end()) { + ASSERT(scoped_route_info_by_key_.contains(iter->second->scopeKey().hash())); + scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); + scoped_route_info_by_name_.erase(iter); + } +} -Router::ConfigConstSharedPtr ScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { - return std::make_shared(); +Router::ConfigConstSharedPtr +ScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { + std::unique_ptr scope_key = scope_key_builder_.computeScopeKey(headers); + if (scope_key == nullptr) { + return nullptr; + } + auto iter = scoped_route_info_by_key_.find(scope_key->hash()); + if (iter != scoped_route_info_by_key_.end()) { + return iter->second->routeConfig(); + } + return nullptr; } } // namespace Router diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 184929f94664..91e67e841b14 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -4,13 +4,17 @@ #include "envoy/api/v2/srds.pb.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/router/rds.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" #include "envoy/thread_local/thread_local.h" +#include "common/common/hash.h" #include "common/protobuf/utility.h" #include "common/router/config_impl.h" -#include "common/router/scoped_config_manager.h" + +#include "absl/numeric/int128.h" +#include "absl/strings/str_format.h" namespace Envoy { namespace Router { @@ -26,15 +30,14 @@ class ScopeKeyFragmentBase { bool operator==(const ScopeKeyFragmentBase& other) const { if (typeid(*this) == typeid(other)) { - return equals(other); + return hash() == other.hash(); } return false; } virtual ~ScopeKeyFragmentBase() = default; -private: - // Returns true if the two fragments equal else false. - virtual bool equals(const ScopeKeyFragmentBase&) const PURE; + // Hash of the fragment. + virtual uint64_t hash() const PURE; }; /** @@ -52,28 +55,39 @@ class ScopeKey { // Caller should guarantee the fragment is not nullptr. void addFragment(std::unique_ptr&& fragment) { ASSERT(fragment != nullptr, "null fragment not allowed in ScopeKey."); + updateHash(*fragment); fragments_.emplace_back(std::move(fragment)); } + uint64_t hash() const { return hash_; } bool operator!=(const ScopeKey& other) const; - bool operator==(const ScopeKey& other) const; private: + // Update the key's hash with the new fragment hash. + void updateHash(const ScopeKeyFragmentBase& fragment) { + std::stringbuf buffer; + buffer.sputn(reinterpret_cast(&hash_), sizeof(hash_)); + const auto& fragment_hash = fragment.hash(); + buffer.sputn(reinterpret_cast(&fragment_hash), sizeof(fragment_hash)); + hash_ = HashUtil::xxHash64(buffer.str()); + } + + uint64_t hash_{0}; std::vector> fragments_; }; // String fragment. class StringKeyFragment : public ScopeKeyFragmentBase { public: - explicit StringKeyFragment(absl::string_view value) : value_(value) {} + explicit StringKeyFragment(absl::string_view value) + : value_(value), hash_(HashUtil::xxHash64(value_)) {} -private: - bool equals(const ScopeKeyFragmentBase& other) const override { - return value_ == static_cast(other).value_; - } + uint64_t hash() const override { return hash_; } +private: const std::string value_; + const uint64_t hash_; }; /** @@ -132,9 +146,27 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { std::vector> fragment_builders_; }; +// ScopedRouteConfiguration and corresponding RouteConfigProvider. +class ScopedRouteInfo { +public: + ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, + ConfigConstSharedPtr&& route_config); + + const ConfigConstSharedPtr& routeConfig() const { return route_config_; } + const ScopeKey& scopeKey() const { return scope_key_; } + const envoy::api::v2::ScopedRouteConfiguration& configProto() const { return config_proto_; } + const std::string& scopeName() const { return config_proto_.name(); } + +private: + envoy::api::v2::ScopedRouteConfiguration config_proto_; + ScopeKey scope_key_; + ConfigConstSharedPtr route_config_; +}; +using ScopedRouteInfoConstSharedPtr = std::shared_ptr; +// Ordered map for consistent config dumping. +using ScopedRouteMap = std::map; + /** - * TODO(AndresGuedez): implement scoped routing logic. - * * Each Envoy worker is assigned an instance of this type. When config updates are received, * addOrUpdateRoutingScope() and removeRoutingScope() are called to update the set of scoped routes. * @@ -153,8 +185,11 @@ class ScopedConfigImpl : public ScopedConfig { Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const override; private: - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder - scope_key_builder_; + ScopeKeyBuilderImpl scope_key_builder_; + // From scope name to cached ScopedRouteInfo. + absl::flat_hash_map scoped_route_info_by_name_; + // Hash by ScopeKey hash to lookup in constant time. + absl::flat_hash_map scoped_route_info_by_key_; }; /** diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc deleted file mode 100644 index 2a5b75f3b29c..000000000000 --- a/source/common/router/scoped_config_manager.cc +++ /dev/null @@ -1,22 +0,0 @@ -#include "common/router/scoped_config_manager.h" - -#include "envoy/common/exception.h" - -#include "common/common/fmt.h" - -namespace Envoy { -namespace Router { - -ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( - const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { - auto scoped_route_info = std::make_shared(config_proto); - scoped_route_map_[config_proto.name()] = scoped_route_info; - return scoped_route_info; -} - -bool ScopedConfigManager::removeRoutingScope(const std::string& name) { - return scoped_route_map_.erase(name) == 0; -} - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h deleted file mode 100644 index 5f8dd6fda878..000000000000 --- a/source/common/router/scoped_config_manager.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/api/v2/srds.pb.h" - -namespace Envoy { -namespace Router { - -// The internal representation of the configuration distributed via the ScopedRouteConfiguration -// proto. -class ScopedRouteInfo { -public: - ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) - : config_proto_(config_proto) {} - - // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. - - const envoy::api::v2::ScopedRouteConfiguration config_proto_; -}; -using ScopedRouteInfoConstSharedPtr = std::shared_ptr; - -// A manager for routing configuration scopes. -// An instance of the manager is owned by each ScopedRdsConfigSubscription. When config updates are -// received (on the main thread), the manager is called to track changes to the set of scoped route -// configurations and build s as needed. -class ScopedConfigManager { -public: - // Ordered map for consistent config dumping. - using ScopedRouteMap = std::map; - - // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the - // set of scopes matched against the scope keys built for each HTTP request. - ScopedRouteInfoConstSharedPtr - addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, - const std::string& version_info); - - // Removes a routing scope from the set of scopes matched against each HTTP request. - bool removeRoutingScope(const std::string& scope_name); - - const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } - -private: - ScopedRouteMap scoped_route_map_; -}; - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 26ef93327284..989e284668c4 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -6,7 +6,12 @@ #include "envoy/api/v2/srds.pb.validate.h" #include "common/common/assert.h" +#include "common/common/cleanup.h" #include "common/common/logger.h" +#include "common/common/utility.h" +#include "common/config/resources.h" +#include "common/init/manager_impl.h" +#include "common/init/watcher_impl.h" // Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all // ConfigProvider related types for consistency. @@ -17,9 +22,7 @@ using Envoy::Config::ConfigProviderPtr; namespace Envoy { namespace Router { - namespace ScopedRoutesConfigProviderUtil { - ConfigProviderPtr create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, @@ -79,13 +82,17 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + envoy::api::v2::core::ConfigSource rds_config_source, + RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), - name_(name), scope_key_builder_(scope_key_builder), + factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), - validation_visitor_(factory_context.messageValidationVisitor()) { + rds_config_source_(std::move(rds_config_source)), + validation_visitor_(factory_context.messageValidationVisitor()), stat_prefix_(stat_prefix), + route_config_provider_manager_(route_config_provider_manager) { subscription_ = factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( scoped_rds.scoped_rds_config_source(), @@ -100,72 +107,220 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( }); } +ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::RdsRouteConfigProviderHelper( + ScopedRdsConfigSubscription& parent, std::string scope_name, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Init::Manager& init_manager) + : parent_(parent), scope_name_(scope_name), + route_provider_(static_cast( + parent_.route_config_provider_manager_ + .createRdsRouteConfigProvider(rds, parent_.factory_context_, parent_.stat_prefix_, + init_manager) + .release())), + rds_update_callback_handle_(route_provider_->subscription().addUpdateCallback([this]() { + // Subscribe to RDS update. + parent_.onRdsConfigUpdate(scope_name_, route_provider_->subscription()); + })) {} + +bool ScopedRdsConfigSubscription::addOrUpdateScopes( + const Protobuf::RepeatedPtrField& resources, + Init::Manager& init_manager, const std::string& version_info, + std::vector& exception_msgs) { + bool any_applied = false; + envoy::config::filter::network::http_connection_manager::v2::Rds rds; + rds.mutable_config_source()->MergeFrom(rds_config_source_); + absl::flat_hash_set unique_resource_names; + for (const auto& resource : resources) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + try { + scoped_route_config = + MessageUtil::anyConvert(resource.resource()); + MessageUtil::validate(scoped_route_config, validation_visitor_); + const std::string scope_name = scoped_route_config.name(); + if (!unique_resource_names.insert(scope_name).second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration '{}' found", scope_name)); + } + // TODO(stevenzzz): Creating a new RdsRouteConfigProvider likely expensive, migrate RDS to + // config-provider-framework to make it light weight. + rds.set_route_config_name(scoped_route_config.route_configuration_name()); + auto rds_config_provider_helper = + std::make_unique(*this, scope_name, rds, init_manager); + auto scoped_route_info = std::make_shared( + std::move(scoped_route_config), rds_config_provider_helper->routeConfig()); + // Detect if there is key conflict between two scopes, in which case Envoy won't be able to + // tell which RouteConfiguration to use. Reject the second scope in the delta form API. + auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); + if (iter != scope_name_by_hash_.end()) { + if (iter->second != scoped_route_info->scopeName()) { + throw EnvoyException( + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + iter->second, scoped_route_info->scopeName())); + } + } + // NOTE: delete previous route provider if any. + route_provider_by_scope_.insert({scope_name, std::move(rds_config_provider_helper)}); + scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); + scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; + applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; + }); + any_applied = true; + ENVOY_LOG(debug, "srds: add/update scoped_route '{}', version: {}", + scoped_route_info->scopeName(), version_info); + } catch (const EnvoyException& e) { + exception_msgs.emplace_back(fmt::format("{}", e.what())); + } + } + return any_applied; +} + +bool ScopedRdsConfigSubscription::removeScopes( + const Protobuf::RepeatedPtrField& scope_names, const std::string& version_info) { + bool any_applied = false; + for (const auto& scope_name : scope_names) { + auto iter = scoped_route_map_.find(scope_name); + if (iter != scoped_route_map_.end()) { + route_provider_by_scope_.erase(scope_name); + scope_name_by_hash_.erase(iter->second->scopeKey().hash()); + scoped_route_map_.erase(iter); + applyConfigUpdate([scope_name](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scope_name); + return config; + }); + any_applied = true; + ENVOY_LOG(debug, "srds: remove scoped route '{}', version: {}", scope_name, version_info); + } + } + return any_applied; +} + void ScopedRdsConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) { - std::vector scoped_routes; - for (const auto& resource_any : resources) { - scoped_routes.emplace_back( - MessageUtil::anyConvert(resource_any)); + // If new route config sources come after the factory_context_.initManager()'s initialize() been + // called, that initManager can't accept new targets. Instead we use a local override which will + // start new subscriptions but not wait on them to be ready. + // NOTE: For now we use a local init-manager, in the future when Envoy supports on-demand xDS, we + // will probably make this init-manager as a member of the subscription. + std::unique_ptr noop_init_manager; + // NOTE: This should be defined after noop_init_manager as it depends on the + // noop_init_manager. + std::unique_ptr resume_rds; + if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { + noop_init_manager = + std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); + // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. + // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either + // by Server init or LDS init. + factory_context_.clusterManager().adsMux().pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + resume_rds = std::make_unique([this, &noop_init_manager, version_info] { + // For new RDS subscriptions created after listener warming up, we don't wait for them to warm + // up. + Init::WatcherImpl noop_watcher( + // Note: we just throw it away. + fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), + []() { /*Do nothing.*/ }); + noop_init_manager->initialize(noop_watcher); + // New RDS subscriptions should have been created, now lift the floodgate. + // Note in the case of partial acceptance, accepted RDS subscriptions should be started + // despite of any error. + factory_context_.clusterManager().adsMux().resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + }); } + std::vector exception_msgs; + bool any_applied = addOrUpdateScopes( + added_resources, + (noop_init_manager == nullptr ? factory_context_.initManager() : *noop_init_manager), + version_info, exception_msgs); + any_applied = removeScopes(removed_resources, version_info) || any_applied; + ConfigSubscriptionCommonBase::onConfigUpdate(); + if (any_applied) { + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + } + stats_.config_reload_.inc(); + if (!exception_msgs.empty()) { + throw EnvoyException(fmt::format("Error adding/updating scoped route(s): {}", + StringUtil::join(exception_msgs, ", "))); + } +} - std::unordered_set resource_names; - for (const auto& scoped_route : scoped_routes) { - if (!resource_names.insert(scoped_route.name()).second) { +void ScopedRdsConfigSubscription::onRdsConfigUpdate(const std::string& scope_name, + RdsRouteConfigSubscription& rds_subscription) { + auto iter = scoped_route_map_.find(scope_name); + ASSERT(iter != scoped_route_map_.end(), + fmt::format("trying to update route config for non-existing scope {}", scope_name)); + auto new_scoped_route_info = std::make_shared( + envoy::api::v2::ScopedRouteConfiguration(iter->second->configProto()), + std::make_shared(rds_subscription.routeConfigUpdate()->routeConfiguration(), + factory_context_, false)); + applyConfigUpdate([new_scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(new_scoped_route_info); + return config; + }); +} + +// TODO(stevenzzzz): see issue #7508, consider generalizing this function as it overlaps with +// CdsApiImpl::onConfigUpdate. +void ScopedRdsConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + absl::flat_hash_map scoped_routes; + absl::flat_hash_map scope_name_by_key_hash; + for (const auto& resource_any : resources) { + // Throws (thus rejects all) on any error. + auto scoped_route = + MessageUtil::anyConvert(resource_any); + MessageUtil::validate(scoped_route, validation_visitor_); + const std::string scope_name = scoped_route.name(); + auto scope_config_inserted = scoped_routes.try_emplace(scope_name, std::move(scoped_route)); + if (!scope_config_inserted.second) { throw EnvoyException( - fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + fmt::format("duplicate scoped route configuration '{}' found", scope_name)); + } + const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config = + scope_config_inserted.first->second; + const uint64_t key_fingerprint = MessageUtil::hash(scoped_route_config.key()); + if (!scope_name_by_key_hash.try_emplace(key_fingerprint, scope_name).second) { + throw EnvoyException( + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + scope_name_by_key_hash[key_fingerprint], scope_name)); } } - for (const auto& scoped_route : scoped_routes) { - MessageUtil::validate(scoped_route, validation_visitor_); - } - - // TODO(AndresGuedez): refactor such that it can be shared with other delta APIs (e.g., CDS). - std::vector exception_msgs; - // We need to keep track of which scoped routes we might need to remove. - ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = - scoped_config_manager_.scopedRouteMap(); - for (auto& scoped_route : scoped_routes) { - const std::string& scoped_route_name = scoped_route.name(); - scoped_routes_to_remove.erase(scoped_route_name); - ScopedRouteInfoConstSharedPtr scoped_route_info = - scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) - -> ConfigProvider::ConfigConstSharedPtr { - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - return config; - }); + ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; + Protobuf::RepeatedPtrField to_add_repeated; + Protobuf::RepeatedPtrField to_remove_repeated; + for (auto& iter : scoped_routes) { + const std::string& scope_name = iter.first; + scoped_routes_to_remove.erase(scope_name); + auto* to_add = to_add_repeated.Add(); + to_add->set_name(scope_name); + to_add->set_version(version_info); + to_add->mutable_resource()->PackFrom(iter.second); } for (const auto& scoped_route : scoped_routes_to_remove) { - const std::string scoped_route_name = scoped_route.first; - ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - scoped_config_manager_.removeRoutingScope(scoped_route_name); - applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) - -> ConfigProvider::ConfigConstSharedPtr { - // In place update. - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scoped_route_name); - return config; - }); + *to_remove_repeated.Add() = scoped_route.first; } - - DeltaConfigSubscriptionInstance::onConfigUpdate(); - setLastConfigInfo(absl::optional({absl::nullopt, version_info})); - stats_.config_reload_.inc(); -} + onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); +} // namespace Router ScopedRdsConfigProvider::ScopedRdsConfigProvider( - ScopedRdsConfigSubscriptionSharedPtr&& subscription, - envoy::api::v2::core::ConfigSource rds_config_source) - : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), - subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())), - rds_config_source_(std::move(rds_config_source)) {} + ScopedRdsConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta) {} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); @@ -179,10 +334,9 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const const ScopedRdsConfigSubscription* typed_subscription = static_cast(subscription.get()); dynamic_config->set_name(typed_subscription->name()); - const ScopedConfigManager::ScopedRouteMap& scoped_route_map = - typed_subscription->scopedRouteMap(); + const ScopedRouteMap& scoped_route_map = typed_subscription->scopedRouteMap(); for (const auto& it : scoped_route_map) { - dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->config_proto_); + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->configProto()); } TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), *dynamic_config->mutable_last_updated()); @@ -223,11 +377,13 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( return std::make_shared( scoped_rds_config_source, manager_identifier, typed_optarg.scoped_routes_name_, typed_optarg.scope_key_builder_, factory_context, stat_prefix, + typed_optarg.rds_config_source_, + static_cast(config_provider_manager) + .route_config_provider_manager(), static_cast(config_provider_manager)); }); - return std::make_unique(std::move(subscription), - typed_optarg.rds_config_source_); + return std::make_unique(std::move(subscription)); } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 3d006b6a7039..878cca0a3280 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -3,10 +3,15 @@ #include #include "envoy/api/v2/srds.pb.h" +#include "envoy/common/callback.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/router/route_config_provider_manager.h" #include "envoy/stats/scope.h" #include "common/config/config_provider_impl.h" +#include "common/init/manager_impl.h" +#include "common/router/rds_impl.h" #include "common/router/scoped_config_impl.h" namespace Envoy { @@ -89,43 +94,90 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + envoy::api::v2::core::ConfigSource rds_config_source, + RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); ~ScopedRdsConfigSubscription() override = default; const std::string& name() const { return name_; } - const ScopedConfigManager::ScopedRouteMap& scopedRouteMap() const { - return scoped_config_manager_.scopedRouteMap(); - } + const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: + // A helper class that takes care the life cycle management of a RDS route provider and the + // update callback handle. + struct RdsRouteConfigProviderHelper { + RdsRouteConfigProviderHelper( + ScopedRdsConfigSubscription& parent, std::string scope_name, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Init::Manager& init_manager); + ~RdsRouteConfigProviderHelper() { rds_update_callback_handle_->remove(); } + ConfigConstSharedPtr routeConfig() { return route_provider_->config(); } + + ScopedRdsConfigSubscription& parent_; + std::string scope_name_; + std::unique_ptr route_provider_; + // This handle_ is owned by the route config provider's RDS subscription, when the helper + // destructs, the handle is deleted as well. + Common::CallbackHandle* rds_update_callback_handle_; + }; + + // Adds or updates scopes, create a new RDS provider for each resource, if an exception is thrown + // during updating, the exception message is collected via the exception messages vector. + // Returns true if any scope updated, false otherwise. + bool addOrUpdateScopes(const Protobuf::RepeatedPtrField& resources, + Init::Manager& init_manager, const std::string& version_info, + std::vector& exception_msgs); + // Removes given scopes from the managed set of scopes. + // Returns true if any scope updated, false otherwise. + bool removeScopes(const Protobuf::RepeatedPtrField& scope_names, + const std::string& version_info); + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks + + // NOTE: state-of-the-world form onConfigUpdate(resources, version_info) will throw an + // EnvoyException on any error and essentially reject an update. While the Delta form + // onConfigUpdate(added_resources, removed_resources, version_info) by design will partially + // accept correct RouteConfiguration from management server. void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override; - void onConfigUpdate(const Protobuf::RepeatedPtrField&, - const Protobuf::RepeatedPtrField&, const std::string&) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException*) override { - ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + DeltaConfigSubscriptionInstance::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource).name(); } - + // Propagate RDS updates to ScopeConfigImpl in workers. + void onRdsConfigUpdate(const std::string& scope_name, + RdsRouteConfigSubscription& rds_subscription); + + // ScopedRouteInfo by scope name. + ScopedRouteMap scoped_route_map_; + // RdsRouteConfigProvider by scope name. + absl::flat_hash_map> + route_provider_by_scope_; + // A map of (hash, scope-name), used to detect the key conflict between scopes. + absl::flat_hash_map scope_name_by_hash_; + // For creating RDS subscriptions. + Server::Configuration::FactoryContext& factory_context_; const std::string name_; std::unique_ptr subscription_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder_; Stats::ScopePtr scope_; ScopedRdsStats stats_; - ScopedConfigManager scoped_config_manager_; + const envoy::api::v2::core::ConfigSource rds_config_source_; ProtobufMessage::ValidationVisitor& validation_visitor_; + const std::string stat_prefix_; + RouteConfigProviderManager& route_config_provider_manager_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; @@ -134,22 +186,21 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr(subscription_.get()); + } }; // A ConfigProviderManager for scoped routing configuration that creates static/inline and dynamic // (xds) config providers. class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderManagerImplBase { public: - ScopedRoutesConfigProviderManager(Server::Admin& admin) - : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes") {} + ScopedRoutesConfigProviderManager( + Server::Admin& admin, Router::RouteConfigProviderManager& route_config_provider_manager) + : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes"), + route_config_provider_manager_(route_config_provider_manager) {} ~ScopedRoutesConfigProviderManager() override = default; @@ -174,6 +225,13 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa std::vector>&& config_protos, Server::Configuration::FactoryContext& factory_context, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; + + RouteConfigProviderManager& route_config_provider_manager() { + return route_config_provider_manager_; + } + +private: + RouteConfigProviderManager& route_config_provider_manager_; }; // The optional argument passed to the ConfigProviderManager::create*() functions. diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index cc03d2392364..82fb5274bb96 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -89,10 +89,12 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( return std::make_shared(context.admin()); }); - std::shared_ptr scoped_routes_config_provider_manager = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), [&context] { - return std::make_shared(context.admin()); + std::shared_ptr scoped_routes_config_provider_manager = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), + [&context, route_config_provider_manager] { + return std::make_shared( + context.admin(), *route_config_provider_manager); }); std::shared_ptr filter_config(new HttpConnectionManagerConfig( @@ -100,10 +102,11 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( *scoped_routes_config_provider_manager)); // This lambda captures the shared_ptrs created above, thus preserving the - // reference count. Moreover, keep in mind the capture list determines - // destruction order. - return [route_config_provider_manager, scoped_routes_config_provider_manager, filter_config, - &context, date_provider](Network::FilterManager& filter_manager) -> void { + // reference count. + // Keep in mind the lambda capture list **doesn't** determine the destruction order, but it's fine + // as these captured objects are also global singletons. + return [scoped_routes_config_provider_manager, route_config_provider_manager, date_provider, + filter_config, &context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl( *filter_config, context.drainDecision(), context.random(), context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), @@ -183,8 +186,6 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( break; case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kScopedRoutes: - ENVOY_LOG(warn, "Scoped routing has been enabled but it is not yet fully implemented! HTTP " - "request routing DOES NOT work (yet) with this configuration."); scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::create( config, context_, stats_prefix_, scoped_routes_config_provider_manager_); break; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 5e7e65eeec70..6bf5e0f9ce04 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -629,6 +629,7 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { const std::string&, const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DeltaDummyConfigSubscriptionSharedPtr subscription = + getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, @@ -709,6 +710,8 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { subscription.onConfigUpdate(untyped_dummy_configs, "2"); // NOTE: the config implementation is append only and _does not_ track updates/removals to the // config proto set, so the expectation is to double the size of the set. + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); EXPECT_EQ(provider1->config()->numProtos(), 4); EXPECT_EQ(provider1->configProtoInfoVector().value().version_, "2"); diff --git a/test/common/router/BUILD b/test/common/router/BUILD index eb07c6558972..2ac71059b8bc 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -85,6 +85,7 @@ envoy_cc_test( ], deps = [ "//source/common/router:scoped_config_lib", + "//test/mocks/router:router_mocks", "//test/test_common:utility_lib", ], ) @@ -96,12 +97,16 @@ envoy_cc_test( "abseil_strings", ], deps = [ + "//include/envoy/config:subscription_interface", + "//include/envoy/init:manager_interface", "//source/common/config:utility_lib", "//source/common/http:message_lib", "//source/common/json:json_loader_lib", "//source/common/router:scoped_rds_lib", "//source/server/http:admin_lib", + "//test/mocks/config:config_mocks", "//test/mocks/init:init_mocks", + "//test/mocks/router:router_mocks", "//test/mocks/server:server_mocks", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index fb74aa9694f7..8bd32f455302 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -266,8 +266,8 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { // Get a RouteConfigProvider. This one should create an entry in the RouteConfigProviderManager. rds_.set_route_config_name("foo_route_config"); rds_.mutable_config_source()->set_path("foo_path"); - provider_ = route_config_provider_manager_->createRdsRouteConfigProvider(rds_, factory_context_, - "foo_prefix."); + provider_ = route_config_provider_manager_->createRdsRouteConfigProvider( + rds_, factory_context_, "foo_prefix.", factory_context_.initManager()); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -419,7 +419,7 @@ name: foo_route_config "1"); RouteConfigProviderPtr provider2 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix"); + rds_, factory_context_, "foo_prefix", factory_context_.initManager()); // provider2 should have route config immediately after create EXPECT_TRUE(provider2->configInfo().has_value()); @@ -433,7 +433,7 @@ name: foo_route_config rds2.set_route_config_name("foo_route_config"); rds2.mutable_config_source()->set_path("bar_path"); RouteConfigProviderPtr provider3 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds2, factory_context_, "foo_prefix"); + rds2, factory_context_, "foo_prefix", factory_context_.initManager()); EXPECT_NE(provider3, provider_); factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(route_configs, "provider3"); diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 9c9a06a25934..cf9bfd83055c 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -2,6 +2,7 @@ #include "common/router/scoped_config_impl.h" +#include "test/mocks/router/mocks.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -11,10 +12,12 @@ namespace Router { namespace { using ::Envoy::Http::TestHeaderMapImpl; +using ::testing::NiceMock; +using ::testing::Return; class FooFragment : public ScopeKeyFragmentBase { -private: - bool equals(const ScopeKeyFragmentBase&) const override { return true; }; +public: + uint64_t hash() const override { return 1; } }; TEST(ScopeKeyFragmentBaseTest, EqualSign) { @@ -24,14 +27,33 @@ TEST(ScopeKeyFragmentBaseTest, EqualSign) { EXPECT_NE(foo, bar); } +TEST(ScopeKeyFragmentBaseTest, HashStable) { + FooFragment foo1; + FooFragment foo2; + + // Two FooFragments equal because their hash equals. + EXPECT_EQ(foo1, foo2); + EXPECT_EQ(foo1.hash(), foo2.hash()); + + // Hash value doesn't change. + StringKeyFragment a("abcdefg"); + auto hash_value = a.hash(); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(hash_value, a.hash()); + EXPECT_EQ(StringKeyFragment("abcdefg").hash(), hash_value); + } +} + TEST(StringKeyFragmentTest, Empty) { StringKeyFragment a(""); StringKeyFragment b(""); EXPECT_EQ(a, b); + EXPECT_EQ(a.hash(), b.hash()); StringKeyFragment non_empty("ABC"); EXPECT_NE(a, non_empty); + EXPECT_NE(a.hash(), non_empty.hash()); } TEST(StringKeyFragmentTest, Normal) { @@ -217,7 +239,9 @@ ScopeKey makeKey(const std::vector& parts) { TEST(ScopeKeyDeathTest, AddNullFragment) { ScopeKey key; +#if !defined(NDEBUG) EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); +#endif } TEST(ScopeKeyTest, Unmatches) { @@ -231,6 +255,10 @@ TEST(ScopeKeyTest, Unmatches) { EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + // Order matters. + EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + EXPECT_NE(makeKey({"a", "c", "b"}), makeKey({"a", "b", "c"})); + // Two keys of different length won't match. EXPECT_NE(makeKey({"a", "b"}), makeKey({"a", "b", "c"})); @@ -243,7 +271,7 @@ TEST(ScopeKeyTest, Matches) { EXPECT_EQ(makeKey({"", ""}), makeKey({"", ""})); EXPECT_EQ(makeKey({"a", "", ""}), makeKey({"a", "", ""})); - // Non empty fragments comparison. + // Non empty fragments comparison. EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); } @@ -317,6 +345,170 @@ TEST(ScopeKeyBuilderImplTest, Parse) { EXPECT_EQ(key, nullptr); } +class ScopedRouteInfoTest : public testing::Test { +public: + void SetUp() override { + std::string yaml_plain = R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: foo + - string_key: bar +)EOF"; + TestUtility::loadFromYaml(yaml_plain, scoped_route_config_); + + route_config_ = std::make_shared>(); + route_config_->name_ = "foo_route"; + } + + envoy::api::v2::RouteConfiguration route_configuration_; + envoy::api::v2::ScopedRouteConfiguration scoped_route_config_; + std::shared_ptr route_config_; + std::unique_ptr info_; +}; + +TEST_F(ScopedRouteInfoTest, Creation) { + envoy::api::v2::ScopedRouteConfiguration config_copy = scoped_route_config_; + info_ = std::make_unique(std::move(scoped_route_config_), route_config_); + EXPECT_EQ(info_->routeConfig().get(), route_config_.get()); + EXPECT_TRUE(TestUtility::protoEqual(info_->configProto(), config_copy)); + EXPECT_EQ(info_->scopeName(), "foo_scope"); + EXPECT_EQ(info_->scopeKey(), makeKey({"foo", "bar"})); +} + +class ScopedConfigImplTest : public testing::Test { +public: + void SetUp() override { + std::string yaml_plain = R"EOF( + fragments: + - header_value_extractor: + name: 'foo_header' + element_separator: ',' + element: + key: 'bar' + separator: '=' + - header_value_extractor: + name: 'bar_header' + element_separator: ';' + index: 2 +)EOF"; + TestUtility::loadFromYaml(yaml_plain, key_builder_config_); + + scope_info_a_ = makeScopedRouteInfo(R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: foo + - string_key: bar +)EOF"); + scope_info_a_v2_ = makeScopedRouteInfo(R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: xyz + - string_key: xyz +)EOF"); + scope_info_b_ = makeScopedRouteInfo(R"EOF( + name: bar_scope + route_configuration_name: bar_route + key: + fragments: + - string_key: bar + - string_key: baz +)EOF"); + } + std::shared_ptr makeScopedRouteInfo(const std::string& route_config_yaml) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + TestUtility::loadFromYaml(route_config_yaml, scoped_route_config); + + std::shared_ptr route_config = std::make_shared>(); + route_config->name_ = scoped_route_config.route_configuration_name(); + return std::make_shared(std::move(scoped_route_config), + std::move(route_config)); + } + + std::shared_ptr scope_info_a_; + std::shared_ptr scope_info_a_v2_; + std::shared_ptr scope_info_b_; + ScopedRoutes::ScopeKeyBuilder key_builder_config_; + std::unique_ptr scoped_config_impl_; +}; + +// Test a ScopedConfigImpl returns the correct route Config. +TEST_F(ScopedConfigImplTest, PickRoute) { + scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); + + // Key (foo, bar) maps to scope_info_a_. + ConfigConstSharedPtr route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + }); + EXPECT_EQ(route_config, scope_info_a_->routeConfig()); + + // Key (bar, baz) maps to scope_info_b_. + route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=bar,"}, + {"bar_header", ";val1;baz;val3"}, + }); + EXPECT_EQ(route_config, scope_info_b_->routeConfig()); + + // No such key (bar, NOT_BAZ). + route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",key=value,bar=bar,"}, + {"bar_header", ";val1;NOT_BAZ;val3"}, + }); + EXPECT_EQ(route_config, nullptr); +} + +// Test a ScopedConfigImpl returns the correct route Config before and after scope config update. +TEST_F(ScopedConfigImplTest, Update) { + scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); + + TestHeaderMapImpl headers{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + }; + // Empty ScopeConfig. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Add scope_key (bar, baz). + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + TestHeaderMapImpl{{"foo_header", ",,key=v,bar=bar,"}, {"bar_header", ";val1;baz"}}), + scope_info_b_->routeConfig()); + + // Add scope_key (foo, bar). + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); + // Found scope_info_a_. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), scope_info_a_->routeConfig()); + + // Update scope foo_scope. + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_v2_); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // foo_scope now is keyed by (xyz, xyz). + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + TestHeaderMapImpl{{"foo_header", ",bar=xyz,foo=bar"}, {"bar_header", ";;xyz"}}), + scope_info_a_v2_->routeConfig()); + + // Remove scope "foo_scope". + scoped_config_impl_->removeRoutingScope("foo_scope"); + // scope_info_a_ is gone. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Now delete some non-existent scopes. + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("foo_scope1")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("base_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("bluh_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("xyz_scope")); +} + } // namespace } // namespace Router } // namespace Envoy diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 0bc871f232b8..00936537bc7f 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -2,25 +2,40 @@ #include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/admin/v2alpha/config_dump.pb.validate.h" +#include "envoy/config/subscription.h" +#include "envoy/init/manager.h" #include "envoy/stats/scope.h" #include "common/router/scoped_rds.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/router/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::AnyNumber; +using testing::ByMove; +using testing::DoAll; +using testing::Eq; using testing::InSequence; +using testing::Invoke; +using testing::IsNull; +using testing::NiceMock; using testing::Return; +using testing::ReturnRefOfCopy; namespace Envoy { namespace Router { namespace { +using ::Envoy::Http::TestHeaderMapImpl; + envoy::api::v2::ScopedRouteConfiguration parseScopedRouteConfigurationFromYaml(const std::string& yaml) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; @@ -44,21 +59,39 @@ parseHttpConnectionManagerFromYaml(const std::string& config_yaml) { class ScopedRoutesTestBase : public testing::Test { protected: ScopedRoutesTestBase() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("routes", _)); + route_config_provider_manager_ = + std::make_unique(factory_context_.admin_); + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); - config_provider_manager_ = - std::make_unique(factory_context_.admin_); + config_provider_manager_ = std::make_unique( + factory_context_.admin_, *route_config_provider_manager_); } ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } + // The delta style API helper. + Protobuf::RepeatedPtrField + anyToResource(Protobuf::RepeatedPtrField& resources, + const std::string& version) { + Protobuf::RepeatedPtrField added_resources; + for (const auto& resource_any : resources) { + auto config = TestUtility::anyConvert(resource_any); + auto* to_add = added_resources.Add(); + to_add->set_name(config.name()); + to_add->set_version(version); + to_add->mutable_resource()->PackFrom(config); + } + return added_resources; + } + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } NiceMock factory_context_; - Upstream::ClusterManager::ClusterInfoMap cluster_map_; - Upstream::MockClusterMockPrioritySet cluster_; + std::unique_ptr route_config_provider_manager_; std::unique_ptr config_provider_manager_; + Event::SimulatedTimeSystem time_system_; - envoy::api::v2::core::ConfigSource rds_config_source_; }; class ScopedRdsTest : public ScopedRoutesTestBase { @@ -66,11 +99,56 @@ class ScopedRdsTest : public ScopedRoutesTestBase { void setup() { InSequence s; + // Since factory_context_.cluster_manager_.subscription_factory_.callbacks_ is taken by the SRDS + // subscription. We need to return a different MockSubscription here for each RDS subscription. + // To build the map from RDS route_config_name to the RDS subscription, we need to get the + // route_config_name by mocking start() on the Config::Subscription. + EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, + subscriptionFromConfigSource(_, _, _, _)) + .Times(AnyNumber()); + EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, + subscriptionFromConfigSource( + _, + Eq(Grpc::Common::typeUrl( + envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name())), + _, _)) + .Times(AnyNumber()) + .WillRepeatedly(Invoke([this](const envoy::api::v2::core::ConfigSource&, absl::string_view, + Stats::Scope&, + Envoy::Config::SubscriptionCallbacks& callbacks) { + auto ret = std::make_unique>(); + rds_subscription_by_config_subscription_[ret.get()] = &callbacks; + EXPECT_CALL(*ret, start(_)) + .WillOnce(Invoke( + [this, config_sub_addr = ret.get()](const std::set& resource_names) { + EXPECT_EQ(resource_names.size(), 1); + auto iter = rds_subscription_by_config_subscription_.find(config_sub_addr); + EXPECT_NE(iter, rds_subscription_by_config_subscription_.end()); + rds_subscription_by_name_[*resource_names.begin()] = iter->second; + })); + return ret; + })); + + ON_CALL(factory_context_.init_manager_, add(_)) + .WillByDefault(Invoke([this](const Init::Target& target) { + target_handles_.push_back(target.createHandle("test")); + })); + ON_CALL(factory_context_.init_manager_, initialize(_)) + .WillByDefault(Invoke([this](const Init::Watcher& watcher) { + for (auto& handle_ : target_handles_) { + handle_->initialize(watcher); + } + })); + const std::string config_yaml = R"EOF( name: foo_scoped_routes scope_key_builder: fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: Addr + element: + key: x-foo-key + separator: ; )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; TestUtility::loadFromYaml(config_yaml, scoped_routes_config); @@ -79,11 +157,44 @@ name: foo_scoped_routes ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), scoped_routes_config.rds_config_source(), scoped_routes_config.scope_key_builder())); - subscription_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; + srds_subscription_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; + } + + // Helper function which pushes an update to given RDS subscription, the start(_) of the + // subscription must have been called. + void pushRdsConfig(const std::string& route_config_name, const std::string& version) { + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: test + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: bluh }} +)EOF"; + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(TestUtility::parseYaml( + fmt::format(route_config_tmpl, route_config_name))); + rds_subscription_by_name_[route_config_name]->onConfigUpdate(resources, version); + } + + ScopedRdsConfigProvider* getScopedRdsProvider() const { + return dynamic_cast(provider_.get()); + } + // Helper function which returns the ScopedRouteMap of the subscription. + const ScopedRouteMap& getScopedRouteMap() const { + return getScopedRdsProvider()->subscription().scopedRouteMap(); } - Envoy::Config::SubscriptionCallbacks* subscription_callbacks_{}; + Envoy::Config::SubscriptionCallbacks* srds_subscription_{}; Envoy::Config::ConfigProviderPtr provider_; + std::list target_handles_; + Init::ExpectableWatcherImpl init_watcher_; + + // RDS mocks. + absl::flat_hash_map + rds_subscription_by_config_subscription_; + absl::flat_hash_map rds_subscription_by_name_; }; TEST_F(ScopedRdsTest, ValidateFail) { @@ -99,7 +210,11 @@ route_configuration_name: foo_routes )EOF"; Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources, "1"), ProtoValidationException); + + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'route_configuration_name' validation: value must be > 1 byte. const std::string config_yaml2 = R"EOF( @@ -111,7 +226,10 @@ name: foo_scope )EOF"; Protobuf::RepeatedPtrField resources2; parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources2, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources2, "1"), ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'key' validation: must define at least 1 fragment. const std::string config_yaml3 = R"EOF( @@ -121,11 +239,15 @@ route_configuration_name: foo_routes )EOF"; Protobuf::RepeatedPtrField resources3; parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources3, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources3, "1"), ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed .*value is " + "required.*"); } -// Tests that multiple uniquely named resources are allowed in config updates. -TEST_F(ScopedRdsTest, MultipleResources) { +// Tests that multiple uniquely named non-conflict resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResourcesStow) { setup(); const std::string config_yaml = R"EOF( @@ -140,20 +262,210 @@ route_configuration_name: foo_routes const std::string config_yaml2 = R"EOF( name: foo_scope2 route_configuration_name: foo_routes +key: + fragments: + - string_key: x-bar-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(resources, "1")); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times(2); // SRDS and RDS "foo_routes" + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + + // Verify the config is a ScopedConfigImpl instance, both scopes point to "" as RDS hasn't kicked + // in yet(NullConfigImpl returned). + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + ""); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + ""); + // RDS updates foo_routes. + pushRdsConfig("foo_routes", "111"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + "foo_routes"); + + // Delete foo_scope2. + resources.RemoveLast(); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(resources, "3")); + EXPECT_EQ(getScopedRouteMap().size(), 1); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + EXPECT_EQ( + 2UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // now scope key "x-bar-key" points to nowhere. + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + IsNull()); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); +} + +// Tests that multiple uniquely named non-conflict resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResourcesDelta) { + setup(); + init_watcher_.expectReady().Times(2); // SRDS and RDS "foo_routes" + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes key: fragments: - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-bar-key )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); - EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "1")); + + // Delta API. + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(anyToResource(resources, "2"), {}, "1")); + factory_context_.init_manager_.initialize(init_watcher_); EXPECT_EQ( 1UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + EXPECT_EQ(getScopedRouteMap().size(), 2); + + // Verify the config is a ScopedConfigImpl instance, both scopes point to "" as RDS hasn't kicked + // in yet(NullConfigImpl returned). + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + ""); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + ""); + // RDS updates foo_routes. + pushRdsConfig("foo_routes", "111"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + "foo_routes"); + + // Delete foo_scope2. + resources.RemoveLast(); + Protobuf::RepeatedPtrField deletes; + *deletes.Add() = "foo_scope2"; + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(anyToResource(resources, "4"), deletes, "2")); + EXPECT_EQ(getScopedRouteMap().size(), 1); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + EXPECT_EQ( + 2UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // now scope key "x-bar-key" points to nowhere. + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + IsNull()); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); +} + +// Tests that conflict resources are detected. +TEST_F(ScopedRdsTest, MultipleResourcesWithKeyConflict) { + setup(); + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(resources, "1"), EnvoyException, + ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); + EXPECT_EQ( + // Fully rejected. + 0UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // Scope key "x-foo-key" points to nowhere. + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + IsNull()); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times( + 1); // Just SRDS, RDS "foo_routes" will initialized by the noop init-manager. + EXPECT_EQ(factory_context_.scope_.counter("foo.rds.foo_routes.config_reload").value(), 0UL); + + // Delta API. + EXPECT_CALL(factory_context_.init_manager_, state()) + .WillOnce(Return(Init::Manager::State::Initialized)); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources, "2"), {}, "2"), EnvoyException, + ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); + EXPECT_EQ( + // Partially reject. + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // foo_scope update is applied. + EXPECT_EQ(getScopedRouteMap().size(), 1UL); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + // Scope key "x-foo-key" points to foo_routes due to partial rejection. + pushRdsConfig("foo_routes", "111"); // Push some real route configuration. + EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.rds.foo_routes.config_reload").value()); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); } // Tests that only one resource is provided during a config update. -TEST_F(ScopedRdsTest, InvalidDuplicateResource) { +TEST_F(ScopedRdsTest, InvalidDuplicateResourceSotw) { setup(); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times(0); const std::string config_yaml = R"EOF( name: foo_scope @@ -165,8 +477,42 @@ route_configuration_name: foo_routes Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); - EXPECT_THROW_WITH_MESSAGE(subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, - "duplicate scoped route configuration foo_scope found"); + EXPECT_THROW_WITH_MESSAGE(srds_subscription_->onConfigUpdate(resources, "1"), EnvoyException, + "duplicate scoped route configuration 'foo_scope' found"); +} + +// Tests that only one resource is provided during a config update. +TEST_F(ScopedRdsTest, InvalidDuplicateResourceDelta) { + setup(); + factory_context_.init_manager_.initialize(init_watcher_); + // After the above initialize, the default init_manager should return "Initialized". + EXPECT_CALL(factory_context_.init_manager_, state()) + .WillOnce(Return(Init::Manager::State::Initialized)); + init_watcher_.expectReady().Times( + 1); // SRDS onConfigUpdate breaks, but first foo_routes will + // kick start if it's initialized post-Server/LDS initialization. + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + EXPECT_THROW_WITH_MESSAGE( + srds_subscription_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route(s): duplicate scoped route configuration 'foo_scope' " + "found"); + EXPECT_EQ( + // Partially reject. + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // foo_scope update is applied. + EXPECT_EQ(getScopedRouteMap().size(), 1UL); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); } // Tests a config update failure. @@ -177,31 +523,37 @@ TEST_F(ScopedRdsTest, ConfigUpdateFailure) { timeSystem().setSystemTime(time); const EnvoyException ex(fmt::format("config failure")); // Verify the failure updates the lastUpdated() timestamp. - subscription_callbacks_->onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &ex); + srds_subscription_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, + &ex); EXPECT_EQ(std::chrono::time_point_cast(provider_->lastUpdated()) .time_since_epoch(), time); } -using ScopedRoutesConfigProviderManagerTest = ScopedRoutesTestBase; +// Tests that the /config_dump handler returns the corresponding scoped routing +// config. +TEST_F(ScopedRdsTest, ConfigDump) { + setup(); + factory_context_.init_manager_.initialize(init_watcher_); + EXPECT_CALL(factory_context_.init_manager_, state()) + .Times(2) // There are two SRDS pushes. + .WillRepeatedly(Return(Init::Manager::State::Initialized)); + init_watcher_.expectReady().Times(1); // SRDS only, no RDS push. -// Tests that the /config_dump handler returns the corresponding scoped routing config. -TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump = TestUtility::downcastAndValidate( *message_ptr); - // No routes at all, no last_updated timestamp + // No routes at all(no SRDS push yet), no last_updated timestamp envoy::admin::v2alpha::ScopedRoutesConfigDump expected_config_dump; TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: dynamic_scoped_route_configs: )EOF", expected_config_dump); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump)); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); @@ -215,7 +567,9 @@ stat_prefix: foo name: $0 scope_key_builder: fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: Addr + index: 0 $1 )EOF"; const std::string inline_scoped_route_configs_yaml = R"EOF( @@ -257,17 +611,9 @@ stat_prefix: foo dynamic_scoped_route_configs: )EOF", expected_config_dump); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); - - const std::string scoped_rds_config_yaml = R"EOF( - scoped_rds: - scoped_rds_config_source: -)EOF"; - Envoy::Config::ConfigProviderPtr dynamic_provider = ScopedRoutesConfigProviderUtil::create( - parseHttpConnectionManagerFromYaml(absl::Substitute( - hcm_base_config_yaml, "foo-dynamic-scoped-routes", scoped_rds_config_yaml)), - factory_context_, "foo.", *config_provider_manager_); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump2)); + // Now SRDS kicks off. Protobuf::RepeatedPtrField resources; resources.Add()->PackFrom(parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo @@ -277,8 +623,7 @@ route_configuration_name: dynamic-foo-route-config )EOF")); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891567)); - factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, - "1"); + srds_subscription_->onConfigUpdate(resources, "1"); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: @@ -296,7 +641,7 @@ route_configuration_name: dynamic-foo-route-config seconds: 1234567891 nanos: 234000000 dynamic_scoped_route_configs: - - name: foo-dynamic-scoped-routes + - name: foo_scoped_routes scoped_route_configs: - name: dynamic-foo route_configuration_name: dynamic-foo-route-config @@ -312,11 +657,10 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump3 = TestUtility::downcastAndValidate( *message_ptr); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump3)); resources.Clear(); - factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, - "2"); + srds_subscription_->onConfigUpdate(resources, "2"); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: - name: foo-scoped-routes @@ -333,7 +677,7 @@ route_configuration_name: dynamic-foo-route-config seconds: 1234567891 nanos: 234000000 dynamic_scoped_route_configs: - - name: foo-dynamic-scoped-routes + - name: foo_scoped_routes last_updated: seconds: 1234567891 nanos: 567000000 @@ -344,14 +688,13 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump4 = TestUtility::downcastAndValidate( *message_ptr); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump4.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } -using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderManagerTest; - // Tests that SRDS only allows creation of delta static config providers. -TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { - // Use match all regex due to lack of distinctive matchable output for coverage test. +TEST_F(ScopedRdsTest, DeltaStaticConfigProviderOnly) { + // Use match all regex due to lack of distinctive matchable output for + // coverage test. EXPECT_DEATH(config_provider_manager_->createStaticConfigProvider( parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 32e916da6d9a..e14e87d08fa7 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -48,7 +48,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, http_connection_manager) { const std::string& scope_key_builder_config_yaml = R"EOF( fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: Addr + element: + key: x-foo-key + separator: ; )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder; @@ -124,6 +128,15 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); } + void sendRdsResponse(const std::string& route_config, const std::string& version) { + envoy::api::v2::DiscoveryResponse response; + response.set_version_info(version); + response.set_type_url(Config::TypeUrl::get().RouteConfiguration); + response.add_resources()->PackFrom( + TestUtility::parseYaml(route_config)); + rds_upstream_info_.stream_->sendGrpcMessage(response); + } + void sendScopedRdsResponse(const std::vector& resource_protos, const std::string& version) { ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); @@ -159,19 +172,31 @@ route_configuration_name: foo_route1 )EOF"; const std::string scope_route2 = R"EOF( name: foo_scope2 -route_configuration_name: foo_route2 +route_configuration_name: foo_route1 key: fragments: - - string_key: x-foo-key + - string_key: x-bar-key +)EOF"; + + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} )EOF"; - on_server_init_function_ = [this, &scope_route1, &scope_route2]() { + on_server_init_function_ = [&]() { createScopedRdsStream(); sendScopedRdsResponse({scope_route1, scope_route2}, "1"); + createRdsStream(); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_1"), "1"); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_2"), "2"); }; initialize(); - - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", @@ -179,20 +204,21 @@ route_configuration_name: foo_route2 const std::string scope_route3 = R"EOF( name: foo_scope3 -route_configuration_name: foo_route3 +route_configuration_name: foo_route1 key: fragments: - string_key: x-baz-key )EOF"; sendScopedRdsResponse({scope_route3}, "2"); - - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 6927017134761466251UL); - - // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented - // at this point. + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 3); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_3"), "3"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 3); + // RDS updates won't affect SRDS. + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 6927017134761466251UL); } // Test that a bad config update updates the corresponding stats. diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 827e9c84f875..291bd3248e9d 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -116,11 +116,25 @@ MockRoute::MockRoute() { } MockRoute::~MockRoute() = default; +MockRouteConfigProvider::MockRouteConfigProvider() { + ON_CALL(*this, config()).WillByDefault(Return(route_config_)); +} +MockRouteConfigProvider::~MockRouteConfigProvider() = default; + MockRouteConfigProviderManager::MockRouteConfigProviderManager() = default; MockRouteConfigProviderManager::~MockRouteConfigProviderManager() = default; -MockScopedConfig::MockScopedConfig() = default; +MockScopedConfig::MockScopedConfig() { + ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); +} MockScopedConfig::~MockScopedConfig() = default; +MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() + : config_(std::make_shared()) { + ON_CALL(*this, getConfig()).WillByDefault(Return(config_)); + ON_CALL(*this, apiType()).WillByDefault(Return(ApiType::Delta)); +} +MockScopedRouteConfigProvider::~MockScopedRouteConfigProvider() = default; + } // namespace Router } // namespace Envoy diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 7e6ce48606ea..75249eeaad47 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -8,6 +8,8 @@ #include #include +#include "envoy/common/time.h" +#include "envoy/config/config_provider.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/dispatcher.h" #include "envoy/json/json_object.h" @@ -31,6 +33,7 @@ namespace Envoy { namespace Router { +using ::testing::NiceMock; class MockDirectResponseEntry : public DirectResponseEntry { public: @@ -380,16 +383,29 @@ class MockConfig : public Config { std::string name_{"fake_config"}; }; +class MockRouteConfigProvider : public RouteConfigProvider { +public: + MockRouteConfigProvider(); + ~MockRouteConfigProvider() override; + + MOCK_METHOD0(config, ConfigConstSharedPtr()); + MOCK_CONST_METHOD0(configInfo, absl::optional()); + MOCK_CONST_METHOD0(lastUpdated, SystemTime()); + MOCK_METHOD0(onConfigUpdate, void()); + + std::shared_ptr> route_config_{new NiceMock()}; +}; + class MockRouteConfigProviderManager : public RouteConfigProviderManager { public: MockRouteConfigProviderManager(); ~MockRouteConfigProviderManager() override; - MOCK_METHOD3(createRdsRouteConfigProvider, + MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix)); + const std::string& stat_prefix, Init::Manager& init_manager)); MOCK_METHOD2(createStaticRouteConfigProvider, RouteConfigProviderPtr(const envoy::api::v2::RouteConfiguration& route_config, Server::Configuration::FactoryContext& factory_context)); @@ -401,6 +417,23 @@ class MockScopedConfig : public ScopedConfig { ~MockScopedConfig() override; MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); + + std::shared_ptr route_config_{new NiceMock()}; +}; + +class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { +public: + MockScopedRouteConfigProvider(); + ~MockScopedRouteConfigProvider() override; + + // Config::ConfigProvider + MOCK_CONST_METHOD0(lastUpdated, SystemTime()); + MOCK_CONST_METHOD0(getConfigProto, Protobuf::Message*()); + MOCK_CONST_METHOD0(getConfigProtos, Envoy::Config::ConfigProvider::ConfigProtoVector()); + MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); + MOCK_CONST_METHOD0(apiType, ApiType()); + + std::shared_ptr config_; }; } // namespace Router