From b34e7fff2dfb46294bb05487283e87a76fbb1030 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 15 Mar 2019 13:20:28 -0700 Subject: [PATCH 1/6] http fault: implement header controlled faults Part of https://github.com/envoyproxy/envoy/issues/5942 Signed-off-by: Matt Klein --- api/envoy/config/filter/fault/v2/fault.proto | 26 +++- .../http_filters/fault_filter.rst | 47 +++++-- docs/root/intro/version_history.rst | 10 +- source/common/config/filter_json.cc | 2 - source/extensions/filters/common/fault/BUILD | 20 +++ .../filters/common/fault/fault_config.cc | 78 +++++++++++ .../filters/common/fault/fault_config.h | 130 ++++++++++++++++++ source/extensions/filters/http/fault/BUILD | 1 + .../filters/http/fault/fault_filter.cc | 73 ++++++---- .../filters/http/fault/fault_filter.h | 29 ++-- .../filters/network/mongo_proxy/BUILD | 1 + .../filters/network/mongo_proxy/config.cc | 9 +- .../filters/network/mongo_proxy/proxy.cc | 37 ++--- .../filters/network/mongo_proxy/proxy.h | 31 +---- test/extensions/filters/common/fault/BUILD | 18 +++ .../filters/common/fault/fault_config_test.cc | 52 +++++++ .../fault/fault_filter_integration_test.cc | 79 ++++++++++- .../filters/http/fault/fault_filter_test.cc | 19 --- .../filters/network/mongo_proxy/proxy_test.cc | 15 +- test/test_common/utility.h | 7 + 20 files changed, 543 insertions(+), 141 deletions(-) create mode 100644 source/extensions/filters/common/fault/BUILD create mode 100644 source/extensions/filters/common/fault/fault_config.cc create mode 100644 source/extensions/filters/common/fault/fault_config.h create mode 100644 test/extensions/filters/common/fault/BUILD create mode 100644 test/extensions/filters/common/fault/fault_config_test.cc diff --git a/api/envoy/config/filter/fault/v2/fault.proto b/api/envoy/config/filter/fault/v2/fault.proto index 89d1dc2c55ff..f27f9d446267 100644 --- a/api/envoy/config/filter/fault/v2/fault.proto +++ b/api/envoy/config/filter/fault/v2/fault.proto @@ -19,19 +19,25 @@ import "gogoproto/gogo.proto"; // Delay specification is used to inject latency into the // HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections. message FaultDelay { + // Fault delays are controlled via an HTTP header (if applicable). See the + // :ref:`http fault filter ` documentation for + // more information. + message HeaderDelay { + } + enum FaultDelayType { - // Fixed delay (step function). + // Unused and deprecated. FIXED = 0; } - // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is - // supported. - FaultDelayType type = 1 [(validate.rules).enum.defined_only = true]; + // Unused and deprecated. Will be removed in the next release. + FaultDelayType type = 1 [deprecated = true]; reserved 2; oneof fault_delay_secifier { option (validate.required) = true; + // Add a fixed delay before forwarding the operation upstream. See // https://developers.google.com/protocol-buffers/docs/proto3#json for // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified @@ -40,6 +46,9 @@ message FaultDelay { // for the specified period. This is required if type is FIXED. google.protobuf.Duration fixed_delay = 3 [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + + // Fault delays are controlled via an HTTP header (if applicable). + HeaderDelay header_delay = 5; } // The percentage of operations/connections/requests on which the delay will be injected. @@ -54,11 +63,20 @@ message FaultRateLimit { uint64 limit_kbps = 1 [(validate.rules).uint64.gte = 1]; } + // Rate limits are controlled via an HTTP header (if applicable). See the + // :ref:`http fault filter ` documentation for + // more information. + message HeaderLimit { + } + oneof limit_type { option (validate.required) = true; // A fixed rate limit. FixedLimit fixed_limit = 1; + + // Rate limits are controlled via an HTTP header (if applicable). + HeaderLimit header_limit = 3; } // The percentage of operations/connections/requests on which the rate limit will be injected. diff --git a/docs/root/configuration/http_filters/fault_filter.rst b/docs/root/configuration/http_filters/fault_filter.rst index 39de89628fe2..fcb5cbe90c80 100644 --- a/docs/root/configuration/http_filters/fault_filter.rst +++ b/docs/root/configuration/http_filters/fault_filter.rst @@ -16,15 +16,6 @@ The scope of failures is restricted to those that are observable by an application communicating over the network. CPU and disk failures on the local host cannot be emulated. -Currently, the fault injection filter has the following limitations: - -* Abort codes are restricted to HTTP status codes only -* Delays are restricted to fixed duration. - -Future versions will include support for restricting faults to specific -routes, injecting *gRPC* and *HTTP/2* specific error codes and delay -durations based on distributions. - Configuration ------------- @@ -36,6 +27,44 @@ Configuration * :ref:`v2 API reference ` * This filter should be configured with the name *envoy.fault*. +.. _config_http_filters_fault_injection_http_header: + +HTTP header fault configuration +------------------------------- + +The fault filter has the capability to allow fault configuration to be specified by the caller. +This is useful in certain scenarios in which it is desired to allow the client to specify its own +fault configuration. The currently supported header controls are: + +* Request delay configuration via the *x-envoy-throttle-request-latency* header. The header value + should be an integer that specifies the number of milliseconds to throttle the latency for. +* Response rate limit configuration via the *x-envoy-throttle-response-throughput* header. The + header value should be an integer that specified the limit in KiB/s and must be > 0. + +.. attention:: + + Allowing header control is inherently dangerous if exposed to untrusted clients. In this case, + it is suggested to use the :ref:`max_active_faults + ` setting to limit the + maximum concurrent faults that can be active at any given time. + +The following is an example configuration that enables header control for both of the above +options: + +.. code-block:: yaml + + name: envoy.fault + config: + max_active_faults: 100 + delay: + header_delay: {} + percentage: + numerator: 100 + response_rate_limit: + header_limit: {} + percentage: + numerator: 100 + .. _config_http_filters_fault_injection_runtime: Runtime diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index a8e0813cd129..9a7ba38e7cf8 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -33,18 +33,20 @@ Version history ` setting, as well as :ref:`statistics ` for the number of active faults and the number of faults the overflowed. -* fault: add :ref:`response rate limit +* fault: added :ref:`response rate limit ` fault injection. +* fault: added :ref:`HTTP header fault configuration + ` to the HTTP fault filter. * governance: extending Envoy deprecation policy from 1 release (0-3 months) to 2 releases (3-6 months). * health check: expected response codes in http health checks are now :ref:`configurable `. * http: added new grpc_http1_reverse_bridge filter for converting gRPC requests into HTTP/1.1 requests. * http: fixed a bug where Content-Length:0 was added to HTTP/1 204 responses. -* outlier_detection: added support for :ref:`outlier detection event protobuf-based logging `. -* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to ::ref:`MySQL proxy` for more details. * http: added :ref:`max request headers size `. The default behaviour is unchanged. * http: added modifyDecodingBuffer/modifyEncodingBuffer to allow modifying the buffered request/response data. -* performance: new buffer implementation (disabled by default; to test it, add "--use-libevent-buffers 0" to the command-line arguments when starting Envoy). * http: added encodeComplete/decodeComplete. These are invoked at the end of the stream, after all data has been encoded/decoded respectively. Default implementation is a no-op. +* outlier_detection: added support for :ref:`outlier detection event protobuf-based logging `. +* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to ::ref:`MySQL proxy` for more details. +* performance: new buffer implementation (disabled by default; to test it, add "--use-libevent-buffers 0" to the command-line arguments when starting Envoy). * ratelimit: removed deprecated rate limit configuration from bootstrap. * redis: added :ref:`hashtagging ` to guarantee a given key's upstream. * redis: added :ref:`latency stats ` for commands. diff --git a/source/common/config/filter_json.cc b/source/common/config/filter_json.cc index 1ee3c2c1f721..79356782afc5 100644 --- a/source/common/config/filter_json.cc +++ b/source/common/config/filter_json.cc @@ -254,7 +254,6 @@ void FilterJson::translateMongoProxy( auto* delay = proto_config.mutable_delay(); auto* percentage = delay->mutable_percentage(); - delay->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED); percentage->set_numerator(static_cast(json_fault->getInteger("percent"))); percentage->set_denominator(envoy::type::FractionalPercent::HUNDRED); JSON_UTIL_SET_DURATION_FROM_FIELD(*json_fault, *delay, fixed_delay, duration); @@ -284,7 +283,6 @@ void FilterJson::translateFaultFilter( if (!json_config_delay->empty()) { auto* delay = proto_config.mutable_delay(); auto* percentage = delay->mutable_percentage(); - delay->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED); percentage->set_numerator( static_cast(json_config_delay->getInteger("fixed_delay_percent"))); percentage->set_denominator(envoy::type::FractionalPercent::HUNDRED); diff --git a/source/extensions/filters/common/fault/BUILD b/source/extensions/filters/common/fault/BUILD new file mode 100644 index 000000000000..b8607b4f861b --- /dev/null +++ b/source/extensions/filters/common/fault/BUILD @@ -0,0 +1,20 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "fault_config_lib", + srcs = ["fault_config.cc"], + hdrs = ["fault_config.h"], + deps = [ + "//include/envoy/http:header_map_interface", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/config/filter/fault/v2:fault_cc", + ], +) diff --git a/source/extensions/filters/common/fault/fault_config.cc b/source/extensions/filters/common/fault/fault_config.cc new file mode 100644 index 000000000000..f6e4f73ad6fe --- /dev/null +++ b/source/extensions/filters/common/fault/fault_config.cc @@ -0,0 +1,78 @@ +#include "extensions/filters/common/fault/fault_config.h" + +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Fault { + +FaultDelayConfig::FaultDelayConfig(const envoy::config::filter::fault::v2::FaultDelay& delay_config) + : percentage_(delay_config.percentage()) { + switch (delay_config.fault_delay_secifier_case()) { + case envoy::config::filter::fault::v2::FaultDelay::kFixedDelay: + provider_ = std::make_unique( + std::chrono::milliseconds(PROTOBUF_GET_MS_REQUIRED(delay_config, fixed_delay))); + break; + case envoy::config::filter::fault::v2::FaultDelay::kHeaderDelay: + provider_ = std::make_unique(); + break; + case envoy::config::filter::fault::v2::FaultDelay::FAULT_DELAY_SECIFIER_NOT_SET: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +absl::optional +FaultDelayConfig::HeaderDelayProvider::duration(const Http::HeaderEntry* header) const { + if (header == nullptr) { + return absl::nullopt; + } + + uint64_t value; + if (!StringUtil::atoull(header->value().getStringView().data(), value)) { + return absl::nullopt; + } + + return std::chrono::milliseconds(value); +} + +FaultRateLimitConfig::FaultRateLimitConfig( + const envoy::config::filter::fault::v2::FaultRateLimit& rate_limit_config) + : percentage_(rate_limit_config.percentage()) { + switch (rate_limit_config.limit_type_case()) { + case envoy::config::filter::fault::v2::FaultRateLimit::kFixedLimit: + provider_ = + std::make_unique(rate_limit_config.fixed_limit().limit_kbps()); + break; + case envoy::config::filter::fault::v2::FaultRateLimit::kHeaderLimit: + provider_ = std::make_unique(); + break; + case envoy::config::filter::fault::v2::FaultRateLimit::LIMIT_TYPE_NOT_SET: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +absl::optional +FaultRateLimitConfig::HeaderRateLimitProvider::rateKbps(const Http::HeaderEntry* header) const { + if (header == nullptr) { + return absl::nullopt; + } + + uint64_t value; + if (!StringUtil::atoull(header->value().getStringView().data(), value)) { + return absl::nullopt; + } + + if (value == 0) { + return absl::nullopt; + } + + return value; +} + +} // namespace Fault +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/fault/fault_config.h b/source/extensions/filters/common/fault/fault_config.h new file mode 100644 index 000000000000..e19512f12ff1 --- /dev/null +++ b/source/extensions/filters/common/fault/fault_config.h @@ -0,0 +1,130 @@ +#pragma once + +#include "envoy/config/filter/fault/v2/fault.pb.h" +#include "envoy/http/header_map.h" + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Fault { + +class HeaderNameValues { +public: + const Http::LowerCaseString ThrottleRequestLatency{"x-envoy-throttle-request-latency"}; + const Http::LowerCaseString ThrottleResponseThroughput{"x-envoy-throttle-response-throughput"}; +}; + +typedef ConstSingleton HeaderNames; + +/** + * Generic configuration for a delay fault. + */ +class FaultDelayConfig { +public: + FaultDelayConfig(const envoy::config::filter::fault::v2::FaultDelay& delay_config); + + const envoy::type::FractionalPercent& percentage() const { return percentage_; } + absl::optional duration(const Http::HeaderEntry* header) const { + return provider_->duration(header); + } + +private: + // Abstract delay provider. + class DelayProvider { + public: + virtual ~DelayProvider() = default; + + // Return the duration to use. Optionally passed an HTTP header that may contain the delay + // depending on the provider implementation. + virtual absl::optional + duration(const Http::HeaderEntry* header) const PURE; + }; + + // Delay provider that uses a fixed delay. + class FixedDelayProvider : public DelayProvider { + public: + FixedDelayProvider(std::chrono::milliseconds delay) : delay_(delay) {} + + // DelayProvider + absl::optional duration(const Http::HeaderEntry*) const override { + return delay_; + } + + private: + const std::chrono::milliseconds delay_; + }; + + // Delay provider the reads a delay from an HTTP header. + class HeaderDelayProvider : public DelayProvider { + public: + // DelayProvider + absl::optional + duration(const Http::HeaderEntry* header) const override; + }; + + using DelayProviderPtr = std::unique_ptr; + + DelayProviderPtr provider_; + const envoy::type::FractionalPercent percentage_; +}; + +using FaultDelayConfigPtr = std::unique_ptr; +using FaultDelayConfigSharedPtr = std::shared_ptr; + +/** + * Generic configuration for a rate limit fault. + */ +class FaultRateLimitConfig { +public: + FaultRateLimitConfig(const envoy::config::filter::fault::v2::FaultRateLimit& rate_limit_config); + + const envoy::type::FractionalPercent& percentage() const { return percentage_; } + absl::optional rateKbps(const Http::HeaderEntry* header) const { + return provider_->rateKbps(header); + } + +private: + // Abstract rate limit provider. + class RateLimitProvider { + public: + virtual ~RateLimitProvider() = default; + + // Return the rate limit to use in KiB/s. Optionally passed an HTTP header that may contain the + // rate limit depending on the provider implementation. + virtual absl::optional rateKbps(const Http::HeaderEntry* header) const PURE; + }; + + // Rate limit provider that uses a fixed rate limit. + class FixedRateLimitProvider : public RateLimitProvider { + public: + FixedRateLimitProvider(uint64_t fixed_rate_kbps) : fixed_rate_kbps_(fixed_rate_kbps) {} + absl::optional rateKbps(const Http::HeaderEntry*) const override { + return fixed_rate_kbps_; + } + + private: + const uint64_t fixed_rate_kbps_; + }; + + // Rate limit provider that reads the rate limit from an HTTP header. + class HeaderRateLimitProvider : public RateLimitProvider { + public: + absl::optional rateKbps(const Http::HeaderEntry* header) const override; + }; + + using RateLimitProviderPtr = std::unique_ptr; + + RateLimitProviderPtr provider_; + const envoy::type::FractionalPercent percentage_; +}; + +using FaultRateLimitConfigPtr = std::unique_ptr; + +} // namespace Fault +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/fault/BUILD b/source/extensions/filters/http/fault/BUILD index 4c282cb5e3fa..a5b48b5af103 100644 --- a/source/extensions/filters/http/fault/BUILD +++ b/source/extensions/filters/http/fault/BUILD @@ -32,6 +32,7 @@ envoy_cc_library( "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", "//source/common/protobuf:utility_lib", + "//source/extensions/filters/common/fault:fault_config_lib", "@envoy_api//envoy/config/filter/http/fault/v2:fault_cc", ], ) diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 737c92733d16..0fa230b60f8b 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -34,9 +34,8 @@ FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPF } if (fault.has_delay()) { - const auto& delay = fault.delay(); - fixed_delay_percentage_ = delay.percentage(); - fixed_duration_ms_ = PROTOBUF_GET_MS_OR_DEFAULT(delay, fixed_delay, 0); + request_delay_config_ = + std::make_unique(fault.delay()); } for (const Http::HeaderUtility::HeaderData& header_map : fault.headers()) { @@ -54,11 +53,8 @@ FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPF } if (fault.has_response_rate_limit()) { - RateLimit rate_limit; - ASSERT(fault.response_rate_limit().has_fixed_limit()); - rate_limit.fixed_rate_kbps_ = fault.response_rate_limit().fixed_limit().limit_kbps(); - rate_limit.percentage_ = fault.response_rate_limit().percentage(); - response_rate_limit_ = rate_limit; + response_rate_limit_ = + std::make_unique(fault.response_rate_limit()); } } @@ -122,13 +118,14 @@ Http::FilterHeadersStatus FaultFilter::decodeHeaders(Http::HeaderMap& headers, b fmt::format("fault.http.{}.abort.http_status", downstream_cluster_); } - maybeSetupResponseRateLimit(); + maybeSetupResponseRateLimit(headers); - absl::optional duration_ms = delayDuration(); - if (duration_ms) { + absl::optional duration = delayDuration(headers); + if (duration) { delay_timer_ = decoder_callbacks_->dispatcher().createTimer([this]() -> void { postDelayInjection(); }); - delay_timer_->enableTimer(std::chrono::milliseconds(duration_ms.value())); + ENVOY_LOG(debug, "fault: delaying request {}ms", duration.value().count()); + delay_timer_->enableTimer(duration.value()); recordDelaysInjectedStats(); decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::DelayInjected); return Http::FilterHeadersStatus::StopIteration; @@ -142,15 +139,21 @@ Http::FilterHeadersStatus FaultFilter::decodeHeaders(Http::HeaderMap& headers, b return Http::FilterHeadersStatus::Continue; } -void FaultFilter::maybeSetupResponseRateLimit() { - if (!fault_settings_->responseRateLimit().has_value()) { +void FaultFilter::maybeSetupResponseRateLimit(const Http::HeaderMap& request_headers) { + if (fault_settings_->responseRateLimit() == nullptr) { + return; + } + + absl::optional rate_kbps = fault_settings_->responseRateLimit()->rateKbps( + request_headers.get(Filters::Common::Fault::HeaderNames::get().ThrottleResponseThroughput)); + if (!rate_kbps.has_value()) { return; } // TODO(mattklein123): Allow runtime override via downstream cluster similar to the other keys. if (!config_->runtime().snapshot().featureEnabled( - RuntimeKeys::get().ResponseRateLimitKey, - fault_settings_->responseRateLimit().value().percentage_)) { + RuntimeKeys::get().ResponseRateLimitPercentKey, + fault_settings_->responseRateLimit()->percentage())) { return; } @@ -159,8 +162,7 @@ void FaultFilter::maybeSetupResponseRateLimit() { config_->stats().response_rl_injected_.inc(); response_limiter_ = std::make_unique( - fault_settings_->responseRateLimit().value().fixed_rate_kbps_, - encoder_callbacks_->encoderBufferLimit(), + rate_kbps.value(), encoder_callbacks_->encoderBufferLimit(), [this] { encoder_callbacks_->onEncoderFilterAboveWriteBufferHighWatermark(); }, [this] { encoder_callbacks_->onEncoderFilterBelowWriteBufferLowWatermark(); }, [this](Buffer::Instance& data, bool end_stream) { @@ -186,11 +188,15 @@ bool FaultFilter::faultOverflow() { } bool FaultFilter::isDelayEnabled() { - bool enabled = config_->runtime().snapshot().featureEnabled(RuntimeKeys::get().DelayPercentKey, - fault_settings_->delayPercentage()); + if (fault_settings_->requestDelay() == nullptr) { + return false; + } + + bool enabled = config_->runtime().snapshot().featureEnabled( + RuntimeKeys::get().DelayPercentKey, fault_settings_->requestDelay()->percentage()); if (!downstream_cluster_delay_percent_key_.empty()) { - enabled |= config_->runtime().snapshot().featureEnabled(downstream_cluster_delay_percent_key_, - fault_settings_->delayPercentage()); + enabled |= config_->runtime().snapshot().featureEnabled( + downstream_cluster_delay_percent_key_, fault_settings_->requestDelay()->percentage()); } return enabled; } @@ -205,22 +211,31 @@ bool FaultFilter::isAbortEnabled() { return enabled; } -absl::optional FaultFilter::delayDuration() { - absl::optional ret; +absl::optional +FaultFilter::delayDuration(const Http::HeaderMap& request_headers) { + absl::optional ret; if (!isDelayEnabled()) { return ret; } - uint64_t duration = config_->runtime().snapshot().getInteger(RuntimeKeys::get().DelayDurationKey, - fault_settings_->delayDuration()); + // See if the delay provider has a default delay, if not there is no delay. + auto config_duration = fault_settings_->requestDelay()->duration( + request_headers.get(Filters::Common::Fault::HeaderNames::get().ThrottleRequestLatency)); + if (!config_duration.has_value()) { + return ret; + } + + std::chrono::milliseconds duration = + std::chrono::milliseconds(config_->runtime().snapshot().getInteger( + RuntimeKeys::get().DelayDurationKey, config_duration.value().count())); if (!downstream_cluster_delay_duration_key_.empty()) { - duration = - config_->runtime().snapshot().getInteger(downstream_cluster_delay_duration_key_, duration); + duration = std::chrono::milliseconds(config_->runtime().snapshot().getInteger( + downstream_cluster_delay_duration_key_, duration.count())); } // Delay only if the duration is >0ms - if (duration > 0) { + if (duration.count() > 0) { ret = duration; } diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index ce6970f7c8e0..58e539c67766 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -17,6 +17,8 @@ #include "common/common/token_bucket_impl.h" #include "common/http/header_utility.h" +#include "extensions/filters/common/fault/fault_config.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { @@ -46,35 +48,32 @@ struct FaultFilterStats { */ class FaultSettings : public Router::RouteSpecificFilterConfig { public: - struct RateLimit { - uint64_t fixed_rate_kbps_; - envoy::type::FractionalPercent percentage_; - }; - FaultSettings(const envoy::config::filter::http::fault::v2::HTTPFault& fault); const std::vector& filterHeaders() const { return fault_filter_headers_; } envoy::type::FractionalPercent abortPercentage() const { return abort_percentage_; } - envoy::type::FractionalPercent delayPercentage() const { return fixed_delay_percentage_; } - uint64_t delayDuration() const { return fixed_duration_ms_; } uint64_t abortCode() const { return http_status_; } + const Filters::Common::Fault::FaultDelayConfig* requestDelay() const { + return request_delay_config_.get(); + } const std::string& upstreamCluster() const { return upstream_cluster_; } const std::unordered_set& downstreamNodes() const { return downstream_nodes_; } absl::optional maxActiveFaults() const { return max_active_faults_; } - const absl::optional& responseRateLimit() const { return response_rate_limit_; } + const Filters::Common::Fault::FaultRateLimitConfig* responseRateLimit() const { + return response_rate_limit_.get(); + } private: envoy::type::FractionalPercent abort_percentage_; uint64_t http_status_{}; // HTTP or gRPC return codes - envoy::type::FractionalPercent fixed_delay_percentage_; - uint64_t fixed_duration_ms_{}; // in milliseconds + Filters::Common::Fault::FaultDelayConfigPtr request_delay_config_; std::string upstream_cluster_; // restrict faults to specific upstream cluster std::vector fault_filter_headers_; std::unordered_set downstream_nodes_{}; // Inject failures for specific downstream absl::optional max_active_faults_; - absl::optional response_rate_limit_; + Filters::Common::Fault::FaultRateLimitConfigPtr response_rate_limit_; }; /** @@ -160,7 +159,7 @@ class StreamRateLimiter : Logger::Loggable { /** * A filter that is capable of faulting an entire request before dispatching it upstream. */ -class FaultFilter : public Http::StreamFilter { +class FaultFilter : public Http::StreamFilter, Logger::Loggable { public: FaultFilter(FaultFilterConfigSharedPtr config); ~FaultFilter(); @@ -200,7 +199,7 @@ class FaultFilter : public Http::StreamFilter { const std::string DelayDurationKey = "fault.http.delay.fixed_duration_ms"; const std::string AbortHttpStatusKey = "fault.http.abort.http_status"; const std::string MaxActiveFaultsKey = "fault.http.max_active_faults"; - const std::string ResponseRateLimitKey = "fault.http.rate_limit.response_percent"; + const std::string ResponseRateLimitPercentKey = "fault.http.rate_limit.response_percent"; }; using RuntimeKeys = ConstSingleton; @@ -215,10 +214,10 @@ class FaultFilter : public Http::StreamFilter { bool matchesDownstreamNodes(const Http::HeaderMap& headers); bool isAbortEnabled(); bool isDelayEnabled(); - absl::optional delayDuration(); + absl::optional delayDuration(const Http::HeaderMap& request_headers); uint64_t abortHttpStatus(); void maybeIncActiveFaults(); - void maybeSetupResponseRateLimit(); + void maybeSetupResponseRateLimit(const Http::HeaderMap& request_headers); FaultFilterConfigSharedPtr config_; Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index 6891198b77b2..36a94de85abc 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -75,6 +75,7 @@ envoy_cc_library( "//source/common/network:filter_lib", "//source/common/protobuf:utility_lib", "//source/common/singleton:const_singleton", + "//source/extensions/filters/common/fault:fault_config_lib", "//source/extensions/filters/network:well_known_names", "@envoy_api//envoy/config/filter/network/mongo_proxy/v2:mongo_proxy_cc", ], diff --git a/source/extensions/filters/network/mongo_proxy/config.cc b/source/extensions/filters/network/mongo_proxy/config.cc index 44cf1dbf0286..a8989947e75a 100644 --- a/source/extensions/filters/network/mongo_proxy/config.cc +++ b/source/extensions/filters/network/mongo_proxy/config.cc @@ -27,11 +27,9 @@ Network::FilterFactoryCb MongoProxyFilterConfigFactory::createFilterFactoryFromP context.dispatcher().timeSource())); } - FaultConfigSharedPtr fault_config; + Filters::Common::Fault::FaultDelayConfigSharedPtr fault_config; if (proto_config.has_delay()) { - auto delay = proto_config.delay(); - ASSERT(delay.has_fixed_delay()); - fault_config = std::make_shared(proto_config.delay()); + fault_config = std::make_shared(proto_config.delay()); } const bool emit_dynamic_metadata = proto_config.emit_dynamic_metadata(); @@ -39,8 +37,7 @@ Network::FilterFactoryCb MongoProxyFilterConfigFactory::createFilterFactoryFromP emit_dynamic_metadata](Network::FilterManager& filter_manager) -> void { filter_manager.addFilter(std::make_shared( stat_prefix, context.scope(), context.runtime(), access_log, fault_config, - context.drainDecision(), context.random(), context.dispatcher().timeSource(), - emit_dynamic_metadata)); + context.drainDecision(), context.dispatcher().timeSource(), emit_dynamic_metadata)); }; } diff --git a/source/extensions/filters/network/mongo_proxy/proxy.cc b/source/extensions/filters/network/mongo_proxy/proxy.cc index b0022ae14cb1..59621b337928 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.cc +++ b/source/extensions/filters/network/mongo_proxy/proxy.cc @@ -56,13 +56,12 @@ void AccessLog::logMessage(const Message& message, bool full, ProxyFilter::ProxyFilter(const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime, AccessLogSharedPtr access_log, - const FaultConfigSharedPtr& fault_config, - const Network::DrainDecision& drain_decision, - Runtime::RandomGenerator& generator, TimeSource& time_source, + const Filters::Common::Fault::FaultDelayConfigSharedPtr& fault_config, + const Network::DrainDecision& drain_decision, TimeSource& time_source, bool emit_dynamic_metadata) : stat_prefix_(stat_prefix), scope_(scope), stats_(generateStats(stat_prefix, scope)), - runtime_(runtime), drain_decision_(drain_decision), generator_(generator), - access_log_(access_log), fault_config_(fault_config), time_source_(time_source), + runtime_(runtime), drain_decision_(drain_decision), access_log_(access_log), + fault_config_(fault_config), time_source_(time_source), emit_dynamic_metadata_(emit_dynamic_metadata) { if (!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().ConnectionLoggingEnabled, 100)) { @@ -365,26 +364,30 @@ DecoderPtr ProdProxyFilter::createDecoder(DecoderCallbacks& callbacks) { return DecoderPtr{new DecoderImpl(callbacks)}; } -absl::optional ProxyFilter::delayDuration() { - absl::optional result; +absl::optional ProxyFilter::delayDuration() { + absl::optional result; if (!fault_config_) { return result; } if (!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().FixedDelayPercent, - fault_config_->delayPercentage().numerator(), - generator_.random(), - ProtobufPercentHelper::fractionalPercentDenominatorToInt( - fault_config_->delayPercentage().denominator()))) { + fault_config_->percentage())) { return result; } - const uint64_t duration = runtime_.snapshot().getInteger( - MongoRuntimeConfig::get().FixedDelayDurationMs, fault_config_->delayDuration()); + // See if the delay provider has a default delay, if not there is no delay. + auto config_duration = fault_config_->duration(nullptr); + if (!config_duration.has_value()) { + return result; + } + + const std::chrono::milliseconds duration = + std::chrono::milliseconds(runtime_.snapshot().getInteger( + MongoRuntimeConfig::get().FixedDelayDurationMs, config_duration.value().count())); // Delay only if the duration is > 0ms. - if (duration > 0) { + if (duration.count() > 0) { result = duration; } @@ -405,12 +408,12 @@ void ProxyFilter::tryInjectDelay() { return; } - const absl::optional delay_ms = delayDuration(); + const absl::optional delay = delayDuration(); - if (delay_ms) { + if (delay) { delay_timer_ = read_callbacks_->connection().dispatcher().createTimer( [this]() -> void { delayInjectionTimerCallback(); }); - delay_timer_->enableTimer(std::chrono::milliseconds(delay_ms.value())); + delay_timer_->enableTimer(delay.value()); stats_.delays_injected_.inc(); } } diff --git a/source/extensions/filters/network/mongo_proxy/proxy.h b/source/extensions/filters/network/mongo_proxy/proxy.h index c1e183e93bb1..6c9a308800f4 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.h +++ b/source/extensions/filters/network/mongo_proxy/proxy.h @@ -23,6 +23,7 @@ #include "common/protobuf/utility.h" #include "common/singleton/const_singleton.h" +#include "extensions/filters/common/fault/fault_config.h" #include "extensions/filters/network/mongo_proxy/codec.h" #include "extensions/filters/network/mongo_proxy/utility.h" @@ -98,24 +99,6 @@ class AccessLog { typedef std::shared_ptr AccessLogSharedPtr; -/** - * Mongo fault configuration. - */ -class FaultConfig { -public: - FaultConfig(const envoy::config::filter::fault::v2::FaultDelay& fault_config) - : delay_percentage_(fault_config.percentage()), - duration_ms_(PROTOBUF_GET_MS_REQUIRED(fault_config, fixed_delay)) {} - envoy::type::FractionalPercent delayPercentage() const { return delay_percentage_; } - uint64_t delayDuration() const { return duration_ms_; } - -private: - envoy::type::FractionalPercent delay_percentage_; - const uint64_t duration_ms_; -}; - -typedef std::shared_ptr FaultConfigSharedPtr; - /** * A sniffing filter for mongo traffic. The current implementation makes a copy of read/written * data, decodes it, and generates stats. @@ -126,9 +109,10 @@ class ProxyFilter : public Network::Filter, Logger::Loggable { public: ProxyFilter(const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime, - AccessLogSharedPtr access_log, const FaultConfigSharedPtr& fault_config, - const Network::DrainDecision& drain_decision, Runtime::RandomGenerator& generator, - TimeSource& time_system, bool emit_dynamic_metadata); + AccessLogSharedPtr access_log, + const Filters::Common::Fault::FaultDelayConfigSharedPtr& fault_config, + const Network::DrainDecision& drain_decision, TimeSource& time_system, + bool emit_dynamic_metadata); ~ProxyFilter(); virtual DecoderPtr createDecoder(DecoderCallbacks& callbacks) PURE; @@ -188,7 +172,7 @@ class ProxyFilter : public Network::Filter, void doDecode(Buffer::Instance& buffer); void logMessage(Message& message, bool full); void onDrainClose(); - absl::optional delayDuration(); + absl::optional delayDuration(); void delayInjectionTimerCallback(); void tryInjectDelay(); @@ -198,14 +182,13 @@ class ProxyFilter : public Network::Filter, MongoProxyStats stats_; Runtime::Loader& runtime_; const Network::DrainDecision& drain_decision_; - Runtime::RandomGenerator& generator_; Buffer::OwnedImpl read_buffer_; Buffer::OwnedImpl write_buffer_; bool sniffing_{true}; std::list active_query_list_; AccessLogSharedPtr access_log_; Network::ReadFilterCallbacks* read_callbacks_{}; - const FaultConfigSharedPtr fault_config_; + const Filters::Common::Fault::FaultDelayConfigSharedPtr fault_config_; Event::TimerPtr delay_timer_; Event::TimerPtr drain_close_timer_; TimeSource& time_source_; diff --git a/test/extensions/filters/common/fault/BUILD b/test/extensions/filters/common/fault/BUILD new file mode 100644 index 000000000000..b32b0d000811 --- /dev/null +++ b/test/extensions/filters/common/fault/BUILD @@ -0,0 +1,18 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "fault_config_test", + srcs = ["fault_config_test.cc"], + deps = [ + "//source/extensions/filters/common/fault:fault_config_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/common/fault/fault_config_test.cc b/test/extensions/filters/common/fault/fault_config_test.cc new file mode 100644 index 000000000000..38066633cf4a --- /dev/null +++ b/test/extensions/filters/common/fault/fault_config_test.cc @@ -0,0 +1,52 @@ +#include "extensions/filters/common/fault/fault_config.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Fault { +namespace { + +TEST(FaultConfigTest, FaultDelayHeaderConfig) { + envoy::config::filter::fault::v2::FaultDelay proto_config; + proto_config.mutable_header_delay(); + FaultDelayConfig config(proto_config); + + // No header. + EXPECT_EQ(absl::nullopt, config.duration(nullptr)); + + // Header with bad data. + Http::TestHeaderMapImpl bad_headers{{"x-envoy-throttle-request-latency", "abc"}}; + EXPECT_EQ(absl::nullopt, + config.duration(bad_headers.get(HeaderNames::get().ThrottleRequestLatency))); +} + +TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { + envoy::config::filter::fault::v2::FaultRateLimit proto_config; + proto_config.mutable_header_limit(); + FaultRateLimitConfig config(proto_config); + + // No header. + EXPECT_EQ(absl::nullopt, config.rateKbps(nullptr)); + + // Header with bad data. + Http::TestHeaderMapImpl bad_headers{{"x-envoy-throttle-response-throughput", "abc"}}; + EXPECT_EQ(absl::nullopt, + config.rateKbps(bad_headers.get(HeaderNames::get().ThrottleResponseThroughput))); + + // Header with zero. + Http::TestHeaderMapImpl zero_headers{{"x-envoy-throttle-response-throughput", "0"}}; + EXPECT_EQ(absl::nullopt, + config.rateKbps(zero_headers.get(HeaderNames::get().ThrottleResponseThroughput))); +} + +} // namespace +} // namespace Fault +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/fault/fault_filter_integration_test.cc b/test/extensions/filters/http/fault/fault_filter_integration_test.cc index 782f88933efe..d0d419e3c9e9 100644 --- a/test/extensions/filters/http/fault/fault_filter_integration_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_integration_test.cc @@ -26,6 +26,20 @@ name: envoy.fault percentage: numerator: 100 )EOF"; + + const std::string header_fault_config_ = + R"EOF( +name: envoy.fault +config: + delay: + header_delay: {} + percentage: + numerator: 100 + response_rate_limit: + header_limit: {} + percentage: + numerator: 100 +)EOF"; }; // Fault integration tests that should run with all protocols, useful for testing various @@ -47,6 +61,9 @@ config: {} codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); auto response = sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 1024); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); } // Response rate limited with no trailers. @@ -65,10 +82,56 @@ TEST_P(FaultIntegrationTestAllProtocols, ResponseRateLimitNoTrailers) { simTime().sleep(std::chrono::milliseconds(63)); decoder->waitForBodyData(1088); - // Advance time and wait for a ticks worth of data and end stream. + // Advance time and wait for a tick worth of data and end stream. simTime().sleep(std::chrono::milliseconds(63)); decoder->waitForBodyData(1152); decoder->waitForEndStream(); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); +} + +// Request delay and response rate limited via header configuration. +TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultConfig) { + initializeFilter(header_fault_config_); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-envoy-throttle-request-latency", "200"}, + {"x-envoy-throttle-response-throughput", "1"}}; + const auto current_time = simTime().monotonicTime(); + IntegrationStreamDecoderPtr decoder = codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + // At least 200ms of simulated time should have elapsed before we got the upstream request. + EXPECT_LE(std::chrono::milliseconds(200), simTime().monotonicTime() - current_time); + + // Verify response body throttling. + upstream_request_->encodeHeaders(default_response_headers_, false); + Buffer::OwnedImpl data(std::string(1025, 'a')); + upstream_request_->encodeData(data, true); + decoder->waitForBodyData(1024); + + // Advance time and wait for a tick worth of data and end stream. + simTime().sleep(std::chrono::milliseconds(63)); + decoder->waitForBodyData(1025); + decoder->waitForEndStream(); + + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); +} + +// Header configuration with no headers, so no fault injection. +TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultConfigNoHeaders) { + initializeFilter(header_fault_config_); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + auto response = + sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 1024); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); } // Fault integration tests that run with HTTP/2 only, used for fully testing trailers. @@ -90,11 +153,11 @@ TEST_P(FaultIntegrationTestHttp2, ResponseRateLimitTrailersBodyFlushed) { upstream_request_->encodeData(data, false); decoder->waitForBodyData(1024); - // Advance time and wait for a ticks worth of data. + // Advance time and wait for a tick worth of data. simTime().sleep(std::chrono::milliseconds(63)); decoder->waitForBodyData(1088); - // Advance time and wait for a ticks worth of data. + // Advance time and wait for a tick worth of data. simTime().sleep(std::chrono::milliseconds(63)); decoder->waitForBodyData(1152); @@ -103,6 +166,9 @@ TEST_P(FaultIntegrationTestHttp2, ResponseRateLimitTrailersBodyFlushed) { upstream_request_->encodeTrailers(trailers); decoder->waitForEndStream(); EXPECT_NE(nullptr, decoder->trailers()); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); } // Rate limiting with trailers received before the body has been flushed. @@ -119,15 +185,18 @@ TEST_P(FaultIntegrationTestHttp2, ResponseRateLimitTrailersBodyNotFlushed) { upstream_request_->encodeTrailers(trailers); decoder->waitForBodyData(1024); - // Advance time and wait for a ticks worth of data. + // Advance time and wait for a tick worth of data. simTime().sleep(std::chrono::milliseconds(63)); decoder->waitForBodyData(1088); - // Advance time and wait for a ticks worth of data, trailers, and end stream. + // Advance time and wait for a tick worth of data, trailers, and end stream. simTime().sleep(std::chrono::milliseconds(63)); decoder->waitForBodyData(1152); decoder->waitForEndStream(); EXPECT_NE(nullptr, decoder->trailers()); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); } } // namespace diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index 72335a27548c..e6d108ff0da0 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -262,13 +262,6 @@ TEST(FaultFilterBadConfigTest, MissingDelayDuration) { faultFilterBadConfigHelper(json); } -MATCHER_P(Percent, rhs, "") { - envoy::type::FractionalPercent expected; - expected.set_numerator(rhs); - expected.set_denominator(envoy::type::FractionalPercent::HUNDRED); - return TestUtility::protoEqual(expected, arg); -} - TEST_F(FaultFilterTest, AbortWithHttpStatus) { envoy::config::filter::http::fault::v2::HTTPFault fault; fault.mutable_abort()->mutable_percentage()->set_numerator(100); @@ -281,13 +274,6 @@ TEST_F(FaultFilterTest, AbortWithHttpStatus) { getInteger("fault.http.max_active_faults", std::numeric_limits::max())) .WillOnce(Return(std::numeric_limits::max())); - // Delay related calls - EXPECT_CALL(runtime_.snapshot_, - featureEnabled("fault.http.delay.fixed_delay_percent", - Matcher(Percent(0)))) - .WillOnce(Return(false)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.delay.fixed_duration_ms", _)).Times(0); EXPECT_CALL(decoder_filter_callbacks_, continueDecoding()).Times(0); EXPECT_CALL(decoder_filter_callbacks_.stream_info_, setResponseFlag(StreamInfo::ResponseFlag::DelayInjected)) @@ -375,7 +361,6 @@ TEST_F(FaultFilterTest, Overflow) { TEST_F(FaultFilterTest, FixedDelayDeprecatedPercentAndNonZeroDuration) { envoy::config::filter::http::fault::v2::HTTPFault fault; - fault.mutable_delay()->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED); fault.mutable_delay()->mutable_percentage()->set_numerator(50); fault.mutable_delay()->mutable_percentage()->set_denominator( envoy::type::FractionalPercent::HUNDRED); @@ -1009,10 +994,6 @@ class FaultFilterRateLimitTest : public FaultFilterTest { featureEnabled("fault.http.rate_limit.response_percent", Matcher(Percent(100)))) .WillOnce(Return(enable_runtime)); - EXPECT_CALL(runtime_.snapshot_, - featureEnabled("fault.http.delay.fixed_delay_percent", - Matcher(Percent(0)))) - .WillOnce(Return(false)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("fault.http.abort.abort_percent", Matcher(Percent(0)))) diff --git a/test/extensions/filters/network/mongo_proxy/proxy_test.cc b/test/extensions/filters/network/mongo_proxy/proxy_test.cc index 5a27208b02f5..3c1b7168ba51 100644 --- a/test/extensions/filters/network/mongo_proxy/proxy_test.cc +++ b/test/extensions/filters/network/mongo_proxy/proxy_test.cc @@ -25,6 +25,7 @@ using testing::_; using testing::AnyNumber; using testing::AtLeast; using testing::Invoke; +using testing::Matcher; using testing::NiceMock; using testing::Property; using testing::Return; @@ -82,7 +83,7 @@ class MongoProxyFilterTest : public testing::Test { void initializeFilter(bool emit_dynamic_metadata = false) { filter_ = std::make_unique("test.", store_, runtime_, access_log_, - fault_config_, drain_decision_, generator_, + fault_config_, drain_decision_, dispatcher_.timeSource(), emit_dynamic_metadata); filter_->initializeReadFilterCallbacks(read_filter_callbacks_); filter_->onNewConnection(); @@ -93,15 +94,16 @@ class MongoProxyFilterTest : public testing::Test { } void setupDelayFault(bool enable_fault) { - envoy::config::filter::fault::v2::FaultDelay fault{}; + envoy::config::filter::fault::v2::FaultDelay fault; fault.mutable_percentage()->set_numerator(50); fault.mutable_percentage()->set_denominator(envoy::type::FractionalPercent::HUNDRED); fault.mutable_fixed_delay()->CopyFrom(Protobuf::util::TimeUtil::MillisecondsToDuration(10)); - fault_config_.reset(new FaultConfig(fault)); + fault_config_.reset(new Filters::Common::Fault::FaultDelayConfig(fault)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled(_, _, _, 100)).Times(AnyNumber()); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("mongo.fault.fixed_delay.percent", 50, _, 100)) + EXPECT_CALL(runtime_.snapshot_, + featureEnabled("mongo.fault.fixed_delay.percent", + Matcher(Percent(50)))) .WillOnce(Return(enable_fault)); if (enable_fault) { @@ -117,12 +119,11 @@ class MongoProxyFilterTest : public testing::Test { std::shared_ptr file_{ new NiceMock()}; AccessLogSharedPtr access_log_; - FaultConfigSharedPtr fault_config_; + Filters::Common::Fault::FaultDelayConfigSharedPtr fault_config_; std::unique_ptr filter_; NiceMock read_filter_callbacks_; Envoy::AccessLog::MockAccessLogManager log_manager_; NiceMock drain_decision_; - NiceMock generator_; TestStreamInfo stream_info_; }; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index d30c0f9a7569..7c4ef7ede800 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -522,4 +522,11 @@ MATCHER_P(ProtoEq, rhs, "") { return TestUtility::protoEqual(arg, rhs); } MATCHER_P(RepeatedProtoEq, rhs, "") { return TestUtility::repeatedPtrFieldEqual(arg, rhs); } +MATCHER_P(Percent, rhs, "") { + envoy::type::FractionalPercent expected; + expected.set_numerator(rhs); + expected.set_denominator(envoy::type::FractionalPercent::HUNDRED); + return TestUtility::protoEqual(expected, arg); +} + } // namespace Envoy From 2bc39f5bf5fe161bd3c07bce0c6d2f4c51edc4a2 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Tue, 19 Mar 2019 13:40:11 -0700 Subject: [PATCH 2/6] add DEPRECATED.md Signed-off-by: Matt Klein --- DEPRECATED.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEPRECATED.md b/DEPRECATED.md index 199e6bc6c01d..17c9838a4fb0 100644 --- a/DEPRECATED.md +++ b/DEPRECATED.md @@ -13,6 +13,10 @@ A logged warning is expected for each deprecated item that is in deprecation win Set the `filter_enabled` field instead. * Use of google.protobuf.Struct for extension opaque configs is deprecated. Use google.protobuf.Any instead or pack google.protobuf.Struct in google.protobuf.Any. +* Use of the `type` field in the `FaultDelay` message (found in + [fault.proto](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/fault/v2/fault.proto)) + has been deprecated. It was never used and setting it has no effect. It will be removed in the + following release. ## Version 1.9.0 (Dec 20, 2018) From eda7fa889a2bbae471d1eb1478b51eaee82c05be Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 21 Mar 2019 12:46:48 -0700 Subject: [PATCH 3/6] comments Signed-off-by: Matt Klein --- .../configuration/http_filters/fault_filter.rst | 8 ++++---- .../extensions/filters/common/fault/fault_config.h | 4 ++-- .../extensions/filters/http/fault/fault_filter.cc | 4 ++-- .../filters/common/fault/fault_config_test.cc | 14 ++++++-------- .../http/fault/fault_filter_integration_test.cc | 4 ++-- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/docs/root/configuration/http_filters/fault_filter.rst b/docs/root/configuration/http_filters/fault_filter.rst index fcb5cbe90c80..75f97dba7721 100644 --- a/docs/root/configuration/http_filters/fault_filter.rst +++ b/docs/root/configuration/http_filters/fault_filter.rst @@ -29,16 +29,16 @@ Configuration .. _config_http_filters_fault_injection_http_header: -HTTP header fault configuration -------------------------------- +Controlling fault injection via HTTP headers +-------------------------------------------- The fault filter has the capability to allow fault configuration to be specified by the caller. This is useful in certain scenarios in which it is desired to allow the client to specify its own fault configuration. The currently supported header controls are: -* Request delay configuration via the *x-envoy-throttle-request-latency* header. The header value +* Request delay configuration via the *x-envoy-fault-delay-request* header. The header value should be an integer that specifies the number of milliseconds to throttle the latency for. -* Response rate limit configuration via the *x-envoy-throttle-response-throughput* header. The +* Response rate limit configuration via the *x-envoy-fault-throughput-response* header. The header value should be an integer that specified the limit in KiB/s and must be > 0. .. attention:: diff --git a/source/extensions/filters/common/fault/fault_config.h b/source/extensions/filters/common/fault/fault_config.h index e19512f12ff1..61b3ada9eda7 100644 --- a/source/extensions/filters/common/fault/fault_config.h +++ b/source/extensions/filters/common/fault/fault_config.h @@ -13,8 +13,8 @@ namespace Fault { class HeaderNameValues { public: - const Http::LowerCaseString ThrottleRequestLatency{"x-envoy-throttle-request-latency"}; - const Http::LowerCaseString ThrottleResponseThroughput{"x-envoy-throttle-response-throughput"}; + const Http::LowerCaseString DelayRequest{"x-envoy-fault-delay-request"}; + const Http::LowerCaseString ThroughputResponse{"x-envoy-fault-throughput-response"}; }; typedef ConstSingleton HeaderNames; diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 0fa230b60f8b..a837dc618226 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -145,7 +145,7 @@ void FaultFilter::maybeSetupResponseRateLimit(const Http::HeaderMap& request_hea } absl::optional rate_kbps = fault_settings_->responseRateLimit()->rateKbps( - request_headers.get(Filters::Common::Fault::HeaderNames::get().ThrottleResponseThroughput)); + request_headers.get(Filters::Common::Fault::HeaderNames::get().ThroughputResponse)); if (!rate_kbps.has_value()) { return; } @@ -221,7 +221,7 @@ FaultFilter::delayDuration(const Http::HeaderMap& request_headers) { // See if the delay provider has a default delay, if not there is no delay. auto config_duration = fault_settings_->requestDelay()->duration( - request_headers.get(Filters::Common::Fault::HeaderNames::get().ThrottleRequestLatency)); + request_headers.get(Filters::Common::Fault::HeaderNames::get().DelayRequest)); if (!config_duration.has_value()) { return ret; } diff --git a/test/extensions/filters/common/fault/fault_config_test.cc b/test/extensions/filters/common/fault/fault_config_test.cc index 38066633cf4a..39df2697246e 100644 --- a/test/extensions/filters/common/fault/fault_config_test.cc +++ b/test/extensions/filters/common/fault/fault_config_test.cc @@ -20,9 +20,8 @@ TEST(FaultConfigTest, FaultDelayHeaderConfig) { EXPECT_EQ(absl::nullopt, config.duration(nullptr)); // Header with bad data. - Http::TestHeaderMapImpl bad_headers{{"x-envoy-throttle-request-latency", "abc"}}; - EXPECT_EQ(absl::nullopt, - config.duration(bad_headers.get(HeaderNames::get().ThrottleRequestLatency))); + Http::TestHeaderMapImpl bad_headers{{"x-envoy-fault-delay-request", "abc"}}; + EXPECT_EQ(absl::nullopt, config.duration(bad_headers.get(HeaderNames::get().DelayRequest))); } TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { @@ -34,14 +33,13 @@ TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { EXPECT_EQ(absl::nullopt, config.rateKbps(nullptr)); // Header with bad data. - Http::TestHeaderMapImpl bad_headers{{"x-envoy-throttle-response-throughput", "abc"}}; - EXPECT_EQ(absl::nullopt, - config.rateKbps(bad_headers.get(HeaderNames::get().ThrottleResponseThroughput))); + Http::TestHeaderMapImpl bad_headers{{"x-envoy-fault-throughput-response", "abc"}}; + EXPECT_EQ(absl::nullopt, config.rateKbps(bad_headers.get(HeaderNames::get().ThroughputResponse))); // Header with zero. - Http::TestHeaderMapImpl zero_headers{{"x-envoy-throttle-response-throughput", "0"}}; + Http::TestHeaderMapImpl zero_headers{{"x-envoy-fault-throughput-response", "0"}}; EXPECT_EQ(absl::nullopt, - config.rateKbps(zero_headers.get(HeaderNames::get().ThrottleResponseThroughput))); + config.rateKbps(zero_headers.get(HeaderNames::get().ThroughputResponse))); } } // namespace diff --git a/test/extensions/filters/http/fault/fault_filter_integration_test.cc b/test/extensions/filters/http/fault/fault_filter_integration_test.cc index d0d419e3c9e9..3aabd7841206 100644 --- a/test/extensions/filters/http/fault/fault_filter_integration_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_integration_test.cc @@ -99,8 +99,8 @@ TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultConfig) { {":path", "/test/long/url"}, {":scheme", "http"}, {":authority", "host"}, - {"x-envoy-throttle-request-latency", "200"}, - {"x-envoy-throttle-response-throughput", "1"}}; + {"x-envoy-fault-delay-request", "200"}, + {"x-envoy-fault-throughput-response", "1"}}; const auto current_time = simTime().monotonicTime(); IntegrationStreamDecoderPtr decoder = codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); From c6ea7fc39dc6fd4bdfc303d3f745786171523a6d Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Tue, 26 Mar 2019 10:51:44 -0700 Subject: [PATCH 4/6] comments Signed-off-by: Matt Klein --- api/envoy/config/filter/http/fault/v2/fault.proto | 4 +++- docs/root/intro/version_history.rst | 2 +- source/extensions/filters/http/fault/fault_filter.cc | 5 +++-- .../filters/common/fault/fault_config_test.cc | 10 ++++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/api/envoy/config/filter/http/fault/v2/fault.proto b/api/envoy/config/filter/http/fault/v2/fault.proto index df4258968ab6..0cbf58114a3e 100644 --- a/api/envoy/config/filter/http/fault/v2/fault.proto +++ b/api/envoy/config/filter/http/fault/v2/fault.proto @@ -80,7 +80,9 @@ message HTTPFault { // amount due to the implementation details. google.protobuf.UInt32Value max_active_faults = 6; - // The response rate limit to be applied to the response body of the stream. + // The response rate limit to be applied to the response body of the stream. When configured, + // the percentage can be overriden by the :ref:`fault.http.rate_limit.response_percent + // ` runtime key. // // .. attention:: // This is a per-stream limit versus a connection level limit. This means that concurrent streams diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c42e26fa1a97..604742746f99 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -48,7 +48,7 @@ Version history * http: added modifyDecodingBuffer/modifyEncodingBuffer to allow modifying the buffered request/response data. * http: added encodeComplete/decodeComplete. These are invoked at the end of the stream, after all data has been encoded/decoded respectively. Default implementation is a no-op. * outlier_detection: added support for :ref:`outlier detection event protobuf-based logging `. -* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to ::ref:`MySQL proxy` for more details. +* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to :ref:`MySQL proxy` for more details. * performance: new buffer implementation (disabled by default; to test it, add "--use-libevent-buffers 0" to the command-line arguments when starting Envoy). * ratelimit: removed deprecated rate limit configuration from bootstrap. * redis: added :ref:`hashtagging ` to guarantee a given key's upstream. diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index a837dc618226..6c10fd2ff5fc 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -121,7 +121,7 @@ Http::FilterHeadersStatus FaultFilter::decodeHeaders(Http::HeaderMap& headers, b maybeSetupResponseRateLimit(headers); absl::optional duration = delayDuration(headers); - if (duration) { + if (duration.has_value()) { delay_timer_ = decoder_callbacks_->dispatcher().createTimer([this]() -> void { postDelayInjection(); }); ENVOY_LOG(debug, "fault: delaying request {}ms", duration.value().count()); @@ -219,7 +219,8 @@ FaultFilter::delayDuration(const Http::HeaderMap& request_headers) { return ret; } - // See if the delay provider has a default delay, if not there is no delay. + // See if the configured delay provider has a default delay, if not there is no delay (e.g., + // header configuration and no/invalid header). auto config_duration = fault_settings_->requestDelay()->duration( request_headers.get(Filters::Common::Fault::HeaderNames::get().DelayRequest)); if (!config_duration.has_value()) { diff --git a/test/extensions/filters/common/fault/fault_config_test.cc b/test/extensions/filters/common/fault/fault_config_test.cc index 39df2697246e..a01a1c569006 100644 --- a/test/extensions/filters/common/fault/fault_config_test.cc +++ b/test/extensions/filters/common/fault/fault_config_test.cc @@ -22,6 +22,11 @@ TEST(FaultConfigTest, FaultDelayHeaderConfig) { // Header with bad data. Http::TestHeaderMapImpl bad_headers{{"x-envoy-fault-delay-request", "abc"}}; EXPECT_EQ(absl::nullopt, config.duration(bad_headers.get(HeaderNames::get().DelayRequest))); + + // Valid header. + Http::TestHeaderMapImpl good_headers{{"x-envoy-fault-delay-request", "123"}}; + EXPECT_EQ(std::chrono::milliseconds(123), + config.duration(good_headers.get(HeaderNames::get().DelayRequest)).value()); } TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { @@ -40,6 +45,11 @@ TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { Http::TestHeaderMapImpl zero_headers{{"x-envoy-fault-throughput-response", "0"}}; EXPECT_EQ(absl::nullopt, config.rateKbps(zero_headers.get(HeaderNames::get().ThroughputResponse))); + + // Valid header. + Http::TestHeaderMapImpl good_headers{{"x-envoy-fault-throughput-response", "123"}}; + EXPECT_EQ(123UL, + config.rateKbps(good_headers.get(HeaderNames::get().ThroughputResponse)).value()); } } // namespace From 70506d18c00802ebcd402bc13cea7183fd9f8868 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Tue, 26 Mar 2019 12:14:13 -0700 Subject: [PATCH 5/6] fix Signed-off-by: Matt Klein --- api/envoy/config/filter/http/fault/v2/fault.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/filter/http/fault/v2/fault.proto b/api/envoy/config/filter/http/fault/v2/fault.proto index 0cbf58114a3e..bc491580bb15 100644 --- a/api/envoy/config/filter/http/fault/v2/fault.proto +++ b/api/envoy/config/filter/http/fault/v2/fault.proto @@ -81,7 +81,7 @@ message HTTPFault { google.protobuf.UInt32Value max_active_faults = 6; // The response rate limit to be applied to the response body of the stream. When configured, - // the percentage can be overriden by the :ref:`fault.http.rate_limit.response_percent + // the percentage can be overridden by the :ref:`fault.http.rate_limit.response_percent // ` runtime key. // // .. attention:: From a79a8b05ac5f00cd26e660916c65660fd6b82e78 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Tue, 26 Mar 2019 12:33:43 -0700 Subject: [PATCH 6/6] comment Signed-off-by: Matt Klein --- .../http_filters/fault_filter.rst | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/root/configuration/http_filters/fault_filter.rst b/docs/root/configuration/http_filters/fault_filter.rst index 75f97dba7721..90b11404b90b 100644 --- a/docs/root/configuration/http_filters/fault_filter.rst +++ b/docs/root/configuration/http_filters/fault_filter.rst @@ -72,26 +72,38 @@ Runtime The HTTP fault injection filter supports the following global runtime settings: +.. attention:: + + Some of the following runtime keys require the filter to be configured for the specific fault + type and some do not. Please consult the documentation for each key for more information. + fault.http.abort.abort_percent % of requests that will be aborted if the headers match. Defaults to the *abort_percent* specified in config. If the config does not contain an - *abort* block, then *abort_percent* defaults to 0. + *abort* block, then *abort_percent* defaults to 0. For historic reasons, this runtime key is + available regardless of whether the filter is :ref:`configured for abort + `. fault.http.abort.http_status HTTP status code that will be used as the of requests that will be aborted if the headers match. Defaults to the HTTP status code specified in the config. If the config does not contain an *abort* block, then - *http_status* defaults to 0. + *http_status* defaults to 0. For historic reasons, this runtime key is + available regardless of whether the filter is :ref:`configured for abort + `. fault.http.delay.fixed_delay_percent % of requests that will be delayed if the headers match. Defaults to the - *delay_percent* specified in the config or 0 otherwise. + *delay_percent* specified in the config or 0 otherwise. This runtime key is only available when + the filter is :ref:`configured for delay + `. fault.http.delay.fixed_duration_ms The delay duration in milliseconds. If not specified, the *fixed_duration_ms* specified in the config will be used. If this field is missing from both the runtime and the config, no delays will be - injected. + injected. This runtime key is only available when the filter is :ref:`configured for delay + `. fault.http.max_active_faults The maximum number of active faults (of all types) that Envoy will will inject via the fault @@ -101,10 +113,10 @@ fault.http.max_active_faults ` setting will be used. fault.http.rate_limit.response_percent - % of requests which will have a response rate limit fault injected, if the filter is - :ref:`configured ` to - do so. Defaults to the value set in the :ref:`percentage - ` field. + % of requests which will have a response rate limit fault injected. Defaults to the value set in + the :ref:`percentage ` field. + This runtime key is only available when the filter is :ref:`configured for response rate limiting + `. *Note*, fault filter runtime settings for the specific downstream cluster override the default ones if present. The following are downstream specific