http fault: add response rate limit injection (#6267)

Part of #5942

Signed-off-by: Matt Klein <mklein@lyft.com>

api/envoy/config/filter/fault/v2/fault.proto

@@ -42,6 +42,25 @@ message FaultDelay {
         [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true];
   }
 
-  // The percentage of operations/connection requests on which the delay will be injected.
-  envoy.type.FractionalPercent percentage = 4;
+  // The percentage of operations/connections/requests on which the delay will be injected.
+  type.FractionalPercent percentage = 4;
+}
+
+// Describes a rate limit to be applied.
+message FaultRateLimit {
+  // Describes a fixed/constant rate limit.
+  message FixedLimit {
+    // The limit supplied in KiB/s.
+    uint64 limit_kbps = 1 [(validate.rules).uint64.gte = 1];
+  }
+
+  oneof limit_type {
+    option (validate.required) = true;
+
+    // A fixed rate limit.
+    FixedLimit fixed_limit = 1;
+  }
+
+  // The percentage of operations/connections/requests on which the rate limit will be injected.
+  type.FractionalPercent percentage = 2;
 }

api/envoy/config/filter/http/fault/v2/fault.proto

@@ -30,13 +30,13 @@ message FaultAbort {
 
   // The percentage of requests/operations/connections that will be aborted with the error code
   // provided.
-  envoy.type.FractionalPercent percentage = 3;
+  type.FractionalPercent percentage = 3;
 }
 
 message HTTPFault {
   // If specified, the filter will inject delays based on the values in the
-  // object. At least *abort* or *delay* must be specified.
-  envoy.config.filter.fault.v2.FaultDelay delay = 1;
+  // object.
+  filter.fault.v2.FaultDelay delay = 1;
 
   // If specified, the filter will abort requests based on the values in
   // the object. At least *abort* or *delay* must be specified.
@@ -79,4 +79,11 @@ message HTTPFault {
   //   limit. It's possible for the number of active faults to rise slightly above the configured
   //   amount due to the implementation details.
   google.protobuf.UInt32Value max_active_faults = 6;
+
+  // The response rate limit to be applied to the response body of the stream.
+  //
+  // .. attention::
+  //  This is a per-stream limit versus a connection level limit. This means that concurrent streams
+  //  will each get an independent limit.
+  filter.fault.v2.FaultRateLimit response_rate_limit = 7;
 }

docs/root/configuration/http_filters/fault_filter.rst

@@ -71,6 +71,12 @@ fault.http.max_active_faults
   cause resource constraint issues. If not specified, the :ref:`max_active_faults
   <envoy_api_field_config.filter.http.fault.v2.HTTPFault.max_active_faults>` setting will be used.
 
+fault.http.rate_limit.response_percent
+  % of requests which will have a response rate limit fault injected, if the filter is
+  :ref:`configured <envoy_api_field_config.filter.http.fault.v2.HTTPFault.response_rate_limit>` to
+  do so. Defaults to the value set in the :ref:`percentage
+  <envoy_api_field_config.filter.fault.v2.FaultRateLimit.percentage>` field.
+
 *Note*, fault filter runtime settings for the specific downstream cluster
 override the default ones if present. The following are downstream specific
 runtime keys:
@@ -100,6 +106,7 @@ owning HTTP connection manager.
 
   delays_injected, Counter, Total requests that were delayed
   aborts_injected, Counter, Total requests that were aborted
+  response_rl_injected, Counter, "Total requests that had a response rate limit selected for injection (actually injection may not occur due to disconnect, reset, no body, etc.)"
   faults_overflow, Counter, Total number of faults that were not injected due to overflowing the :ref:`max_active_faults <envoy_api_field_config.filter.http.fault.v2.HTTPFault.max_active_faults>` setting
   active_faults, Gauge, Total number of faults active at the current time
   <downstream-cluster>.delays_injected, Counter, Total delayed requests for the given downstream cluster

docs/root/intro/version_history.rst

@@ -32,6 +32,8 @@ Version history
   <envoy_api_field_config.filter.http.fault.v2.HTTPFault.max_active_faults>` setting, as well as
   :ref:`statistics <config_http_filters_fault_injection_stats>` for the number of active faults
   and the number of faults the overflowed.
+* fault: add :ref:`response rate limit
+  <envoy_api_field_config.filter.http.fault.v2.HTTPFault.response_rate_limit>` fault injection.
 * governance: extending Envoy deprecation policy from 1 release (0-3 months) to 2 releases (3-6 months).
 * health check: expected response codes in http health checks are now :ref:`configurable <envoy_api_msg_core.HealthCheck.HttpHealthCheck>`.
 * http: added new grpc_http1_reverse_bridge filter for converting gRPC requests into HTTP/1.1 requests.

include/envoy/event/timer.h

@@ -31,6 +31,11 @@ class Timer {
    * Enable a pending timeout. If a timeout is already pending, it will be reset to the new timeout.
    */
   virtual void enableTimer(const std::chrono::milliseconds& d) PURE;
+
+  /**
+   * Return whether the timer is currently armed.
+   */
+  virtual bool enabled() PURE;
 };
 
 typedef std::unique_ptr<Timer> TimerPtr;

source/common/event/timer_impl.cc

@@ -31,5 +31,7 @@ void TimerImpl::enableTimer(const std::chrono::milliseconds& d) {
   }
 }
 
+bool TimerImpl::enabled() { return 0 != evtimer_pending(&raw_event_, nullptr); }
+
 } // namespace Event
 } // namespace Envoy

source/common/event/timer_impl.h

@@ -20,6 +20,7 @@ class TimerImpl : public Timer, ImplBase {
   // Timer
   void disableTimer() override;
   void enableTimer(const std::chrono::milliseconds& d) override;
+  bool enabled() override;
 
 private:
   TimerCb cb_;

source/extensions/filters/http/fault/BUILD

@@ -23,8 +23,10 @@ envoy_cc_library(
         "//include/envoy/runtime:runtime_interface",
         "//include/envoy/stats:stats_interface",
         "//include/envoy/stats:stats_macros",
+        "//source/common/buffer:watermark_buffer_lib",
         "//source/common/common:assert_lib",
         "//source/common/common:empty_string",
+        "//source/common/common:token_bucket_impl_lib",
         "//source/common/http:codes_lib",
         "//source/common/http:header_map_lib",
         "//source/common/http:header_utility_lib",

source/extensions/filters/http/fault/config.cc

@@ -15,10 +15,10 @@ namespace Fault {
 Http::FilterFactoryCb FaultFilterFactory::createFilterFactoryFromProtoTyped(
     const envoy::config::filter::http::fault::v2::HTTPFault& config,
     const std::string& stats_prefix, Server::Configuration::FactoryContext& context) {
-  FaultFilterConfigSharedPtr filter_config(
-      new FaultFilterConfig(config, context.runtime(), stats_prefix, context.scope()));
+  FaultFilterConfigSharedPtr filter_config(new FaultFilterConfig(
+      config, context.runtime(), stats_prefix, context.scope(), context.timeSource()));
   return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void {
-    callbacks.addStreamDecoderFilter(std::make_shared<FaultFilter>(filter_config));
+    callbacks.addStreamFilter(std::make_shared<FaultFilter>(filter_config));
   };
 }