Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

route match: Add runtime_fraction field for more granular routing #4217

Merged
merged 15 commits into from
Sep 19, 2018
2 changes: 2 additions & 0 deletions api/envoy/api/v2/route/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ api_proto_library_internal(
deps = [
"//envoy/api/v2/core:base",
"//envoy/type:range",
"//envoy/type:runtime_percent",
],
)

Expand All @@ -18,5 +19,6 @@ api_go_proto_library(
deps = [
"//envoy/api/v2/core:base_go_proto",
"//envoy/type:range_go_proto",
"//envoy/type:runtime_percent",
],
)
38 changes: 28 additions & 10 deletions api/envoy/api/v2/route/route.proto
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ option java_generic_services = true;

import "envoy/api/v2/core/base.proto";
import "envoy/type/range.proto";
import "envoy/type/runtime_percent.proto";

import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
Expand Down Expand Up @@ -274,16 +275,33 @@ message RouteMatch {
// is true.
google.protobuf.BoolValue case_sensitive = 4;

// Indicates that the route should additionally match on a runtime key. An
// integer between 0-100. Every time the route is considered for a match, a
// random number between 0-99 is selected. If the number is <= the value found
// in the key (checked first) or, if the key is not present, the default
// value, the route is a match (assuming everything also about the route
// matches). A runtime route configuration can be used to roll out route changes in a
// gradual manner without full code/config deploys. Refer to the
// :ref:`traffic shifting <config_http_conn_man_route_table_traffic_splitting_shift>` docs
// for additional documentation.
core.RuntimeUInt32 runtime = 5;
oneof runtime_specifier {
// Indicates that the route should additionally match on a runtime key. An integer between
// 0-100. Every time the route is considered for a match, a random number between 0-99 is
// selected. If the number is <= the value found in the key (checked first) or, if the key is
// not present, the default value, the route is a match (assuming everything also about the
// route matches). A runtime route configuration can be used to roll out route changes in a
// gradual manner without full code/config deploys. Refer to the :ref:`traffic shifting
// <config_http_conn_man_route_table_traffic_splitting_shift>` docs for additional
// documentation.
//
// .. attention::
//
// **This field is deprecated**. Set the
// :ref:`runtime_fraction<envoy_api_field_route.RouteMatch.runtime_fraction>` field instead.
core.RuntimeUInt32 runtime = 5 [deprecated = true];
tonya11en marked this conversation as resolved.
Show resolved Hide resolved

// Indicates that the route should additionally match on a runtime key. Every time the route
// is considered for a match, it must also fall under the percentage of matches indicated by
// this field. For some fraction N/D, a random number in the range [0,D) is selected. If the
// number is <= the value of the numberator N, or if the key is not present, the default
// value, the router continues to evaluate the remaining match criteria. A runtime_fraction
// route configuration can be used to roll out route changes in a gradual manner (with more
// granularity than the deprecated runtime field) without full code/config deploys. Refer to
// the :ref:`traffic shifting <config_http_conn_man_route_table_traffic_splitting_shift>` docs
// for additional documentation.
envoy.type.RuntimeFractionalPercent runtime_fraction = 8;
}

// Specifies a set of headers that the route should match on. The router will
// check the request’s headers against all the specified headers in the route
Expand Down
12 changes: 12 additions & 0 deletions api/envoy/type/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,18 @@ api_go_proto_library(
proto = ":http_status",
)

api_proto_library_internal(
name = "runtime_percent",
srcs = ["runtime_percent.proto"],
visibility = ["//visibility:public"],
deps = [":percent"],
)

api_go_proto_library(
name = "runtime_percent",
proto = ":runtime_percent",
)

api_proto_library_internal(
name = "percent",
srcs = ["percent.proto"],
Expand Down
23 changes: 23 additions & 0 deletions api/envoy/type/runtime_percent.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
syntax = "proto3";

package envoy.type;

import "validate/validate.proto";
import "gogoproto/gogo.proto";

import "envoy/type/percent.proto";

option (gogoproto.equal_all) = true;

// [#protodoc-title: RuntimePercent]

// Runtime derived FractionalPercent with defaults for when the numerator or denominator is not
// specified via a runtime key.
message RuntimeFractionalPercent {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be in api/envoy/api/v2/core/base.proto so as to be next to RuntimeUint32?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't do that because I didn't want to introduce a dependency on percent.proto. It was originally in percent.proto, but an earlier comment from @danielhochman suggested I move it out into its own file.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's fine for base to depend on percent.proto (just not the other way around).

// Default value if the runtime value's for the numerator/denominator keys are not available.
envoy.type.FractionalPercent default_value = 1 [(validate.rules).message.required = true];

// Runtime key for a textual representation of a :ref:`fractional percent
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice to see this approach. I actually think we should use the JSON/YAML encoding on second thoughts, since text proto is not a standard. If you go this route, we have some helpers to make it easy to parse, see https://github.com/envoyproxy/envoy/blob/master/source/common/protobuf/utility.h#L210. This will also validate the proto for you for constraints, which is missing in the parsing below.

// types<envoy_api_enum_type.FractionalPercent`.
string runtime_key = 2;
htuch marked this conversation as resolved.
Show resolved Hide resolved
}
1 change: 1 addition & 0 deletions docs/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ PROTO_RST="
/envoy/service/auth/v2alpha/external_auth/envoy/service/auth/v2alpha/external_auth.proto.rst
/envoy/type/http_status/envoy/type/http_status.proto.rst
/envoy/type/percent/envoy/type/percent.proto.rst
/envoy/type/runtime_percent/envoy/type/runtime_percent.proto.rst
/envoy/type/range/envoy/type/range.proto.rst
/envoy/type/matcher/metadata/envoy/type/matcher/metadata.proto.rst
/envoy/type/matcher/value/envoy/type/matcher/value.proto.rst
Expand Down
1 change: 1 addition & 0 deletions docs/root/api-v2/types/types.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ Types

../type/http_status.proto
../type/percent.proto
../type/runtime_percent.proto
../type/range.proto
../type/matcher/metadata.proto
../type/matcher/number.proto
Expand Down
7 changes: 4 additions & 3 deletions source/common/access_log/access_log_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,15 @@ bool RuntimeFilter::evaluate(const RequestInfo::RequestInfo&,
const Http::HeaderEntry* uuid = request_header.RequestId();
uint64_t random_value;
if (use_independent_randomness_ || uuid == nullptr ||
!UuidUtils::uuidModBy(uuid->value().c_str(), random_value,
ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent_))) {
!UuidUtils::uuidModBy(
uuid->value().c_str(), random_value,
ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent_.denominator()))) {
random_value = random_.random();
}

return runtime_.snapshot().featureEnabled(
runtime_key_, percent_.numerator(), random_value,
ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent_));
ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent_.denominator()));
}

OperatorFilter::OperatorFilter(const Protobuf::RepeatedPtrField<
Expand Down
5 changes: 3 additions & 2 deletions source/common/protobuf/utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ uint64_t convertPercent(double percent, uint64_t max_value) {
return max_value * (percent / 100.0);
}

uint64_t fractionalPercentDenominatorToInt(const envoy::type::FractionalPercent& percent) {
switch (percent.denominator()) {
uint64_t fractionalPercentDenominatorToInt(
const envoy::type::FractionalPercent::DenominatorType& denominator) {
switch (denominator) {
case envoy::type::FractionalPercent::HUNDRED:
return 100;
case envoy::type::FractionalPercent::TEN_THOUSAND:
Expand Down
5 changes: 3 additions & 2 deletions source/common/protobuf/utility.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,11 @@ uint64_t convertPercent(double percent, uint64_t max_value);

/**
* Convert a fractional percent denominator enum into an integer.
* @param percent supplies percent to convert.
* @param denominator supplies denominator to convert.
* @return the converted denominator.
*/
uint64_t fractionalPercentDenominatorToInt(const envoy::type::FractionalPercent& percent);
uint64_t fractionalPercentDenominatorToInt(
const envoy::type::FractionalPercent::DenominatorType& denominator);

} // namespace ProtobufPercentHelper
} // namespace Envoy
Expand Down
42 changes: 33 additions & 9 deletions source/common/router/config_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "common/common/empty_string.h"
#include "common/common/fmt.h"
#include "common/common/hash.h"
#include "common/common/logger.h"
#include "common/common/utility.h"
#include "common/config/metadata.h"
#include "common/config/rds_json.h"
Expand All @@ -26,6 +27,7 @@
#include "common/http/headers.h"
#include "common/http/utility.h"
#include "common/http/websocket/ws_handler_impl.h"
#include "common/protobuf/protobuf.h"
#include "common/protobuf/utility.h"
#include "common/router/retry_state_impl.h"

Expand Down Expand Up @@ -260,7 +262,7 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost,
timeout_(PROTOBUF_GET_MS_OR_DEFAULT(route.route(), timeout, DEFAULT_ROUTE_TIMEOUT_MS)),
idle_timeout_(PROTOBUF_GET_OPTIONAL_MS(route.route(), idle_timeout)),
max_grpc_timeout_(PROTOBUF_GET_OPTIONAL_MS(route.route(), max_grpc_timeout)),
runtime_(loadRuntimeData(route.match())), loader_(factory_context.runtime()),
loader_(factory_context.runtime()), runtime_(loadRuntimeData(route.match())),
host_redirect_(route.redirect().host_redirect()),
path_redirect_(route.redirect().path_redirect()),
https_redirect_(route.redirect().https_redirect()),
Expand Down Expand Up @@ -345,8 +347,11 @@ bool RouteEntryImplBase::matchRoute(const Http::HeaderMap& headers, uint64_t ran
bool matches = true;

if (runtime_) {
matches &= loader_.snapshot().featureEnabled(runtime_.value().key_, runtime_.value().default_,
random_value);
matches &= random_value % runtime_->denominator_val_ < runtime_->numerator_val_;
if (!matches) {
// No need to waste further cycles calculating a route match.
return false;
}
}

matches &= Http::HeaderUtility::matchHeaders(headers, config_headers_);
Expand Down Expand Up @@ -404,14 +409,33 @@ void RouteEntryImplBase::finalizeResponseHeaders(
absl::optional<RouteEntryImplBase::RuntimeData>
RouteEntryImplBase::loadRuntimeData(const envoy::api::v2::route::RouteMatch& route_match) {
absl::optional<RuntimeData> runtime;
if (route_match.has_runtime()) {
RuntimeData data;
data.key_ = route_match.runtime().runtime_key();
data.default_ = route_match.runtime().default_value();
runtime = data;
RuntimeData runtime_data;

if (route_match.runtime_specifier_case() == envoy::api::v2::route::RouteMatch::kRuntimeFraction) {
envoy::type::FractionalPercent fractional_percent;
const std::string& fraction_text =
loader_.snapshot().get(route_match.runtime_fraction().runtime_key());
if (!Protobuf::TextFormat::ParseFromString(fraction_text, &fractional_percent)) {
// Since we failed to parse the textual protobuf, let's load up the default value.
fractional_percent = route_match.runtime_fraction().default_value();
ENVOY_LOG(error, "failed to parse string value for runtime key {}",
route_match.runtime_fraction().runtime_key());
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I read this right, failure to convert the denominator key into a valid denominator value causes us to use the numerator key's value with the default value's denominator. That seems like it might cause some pretty unexpected behavior. Is it reasonable to just switch using the default value altogether in this case?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think what you propose is the sane thing to do. I'll change the logic.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually- scratch that. A user would have unexpected behavior either way if they misspell a denominator type and it switches to a default value. I'm not sure one way is better than the other. @htuch what do you think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure what the best practice to alert when runtime is wrong; how do folks operationally determine when a runtime change has taken effect? Via /runtime?

One thought that occurred to me just now is that instead of having separate values for numerator/denominator, we have a single values, which is a textual representation of a FractionalPercent proto. That way, we can just use PGV validation on this to check sanity. It also solves an update atomicity issue.

runtime_data.numerator_val_ = fractional_percent.numerator();
runtime_data.denominator_val_ =
ProtobufPercentHelper::fractionalPercentDenominatorToInt(fractional_percent.denominator());
} else if (route_match.runtime_specifier_case() == envoy::api::v2::route::RouteMatch::kRuntime) {
// For backwards compatibility, the deprecated 'runtime' field must be converted to a
// RuntimeData format with a variable denominator type. The 'runtime' field assumes a percentage
// (0-100), so the hard-coded denominator value reflects this.
runtime_data.denominator_val_ = 100;
runtime_data.numerator_val_ = loader_.snapshot().getInteger(
route_match.runtime().runtime_key(), route_match.runtime().default_value());
} else {
return runtime;
}

return runtime;
return runtime_data;
}

void RouteEntryImplBase::finalizePathHeader(Http::HeaderMap& headers,
Expand Down
12 changes: 6 additions & 6 deletions source/common/router/config_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,8 @@ class RouteEntryImplBase : public RouteEntry,
public DirectResponseEntry,
public Route,
public PathMatchCriterion,
public std::enable_shared_from_this<RouteEntryImplBase> {
public std::enable_shared_from_this<RouteEntryImplBase>,
Logger::Loggable<Logger::Id::router> {
public:
/**
* @throw EnvoyException with reason if the route configuration contains any errors
Expand Down Expand Up @@ -362,8 +363,8 @@ class RouteEntryImplBase : public RouteEntry,

private:
struct RuntimeData {
std::string key_{};
uint64_t default_{};
uint64_t numerator_val_{};
uint64_t denominator_val_{};
};

class DynamicRouteEntry : public RouteEntry, public Route {
Expand Down Expand Up @@ -494,8 +495,7 @@ class RouteEntryImplBase : public RouteEntry,

typedef std::shared_ptr<WeightedClusterEntry> WeightedClusterEntrySharedPtr;

static absl::optional<RuntimeData>
loadRuntimeData(const envoy::api::v2::route::RouteMatch& route);
absl::optional<RuntimeData> loadRuntimeData(const envoy::api::v2::route::RouteMatch& route);

static std::multimap<std::string, std::string>
parseOpaqueConfig(const envoy::api::v2::route::Route& route);
Expand All @@ -516,8 +516,8 @@ class RouteEntryImplBase : public RouteEntry,
const std::chrono::milliseconds timeout_;
const absl::optional<std::chrono::milliseconds> idle_timeout_;
const absl::optional<std::chrono::milliseconds> max_grpc_timeout_;
const absl::optional<RuntimeData> runtime_;
Runtime::Loader& loader_;
const absl::optional<RuntimeData> runtime_;
const std::string host_redirect_;
const std::string path_redirect_;
const bool https_redirect_;
Expand Down
10 changes: 6 additions & 4 deletions source/extensions/filters/http/fault/fault_filter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -134,13 +134,14 @@ bool FaultFilter::isDelayEnabled() {
bool enabled = config_->runtime().snapshot().featureEnabled(
DELAY_PERCENT_KEY, fault_settings_->delayPercentage().numerator(),
config_->randomGenerator().random(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(fault_settings_->delayPercentage()));
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_settings_->delayPercentage().denominator()));
if (!downstream_cluster_delay_percent_key_.empty()) {
enabled |= config_->runtime().snapshot().featureEnabled(
downstream_cluster_delay_percent_key_, fault_settings_->delayPercentage().numerator(),
config_->randomGenerator().random(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_settings_->delayPercentage()));
fault_settings_->delayPercentage().denominator()));
}
return enabled;
}
Expand All @@ -149,13 +150,14 @@ bool FaultFilter::isAbortEnabled() {
bool enabled = config_->runtime().snapshot().featureEnabled(
ABORT_PERCENT_KEY, fault_settings_->abortPercentage().numerator(),
config_->randomGenerator().random(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(fault_settings_->abortPercentage()));
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_settings_->abortPercentage().denominator()));
if (!downstream_cluster_abort_percent_key_.empty()) {
enabled |= config_->runtime().snapshot().featureEnabled(
downstream_cluster_abort_percent_key_, fault_settings_->abortPercentage().numerator(),
config_->randomGenerator().random(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_settings_->abortPercentage()));
fault_settings_->abortPercentage().denominator()));
}
return enabled;
}
Expand Down
2 changes: 1 addition & 1 deletion source/extensions/filters/network/mongo_proxy/proxy.cc
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ absl::optional<uint64_t> ProxyFilter::delayDuration() {
fault_config_->delayPercentage().numerator(),
generator_.random(),
ProtobufPercentHelper::fractionalPercentDenominatorToInt(
fault_config_->delayPercentage()))) {
fault_config_->delayPercentage().denominator()))) {
return result;
}

Expand Down
Loading