diff --git a/docs/configuration/cluster_manager/cluster_runtime.rst b/docs/configuration/cluster_manager/cluster_runtime.rst index 1f717c5d7af8..ab3d069caf63 100644 --- a/docs/configuration/cluster_manager/cluster_runtime.rst +++ b/docs/configuration/cluster_manager/cluster_runtime.rst @@ -31,3 +31,16 @@ upstream.use_http2 upstream.weight_enabled Binary switch to turn on or off weighted load balancing. If set to non 0, weighted load balancing is enabled. Defaults to enabled. + +upstream.zone_routing.enabled + % of requests that will be routed to the same upstream zone. Defaults to 100% of requests. + +upstream.zone_routing.percent_diff + Zone aware routing will be used only if the percent of upstream hosts in the same zone is within + percent_diff of expected. Expected is calculated as 100 / number_of_zones. This prevents Envoy + from using same zone routing if the zones are not balanced well. Defaults to 3% allowed deviation. + +upstream.zone_routing.healthy_panic_threshold + Defines the :ref:`zone healthy panic threshold ` + percentage. Defaults to 80%. If the % of healthy hosts in the current zone falls below this % + all healthy hosts will be used for routing. diff --git a/docs/intro/arch_overview/load_balancing.rst b/docs/intro/arch_overview/load_balancing.rst index ab3637574645..998a85b4697b 100644 --- a/docs/intro/arch_overview/load_balancing.rst +++ b/docs/intro/arch_overview/load_balancing.rst @@ -50,3 +50,13 @@ health status and balance amongst all hosts. This is known as the *panic thresho panic threshold is 50%. This is :ref:`configurable ` via runtime. The panic threshold is used to avoid a situation in which host failures cascade throughout the cluster as load increases. + +.. _arch_overview_load_balancing_zone_panic_threshold: + +Zone aware routing and local zone panic threshold +------------------------------------------------- + +By default Envoy performs zone aware routing where it will send traffic to the same upstream zone. +This is only done if the zones are well balanced (defaults to 3% allowed deviation) and if there +are enough healthy hosts in the local zone (the *panic threshold* which defaults to 80%). These are +:ref:`configurable ` via runtime. diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index f41c9ef84938..53ad7df1b5e0 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -160,7 +160,11 @@ class HostSet { COUNTER(update_attempt) \ COUNTER(update_success) \ COUNTER(update_failure) \ - GAUGE (max_host_weight) + GAUGE (max_host_weight) \ + GAUGE (upstream_zone_count) \ + COUNTER(upstream_zone_above_threshold) \ + COUNTER(upstream_zone_healthy_panic) \ + COUNTER(upstream_zone_within_threshold) // clang-format on /** diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 5c57136b630f..c03e3c0daf06 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -14,18 +14,49 @@ const std::vector& LoadBalancerBase::hostsToUse() { return host_set_.hosts(); } - uint64_t threshold = + uint64_t global_panic_threshold = std::min(100UL, runtime_.snapshot().getInteger("upstream.healthy_panic_threshold", 50)); - double healthy_percent = - (host_set_.healthyHosts().size() / static_cast(host_set_.hosts().size())) * 100; + double healthy_percent = 100.0 * host_set_.healthyHosts().size() / host_set_.hosts().size(); // If the % of healthy hosts in the cluster is less than our panic threshold, we use all hosts. - if (healthy_percent < threshold) { + if (healthy_percent < global_panic_threshold) { stats_.upstream_rq_lb_healthy_panic_.inc(); return host_set_.hosts(); - } else { + } + + // Early exit if we cannot perform zone aware routing. + if (stats_.upstream_zone_count_.value() < 2 || host_set_.localZoneHealthyHosts().empty() || + !runtime_.snapshot().featureEnabled("upstream.zone_routing.enabled", 100)) { return host_set_.healthyHosts(); } + + double zone_to_all_percent = + 100.0 * host_set_.localZoneHealthyHosts().size() / host_set_.healthyHosts().size(); + double expected_percent = 100.0 / stats_.upstream_zone_count_.value(); + + uint64_t zone_percent_diff = + runtime_.snapshot().getInteger("upstream.zone_routing.percent_diff", 3); + + // Hosts should be roughly equally distributed between zones. + if (std::abs(zone_to_all_percent - expected_percent) > zone_percent_diff) { + stats_.upstream_zone_above_threshold_.inc(); + + return host_set_.healthyHosts(); + } + + stats_.upstream_zone_within_threshold_.inc(); + + uint64_t zone_panic_threshold = + runtime_.snapshot().getInteger("upstream.zone_routing.healthy_panic_threshold", 80); + double zone_healthy_percent = + 100.0 * host_set_.localZoneHealthyHosts().size() / host_set_.localZoneHosts().size(); + if (zone_healthy_percent < zone_panic_threshold) { + stats_.upstream_zone_healthy_panic_.inc(); + + return host_set_.healthyHosts(); + } + + return host_set_.localZoneHealthyHosts(); } ConstHostPtr RoundRobinLoadBalancer::chooseHost() { diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 6a0e267672d1..549376349d49 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -177,6 +177,7 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const std::vector& n std::vector& hosts_removed, bool depend_on_hc) { uint64_t max_host_weight = 1; + std::unordered_set zones; // Go through and see if the list we have is different from what we just got. If it is, we // make a new host list and raise a change notification. This uses an N^2 search given that @@ -188,6 +189,7 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const std::vector& n // If we find a host matched based on URL, we keep it. However we do change weight inline so // do that here. if ((*i)->url() == host->url()) { + zones.insert((*i)->zone()); if (host->weight() > max_host_weight) { max_host_weight = host->weight(); } @@ -205,6 +207,7 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const std::vector& n if (host->weight() > max_host_weight) { max_host_weight = host->weight(); } + zones.insert(host->zone()); final_hosts.push_back(host); hosts_added.push_back(host); @@ -221,6 +224,7 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const std::vector& n if ((*i)->weight() > max_host_weight) { max_host_weight = (*i)->weight(); } + zones.insert((*i)->zone()); final_hosts.push_back(*i); i = current_hosts.erase(i); @@ -231,6 +235,7 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const std::vector& n } stats_.max_host_weight_.set(max_host_weight); + stats_.upstream_zone_count_.set(zones.size()); if (!hosts_added.empty() || !current_hosts.empty()) { hosts_removed = std::move(current_hosts); diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 94f132bde776..5f7ccdb1839a 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -66,6 +66,112 @@ TEST_F(RoundRobinLoadBalancerTest, MaxUnhealthyPanic) { EXPECT_EQ(3UL, stats_.upstream_rq_lb_healthy_panic_.value()); } +TEST_F(RoundRobinLoadBalancerTest, ZoneAwareRoutingDone) { + cluster_.healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:82")}; + cluster_.hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:82")}; + cluster_.local_zone_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81")}; + cluster_.local_zone_healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81")}; + stats_.upstream_zone_count_.set(3UL); + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.healthy_panic_threshold", 80)) + .WillRepeatedly(Return(80)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.percent_diff", 3)) + .WillRepeatedly(Return(2)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + + // There is only one host in the given zone for zone aware routing. + EXPECT_EQ(cluster_.local_zone_healthy_hosts_[0], lb_.chooseHost()); + EXPECT_EQ(1UL, stats_.upstream_zone_within_threshold_.value()); + + EXPECT_EQ(cluster_.local_zone_healthy_hosts_[0], lb_.chooseHost()); + EXPECT_EQ(2UL, stats_.upstream_zone_within_threshold_.value()); + + // Disable runtime global zone routing. + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(false)); + EXPECT_EQ(cluster_.healthy_hosts_[2], lb_.chooseHost()); + EXPECT_EQ(2UL, stats_.upstream_zone_within_threshold_.value()); +} + +TEST_F(RoundRobinLoadBalancerTest, NoZoneAwareRoutingOneZone) { + cluster_.healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80")}; + cluster_.hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80")}; + cluster_.local_zone_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80")}; + cluster_.local_zone_healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80")}; + stats_.upstream_zone_count_.set(1UL); + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)).Times(0); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.healthy_panic_threshold", 80)) + .Times(0); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.percent_diff", 3)).Times(0); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + + EXPECT_EQ(cluster_.healthy_hosts_[0], lb_.chooseHost()); + EXPECT_EQ(0UL, stats_.upstream_zone_within_threshold_.value()); + EXPECT_EQ(0UL, stats_.upstream_zone_above_threshold_.value()); +} + +TEST_F(RoundRobinLoadBalancerTest, ZoneAwareRoutingNotHealthy) { + cluster_.healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:82")}; + cluster_.hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:82")}; + cluster_.local_zone_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81")}; + cluster_.local_zone_healthy_hosts_ = {}; + stats_.upstream_zone_count_.set(3UL); + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + + // Should not be called due to early exit. + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.healthy_panic_threshold", 80)) + .Times(0); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.percent_diff", 3)).Times(0); + + // local zone has no healthy hosts, take from the all healthy hosts. + EXPECT_EQ(cluster_.healthy_hosts_[0], lb_.chooseHost()); + EXPECT_EQ(cluster_.healthy_hosts_[1], lb_.chooseHost()); +} + +TEST_F(RoundRobinLoadBalancerTest, ZoneAwareRoutingNotEnoughHealthy) { + cluster_.healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:82")}; + cluster_.hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:80"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81"), + newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:82")}; + cluster_.local_zone_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81")}; + cluster_.local_zone_healthy_hosts_ = {newTestHost(Upstream::MockCluster{}, "tcp://127.0.0.1:81")}; + stats_.upstream_zone_count_.set(2UL); + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.healthy_panic_threshold", 80)) + .WillRepeatedly(Return(80)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.percent_diff", 3)) + .WillRepeatedly(Return(3)); + + // Not enough healthy hosts in local zone. + EXPECT_EQ(cluster_.healthy_hosts_[0], lb_.chooseHost()); + EXPECT_EQ(1UL, stats_.upstream_zone_above_threshold_.value()); + EXPECT_EQ(cluster_.healthy_hosts_[1], lb_.chooseHost()); + EXPECT_EQ(2UL, stats_.upstream_zone_above_threshold_.value()); +} + class LeastRequestLoadBalancerTest : public testing::Test { public: LeastRequestLoadBalancerTest() : stats_(ClusterImplBase::generateStats("", stats_store_)) {} @@ -135,7 +241,7 @@ TEST_F(LeastRequestLoadBalancerTest, Normal) { EXPECT_EQ(cluster_.healthy_hosts_[1], lb_.chooseHost()); } -TEST_F(LeastRequestLoadBalancerTest, WeightImbalanceRuntimOff) { +TEST_F(LeastRequestLoadBalancerTest, WeightImbalanceRuntimeOff) { // Disable weight balancing. EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.weight_enabled", 1)) .WillRepeatedly(Return(0)); diff --git a/test/common/upstream/sds_test.cc b/test/common/upstream/sds_test.cc index ef287a8cffcb..8fe831ca096f 100644 --- a/test/common/upstream/sds_test.cc +++ b/test/common/upstream/sds_test.cc @@ -130,6 +130,7 @@ TEST_F(SdsTest, NoHealthChecker) { EXPECT_EQ("us-east-1d", canary_host->zone()); EXPECT_EQ(1U, canary_host->weight()); EXPECT_EQ(1UL, cluster_->stats().max_host_weight_.value()); + EXPECT_EQ(3UL, cluster_->stats().upstream_zone_count_.value()); // Test response with weight change. We should still have the same host. setupRequest(); @@ -147,6 +148,7 @@ TEST_F(SdsTest, NoHealthChecker) { EXPECT_EQ("us-east-1d", canary_host->zone()); EXPECT_EQ(50U, canary_host->weight()); EXPECT_EQ(50UL, cluster_->stats().max_host_weight_.value()); + EXPECT_EQ(3UL, cluster_->stats().upstream_zone_count_.value()); // Now test the failure case, our cluster size should not change. setupRequest(); @@ -157,6 +159,7 @@ TEST_F(SdsTest, NoHealthChecker) { EXPECT_EQ(13UL, cluster_->hosts().size()); EXPECT_EQ(50U, canary_host->weight()); EXPECT_EQ(50UL, cluster_->stats().max_host_weight_.value()); + EXPECT_EQ(3UL, cluster_->stats().upstream_zone_count_.value()); // 503 response. setupRequest(); @@ -169,6 +172,7 @@ TEST_F(SdsTest, NoHealthChecker) { EXPECT_EQ(13UL, cluster_->hosts().size()); EXPECT_EQ(50U, canary_host->weight()); EXPECT_EQ(50UL, cluster_->stats().max_host_weight_.value()); + EXPECT_EQ(3UL, cluster_->stats().upstream_zone_count_.value()); } TEST_F(SdsTest, HealthChecker) { diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index bf0ff5b2a440..5396113decb6 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -25,6 +25,8 @@ MockCluster::MockCluster() ON_CALL(*this, connectTimeout()).WillByDefault(Return(std::chrono::milliseconds(1))); ON_CALL(*this, hosts()).WillByDefault(ReturnRef(hosts_)); ON_CALL(*this, healthyHosts()).WillByDefault(ReturnRef(healthy_hosts_)); + ON_CALL(*this, localZoneHosts()).WillByDefault(ReturnRef(local_zone_hosts_)); + ON_CALL(*this, localZoneHealthyHosts()).WillByDefault(ReturnRef(local_zone_healthy_hosts_)); ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, altStatName()).WillByDefault(ReturnRef(alt_stat_name_)); ON_CALL(*this, lbType()).WillByDefault(Return(Upstream::LoadBalancerType::RoundRobin)); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 98d6f4e81072..62cb1ac2525f 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -47,6 +47,8 @@ class MockCluster : public Cluster { std::vector hosts_; std::vector healthy_hosts_; + std::vector local_zone_hosts_; + std::vector local_zone_healthy_hosts_; std::string name_{"fake_cluster"}; std::string alt_stat_name_{"fake_alt_cluster"}; std::list callbacks_;