From 8b5f6e861cfc5b6345e6887f88120c918802aad9 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Thu, 3 Oct 2024 11:43:05 -0500 Subject: [PATCH 01/77] mobile: Fix broken link (#36436) Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- .../ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt index 354ef87073..6cfc8ce170 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt @@ -53,7 +53,7 @@ class ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest { HttpTestServerFactory.start( HttpTestServerFactory.Type.HTTP1_WITHOUT_TLS, 0, - // http://go/mdn/HTTP/Proxy_servers_and_tunneling/Proxy_Auto-Configuration_PAC_file + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Proxy_servers_and_tunneling/Proxy_Auto-Configuration_PAC_file mapOf("Content-Type" to "application/x-ns-proxy-autoconfig"), """ function FindProxyForURL(url, host) { From 37b725d94380543b0c44a721dcfda5666539308f Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:33:52 -0400 Subject: [PATCH 02/77] Handle encode metadata after recreated stream (#36427) Commit Message: Handle encode metadata after recreated stream Risk Level: Low Testing: Integration test Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A --------- Signed-off-by: tyxia --- source/common/http/filter_manager.cc | 10 +++ source/common/http/filter_manager.h | 2 + test/integration/BUILD | 3 + test/integration/filter_integration_test.cc | 80 +++++++++++++++++++ test/integration/filters/BUILD | 33 ++++++++ .../filters/add_encode_metadata_filter.cc | 41 ++++++++++ .../filters/encoder_recreate_stream_filter.cc | 55 +++++++++++++ 7 files changed, 224 insertions(+) create mode 100644 test/integration/filters/add_encode_metadata_filter.cc create mode 100644 test/integration/filters/encoder_recreate_stream_filter.cc diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index 35066ddb88..89c58323ef 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -1696,6 +1696,10 @@ bool ActiveStreamDecoderFilter::recreateStream(const ResponseHeaderMap* headers) return false; } + parent_.state_.decoder_filter_chain_aborted_ = true; + parent_.state_.encoder_filter_chain_aborted_ = true; + parent_.state_.recreated_stream_ = true; + parent_.streamInfo().setResponseCodeDetails( StreamInfo::ResponseCodeDetails::get().InternalRedirect); @@ -1762,6 +1766,12 @@ void ActiveStreamEncoderFilter::drainSavedResponseMetadata() { } void ActiveStreamEncoderFilter::handleMetadataAfterHeadersCallback() { + if (parent_.state_.recreated_stream_) { + // The stream has been recreated. In this case, there's no reason to encode saved metadata. + getSavedResponseMetadata()->clear(); + return; + } + // If we drain accumulated metadata, the iteration must start with the current filter. const bool saved_state = iterate_from_current_filter_; iterate_from_current_filter_ = true; diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 1798106f86..94500f13d1 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -894,6 +894,8 @@ class FilterManager : public ScopeTrackedObject, bool decoder_filter_chain_aborted_{}; bool encoder_filter_chain_aborted_{}; bool saw_downstream_reset_{}; + // True when the stream was recreated. + bool recreated_stream_{}; // The following 3 members are booleans rather than part of the space-saving bitfield as they // are passed as arguments to functions expecting bools. Extend State using the bitfield diff --git a/test/integration/BUILD b/test/integration/BUILD index 58a33e4a52..374b9f7b3f 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -798,13 +798,16 @@ envoy_cc_test( "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", "//test/integration/filters:add_body_filter_config_lib", + "//test/integration/filters:add_encode_metadata_filter_lib", "//test/integration/filters:add_invalid_data_filter_lib", "//test/integration/filters:assert_non_reentrant_filter_lib", "//test/integration/filters:buffer_continue_filter_lib", "//test/integration/filters:continue_after_local_reply_filter_lib", "//test/integration/filters:continue_headers_only_inject_body", "//test/integration/filters:encoder_decoder_buffer_filter_lib", + "//test/integration/filters:encoder_recreate_stream_filter_lib", "//test/integration/filters:invalid_header_filter_lib", + "//test/integration/filters:local_reply_during_decoding_filter_lib", "//test/integration/filters:local_reply_during_encoding_data_filter_lib", "//test/integration/filters:local_reply_during_encoding_filter_lib", "//test/integration/filters:local_reply_with_metadata_filter_lib", diff --git a/test/integration/filter_integration_test.cc b/test/integration/filter_integration_test.cc index e83a9d2311..5028b5eca5 100644 --- a/test/integration/filter_integration_test.cc +++ b/test/integration/filter_integration_test.cc @@ -1575,5 +1575,85 @@ TEST_P(FilterIntegrationTest, FilterAddsDataToHeaderOnlyRequestWithIndependentHa testFilterAddsDataAndTrailersToHeaderOnlyRequest(); } +// Add metadata in the first filter before recreate the stream in the second filter, +// on response path. +TEST_P(FilterIntegrationTest, RecreateStreamAfterEncodeMetadata) { + // recreateStream is not supported in Upstream filter chain. + if (!testing_downstream_filter_) { + return; + } + + prependFilter("{ name: add-metadata-encode-headers-filter }"); + prependFilter("{ name: encoder-recreate-stream-filter }"); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, true); + + // Second upstream request is triggered by recreateStream. + FakeStreamPtr upstream_request_2; + // Wait for the next stream on the upstream connection. + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_2)); + // Wait for the stream to be completely received. + ASSERT_TRUE(upstream_request_2->waitForEndStream(*dispatcher_)); + upstream_request_2->encodeHeaders(default_response_headers_, true); + + // Wait for the response to be completely received. + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + + // Verify the metadata is received. + std::set expected_metadata_keys = {"headers", "duplicate"}; + EXPECT_EQ(response->metadataMap().size(), expected_metadata_keys.size()); + for (const auto& key : expected_metadata_keys) { + // keys are the same as their corresponding values. + auto it = response->metadataMap().find(key); + ASSERT_FALSE(it == response->metadataMap().end()) << "key: " << key; + EXPECT_EQ(response->metadataMap().find(key)->second, key); + } +} + +// Add metadata in the first filter on local reply path. +TEST_P(FilterIntegrationTest, EncodeMetadataOnLocalReply) { + // Local replies are not seen by upstream HTTP filters. add-metadata-encode-headers-filter will + // not be invoked if it is installed in upstream filter chain. + // Thus, this test is only applicable to downstream filter chain. + if (!testing_downstream_filter_) { + return; + } + + prependFilter("{ name: local-reply-during-decode }"); + prependFilter("{ name: add-metadata-encode-headers-filter }"); + + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("500", response->headers().getStatusValue()); + + // Verify the metadata is received. + std::set expected_metadata_keys = {"headers", "duplicate"}; + EXPECT_EQ(response->metadataMap().size(), expected_metadata_keys.size()); + for (const auto& key : expected_metadata_keys) { + // keys are the same as their corresponding values. + auto it = response->metadataMap().find(key); + ASSERT_FALSE(it == response->metadataMap().end()) << "key: " << key; + EXPECT_EQ(response->metadataMap().find(key)->second, key); + } +} + } // namespace } // namespace Envoy diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index 35e062867d..6dd174614b 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -1013,3 +1013,36 @@ envoy_cc_test_library( "//test/extensions/filters/http/common:empty_http_filter_config_lib", ], ) + +envoy_cc_test_library( + name = "add_encode_metadata_filter_lib", + srcs = [ + "add_encode_metadata_filter.cc", + ], + deps = [ + ":common_lib", + "//envoy/event:timer_interface", + "//envoy/http:filter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) + +envoy_cc_test_library( + name = "encoder_recreate_stream_filter_lib", + srcs = [ + "encoder_recreate_stream_filter.cc", + ], + deps = [ + ":common_lib", + "//envoy/event:timer_interface", + "//envoy/http:filter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/common/router:string_accessor_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) diff --git a/test/integration/filters/add_encode_metadata_filter.cc b/test/integration/filters/add_encode_metadata_filter.cc new file mode 100644 index 0000000000..5f56ac39b0 --- /dev/null +++ b/test/integration/filters/add_encode_metadata_filter.cc @@ -0,0 +1,41 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +// A filter add encoded metadata in encodeHeaders. +class AddEncodeMetadataFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "add-metadata-encode-headers-filter"; + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override { + Http::MetadataMap metadata_map = {{"headers", "headers"}, {"duplicate", "duplicate"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); + return Http::FilterHeadersStatus::Continue; + } + + Http::FilterDataStatus encodeData(Buffer::Instance&, bool) override { + return Http::FilterDataStatus::Continue; + } +}; + +constexpr char AddEncodeMetadataFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy diff --git a/test/integration/filters/encoder_recreate_stream_filter.cc b/test/integration/filters/encoder_recreate_stream_filter.cc new file mode 100644 index 0000000000..ebcbb4e40e --- /dev/null +++ b/test/integration/filters/encoder_recreate_stream_filter.cc @@ -0,0 +1,55 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/router/string_accessor_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +class EncoderRecreateStreamFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "encoder-recreate-stream-filter"; + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override { + const auto* filter_state = + decoder_callbacks_->streamInfo().filterState()->getDataReadOnly( + "test_key"); + + if (filter_state != nullptr) { + return ::Envoy::Http::FilterHeadersStatus::Continue; + } + + decoder_callbacks_->streamInfo().filterState()->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::Request); + + if (decoder_callbacks_->recreateStream(nullptr)) { + return ::Envoy::Http::FilterHeadersStatus::StopIteration; + } + + return ::Envoy::Http::FilterHeadersStatus::Continue; + } + + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { + decoder_callbacks_ = &callbacks; + } +}; + +// perform static registration +constexpr char EncoderRecreateStreamFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy From 24fe164d02a9c391c8e4678a5a9d7a98912a64a5 Mon Sep 17 00:00:00 2001 From: code Date: Fri, 4 Oct 2024 02:40:33 +0800 Subject: [PATCH 03/77] refactoring: refactored the FilterState object field support (#36399) The filter state reflection provides a great feature to access the inner status/property of filter state. However, it has two limitations: 1. It requires the object key be same with the factory key. This limitation make we cannot set multiple objects that with same type. 2. It is a little complex to enable the Field support. We need to define additional reflection class and a factory class. This PR make things much simpler. Risk Level: low. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. --------- Signed-off-by: wangbaiping --- envoy/stream_info/filter_state.h | 26 +++++--------- .../common/formatter/stream_info_formatter.cc | 19 ++-------- .../common/formatter/stream_info_formatter.h | 1 - .../network/filter_state_dst_address.cc | 36 ++++++------------- .../common/network/filter_state_dst_address.h | 6 ++-- .../extensions/filters/common/expr/context.cc | 25 +++++-------- .../formatter/substitution_formatter_test.cc | 30 ++-------------- .../original_dst/original_dst_cluster_test.cc | 6 ++-- .../filters/common/expr/context_test.cc | 17 +++++++++ 9 files changed, 56 insertions(+), 110 deletions(-) diff --git a/envoy/stream_info/filter_state.h b/envoy/stream_info/filter_state.h index 7e7cfd7fde..f43ffe00eb 100644 --- a/envoy/stream_info/filter_state.h +++ b/envoy/stream_info/filter_state.h @@ -87,6 +87,8 @@ class FilterState { class Object { public: + using FieldType = absl::variant; + virtual ~Object() = default; /** @@ -102,21 +104,17 @@ class FilterState { * This method can be used to get an unstructured serialization result. */ virtual absl::optional serializeAsString() const { return absl::nullopt; } - }; - - /** - * Generic reflection support for the filter state objects. - */ - class ObjectReflection { - public: - virtual ~ObjectReflection() = default; - using FieldType = absl::variant; + /** + * @return bool true if the object supports field access. False if the object does not support + * field access. Default implementation returns false. + */ + virtual bool hasFieldSupport() const { return false; } /** - * @return FieldType a field value for a field name. + * @return FieldType a single state property or field value for a name. */ - virtual FieldType getField(absl::string_view) const PURE; + virtual FieldType getField(absl::string_view) const { return absl::monostate{}; } }; /** @@ -134,12 +132,6 @@ class FilterState { * is malformed. */ virtual std::unique_ptr createFromBytes(absl::string_view data) const PURE; - - /** - * @return std::unique_ptr for the runtime object - * Note that the reflection object is a view and should not outlive the object. - */ - virtual std::unique_ptr reflect(const Object*) const { return nullptr; } }; struct FilterObject { diff --git a/source/common/formatter/stream_info_formatter.cc b/source/common/formatter/stream_info_formatter.cc index c7f532db00..5ff679a731 100644 --- a/source/common/formatter/stream_info_formatter.cc +++ b/source/common/formatter/stream_info_formatter.cc @@ -192,7 +192,6 @@ FilterStateFormatter::FilterStateFormatter(absl::string_view key, absl::optional if (!field_name.empty()) { format_ = FilterStateFormat::Field; field_name_ = std::string(field_name); - factory_ = Registry::FactoryRegistry::getFactory(key); } else if (serialize_as_string) { format_ = FilterStateFormat::String; } else { @@ -264,14 +263,7 @@ FilterStateFormatter::format(const StreamInfo::StreamInfo& stream_info) const { #endif } case FilterStateFormat::Field: { - if (!factory_) { - return absl::nullopt; - } - const auto reflection = factory_->reflect(state); - if (!reflection) { - return absl::nullopt; - } - auto field_value = reflection->getField(field_name_); + auto field_value = state->getField(field_name_); auto string_value = absl::visit(StringFieldVisitor(), field_value); if (!string_value) { return absl::nullopt; @@ -315,14 +307,7 @@ FilterStateFormatter::formatValue(const StreamInfo::StreamInfo& stream_info) con return SubstitutionFormatUtils::unspecifiedValue(); } case FilterStateFormat::Field: { - if (!factory_) { - return SubstitutionFormatUtils::unspecifiedValue(); - } - const auto reflection = factory_->reflect(state); - if (!reflection) { - return SubstitutionFormatUtils::unspecifiedValue(); - } - auto field_value = reflection->getField(field_name_); + auto field_value = state->getField(field_name_); auto string_value = absl::visit(StringFieldVisitor(), field_value); if (!string_value) { return SubstitutionFormatUtils::unspecifiedValue(); diff --git a/source/common/formatter/stream_info_formatter.h b/source/common/formatter/stream_info_formatter.h index 0cc80e0d91..10f4288c9d 100644 --- a/source/common/formatter/stream_info_formatter.h +++ b/source/common/formatter/stream_info_formatter.h @@ -110,7 +110,6 @@ class FilterStateFormatter : public StreamInfoFormatterProvider { const bool is_upstream_; FilterStateFormat format_; std::string field_name_; - StreamInfo::FilterState::ObjectFactory* factory_; }; class CommonDurationFormatter : public StreamInfoFormatterProvider { diff --git a/source/common/network/filter_state_dst_address.cc b/source/common/network/filter_state_dst_address.cc index b164573e5f..231f3e0b1c 100644 --- a/source/common/network/filter_state_dst_address.cc +++ b/source/common/network/filter_state_dst_address.cc @@ -9,39 +9,25 @@ absl::optional AddressObject::hash() const { return HashUtil::xxHash64(address_->asStringView()); } -class AddressObjectReflection : public StreamInfo::FilterState::ObjectReflection { -public: - AddressObjectReflection(const AddressObject* object) : object_(object) {} - FieldType getField(absl::string_view field_name) const override { - const auto* ip = object_->address_->ip(); - if (!ip) { - return {}; - } - if (field_name == "ip") { - return ip->addressAsString(); - } else if (field_name == "port") { - return int64_t(ip->port()); - } +StreamInfo::FilterState::Object::FieldType +AddressObject::getField(absl::string_view field_name) const { + const auto* ip = address_->ip(); + if (!ip) { return {}; } - -private: - const AddressObject* object_; -}; + if (field_name == "ip") { + return ip->addressAsString(); + } else if (field_name == "port") { + return int64_t(ip->port()); + } + return {}; +} std::unique_ptr BaseAddressObjectFactory::createFromBytes(absl::string_view data) const { const auto address = Utility::parseInternetAddressAndPortNoThrow(std::string(data)); return address ? std::make_unique(address) : nullptr; } -std::unique_ptr -BaseAddressObjectFactory::reflect(const StreamInfo::FilterState::Object* data) const { - const auto* object = dynamic_cast(data); - if (object) { - return std::make_unique(object); - } - return nullptr; -} } // namespace Network } // namespace Envoy diff --git a/source/common/network/filter_state_dst_address.h b/source/common/network/filter_state_dst_address.h index ec6c565fd6..b35cb25b9f 100644 --- a/source/common/network/filter_state_dst_address.h +++ b/source/common/network/filter_state_dst_address.h @@ -17,13 +17,15 @@ class AddressObject : public StreamInfo::FilterState::Object, public Hashable { absl::optional serializeAsString() const override { return address_ ? absl::make_optional(address_->asString()) : absl::nullopt; } + bool hasFieldSupport() const override { return true; } + FieldType getField(absl::string_view field_name) const override; + // Implements hashing interface because the value is applied once per upstream connection. // Multiple streams sharing the upstream connection must have the same address object. absl::optional hash() const override; private: const Network::Address::InstanceConstSharedPtr address_; - friend class AddressObjectReflection; }; /** @@ -33,8 +35,6 @@ class BaseAddressObjectFactory : public StreamInfo::FilterState::ObjectFactory { public: std::unique_ptr createFromBytes(absl::string_view data) const override; - std::unique_ptr - reflect(const StreamInfo::FilterState::Object* data) const override; }; } // namespace Network diff --git a/source/extensions/filters/common/expr/context.cc b/source/extensions/filters/common/expr/context.cc index 10438dff4d..a13a39656a 100644 --- a/source/extensions/filters/common/expr/context.cc +++ b/source/extensions/filters/common/expr/context.cc @@ -300,13 +300,12 @@ absl::optional PeerWrapper::operator[](CelValue key) const { class FilterStateObjectWrapper : public google::api::expr::runtime::CelMap { public: - FilterStateObjectWrapper(const StreamInfo::FilterState::ObjectReflection* reflection) - : reflection_(reflection) {} + FilterStateObjectWrapper(const StreamInfo::FilterState::Object* object) : object_(object) {} absl::optional operator[](CelValue key) const override { - if (reflection_ == nullptr || !key.IsString()) { + if (object_ == nullptr || !key.IsString()) { return {}; } - auto field_value = reflection_->getField(key.StringOrDie().value()); + auto field_value = object_->getField(key.StringOrDie().value()); return absl::visit(Visitor{}, field_value); } // Default stubs. @@ -325,7 +324,7 @@ class FilterStateObjectWrapper : public google::api::expr::runtime::CelMap { } absl::optional operator()(absl::monostate) { return {}; } }; - const StreamInfo::FilterState::ObjectReflection* reflection_; + const StreamInfo::FilterState::Object* object_; }; absl::optional FilterStateWrapper::operator[](CelValue key) const { @@ -339,17 +338,11 @@ absl::optional FilterStateWrapper::operator[](CelValue key) const { if (cel_state) { return cel_state->exprValue(&arena_, false); } else if (object != nullptr) { - // Attempt to find the reflection object. - auto factory = - Registry::FactoryRegistry::getFactory(value); - if (factory) { - auto reflection = factory->reflect(object); - if (reflection) { - auto* raw_reflection = reflection.release(); - arena_.Own(raw_reflection); - return CelValue::CreateMap( - ProtobufWkt::Arena::Create(&arena_, raw_reflection)); - } + // TODO(wbpcode): the implementation of cannot handle the case where the object has provided + // field support, but callers only want to access the whole object. + if (object->hasFieldSupport()) { + return CelValue::CreateMap( + ProtobufWkt::Arena::Create(&arena_, object)); } absl::optional serialized = object->serializeAsString(); if (serialized.has_value()) { diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 41b3e2fcba..11bfd2135c 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -123,18 +123,10 @@ class TestSerializedStringFilterState : public StreamInfo::FilterState::Object { message->set_value(raw_string_ + " By TYPED"); return message; } - -private: - std::string raw_string_; - friend class TestSerializedStringReflection; -}; - -class TestSerializedStringReflection : public StreamInfo::FilterState::ObjectReflection { -public: - TestSerializedStringReflection(const TestSerializedStringFilterState* data) : data_(data) {} + bool hasFieldSupport() const override { return true; } FieldType getField(absl::string_view field_name) const override { if (field_name == "test_field") { - return data_->raw_string_; + return raw_string_; } else if (field_name == "test_num") { return 137; } @@ -142,25 +134,9 @@ class TestSerializedStringReflection : public StreamInfo::FilterState::ObjectRef } private: - const TestSerializedStringFilterState* data_; -}; - -class TestSerializedStringFilterStateFactory : public StreamInfo::FilterState::ObjectFactory { -public: - std::string name() const override { return "test_key"; } - std::unique_ptr - createFromBytes(absl::string_view) const override { - return nullptr; - } - std::unique_ptr - reflect(const StreamInfo::FilterState::Object* data) const override { - return std::make_unique( - dynamic_cast(data)); - } + std::string raw_string_; }; -REGISTER_FACTORY(TestSerializedStringFilterStateFactory, StreamInfo::FilterState::ObjectFactory); - // Test tests multiple versions of variadic template method parseSubcommand // extracting tokens. TEST(SubstitutionFormatParser, commandParser) { diff --git a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc index d0e1cd8e82..ba0777efa7 100644 --- a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc +++ b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc @@ -1163,10 +1163,8 @@ TEST(DestinationAddress, ObjectFactory) { auto object = factory->createFromBytes(address); ASSERT_NE(nullptr, object); EXPECT_EQ(address, object->serializeAsString()); - auto mirror = factory->reflect(object.get()); - ASSERT_NE(nullptr, mirror); - EXPECT_THAT(mirror->getField("ip"), testing::VariantWith("10.0.0.10")); - EXPECT_THAT(mirror->getField("port"), testing::VariantWith(8080)); + EXPECT_THAT(object->getField("ip"), testing::VariantWith("10.0.0.10")); + EXPECT_THAT(object->getField("port"), testing::VariantWith(8080)); EXPECT_EQ(nullptr, factory->createFromBytes("foo")); } diff --git a/test/extensions/filters/common/expr/context_test.cc b/test/extensions/filters/common/expr/context_test.cc index c766d47e25..f7cec4fb53 100644 --- a/test/extensions/filters/common/expr/context_test.cc +++ b/test/extensions/filters/common/expr/context_test.cc @@ -777,6 +777,8 @@ TEST(Context, FilterStateAttributes) { StreamInfo::FilterStateImpl filter_state(StreamInfo::FilterState::LifeSpan::FilterChain); ProtobufWkt::Arena arena; FilterStateWrapper wrapper(arena, filter_state); + auto status_or = wrapper.ListKeys(&arena); + EXPECT_EQ(status_or.status().message(), "ListKeys() is not implemented"); const std::string key = "filter_state_key"; const std::string serialized = "filter_state_value"; @@ -941,6 +943,21 @@ TEST(Context, XDSAttributes) { } } +TEST(Context, EmptyXdsWrapper) { + Protobuf::Arena arena; + XDSWrapper wrapper(arena, nullptr, nullptr); + + { + const auto value = wrapper[CelValue::CreateStringView(Node)]; + EXPECT_FALSE(value.has_value()); + } + + { + const auto value = wrapper[CelValue::CreateStringView(ClusterName)]; + EXPECT_FALSE(value.has_value()); + } +} + } // namespace } // namespace Expr } // namespace Common From 505f125d821a25ba4f906c67acd20b68aa6f7096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Thu, 3 Oct 2024 16:21:20 -0400 Subject: [PATCH 04/77] gcc: add a `--host_linkopt` to use `gold` too (#36438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this, when building in the envoy docker without RBE, I see gcc trying to use `lld` instead, and it's (a) not in `$PATH` and (b) counter to what we had set in `--linkopt` for the gcc config. Risk Level: low Testing: local build in envoy docker Signed-off-by: Alejandro R. Sedeño --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index c01fb9a246..637ac2a593 100644 --- a/.bazelrc +++ b/.bazelrc @@ -91,7 +91,7 @@ build:clang-pch --spawn_strategy=local build:clang-pch --define=ENVOY_CLANG_PCH=1 # Use gold linker for gcc compiler. -build:gcc --linkopt=-fuse-ld=gold +build:gcc --linkopt=-fuse-ld=gold --host_linkopt=-fuse-ld=gold build:gcc --test_env=HEAPCHECK= build:gcc --action_env=BAZEL_COMPILER=gcc build:gcc --action_env=CC=gcc --action_env=CXX=g++ From 847e2a798625379b909cf7eaad82460e6d65d7a1 Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 4 Oct 2024 02:09:30 -0400 Subject: [PATCH 05/77] Update QUICHE from 171f6f89a to eaeaa74b2 (#36440) Update QUICHE from 171f6f89a to eaeaa74b2 https://github.com/google/quiche/compare/171f6f89a..eaeaa74b2 ``` $ git log 171f6f89a..eaeaa74b2 --date=short --no-merges --format="%ad %al %s" 2024-10-02 wub Deprecate --gfe2_reloadable_flag_quic_new_error_code_when_packets_buffered_too_long. 2024-10-02 fayang No public description 2024-10-02 fayang No public description 2024-10-01 birenroy Adds test cases exercising response-complete-before-request for nghttp2 and oghttp2. 2024-10-01 martinduke Add parser/framer support for SUBSCRIBE_NAMESPACE, SUBSCRIBE_NAMESPACE_OK, SUBSCRIBE_NAMESPACE_ERROR, UNSUBSCRIBE_NAMESPACE. 2024-10-01 birenroy Removes the last library in //third_party/spdy/core, and deletes the package. 2024-10-01 wub Add QUIC connection options for testing: - CHP1: Add 1-packet padding to CHLO. - CHP2: Add 2-packet padding to CHLO. 2024-09-30 asedeno Don't set IPv4 socket options on dual-stack sockets on `__APPLE__` platforms. 2024-09-30 vasilvv Update WebTransport header names. 2024-09-30 vasilvv Simplify some of the framing code. 2024-09-30 vasilvv Record QUIC traces in moqt_simulator. 2024-09-30 martinduke Implement MoQT Peeps and Object message changes for draft-06. This is the minimum for interoperability; this code always sends subgroup_id = 0 and ignores the incoming subgroup_id. 2024-09-30 martinduke Update existing messages for draft-06. This is mostly turning track_namespace into a tuple. Also generalizes Subscribe parameters. ``` Risk Level: low, Testing: existing tests passed Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A --------- Signed-off-by: Dan Zhang Co-authored-by: Dan Zhang --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b5233988c6..503403a344 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -153,7 +153,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # # !!! NOTE !!! # Anytime the FIPS BoringSSL version is upgraded, `bazel/external/boringssl_fips.genrule_cmd` must be updated to use the toolchain - # specified in the associated accredidation certificate, which can be found linked from + # specified in the associated accreditation certificate, which can be found linked from # https://boringssl.googlesource.com/boringssl/+/refs/heads/master/crypto/fipsmodule/FIPS.md, for example # https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4735. version = "fips-20220613", @@ -1208,12 +1208,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "171f6f89a6a119e8763f1216f8d85347f997cd3b", - sha256 = "3e0fec32dfa9c7568d4703516ee14c9e2316379e0a35f723d17a988be178e532", + version = "eaeaa74b2b4bf4cd9f7a2f44ba8f323fdc55f66a", + sha256 = "1383267a64cb18fca62868e7b54118c223e164d9c0533b11a9a31c779c626f95", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2024-09-26", + release_date = "2024-10-02", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 42068a59e37d0d6c313340d0175cb08cce2574bb Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 4 Oct 2024 11:34:47 -0700 Subject: [PATCH 06/77] xds: make certificate provider instance name required (#36441) Commit Message: xds: make certificate provider instance name required Additional Description: The comment saying that there is a default value for this field is misleading. All existing gRPC implementations have this as a required field. Note that this change does not affect Envoy, since Envoy does not yet support this field. However, I've added a PGV annotation (used by Envoy but not by gRPC) to avoid confusion when Envoy eventually adds support for this field. Risk Level: Low Testing: N/A Docs Changes: Included in PR Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Mark D. Roth --- api/envoy/extensions/transport_sockets/tls/v3/common.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/transport_sockets/tls/v3/common.proto b/api/envoy/extensions/transport_sockets/tls/v3/common.proto index c1a3f5b33b..46d86b6585 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -290,12 +290,12 @@ message TlsSessionTicketKeys { // respect to the TLS handshake. // [#not-implemented-hide:] message CertificateProviderPluginInstance { - // Provider instance name. If not present, defaults to "default". + // Provider instance name. // // Instance names should generally be defined not in terms of the underlying provider // implementation (e.g., "file_watcher") but rather in terms of the function of the // certificates (e.g., "foo_deployment_identity"). - string instance_name = 1; + string instance_name = 1 [(validate.rules).string = {min_len: 1}]; // Opaque name used to specify certificate instances or types. For example, "ROOTCA" to specify // a root-certificate (validation context) or "example.com" to specify a certificate for a From 1173629e531abf758f011c2da15da739f72881c6 Mon Sep 17 00:00:00 2001 From: Greg Greenway Date: Fri, 4 Oct 2024 15:53:11 -0700 Subject: [PATCH 07/77] http: allow runtime override of default for max response headers kb (#36439) Also, update docs and tests for similar runtime overrides that already existed This is a followup to #36231 Risk Level: Low Testing: New tests, plus more tests for existing untested code Docs Changes: Updated proto docs, including adding docs for existing feature Release Notes: updated Signed-off-by: Greg Greenway --- api/envoy/config/core/v3/protocol.proto | 3 + .../v3/http_connection_manager.proto | 1 + changelogs/current.yaml | 3 +- envoy/http/codec.h | 2 + source/common/upstream/upstream_impl.cc | 13 +++- test/common/upstream/upstream_impl_test.cc | 63 +++++++++++++++++++ .../http_connection_manager/config_test.cc | 23 +++++++ 7 files changed, 105 insertions(+), 3 deletions(-) diff --git a/api/envoy/config/core/v3/protocol.proto b/api/envoy/config/core/v3/protocol.proto index e566278600..423ec11b8c 100644 --- a/api/envoy/config/core/v3/protocol.proto +++ b/api/envoy/config/core/v3/protocol.proto @@ -262,6 +262,8 @@ message HttpProtocolOptions { // The maximum number of headers (request headers if configured on HttpConnectionManager, // response headers when configured on a cluster). // If unconfigured, the default maximum number of headers allowed is 100. + // The default value for requests can be overridden by setting runtime key ``envoy.reloadable_features.max_request_headers_count``. + // The default value for responses can be overridden by setting runtime key ``envoy.reloadable_features.max_response_headers_count``. // Downstream requests that exceed this limit will receive a 431 response for HTTP/1.x and cause a stream // reset for HTTP/2. // Upstream responses that exceed this limit will result in a 503 response. @@ -270,6 +272,7 @@ message HttpProtocolOptions { // The maximum size of response headers. // If unconfigured, the default is 60 KiB, except for HTTP/1 response headers which have a default // of 80KiB. + // The default value can be overridden by setting runtime key ``envoy.reloadable_features.max_response_headers_size_kb``. // Responses that exceed this limit will result in a 503 response. // In Envoy, this setting is only valid when configured on an upstream cluster, not on the // :ref:`HTTP Connection Manager diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 3d438ae878..3b49f13295 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -493,6 +493,7 @@ message HttpConnectionManager { // The maximum request headers size for incoming connections. // If unconfigured, the default max request headers allowed is 60 KiB. + // The default value can be overridden by setting runtime key ``envoy.reloadable_features.max_request_headers_size_kb``. // Requests that exceed this limit will receive a 431 response. // // Note: currently some protocol codecs impose limits on the maximum size of a single header: diff --git a/changelogs/current.yaml b/changelogs/current.yaml index b773fb76f6..200f299a12 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -306,7 +306,8 @@ new_features: - area: http change: | Added configuration setting for the :ref:`maximum size of response headers - ` in responses. + ` in responses. The default can + be overridden with runtime key ``envoy.reloadable_features.max_response_headers_size_kb``. - area: http_11_proxy change: | Added the option to configure the transport socket via locality or endpoint metadata. diff --git a/envoy/http/codec.h b/envoy/http/codec.h index 7408d4085a..0297a8a323 100644 --- a/envoy/http/codec.h +++ b/envoy/http/codec.h @@ -46,6 +46,8 @@ const char MaxResponseHeadersCountOverrideKey[] = "envoy.reloadable_features.max_response_headers_count"; const char MaxRequestHeadersSizeOverrideKey[] = "envoy.reloadable_features.max_request_headers_size_kb"; +const char MaxResponseHeadersSizeOverrideKey[] = + "envoy.reloadable_features.max_response_headers_size_kb"; class Stream; class RequestDecoder; diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 195d2ba59a..49f7bdaa93 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1229,8 +1229,17 @@ ClusterInfoImpl::ClusterInfoImpl( http_protocol_options_->common_http_protocol_options_, max_headers_count, runtime_.snapshot().getInteger(Http::MaxResponseHeadersCountOverrideKey, Http::DEFAULT_MAX_HEADERS_COUNT))), - max_response_headers_kb_(PROTOBUF_GET_OPTIONAL_WRAPPED( - http_protocol_options_->common_http_protocol_options_, max_response_headers_kb)), + max_response_headers_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( + http_protocol_options_->common_http_protocol_options_, max_response_headers_kb, + [&]() -> absl::optional { + constexpr uint64_t unspecified = 0; + uint64_t runtime_val = runtime_.snapshot().getInteger( + Http::MaxResponseHeadersSizeOverrideKey, unspecified); + if (runtime_val == unspecified) { + return absl::nullopt; + } + return runtime_val; + }())), type_(config.type()), drain_connections_on_host_removal_(config.ignore_health_on_host_removal()), connection_pool_per_downstream_connection_( diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 38917ad1be..fc78e4c718 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -5594,6 +5594,69 @@ TEST_F(ClusterInfoImplTest, Http2AutoWithNonAlpnMatcherAndValidationOff) { EXPECT_NO_THROW(makeCluster(yaml + auto_http2)); } +TEST_F(ClusterInfoImplTest, MaxResponseHeadersDefault) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + )EOF", + Network::Address::IpVersion::v4); + + auto cluster = makeCluster(yaml); + EXPECT_FALSE(cluster->info()->maxResponseHeadersKb().has_value()); + EXPECT_EQ(100, cluster->info()->maxResponseHeadersCount()); +} + +// Test that the runtime override for the defaults is used when specified. +TEST_F(ClusterInfoImplTest, MaxResponseHeadersRuntimeOverride) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + )EOF", + Network::Address::IpVersion::v4); + + EXPECT_CALL(runtime_.snapshot_, + getInteger("envoy.reloadable_features.max_response_headers_size_kb", _)) + .WillRepeatedly(Return(123)); + EXPECT_CALL(runtime_.snapshot_, + getInteger("envoy.reloadable_features.max_response_headers_count", _)) + .WillRepeatedly(Return(456)); + + auto cluster = makeCluster(yaml); + EXPECT_EQ(absl::make_optional(uint16_t(123)), cluster->info()->maxResponseHeadersKb()); + EXPECT_EQ(456, cluster->info()->maxResponseHeadersCount()); +} + +// Test that the runtime override is ignored if there is a configured value. +TEST_F(ClusterInfoImplTest, MaxResponseHeadersRuntimeOverrideIgnored) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + common_http_protocol_options: + max_response_headers_kb: 1 + max_headers_count: 2 + explicit_http_config: + http2_protocol_options: {} + )EOF", + Network::Address::IpVersion::v4); + + EXPECT_CALL(runtime_.snapshot_, + getDouble("envoy.reloadable_features.max_response_headers_size_kb", _)) + .WillRepeatedly(Return(123)); + EXPECT_CALL(runtime_.snapshot_, + getDouble("envoy.reloadable_features.max_response_headers_count", _)) + .WillRepeatedly(Return(456)); + + auto cluster = makeCluster(yaml); + EXPECT_EQ(absl::make_optional(uint16_t(1)), cluster->info()->maxResponseHeadersKb()); + EXPECT_EQ(2, cluster->info()->maxResponseHeadersCount()); +} + TEST_F(ClusterInfoImplTest, UpstreamFilterTypedAndDynamicConfigThrows) { const std::string yaml = R"EOF( name: name diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 9805d0f72c..9c8aa06424 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -869,6 +869,29 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbMaxConfiguredViaRunti EXPECT_EQ(9000, config.maxRequestHeadersKb()); } +TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersCountMaxConfiguredViaRuntime) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + )EOF"; + + ON_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + getInteger("envoy.reloadable_features.max_request_headers_count", _)) + .WillByDefault(Return(42)); + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + &scoped_routes_config_provider_manager_, tracer_manager_, + filter_config_provider_manager_, creation_status_); + ASSERT_TRUE(creation_status_.ok()); + EXPECT_EQ(42, config.maxRequestHeadersCount()); +} + // Validated that an explicit zero stream idle timeout disables. TEST_F(HttpConnectionManagerConfigTest, DisabledStreamIdleTimeout) { const std::string yaml_string = R"EOF( From 9244dc957b1f9e44337399b0fe01198eb171e76a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Fri, 4 Oct 2024 19:59:32 -0400 Subject: [PATCH 08/77] gcc: remove -Wdangling-reference workaround (#36452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The false positive warnings have been resolved in current versions of gcc. Risk Level: low Testing: CI Signed-off-by: Alejandro R. Sedeño --- .../access_loggers/grpc/grpc_access_log_utils.cc | 7 ------- source/extensions/common/dubbo/hessian2_utils.h | 8 -------- .../filters/network/dubbo_proxy/hessian_utils.h | 9 --------- .../health_checkers/grpc/health_checker_impl.cc | 7 ------- .../health_checkers/http/health_checker_impl.h | 7 ------- source/extensions/health_checkers/thrift/thrift.h | 7 ------- 6 files changed, 45 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc index 2aab12ea6a..ac0bcb9d0d 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -275,14 +275,7 @@ void Utility::extractCommonAccessLogProperties( } if (stream_info.upstreamInfo().has_value()) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const auto& upstream_info = stream_info.upstreamInfo().value().get(); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif if (upstream_info.upstreamHost() != nullptr) { Network::Utility::addressToProtobufAddress( *upstream_info.upstreamHost()->address(), diff --git a/source/extensions/common/dubbo/hessian2_utils.h b/source/extensions/common/dubbo/hessian2_utils.h index 30a0f54335..a51a224bc8 100644 --- a/source/extensions/common/dubbo/hessian2_utils.h +++ b/source/extensions/common/dubbo/hessian2_utils.h @@ -5,15 +5,7 @@ #include "envoy/buffer/buffer.h" #include "absl/strings/string_view.h" - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif #include "hessian2/basic_codec/object_codec.hpp" -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif #include "hessian2/codec.hpp" #include "hessian2/object.hpp" #include "hessian2/reader.hpp" diff --git a/source/extensions/filters/network/dubbo_proxy/hessian_utils.h b/source/extensions/filters/network/dubbo_proxy/hessian_utils.h index da30248da3..8385eb37c0 100644 --- a/source/extensions/filters/network/dubbo_proxy/hessian_utils.h +++ b/source/extensions/filters/network/dubbo_proxy/hessian_utils.h @@ -5,16 +5,7 @@ #include "envoy/buffer/buffer.h" #include "absl/strings/string_view.h" - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif #include "hessian2/basic_codec/object_codec.hpp" -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - #include "hessian2/codec.hpp" #include "hessian2/object.hpp" #include "hessian2/reader.hpp" diff --git a/source/extensions/health_checkers/grpc/health_checker_impl.cc b/source/extensions/health_checkers/grpc/health_checker_impl.cc index f9df90eaa1..e5873e8ec0 100644 --- a/source/extensions/health_checkers/grpc/health_checker_impl.cc +++ b/source/extensions/health_checkers/grpc/health_checker_impl.cc @@ -196,15 +196,8 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onInterval() { request_encoder_ = &client_->newStream(*this); request_encoder_->getStream().addCallbacks(*this); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const std::string& authority = getHostname(host_, parent_.authority_value_, parent_.cluster_.info()); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif auto headers_message = Grpc::Common::prepareHeaders(authority, parent_.service_method_.service()->full_name(), parent_.service_method_.name(), absl::nullopt); diff --git a/source/extensions/health_checkers/http/health_checker_impl.h b/source/extensions/health_checkers/http/health_checker_impl.h index d4a03688c1..939062b736 100644 --- a/source/extensions/health_checkers/http/health_checker_impl.h +++ b/source/extensions/health_checkers/http/health_checker_impl.h @@ -141,14 +141,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { Http::CodecClientPtr client_; Http::ResponseHeaderMapPtr response_headers_; Buffer::InstancePtr response_body_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const std::string& hostname_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif Network::ConnectionInfoProviderSharedPtr local_connection_info_provider_; // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. const Http::Protocol protocol_; diff --git a/source/extensions/health_checkers/thrift/thrift.h b/source/extensions/health_checkers/thrift/thrift.h index 329a01e9d0..9c75e9df56 100644 --- a/source/extensions/health_checkers/thrift/thrift.h +++ b/source/extensions/health_checkers/thrift/thrift.h @@ -57,14 +57,7 @@ class ThriftHealthChecker : public Upstream::HealthCheckerImplBase { private: ThriftHealthChecker& parent_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const std::string& hostname_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif ClientPtr client_; bool expect_close_{}; }; From 16cd72ea238a70e77d6f0e5f36224ebe38a6ca88 Mon Sep 17 00:00:00 2001 From: Bin Wu <46450037+wu-bin@users.noreply.github.com> Date: Sat, 5 Oct 2024 09:43:07 -0400 Subject: [PATCH 09/77] Add `ENVOY_EXECUTION_SCOPE`. (#36056) Add `ENVOY_EXECUTION_SCOPE` to mark the start and end of a Envoy::Tracing::Span or Http::FilterContext, which is active in the current thread. This macro only takes effect when `ENVOY_ENABLE_EXECUTION_SCOPE` is defined. Commit Message: Add `ENVOY_EXECUTION_SCOPE`. Additional Description: Risk Level: No. It is no-op unless `ENVOY_ENABLE_EXECUTION_SCOPE` is defined. Testing: Unit test in test/common/common/execution_context_test.cc. Docs Changes: N/A Release Notes: N/A Platform Specific Features: [Optional Runtime guard:] [Optional Fixes #Issue] [Optional Fixes commit #PR or SHA] [Optional Deprecated:] [Optional [API Considerations](https://github.com/envoyproxy/envoy/blob/main/api/review_checklist.md):] --------- Signed-off-by: Bin Wu --- envoy/common/BUILD | 2 + envoy/common/execution_context.h | 62 ++++++++++++++--- source/common/common/cleanup.h | 4 ++ source/common/http/conn_manager_impl.cc | 11 +++ source/common/http/filter_manager.cc | 9 +++ test/common/common/BUILD | 2 + test/common/common/execution_context_test.cc | 73 +++++++++++++++++++- 7 files changed, 152 insertions(+), 11 deletions(-) diff --git a/envoy/common/BUILD b/envoy/common/BUILD index 12c71b554b..c11152a639 100644 --- a/envoy/common/BUILD +++ b/envoy/common/BUILD @@ -127,6 +127,8 @@ envoy_cc_library( deps = [ ":pure_lib", ":scope_tracker_interface", + "//source/common/common:cleanup_lib", + "//source/common/common:macros", ], ) diff --git a/envoy/common/execution_context.h b/envoy/common/execution_context.h index b723a221bc..3580f8f2bd 100644 --- a/envoy/common/execution_context.h +++ b/envoy/common/execution_context.h @@ -6,6 +6,8 @@ #include "envoy/common/scope_tracker.h" #include "envoy/stream_info/stream_info.h" +#include "source/common/common/cleanup.h" +#include "source/common/common/macros.h" #include "source/common/common/non_copyable.h" namespace Envoy { @@ -15,6 +17,14 @@ namespace Envoy { static constexpr absl::string_view kConnectionExecutionContextFilterStateName = "envoy.network.connection_execution_context"; +namespace Http { +struct FilterContext; +} + +namespace Tracing { +class Span; +} + class ScopedExecutionContext; // ExecutionContext can be inherited by subclasses to represent arbitrary information associated @@ -22,6 +32,27 @@ class ScopedExecutionContext; // starts/ends. For an example usage, please see // https://github.com/envoyproxy/envoy/issues/32012. class ExecutionContext : public StreamInfo::FilterState::Object, NonCopyable { +public: + static void setEnabled(bool value) { enabled().store(value, std::memory_order_relaxed); } + + static bool isEnabled() { return enabled().load(std::memory_order_relaxed); } + + static ExecutionContext* fromStreamInfo(OptRef info) { + if (!isEnabled() || !info.has_value()) { + return nullptr; + } + const auto* const_context = info->filterState().getDataReadOnly( + kConnectionExecutionContextFilterStateName); + return const_cast(const_context); + } + + // Called when enters a scope in which |*span| is active. + // Returns an object that can do some cleanup when exits the scope. + virtual Envoy::Cleanup onScopeEnter(Envoy::Tracing::Span* span) PURE; + // Called when enters a scope in which |*filter_context| is active. + // Returns an object that can do some cleanup when exits the scope. + virtual Envoy::Cleanup onScopeEnter(const Http::FilterContext* filter_context) PURE; + protected: // Called when the current thread starts to run code on behalf of the owner of this object. // protected because it should only be called by ScopedExecutionContext. @@ -30,6 +61,9 @@ class ExecutionContext : public StreamInfo::FilterState::Object, NonCopyable { // protected because it should only be called by ScopedExecutionContext. virtual void deactivate() PURE; +private: + static std::atomic& enabled() { MUTABLE_CONSTRUCT_ON_FIRST_USE(std::atomic); } + friend class ScopedExecutionContext; }; @@ -47,7 +81,8 @@ class ScopedExecutionContext : NonCopyable { public: ScopedExecutionContext() : ScopedExecutionContext(nullptr) {} ScopedExecutionContext(const ScopeTrackedObject* object) - : context_(object != nullptr ? getExecutionContext(object->trackedStream()) : nullptr) { + : context_(object != nullptr ? ExecutionContext::fromStreamInfo(object->trackedStream()) + : nullptr) { if (context_ != nullptr) { context_->activate(); } @@ -66,18 +101,25 @@ class ScopedExecutionContext : NonCopyable { bool isNull() const { return context_ == nullptr; } private: - ExecutionContext* getExecutionContext(OptRef info) { - if (!info.has_value()) { - return nullptr; - } - const auto* const_context = info->filterState().getDataReadOnly( - kConnectionExecutionContextFilterStateName); - return const_cast(const_context); - } - ExecutionContext* context_; }; +#define ENVOY_EXECUTION_SCOPE_CAT_(a, b) a##b +#define ENVOY_EXECUTION_SCOPE_CAT(a, b) ENVOY_EXECUTION_SCOPE_CAT_(a, b) +// Invoked when |scopedObject| is active from the current line to the end of the current c++ scope. +// |trackedStream| is a OptRef from which a ExecutionContext is extracted. +// |scopedObject| is a pointer to a Envoy::Tracing::Span or a Http::FilterContext. +#define ENVOY_EXECUTION_SCOPE(trackedStream, scopedObject) \ + Envoy::Cleanup ENVOY_EXECUTION_SCOPE_CAT(on_scope_exit_, __LINE__) = \ + [execution_context = ExecutionContext::fromStreamInfo(trackedStream), \ + scoped_object = (scopedObject)] { \ + if (execution_context == nullptr) { \ + return Envoy::Cleanup::Noop(); \ + } \ + return execution_context->onScopeEnter(scoped_object); \ + }() +#else +#define ENVOY_EXECUTION_SCOPE(trackedStream, scopedObject) #endif } // namespace Envoy diff --git a/source/common/common/cleanup.h b/source/common/common/cleanup.h index 3d34bf51b8..bdefb1cbb2 100644 --- a/source/common/common/cleanup.h +++ b/source/common/common/cleanup.h @@ -24,6 +24,10 @@ class Cleanup { bool cancelled() { return cancelled_; } + static Cleanup Noop() { + return Cleanup([] {}); + } + private: std::function f_; bool cancelled_{false}; diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index e06bca9c6a..84e358b01c 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1416,6 +1416,7 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt traceRequest(); } + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); if (!connection_manager_.shouldDeferRequestProxyingToNextIoCycle()) { filter_manager_.decodeHeaders(*request_headers_, end_stream); } else { @@ -1487,6 +1488,7 @@ void ConnectionManagerImpl::ActiveStream::traceRequest() { void ConnectionManagerImpl::ActiveStream::decodeData(Buffer::Instance& data, bool end_stream) { ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); maybeRecordLastByteReceived(end_stream); filter_manager_.streamInfo().addBytesReceived(data.length()); if (!state_.deferred_to_next_io_iteration_) { @@ -1504,6 +1506,7 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(RequestTrailerMapPtr&& ENVOY_STREAM_LOG(debug, "request trailers complete:\n{}", *this, *trailers); ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); resetIdleTimer(); ASSERT(!request_trailers_); @@ -1524,6 +1527,7 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(RequestTrailerMapPtr&& } void ConnectionManagerImpl::ActiveStream::decodeMetadata(MetadataMapPtr&& metadata_map) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); resetIdleTimer(); if (!state_.deferred_to_next_io_iteration_) { // After going through filters, the ownership of metadata_map will be passed to terminal filter. @@ -1728,6 +1732,7 @@ void ConnectionManagerImpl::ActiveStream::onLocalReply(Code code) { } void ConnectionManagerImpl::ActiveStream::encode1xxHeaders(ResponseHeaderMap& response_headers) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); // Strip the T-E headers etc. Defer other header additions as well as drain-close logic to the // continuation headers. ConnectionManagerUtility::mutateResponseHeaders( @@ -1746,6 +1751,7 @@ void ConnectionManagerImpl::ActiveStream::encode1xxHeaders(ResponseHeaderMap& re void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& headers, bool end_stream) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); // Base headers. // We want to preserve the original date header, but we add a date header if it is absent @@ -1891,6 +1897,7 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade } void ConnectionManagerImpl::ActiveStream::encodeData(Buffer::Instance& data, bool end_stream) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); ENVOY_STREAM_LOG(trace, "encoding data via codec (size={} end_stream={})", *this, data.length(), end_stream); @@ -1899,12 +1906,14 @@ void ConnectionManagerImpl::ActiveStream::encodeData(Buffer::Instance& data, boo } void ConnectionManagerImpl::ActiveStream::encodeTrailers(ResponseTrailerMap& trailers) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); ENVOY_STREAM_LOG(debug, "encoding trailers via codec:\n{}", *this, trailers); response_encoder_->encodeTrailers(trailers); } void ConnectionManagerImpl::ActiveStream::encodeMetadata(MetadataMapPtr&& metadata) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); MetadataMapVector metadata_map_vector; metadata_map_vector.emplace_back(std::move(metadata)); ENVOY_STREAM_LOG(debug, "encoding metadata via codec:\n{}", *this, metadata_map_vector); @@ -2182,6 +2191,7 @@ void ConnectionManagerImpl::ActiveStream::onRequestDataTooLarge() { void ConnectionManagerImpl::ActiveStream::recreateStream( StreamInfo::FilterStateSharedPtr filter_state) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); ResponseEncoder* response_encoder = response_encoder_; response_encoder_ = nullptr; @@ -2253,6 +2263,7 @@ bool ConnectionManagerImpl::ActiveStream::onDeferredRequestProcessing() { if (!state_.deferred_to_next_io_iteration_) { return false; } + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); state_.deferred_to_next_io_iteration_ = false; bool end_stream = state_.deferred_end_stream_ && deferred_data_ == nullptr && deferred_request_trailers_ == nullptr && deferred_metadata_.empty(); diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index 89c58323ef..d158ea8088 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -542,6 +542,7 @@ void FilterManager::decodeHeaders(ActiveStreamDecoderFilter* filter, RequestHead (*entry)->end_stream_); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); ASSERT(!(state_.filter_call_state_ & FilterCallState::DecodeHeaders)); state_.filter_call_state_ |= FilterCallState::DecodeHeaders; (*entry)->end_stream_ = (end_stream && continue_data_entry == decoder_filters_.end()); @@ -653,6 +654,7 @@ void FilterManager::decodeData(ActiveStreamDecoderFilter* filter, Buffer::Instan (*entry)->end_stream_); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame types, return now. if (handleDataIfStopAll(**entry, data, state_.decoder_filters_streaming_)) { return; @@ -803,6 +805,7 @@ void FilterManager::decodeTrailers(ActiveStreamDecoderFilter* filter, RequestTra ASSERT(!state_.decoder_filter_chain_complete_ || entry == decoder_filters_.end()); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, return now. if ((*entry)->stoppedAll()) { return; @@ -846,6 +849,7 @@ void FilterManager::decodeMetadata(ActiveStreamDecoderFilter* filter, MetadataMa ASSERT(!(state_.filter_call_state_ & FilterCallState::DecodeMetadata)); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, stores metadata and returns. // If the filter pointed by entry hasn't returned from decodeHeaders, stores newly added // metadata in case decodeHeaders returns StopAllIteration. The latter can happen when headers @@ -1173,6 +1177,7 @@ void FilterManager::encode1xxHeaders(ActiveStreamEncoderFilter* filter, std::list::iterator entry = commonEncodePrefix(filter, false, FilterIterationStartState::AlwaysStartFromNext); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); ASSERT(!(state_.filter_call_state_ & FilterCallState::Encode1xxHeaders)); state_.filter_call_state_ |= FilterCallState::Encode1xxHeaders; const Filter1xxHeadersStatus status = (*entry)->handle_->encode1xxHeaders(headers); @@ -1222,6 +1227,7 @@ void FilterManager::encodeHeaders(ActiveStreamEncoderFilter* filter, ResponseHea std::list::iterator continue_data_entry = encoder_filters_.end(); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); ASSERT(!(state_.filter_call_state_ & FilterCallState::EncodeHeaders)); state_.filter_call_state_ |= FilterCallState::EncodeHeaders; (*entry)->end_stream_ = (end_stream && continue_data_entry == encoder_filters_.end()); @@ -1306,6 +1312,7 @@ void FilterManager::encodeMetadata(ActiveStreamEncoderFilter* filter, commonEncodePrefix(filter, false, FilterIterationStartState::CanStartFromCurrent); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, stores metadata and returns. // If the filter pointed by entry hasn't returned from encodeHeaders, stores newly added // metadata in case encodeHeaders returns StopAllIteration. The latter can happen when headers @@ -1394,6 +1401,7 @@ void FilterManager::encodeData(ActiveStreamEncoderFilter* filter, Buffer::Instan const bool trailers_exists_at_start = filter_manager_callbacks_.responseTrailers().has_value(); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, return now. if (handleDataIfStopAll(**entry, data, state_.encoder_filters_streaming_)) { return; @@ -1464,6 +1472,7 @@ void FilterManager::encodeTrailers(ActiveStreamEncoderFilter* filter, std::list::iterator entry = commonEncodePrefix(filter, true, FilterIterationStartState::CanStartFromCurrent); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, return now. if ((*entry)->stoppedAll()) { return; diff --git a/test/common/common/BUILD b/test/common/common/BUILD index d15e11f00f..f1d14259d0 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -603,6 +603,8 @@ envoy_cc_test( deps = [ "//envoy/common:execution_context", "//source/common/api:api_lib", + "//source/common/http:conn_manager_lib", + "//source/common/http:filter_manager_lib", "//test/mocks:common_lib", "//test/mocks/stream_info:stream_info_mocks", ], diff --git a/test/common/common/execution_context_test.cc b/test/common/common/execution_context_test.cc index 8aed2f00ea..66dd126ce4 100644 --- a/test/common/common/execution_context_test.cc +++ b/test/common/common/execution_context_test.cc @@ -1,9 +1,13 @@ #include #include "envoy/common/execution_context.h" +#include "envoy/http/filter_factory.h" #include "source/common/api/api_impl.h" #include "source/common/common/scope_tracker.h" +#include "source/common/http/conn_manager_impl.h" +#include "source/common/http/filter_manager.h" +#include "source/common/tracing/null_span_impl.h" #include "test/mocks/common.h" #include "test/mocks/stream_info/mocks.h" @@ -14,8 +18,21 @@ namespace Envoy { +thread_local const Http::FilterContext* current_filter_context = nullptr; + class TestExecutionContext : public ExecutionContext { public: + Envoy::Cleanup onScopeEnter(Envoy::Tracing::Span*) override { return Envoy::Cleanup::Noop(); } + + Envoy::Cleanup onScopeEnter(const Http::FilterContext* filter_context) override { + if (filter_context == nullptr) { + return Envoy::Cleanup::Noop(); + } + const Http::FilterContext* old_filter_context = current_filter_context; + current_filter_context = filter_context; + return Envoy::Cleanup([old_filter_context]() { current_filter_context = old_filter_context; }); + } + int activationDepth() const { return activation_depth_; } int activationGenerations() const { return activation_generations_; } @@ -39,6 +56,11 @@ class TestExecutionContext : public ExecutionContext { class ExecutionContextTest : public testing::Test { public: + static void SetUpTestCase() { + EXPECT_FALSE(ExecutionContext::isEnabled()); + ExecutionContext::setEnabled(true); + } + ExecutionContextTest() { ON_CALL(tracked_object_, trackedStream()) .WillByDefault(testing::Return(OptRef(stream_info_))); @@ -114,7 +136,6 @@ TEST_F(ExecutionContextTest, DisjointScopes) { } TEST_F(ExecutionContextTest, InScopeTrackerScopeState) { - Api::ApiPtr api(Api::createApiForTest()); Event::DispatcherPtr dispatcher(api->allocateDispatcher("test_thread")); EXPECT_CALL(tracked_object_, trackedStream()) @@ -137,4 +158,54 @@ TEST_F(ExecutionContextTest, InScopeTrackerScopeState) { { ScopeTrackerScopeState scope(&tracked_object_, *dispatcher); } } +TEST_F(ExecutionContextTest, NoopScope) { + OptRef null_stream_info; + Http::FilterContext* null_filter_context = nullptr; + Tracing::Span* null_tracing_span = nullptr; + ENVOY_EXECUTION_SCOPE(null_stream_info, null_filter_context); + ENVOY_EXECUTION_SCOPE(null_stream_info, null_tracing_span); + + Http::FilterContext filter_context; + ENVOY_EXECUTION_SCOPE(null_stream_info, &filter_context); + ENVOY_EXECUTION_SCOPE(null_stream_info, &Tracing::NullSpan::instance()); + + setWithContext(); + EXPECT_CALL(tracked_object_, trackedStream()) + .Times(2) + .WillRepeatedly(testing::Return(OptRef(stream_info_))); + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), null_filter_context); + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), null_tracing_span); +} + +TEST_F(ExecutionContextTest, FilterScope) { + setWithContext(); + EXPECT_CALL(tracked_object_, trackedStream()) + .Times(2) + .WillRepeatedly(testing::Return(OptRef(stream_info_))); + + Http::FilterContext outer_filter_context{"outer_filter"}; + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), &outer_filter_context); + EXPECT_EQ(current_filter_context, &outer_filter_context); + + { + Http::FilterContext inner_filter_context{"inner_filter"}; + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), &inner_filter_context); + EXPECT_EQ(current_filter_context, &inner_filter_context); + } + + EXPECT_EQ(current_filter_context, &outer_filter_context); +} + +// Make sure source/common/http/conn_manager_impl.cc compiles with ENVOY_ENABLE_EXECUTION_CONTEXT. +TEST_F(ExecutionContextTest, ConnectionManagerImplCompiles) { + const Http::ConnectionManagerImpl* impl = nullptr; + EXPECT_EQ(impl, impl); +} + +// Make sure source/common/http/filter_manager.cc compiles with ENVOY_ENABLE_EXECUTION_CONTEXT. +TEST_F(ExecutionContextTest, FilterManagerCompiles) { + const Http::FilterManager* manager = nullptr; + EXPECT_EQ(manager, manager); +} + } // namespace Envoy From 33679e411dbb4698eae07f7926e595fc13e2805d Mon Sep 17 00:00:00 2001 From: blake-snyder Date: Sat, 5 Oct 2024 06:49:15 -0700 Subject: [PATCH 10/77] Add support for multiple formats of ORCA headers. (#35894) Commit Message: Add support for multiple formats of ORCA headers. Additional Description: Add support for multiple formats of ORCA headers. ORCA parsing introduced in https://github.com/envoyproxy/envoy/pull/35422 [Original Design Proposal](https://github.com/envoyproxy/envoy/issues/6614) [Using ORCA load reports in Envoy](https://docs.google.com/document/d/1gb_2pcNnEzTgo1EJ6w1Ol7O-EH-O_Ysu5o215N9MTAg/edit#heading=h.bi4e79pb39fe) Risk Level: Low Testing: See included unit tests. Docs Changes: N/A Release Notes: N/A Platform Specific Features: JSON format unsupported on Mobile. CC @efimki @adisuissa @wbpcode --------- Signed-off-by: blake-snyder --- source/common/orca/BUILD | 3 + source/common/orca/orca_parser.cc | 154 +++++++++++++++++++-- source/common/orca/orca_parser.h | 17 ++- test/common/orca/orca_parser_test.cc | 195 ++++++++++++++++++++++++++- 4 files changed, 354 insertions(+), 15 deletions(-) diff --git a/source/common/orca/BUILD b/source/common/orca/BUILD index 79aa433408..56d750828f 100644 --- a/source/common/orca/BUILD +++ b/source/common/orca/BUILD @@ -13,8 +13,11 @@ envoy_cc_library( srcs = ["orca_parser.cc"], hdrs = ["orca_parser.h"], deps = [ + "//envoy/common:exception_lib", "//envoy/http:header_map_interface", "//source/common/common:base64_lib", + "//source/common/http:header_utility_lib", + "//source/common/protobuf:utility_lib_header", "@com_github_cncf_xds//xds/data/orca/v3:pkg_cc_proto", "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/status:statusor", diff --git a/source/common/orca/orca_parser.cc b/source/common/orca/orca_parser.cc index a18061bb25..2dd29bc794 100644 --- a/source/common/orca/orca_parser.cc +++ b/source/common/orca/orca_parser.cc @@ -1,13 +1,27 @@ #include "source/common/orca/orca_parser.h" +#include +#include #include +#include +#include +#include "envoy/common/exception.h" #include "envoy/http/header_map.h" #include "source/common/common/base64.h" #include "source/common/common/fmt.h" +#include "source/common/http/header_utility.h" +#include "source/common/protobuf/utility.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" using ::Envoy::Http::HeaderMap; using xds::data::orca::v3::OrcaLoadReport; @@ -17,29 +31,147 @@ namespace Orca { namespace { +const Http::LowerCaseString& endpointLoadMetricsHeader() { + CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, kEndpointLoadMetricsHeader); +} + const Http::LowerCaseString& endpointLoadMetricsHeaderBin() { CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, kEndpointLoadMetricsHeaderBin); } +absl::Status tryCopyNamedMetricToOrcaLoadReport(absl::string_view metric_name, double metric_value, + OrcaLoadReport& orca_load_report) { + if (metric_name.empty()) { + return absl::InvalidArgumentError("named metric key is empty."); + } + + orca_load_report.mutable_named_metrics()->insert({std::string(metric_name), metric_value}); + return absl::OkStatus(); +} + +std::vector parseCommaDelimitedHeader(const absl::string_view entry) { + std::vector values; + std::vector tokens = + Envoy::Http::HeaderUtility::parseCommaDelimitedHeader(entry); + values.insert(values.end(), tokens.begin(), tokens.end()); + return values; +} + +absl::Status tryCopyMetricToOrcaLoadReport(absl::string_view metric_name, + absl::string_view metric_value, + OrcaLoadReport& orca_load_report) { + if (metric_name.empty()) { + return absl::InvalidArgumentError("metric names cannot be empty strings"); + } + + if (metric_value.empty()) { + return absl::InvalidArgumentError("metric values cannot be empty strings"); + } + + double value; + if (!absl::SimpleAtod(metric_value, &value)) { + return absl::InvalidArgumentError(fmt::format( + "unable to parse custom backend load metric value({}): {}", metric_name, metric_value)); + } + + if (std::isnan(value)) { + return absl::InvalidArgumentError( + fmt::format("custom backend load metric value({}) cannot be NaN.", metric_name)); + } + + if (std::isinf(value)) { + return absl::InvalidArgumentError( + fmt::format("custom backend load metric value({}) cannot be infinity.", metric_name)); + } + + if (absl::StartsWith(metric_name, kNamedMetricsFieldPrefix)) { + auto metric_name_without_prefix = absl::StripPrefix(metric_name, kNamedMetricsFieldPrefix); + return tryCopyNamedMetricToOrcaLoadReport(metric_name_without_prefix, value, orca_load_report); + } + + if (metric_name == kCpuUtilizationField) { + orca_load_report.set_cpu_utilization(value); + } else if (metric_name == kMemUtilizationField) { + orca_load_report.set_mem_utilization(value); + } else if (metric_name == kApplicationUtilizationField) { + orca_load_report.set_application_utilization(value); + } else if (metric_name == kEpsField) { + orca_load_report.set_eps(value); + } else if (metric_name == kRpsFractionalField) { + orca_load_report.set_rps_fractional(value); + } else { + return absl::InvalidArgumentError(absl::StrCat("unsupported metric name: ", metric_name)); + } + return absl::OkStatus(); +} + +absl::Status tryParseNativeHttpEncoded(const absl::string_view header, + OrcaLoadReport& orca_load_report) { + const std::vector values = parseCommaDelimitedHeader(header); + + // Check for duplicate metric names here because OrcaLoadReport fields are not + // marked as optional and therefore don't differentiate between unset and + // default values. + absl::flat_hash_set metric_names; + for (const auto value : values) { + std::pair entry = + absl::StrSplit(value, absl::MaxSplits(':', 1), absl::SkipWhitespace()); + if (metric_names.contains(entry.first)) { + return absl::AlreadyExistsError( + absl::StrCat(kEndpointLoadMetricsHeader, " contains duplicate metric: ", entry.first)); + } + RETURN_IF_NOT_OK(tryCopyMetricToOrcaLoadReport(entry.first, entry.second, orca_load_report)); + metric_names.insert(entry.first); + } + return absl::OkStatus(); +} + +absl::Status tryParseSerializedBinary(const absl::string_view header, + OrcaLoadReport& orca_load_report) { + if (header.empty()) { + return absl::InvalidArgumentError("ORCA binary header value is empty"); + } + const std::string decoded_value = Envoy::Base64::decode(header); + if (decoded_value.empty()) { + return absl::InvalidArgumentError( + fmt::format("unable to decode ORCA binary header value: {}", header)); + } + if (!orca_load_report.ParseFromString(decoded_value)) { + return absl::InvalidArgumentError( + fmt::format("unable to parse binaryheader to OrcaLoadReport: {}", header)); + } + return absl::OkStatus(); +} + } // namespace absl::StatusOr parseOrcaLoadReportHeaders(const HeaderMap& headers) { OrcaLoadReport load_report; - // Binary protobuf format. + // Binary protobuf format. Legacy header from gRPC implementation. if (const auto header_bin = headers.get(endpointLoadMetricsHeaderBin()); !header_bin.empty()) { const auto header_value = header_bin[0]->value().getStringView(); - if (header_value.empty()) { - return absl::InvalidArgumentError("ORCA binary header value is empty"); - } - const std::string decoded_value = Envoy::Base64::decode(header_value); - if (decoded_value.empty()) { - return absl::InvalidArgumentError( - fmt::format("unable to decode ORCA binary header value: {}", header_value)); - } - if (!load_report.ParseFromString(decoded_value)) { + RETURN_IF_NOT_OK(tryParseSerializedBinary(header_value, load_report)); + } else if (const auto header = headers.get(endpointLoadMetricsHeader()); !header.empty()) { + std::pair split_header = + absl::StrSplit(header[0]->value().getStringView(), absl::MaxSplits(' ', 1)); + + if (split_header.first == kHeaderFormatPrefixBin) { // Binary protobuf format. + RETURN_IF_NOT_OK(tryParseSerializedBinary(split_header.second, load_report)); + } else if (split_header.first == kHeaderFormatPrefixText) { // Native HTTP format. + RETURN_IF_NOT_OK(tryParseNativeHttpEncoded(split_header.second, load_report)); + } else if (split_header.first == kHeaderFormatPrefixJson) { // JSON format. +#if defined(ENVOY_ENABLE_FULL_PROTOS) && defined(ENVOY_ENABLE_YAML) + const std::string json_string = std::string(split_header.second); + bool has_unknown_field = false; + RETURN_IF_ERROR( + Envoy::MessageUtil::loadFromJsonNoThrow(json_string, load_report, has_unknown_field)); +#else + IS_ENVOY_BUG("JSON formatted ORCA header support not implemented for this build"); +#endif // !ENVOY_ENABLE_FULL_PROTOS || !ENVOY_ENABLE_YAML + } else { return absl::InvalidArgumentError( - fmt::format("unable to parse binaryheader to OrcaLoadReport: {}", header_value)); + fmt::format("unsupported ORCA header format: {}", split_header.first)); } } else { return absl::NotFoundError("no ORCA data sent from the backend"); diff --git a/source/common/orca/orca_parser.h b/source/common/orca/orca_parser.h index 86fd239440..6c6f455275 100644 --- a/source/common/orca/orca_parser.h +++ b/source/common/orca/orca_parser.h @@ -8,11 +8,24 @@ namespace Envoy { namespace Orca { -// Header used to send ORCA load metrics from the backend. +// Headers used to send ORCA load metrics from the backend. +static constexpr absl::string_view kEndpointLoadMetricsHeader = "endpoint-load-metrics"; static constexpr absl::string_view kEndpointLoadMetricsHeaderBin = "endpoint-load-metrics-bin"; +// Prefix used to determine format expected in kEndpointLoadMetricsHeader. +static constexpr absl::string_view kHeaderFormatPrefixBin = "BIN"; +static constexpr absl::string_view kHeaderFormatPrefixJson = "JSON"; +static constexpr absl::string_view kHeaderFormatPrefixText = "TEXT"; +// The following fields are the names of the metrics tracked in the ORCA load +// report proto. +static constexpr absl::string_view kApplicationUtilizationField = "application_utilization"; +static constexpr absl::string_view kCpuUtilizationField = "cpu_utilization"; +static constexpr absl::string_view kMemUtilizationField = "mem_utilization"; +static constexpr absl::string_view kEpsField = "eps"; +static constexpr absl::string_view kRpsFractionalField = "rps_fractional"; +static constexpr absl::string_view kNamedMetricsFieldPrefix = "named_metrics."; // Parses ORCA load metrics from a header map into an OrcaLoadReport proto. -// Supports serialized binary formats. +// Supports native HTTP, JSON and serialized binary formats. absl::StatusOr parseOrcaLoadReportHeaders(const Envoy::Http::HeaderMap& headers); } // namespace Orca diff --git a/test/common/orca/orca_parser_test.cc b/test/common/orca/orca_parser_test.cc index 6b56f72d55..b86fcc7d31 100644 --- a/test/common/orca/orca_parser_test.cc +++ b/test/common/orca/orca_parser_test.cc @@ -1,4 +1,4 @@ -#include +#include #include "source/common/common/base64.h" #include "source/common/orca/orca_parser.h" @@ -7,12 +7,25 @@ #include "test/test_common/utility.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "xds/data/orca/v3/orca_load_report.pb.h" namespace Envoy { namespace Orca { namespace { +const std::string formattedHeaderPrefixText() { + CONSTRUCT_ON_FIRST_USE(std::string, absl::StrCat(kHeaderFormatPrefixText, " ")); +} + +const std::string formattedHeaderPrefixJson() { + CONSTRUCT_ON_FIRST_USE(std::string, absl::StrCat(kHeaderFormatPrefixJson, " ")); +} + +const std::string formattedHeaderPrefixBin() { + CONSTRUCT_ON_FIRST_USE(std::string, absl::StrCat(kHeaderFormatPrefixBin, " ")); +} + // Returns an example OrcaLoadReport proto with all fields populated. static xds::data::orca::v3::OrcaLoadReport exampleOrcaLoadReport() { xds::data::orca::v3::OrcaLoadReport orca_load_report; @@ -42,7 +55,157 @@ TEST(OrcaParserUtilTest, MissingOrcaHeaders) { StatusHelpers::HasStatus(absl::NotFoundError("no ORCA data sent from the backend"))); } -TEST(OrcaParserUtilTest, BinaryHeader) { +TEST(OrcaParserUtilTest, InvalidOrcaHeaderPrefix) { + // Verify that error is returned when unknown/invalid prefix is found in ORCA + // header value. + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), "BAD random-value"}}; + EXPECT_THAT( + parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError("unsupported ORCA header format: BAD"))); +} + +TEST(OrcaParserUtilTest, EmptyOrcaHeader) { + Http::TestRequestHeaderMapImpl headers{{std::string(kEndpointLoadMetricsHeader), ""}}; + EXPECT_THAT( + parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError("unsupported ORCA header format: "))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), + "cpu_utilization:0.7,application_utilization:0.8,mem_utilization:0.9," + "rps_fractional:1000,eps:2," + "named_metrics.foo:123,named_metrics.bar:0.2")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderIncorrectFieldType) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "cpu_utilization:\"0.7\"")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus( + absl::InvalidArgumentError("unable to parse custom backend load metric " + "value(cpu_utilization): \"0.7\""))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderNanMetricValue) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), + "cpu_utilization:", std::numeric_limits::quiet_NaN())}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError( + "custom backend load metric value(cpu_utilization) cannot be NaN."))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderInfinityMetricValue) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), + "cpu_utilization:", std::numeric_limits::infinity())}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError( + "custom backend load metric value(cpu_utilization) cannot be " + "infinity."))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderContainsDuplicateMetric) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "cpu_utilization:0.7,cpu_utilization:0.8")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::AlreadyExistsError(absl::StrCat( + kEndpointLoadMetricsHeader, " contains duplicate metric: cpu_utilization")))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderUnsupportedMetric) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "cpu_utilization:0.7,unsupported_metric:0.8")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus( + absl::InvalidArgumentError("unsupported metric name: unsupported_metric"))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderContainsDuplicateNamedMetric) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat( + formattedHeaderPrefixText(), + "named_metrics.foo:123,named_metrics.duplicate:123,named_metrics.duplicate:0.2")}}; + EXPECT_THAT( + parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::AlreadyExistsError(absl::StrCat( + kEndpointLoadMetricsHeader, " contains duplicate metric: named_metrics.duplicate")))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderContainsEmptyNamedMetricKey) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "named_metrics.:123")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError("named metric key is empty."))); +} + +TEST(OrcaParserUtilTest, InvalidNativeHttpEncodedHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "not-a-list-of-key-value-pairs")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus( + absl::InvalidArgumentError("metric values cannot be empty strings"))); +} + +TEST(OrcaParserUtilTest, JsonHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), + "{\"cpu_utilization\": 0.7, \"application_utilization\": 0.8, " + "\"mem_utilization\": 0.9, \"rps_fractional\": 1000, \"eps\": 2, " + "\"named_metrics\": {\"foo\": 123,\"bar\": 0.2}}")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + +TEST(OrcaParserUtilTest, InvalidJsonHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), "JSON not-a-valid-json-string")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("invalid JSON"))); +} + +TEST(OrcaParserUtilTest, JsonHeaderUnknownField) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), + "{\"cpu_utilization\": 0.7, \"application_utilization\": 0.8, " + "\"mem_utilization\": 0.9, \"rps_fractional\": 1000, \"eps\": 2, " + "\"unknown_field\": 2," + "\"named_metrics\": {\"foo\": 123,\"bar\": 0.2}}")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("invalid JSON"))); +} + +TEST(OrcaParserUtilTest, JsonHeaderIncorrectFieldType) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), "{\"cpu_utilization\": \"0.7\"")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("invalid JSON"))); +} + +TEST(OrcaParserUtilTest, LegacyBinaryHeader) { + // Verify processing of headers sent in legacy ORCA header inherited from gRPC + // implementation works as intended. const std::string proto_string = TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); const auto orca_load_report_header_bin = @@ -53,6 +216,20 @@ TEST(OrcaParserUtilTest, BinaryHeader) { StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); } +TEST(OrcaParserUtilTest, BinaryHeader) { + // Verify serialized binary header processing when using default ORCA header + // and appropriate format prefix in the header value. + const std::string proto_string = + TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); + const auto orca_load_report_header_bin = + Envoy::Base64::encode(proto_string.c_str(), proto_string.length()); + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixBin(), orca_load_report_header_bin)}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + TEST(OrcaParserUtilTest, InvalidBinaryHeader) { const std::string proto_string = TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); @@ -84,6 +261,20 @@ TEST(OrcaParserUtilTest, EmptyBinaryHeader) { testing::HasSubstr("ORCA binary header value is empty"))); } +TEST(OrcaParserUtilTest, BinHeaderPrecedence) { + // Verifies that the order of precedence (binary proto over native http + // format) is observed when multiple ORCA headers are sent from the backend. + const std::string proto_string = + TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); + const auto orca_load_report_header_bin = + Envoy::Base64::encode(proto_string.c_str(), proto_string.length()); + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), "cpu_utilization:0.7"}, + {std::string(kEndpointLoadMetricsHeaderBin), orca_load_report_header_bin}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + } // namespace } // namespace Orca } // namespace Envoy From e48666365c8a0b3a62343602fd9380d58a7afd95 Mon Sep 17 00:00:00 2001 From: code Date: Sun, 6 Oct 2024 07:27:24 +0800 Subject: [PATCH 11/77] local rate limit: add new rate_limits support to the filter (#36099) Commit Message: local rate limit: add new rate_limits api to the filter's api Additional Description: In the previous local rate limit, the [rate_limits](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-field-config-route-v3-virtualhost-rate-limits) field of route is used to generate the descriptor entries. Then the generated entries will be used to match a token bucket which is configured in the filter configs (route level, vhost level, etc). However, it make the configuration very complex, and cannot cover some common scenarios easily. For example, give a specific virtual host X and a special route Y that under this virtual host X. We want to provides a virtual host level rate limit for the specific virtual host X, and a route level rate limit for the specific route Y. We hope the configuration of virtual host could works for all routes except the Y. For most filters, this requirement could be achieved by getting the most specific filter config and applying it. But for the local rate limit, thing become very complex. Because the rate limit configuration is split into `rate_limits` field of route and the filter config. The local rate limit need to handle these relationship carefully. This PR try to simplify it. Risk Level: low. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. --------- Signed-off-by: wangbaiping Signed-off-by: code Co-authored-by: Matt Klein --- .../filters/http/local_ratelimit/v3/BUILD | 1 + .../local_ratelimit/v3/local_rate_limit.proto | 22 +- changelogs/current.yaml | 8 + source/common/router/router_ratelimit.cc | 62 +- source/common/router/router_ratelimit.h | 31 +- .../filters/common/ratelimit_config/BUILD | 22 + .../ratelimit_config/ratelimit_config.cc | 143 +++ .../ratelimit_config/ratelimit_config.h | 57 + .../filters/http/local_ratelimit/BUILD | 1 + .../filters/http/local_ratelimit/config.cc | 11 +- .../http/local_ratelimit/local_ratelimit.cc | 35 +- .../http/local_ratelimit/local_ratelimit.h | 22 +- .../filters/common/ratelimit_config/BUILD | 37 + .../ratelimit_config/ratelimit_config_test.cc | 1121 +++++++++++++++++ .../ratelimit_config_test.proto | 9 + .../filters/http/local_ratelimit/BUILD | 3 +- .../http/local_ratelimit/filter_test.cc | 86 +- 17 files changed, 1594 insertions(+), 77 deletions(-) create mode 100644 source/extensions/filters/common/ratelimit_config/BUILD create mode 100644 source/extensions/filters/common/ratelimit_config/ratelimit_config.cc create mode 100644 source/extensions/filters/common/ratelimit_config/ratelimit_config.h create mode 100644 test/extensions/filters/common/ratelimit_config/BUILD create mode 100644 test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc create mode 100644 test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD b/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD index 1ef2f0c9bf..ac9fd7c8ab 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD @@ -7,6 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", + "//envoy/config/route/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/type/v3:pkg", "@com_github_cncf_xds//udpa/annotations:pkg", diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto index a32475f352..82e38ed91d 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.local_ratelimit.v3; import "envoy/config/core/v3/base.proto"; +import "envoy/config/route/v3/route_components.proto"; import "envoy/extensions/common/ratelimit/v3/ratelimit.proto"; import "envoy/type/v3/http_status.proto"; import "envoy/type/v3/token_bucket.proto"; @@ -22,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Local Rate limit :ref:`configuration overview `. // [#extension: envoy.filters.http.local_ratelimit] -// [#next-free-field: 17] +// [#next-free-field: 18] message LocalRateLimit { // The human readable prefix to use when emitting stats. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -147,4 +148,23 @@ message LocalRateLimit { // of the default ``UNAVAILABLE`` gRPC code for a rate limited gRPC call. The // HTTP code will be 200 for a gRPC response. bool rate_limited_as_resource_exhausted = 15; + + // Rate limit configuration that is used to generate a list of descriptor entries based on + // the request context. The generated entries will be used to find one or multiple matched rate + // limit rule from the ``descriptors``. + // If this is set, then + // :ref:`VirtualHost.rate_limits` or + // :ref:`RouteAction.rate_limits` fields + // will be ignored. + // + // .. note:: + // Not all configuration fields of + // :ref:`rate limit config ` is supported at here. + // Following fields are not supported: + // + // 1. :ref:`rate limit stage `. + // 2. :ref:`dynamic metadata `. + // 3. :ref:`disable_key `. + // 4. :ref:`override limit `. + repeated config.route.v3.RateLimit rate_limits = 17; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 200f299a12..9b844af994 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -432,6 +432,14 @@ new_features: change: | Added two new methods ``oidsPeerCertificate()`` and ``oidsLocalCertificate()`` to SSL connection object API :ref:`SSL connection info object `. +- area: local_ratelimit + change: | + Add the :ref:`rate_limits + ` + field to generate rate limit descriptors. If this field is set, the + :ref:`VirtualHost.rate_limits` or + :ref:`RouteAction.rate_limits` fields + will be ignored. - area: basic_auth change: | Added support to provide an override diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index db2dcc45e2..77c285a85a 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -12,8 +12,6 @@ #include "source/common/common/empty_string.h" #include "source/common/config/metadata.h" #include "source/common/config/utility.h" -#include "source/common/http/matching/data_impl.h" -#include "source/common/matcher/matcher.h" #include "source/common/protobuf/utility.h" namespace Envoy { @@ -39,44 +37,24 @@ bool populateDescriptor(const std::vector& act return result; } -class RateLimitDescriptorValidationVisitor - : public Matcher::MatchTreeValidationVisitor { -public: - absl::Status performDataInputValidation(const Matcher::DataInputFactory&, - absl::string_view) override { - return absl::OkStatus(); +} // namespace + +// Ratelimit::DescriptorProducer +bool MatchInputRateLimitDescriptor::populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, + const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const { + Http::Matching::HttpMatchingDataImpl data(info); + data.onRequestHeaders(headers); + auto result = data_input_->get(data); + if (!absl::holds_alternative(result.data_)) { + return false; } -}; - -class MatchInputRateLimitDescriptor : public RateLimit::DescriptorProducer { -public: - MatchInputRateLimitDescriptor(const std::string& descriptor_key, - Matcher::DataInputPtr&& data_input) - : descriptor_key_(descriptor_key), data_input_(std::move(data_input)) {} - - // Ratelimit::DescriptorProducer - bool populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, const std::string&, - const Http::RequestHeaderMap& headers, - const StreamInfo::StreamInfo& info) const override { - Http::Matching::HttpMatchingDataImpl data(info); - data.onRequestHeaders(headers); - auto result = data_input_->get(data); - if (absl::holds_alternative(result.data_)) { - return false; - } - const std::string& str = absl::get(result.data_); - if (!str.empty()) { - descriptor_entry = {descriptor_key_, str}; - } - return true; + if (absl::string_view str = absl::get(result.data_); !str.empty()) { + descriptor_entry = {descriptor_key_, std::string(str)}; } - -private: - const std::string descriptor_key_; - Matcher::DataInputPtr data_input_; -}; - -} // namespace + return true; +} const uint64_t RateLimitPolicyImpl::MAX_STAGE_NUMBER = 10UL; @@ -256,7 +234,8 @@ QueryParameterValueMatchAction::QueryParameterValueMatchAction( : descriptor_value_(action.descriptor_value()), descriptor_key_(!action.descriptor_key().empty() ? action.descriptor_key() : "query_match"), expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)), - action_query_parameters_(buildQueryParameterMatcherVector(action, context)) {} + action_query_parameters_( + buildQueryParameterMatcherVector(action.query_parameters(), context)) {} bool QueryParameterValueMatchAction::populateDescriptor( RateLimit::DescriptorEntry& descriptor_entry, const std::string&, @@ -274,10 +253,11 @@ bool QueryParameterValueMatchAction::populateDescriptor( std::vector QueryParameterValueMatchAction::buildQueryParameterMatcherVector( - const envoy::config::route::v3::RateLimit::Action::QueryParameterValueMatch& action, + const Protobuf::RepeatedPtrField& + query_parameters, Server::Configuration::CommonFactoryContext& context) { std::vector ret; - for (const auto& query_parameter : action.query_parameters()) { + for (const auto& query_parameter : query_parameters) { ret.push_back(std::make_unique(query_parameter, context)); } return ret; diff --git a/source/common/router/router_ratelimit.h b/source/common/router/router_ratelimit.h index b069af8d7e..3fb5149a4c 100644 --- a/source/common/router/router_ratelimit.h +++ b/source/common/router/router_ratelimit.h @@ -13,6 +13,8 @@ #include "source/common/config/metadata.h" #include "source/common/http/header_utility.h" +#include "source/common/http/matching/data_impl.h" +#include "source/common/matcher/matcher.h" #include "source/common/network/cidr_range.h" #include "source/common/protobuf/utility.h" #include "source/common/router/config_utility.h" @@ -146,6 +148,7 @@ class MetaDataAction : public RateLimit::DescriptorProducer { MetaDataAction(const envoy::config::route::v3::RateLimit::Action::MetaData& action); // for maintaining backward compatibility with the deprecated DynamicMetaData action MetaDataAction(const envoy::config::route::v3::RateLimit::Action::DynamicMetaData& action); + // Ratelimit::DescriptorProducer bool populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, const std::string& local_service_cluster, @@ -198,7 +201,8 @@ class QueryParameterValueMatchAction : public RateLimit::DescriptorProducer { const StreamInfo::StreamInfo& info) const override; std::vector buildQueryParameterMatcherVector( - const envoy::config::route::v3::RateLimit::Action::QueryParameterValueMatch& action, + const Protobuf::RepeatedPtrField& + query_parameters, Server::Configuration::CommonFactoryContext& context); private: @@ -208,6 +212,31 @@ class QueryParameterValueMatchAction : public RateLimit::DescriptorProducer { const std::vector action_query_parameters_; }; +class RateLimitDescriptorValidationVisitor + : public Matcher::MatchTreeValidationVisitor { +public: + absl::Status performDataInputValidation(const Matcher::DataInputFactory&, + absl::string_view) override { + return absl::OkStatus(); + } +}; + +class MatchInputRateLimitDescriptor : public RateLimit::DescriptorProducer { +public: + MatchInputRateLimitDescriptor(const std::string& descriptor_key, + Matcher::DataInputPtr&& data_input) + : descriptor_key_(descriptor_key), data_input_(std::move(data_input)) {} + + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; + +private: + const std::string descriptor_key_; + Matcher::DataInputPtr data_input_; +}; + /* * Implementation of RateLimitPolicyEntry that holds the action for the configuration. */ diff --git a/source/extensions/filters/common/ratelimit_config/BUILD b/source/extensions/filters/common/ratelimit_config/BUILD new file mode 100644 index 0000000000..a89b1d3676 --- /dev/null +++ b/source/extensions/filters/common/ratelimit_config/BUILD @@ -0,0 +1,22 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "ratelimit_config_lib", + srcs = ["ratelimit_config.cc"], + hdrs = ["ratelimit_config.h"], + deps = [ + "//envoy/ratelimit:ratelimit_interface", + "//source/common/router:router_ratelimit_lib", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/strings", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/common/ratelimit_config/ratelimit_config.cc b/source/extensions/filters/common/ratelimit_config/ratelimit_config.cc new file mode 100644 index 0000000000..c3fec0f4f6 --- /dev/null +++ b/source/extensions/filters/common/ratelimit_config/ratelimit_config.cc @@ -0,0 +1,143 @@ +#include "source/extensions/filters/common/ratelimit_config/ratelimit_config.h" + +#include "source/common/config/utility.h" +#include "source/common/http/matching/data_impl.h" +#include "source/common/matcher/matcher.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { + +RateLimitPolicy::RateLimitPolicy(const ProtoRateLimit& config, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit) { + if (config.has_stage() || !config.disable_key().empty()) { + creation_status = + absl::InvalidArgumentError("'stage' field and 'disable_key' field are not supported"); + return; + } + + if (config.has_limit()) { + if (no_limit) { + creation_status = absl::InvalidArgumentError("'limit' field is not supported"); + return; + } + } + + for (const ProtoRateLimit::Action& action : config.actions()) { + switch (action.action_specifier_case()) { + case ProtoRateLimit::Action::ActionSpecifierCase::kSourceCluster: + actions_.emplace_back(new Envoy::Router::SourceClusterAction()); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kDestinationCluster: + actions_.emplace_back(new Envoy::Router::DestinationClusterAction()); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kRequestHeaders: + actions_.emplace_back(new Envoy::Router::RequestHeadersAction(action.request_headers())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kRemoteAddress: + actions_.emplace_back(new Envoy::Router::RemoteAddressAction()); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kGenericKey: + actions_.emplace_back(new Envoy::Router::GenericKeyAction(action.generic_key())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kMetadata: + actions_.emplace_back(new Envoy::Router::MetaDataAction(action.metadata())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kHeaderValueMatch: + actions_.emplace_back( + new Envoy::Router::HeaderValueMatchAction(action.header_value_match(), context)); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kExtension: { + ProtobufMessage::ValidationVisitor& validator = context.messageValidationVisitor(); + auto* factory = + Envoy::Config::Utility::getFactory( + action.extension()); + if (!factory) { + // If no descriptor extension is found, fallback to using HTTP matcher + // input functions. Note that if the same extension name or type was + // dual registered as an extension descriptor and an HTTP matcher input + // function, the descriptor extension takes priority. + Router::RateLimitDescriptorValidationVisitor validation_visitor; + Matcher::MatchInputFactory input_factory(validator, + validation_visitor); + Matcher::DataInputFactoryCb data_input_cb = + input_factory.createDataInput(action.extension()); + actions_.emplace_back(std::make_unique( + action.extension().name(), data_input_cb())); + break; + } + auto message = Envoy::Config::Utility::translateAnyToFactoryConfig( + action.extension().typed_config(), validator, *factory); + Envoy::RateLimit::DescriptorProducerPtr producer = + factory->createDescriptorProducerFromProto(*message, context); + if (producer) { + actions_.emplace_back(std::move(producer)); + } else { + creation_status = absl::InvalidArgumentError( + absl::StrCat("Rate limit descriptor extension failed: ", action.extension().name())); + return; + } + break; + } + case ProtoRateLimit::Action::ActionSpecifierCase::kMaskedRemoteAddress: + actions_.emplace_back(new Router::MaskedRemoteAddressAction(action.masked_remote_address())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kQueryParameterValueMatch: + actions_.emplace_back(new Router::QueryParameterValueMatchAction( + action.query_parameter_value_match(), context)); + break; + default: + creation_status = absl::InvalidArgumentError(fmt::format( + "Unsupported rate limit action: {}", static_cast(action.action_specifier_case()))); + return; + } + } +} + +void RateLimitPolicy::populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& stream_info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const { + Envoy::RateLimit::LocalDescriptor descriptor; + for (const Envoy::RateLimit::DescriptorProducerPtr& action : actions_) { + Envoy::RateLimit::DescriptorEntry entry; + if (!action->populateDescriptor(entry, local_service_cluster, headers, stream_info)) { + return; + } + if (!entry.key_.empty()) { + descriptor.entries_.emplace_back(std::move(entry)); + } + } + descriptors.emplace_back(std::move(descriptor)); +} + +RateLimitConfig::RateLimitConfig(const Protobuf::RepeatedPtrField& configs, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit) { + for (const ProtoRateLimit& config : configs) { + auto descriptor_generator = + std::make_unique(config, context, creation_status, no_limit); + if (!creation_status.ok()) { + return; + } + rate_limit_policies_.emplace_back(std::move(descriptor_generator)); + } +} + +void RateLimitConfig::populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& stream_info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const { + for (const auto& generator : rate_limit_policies_) { + generator->populateDescriptors(headers, stream_info, local_service_cluster, descriptors); + } +} + +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/ratelimit_config/ratelimit_config.h b/source/extensions/filters/common/ratelimit_config/ratelimit_config.h new file mode 100644 index 0000000000..743e69360c --- /dev/null +++ b/source/extensions/filters/common/ratelimit_config/ratelimit_config.h @@ -0,0 +1,57 @@ +#pragma once + +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/ratelimit/ratelimit.h" + +#include "source/common/router/router_ratelimit.h" + +#include "absl/container/inlined_vector.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { + +using ProtoRateLimit = envoy::config::route::v3::RateLimit; +using RateLimitDescriptors = std::vector; + +class RateLimitPolicy : Logger::Loggable { +public: + RateLimitPolicy(const ProtoRateLimit& config, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit = true); + + void populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const; + +private: + std::vector actions_; +}; + +class RateLimitConfig : Logger::Loggable { +public: + RateLimitConfig(const Protobuf::RepeatedPtrField& configs, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit = true); + + bool empty() const { return rate_limit_policies_.empty(); } + + size_t size() const { return rate_limit_policies_.size(); } + + void populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const; + +private: + std::vector> rate_limit_policies_; +}; + +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/local_ratelimit/BUILD b/source/extensions/filters/http/local_ratelimit/BUILD index 88d4042e98..f20a2b0411 100644 --- a/source/extensions/filters/http/local_ratelimit/BUILD +++ b/source/extensions/filters/http/local_ratelimit/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//source/common/runtime:runtime_lib", "//source/extensions/filters/common/local_ratelimit:local_ratelimit_lib", "//source/extensions/filters/common/ratelimit:ratelimit_lib", + "//source/extensions/filters/common/ratelimit_config:ratelimit_config_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", "//source/extensions/filters/http/common:ratelimit_headers_lib", "@envoy_api//envoy/extensions/common/ratelimit/v3:pkg_cc_proto", diff --git a/source/extensions/filters/http/local_ratelimit/config.cc b/source/extensions/filters/http/local_ratelimit/config.cc index e0990cf459..2228619acc 100644 --- a/source/extensions/filters/http/local_ratelimit/config.cc +++ b/source/extensions/filters/http/local_ratelimit/config.cc @@ -15,12 +15,9 @@ namespace LocalRateLimitFilter { Http::FilterFactoryCb LocalRateLimitFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { - auto& server_context = context.serverFactoryContext(); - FilterConfigSharedPtr filter_config = std::make_shared( - proto_config, server_context.localInfo(), server_context.mainThreadDispatcher(), - server_context.clusterManager(), server_context.singletonManager(), context.scope(), - server_context.runtime()); + FilterConfigSharedPtr filter_config = + std::make_shared(proto_config, context.serverFactoryContext(), context.scope()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; @@ -30,9 +27,7 @@ Router::RouteSpecificFilterConfigConstSharedPtr LocalRateLimitFilterConfig::createRouteSpecificFilterConfigTyped( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& proto_config, Server::Configuration::ServerFactoryContext& context, ProtobufMessage::ValidationVisitor&) { - return std::make_shared( - proto_config, context.localInfo(), context.mainThreadDispatcher(), context.clusterManager(), - context.singletonManager(), context.scope(), context.runtime(), true); + return std::make_shared(proto_config, context, context.scope(), true); } /** diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc index 8356fa49b6..0e12da02bf 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc @@ -23,10 +23,8 @@ const std::string& PerConnectionRateLimiter::key() { FilterConfig::FilterConfig( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& config, - const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, - Upstream::ClusterManager& cm, Singleton::Manager& singleton_manager, Stats::Scope& scope, - Runtime::Loader& runtime, const bool per_route) - : dispatcher_(dispatcher), status_(toErrorCode(config.status().code())), + Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, const bool per_route) + : dispatcher_(context.mainThreadDispatcher()), status_(toErrorCode(config.status().code())), stats_(generateStats(config.stat_prefix(), scope)), fill_interval_(std::chrono::milliseconds( PROTOBUF_GET_MS_OR_DEFAULT(config.token_bucket(), fill_interval, 0))), @@ -38,7 +36,7 @@ FilterConfig::FilterConfig( config.has_always_consume_default_token_bucket() ? config.always_consume_default_token_bucket().value() : true), - local_info_(local_info), runtime_(runtime), + local_info_(context.localInfo()), runtime_(context.runtime()), filter_enabled_( config.has_filter_enabled() ? absl::optional( @@ -73,20 +71,35 @@ FilterConfig::FilterConfig( throw EnvoyException("local rate limit token bucket must be set for per filter configs"); } + absl::Status creation_status; + rate_limit_config_ = std::make_unique( + config.rate_limits(), context, creation_status); + THROW_IF_NOT_OK_REF(creation_status); + + if (rate_limit_config_->empty()) { + if (!config.descriptors().empty()) { + ENVOY_LOG_FIRST_N( + warn, 20, + "'descriptors' is set but only used by route configuration. Please configure the local " + "rate limit filter using the embedded 'rate_limits' field as route configuration for " + "local rate limits will be ignored in the future."); + } + } + Filters::Common::LocalRateLimit::ShareProviderSharedPtr share_provider; if (config.has_local_cluster_rate_limit()) { if (rate_limit_per_connection_) { throw EnvoyException("local_cluster_rate_limit is set and " "local_rate_limit_per_downstream_connection is set to true"); } - if (!cm.localClusterName().has_value()) { + if (!context.clusterManager().localClusterName().has_value()) { throw EnvoyException("local_cluster_rate_limit is set but no local cluster name is present"); } // If the local cluster name is set then the relevant cluster must exist or the cluster // manager will fail to initialize. share_provider_manager_ = Filters::Common::LocalRateLimit::ShareProviderManager::singleton( - dispatcher, cm, singleton_manager); + dispatcher_, context.clusterManager(), context.singletonManager()); if (!share_provider_manager_) { throw EnvoyException("local_cluster_rate_limit is set but no local cluster is present"); } @@ -95,7 +108,7 @@ FilterConfig::FilterConfig( } rate_limiter_ = std::make_unique( - fill_interval_, max_tokens_, tokens_per_fill_, dispatcher, descriptors_, + fill_interval_, max_tokens_, tokens_per_fill_, dispatcher_, descriptors_, always_consume_default_token_bucket_, std::move(share_provider)); } @@ -137,7 +150,11 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, std::vector descriptors; if (used_config_->hasDescriptors()) { - populateDescriptors(descriptors, headers); + if (used_config_->hasRateLimitConfigs()) { + used_config_->populateDescriptors(headers, decoder_callbacks_->streamInfo(), descriptors); + } else { + populateDescriptors(descriptors, headers); + } } if (ENVOY_LOG_CHECK_LEVEL(debug)) { diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h index d23f7228f0..2b3af4ec7f 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h @@ -19,6 +19,7 @@ #include "source/common/runtime/runtime_protos.h" #include "source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.h" #include "source/extensions/filters/common/ratelimit/ratelimit.h" +#include "source/extensions/filters/common/ratelimit_config/ratelimit_config.h" #include "source/extensions/filters/http/common/pass_through_filter.h" namespace Envoy { @@ -69,12 +70,12 @@ class PerConnectionRateLimiter : public StreamInfo::FilterState::Object { /** * Global configuration for the HTTP local rate limit filter. */ -class FilterConfig : public Router::RouteSpecificFilterConfig { +class FilterConfig : public Router::RouteSpecificFilterConfig, + Logger::Loggable { public: FilterConfig(const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& config, - const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, - Upstream::ClusterManager& cm, Singleton::Manager& singleton_manager, - Stats::Scope& scope, Runtime::Loader& runtime, bool per_route = false); + Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, + const bool per_route = false); ~FilterConfig() override { // Ensure that the LocalRateLimiterImpl instance will be destroyed on the thread where its inner // timer is created and running. @@ -113,6 +114,18 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { return rate_limited_grpc_status_; } + bool hasRateLimitConfigs() const { + ASSERT(rate_limit_config_ != nullptr); + return !rate_limit_config_->empty(); + } + + void populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info, + Filters::Common::RateLimit::RateLimitDescriptors& descriptors) const { + ASSERT(rate_limit_config_ != nullptr); + rate_limit_config_->populateDescriptors(headers, info, local_info_.clusterName(), descriptors); + } + private: friend class FilterTest; @@ -150,6 +163,7 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { const bool enable_x_rate_limit_headers_; const envoy::extensions::common::ratelimit::v3::VhRateLimitsOptions vh_rate_limits_; const absl::optional rate_limited_grpc_status_; + std::unique_ptr rate_limit_config_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/common/ratelimit_config/BUILD b/test/extensions/filters/common/ratelimit_config/BUILD new file mode 100644 index 0000000000..bffabf03a6 --- /dev/null +++ b/test/extensions/filters/common/ratelimit_config/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", + "envoy_proto_library", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_proto_library( + name = "ratelimit_config_test_proto", + srcs = ["ratelimit_config_test.proto"], + deps = [ + "@envoy_api//envoy/config/route/v3:pkg", + ], +) + +envoy_cc_test( + name = "ratelimit_config_test", + srcs = ["ratelimit_config_test.cc"], + deps = [ + ":ratelimit_config_test_proto_cc_proto", + "//source/common/http:header_map_lib", + "//source/common/protobuf:utility_lib", + "//source/common/router:config_lib", + "//source/extensions/filters/common/ratelimit_config:ratelimit_config_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/ratelimit:ratelimit_mocks", + "//test/mocks/router:router_mocks", + "//test/mocks/server:instance_mocks", + "//test/test_common:registry_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc new file mode 100644 index 0000000000..aea1832038 --- /dev/null +++ b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc @@ -0,0 +1,1121 @@ +#include +#include +#include + +#include "envoy/config/route/v3/route.pb.h" +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/config/route/v3/route_components.pb.validate.h" + +#include "source/common/http/header_map_impl.h" +#include "source/common/network/address_impl.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/filters/common/ratelimit_config/ratelimit_config.h" + +#include "test/extensions/filters/common/ratelimit_config/ratelimit_config_test.pb.h" +#include "test/extensions/filters/common/ratelimit_config/ratelimit_config_test.pb.validate.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/ratelimit/mocks.h" +#include "test/mocks/router/mocks.h" +#include "test/mocks/server/instance.h" +#include "test/test_common/printers.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { +namespace { + +ProtoRateLimit parseRateLimitFromV3Yaml(const std::string& yaml_string) { + ProtoRateLimit rate_limit; + TestUtility::loadFromYaml(yaml_string, rate_limit); + TestUtility::validate(rate_limit); + return rate_limit; +} + +TEST(BadRateLimitConfiguration, MissingActions) { + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml("{}"), EnvoyException, + "value must contain at least"); +} + +TEST(BadRateLimitConfiguration, ActionsMissingRequiredFields) { + const std::string yaml_one = R"EOF( +actions: +- request_headers: {} + )EOF"; + + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml(yaml_one), EnvoyException, + "value length must be at least"); + + const std::string yaml_two = R"EOF( +actions: +- request_headers: + header_name: test + )EOF"; + + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml(yaml_two), EnvoyException, + "value length must be at least"); + + const std::string yaml_three = R"EOF( +actions: +- request_headers: + descriptor_key: test + )EOF"; + + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml(yaml_three), EnvoyException, + "value length must be at least"); +} + +class RateLimitConfigTest : public testing::Test { +public: + void setupTest(const std::string& yaml) { + test::extensions::filters::common::ratelimit_config::TestRateLimitConfig proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + config_ = std::make_unique( + proto_config.rate_limits(), factory_context_, creation_status_); + stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_)); + } + + NiceMock factory_context_; + ProtobufMessage::NullValidationVisitorImpl any_validation_visitor_; + absl::Status creation_status_{}; + std::unique_ptr config_; + Http::TestRequestHeaderMapImpl headers_; + std::shared_ptr route_{new NiceMock()}; + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv4Instance("10.0.0.1")}; + NiceMock stream_info_; +}; + +TEST_F(RateLimitConfigTest, DisableKeyIsNotAllowed) { + { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + stage: 2 + disable_key: anything + limit: + dynamic_metadata: + metadata_key: + key: key + path: + - key: key + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + EXPECT_FALSE(creation_status_.ok()); + EXPECT_EQ(creation_status_.message(), + "'stage' field and 'disable_key' field are not supported"); + } +} + +TEST_F(RateLimitConfigTest, LimitIsNotAllowed) { + { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + limit: + dynamic_metadata: + metadata_key: + key: key + path: + - key: key + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + EXPECT_FALSE(creation_status_.ok()); + EXPECT_EQ(creation_status_.message(), "'limit' field is not supported"); + } +} + +TEST_F(RateLimitConfigTest, NoAction) { + { + const std::string yaml = R"EOF( +actions: +- {} + )EOF"; + + ProtoRateLimit rate_limit; + TestUtility::loadFromYaml(yaml, rate_limit); + + absl::Status creation_status; + RateLimitPolicy policy(rate_limit, factory_context_, creation_status); + + EXPECT_TRUE(absl::StartsWith(creation_status.message(), "Unsupported rate limit action:")); + } + + { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + - {} + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + + EXPECT_TRUE(absl::StartsWith(creation_status_.message(), "Unsupported rate limit action:")); + } +} + +TEST_F(RateLimitConfigTest, EmptyRateLimit) { + const std::string yaml = R"EOF( +rate_limits: [] + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + + EXPECT_TRUE(config_->empty()); +} + +TEST_F(RateLimitConfigTest, SinglePolicy) { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + + EXPECT_EQ(1U, config_->size()); + + std::vector descriptors; + config_->populateDescriptors(headers_, stream_info_, "", descriptors); + EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), + testing::ContainerEq(descriptors)); +} + +TEST_F(RateLimitConfigTest, MultiplePoliciesAndMultipleActions) { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + - destination_cluster: {} + - actions: + - destination_cluster: {} + )EOF"; + + setupTest(yaml); + + std::vector descriptors; + + config_->populateDescriptors(headers_, stream_info_, "", descriptors); + + EXPECT_THAT(std::vector( + {Envoy::RateLimit::LocalDescriptor{ + {{"remote_address", "10.0.0.1"}, {"destination_cluster", "fake_cluster"}}}, + Envoy::RateLimit::LocalDescriptor{{{"destination_cluster", "fake_cluster"}}}}), + testing::ContainerEq(descriptors)); +} + +class RateLimitPolicyTest : public testing::Test { +public: + void setupTest(const std::string& yaml) { + rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml), + factory_context_, creation_status_); + descriptors_.clear(); + stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_)); + } + + NiceMock factory_context_; + std::unique_ptr rate_limit_entry_; + absl::Status creation_status_{}; + Http::TestRequestHeaderMapImpl headers_; + std::shared_ptr route_{new NiceMock()}; + + std::vector descriptors_; + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv4Instance("10.0.0.1")}; + NiceMock stream_info_; +}; + +class RateLimitPolicyIpv6Test : public testing::Test { +public: + void setupTest(const std::string& yaml) { + absl::Status creation_status; + rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml), + factory_context_, creation_status); + THROW_IF_NOT_OK(creation_status); // NOLINT + descriptors_.clear(); + stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_)); + } + + NiceMock factory_context_; + std::unique_ptr rate_limit_entry_; + Http::TestRequestHeaderMapImpl headers_; + std::vector descriptors_; + std::shared_ptr route_{new NiceMock()}; + + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv6Instance("2001:abcd:ef01:2345:6789:abcd:ef01:234")}; + NiceMock stream_info_; +}; + +TEST_F(RateLimitPolicyTest, RemoteAddress) { + const std::string yaml = R"EOF( +actions: +- remote_address: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MaskedRemoteAddressIpv4Default) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "10.0.0.1/32"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MaskedRemoteAddressIpv4) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: + v4_prefix_mask_len: 16 + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "10.0.0.0/16"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyIpv6Test, MaskedRemoteAddressIpv6Default) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "2001:abcd:ef01:2345:6789:abcd:ef01:234/128"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyIpv6Test, MaskedRemoteAddressIpv6) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: + v6_prefix_mask_len: 64 + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "2001:abcd:ef01:2345::/64"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Verify no descriptor is emitted if remote is a pipe. +TEST_F(RateLimitPolicyTest, PipeAddress) { + const std::string yaml = R"EOF( +actions: +- remote_address: {} + )EOF"; + + setupTest(yaml); + + stream_info_.downstream_connection_info_provider_->setRemoteAddress( + *Network::Address::PipeInstance::create("/hello")); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, SourceService) { + const std::string yaml = R"EOF( +actions: +- source_cluster: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"source_cluster", "service_cluster"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, DestinationService) { + const std::string yaml = R"EOF( +actions: +- destination_cluster: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"destination_cluster", "fake_cluster"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, RequestHeaders) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header-name + descriptor_key: my_header_name + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"my_header_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Validate that a descriptor is added if the missing request header +// has skip_if_absent set to true +TEST_F(RateLimitPolicyTest, RequestHeadersWithSkipIfAbsent) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header-name + descriptor_key: my_header_name + skip_if_absent: false +- request_headers: + header_name: x-header + descriptor_key: my_header + skip_if_absent: true + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"my_header_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Tests if the descriptors are added if one of the headers is missing +// and skip_if_absent is set to default value which is false +TEST_F(RateLimitPolicyTest, RequestHeadersWithDefaultSkipIfAbsent) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header-name + descriptor_key: my_header_name + skip_if_absent: false +- request_headers: + header_name: x-header + descriptor_key: my_header + skip_if_absent: false + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{"x-header-test", "test_value"}}; + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, RequestHeadersNoMatch) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header + descriptor_key: my_header_name + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, RateLimitKey) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_value: fake_key + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"generic_key", "fake_key"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, GenericKeyWithSetDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: fake_key + descriptor_value: fake_value + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, GenericKeyWithEmptyDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: "" + descriptor_value: fake_value + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"generic_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataMatchDynamicSourceByDefault) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataMatchDynamicSource) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + source: DYNAMIC + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataMatchRouteEntrySource) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + source: ROUTE_ENTRY + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, route_->metadata_); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Tests that the default_value is used in the descriptor when the metadata_key is empty. +TEST_F(RateLimitPolicyTest, MetaDataNoMatchWithDefaultValue) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataNoMatch) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, MetaDataEmptyValue) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: "" + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +// Tests that no descriptors are generated when both the metadata_key and default_value are empty. +TEST_F(RateLimitPolicyTest, MetaDataAndDefaultValueEmpty) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: fake_key + descriptor_value: fake_value +- metadata: + descriptor_key: fake_key + default_value: "" + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: "" + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +// Tests that no descriptor is generated when both the metadata_key and default_value are empty, +// and skip_if_absent is set to true. +TEST_F(RateLimitPolicyTest, MetaDataAndDefaultValueEmptySkipIfAbsent) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: fake_key + descriptor_value: fake_value +- metadata: + descriptor_key: fake_key + default_value: "" + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + skip_if_absent: true + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: "" + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataNonStringNoMatch) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: + foo: bar + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatch) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"header_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_key: fake_key + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchNoMatch) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "not_same_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchHeadersNotPresent) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + expect_match: false + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "not_same_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"header_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchHeadersPresent) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + expect_match: false + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatch) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=test_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"query_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_key: fake_key + descriptor_value: fake_value + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=test_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchNoMatch) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=not_same_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchExpectNoMatch) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + expect_match: false + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=not_same_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"query_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchExpectNoMatchFailed) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + expect_match: false + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=test_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, CompoundActions) { + const std::string yaml = R"EOF( +actions: +- destination_cluster: {} +- source_cluster: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector( + {{{{"destination_cluster", "fake_cluster"}, {"source_cluster", "service_cluster"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, CompoundActionsNoDescriptor) { + const std::string yaml = R"EOF( +actions: +- destination_cluster: {} +- header_value_match: + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +const std::string RequestHeaderMatchInputDescriptor = R"EOF( +actions: +- extension: + name: my_header_name + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: x-header-name + )EOF"; + +TEST_F(RateLimitPolicyTest, RequestMatchInput) { + setupTest(RequestHeaderMatchInputDescriptor); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"my_header_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, RequestMatchInputEmpty) { + setupTest(RequestHeaderMatchInputDescriptor); + headers_.setCopy(Http::LowerCaseString("x-header-name"), ""); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_FALSE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, RequestMatchInputSkip) { + setupTest(RequestHeaderMatchInputDescriptor); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +class ExtensionDescriptorFactory : public Envoy::RateLimit::DescriptorProducerFactory { +public: + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + std::string name() const override { return "test.descriptor_producer"; } + + Envoy::RateLimit::DescriptorProducerPtr + createDescriptorProducerFromProto(const Protobuf::Message&, + Server::Configuration::CommonFactoryContext&) override { + return return_valid_producer_ ? std::make_unique() : nullptr; + } + bool return_valid_producer_{true}; +}; + +TEST_F(RateLimitPolicyTest, ExtensionDescriptorProducer) { + const std::string ExtensionDescriptor = R"EOF( +actions: +- extension: + name: test.descriptor_producer + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + key: value + )EOF"; + + { + ExtensionDescriptorFactory factory; + Registry::InjectFactory registration(factory); + + setupTest(ExtensionDescriptor); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + EXPECT_THAT( + std::vector({{{{"source_cluster", "service_cluster"}}}}), + testing::ContainerEq(descriptors_)); + } + + { + ExtensionDescriptorFactory factory; + factory.return_valid_producer_ = false; + Registry::InjectFactory registration(factory); + + setupTest(ExtensionDescriptor); + + EXPECT_TRUE( + absl::StartsWith(creation_status_.message(), "Rate limit descriptor extension failed:")); + } +} + +} // namespace +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto new file mode 100644 index 0000000000..08d48af9cb --- /dev/null +++ b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package test.extensions.filters.common.ratelimit_config; + +import "envoy/config/route/v3/route_components.proto"; + +message TestRateLimitConfig { + repeated envoy.config.route.v3.RateLimit rate_limits = 1; +} diff --git a/test/extensions/filters/http/local_ratelimit/BUILD b/test/extensions/filters/http/local_ratelimit/BUILD index bb876c07f7..e71f31db08 100644 --- a/test/extensions/filters/http/local_ratelimit/BUILD +++ b/test/extensions/filters/http/local_ratelimit/BUILD @@ -22,8 +22,7 @@ envoy_extension_cc_test( "//source/extensions/filters/http/local_ratelimit:local_ratelimit_lib", "//test/common/stream_info:test_util", "//test/mocks/http:http_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/upstream:cluster_manager_mocks", + "//test/mocks/server:server_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/filters/http/local_ratelimit/v3:pkg_cc_proto", diff --git a/test/extensions/filters/http/local_ratelimit/filter_test.cc b/test/extensions/filters/http/local_ratelimit/filter_test.cc index 4495dcaf7a..cfecb31560 100644 --- a/test/extensions/filters/http/local_ratelimit/filter_test.cc +++ b/test/extensions/filters/http/local_ratelimit/filter_test.cc @@ -4,8 +4,7 @@ #include "source/extensions/filters/http/local_ratelimit/local_ratelimit.h" #include "test/mocks/http/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/upstream/cluster_manager.h" +#include "test/mocks/server/mocks.h" #include "test/test_common/test_runtime.h" #include "test/test_common/thread_factory_for_test.h" @@ -64,12 +63,12 @@ class FilterTest : public testing::Test { void setupPerRoute(const std::string& yaml, const bool enabled = true, const bool enforced = true, const bool per_route = false) { EXPECT_CALL( - runtime_.snapshot_, + factory_context_.runtime_loader_.snapshot_, featureEnabled(absl::string_view("test_enabled"), testing::Matcher(Percent(100)))) .WillRepeatedly(testing::Return(enabled)); EXPECT_CALL( - runtime_.snapshot_, + factory_context_.runtime_loader_.snapshot_, featureEnabled(absl::string_view("test_enforced"), testing::Matcher(Percent(100)))) .WillRepeatedly(testing::Return(enforced)); @@ -80,8 +79,7 @@ class FilterTest : public testing::Test { envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit config; TestUtility::loadFromYaml(yaml, config); config_ = - std::make_shared(config, local_info_, dispatcher_, cm_, singleton_manager_, - *stats_.rootScope(), runtime_, per_route); + std::make_shared(config, factory_context_, *stats_.rootScope(), per_route); filter_ = std::make_shared(config_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); @@ -103,10 +101,8 @@ class FilterTest : public testing::Test { testing::NiceMock decoder_callbacks_; testing::NiceMock decoder_callbacks_2_; NiceMock dispatcher_; - NiceMock runtime_; - NiceMock local_info_; - NiceMock cm_; - Singleton::ManagerImpl singleton_manager_; + + NiceMock factory_context_; std::shared_ptr config_; std::shared_ptr filter_; @@ -115,7 +111,7 @@ class FilterTest : public testing::Test { TEST_F(FilterTest, Runtime) { setup(fmt::format(config_yaml, "false", "1", "false", "\"OFF\""), false, false); - EXPECT_EQ(&runtime_, &(config_->runtime())); + EXPECT_EQ(&factory_context_.runtime_loader_, &(config_->runtime())); } TEST_F(FilterTest, ToErrorCode) { @@ -400,6 +396,56 @@ enable_x_ratelimit_headers: {} stage: {} )"; +static constexpr absl::string_view inlined_descriptor_config_yaml = R"( +stat_prefix: test +token_bucket: + max_tokens: {} + tokens_per_fill: 1 + fill_interval: 60s +filter_enabled: + runtime_key: test_enabled + default_value: + numerator: 100 + denominator: HUNDRED +filter_enforced: + runtime_key: test_enforced + default_value: + numerator: 100 + denominator: HUNDRED +response_headers_to_add: + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-test-rate-limit + value: 'true' +enable_x_ratelimit_headers: {} +descriptors: +- entries: + - key: hello + value: world + - key: foo + value: bar + token_bucket: + max_tokens: 10 + tokens_per_fill: 10 + fill_interval: 60s +- entries: + - key: foo2 + value: bar2 + token_bucket: + max_tokens: {} + tokens_per_fill: 1 + fill_interval: 60s +rate_limits: +- actions: + - header_value_match: + descriptor_key: foo2 + descriptor_value: bar2 + headers: + - name: x-header-name + string_match: + exact: test_value + )"; + static constexpr absl::string_view consume_default_token_config_yaml = R"( stat_prefix: test token_bucket: @@ -938,6 +984,24 @@ TEST_F(DescriptorFilterTest, IncludeVirtualHostRateLimitsSetTrue) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); } +TEST_F(DescriptorFilterTest, UseInlinedRateLimitConfig) { + setUpTest(fmt::format(inlined_descriptor_config_yaml, "10", R"("OFF")", "1")); + + auto headers = Http::TestRequestHeaderMapImpl(); + // Requests will not be blocked because the requests don't match any descriptor and + // the global token bucket has enough tokens. + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + + headers.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + // Only one request is allowed in 60s for the matched request. + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); +} + } // namespace LocalRateLimitFilter } // namespace HttpFilters } // namespace Extensions From c0d332e106084efb7d89bf87f1455e0caa84ef76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:38:29 +0100 Subject: [PATCH 12/77] build(deps): bump yarl from 1.13.0 to 1.13.1 in /tools/base (#36376) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 186 ++++++++++++++++++------------------ 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 505741eccd..91b6fbec51 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1515,99 +1515,99 @@ yapf==0.40.2 \ # via # -r requirements.in # envoy-code-check -yarl==1.13.0 \ - --hash=sha256:01953b5686e5868fd0d8eaea4e484482c158597b8ddb9d9d4d048303fa3334c7 \ - --hash=sha256:01d3941d416e71ce65f33393beb50e93c1c9e8e516971b6653c96df6eb599a2c \ - --hash=sha256:02f117a63d11c8c2ada229029f8bb444a811e62e5041da962de548f26ac2c40f \ - --hash=sha256:0675a9cf65176e11692b20a516d5f744849251aa24024f422582d2d1bf7c8c82 \ - --hash=sha256:08037790f973367431b9406a7b9d940e872cca12e081bce3b7cea068daf81f0a \ - --hash=sha256:08a3b0b8d10092dade46424fe775f2c9bc32e5a985fdd6afe410fe28598db6b2 \ - --hash=sha256:08d15aff3477fecb7a469d1fdf5939a686fbc5a16858022897d3e9fc99301f19 \ - --hash=sha256:0b489858642e4e92203941a8fdeeb6373c0535aa986200b22f84d4b39cd602ba \ - --hash=sha256:0bd3caf554a52da78ec08415ebedeb6b9636436ca2afda9b5b9ff4a533478940 \ - --hash=sha256:0db15ce35dfd100bc9ab40173f143fbea26c84d7458d63206934fe5548fae25d \ - --hash=sha256:1129737da2291c9952a93c015e73583dd66054f3ae991c8674f6e39c46d95dd3 \ - --hash=sha256:12c92576633027f297c26e52aba89f6363b460f483d85cf7c14216eb55d06d02 \ - --hash=sha256:13a9cd39e47ca4dc25139d3c63fe0dc6acf1b24f9d94d3b5197ac578fbfd84bf \ - --hash=sha256:144b9e9164f21da81731c970dbda52245b343c0f67f3609d71013dd4d0db9ebf \ - --hash=sha256:1932c7bfa537f89ad5ca3d1e7e05de3388bb9e893230a384159fb974f6e9f90c \ - --hash=sha256:24b78a1f57780eeeb17f5e1be851ab9fa951b98811e1bb4b5a53f74eec3e2666 \ - --hash=sha256:26214b0a9b8f4b7b04e67eee94a82c9b4e5c721f4d1ce7e8c87c78f0809b7684 \ - --hash=sha256:2842a89b697d8ca3dda6a25b4e4d835d14afe25a315c8a79dbdf5f70edfd0960 \ - --hash=sha256:2cec7b52903dcf9008311167036775346dcb093bb15ed7ec876debc3095e7dab \ - --hash=sha256:2d7a44ae252efb0fcd79ac0997416721a44345f53e5aec4a24f489d983aa00e3 \ - --hash=sha256:2f6f4a352d0beea5dd39315ab16fc26f0122d13457a7e65ad4f06c7961dcf87a \ - --hash=sha256:31748bee7078db26008bf94d39693c682a26b5c3a80a67194a4c9c8fe3b5cf47 \ - --hash=sha256:33e2f5ef965e69a1f2a1b0071a70c4616157da5a5478f3c2f6e185e06c56a322 \ - --hash=sha256:3590ed9c7477059aea067a58ec87b433bbd47a2ceb67703b1098cca1ba075f0d \ - --hash=sha256:3628e4e572b1db95285a72c4be102356f2dfc6214d9f126de975fd51b517ae55 \ - --hash=sha256:37049eb26d637a5b2f00562f65aad679f5d231c4c044edcd88320542ad66a2d9 \ - --hash=sha256:38a3b742c923fe2cab7d2e2c67220d17da8d0433e8bfe038356167e697ef5524 \ - --hash=sha256:3a9b2650425b2ab9cc68865978963601b3c2414e1d94ef04f193dd5865e1bd79 \ - --hash=sha256:3ff602aa84420b301c083ae7f07df858ae8e371bf3be294397bda3e0b27c6290 \ - --hash=sha256:419c22b419034b4ee3ba1c27cbbfef01ca8d646f9292f614f008093143334cdc \ - --hash=sha256:4483680e129b2a4250be20947b554cd5f7140fa9e5a1e4f1f42717cf91f8676a \ - --hash=sha256:49bee8c99586482a238a7b2ec0ef94e5f186bfdbb8204d14a3dd31867b3875ce \ - --hash=sha256:4c73e0f8375b75806b8771890580566a2e6135e6785250840c4f6c45b69eb72d \ - --hash=sha256:4ffd8a9758b5df7401a49d50e76491f4c582cf7350365439563062cdff45bf16 \ - --hash=sha256:517f9d90ca0224bb7002266eba6e70d8fcc8b1d0c9321de2407e41344413ed46 \ - --hash=sha256:52b7bb09bb48f7855d574480e2efc0c30d31cab4e6ffc6203e2f7ffbf2e4496a \ - --hash=sha256:5378cb60f4209505f6aa60423c174336bd7b22e0d8beb87a2a99ad50787f1341 \ - --hash=sha256:5540b4896b244a6539f22b613b32b5d1b737e08011aa4ed56644cb0519d687df \ - --hash=sha256:561a5f6c054927cf5a993dd7b032aeebc10644419e65db7dd6bdc0b848806e65 \ - --hash=sha256:580fdb2ea48a40bcaa709ee0dc71f64e7a8f23b44356cc18cd9ce55dc3bc3212 \ - --hash=sha256:6072ff51eeb7938ecac35bf24fc465be00e75217eaa1ffad3cc7620accc0f6f4 \ - --hash=sha256:612bd8d2267558bea36347e4e6e3a96f436bdc5c011f1437824be4f2e3abc5e1 \ - --hash=sha256:66c028066be36d54e7a0a38e832302b23222e75db7e65ed862dc94effc8ef062 \ - --hash=sha256:73777f145cd591e1377bf8d8a97e5f8e39c9742ad0f100c898bba1f963aef662 \ - --hash=sha256:784d6e50ea96b3bbb078eb7b40d8c0e3674c2f12da4f0061f889b2cfdbab8f37 \ - --hash=sha256:79de5f8432b53d1261d92761f71dfab5fc7e1c75faa12a3535c27e681dacfa9d \ - --hash=sha256:801fb5dfc05910cd5ef4806726e2129d8c9a16cdfa26a8166697da0861e59dfc \ - --hash=sha256:8986fa2be78193dc8b8c27bd0d3667fe612f7232844872714c4200499d5225ca \ - --hash=sha256:8a67f20e97462dee8a89e9b997a78932959d2ed991e8f709514cb4160143e7b1 \ - --hash=sha256:8ab16c9e94726fdfcbf5b37a641c9d9d0b35cc31f286a2c3b9cad6451cb53b2b \ - --hash=sha256:91251614cca1ba4ab0507f1ba5f5a44e17a5e9a4c7f0308ea441a994bdac3fc7 \ - --hash=sha256:92a26956d268ad52bd2329c2c674890fe9e8669b41d83ed136e7037b1a29808e \ - --hash=sha256:92abbe37e3fb08935e0e95ac5f83f7b286a6f2575f542225ec7afde405ed1fa1 \ - --hash=sha256:9671d0d65f86e0a0eee59c5b05e381c44e3d15c36c2a67da247d5d82875b4e4e \ - --hash=sha256:9d2845f1a37438a8e11e4fcbbf6ffd64bc94dc9cb8c815f72d0eb6f6c622deb0 \ - --hash=sha256:a9a1a600e8449f3a24bc7dca513be8d69db173fe842e8332a7318b5b8757a6af \ - --hash=sha256:aa187a8599e0425f26b25987d884a8b67deb5565f1c450c3a6e8d3de2cdc8715 \ - --hash=sha256:aaf10e525e461f43831d82149d904f35929d89f3ccd65beaf7422aecd500dd39 \ - --hash=sha256:ab3ee57b25ce15f79ade27b7dfb5e678af26e4b93be5a4e22655acd9d40b81ba \ - --hash=sha256:acf27399c94270103d68f86118a183008d601e4c2c3a7e98dcde0e3b0163132f \ - --hash=sha256:acf8c219a59df22609cfaff4a7158a0946f273e3b03a5385f1fdd502496f0cff \ - --hash=sha256:b536c2ac042add7f276d4e5857b08364fc32f28e02add153f6f214de50f12d07 \ - --hash=sha256:b9648e5ae280babcac867b16e845ce51ed21f8c43bced2ca40cff7eee983d6d4 \ - --hash=sha256:bcb374db7a609484941c01380c1450728ec84d9c3e68cd9a5feaecb52626c4be \ - --hash=sha256:be828e92ae67a21d6a252aecd65668dddbf3bb5d5278660be607647335001119 \ - --hash=sha256:bf7a9b31729b97985d4a796808859dfd0e37b55f1ca948d46a568e56e51dd8fb \ - --hash=sha256:bff0d468664cdf7b2a6bfd5e17d4a7025edb52df12e0e6e17223387b421d425c \ - --hash=sha256:c2518660bd8166e770b76ce92514b491b8720ae7e7f5f975cd888b1592894d2c \ - --hash=sha256:c7d35ff2a5a51bc6d40112cdb4ca3fd9636482ce8c6ceeeee2301e34f7ed7556 \ - --hash=sha256:ca71238af0d247d07747cb7202a9359e6e1d6d9e277041e1ad2d9f36b3a111a6 \ - --hash=sha256:cdcdd49136d423ee5234c9360eae7063d3120a429ee984d7d9da821c012da4d7 \ - --hash=sha256:cf4f3a87bd52f8f33b0155cd0f6f22bdf2092d88c6c6acbb1aee3bc206ecbe35 \ - --hash=sha256:d04ea92a3643a9bb28aa6954fff718342caab2cc3d25d0160fe16e26c4a9acb7 \ - --hash=sha256:d42227711a4180d0c22cec30fd81d263d7bb378389d8e70b5f4c597e8abae202 \ - --hash=sha256:d78ebad57152d301284761b03a708aeac99c946a64ba967d47cbcc040e36688b \ - --hash=sha256:d807417ceebafb7ce18085a1205d28e8fcb1435a43197d7aa3fab98f5bfec5ef \ - --hash=sha256:d95fcc9508390db73a0f1c7e78d9a1b1a3532a3f34ceff97c0b3b04140fbe6e4 \ - --hash=sha256:db463fce425f935eee04a9182c74fdf9ed90d3bd2079d4a17f8fb7a2d7c11009 \ - --hash=sha256:db90702060b1cdb7c7609d04df5f68a12fd5581d013ad379e58e0c2e651d92b8 \ - --hash=sha256:de97ee57e00a82ebb8c378fc73c5d9a773e4c2cec8079ff34ebfef61c8ba5b11 \ - --hash=sha256:deb70c006be548076d628630aad9a3ef3a1b2c28aaa14b395cf0939b9124252e \ - --hash=sha256:e3b4293f02129cc2f5068f3687ef294846a79c9d19fabaa9bfdfeeebae11c001 \ - --hash=sha256:e480a12cec58009eeaeee7f48728dc8f629f8e0f280d84957d42c361969d84da \ - --hash=sha256:e4dddf99a853b3f60f3ce6363fb1ad94606113743cf653f116a38edd839a4461 \ - --hash=sha256:e5462756fb34c884ca9d4875b6d2ec80957a767123151c467c97a9b423617048 \ - --hash=sha256:e557e2681b47a0ecfdfbea44743b3184d94d31d5ce0e4b13ff64ce227a40f86e \ - --hash=sha256:ebb2236f8098205f59774a28e25a84601a4beb3e974157d418ee6c470d73e0dc \ - --hash=sha256:f3ef76df654f3547dcb76ba550f9ca59826588eecc6bd7df16937c620df32060 \ - --hash=sha256:f603216d62e9680bfac7fb168ef9673fd98abbb50c43e73d97615dfa1afebf57 \ - --hash=sha256:f997004ff530b5381290e82b212a93bd420fefe5a605872dc16fc7e4a7f4251e \ - --hash=sha256:fda4404bbb6f91e327827f4483d52fe24f02f92de91217954cf51b1cb9ee9c41 \ - --hash=sha256:fe6946c3cbcfbed67c5e50dae49baff82ad054aaa10ff7a4db8dfac646b7b479 +yarl==1.13.1 \ + --hash=sha256:08d7148ff11cb8e886d86dadbfd2e466a76d5dd38c7ea8ebd9b0e07946e76e4b \ + --hash=sha256:098b870c18f1341786f290b4d699504e18f1cd050ed179af8123fd8232513424 \ + --hash=sha256:11b3ca8b42a024513adce810385fcabdd682772411d95bbbda3b9ed1a4257644 \ + --hash=sha256:1891d69a6ba16e89473909665cd355d783a8a31bc84720902c5911dbb6373465 \ + --hash=sha256:1bbb418f46c7f7355084833051701b2301092e4611d9e392360c3ba2e3e69f88 \ + --hash=sha256:1d0828e17fa701b557c6eaed5edbd9098eb62d8838344486248489ff233998b8 \ + --hash=sha256:1d8e3ca29f643dd121f264a7c89f329f0fcb2e4461833f02de6e39fef80f89da \ + --hash=sha256:1fa56f34b2236f5192cb5fceba7bbb09620e5337e0b6dfe2ea0ddbd19dd5b154 \ + --hash=sha256:216a6785f296169ed52cd7dcdc2612f82c20f8c9634bf7446327f50398732a51 \ + --hash=sha256:22b739f99c7e4787922903f27a892744189482125cc7b95b747f04dd5c83aa9f \ + --hash=sha256:2430cf996113abe5aee387d39ee19529327205cda975d2b82c0e7e96e5fdabdc \ + --hash=sha256:269c201bbc01d2cbba5b86997a1e0f73ba5e2f471cfa6e226bcaa7fd664b598d \ + --hash=sha256:298c1eecfd3257aa16c0cb0bdffb54411e3e831351cd69e6b0739be16b1bdaa8 \ + --hash=sha256:2a93a4557f7fc74a38ca5a404abb443a242217b91cd0c4840b1ebedaad8919d4 \ + --hash=sha256:2b2442a415a5f4c55ced0fade7b72123210d579f7d950e0b5527fc598866e62c \ + --hash=sha256:2db874dd1d22d4c2c657807562411ffdfabec38ce4c5ce48b4c654be552759dc \ + --hash=sha256:309c104ecf67626c033845b860d31594a41343766a46fa58c3309c538a1e22b2 \ + --hash=sha256:31497aefd68036d8e31bfbacef915826ca2e741dbb97a8d6c7eac66deda3b606 \ + --hash=sha256:373f16f38721c680316a6a00ae21cc178e3a8ef43c0227f88356a24c5193abd6 \ + --hash=sha256:396e59b8de7e4d59ff5507fb4322d2329865b909f29a7ed7ca37e63ade7f835c \ + --hash=sha256:3bb83a0f12701c0b91112a11148b5217617982e1e466069d0555be9b372f2734 \ + --hash=sha256:3de86547c820e4f4da4606d1c8ab5765dd633189791f15247706a2eeabc783ae \ + --hash=sha256:3fdbf0418489525231723cdb6c79e7738b3cbacbaed2b750cb033e4ea208f220 \ + --hash=sha256:40c6e73c03a6befb85b72da213638b8aaa80fe4136ec8691560cf98b11b8ae6e \ + --hash=sha256:44a4c40a6f84e4d5955b63462a0e2a988f8982fba245cf885ce3be7618f6aa7d \ + --hash=sha256:44b07e1690f010c3c01d353b5790ec73b2f59b4eae5b0000593199766b3f7a5c \ + --hash=sha256:45d23c4668d4925688e2ea251b53f36a498e9ea860913ce43b52d9605d3d8177 \ + --hash=sha256:45f209fb4bbfe8630e3d2e2052535ca5b53d4ce2d2026bed4d0637b0416830da \ + --hash=sha256:4afdf84610ca44dcffe8b6c22c68f309aff96be55f5ea2fa31c0c225d6b83e23 \ + --hash=sha256:4feaaa4742517eaceafcbe74595ed335a494c84634d33961214b278126ec1485 \ + --hash=sha256:576365c9f7469e1f6124d67b001639b77113cfd05e85ce0310f5f318fd02fe85 \ + --hash=sha256:5820bd4178e6a639b3ef1db8b18500a82ceab6d8b89309e121a6859f56585b05 \ + --hash=sha256:5989a38ba1281e43e4663931a53fbf356f78a0325251fd6af09dd03b1d676a09 \ + --hash=sha256:5a9bacedbb99685a75ad033fd4de37129449e69808e50e08034034c0bf063f99 \ + --hash=sha256:5b66c87da3c6da8f8e8b648878903ca54589038a0b1e08dde2c86d9cd92d4ac9 \ + --hash=sha256:5c5e32fef09ce101fe14acd0f498232b5710effe13abac14cd95de9c274e689e \ + --hash=sha256:658e8449b84b92a4373f99305de042b6bd0d19bf2080c093881e0516557474a5 \ + --hash=sha256:6a2acde25be0cf9be23a8f6cbd31734536a264723fca860af3ae5e89d771cd71 \ + --hash=sha256:6a5185ad722ab4dd52d5fb1f30dcc73282eb1ed494906a92d1a228d3f89607b0 \ + --hash=sha256:6b7f6e699304717fdc265a7e1922561b02a93ceffdaefdc877acaf9b9f3080b8 \ + --hash=sha256:703b0f584fcf157ef87816a3c0ff868e8c9f3c370009a8b23b56255885528f10 \ + --hash=sha256:7055bbade838d68af73aea13f8c86588e4bcc00c2235b4b6d6edb0dbd174e246 \ + --hash=sha256:78f271722423b2d4851cf1f4fa1a1c4833a128d020062721ba35e1a87154a049 \ + --hash=sha256:7addd26594e588503bdef03908fc207206adac5bd90b6d4bc3e3cf33a829f57d \ + --hash=sha256:81bad32c8f8b5897c909bf3468bf601f1b855d12f53b6af0271963ee67fff0d2 \ + --hash=sha256:82e692fb325013a18a5b73a4fed5a1edaa7c58144dc67ad9ef3d604eccd451ad \ + --hash=sha256:84bbcdcf393139f0abc9f642bf03f00cac31010f3034faa03224a9ef0bb74323 \ + --hash=sha256:86c438ce920e089c8c2388c7dcc8ab30dfe13c09b8af3d306bcabb46a053d6f7 \ + --hash=sha256:8be8cdfe20787e6a5fcbd010f8066227e2bb9058331a4eccddec6c0db2bb85b2 \ + --hash=sha256:8c723c91c94a3bc8033dd2696a0f53e5d5f8496186013167bddc3fb5d9df46a3 \ + --hash=sha256:8ca53632007c69ddcdefe1e8cbc3920dd88825e618153795b57e6ebcc92e752a \ + --hash=sha256:8f722f30366474a99745533cc4015b1781ee54b08de73260b2bbe13316079851 \ + --hash=sha256:942c80a832a79c3707cca46bd12ab8aa58fddb34b1626d42b05aa8f0bcefc206 \ + --hash=sha256:94a993f976cdcb2dc1b855d8b89b792893220db8862d1a619efa7451817c836b \ + --hash=sha256:95c6737f28069153c399d875317f226bbdea939fd48a6349a3b03da6829fb550 \ + --hash=sha256:9915300fe5a0aa663c01363db37e4ae8e7c15996ebe2c6cce995e7033ff6457f \ + --hash=sha256:9a18595e6a2ee0826bf7dfdee823b6ab55c9b70e8f80f8b77c37e694288f5de1 \ + --hash=sha256:9c8854b9f80693d20cec797d8e48a848c2fb273eb6f2587b57763ccba3f3bd4b \ + --hash=sha256:9cec42a20eae8bebf81e9ce23fb0d0c729fc54cf00643eb251ce7c0215ad49fe \ + --hash=sha256:9d2e1626be8712333a9f71270366f4a132f476ffbe83b689dd6dc0d114796c74 \ + --hash=sha256:9d74f3c335cfe9c21ea78988e67f18eb9822f5d31f88b41aec3a1ec5ecd32da5 \ + --hash=sha256:9fb4134cc6e005b99fa29dbc86f1ea0a298440ab6b07c6b3ee09232a3b48f495 \ + --hash=sha256:a0ae6637b173d0c40b9c1462e12a7a2000a71a3258fa88756a34c7d38926911c \ + --hash=sha256:a31d21089894942f7d9a8df166b495101b7258ff11ae0abec58e32daf8088813 \ + --hash=sha256:a3442c31c11088e462d44a644a454d48110f0588de830921fd201060ff19612a \ + --hash=sha256:ab9524e45ee809a083338a749af3b53cc7efec458c3ad084361c1dbf7aaf82a2 \ + --hash=sha256:b1481c048fe787f65e34cb06f7d6824376d5d99f1231eae4778bbe5c3831076d \ + --hash=sha256:b8c837ab90c455f3ea8e68bee143472ee87828bff19ba19776e16ff961425b57 \ + --hash=sha256:bbf2c3f04ff50f16404ce70f822cdc59760e5e2d7965905f0e700270feb2bbfc \ + --hash=sha256:bbf9c2a589be7414ac4a534d54e4517d03f1cbb142c0041191b729c2fa23f320 \ + --hash=sha256:bcd5bf4132e6a8d3eb54b8d56885f3d3a38ecd7ecae8426ecf7d9673b270de43 \ + --hash=sha256:c14c16831b565707149c742d87a6203eb5597f4329278446d5c0ae7a1a43928e \ + --hash=sha256:c49f3e379177f4477f929097f7ed4b0622a586b0aa40c07ac8c0f8e40659a1ac \ + --hash=sha256:c92b89bffc660f1274779cb6fbb290ec1f90d6dfe14492523a0667f10170de26 \ + --hash=sha256:cd66152561632ed4b2a9192e7f8e5a1d41e28f58120b4761622e0355f0fe034c \ + --hash=sha256:cf1ad338620249f8dd6d4b6a91a69d1f265387df3697ad5dc996305cf6c26fb2 \ + --hash=sha256:d07b52c8c450f9366c34aa205754355e933922c79135125541daae6cbf31c799 \ + --hash=sha256:d0d12fe78dcf60efa205e9a63f395b5d343e801cf31e5e1dda0d2c1fb618073d \ + --hash=sha256:d4ee1d240b84e2f213565f0ec08caef27a0e657d4c42859809155cf3a29d1735 \ + --hash=sha256:d959fe96e5c2712c1876d69af0507d98f0b0e8d81bee14cfb3f6737470205419 \ + --hash=sha256:dcaef817e13eafa547cdfdc5284fe77970b891f731266545aae08d6cce52161e \ + --hash=sha256:df4e82e68f43a07735ae70a2d84c0353e58e20add20ec0af611f32cd5ba43fb4 \ + --hash=sha256:ec8cfe2295f3e5e44c51f57272afbd69414ae629ec7c6b27f5a410efc78b70a0 \ + --hash=sha256:ec9dd328016d8d25702a24ee274932aebf6be9787ed1c28d021945d264235b3c \ + --hash=sha256:ef9b85fa1bc91c4db24407e7c4da93a5822a73dd4513d67b454ca7064e8dc6a3 \ + --hash=sha256:f3bf60444269345d712838bb11cc4eadaf51ff1a364ae39ce87a5ca8ad3bb2c8 \ + --hash=sha256:f452cc1436151387d3d50533523291d5f77c6bc7913c116eb985304abdbd9ec9 \ + --hash=sha256:f7917697bcaa3bc3e83db91aa3a0e448bf5cde43c84b7fc1ae2427d2417c0224 \ + --hash=sha256:f90575e9fe3aae2c1e686393a9689c724cd00045275407f71771ae5d690ccf38 \ + --hash=sha256:fb382fd7b4377363cc9f13ba7c819c3c78ed97c36a82f16f3f92f108c787cbbf \ + --hash=sha256:fb9f59f3848edf186a76446eb8bcf4c900fe147cb756fbbd730ef43b2e67c6a7 \ + --hash=sha256:fc2931ac9ce9c61c9968989ec831d3a5e6fcaaff9474e7cfa8de80b7aff5a093 # via # -r requirements.in # aiohttp From 58ce825e14b679a79694fbd872e59e20ed3747c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:39:03 +0100 Subject: [PATCH 13/77] build(deps): bump github/codeql-action from 3.26.9 to 3.26.11 (#36443) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 5101ab9d19..78bacf69ee 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -34,7 +34,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -73,4 +73,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index ff5dad1275..b2d59ba7c7 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -65,7 +65,7 @@ jobs: - name: Initialize CodeQL if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/init@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 with: languages: cpp @@ -108,4 +108,4 @@ jobs: - name: Perform CodeQL Analysis if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/analyze@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index f3075f3ebd..88324734fb 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -40,6 +40,6 @@ jobs: retention-days: 5 - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/upload-sarif@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11 with: sarif_file: results.sarif From 9b8c915c1784013b16590d415fbb39571f981981 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:40:25 +0100 Subject: [PATCH 14/77] build(deps): bump aiohttp from 3.10.6 to 3.10.9 in /tools/base (#36461) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 184 ++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 91b6fbec51..023c1c732a 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -92,98 +92,98 @@ aiohappyeyeballs==2.4.0 \ --hash=sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2 \ --hash=sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd # via aiohttp -aiohttp==3.10.6 \ - --hash=sha256:02108326574ff60267b7b35b17ac5c0bbd0008ccb942ce4c48b657bb90f0b8aa \ - --hash=sha256:029a019627b37fa9eac5c75cc54a6bb722c4ebbf5a54d8c8c0fb4dd8facf2702 \ - --hash=sha256:03fa40d1450ee5196e843315ddf74a51afc7e83d489dbfc380eecefea74158b1 \ - --hash=sha256:0749c4d5a08a802dd66ecdf59b2df4d76b900004017468a7bb736c3b5a3dd902 \ - --hash=sha256:0754690a3a26e819173a34093798c155bafb21c3c640bff13be1afa1e9d421f9 \ - --hash=sha256:0a75d5c9fb4f06c41d029ae70ad943c3a844c40c0a769d12be4b99b04f473d3d \ - --hash=sha256:0b82c8ebed66ce182893e7c0b6b60ba2ace45b1df104feb52380edae266a4850 \ - --hash=sha256:0be3115753baf8b4153e64f9aa7bf6c0c64af57979aa900c31f496301b374570 \ - --hash=sha256:14477c4e52e2f17437b99893fd220ffe7d7ee41df5ebf931a92b8ca82e6fd094 \ - --hash=sha256:164ecd32e65467d86843dbb121a6666c3deb23b460e3f8aefdcaacae79eb718a \ - --hash=sha256:1cb045ec5961f51af3e2c08cd6fe523f07cc6e345033adee711c49b7b91bb954 \ - --hash=sha256:1e52e59ed5f4cc3a3acfe2a610f8891f216f486de54d95d6600a2c9ba1581f4d \ - --hash=sha256:217791c6a399cc4f2e6577bb44344cba1f5714a2aebf6a0bea04cfa956658284 \ - --hash=sha256:25d92f794f1332f656e3765841fc2b7ad5c26c3f3d01e8949eeb3495691cf9f4 \ - --hash=sha256:2708baccdc62f4b1251e59c2aac725936a900081f079b88843dabcab0feeeb27 \ - --hash=sha256:27cf19a38506e2e9f12fc17e55f118f04897b0a78537055d93a9de4bf3022e3d \ - --hash=sha256:289fa8a20018d0d5aa9e4b35d899bd51bcb80f0d5f365d9a23e30dac3b79159b \ - --hash=sha256:2cd5290ab66cfca2f90045db2cc6434c1f4f9fbf97c9f1c316e785033782e7d2 \ - --hash=sha256:2dd56e3c43660ed3bea67fd4c5025f1ac1f9ecf6f0b991a6e5efe2e678c490c5 \ - --hash=sha256:3427031064b0d5c95647e6369c4aa3c556402f324a3e18107cb09517abe5f962 \ - --hash=sha256:3468b39f977a11271517c6925b226720e148311039a380cc9117b1e2258a721f \ - --hash=sha256:370e2d47575c53c817ee42a18acc34aad8da4dbdaac0a6c836d58878955f1477 \ - --hash=sha256:3d2665c5df629eb2f981dab244c01bfa6cdc185f4ffa026639286c4d56fafb54 \ - --hash=sha256:3e15e33bfc73fa97c228f72e05e8795e163a693fd5323549f49367c76a6e5883 \ - --hash=sha256:3fb4216e3ec0dbc01db5ba802f02ed78ad8f07121be54eb9e918448cc3f61b7c \ - --hash=sha256:40271a2a375812967401c9ca8077de9368e09a43a964f4dce0ff603301ec9358 \ - --hash=sha256:438c5863feb761f7ca3270d48c292c334814459f61cc12bab5ba5b702d7c9e56 \ - --hash=sha256:4407a80bca3e694f2d2a523058e20e1f9f98a416619e04f6dc09dc910352ac8b \ - --hash=sha256:444d1704e2af6b30766debed9be8a795958029e552fe77551355badb1944012c \ - --hash=sha256:4611db8c907f90fe86be112efdc2398cd7b4c8eeded5a4f0314b70fdea8feab0 \ - --hash=sha256:473961b3252f3b949bb84873d6e268fb6d8aa0ccc6eb7404fa58c76a326bb8e1 \ - --hash=sha256:4752df44df48fd42b80f51d6a97553b482cda1274d9dc5df214a3a1aa5d8f018 \ - --hash=sha256:47647c8af04a70e07a2462931b0eba63146a13affa697afb4ecbab9d03a480ce \ - --hash=sha256:482f74057ea13d387a7549d7a7ecb60e45146d15f3e58a2d93a0ad2d5a8457cd \ - --hash=sha256:4bef1480ee50f75abcfcb4b11c12de1005968ca9d0172aec4a5057ba9f2b644f \ - --hash=sha256:4fabdcdc781a36b8fd7b2ca9dea8172f29a99e11d00ca0f83ffeb50958da84a1 \ - --hash=sha256:5582de171f0898139cf51dd9fcdc79b848e28d9abd68e837f0803fc9f30807b1 \ - --hash=sha256:58c5d7318a136a3874c78717dd6de57519bc64f6363c5827c2b1cb775bea71dd \ - --hash=sha256:5db26bbca8e7968c4c977a0c640e0b9ce7224e1f4dcafa57870dc6ee28e27de6 \ - --hash=sha256:614fc21e86adc28e4165a6391f851a6da6e9cbd7bb232d0df7718b453a89ee98 \ - --hash=sha256:6419728b08fb6380c66a470d2319cafcec554c81780e2114b7e150329b9a9a7f \ - --hash=sha256:669c0efe7e99f6d94d63274c06344bd0e9c8daf184ce5602a29bc39e00a18720 \ - --hash=sha256:66bc81361131763660b969132a22edce2c4d184978ba39614e8f8f95db5c95f8 \ - --hash=sha256:671745ea7db19693ce867359d503772177f0b20fa8f6ee1e74e00449f4c4151d \ - --hash=sha256:682836fc672972cc3101cc9e30d49c5f7e8f1d010478d46119fe725a4545acfd \ - --hash=sha256:6a504d7cdb431a777d05a124fd0b21efb94498efa743103ea01b1e3136d2e4fb \ - --hash=sha256:6a86610174de8a85a920e956e2d4f9945e7da89f29a00e95ac62a4a414c4ef4e \ - --hash=sha256:6b50b367308ca8c12e0b50cba5773bc9abe64c428d3fd2bbf5cd25aab37c77bf \ - --hash=sha256:7475da7a5e2ccf1a1c86c8fee241e277f4874c96564d06f726d8df8e77683ef7 \ - --hash=sha256:7641920bdcc7cd2d3ddfb8bb9133a6c9536b09dbd49490b79e125180b2d25b93 \ - --hash=sha256:79a9f42efcc2681790595ab3d03c0e52d01edc23a0973ea09f0dc8d295e12b8e \ - --hash=sha256:7ea35d849cdd4a9268f910bff4497baebbc1aa3f2f625fd8ccd9ac99c860c621 \ - --hash=sha256:8198b7c002aae2b40b2d16bfe724b9a90bcbc9b78b2566fc96131ef4e382574d \ - --hash=sha256:81b292f37969f9cc54f4643f0be7dacabf3612b3b4a65413661cf6c350226787 \ - --hash=sha256:844d48ff9173d0b941abed8b2ea6a412f82b56d9ab1edb918c74000c15839362 \ - --hash=sha256:8617c96a20dd57e7e9d398ff9d04f3d11c4d28b1767273a5b1a018ada5a654d3 \ - --hash=sha256:8a637d387db6fdad95e293fab5433b775fd104ae6348d2388beaaa60d08b38c4 \ - --hash=sha256:92351aa5363fc3c1f872ca763f86730ced32b01607f0c9662b1fa711087968d0 \ - --hash=sha256:9843d683b8756971797be171ead21511d2215a2d6e3c899c6e3107fbbe826791 \ - --hash=sha256:995ab1a238fd0d19dc65f2d222e5eb064e409665c6426a3e51d5101c1979ee84 \ - --hash=sha256:9bd6b2033993d5ae80883bb29b83fb2b432270bbe067c2f53cc73bb57c46065f \ - --hash=sha256:9d26da22a793dfd424be1050712a70c0afd96345245c29aced1e35dbace03413 \ - --hash=sha256:a976ef488f26e224079deb3d424f29144c6d5ba4ded313198169a8af8f47fb82 \ - --hash=sha256:a9f196c970db2dcde4f24317e06615363349dc357cf4d7a3b0716c20ac6d7bcd \ - --hash=sha256:b169f8e755e541b72e714b89a831b315bbe70db44e33fead28516c9e13d5f931 \ - --hash=sha256:b504c08c45623bf5c7ca41be380156d925f00199b3970efd758aef4a77645feb \ - --hash=sha256:ba18573bb1de1063d222f41de64a0d3741223982dcea863b3f74646faf618ec7 \ - --hash=sha256:ba3662d41abe2eab0eeec7ee56f33ef4e0b34858f38abf24377687f9e1fb00a5 \ - --hash=sha256:bd294dcdc1afdc510bb51d35444003f14e327572877d016d576ac3b9a5888a27 \ - --hash=sha256:bdbeff1b062751c2a2a55b171f7050fb7073633c699299d042e962aacdbe1a07 \ - --hash=sha256:bf861da9a43d282d6dd9dcd64c23a0fccf2c5aa5cd7c32024513c8c79fb69de3 \ - --hash=sha256:c82a94ddec996413a905f622f3da02c4359952aab8d817c01cf9915419525e95 \ - --hash=sha256:c91781d969fbced1993537f45efe1213bd6fccb4b37bfae2a026e20d6fbed206 \ - --hash=sha256:c9721cdd83a994225352ca84cd537760d41a9da3c0eacb3ff534747ab8fba6d0 \ - --hash=sha256:cca776a440795db437d82c07455761c85bbcf3956221c3c23b8c93176c278ce7 \ - --hash=sha256:cf8b8560aa965f87bf9c13bf9fed7025993a155ca0ce8422da74bf46d18c2f5f \ - --hash=sha256:d2578ef941be0c2ba58f6f421a703527d08427237ed45ecb091fed6f83305336 \ - --hash=sha256:d2b3935a22c9e41a8000d90588bed96cf395ef572dbb409be44c6219c61d900d \ - --hash=sha256:d4dfa5ad4bce9ca30a76117fbaa1c1decf41ebb6c18a4e098df44298941566f9 \ - --hash=sha256:d7f408c43f5e75ea1edc152fb375e8f46ef916f545fb66d4aebcbcfad05e2796 \ - --hash=sha256:dc1a16f3fc1944c61290d33c88dc3f09ba62d159b284c38c5331868425aca426 \ - --hash=sha256:e0009258e97502936d3bd5bf2ced15769629097d0abb81e6495fba1047824fe0 \ - --hash=sha256:e05b39158f2af0e2438cc2075cfc271f4ace0c3cc4a81ec95b27a0432e161951 \ - --hash=sha256:e1f80cd17d81a404b6e70ef22bfe1870bafc511728397634ad5f5efc8698df56 \ - --hash=sha256:e2e7d5591ea868d5ec82b90bbeb366a198715672841d46281b623e23079593db \ - --hash=sha256:f3af26f86863fad12e25395805bb0babbd49d512806af91ec9708a272b696248 \ - --hash=sha256:f52e54fd776ad0da1006708762213b079b154644db54bcfc62f06eaa5b896402 \ - --hash=sha256:f8b8e49fe02f744d38352daca1dbef462c3874900bd8166516f6ea8e82b5aacf \ - --hash=sha256:fb138fbf9f53928e779650f5ed26d0ea1ed8b2cab67f0ea5d63afa09fdc07593 \ - --hash=sha256:fe517113fe4d35d9072b826c3e147d63c5f808ca8167d450b4f96c520c8a1d8d \ - --hash=sha256:ff99ae06eef85c7a565854826114ced72765832ee16c7e3e766c5e4c5b98d20e +aiohttp==3.10.9 \ + --hash=sha256:02d1d6610588bcd743fae827bd6f2e47e0d09b346f230824b4c6fb85c6065f9c \ + --hash=sha256:03690541e4cc866eef79626cfa1ef4dd729c5c1408600c8cb9e12e1137eed6ab \ + --hash=sha256:0bc059ecbce835630e635879f5f480a742e130d9821fbe3d2f76610a6698ee25 \ + --hash=sha256:0c21c82df33b264216abffff9f8370f303dab65d8eee3767efbbd2734363f677 \ + --hash=sha256:1298b854fd31d0567cbb916091be9d3278168064fca88e70b8468875ef9ff7e7 \ + --hash=sha256:1321658f12b6caffafdc35cfba6c882cb014af86bef4e78c125e7e794dfb927b \ + --hash=sha256:143b0026a9dab07a05ad2dd9e46aa859bffdd6348ddc5967b42161168c24f857 \ + --hash=sha256:16e6a51d8bc96b77f04a6764b4ad03eeef43baa32014fce71e882bd71302c7e4 \ + --hash=sha256:172ad884bb61ad31ed7beed8be776eb17e7fb423f1c1be836d5cb357a096bf12 \ + --hash=sha256:17c272cfe7b07a5bb0c6ad3f234e0c336fb53f3bf17840f66bd77b5815ab3d16 \ + --hash=sha256:1a0ee6c0d590c917f1b9629371fce5f3d3f22c317aa96fbdcce3260754d7ea21 \ + --hash=sha256:2746d8994ebca1bdc55a1e998feff4e94222da709623bb18f6e5cfec8ec01baf \ + --hash=sha256:2914caa46054f3b5ff910468d686742ff8cff54b8a67319d75f5d5945fd0a13d \ + --hash=sha256:2bbf94d4a0447705b7775417ca8bb8086cc5482023a6e17cdc8f96d0b1b5aba6 \ + --hash=sha256:2bd9f3eac515c16c4360a6a00c38119333901b8590fe93c3257a9b536026594d \ + --hash=sha256:2c33fa6e10bb7ed262e3ff03cc69d52869514f16558db0626a7c5c61dde3c29f \ + --hash=sha256:2d37f4718002863b82c6f391c8efd4d3a817da37030a29e2682a94d2716209de \ + --hash=sha256:3668d0c2a4d23fb136a753eba42caa2c0abbd3d9c5c87ee150a716a16c6deec1 \ + --hash=sha256:36d4fba838be5f083f5490ddd281813b44d69685db910907636bc5dca6322316 \ + --hash=sha256:40ff5b7660f903dc587ed36ef08a88d46840182d9d4b5694e7607877ced698a1 \ + --hash=sha256:42775de0ca04f90c10c5c46291535ec08e9bcc4756f1b48f02a0657febe89b10 \ + --hash=sha256:482c85cf3d429844396d939b22bc2a03849cb9ad33344689ad1c85697bcba33a \ + --hash=sha256:4e6cb75f8ddd9c2132d00bc03c9716add57f4beff1263463724f6398b813e7eb \ + --hash=sha256:4edc3fd701e2b9a0d605a7b23d3de4ad23137d23fc0dbab726aa71d92f11aaaf \ + --hash=sha256:4fd16b30567c5b8e167923be6e027eeae0f20cf2b8a26b98a25115f28ad48ee0 \ + --hash=sha256:5002a02c17fcfd796d20bac719981d2fca9c006aac0797eb8f430a58e9d12431 \ + --hash=sha256:51d0a4901b27272ae54e42067bc4b9a90e619a690b4dc43ea5950eb3070afc32 \ + --hash=sha256:558b3d223fd631ad134d89adea876e7fdb4c93c849ef195049c063ada82b7d08 \ + --hash=sha256:5c070430fda1a550a1c3a4c2d7281d3b8cfc0c6715f616e40e3332201a253067 \ + --hash=sha256:5f392ef50e22c31fa49b5a46af7f983fa3f118f3eccb8522063bee8bfa6755f8 \ + --hash=sha256:60555211a006d26e1a389222e3fab8cd379f28e0fbf7472ee55b16c6c529e3a6 \ + --hash=sha256:608cecd8d58d285bfd52dbca5b6251ca8d6ea567022c8a0eaae03c2589cd9af9 \ + --hash=sha256:60ad5b8a7452c0f5645c73d4dad7490afd6119d453d302cd5b72b678a85d6044 \ + --hash=sha256:63649309da83277f06a15bbdc2a54fbe75efb92caa2c25bb57ca37762789c746 \ + --hash=sha256:6ebdc3b3714afe1b134b3bbeb5f745eed3ecbcff92ab25d80e4ef299e83a5465 \ + --hash=sha256:6f3c6648aa123bcd73d6f26607d59967b607b0da8ffcc27d418a4b59f4c98c7c \ + --hash=sha256:7003f33f5f7da1eb02f0446b0f8d2ccf57d253ca6c2e7a5732d25889da82b517 \ + --hash=sha256:776e9f3c9b377fcf097c4a04b241b15691e6662d850168642ff976780609303c \ + --hash=sha256:85711eec2d875cd88c7eb40e734c4ca6d9ae477d6f26bd2b5bb4f7f60e41b156 \ + --hash=sha256:87d1e4185c5d7187684d41ebb50c9aeaaaa06ca1875f4c57593071b0409d2444 \ + --hash=sha256:8a3f063b41cc06e8d0b3fcbbfc9c05b7420f41287e0cd4f75ce0a1f3d80729e6 \ + --hash=sha256:8b3fb28a9ac8f2558760d8e637dbf27aef1e8b7f1d221e8669a1074d1a266bb2 \ + --hash=sha256:8bd9125dd0cc8ebd84bff2be64b10fdba7dc6fd7be431b5eaf67723557de3a31 \ + --hash=sha256:8be1a65487bdfc285bd5e9baf3208c2132ca92a9b4020e9f27df1b16fab998a9 \ + --hash=sha256:8cc0d13b4e3b1362d424ce3f4e8c79e1f7247a00d792823ffd640878abf28e56 \ + --hash=sha256:8d9d10d10ec27c0d46ddaecc3c5598c4db9ce4e6398ca872cdde0525765caa2f \ + --hash=sha256:8debb45545ad95b58cc16c3c1cc19ad82cffcb106db12b437885dbee265f0ab5 \ + --hash=sha256:91aa966858593f64c8a65cdefa3d6dc8fe3c2768b159da84c1ddbbb2c01ab4ef \ + --hash=sha256:9331dd34145ff105177855017920dde140b447049cd62bb589de320fd6ddd582 \ + --hash=sha256:99f9678bf0e2b1b695e8028fedac24ab6770937932eda695815d5a6618c37e04 \ + --hash=sha256:9fdf5c839bf95fc67be5794c780419edb0dbef776edcfc6c2e5e2ffd5ee755fa \ + --hash=sha256:a14e4b672c257a6b94fe934ee62666bacbc8e45b7876f9dd9502d0f0fe69db16 \ + --hash=sha256:a19caae0d670771ea7854ca30df76f676eb47e0fd9b2ee4392d44708f272122d \ + --hash=sha256:a35ed3d03910785f7d9d6f5381f0c24002b2b888b298e6f941b2fc94c5055fcd \ + --hash=sha256:a61df62966ce6507aafab24e124e0c3a1cfbe23c59732987fc0fd0d71daa0b88 \ + --hash=sha256:a6e00c8a92e7663ed2be6fcc08a2997ff06ce73c8080cd0df10cc0321a3168d7 \ + --hash=sha256:ac3196952c673822ebed8871cf8802e17254fff2a2ed4835d9c045d9b88c5ec7 \ + --hash=sha256:ac74e794e3aee92ae8f571bfeaa103a141e409863a100ab63a253b1c53b707eb \ + --hash=sha256:ad3675c126f2a95bde637d162f8231cff6bc0bc9fbe31bd78075f9ff7921e322 \ + --hash=sha256:aeebd3061f6f1747c011e1d0b0b5f04f9f54ad1a2ca183e687e7277bef2e0da2 \ + --hash=sha256:ba1a599255ad6a41022e261e31bc2f6f9355a419575b391f9655c4d9e5df5ff5 \ + --hash=sha256:bbdb8def5268f3f9cd753a265756f49228a20ed14a480d151df727808b4531dd \ + --hash=sha256:c2555e4949c8d8782f18ef20e9d39730d2656e218a6f1a21a4c4c0b56546a02e \ + --hash=sha256:c2695c61cf53a5d4345a43d689f37fc0f6d3a2dc520660aec27ec0f06288d1f9 \ + --hash=sha256:c2b627d3c8982691b06d89d31093cee158c30629fdfebe705a91814d49b554f8 \ + --hash=sha256:c46131c6112b534b178d4e002abe450a0a29840b61413ac25243f1291613806a \ + --hash=sha256:c54dc329cd44f7f7883a9f4baaefe686e8b9662e2c6c184ea15cceee587d8d69 \ + --hash=sha256:c7d7cafc11d70fdd8801abfc2ff276744ae4cb39d8060b6b542c7e44e5f2cfc2 \ + --hash=sha256:cb0b2d5d51f96b6cc19e6ab46a7b684be23240426ae951dcdac9639ab111b45e \ + --hash=sha256:d15a29424e96fad56dc2f3abed10a89c50c099f97d2416520c7a543e8fddf066 \ + --hash=sha256:d1f5c9169e26db6a61276008582d945405b8316aae2bb198220466e68114a0f5 \ + --hash=sha256:d271f770b52e32236d945911b2082f9318e90ff835d45224fa9e28374303f729 \ + --hash=sha256:d646fdd74c25bbdd4a055414f0fe32896c400f38ffbdfc78c68e62812a9e0257 \ + --hash=sha256:d6e395c3d1f773cf0651cd3559e25182eb0c03a2777b53b4575d8adc1149c6e9 \ + --hash=sha256:d7c071235a47d407b0e93aa6262b49422dbe48d7d8566e1158fecc91043dd948 \ + --hash=sha256:d97273a52d7f89a75b11ec386f786d3da7723d7efae3034b4dda79f6f093edc1 \ + --hash=sha256:dcf354661f54e6a49193d0b5653a1b011ba856e0b7a76bda2c33e4c6892f34ea \ + --hash=sha256:e3e7fabedb3fe06933f47f1538df7b3a8d78e13d7167195f51ca47ee12690373 \ + --hash=sha256:e525b69ee8a92c146ae5b4da9ecd15e518df4d40003b01b454ad694a27f498b5 \ + --hash=sha256:e709d6ac598c5416f879bb1bae3fd751366120ac3fa235a01de763537385d036 \ + --hash=sha256:e83dfefb4f7d285c2d6a07a22268344a97d61579b3e0dce482a5be0251d672ab \ + --hash=sha256:e86260b76786c28acf0b5fe31c8dca4c2add95098c709b11e8c35b424ebd4f5b \ + --hash=sha256:e883b61b75ca6efc2541fcd52a5c8ccfe288b24d97e20ac08fdf343b8ac672ea \ + --hash=sha256:f0a44bb40b6aaa4fb9a5c1ee07880570ecda2065433a96ccff409c9c20c1624a \ + --hash=sha256:f82ace0ec57c94aaf5b0e118d4366cff5889097412c75aa14b4fd5fc0c44ee3e \ + --hash=sha256:f9ca09414003c0e96a735daa1f071f7d7ed06962ef4fa29ceb6c80d06696d900 \ + --hash=sha256:fa430b871220dc62572cef9c69b41e0d70fcb9d486a4a207a5de4c1f25d82593 \ + --hash=sha256:fc262c3df78c8ff6020c782d9ce02e4bcffe4900ad71c0ecdad59943cba54442 \ + --hash=sha256:fcd546782d03181b0b1d20b43d612429a90a68779659ba8045114b867971ab71 \ + --hash=sha256:fd4ceeae2fb8cabdd1b71c82bfdd39662473d3433ec95b962200e9e752fb70d0 \ + --hash=sha256:fec5fac7aea6c060f317f07494961236434928e6f4374e170ef50b3001e14581 # via # -r requirements.in # aio-api-github From 3415fea16992a08356ade5c4d111e5a49114e743 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:20:58 +0100 Subject: [PATCH 15/77] build(deps): bump icalendar from 5.0.13 to 6.0.0 in /tools/base (#36378) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 023c1c732a..344fb1732e 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -764,9 +764,9 @@ humanfriendly==10.0 \ --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc # via coloredlogs -icalendar==5.0.13 \ - --hash=sha256:5ded5415e2e1edef5ab230024a75878a7a81d518a3b1ae4f34bf20b173c84dc2 \ - --hash=sha256:92799fde8cce0b61daa8383593836d1e19136e504fa1671f471f98be9b029706 +icalendar==6.0.0 \ + --hash=sha256:567e718551d800362db04ca09777295336e1803f6fc6bc0a7a5e258917fa8ed0 \ + --hash=sha256:7ddf60d343f3c1f716de9b62f6e80ffd95d03cab62464894a0539feab7b5c76e # via -r requirements.in idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ @@ -1179,7 +1179,6 @@ pytz==2024.2 \ # via # aio-core # envoy-base-utils - # icalendar pyu2f==0.1.5 \ --hash=sha256:a3caa3a11842fc7d5746376f37195e6af5f17c0a15737538bb1cebf656fb306b # via google-reauth @@ -1380,6 +1379,10 @@ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via pygithub +tzdata==2024.2 \ + --hash=sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc \ + --hash=sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd + # via icalendar uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e From 10c8741f4e36538fb85c8b03c197854e7fa397f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:21:18 +0100 Subject: [PATCH 16/77] build(deps): bump frozendict from 2.4.4 to 2.4.5 in /tools/base (#36460) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 73 ++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 344fb1732e..347aa8ff8e 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -586,40 +586,45 @@ flake8==7.1.1 \ # -r requirements.in # envoy-code-check # pep8-naming -frozendict==2.4.4 \ - --hash=sha256:07c3a5dee8bbb84cba770e273cdbf2c87c8e035903af8f781292d72583416801 \ - --hash=sha256:12a342e439aef28ccec533f0253ea53d75fe9102bd6ea928ff530e76eac38906 \ - --hash=sha256:1697793b5f62b416c0fc1d94638ec91ed3aa4ab277f6affa3a95216ecb3af170 \ - --hash=sha256:199a4d32194f3afed6258de7e317054155bc9519252b568d9cfffde7e4d834e5 \ - --hash=sha256:259528ba6b56fa051bc996f1c4d8b57e30d6dd3bc2f27441891b04babc4b5e73 \ - --hash=sha256:2b70b431e3a72d410a2cdf1497b3aba2f553635e0c0f657ce311d841bf8273b6 \ - --hash=sha256:2bd009cf4fc47972838a91e9b83654dc9a095dc4f2bb3a37c3f3124c8a364543 \ - --hash=sha256:2d8536e068d6bf281f23fa835ac07747fb0f8851879dd189e9709f9567408b4d \ - --hash=sha256:3148062675536724502c6344d7c485dd4667fdf7980ca9bd05e338ccc0c4471e \ - --hash=sha256:3f7c031b26e4ee6a3f786ceb5e3abf1181c4ade92dce1f847da26ea2c96008c7 \ - --hash=sha256:4297d694eb600efa429769125a6f910ec02b85606f22f178bafbee309e7d3ec7 \ - --hash=sha256:4a59578d47b3949437519b5c39a016a6116b9e787bb19289e333faae81462e59 \ - --hash=sha256:4ae8d05c8d0b6134bfb6bfb369d5fa0c4df21eabb5ca7f645af95fdc6689678e \ - --hash=sha256:5d58d9a8d9e49662c6dafbea5e641f97decdb3d6ccd76e55e79818415362ba25 \ - --hash=sha256:63aa49f1919af7d45fb8fd5dec4c0859bc09f46880bd6297c79bb2db2969b63d \ - --hash=sha256:6874fec816b37b6eb5795b00e0574cba261bf59723e2de607a195d5edaff0786 \ - --hash=sha256:6eb716e6a6d693c03b1d53280a1947716129f5ef9bcdd061db5c17dea44b80fe \ - --hash=sha256:705efca8d74d3facbb6ace80ab3afdd28eb8a237bfb4063ed89996b024bc443d \ - --hash=sha256:78c94991944dd33c5376f720228e5b252ee67faf3bac50ef381adc9e51e90d9d \ - --hash=sha256:7f79c26dff10ce11dad3b3627c89bb2e87b9dd5958c2b24325f16a23019b8b94 \ - --hash=sha256:7fee9420475bb6ff357000092aa9990c2f6182b2bab15764330f4ad7de2eae49 \ - --hash=sha256:812ab17522ba13637826e65454115a914c2da538356e85f43ecea069813e4b33 \ - --hash=sha256:85375ec6e979e6373bffb4f54576a68bf7497c350861d20686ccae38aab69c0a \ - --hash=sha256:87ebcde21565a14fe039672c25550060d6f6d88cf1f339beac094c3b10004eb0 \ - --hash=sha256:93a7b19afb429cbf99d56faf436b45ef2fa8fe9aca89c49eb1610c3bd85f1760 \ - --hash=sha256:b3b967d5065872e27b06f785a80c0ed0a45d1f7c9b85223da05358e734d858ca \ - --hash=sha256:c6bf9260018d653f3cab9bd147bd8592bf98a5c6e338be0491ced3c196c034a3 \ - --hash=sha256:c8f92425686323a950337da4b75b4c17a3327b831df8c881df24038d560640d4 \ - --hash=sha256:d13b4310db337f4d2103867c5a05090b22bc4d50ca842093779ef541ea9c9eea \ - --hash=sha256:d9647563e76adb05b7cde2172403123380871360a114f546b4ae1704510801e5 \ - --hash=sha256:dc2228874eacae390e63fd4f2bb513b3144066a977dc192163c9f6c7f6de6474 \ - --hash=sha256:e1b941132d79ce72d562a13341d38fc217bc1ee24d8c35a20d754e79ff99e038 \ - --hash=sha256:fefeb700bc7eb8b4c2dc48704e4221860d254c8989fb53488540bc44e44a1ac2 +frozendict==2.4.5 \ + --hash=sha256:043bce9f5e7e8df8313d0c961b2da3346c0b002311d88c74cadae2b1f8fd2904 \ + --hash=sha256:0483776fa25f0c3f9ff75a89b5276a582231f83baea85b091dfefde99b95ac5c \ + --hash=sha256:05786d9cff61ee95149c15cdfc10cf4b4cb10e1eab1d4648b0c99ed58fedb86d \ + --hash=sha256:08e8d4ebc950916b5cdeedcb697cf893c5903d9cb8ab27dffe41072a08e1cfc1 \ + --hash=sha256:14755127b988b428f95f11fa626374e247f8a40316697218f1e2f44c107c5027 \ + --hash=sha256:1f364cfe5ef97523a4434e9f458bb4821594d3531d898621e5acae43463dcb5e \ + --hash=sha256:20cf1db30ca0c8f76a05ec1256132f505cf6f93eb382a5961c738377037b5b9d \ + --hash=sha256:24953a1cfe344415e7557413e493e21fa9c4cecbc13284b6f334837aa5601c9f \ + --hash=sha256:2b51e0aae859fc8d56138eac4be121a76647fa66ea620af8a62d9d6938729441 \ + --hash=sha256:35730ff269b8a6eb07c3d91d5fd6a7f5c32e08bef665fe1ef51012bdb2702196 \ + --hash=sha256:3702438d81deab127963ab41dca1738a0fbf11038a50a25c5f814943a2d25de5 \ + --hash=sha256:3af1b09bad761d5500b096b757c346591b5dfdc8443aa9642c2c02b4885dc09e \ + --hash=sha256:4b3eea1678607174c468fe4e3b903625bccfb3215ae2b0138220dc1f6ef71f37 \ + --hash=sha256:4f073c926b1f88fa85ed85222101f61f6c4b2180c95d1528ca6ecc7cef835442 \ + --hash=sha256:6be054ba76e8a49c6846a7369db3eaaa0593c29a644026c25923f13fdea06483 \ + --hash=sha256:6d10ad91ded20fae8737fddcfed1b6358207f95292a335f96c4cd365d97f5e30 \ + --hash=sha256:73ff80ce3f95c3ec60b38ef4ddf5cb8e20ea3edce2f56e89af7c4c45013684bf \ + --hash=sha256:754924d53b1fd2dae7b15f9c86b0610334a840ae728cc717f24aa040fc94136f \ + --hash=sha256:76c7af5f9db9ae01531edaf38fd3e8faae1bb6b94abbf8fa5bd0326f52e777df \ + --hash=sha256:8582f26ca862bb1e40ed790d934adf6afcfc904b0425dcfe01aed2103bed27fb \ + --hash=sha256:95bf788d65e3a50d9deef5c68e568c59acf473fea8a7a4d7e405a7e63ac85cdd \ + --hash=sha256:98b1483980c47bb74ef777ab748031a69eb41c43ef44494e9db79111638fdb84 \ + --hash=sha256:a5448639058157ccd7f26ba83e441066c6ae515beb68b9b99d5b2dbb0bb36b19 \ + --hash=sha256:a81d80ded07f0e34a3fa2b78235a48259b886ad20a96da5526a9404d192c6eb3 \ + --hash=sha256:a967730a115cb8b5d028e0d07cd680c04324e913ee3d1caef9e039ca740343c4 \ + --hash=sha256:af4bdffecff350a68f1a966dec84459d3ea8e2606c14fb3ebe937b8c6c9e9129 \ + --hash=sha256:b8481d83a7219e9e14c719113ea2d8321d7840af35af58c72b3864b7123e7de3 \ + --hash=sha256:bbf9dcb69a44519308c775bcf72b8417c3102f415f77df19b8eddb97b8f88d78 \ + --hash=sha256:d2b2a9c809ebb54c8c0fe2acc38d45299f3e9a9d90061773468896221cab1eaa \ + --hash=sha256:de1126d6dd39d7939be388161b973ffadb9c6e97f8f9fdcb52ba790828621191 \ + --hash=sha256:dee64c44e1a146171bdd2bf4501210f273a5ab73cc2e924a01f81b14e4612f94 \ + --hash=sha256:e591124818f2d5d8f2d284a7b78604f4180e73d770c33252370edccff5969293 \ + --hash=sha256:eb58e0c41bc6bb60a8b0852d50da8836380fb78311608dc26791e3e9aa25bcfe \ + --hash=sha256:ee3e6f4013446d17c580ce0bac41a1f81371c21dd6e34a85c0f26bae94fdfd9b \ + --hash=sha256:f1385852cb05a31ae76c972caa3a07679414b01a664848bde2bae71c9910f045 \ + --hash=sha256:f2ff03ed21657cde7ea04fffc76f0c1736100a03671e4682722685e63486cd96 \ + --hash=sha256:f8d677a6066ca3289181028fccabf77413addddaa7a6629485d52aeaa4f4899c \ + --hash=sha256:fd7add309789595c044c0155a0bddfa9d20c77f65de1e33a14aa3033b936ef63 # via # -r requirements.in # aio-run-runner From b4f6137c6ea53d503971aa9e973ee9f40346fd97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:21:35 +0100 Subject: [PATCH 17/77] build(deps): bump kafka-python-ng from 2.2.2 to 2.2.3 in /tools/base (#36429) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 347aa8ff8e..99c5b90a40 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -795,9 +795,9 @@ jinja2==3.1.4 \ # envoy-base-utils # envoy-dependency-check # sphinx -kafka-python-ng==2.2.2 \ - --hash=sha256:3fab1a03133fade1b6fd5367ff726d980e59031c4aaca9bf02c516840a4f8406 \ - --hash=sha256:87ad3a766e2c0bec71d9b99bdd9e9c5cda62d96cfda61a8ca16510484d6ad7d4 +kafka-python-ng==2.2.3 \ + --hash=sha256:adc6e82147c441ca4ae1f22e291fc08efab0d10971cbd4aa1481d2ffa38e9480 \ + --hash=sha256:f79f28e10ade9b5a9860b2ec15b7cc8dc510d5702f5a399430478cff5f93a05a # via -r requirements.in markupsafe==2.1.5 \ --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ From 7bf87fd1292c79d1d5fa2f681d03c66a2ccd4e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Mon, 7 Oct 2024 09:46:40 -0400 Subject: [PATCH 18/77] Adjust RBE resources for a memory-hungry test (#36453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit //test/extensions/load_balancing_policies/client_side_weighted_round_robin:integration_test Risk Level: low Testing: CI Signed-off-by: Alejandro R. Sedeño --- .../client_side_weighted_round_robin/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD b/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD index e228a82640..0a6739e131 100644 --- a/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD +++ b/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD @@ -42,6 +42,7 @@ envoy_extension_cc_test( size = "large", srcs = ["integration_test.cc"], extension_names = ["envoy.load_balancing_policies.client_side_weighted_round_robin"], + rbe_pool = "2core", deps = [ "//source/common/protobuf", "//source/extensions/load_balancing_policies/client_side_weighted_round_robin:config", From 68b4559d126d631817a750e356269a7739417e7b Mon Sep 17 00:00:00 2001 From: code Date: Mon, 7 Oct 2024 22:30:10 +0800 Subject: [PATCH 19/77] utility: new utility method to convert proto value to string (#36334) Commit Message: utility: new utility method to convert proto value to string Additional Description: New utility method to convert the proto value to json. This could work even the `ENVOY_ENABLE_YAML` is not set and is exception free. Risk Level: low. Testing: unit test. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. --------- Signed-off-by: wangbaiping --- source/common/json/BUILD | 10 +++ source/common/json/json_streamer.h | 5 ++ source/common/json/json_utility.cc | 88 +++++++++++++++++++++++++++ source/common/json/json_utility.h | 22 +++++++ test/common/json/BUILD | 9 +++ test/common/json/json_utility_test.cc | 88 +++++++++++++++++++++++++++ 6 files changed, 222 insertions(+) create mode 100644 source/common/json/json_utility.cc create mode 100644 source/common/json/json_utility.h create mode 100644 test/common/json/json_utility_test.cc diff --git a/source/common/json/BUILD b/source/common/json/BUILD index 02ae19bfab..cd431c83d0 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -64,3 +64,13 @@ envoy_cc_library( "@com_google_absl//absl/strings", ], ) + +envoy_cc_library( + name = "json_utility_lib", + srcs = ["json_utility.cc"], + hdrs = ["json_utility.h"], + deps = [ + ":json_streamer_lib", + "//source/common/protobuf:utility_lib_header", + ], +) diff --git a/source/common/json/json_streamer.h b/source/common/json/json_streamer.h index f63d1d439d..6c3a6790b9 100644 --- a/source/common/json/json_streamer.h +++ b/source/common/json/json_streamer.h @@ -456,5 +456,10 @@ template class StreamerBase { */ using BufferStreamer = StreamerBase; +/** + * A Streamer that streams to a string. + */ +using StringStreamer = StreamerBase; + } // namespace Json } // namespace Envoy diff --git a/source/common/json/json_utility.cc b/source/common/json/json_utility.cc new file mode 100644 index 0000000000..d0110e506c --- /dev/null +++ b/source/common/json/json_utility.cc @@ -0,0 +1,88 @@ +#include "source/common/json/json_utility.h" + +namespace Envoy { +namespace Json { + +namespace { + +void structValueToJson(const ProtobufWkt::Struct& struct_value, StringStreamer::Map& level); +void listValueToJson(const ProtobufWkt::ListValue& list_value, StringStreamer::Array& level); + +void valueToJson(const ProtobufWkt::Value& value, StringStreamer::Level& level) { + switch (value.kind_case()) { + case ProtobufWkt::Value::KIND_NOT_SET: + case ProtobufWkt::Value::kNullValue: + level.addNull(); + break; + case ProtobufWkt::Value::kNumberValue: + level.addNumber(value.number_value()); + break; + case ProtobufWkt::Value::kStringValue: + level.addString(value.string_value()); + break; + case ProtobufWkt::Value::kBoolValue: + level.addBool(value.bool_value()); + break; + case ProtobufWkt::Value::kStructValue: + structValueToJson(value.struct_value(), *level.addMap()); + break; + case ProtobufWkt::Value::kListValue: + listValueToJson(value.list_value(), *level.addArray()); + break; + } +} + +void structValueToJson(const ProtobufWkt::Struct& struct_value, StringStreamer::Map& map) { + using PairRefWrapper = + std::reference_wrapper::value_type>; + absl::InlinedVector sorted_fields; + sorted_fields.reserve(struct_value.fields_size()); + + for (const auto& field : struct_value.fields()) { + sorted_fields.emplace_back(field); + } + // Sort the keys to make the output deterministic. + std::sort(sorted_fields.begin(), sorted_fields.end(), + [](PairRefWrapper a, PairRefWrapper b) { return a.get().first < b.get().first; }); + + for (const PairRefWrapper field : sorted_fields) { + map.addKey(field.get().first); + valueToJson(field.get().second, map); + } +} + +void listValueToJson(const ProtobufWkt::ListValue& list_value, StringStreamer::Array& arr) { + for (const ProtobufWkt::Value& value : list_value.values()) { + valueToJson(value, arr); + } +} + +} // namespace + +void Utility::appendValueToString(const ProtobufWkt::Value& value, std::string& dest) { + StringStreamer streamer(dest); + switch (value.kind_case()) { + case ProtobufWkt::Value::KIND_NOT_SET: + case ProtobufWkt::Value::kNullValue: + streamer.addNull(); + break; + case ProtobufWkt::Value::kNumberValue: + streamer.addNumber(value.number_value()); + break; + case ProtobufWkt::Value::kStringValue: + streamer.addString(value.string_value()); + break; + case ProtobufWkt::Value::kBoolValue: + streamer.addBool(value.bool_value()); + break; + case ProtobufWkt::Value::kStructValue: + structValueToJson(value.struct_value(), *streamer.makeRootMap()); + break; + case ProtobufWkt::Value::kListValue: + listValueToJson(value.list_value(), *streamer.makeRootArray()); + break; + } +} + +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_utility.h b/source/common/json/json_utility.h new file mode 100644 index 0000000000..3797a3befd --- /dev/null +++ b/source/common/json/json_utility.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +#include "source/common/json/json_streamer.h" +#include "source/common/protobuf/protobuf.h" + +namespace Envoy { +namespace Json { + +class Utility { +public: + /** + * Convert a ProtobufWkt::Value to a JSON string. + * @param value message of type type.googleapis.com/google.protobuf.Value + * @param dest JSON string. + */ + static void appendValueToString(const ProtobufWkt::Value& value, std::string& dest); +}; + +} // namespace Json +} // namespace Envoy diff --git a/test/common/json/BUILD b/test/common/json/BUILD index 91bf779da9..8bbe61deb4 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -91,6 +91,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "json_utility_test", + srcs = ["json_utility_test.cc"], + deps = [ + "//source/common/json:json_utility_lib", + "//source/common/protobuf:utility_lib", + ], +) + envoy_cc_test_binary( name = "gen_excluded_unicodes", srcs = ["gen_excluded_unicodes.cc"], diff --git a/test/common/json/json_utility_test.cc b/test/common/json/json_utility_test.cc new file mode 100644 index 0000000000..036efdd487 --- /dev/null +++ b/test/common/json/json_utility_test.cc @@ -0,0 +1,88 @@ +#include "source/common/json/json_utility.h" +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Json { +namespace { + +std::string toJson(const ProtobufWkt::Value& v) { + std::string json_string; + Utility::appendValueToString(v, json_string); + return json_string; +} + +TEST(JsonUtilityTest, AppendValueToString) { + ProtobufWkt::Value v; + + // null + EXPECT_EQ(toJson(v), "null"); + + v.set_null_value(ProtobufWkt::NULL_VALUE); + EXPECT_EQ(toJson(v), "null"); + + // bool + v.set_bool_value(true); + EXPECT_EQ(toJson(v), "true"); + + v.set_bool_value(false); + EXPECT_EQ(toJson(v), "false"); + + // number + v.set_number_value(1); + EXPECT_EQ(toJson(v), "1"); + + v.set_number_value(1.1); + EXPECT_EQ(toJson(v), "1.1"); + + // string + v.set_string_value("foo"); + EXPECT_EQ(toJson(v), "\"foo\""); + + // struct + auto* struct_value = v.mutable_struct_value(); + EXPECT_EQ(toJson(v), R"EOF({})EOF"); + + struct_value->mutable_fields()->insert({"foo", ValueUtil::stringValue("bar")}); + EXPECT_EQ(toJson(v), R"EOF({"foo":"bar"})EOF"); + + // list + auto* list_value = v.mutable_list_value(); + + EXPECT_EQ(toJson(v), R"EOF([])EOF"); + + list_value->add_values()->set_string_value("foo"); + list_value->add_values()->set_string_value("bar"); + + EXPECT_EQ(toJson(v), R"EOF(["foo","bar"])EOF"); + + // Complex structure + const std::string yaml = R"EOF( + a: + a: + - a: 1 + b: 2 + b: + - a: 3 + b: 4 + - a: 5 + c: true + d: [5, 3.14] + e: foo + f: 1.1 + b: [1, 2, 3] + c: bar + )EOF"; + + MessageUtil::loadFromYaml(yaml, v, ProtobufMessage::getNullValidationVisitor()); + + EXPECT_EQ( + toJson(v), + R"EOF({"a":{"a":[{"a":1,"b":2}],"b":[{"a":3,"b":4},{"a":5}],"c":true,"d":[5,"3.14"],"e":"foo","f":"1.1"},"b":[1,2,3],"c":"bar"})EOF"); +} + +} // namespace +} // namespace Json +} // namespace Envoy From 78f832db10751bb803f6a339f7515d7e1d724c8f Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 7 Oct 2024 11:11:29 -0400 Subject: [PATCH 20/77] Reapply "udp: set Don't Fragment(DF) bit in IP packet header (#36341)" (#36437) Commit Message: removed the static_assert as it is expected that some platform, especially some iOS versions supports neither of the socket options. In this case, Envoy wont' set DF bit. Additional Description: reland #36341 Risk Level: low Testing: new unit tests Docs Changes: N/A Release Notes: Yes Platform Specific Features: N/A Runtime guard: envoy.reloadable_features.udp_set_do_not_fragment --------- Signed-off-by: Dan Zhang Co-authored-by: Dan Zhang --- changelogs/current.yaml | 4 + envoy/network/socket.h | 19 ++++ .../common/listener_manager/listener_impl.cc | 7 ++ .../common/network/socket_option_factory.cc | 27 ++++++ source/common/network/socket_option_factory.h | 6 ++ source/common/quic/envoy_quic_utils.cc | 31 +++++- source/common/runtime/runtime_features.cc | 1 + .../listener_manager_impl_quic_only_test.cc | 95 +++++++++++++------ test/common/quic/BUILD | 1 + .../client_connection_factory_impl_test.cc | 76 +++++++++++++++ test/common/quic/envoy_quic_utils_test.cc | 73 +++++++++++++- tools/spelling/spelling_dictionary.txt | 1 + 12 files changed, 306 insertions(+), 35 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 9b844af994..20f1e7683b 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -116,6 +116,10 @@ minor_behavior_changes: change: | Changes the default value of ``envoy.reloadable_features.http2_use_oghttp2`` to ``false``. This changes the codec used for HTTP/2 requests and responses to address to address stability concerns. This behavior can be reverted by setting the feature to ``true``. +- area: udp + change: | + Set Don't Fragment (DF) flag bit on IP packet header on UDP listener sockets and QUIC upstream connection sockets. This behavior + can be reverted by setting ``envoy.reloadable_features.udp_set_do_not_fragment`` to false. - area: access_log change: | Sanitize SNI for potential log injection. The invalid character will be replaced by ``_`` with an ``invalid:`` marker. If runtime diff --git a/envoy/network/socket.h b/envoy/network/socket.h index 39c4927549..bed7b7be35 100644 --- a/envoy/network/socket.h +++ b/envoy/network/socket.h @@ -175,6 +175,25 @@ static_assert(IP_RECVDSTADDR == IP_SENDSRCADDR); #define ENVOY_ATTACH_REUSEPORT_CBPF Network::SocketOptionName() #endif +#if !defined(ANDROID) && defined(__APPLE__) +// Only include TargetConditionals after testing ANDROID as some Android builds +// on the Mac have this header available and it's not needed unless the target +// is really an Apple platform. +#include +#if !defined(TARGET_OS_IPHONE) || !TARGET_OS_IPHONE +// MAC_OS +#define ENVOY_IP_DONTFRAG ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_DONTFRAG) +#define ENVOY_IPV6_DONTFRAG ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_DONTFRAG) +#endif +#endif + +#if !defined(ENVOY_IP_DONTFRAG) && defined(IP_PMTUDISC_DO) +#define ENVOY_IP_MTU_DISCOVER ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_MTU_DISCOVER) +#define ENVOY_IP_MTU_DISCOVER_VALUE IP_PMTUDISC_DO +#define ENVOY_IPV6_MTU_DISCOVER ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_MTU_DISCOVER) +#define ENVOY_IPV6_MTU_DISCOVER_VALUE IPV6_PMTUDISC_DO +#endif + /** * Interface representing a single filter chain info. */ diff --git a/source/common/listener_manager/listener_impl.cc b/source/common/listener_manager/listener_impl.cc index 1e2c8cd03e..23350a0c22 100644 --- a/source/common/listener_manager/listener_impl.cc +++ b/source/common/listener_manager/listener_impl.cc @@ -677,6 +677,13 @@ void ListenerImpl::buildListenSocketOptions( addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildUdpGroOptions()); } + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + addListenSocketOptions( + listen_socket_options_list_[i], + Network::SocketOptionFactory::buildDoNotFragmentOptions( + /*mapped_v6*/ addresses_[i]->ip()->version() == Network::Address::IpVersion::v6 && + !addresses_[i]->ip()->ipv6()->v6only())); + } // Additional factory specific options. ASSERT(udp_listener_config_->listener_factory_ != nullptr, diff --git a/source/common/network/socket_option_factory.cc b/source/common/network/socket_option_factory.cc index 094ebca074..83fc9bfc30 100644 --- a/source/common/network/socket_option_factory.cc +++ b/source/common/network/socket_option_factory.cc @@ -181,5 +181,32 @@ std::unique_ptr SocketOptionFactory::buildIpRecvTosOptions() { return options; } +std::unique_ptr +SocketOptionFactory::buildDoNotFragmentOptions(bool supports_v4_mapped_v6_addresses) { + auto options = std::make_unique(); +#ifdef ENVOY_IP_DONTFRAG + options->push_back(std::make_shared( + envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_IP_DONTFRAG, ENVOY_IPV6_DONTFRAG, + 1)); + // v4 mapped v6 addresses don't support ENVOY_IP_DONTFRAG on MAC OS. + (void)supports_v4_mapped_v6_addresses; +#elif defined(ENVOY_IP_MTU_DISCOVER) + options->push_back(std::make_shared( + envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_IP_MTU_DISCOVER, + ENVOY_IP_MTU_DISCOVER_VALUE, ENVOY_IPV6_MTU_DISCOVER, ENVOY_IPV6_MTU_DISCOVER_VALUE)); + + if (supports_v4_mapped_v6_addresses) { + ENVOY_LOG_MISC(trace, "Also apply the V4 option to v6 socket to support v4-mapped addresses."); + options->push_back( + std::make_shared(envoy::config::core::v3::SocketOption::STATE_PREBIND, + ENVOY_IP_MTU_DISCOVER, ENVOY_IP_MTU_DISCOVER_VALUE)); + } +#else + (void)supports_v4_mapped_v6_addresses; + ENVOY_LOG_MISC(trace, "Platform supports neither socket option IP_DONTFRAG nor IP_MTU_DISCOVER"); +#endif + return options; +} + } // namespace Network } // namespace Envoy diff --git a/source/common/network/socket_option_factory.h b/source/common/network/socket_option_factory.h index 90f521bc67..94bfd5f839 100644 --- a/source/common/network/socket_option_factory.h +++ b/source/common/network/socket_option_factory.h @@ -38,6 +38,12 @@ class SocketOptionFactory : Logger::Loggable { static std::unique_ptr buildUdpGroOptions(); static std::unique_ptr buildZeroSoLingerOptions(); static std::unique_ptr buildIpRecvTosOptions(); + /** + * @param supports_v4_mapped_v6_addresses true if this option is to be applied to a v6 socket with + * v4-mapped v6 address(i.e. ::ffff:172.21.0.6) support. + */ + static std::unique_ptr + buildDoNotFragmentOptions(bool supports_v4_mapped_v6_addresses); }; } // namespace Network } // namespace Envoy diff --git a/source/common/quic/envoy_quic_utils.cc b/source/common/quic/envoy_quic_utils.cc index 098abd6b79..28d59302e5 100644 --- a/source/common/quic/envoy_quic_utils.cc +++ b/source/common/quic/envoy_quic_utils.cc @@ -17,11 +17,13 @@ namespace Quic { namespace { Network::Address::InstanceConstSharedPtr -getLoopbackAddress(const Network::Address::IpVersion version) { - if (version == Network::Address::IpVersion::v6) { - return std::make_shared("::1"); +getLoopbackAddress(Network::Address::InstanceConstSharedPtr peer_address) { + if (peer_address->ip()->version() == Network::Address::IpVersion::v6) { + return std::make_shared( + "::1", 0, &peer_address->socketInterface(), peer_address->ip()->ipv6()->v6only()); } - return std::make_shared("127.0.0.1"); + return std::make_shared("127.0.0.1", + &peer_address->socketInterface()); } } // namespace @@ -200,7 +202,7 @@ createConnectionSocket(const Network::Address::InstanceConstSharedPtr& peer_addr Network::Socket::Type::Datagram, // Use the loopback address if `local_addr` is null, to pass in the socket interface used to // create the IoHandle, without having to make the more expensive `getifaddrs` call. - local_addr ? local_addr : getLoopbackAddress(peer_addr->ip()->version()), peer_addr, + local_addr ? local_addr : getLoopbackAddress(peer_addr), peer_addr, Network::SocketCreationOptions{false, max_addresses_cache_size}); connection_socket->setDetectedTransportProtocol("quic"); if (!connection_socket->isOpen()) { @@ -215,6 +217,25 @@ createConnectionSocket(const Network::Address::InstanceConstSharedPtr& peer_addr if (prefer_gro && Api::OsSysCallsSingleton::get().supportsUdpGro()) { connection_socket->addOptions(Network::SocketOptionFactory::buildUdpGroOptions()); } + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int v6_only = 0; + if (connection_socket->ipVersion().has_value() && + connection_socket->ipVersion().value() == Network::Address::IpVersion::v6) { + socklen_t v6_only_len = sizeof(v6_only); + Api::SysCallIntResult result = + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_V6ONLY, &v6_only, &v6_only_len); + if (result.return_value_ != 0) { + ENVOY_LOG_MISC( + error, "Failed to get IPV6_V6ONLY socket option, getsockopt() returned {}, errno {}", + result.return_value_, result.errno_); + connection_socket->close(); + return connection_socket; + } + } + connection_socket->addOptions(Network::SocketOptionFactory::buildDoNotFragmentOptions( + /*mapped_v6*/ connection_socket->ipVersion().value() == Network::Address::IpVersion::v6 && + v6_only == 0)); + } if (options != nullptr) { connection_socket->addOptions(options); } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 6ec39cd77d..8306957141 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -94,6 +94,7 @@ RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); RUNTIME_GUARD(envoy_reloadable_features_strict_duration_validation); RUNTIME_GUARD(envoy_reloadable_features_tcp_tunneling_send_downstream_fin_on_upstream_trailers); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); +RUNTIME_GUARD(envoy_reloadable_features_udp_set_do_not_fragment); RUNTIME_GUARD(envoy_reloadable_features_udp_socket_apply_aggregated_read_limit); RUNTIME_GUARD(envoy_reloadable_features_uhv_allow_malformed_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_upstream_remote_address_use_connection); diff --git a/test/common/listener_manager/listener_manager_impl_quic_only_test.cc b/test/common/listener_manager/listener_manager_impl_quic_only_test.cc index 05dd2cf623..f088a12dd0 100644 --- a/test/common/listener_manager/listener_manager_impl_quic_only_test.cc +++ b/test/common/listener_manager/listener_manager_impl_quic_only_test.cc @@ -24,6 +24,21 @@ class MockSupportsUdpGso : public Api::OsSysCallsImpl { class ListenerManagerImplQuicOnlyTest : public ListenerManagerImplTest { public: + size_t expectedNumSocketOptions() { + // SO_REUSEPORT, IP_PKTINFO and IP_MTU_DISCOVER/IP_DONTFRAG. + const size_t num_platform_independent_socket_options = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment") ? 3 : 2; + size_t num_platform_dependent_socket_options = 0; +#ifdef SO_RXQ_OVFL + ++num_platform_dependent_socket_options; +#endif + if (Api::OsSysCallsSingleton::get().supportsUdpGro()) { + // SO_REUSEPORT + ++num_platform_dependent_socket_options; + } + return num_platform_dependent_socket_options + num_platform_independent_socket_options; + } + NiceMock udp_gso_syscall_; TestThreadsafeSingletonInjector os_calls{&udp_gso_syscall_}; Api::OsSysCallsImpl os_sys_calls_actual_; @@ -106,13 +121,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { .WillByDefault(Return(os_sys_calls_actual_.supportsUdpGso())); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -138,6 +147,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0] @@ -195,13 +216,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicWriterFromConfig) { ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -227,6 +242,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicWriterFromConfig) { } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0] @@ -303,13 +330,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithExplictConnection ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -335,6 +356,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithExplictConnection } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0] @@ -359,13 +392,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFilterFromConfig) { ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -391,6 +418,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFilterFromConfig) { } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); // Verify that the right filter chain type is installed. diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 2f1c560192..36c0a8f1b5 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -323,6 +323,7 @@ envoy_cc_test( "//test/mocks/upstream:cluster_info_mocks", "//test/mocks/upstream:transport_socket_match_mocks", "//test/test_common:test_runtime_lib", + "//test/test_common:threadsafe_singleton_injector_lib", ], ) diff --git a/test/common/quic/client_connection_factory_impl_test.cc b/test/common/quic/client_connection_factory_impl_test.cc index 5f505584b9..7b407eb46f 100644 --- a/test/common/quic/client_connection_factory_impl_test.cc +++ b/test/common/quic/client_connection_factory_impl_test.cc @@ -14,6 +14,7 @@ #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "quiche/quic/core/crypto/quic_client_session_cache.h" #include "quiche/quic/core/deterministic_connection_id_generator.h" @@ -134,6 +135,49 @@ TEST_P(QuicNetworkConnectionTest, SocketOptions) { client_connection->close(Network::ConnectionCloseType::NoFlush); } +TEST_P(QuicNetworkConnectionTest, PreBindSocketOptionsFailure) { + initialize(); + + auto socket_option = std::make_shared(); + auto socket_options = std::make_shared(); + socket_options->push_back(socket_option); + EXPECT_CALL(*socket_option, setOption(_, envoy::config::core::v3::SocketOption::STATE_PREBIND)) + .WillOnce(Return(false)); + + std::unique_ptr client_connection = createQuicNetworkConnection( + *quic_info_, crypto_config_, + quic::QuicServerId{factory_->clientContextConfig()->serverNameIndication(), PEER_PORT}, + dispatcher_, test_address_, test_address_, quic_stat_names_, {}, *store_.rootScope(), + socket_options, nullptr, connection_id_generator_, *factory_); + EnvoyQuicClientSession* session = static_cast(client_connection.get()); + session->Initialize(); + client_connection->connect(); + EXPECT_FALSE(session->connection()->connected()); + EXPECT_EQ(client_connection->state(), Network::Connection::State::Closed); +} + +TEST_P(QuicNetworkConnectionTest, PostBindSocketOptionsFailure) { + initialize(); + + auto socket_option = std::make_shared(); + auto socket_options = std::make_shared(); + socket_options->push_back(socket_option); + EXPECT_CALL(*socket_option, setOption(_, envoy::config::core::v3::SocketOption::STATE_PREBIND)); + EXPECT_CALL(*socket_option, setOption(_, envoy::config::core::v3::SocketOption::STATE_BOUND)) + .WillOnce(Return(false)); + + std::unique_ptr client_connection = createQuicNetworkConnection( + *quic_info_, crypto_config_, + quic::QuicServerId{factory_->clientContextConfig()->serverNameIndication(), PEER_PORT}, + dispatcher_, test_address_, test_address_, quic_stat_names_, {}, *store_.rootScope(), + socket_options, nullptr, connection_id_generator_, *factory_); + EnvoyQuicClientSession* session = static_cast(client_connection.get()); + session->Initialize(); + client_connection->connect(); + EXPECT_FALSE(session->connection()->connected()); + EXPECT_EQ(client_connection->state(), Network::Connection::State::Closed); +} + TEST_P(QuicNetworkConnectionTest, LocalAddress) { initialize(); Network::Address::InstanceConstSharedPtr local_addr = @@ -151,9 +195,41 @@ TEST_P(QuicNetworkConnectionTest, LocalAddress) { EXPECT_TRUE(client_connection->connecting()); EXPECT_EQ(Network::Connection::State::Open, client_connection->state()); EXPECT_THAT(client_connection->connectionInfoProvider().localAddress(), testing::NotNull()); + if (GetParam() == Network::Address::IpVersion::v6) { + EXPECT_TRUE(client_connection->connectionInfoProvider().localAddress()->ip()->ipv6()->v6only()); + } client_connection->close(Network::ConnectionCloseType::NoFlush); } +class MockGetSockOptSysCalls : public Api::OsSysCallsImpl { +public: + MOCK_METHOD(Api::SysCallIntResult, getsockopt, + (os_fd_t sockfd, int level, int optname, void* optval, socklen_t* optlen)); +}; + +TEST_P(QuicNetworkConnectionTest, GetV6OnlySocketOptionFailure) { + if (GetParam() == Network::Address::IpVersion::v4) { + return; + } + initialize(); + MockGetSockOptSysCalls os_sys_calls; + TestThreadsafeSingletonInjector singleton_injector{&os_sys_calls}; + + EXPECT_CALL(os_sys_calls, getsockopt(_, IPPROTO_IPV6, IPV6_V6ONLY, _, _)) + .WillOnce(Return(Api::SysCallIntResult{-1, SOCKET_ERROR_NOT_SUP})); + std::unique_ptr client_connection = createQuicNetworkConnection( + *quic_info_, crypto_config_, + quic::QuicServerId{factory_->clientContextConfig()->serverNameIndication(), PEER_PORT}, + dispatcher_, test_address_, test_address_, quic_stat_names_, {}, *store_.rootScope(), nullptr, + nullptr, connection_id_generator_, *factory_); + EnvoyQuicClientSession* session = static_cast(client_connection.get()); + session->Initialize(); + client_connection->connect(); + EXPECT_TRUE(client_connection->connecting()); + EXPECT_FALSE(session->connection()->connected()); + EXPECT_EQ(client_connection->state(), Network::Connection::State::Closed); +} + TEST_P(QuicNetworkConnectionTest, Srtt) { initialize(); diff --git a/test/common/quic/envoy_quic_utils_test.cc b/test/common/quic/envoy_quic_utils_test.cc index 3fbfe54e07..c3ea2ff3c4 100644 --- a/test/common/quic/envoy_quic_utils_test.cc +++ b/test/common/quic/envoy_quic_utils_test.cc @@ -297,15 +297,58 @@ TEST(EnvoyQuicUtilsTest, CreateConnectionSocket) { EXPECT_TRUE(connection_socket->isOpen()); EXPECT_TRUE(connection_socket->ioHandle().wasConnected()); EXPECT_EQ("127.0.0.1", no_local_addr->ip()->addressAsString()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int value = 0; + socklen_t val_length = sizeof(value); +#ifdef ENVOY_IP_DONTFRAG + RELEASE_ASSERT(connection_socket->getSocketOption(IPPROTO_IP, IP_DONTFRAG, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_DONTFRAG"); + EXPECT_EQ(value, 1); +#else + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IP, IP_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_MTU_DISCOVER"); + EXPECT_EQ(value, IP_PMTUDISC_DO); +#endif + } connection_socket->close(); Network::Address::InstanceConstSharedPtr local_addr_v6 = - std::make_shared("::1"); + std::make_shared("::1", 0, nullptr, /*v6only*/ true); Network::Address::InstanceConstSharedPtr peer_addr_v6 = - std::make_shared("::1", 54321, nullptr); + std::make_shared("::1", 54321, nullptr, /*v6only*/ false); connection_socket = createConnectionSocket(peer_addr_v6, local_addr_v6, nullptr); EXPECT_TRUE(connection_socket->isOpen()); EXPECT_TRUE(connection_socket->ioHandle().wasConnected()); + EXPECT_TRUE(local_addr_v6->ip()->ipv6()->v6only()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int value = 0; + socklen_t val_length = sizeof(value); +#ifdef ENVOY_IP_DONTFRAG + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_DONTFRAG, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_DONTFRAG"); + ; + EXPECT_EQ(value, 1); +#else + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_MTU_DISCOVER"); + EXPECT_EQ(value, IPV6_PMTUDISC_DO); + // The v4 socket option is not applied to v6-only socket. + value = 0; + val_length = sizeof(value); + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IP, IP_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_MTU_DISCOVER"); + EXPECT_NE(value, IP_PMTUDISC_DO); +#endif + } connection_socket->close(); Network::Address::InstanceConstSharedPtr no_local_addr_v6 = nullptr; @@ -313,6 +356,32 @@ TEST(EnvoyQuicUtilsTest, CreateConnectionSocket) { EXPECT_TRUE(connection_socket->isOpen()); EXPECT_TRUE(connection_socket->ioHandle().wasConnected()); EXPECT_EQ("::1", no_local_addr_v6->ip()->addressAsString()); + EXPECT_FALSE(no_local_addr_v6->ip()->ipv6()->v6only()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int value = 0; + socklen_t val_length = sizeof(value); +#ifdef ENVOY_IP_DONTFRAG + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_DONTFRAG, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_DONTFRAG"); + EXPECT_EQ(value, 1); +#else + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_MTU_DISCOVER"); + EXPECT_EQ(value, IPV6_PMTUDISC_DO); + // The v4 socket option is also applied to dual stack socket. + value = 0; + val_length = sizeof(value); + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IP, IP_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_MTU_DISCOVER"); + EXPECT_EQ(value, IP_PMTUDISC_DO); +#endif + } connection_socket->close(); } diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 452a48b9a2..cd5d25bc3f 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -45,6 +45,7 @@ deadcode DFP Dynatrace DOM +DONTFRAG Gasd GiB IPTOS From 5fb93b9655b2300e6ec4cf8793376f3549888650 Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Mon, 7 Oct 2024 12:16:15 -0400 Subject: [PATCH 21/77] mobile: Enable integration tests to run both a HTTP and a proxy server (#36454) This will be used by a subsequent change to test the Apple PAC proxy resolver. Signed-off-by: Ali Beyad --- .../integration/test_server_interface.cc | 86 +++++++++++++++---- .../integration/test_server_interface.h | 43 +++++++--- mobile/test/objective-c/EnvoyTestServer.h | 14 +-- mobile/test/objective-c/EnvoyTestServer.mm | 26 ++++-- .../integration/CancelGRPCStreamTest.swift | 4 +- .../swift/integration/CancelStreamTest.swift | 4 +- .../integration/EndToEndNetworkingTest.swift | 4 +- .../integration/FilterResetIdleTest.swift | 4 +- .../integration/GRPCReceiveErrorTest.swift | 4 +- .../swift/integration/IdleTimeoutTest.swift | 4 +- .../swift/integration/KeyValueStoreTest.swift | 4 +- .../swift/integration/ReceiveDataTest.swift | 4 +- .../ResetConnectivityStateTest.swift | 4 +- .../test/swift/integration/SendDataTest.swift | 4 +- .../swift/integration/SendHeadersTest.swift | 4 +- .../swift/integration/SendTrailersTest.swift | 4 +- .../integration/SetEventTrackerTest.swift | 4 +- .../SetEventTrackerTestNoTracker.swift | 4 +- .../swift/integration/SetLoggerTest.swift | 4 +- .../proxying/HTTPRequestUsingProxyTest.swift | 20 ++--- 20 files changed, 166 insertions(+), 83 deletions(-) diff --git a/mobile/test/common/integration/test_server_interface.cc b/mobile/test/common/integration/test_server_interface.cc index 91dc5f46e7..c704d1c8f9 100644 --- a/mobile/test/common/integration/test_server_interface.cc +++ b/mobile/test/common/integration/test_server_interface.cc @@ -2,39 +2,91 @@ // NOLINT(namespace-envoy) -static std::shared_ptr strong_test_server_; -static std::weak_ptr weak_test_server_; +static std::shared_ptr strong_test_http_server_; +static std::weak_ptr weak_test_http_server_; +static std::shared_ptr strong_test_proxy_server_; +static std::weak_ptr weak_test_proxy_server_; -static std::shared_ptr test_server() { return weak_test_server_.lock(); } +static std::shared_ptr test_http_server() { + if (strong_test_http_server_ == nullptr) { + return nullptr; + } + return weak_test_http_server_.lock(); +} + +static std::shared_ptr test_proxy_server() { + if (strong_test_proxy_server_ == nullptr) { + return nullptr; + } + return weak_test_proxy_server_.lock(); +} -void start_server(Envoy::TestServerType test_server_type) { - strong_test_server_ = std::make_shared(); - weak_test_server_ = strong_test_server_; +void start_http_server(Envoy::TestServerType test_server_type) { + strong_test_http_server_ = std::make_shared(); + weak_test_http_server_ = strong_test_http_server_; - if (auto server = test_server()) { + ASSERT(test_server_type == Envoy::TestServerType::HTTP1_WITHOUT_TLS || + test_server_type == Envoy::TestServerType::HTTP1_WITH_TLS || + test_server_type == Envoy::TestServerType::HTTP2_WITH_TLS || + test_server_type == Envoy::TestServerType::HTTP3, + "Cannot start a proxy server with start_http_server. Use start_proxy_server instead."); + if (auto server = test_http_server()) { server->start(test_server_type, 0); } } -void shutdown_server() { - // Reset the primary handle to the test_server, +void start_proxy_server(Envoy::TestServerType test_server_type) { + strong_test_proxy_server_ = std::make_shared(); + weak_test_proxy_server_ = strong_test_proxy_server_; + + ASSERT(test_server_type == Envoy::TestServerType::HTTP_PROXY || + test_server_type == Envoy::TestServerType::HTTPS_PROXY, + "Cannot start a HTTP server with start_proxy_server. Use start_http_server instead."); + if (auto server = test_proxy_server()) { + server->start(test_server_type, 0); + } +} + +void shutdown_http_server() { + if (strong_test_http_server_ == nullptr) { + return; + } + // Reset the primary handle to the test_http_server, // but retain it long enough to synchronously shutdown. - auto server = strong_test_server_; - strong_test_server_.reset(); + auto server = strong_test_http_server_; + strong_test_http_server_.reset(); server->shutdown(); } -int get_server_port() { - if (auto server = test_server()) { +void shutdown_proxy_server() { + if (strong_test_proxy_server_ == nullptr) { + return; + } + // Reset the primary handle to the test_proxy_server, + // but retain it long enough to synchronously shutdown. + auto server = strong_test_proxy_server_; + strong_test_proxy_server_.reset(); + server->shutdown(); +} + +int get_http_server_port() { + if (auto server = test_http_server()) { + return server->getPort(); + } + return -1; // failure +} + +int get_proxy_server_port() { + if (auto server = test_proxy_server()) { return server->getPort(); } return -1; // failure } -void set_headers_and_data(absl::string_view header_key, absl::string_view header_value, - absl::string_view response_body) { - if (auto server = test_server()) { - // start_server() must be called before headers and data can be added. +void set_http_headers_and_data(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body) { + if (auto server = test_http_server()) { + // start_http_server() must be called before headers and data can be added. ASSERT(server); server->setHeadersAndData(header_key, header_value, response_body); } diff --git a/mobile/test/common/integration/test_server_interface.h b/mobile/test/common/integration/test_server_interface.h index 0b51d2a6e5..21e43b722b 100644 --- a/mobile/test/common/integration/test_server_interface.h +++ b/mobile/test/common/integration/test_server_interface.h @@ -9,28 +9,47 @@ extern "C" { // functions #endif /** - * Starts the server. Can only have one server active per JVM. This is blocking until the port can - * start accepting requests. + * Starts the HTTP server. Can only have one HTTP server active per process. This is blocking + * until the port can start accepting requests. */ -void start_server(Envoy::TestServerType test_server_type); +void start_http_server(Envoy::TestServerType test_server_type); /** - * Shutdowns the server. Can be restarted later. This is blocking until the server has freed all - * resources. + * Starts the Proxy server. Can only have one proxy server active per process. This is blocking + * until the port can start accepting requests. */ -void shutdown_server(); +void start_proxy_server(Envoy::TestServerType test_server_type); /** - * Returns the port that got attributed. Can only be called once the server has been started. + * Shuts down the HTTP server. Can be restarted later. This is blocking until the server has freed + * all resources. */ -int get_server_port(); +void shutdown_http_server(); /** - * Set response data for server to return for any URL. Can only be called once the server has been - * started. + * Shuts down the proxy server. Can be restarted later. This is blocking until the server has freed + * all resources. */ -void set_headers_and_data(absl::string_view header_key, absl::string_view header_value, - absl::string_view response_body); +void shutdown_proxy_server(); + +/** + * Returns the port that got attributed to the HTTP server. Can only be called once the server has + * been started. + */ +int get_http_server_port(); + +/** + * Returns the port that got attributed to the proxy server. Can only be called once the server has + * been started. + */ +int get_proxy_server_port(); + +/** + * Set response data for the HTTP server to return for any URL. Can only be called once the server + * has been started. + */ +void set_http_headers_and_data(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body); #ifdef __cplusplus } // functions diff --git a/mobile/test/objective-c/EnvoyTestServer.h b/mobile/test/objective-c/EnvoyTestServer.h index dbc9758d9e..8d51cf67dd 100644 --- a/mobile/test/objective-c/EnvoyTestServer.h +++ b/mobile/test/objective-c/EnvoyTestServer.h @@ -10,17 +10,21 @@ // (https://docs.engflow.com/re/client/platform-options-reference.html#sandboxallowed). @interface EnvoyTestServer : NSObject -// Get the port of the upstream server. -+ (NSInteger)getEnvoyPort; +// Get the port of the upstream HTTP server. ++ (NSInteger)getHttpPort; +// Get the port of the upstream proxy server. ++ (NSInteger)getProxyPort; // Starts a server with HTTP1 and no TLS. + (void)startHttp1PlaintextServer; // Starts a server as a HTTP proxy. + (void)startHttpProxyServer; // Starts a server as a HTTPS proxy. + (void)startHttpsProxyServer; -// Shut down and clean up server. -+ (void)shutdownTestServer; -// Add response data to the upstream. +// Shut down and clean up the HTTP server. ++ (void)shutdownTestHttpServer; +// Shut down and clean up the Proxy server. ++ (void)shutdownTestProxyServer; +// Add response data to the HTTP server. + (void)setHeadersAndData:(NSString *)header_key header_value:(NSString *)header_value response_body:(NSString *)response_body; diff --git a/mobile/test/objective-c/EnvoyTestServer.mm b/mobile/test/objective-c/EnvoyTestServer.mm index e1d64714fd..f07f707964 100644 --- a/mobile/test/objective-c/EnvoyTestServer.mm +++ b/mobile/test/objective-c/EnvoyTestServer.mm @@ -3,31 +3,39 @@ @implementation EnvoyTestServer -+ (NSInteger)getEnvoyPort { - return get_server_port(); ++ (NSInteger)getHttpPort { + return get_http_server_port(); +} + ++ (NSInteger)getProxyPort { + return get_proxy_server_port(); } + (void)startHttp1PlaintextServer { - start_server(Envoy::TestServerType::HTTP1_WITHOUT_TLS); + start_http_server(Envoy::TestServerType::HTTP1_WITHOUT_TLS); } + (void)startHttpProxyServer { - start_server(Envoy::TestServerType::HTTP_PROXY); + start_proxy_server(Envoy::TestServerType::HTTP_PROXY); } + (void)startHttpsProxyServer { - start_server(Envoy::TestServerType::HTTPS_PROXY); + start_proxy_server(Envoy::TestServerType::HTTPS_PROXY); +} + ++ (void)shutdownTestHttpServer { + shutdown_http_server(); } -+ (void)shutdownTestServer { - shutdown_server(); ++ (void)shutdownTestProxyServer { + shutdown_proxy_server(); } + (void)setHeadersAndData:(NSString *)header_key header_value:(NSString *)header_value response_body:(NSString *)response_body { - set_headers_and_data([header_key UTF8String], [header_value UTF8String], - [response_body UTF8String]); + set_http_headers_and_data([header_key UTF8String], [header_value UTF8String], + [response_body UTF8String]); } @end diff --git a/mobile/test/swift/integration/CancelGRPCStreamTest.swift b/mobile/test/swift/integration/CancelGRPCStreamTest.swift index 6b35c002f0..2ee52fa208 100644 --- a/mobile/test/swift/integration/CancelGRPCStreamTest.swift +++ b/mobile/test/swift/integration/CancelGRPCStreamTest.swift @@ -70,7 +70,7 @@ final class CancelGRPCStreamTests: XCTestCase { let requestHeaders = GRPCRequestHeadersBuilder( scheme: "http", - authority: "localhost:" + String(EnvoyTestServer.getEnvoyPort()), + authority: "localhost:" + String(EnvoyTestServer.getHttpPort()), path: "/") .build() @@ -87,6 +87,6 @@ final class CancelGRPCStreamTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/CancelStreamTest.swift b/mobile/test/swift/integration/CancelStreamTest.swift index a53f165b2c..44314123fc 100644 --- a/mobile/test/swift/integration/CancelStreamTest.swift +++ b/mobile/test/swift/integration/CancelStreamTest.swift @@ -68,7 +68,7 @@ final class CancelStreamTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/") .build() @@ -86,6 +86,6 @@ final class CancelStreamTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/EndToEndNetworkingTest.swift b/mobile/test/swift/integration/EndToEndNetworkingTest.swift index 638eefc623..15e89d6501 100644 --- a/mobile/test/swift/integration/EndToEndNetworkingTest.swift +++ b/mobile/test/swift/integration/EndToEndNetworkingTest.swift @@ -23,7 +23,7 @@ final class EndToEndNetworkingTest: XCTestCase { "x-response-foo", header_value: "aaa", response_body: "hello world") let headersExpectation = self.expectation(description: "Response headers received") let dataExpectation = self.expectation(description: "Response data received") - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder( method: .get, scheme: "http", authority: "localhost:" + port, path: "/" ) @@ -60,6 +60,6 @@ final class EndToEndNetworkingTest: XCTestCase { XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/FilterResetIdleTest.swift b/mobile/test/swift/integration/FilterResetIdleTest.swift index da7f5a5288..7f59c66340 100644 --- a/mobile/test/swift/integration/FilterResetIdleTest.swift +++ b/mobile/test/swift/integration/FilterResetIdleTest.swift @@ -122,7 +122,7 @@ final class FilterResetIdleTests: XCTestCase { cancelExpectation.isInverted = true EnvoyTestServer.startHttp1PlaintextServer() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let engine = EngineBuilder() .setLogLevel(.debug) @@ -166,6 +166,6 @@ final class FilterResetIdleTests: XCTestCase { ) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/GRPCReceiveErrorTest.swift b/mobile/test/swift/integration/GRPCReceiveErrorTest.swift index a1266db281..99efaaac3d 100644 --- a/mobile/test/swift/integration/GRPCReceiveErrorTest.swift +++ b/mobile/test/swift/integration/GRPCReceiveErrorTest.swift @@ -86,7 +86,7 @@ final class GRPCReceiveErrorTests: XCTestCase { let requestHeaders = GRPCRequestHeadersBuilder( scheme: "http", - authority: "localhost:" + String(EnvoyTestServer.getEnvoyPort()), + authority: "localhost:" + String(EnvoyTestServer.getHttpPort()), path: "/pb.api.v1.Foo/GetBar") .build() let message = Data([1, 2, 3, 4, 5]) @@ -114,6 +114,6 @@ final class GRPCReceiveErrorTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/IdleTimeoutTest.swift b/mobile/test/swift/integration/IdleTimeoutTest.swift index 95324f9195..9735c12024 100644 --- a/mobile/test/swift/integration/IdleTimeoutTest.swift +++ b/mobile/test/swift/integration/IdleTimeoutTest.swift @@ -95,7 +95,7 @@ final class IdleTimeoutTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder( method: .get, scheme: "http", authority: "localhost:" + port, path: "/" ) @@ -119,6 +119,6 @@ final class IdleTimeoutTests: XCTestCase { ) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/KeyValueStoreTest.swift b/mobile/test/swift/integration/KeyValueStoreTest.swift index 595bae1345..d29363572e 100644 --- a/mobile/test/swift/integration/KeyValueStoreTest.swift +++ b/mobile/test/swift/integration/KeyValueStoreTest.swift @@ -66,7 +66,7 @@ final class KeyValueStoreTests: XCTestCase { let requestHeaders = RequestHeadersBuilder( method: .get, scheme: "http", - authority: "localhost:" + String(EnvoyTestServer.getEnvoyPort()), path: "/simple.txt" + authority: "localhost:" + String(EnvoyTestServer.getHttpPort()), path: "/simple.txt" ) .build() @@ -84,6 +84,6 @@ final class KeyValueStoreTests: XCTestCase { ) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/ReceiveDataTest.swift b/mobile/test/swift/integration/ReceiveDataTest.swift index 483c947e72..0ac82ce0c8 100644 --- a/mobile/test/swift/integration/ReceiveDataTest.swift +++ b/mobile/test/swift/integration/ReceiveDataTest.swift @@ -33,7 +33,7 @@ final class ReceiveDataTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -67,6 +67,6 @@ final class ReceiveDataTests: XCTestCase { XCTAssertEqual(actualResponseBody, directResponseBody) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/ResetConnectivityStateTest.swift b/mobile/test/swift/integration/ResetConnectivityStateTest.swift index ddeb5bb059..51e357c882 100644 --- a/mobile/test/swift/integration/ResetConnectivityStateTest.swift +++ b/mobile/test/swift/integration/ResetConnectivityStateTest.swift @@ -30,7 +30,7 @@ final class ResetConnectivityStateTest: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -98,6 +98,6 @@ final class ResetConnectivityStateTest: XCTestCase { XCTAssertTrue(resultEndStream2) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SendDataTest.swift b/mobile/test/swift/integration/SendDataTest.swift index 4a7b736b6f..62040a432f 100644 --- a/mobile/test/swift/integration/SendDataTest.swift +++ b/mobile/test/swift/integration/SendDataTest.swift @@ -41,7 +41,7 @@ final class SendDataTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -67,6 +67,6 @@ final class SendDataTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SendHeadersTest.swift b/mobile/test/swift/integration/SendHeadersTest.swift index 46de840b92..07c6247607 100644 --- a/mobile/test/swift/integration/SendHeadersTest.swift +++ b/mobile/test/swift/integration/SendHeadersTest.swift @@ -33,7 +33,7 @@ final class SendHeadersTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -62,6 +62,6 @@ final class SendHeadersTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SendTrailersTest.swift b/mobile/test/swift/integration/SendTrailersTest.swift index 5f1caa7a24..ca67509351 100644 --- a/mobile/test/swift/integration/SendTrailersTest.swift +++ b/mobile/test/swift/integration/SendTrailersTest.swift @@ -47,7 +47,7 @@ final class SendTrailersTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -73,6 +73,6 @@ final class SendTrailersTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SetEventTrackerTest.swift b/mobile/test/swift/integration/SetEventTrackerTest.swift index beec655a17..d3b8c67426 100644 --- a/mobile/test/swift/integration/SetEventTrackerTest.swift +++ b/mobile/test/swift/integration/SetEventTrackerTest.swift @@ -42,7 +42,7 @@ final class SetEventTrackerTest: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -55,6 +55,6 @@ final class SetEventTrackerTest: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [eventExpectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift b/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift index 33f5035191..d37ce77046 100644 --- a/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift +++ b/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift @@ -36,7 +36,7 @@ final class SetEventTrackerTestNoTracker: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -52,6 +52,6 @@ final class SetEventTrackerTestNoTracker: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SetLoggerTest.swift b/mobile/test/swift/integration/SetLoggerTest.swift index 96836ebf9d..0d31beac5b 100644 --- a/mobile/test/swift/integration/SetLoggerTest.swift +++ b/mobile/test/swift/integration/SetLoggerTest.swift @@ -25,7 +25,7 @@ final class LoggerTests: XCTestCase { description: "Run received log event via event tracker") EnvoyTestServer.startHttp1PlaintextServer() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let engine = EngineBuilder() .setLogLevel(.debug) @@ -64,6 +64,6 @@ final class LoggerTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [logEventExpectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift b/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift index 67f3e215bb..48f5d17a08 100644 --- a/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift +++ b/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift @@ -53,7 +53,7 @@ final class HTTPRequestUsingProxyTest: XCTestCase { func testHTTPRequestUsingProxy() throws { EnvoyTestServer.startHttpProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") @@ -77,13 +77,13 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } // https://github.com/envoyproxy/envoy/issues/33014 func skipped_testHTTPSRequestUsingProxy() throws { EnvoyTestServer.startHttpsProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") let responseHeadersExpectation = @@ -136,13 +136,13 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } // https://github.com/envoyproxy/envoy/issues/33014 func skipped_testHTTPSRequestUsingPacFileUrlResolver() throws { EnvoyTestServer.startHttpsProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") let responseHeadersExpectation = @@ -195,12 +195,12 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } func testTwoHTTPRequestsUsingProxy() throws { EnvoyTestServer.startHttpProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") @@ -227,12 +227,12 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } func testHTTPRequestUsingProxyCancelStream() throws { EnvoyTestServer.startHttpProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") @@ -271,7 +271,7 @@ final class HTTPRequestUsingProxyTest: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [cancelExpectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } // TODO(abeyad): Add test for proxy system settings updated. From 0aff2110ee0aafed48457192382418af6c1910ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Mon, 7 Oct 2024 12:28:14 -0400 Subject: [PATCH 22/77] Bump abseil to LTS 20240722.0. (#36317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also add one additional patch to CEL to handle `absl::StrCat` and friends moving to `absl/string/str_cat.h`. Signed-off-by: Alejandro R. Sedeño --- bazel/cel-cpp.patch | 12 ++++++++++++ bazel/external/quiche.BUILD | 1 + bazel/repository_locations.bzl | 6 +++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/bazel/cel-cpp.patch b/bazel/cel-cpp.patch index 34dd41033e..a2ce70b03c 100644 --- a/bazel/cel-cpp.patch +++ b/bazel/cel-cpp.patch @@ -1,3 +1,15 @@ +diff --git a/base/attribute.h b/base/attribute.h +index 9462c180..d6dcce83 100644 +--- a/base/attribute.h ++++ b/base/attribute.h +@@ -23,6 +23,7 @@ + #include + + #include "absl/status/statusor.h" ++#include "absl/strings/str_cat.h" + #include "absl/strings/string_view.h" + #include "absl/types/optional.h" + #include "absl/types/span.h" diff --git a/base/memory.h b/base/memory.h index 3552e19..0fbe618 100644 --- a/base/memory.h diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 3a334c4d4e..09e998ef06 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -4775,6 +4775,7 @@ envoy_quiche_platform_impl_cc_library( "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/log:flags", ], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 503403a344..1c93f3536d 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -181,12 +181,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Abseil", project_desc = "Open source collection of C++ libraries drawn from the most fundamental pieces of Google’s internal codebase", project_url = "https://abseil.io/", - version = "20230802.1", - sha256 = "987ce98f02eefbaf930d6e38ab16aa05737234d7afbab2d5c4ea7adbe50c28ed", + version = "20240722.0", + sha256 = "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3", strip_prefix = "abseil-cpp-{version}", urls = ["https://github.com/abseil/abseil-cpp/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2023-09-18", + release_date = "2024-08-01", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/abseil/abseil-cpp/blob/{version}/LICENSE", From fc10098a84d73a97470c08b34b5b0d11a4aa38ad Mon Sep 17 00:00:00 2001 From: Kuat Date: Mon, 7 Oct 2024 14:10:49 -0700 Subject: [PATCH 23/77] stream info: add bool string serlalizer (#36451) Change-Id: I2065f5ed863ca4b6b439f00dc39de0dcdbe2eb4c Commit Message: add missing serializer for bool object. I discovered this gap while using Wasm `get_property`. Risk Level: low Testing: yes Signed-off-by: Kuat Yessenov --- source/common/stream_info/bool_accessor_impl.h | 4 ++++ test/common/stream_info/bool_accessor_impl_test.cc | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/source/common/stream_info/bool_accessor_impl.h b/source/common/stream_info/bool_accessor_impl.h index 868bfefab9..8de1563fae 100644 --- a/source/common/stream_info/bool_accessor_impl.h +++ b/source/common/stream_info/bool_accessor_impl.h @@ -19,6 +19,10 @@ class BoolAccessorImpl : public BoolAccessor { return message; } + absl::optional serializeAsString() const override { + return value_ ? "true" : "false"; + } + // From BoolAccessor. bool value() const override { return value_; } diff --git a/test/common/stream_info/bool_accessor_impl_test.cc b/test/common/stream_info/bool_accessor_impl_test.cc index dcc8c5ccc9..4f0845f05e 100644 --- a/test/common/stream_info/bool_accessor_impl_test.cc +++ b/test/common/stream_info/bool_accessor_impl_test.cc @@ -22,6 +22,12 @@ TEST(BoolAccessorImplTest, TestProto) { EXPECT_NE(nullptr, message); } +TEST(BoolAccessorImplTest, TestString) { + BoolAccessorImpl accessor(true); + auto str = accessor.serializeAsString(); + EXPECT_EQ("true", str); +} + } // namespace } // namespace StreamInfo } // namespace Envoy From 0d6e45387fce8dc816161ea5a2621e9ec2219a19 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Mon, 7 Oct 2024 17:12:02 -0500 Subject: [PATCH 24/77] Update QUICHE from eaeaa74b2 to de8f411c1 (#36470) https://github.com/google/quiche/compare/eaeaa74b2..de8f411c1 ``` $ git log eaeaa74b2..de8f411c1 --date=short --no-merges --format="%ad %al %s" 2024-10-07 quiche-dev Replace deprecated HexStringToBytes API 2024-10-07 quiche-dev Enabling rolled out flags. 2024-10-07 martinduke Add length field to MoQT control messages. 2024-10-03 quiche-dev Add missing "override" specifier in mock methods. 2024-10-03 quiche-dev Add a new OnParsedClientHelloReceived method to QuicConnection and QuicConnectionDebugVisitor. ``` Signed-off-by: Fredy Wijaya --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1c93f3536d..b6e02d489b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1208,12 +1208,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "eaeaa74b2b4bf4cd9f7a2f44ba8f323fdc55f66a", - sha256 = "1383267a64cb18fca62868e7b54118c223e164d9c0533b11a9a31c779c626f95", + version = "de8f411c1b387499c220ffd6702c43dd7ccaca00", + sha256 = "03a2855a70a3f22e0b723cc63f4d6b1817e894f35c2a441981c7f8152196713e", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2024-10-02", + release_date = "2024-10-07", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 563ec607146fc8c609ab2eaade44cb2d92cc1b17 Mon Sep 17 00:00:00 2001 From: Thomas Habets Date: Tue, 8 Oct 2024 00:58:25 +0100 Subject: [PATCH 25/77] Make DownstreamTiming a struct, instead of a class (#36473) Commit Message: Make DownstreamTiming a struct, instead of a class Additional Description:This makes it consistent with UpstreamTiming, and makes it easier to dependency inject timing info for testing. Risk Level: WCPGW Testing: None Docs Changes: None Release Notes: None Platform Specific Features: None Signed-off-by: Thomas Habets --- envoy/stream_info/stream_info.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index 39ffd18cef..2b14446c28 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -354,8 +354,7 @@ struct UpstreamTiming { absl::optional upstream_handshake_complete_; }; -class DownstreamTiming { -public: +struct DownstreamTiming { void setValue(absl::string_view key, MonotonicTime value) { timings_[key] = value; } absl::optional getValue(absl::string_view value) const { @@ -410,7 +409,6 @@ class DownstreamTiming { last_downstream_header_rx_byte_received_ = time_source.monotonicTime(); } -private: absl::flat_hash_map timings_; // The time when the last byte of the request was received. absl::optional last_downstream_rx_byte_received_; From 187cd09e898bcf3b17682dc8a835dffabfc3533c Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Mon, 7 Oct 2024 20:25:29 -0500 Subject: [PATCH 26/77] getaddrinfo: Add trace info in the DNS resolution details (#36312) Commit Message: This PR adds trace info in the DNS resolution details by setting the `envoy.enable_dfp_dns_trace` runtime flag to true. This can be useful for debugging issues, such as debugging resolve timeout issue. Additional Description: - `PendingQuery::cancel` holds per-query lock instead of a lock for the whole `GetAddrInfoDnsResolver`. - The `getaddrinfo` implementation has been updated to use `std::unique_ptr` instead of `std::shared_ptr` to store the `PendingQuery` given that the `PostCb` now uses `absl::AnyInvocable` instead of `std::function`. Risk Level: low Testing: unit and integration tests Docs Changes: inline Release Notes: inline Platform Specific Features: dynamic forward proxy, mobile --------- Signed-off-by: Fredy Wijaya --- changelogs/current.yaml | 3 + envoy/network/BUILD | 6 +- envoy/network/dns.h | 22 ++ mobile/library/cc/engine_builder.cc | 1 + .../dynamic_forward_proxy/dns_cache_impl.cc | 35 +++- .../dynamic_forward_proxy/dns_cache_impl.h | 5 +- .../dns_resolver/apple/apple_dns_impl.h | 2 + .../network/dns_resolver/cares/dns_impl.h | 4 + .../dns_resolver/getaddrinfo/getaddrinfo.cc | 51 +++-- .../dns_resolver/getaddrinfo/getaddrinfo.h | 44 +++- .../proxy_filter_integration_test.cc | 79 ++++++- .../dynamic_forward_proxy/test_resolver.cc | 2 +- .../dynamic_forward_proxy/test_resolver.h | 26 +-- .../getaddrinfo/getaddrinfo_test.cc | 196 ++++++++++++------ test/mocks/network/mocks.h | 2 + 15 files changed, 362 insertions(+), 116 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 20f1e7683b..fdf30064d8 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -399,6 +399,9 @@ new_features: to configure the number of retries. If this field is not provided, the ``getaddrinfo`` resolver will retry indefinitely until it succeeds or the DNS query times out. This behavior can be reverted by setting the runtime guard ``envoy.reloadable_features.getaddrinfo_num_retries`` to false. +- area: getaddrinfo + change: | + Added ``envoy.enable_dfp_dns_trace`` runtime flag to enable DNS query trace in the DNS resolution details. - area: geoip change: | Added ``envoy.reloadable_features.mmdb_files_reload_enabled`` runtime flag that enables reload of mmdb files by default. diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 95ab4e4677..13bbd44570 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -85,7 +85,11 @@ envoy_cc_library( envoy_cc_library( name = "dns_interface", hdrs = ["dns.h"], - deps = ["//envoy/network:address_interface"], + deps = [ + "//envoy/common:optref_lib", + "//envoy/common:time_interface", + "//envoy/network:address_interface", + ], ) envoy_cc_library( diff --git a/envoy/network/dns.h b/envoy/network/dns.h index c7847babcc..d2781c853e 100644 --- a/envoy/network/dns.h +++ b/envoy/network/dns.h @@ -6,7 +6,9 @@ #include #include +#include "envoy/common/optref.h" #include "envoy/common/pure.h" +#include "envoy/common/time.h" #include "envoy/network/address.h" #include "absl/types/variant.h" @@ -30,11 +32,31 @@ class ActiveDnsQuery { Timeout }; + /** Store the trace information. */ + struct Trace { + /** + * An identifier to store the trace information. The trace is `uint8_t` because the value can + * vary depending on the DNS resolver implementation. + */ + uint8_t trace_; + /** Store the current time of this trace. */ + MonotonicTime time_; + }; + /** * Cancel an outstanding DNS request. * @param reason supplies the cancel reason. */ virtual void cancel(CancelReason reason) PURE; + + /** + * Add a trace for the DNS query. The trace lifetime is tied to the lifetime of `ActiveQuery` and + * `ActiveQuery` will be destroyed upon query completion or cancellation. + */ + virtual void addTrace(uint8_t trace) PURE; + + /** Return the DNS query traces. */ + virtual OptRef> getTraces() PURE; }; /** diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index 8815ab4315..26fafc8188 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -851,6 +851,7 @@ std::unique_ptr EngineBuilder::generate } (*runtime_values.mutable_fields())["disallow_global_stats"].set_bool_value(true); + (*runtime_values.mutable_fields())["enable_dfp_dns_trace"].set_bool_value(true); ProtobufWkt::Struct& overload_values = *(*envoy_layer.mutable_fields())["overload"].mutable_struct_value(); (*overload_values.mutable_fields())["global_downstream_max_connections"].set_string_value( diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index d7841bad16..44b2b98e8f 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -66,6 +66,8 @@ DnsCacheImpl::DnsCacheImpl( ENVOY_LOG(debug, "DNS pre-resolve starting for host {}", host); startCacheLoad(host, hostname.port_value(), false, false); } + enable_dfp_dns_trace_ = context.serverFactoryContext().runtime().snapshot().getBoolean( + "envoy.enable_dfp_dns_trace", false); } DnsCacheImpl::~DnsCacheImpl() { @@ -256,11 +258,10 @@ DnsCacheImpl::PrimaryHostInfo& DnsCacheImpl::getPrimaryHost(const std::string& h void DnsCacheImpl::onResolveTimeout(const std::string& host) { ASSERT(main_thread_dispatcher_.isThreadSafe()); - auto& primary_host = getPrimaryHost(host); ENVOY_LOG_EVENT(debug, "dns_cache_resolve_timeout", "host='{}' resolution timeout", host); stats_.dns_query_timeout_.inc(); - primary_host.active_query_->cancel(Network::ActiveDnsQuery::CancelReason::Timeout); - finishResolve(host, Network::DnsResolver::ResolutionStatus::Failure, "resolve_timeout", {}); + finishResolve(host, Network::DnsResolver::ResolutionStatus::Failure, "resolve_timeout", {}, + absl::nullopt, /* is_proxy_lookup= */ false, /* is_timeout= */ true); } void DnsCacheImpl::onReResolveAlarm(const std::string& host) { @@ -379,7 +380,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, absl::string_view details, std::list&& response, absl::optional resolution_time, - bool is_proxy_lookup) { + bool is_proxy_lookup, bool is_timeout) { ASSERT(main_thread_dispatcher_.isThreadSafe()); if (Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.dns_cache_set_ip_version_to_remove")) { @@ -417,6 +418,28 @@ void DnsCacheImpl::finishResolve(const std::string& host, return primary_host_it->second.get(); }(); + std::string details_with_maybe_trace = std::string(details); + if (primary_host_info != nullptr && primary_host_info->active_query_ != nullptr) { + if (enable_dfp_dns_trace_) { + OptRef> traces = + primary_host_info->active_query_->getTraces(); + if (traces.has_value()) { + std::vector string_traces; + string_traces.reserve(traces.ref().size()); + std::transform(traces.ref().begin(), traces.ref().end(), std::back_inserter(string_traces), + [](const auto& trace) { + return absl::StrCat(trace.trace_, "=", + trace.time_.time_since_epoch().count()); + }); + details_with_maybe_trace = absl::StrCat(details, ":", absl::StrJoin(string_traces, ",")); + } + } + // `cancel` must be called last because the `ActiveQuery` will be destroyed afterward. + if (is_timeout) { + primary_host_info->active_query_->cancel(Network::ActiveDnsQuery::CancelReason::Timeout); + } + } + bool first_resolve = false; if (!from_cache) { @@ -480,7 +503,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, current_address ? current_address->asStringView() : "", new_address ? new_address->asStringView() : ""); primary_host_info->host_info_->setAddresses(new_address, std::move(address_list)); - primary_host_info->host_info_->setDetails(std::string(details)); + primary_host_info->host_info_->setDetails(details_with_maybe_trace); runAddUpdateCallbacks(host, primary_host_info->host_info_); primary_host_info->host_info_->setFirstResolveComplete(); @@ -490,7 +513,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, // We only set details here if current address is null because but // non-null->null resolutions we don't update the address so will use a // previously resolved address + details. - primary_host_info->host_info_->setDetails(std::string(details)); + primary_host_info->host_info_->setDetails(details_with_maybe_trace); } if (first_resolve) { diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index fe8d681944..15e0b41408 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -145,7 +145,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable&& response, absl::optional resolution_time = {}, - bool is_proxy_lookup = false); + bool is_proxy_lookup = false, bool is_timeout = false); void runAddUpdateCallbacks(const std::string& host, const DnsHostInfoSharedPtr& host_info); void runResolutionCompleteCallbacks(const std::string& host, const DnsHostInfoSharedPtr& host_info, @@ -270,6 +270,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable ip_version_to_remove_ ABSL_GUARDED_BY(ip_version_to_remove_lock_) = absl::nullopt; + bool enable_dfp_dns_trace_; }; } // namespace DynamicForwardProxy diff --git a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h index d8f01bd736..448c0a5946 100644 --- a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h +++ b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h @@ -100,6 +100,8 @@ class AppleDnsResolverImpl : public DnsResolver, protected Logger::Loggable> getTraces() override { return {}; } static DnsResponse buildDnsResponse(const struct sockaddr* address, uint32_t ttl); diff --git a/source/extensions/network/dns_resolver/cares/dns_impl.h b/source/extensions/network/dns_resolver/cares/dns_impl.h index a6ead67415..e61efc3c52 100644 --- a/source/extensions/network/dns_resolver/cares/dns_impl.h +++ b/source/extensions/network/dns_resolver/cares/dns_impl.h @@ -68,6 +68,7 @@ class DnsResolverImpl : public DnsResolver, protected Logger::Loggable> getTraces() override { return {}; } + // Does the object own itself? Resource reclamation occurs via self-deleting // on query completion or error. bool owned_ = false; diff --git a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.cc b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.cc index 398e60fba2..6412bc36e2 100644 --- a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.cc +++ b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.cc @@ -35,16 +35,21 @@ ActiveDnsQuery* GetAddrInfoDnsResolver::resolve(const std::string& dns_name, DnsLookupFamily dns_lookup_family, ResolveCb callback) { ENVOY_LOG(debug, "adding new query [{}] to pending queries", dns_name); - absl::MutexLock guard(&mutex_); - auto new_query = std::make_unique(dns_name, dns_lookup_family, callback, mutex_); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.getaddrinfo_num_retries") && - config_.has_num_retries()) { - // + 1 to include the initial query. - pending_queries_.push_back({std::move(new_query), config_.num_retries().value() + 1}); - } else { - pending_queries_.push_back({std::move(new_query), absl::nullopt}); + auto new_query = std::make_unique(dns_name, dns_lookup_family, callback); + ActiveDnsQuery* active_query; + { + absl::MutexLock guard(&mutex_); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.getaddrinfo_num_retries") && + config_.has_num_retries()) { + // + 1 to include the initial query. + pending_queries_.push_back({std::move(new_query), config_.num_retries().value() + 1}); + } else { + pending_queries_.push_back({std::move(new_query), absl::nullopt}); + } + active_query = pending_queries_.back().pending_query_.get(); } - return pending_queries_.back().pending_query_.get(); + active_query->addTrace(static_cast(GetAddrInfoTrace::NotStarted)); + return active_query; } std::pair> @@ -117,7 +122,7 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { ENVOY_LOG(debug, "starting getaddrinfo resolver thread"); while (true) { - PendingQuerySharedPtr next_query; + std::unique_ptr next_query; absl::optional num_retries; const bool reresolve = Runtime::runtimeFeatureEnabled("envoy.reloadable_features.dns_reresolve_on_eai_again"); @@ -134,10 +139,10 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { } PendingQueryInfo pending_query_info = std::move(pending_queries_.front()); - next_query = pending_query_info.pending_query_; + next_query = std::move(pending_query_info.pending_query_); num_retries = pending_query_info.num_retries_; pending_queries_.pop_front(); - if (reresolve && next_query->cancelled_) { + if (reresolve && next_query->isCancelled()) { continue; } } @@ -148,6 +153,7 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { std::pair> response; std::string details; { + next_query->addTrace(static_cast(GetAddrInfoTrace::Starting)); addrinfo hints; memset(&hints, 0, sizeof(hints)); hints.ai_flags = AI_ADDRCONFIG; @@ -161,25 +167,34 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { &hints, &addrinfo_result_do_not_use); auto addrinfo_wrapper = AddrInfoWrapper(addrinfo_result_do_not_use); if (rc.return_value_ == 0) { + next_query->addTrace(static_cast(GetAddrInfoTrace::Success)); response = processResponse(*next_query, addrinfo_wrapper.get()); } else if (reresolve && rc.return_value_ == EAI_AGAIN) { - absl::MutexLock guard(&mutex_); if (num_retries.has_value()) { (*num_retries)--; } if (!num_retries.has_value()) { ENVOY_LOG(debug, "retrying query [{}]", next_query->dns_name_); - pending_queries_.push_back({std::move(next_query), absl::nullopt}); + next_query->addTrace(static_cast(GetAddrInfoTrace::Retrying)); + { + absl::MutexLock guard(&mutex_); + pending_queries_.push_back({std::move(next_query), absl::nullopt}); + } continue; } if (*num_retries > 0) { ENVOY_LOG(debug, "retrying query [{}], num_retries: {}", next_query->dns_name_, *num_retries); - pending_queries_.push_back({std::move(next_query), *num_retries}); + next_query->addTrace(static_cast(GetAddrInfoTrace::Retrying)); + { + absl::MutexLock guard(&mutex_); + pending_queries_.push_back({std::move(next_query), *num_retries}); + } continue; } ENVOY_LOG(debug, "not retrying query [{}] because num_retries: {}", next_query->dns_name_, *num_retries); + next_query->addTrace(static_cast(GetAddrInfoTrace::DoneRetrying)); response = std::make_pair(ResolutionStatus::Failure, std::list()); } else if (treat_nodata_noname_as_success && (rc.return_value_ == EAI_NONAME || rc.return_value_ == EAI_NODATA)) { @@ -190,10 +205,12 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { // https://github.com/envoyproxy/envoy/blob/099d85925b32ce8bf06e241ee433375a0a3d751b/source/extensions/network/dns_resolver/cares/dns_impl.h#L109-L111. ENVOY_LOG(debug, "getaddrinfo for host={} has no results rc={}", next_query->dns_name_, gai_strerror(rc.return_value_)); + next_query->addTrace(static_cast(GetAddrInfoTrace::NoResult)); response = std::make_pair(ResolutionStatus::Completed, std::list()); } else { ENVOY_LOG(debug, "getaddrinfo failed for host={} with rc={} errno={}", next_query->dns_name_, gai_strerror(rc.return_value_), errorDetails(rc.errno_)); + next_query->addTrace(static_cast(GetAddrInfoTrace::Failed)); response = std::make_pair(ResolutionStatus::Failure, std::list()); } details = gai_strerror(rc.return_value_); @@ -201,9 +218,11 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { dispatcher_.post([finished_query = std::move(next_query), response = std::move(response), details = std::string(details)]() mutable { - if (finished_query->cancelled_) { + if (finished_query->isCancelled()) { + finished_query->addTrace(static_cast(GetAddrInfoTrace::Cancelled)); ENVOY_LOG(debug, "dropping cancelled query [{}]", finished_query->dns_name_); } else { + finished_query->addTrace(static_cast(GetAddrInfoTrace::Callback)); finished_query->callback_(response.first, std::move(details), std::move(response.second)); } }); diff --git a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h index 56f71a945c..27f29a6c6a 100644 --- a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h +++ b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h @@ -12,6 +12,19 @@ namespace Network { DECLARE_FACTORY(GetAddrInfoDnsResolverFactory); +// Trace information for getaddrinfo. +enum class GetAddrInfoTrace : uint8_t { + NotStarted = 0, + Starting = 1, + Success = 2, + Failed = 3, + NoResult = 4, + Retrying = 5, + DoneRetrying = 6, + Cancelled = 7, + Callback = 8, +}; + // This resolver uses getaddrinfo() on a dedicated resolution thread. Thus, it is only suitable // currently for relatively low rate resolutions. In the future a thread pool could be added if // desired. @@ -34,28 +47,41 @@ class GetAddrInfoDnsResolver : public DnsResolver, public Logger::Loggable> getTraces() override { + absl::MutexLock lock(&mutex_); + return traces_; + } + + bool isCancelled() { + absl::MutexLock lock(&mutex_); + return cancelled_; + } + + absl::Mutex mutex_; const std::string dns_name_; const DnsLookupFamily dns_lookup_family_; ResolveCb callback_; bool cancelled_{false}; + std::vector traces_; }; - // Must be a shared_ptr for passing around via post. - using PendingQuerySharedPtr = std::shared_ptr; struct PendingQueryInfo { - PendingQuerySharedPtr pending_query_; + std::unique_ptr pending_query_; // Empty means it will retry indefinitely until it succeeds. absl::optional num_retries_; }; diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 5bb5ea66e4..81d4bcee54 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -40,7 +40,7 @@ class ProxyFilterIntegrationTest : public testing::TestWithParam inject_factory(factory); + Registry::InjectFactory::forceAllowDuplicates(); + config_helper_.addRuntimeOverride("envoy.enable_dfp_dns_trace", "true"); + useAccessLog("%RESPONSE_CODE_DETAILS%"); + + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); + + config_helper_.prependFilter(fmt::format(R"EOF( + name: stream-info-to-headers-filter +)EOF")); + + upstream_tls_ = false; // upstream creation doesn't handle autonomous_upstream_ + autonomous_upstream_ = true; + std::string resolver_config = R"EOF( + typed_dns_resolver_config: + name: envoy.network.dns_resolver.getaddrinfo + typed_config: + "@type": type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig)EOF"; + initializeWithArgs(1024, 1024, "", resolver_config, false, 0.000000001); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_), + HasSubstr("dns_resolution_failure{resolve_timeout:")); +} + +TEST_P(ProxyFilterIntegrationTest, GetAddrInfoResolveTimeoutWithoutTrace) { + Network::OverrideAddrInfoDnsResolverFactory factory; + Registry::InjectFactory inject_factory(factory); + Registry::InjectFactory::forceAllowDuplicates(); + useAccessLog("%RESPONSE_CODE_DETAILS%"); + + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); + + config_helper_.prependFilter(fmt::format(R"EOF( + name: stream-info-to-headers-filter +)EOF")); + + upstream_tls_ = false; // upstream creation doesn't handle autonomous_upstream_ + autonomous_upstream_ = true; + std::string resolver_config = R"EOF( + typed_dns_resolver_config: + name: envoy.network.dns_resolver.getaddrinfo + typed_config: + "@type": type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig)EOF"; + initializeWithArgs(1024, 1024, "", resolver_config, false, 0.000000001); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_), + HasSubstr("dns_resolution_failure{resolve_timeout}")); +} + TEST_P(ProxyFilterIntegrationTest, ParallelRequests) { setDownstreamProtocol(Http::CodecType::HTTP2); setUpstreamProtocol(Http::CodecType::HTTP2); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc index 23ff9bd712..88b73ee944 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc @@ -4,7 +4,7 @@ namespace Envoy { namespace Network { absl::Mutex TestResolver::resolution_mutex_; -std::list dns_override)>> +std::list dns_override)>> TestResolver::blocked_resolutions_; REGISTER_FACTORY(TestResolverFactory, DnsResolverFactory); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h index 7248799887..41b47d10eb 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h +++ b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h @@ -27,7 +27,7 @@ class TestResolver : public GetAddrInfoDnsResolver { if (blocked_resolutions_.empty()) { continue; } - auto run = blocked_resolutions_.front(); + auto run = std::move(blocked_resolutions_.front()); blocked_resolutions_.pop_front(); run(dns_override); return; @@ -36,21 +36,23 @@ class TestResolver : public GetAddrInfoDnsResolver { ActiveDnsQuery* resolve(const std::string& dns_name, DnsLookupFamily dns_lookup_family, ResolveCb callback) override { - auto new_query = new PendingQuery(dns_name, dns_lookup_family, callback, mutex_); - + std::unique_ptr new_query = + std::make_unique(dns_name, dns_lookup_family, callback); + PendingQuery* raw_new_query = new_query.get(); absl::MutexLock guard(&resolution_mutex_); - blocked_resolutions_.push_back([&, new_query](absl::optional dns_override) { - absl::MutexLock guard(&mutex_); - if (dns_override.has_value()) { - *const_cast(&new_query->dns_name_) = dns_override.value(); - } - pending_queries_.push_back({std::unique_ptr{new_query}, absl::nullopt}); - }); - return new_query; + blocked_resolutions_.push_back( + [&, query = std::move(new_query)](absl::optional dns_override) mutable { + absl::MutexLock guard(&mutex_); + if (dns_override.has_value()) { + *const_cast(&query->dns_name_) = dns_override.value(); + } + pending_queries_.push_back(PendingQueryInfo{std::move(query), absl::nullopt}); + }); + return raw_new_query; } static absl::Mutex resolution_mutex_; - static std::list dns_override)>> + static std::list dns_override)>> blocked_resolutions_ ABSL_GUARDED_BY(resolution_mutex_); }; diff --git a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc index b2a830cd21..840e11c0c1 100644 --- a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc +++ b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc @@ -2,6 +2,7 @@ #include "source/common/network/dns_resolver/dns_factory_util.h" #include "source/common/network/utility.h" +#include "source/extensions/network//dns_resolver/getaddrinfo/getaddrinfo.h" #include "test/mocks/api/mocks.h" #include "test/test_common/threadsafe_singleton_injector.h" @@ -9,6 +10,7 @@ #include "gtest/gtest.h" +using testing::ElementsAre; using testing::NiceMock; using testing::Return; @@ -104,18 +106,31 @@ class GetAddrInfoDnsImplTest : public testing::Test { DnsResolverSharedPtr resolver_; NiceMock os_sys_calls_; envoy::extensions::network::dns_resolver::getaddrinfo::v3::GetAddrInfoDnsResolverConfig config_; + ActiveDnsQuery* active_dns_query_; }; +MATCHER_P(HasTrace, expected_trace, "") { + return arg.trace_ == static_cast(expected_trace); +} + TEST_F(GetAddrInfoDnsImplTest, LocalhostResolve) { // See https://github.com/envoyproxy/envoy/issues/28504. DISABLE_UNDER_WINDOWS; - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - verifyRealGaiResponse(status, std::move(response)); - dispatcher_->exit(); - }); + initialize(); + + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + verifyRealGaiResponse(status, std::move(response)); + EXPECT_THAT(active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } @@ -124,75 +139,106 @@ TEST_F(GetAddrInfoDnsImplTest, Cancel) { // See https://github.com/envoyproxy/envoy/issues/28504. DISABLE_UNDER_WINDOWS; + initialize(); + auto query = resolver_->resolve( "localhost", DnsLookupFamily::All, [](DnsResolver::ResolutionStatus, absl::string_view, std::list&&) { FAIL(); }); query->cancel(ActiveDnsQuery::CancelReason::QueryAbandoned); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - verifyRealGaiResponse(status, std::move(response)); - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + verifyRealGaiResponse(status, std::move(response)); + EXPECT_THAT(active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, Failure) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .WillOnce(Return(Api::SysCallIntResult{EAI_FAIL, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view details, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); - EXPECT_EQ("Non-recoverable failure in name resolution", details); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view details, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); + EXPECT_EQ("Non-recoverable failure in name resolution", details); + EXPECT_TRUE(response.empty()); + EXPECT_THAT(active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Failed), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, NoData) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .WillOnce(Return(Api::SysCallIntResult{EAI_NODATA, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + EXPECT_THAT(active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::NoResult), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, NoName) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .WillOnce(Return(Api::SysCallIntResult{EAI_NONAME, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + EXPECT_THAT(active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::NoResult), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, TryAgainIndefinitelyAndSuccess) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); // 2 calls - one EAGAIN, one success. @@ -200,14 +246,20 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainIndefinitelyAndSuccess) { .Times(2) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{0, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = resolver_->resolve( + "localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + EXPECT_THAT( + active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } @@ -232,6 +284,8 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainThenCancel) { } TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndSuccess) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); config_.mutable_num_retries()->set_value(3); @@ -244,19 +298,29 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndSuccess) { .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{0, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = resolver_->resolve( + "localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + EXPECT_THAT( + active_dns_query_->getTraces().ref(), + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndFailure) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); config_.mutable_num_retries()->set_value(3); @@ -269,15 +333,23 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndFailure) { .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view details, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); - EXPECT_FALSE(details.empty()); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = resolver_->resolve( + "localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view details, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); + EXPECT_FALSE(details.empty()); + EXPECT_TRUE(response.empty()); + EXPECT_THAT( + active_dns_query_->getTraces().ref(), + ElementsAre( + HasTrace(GetAddrInfoTrace::NotStarted), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::DoneRetrying), HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 7174b21667..c3f4bfd5c5 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -43,6 +43,8 @@ class MockActiveDnsQuery : public ActiveDnsQuery { // Network::ActiveDnsQuery MOCK_METHOD(void, cancel, (CancelReason reason)); + MOCK_METHOD(void, addTrace, (uint8_t)); + MOCK_METHOD(OptRef>, getTraces, ()); }; class MockFilterManager : public FilterManager { From e9574e656eab86050aaa7422c79da8c190babb04 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:00:57 -0400 Subject: [PATCH 27/77] ext_proc: fix typo in log (#36449) Commit Message: fix typo in log Signed-off-by: tyxia --- source/extensions/filters/http/ext_proc/processor_state.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/http/ext_proc/processor_state.cc b/source/extensions/filters/http/ext_proc/processor_state.cc index b0c569c7d1..efe972a4fc 100644 --- a/source/extensions/filters/http/ext_proc/processor_state.cc +++ b/source/extensions/filters/http/ext_proc/processor_state.cc @@ -499,7 +499,7 @@ void EncodingProcessorState::requestWatermark() { void EncodingProcessorState::clearWatermark() { if (watermark_requested_) { - ENVOY_LOG(debug, "Watermark lowered on decoding"); + ENVOY_LOG(debug, "Watermark lowered on encoding"); watermark_requested_ = false; encoder_callbacks_->onEncoderFilterBelowWriteBufferLowWatermark(); } From 18897c7ebc1864ad700a85a74d2fb4e5969160f7 Mon Sep 17 00:00:00 2001 From: Thomas Habets Date: Tue, 8 Oct 2024 14:28:23 +0100 Subject: [PATCH 28/77] Add getter for last downstream header byte received (#36472) Commit Message: Add getter for last downstream header byte received Additional Description: Used for latency measurements Risk Level: n/a Testing: None Docs Changes: None Release Notes: None Platform Specific Features: None Signed-off-by: Thomas Habets --- source/common/stream_info/utility.cc | 8 ++++++++ source/common/stream_info/utility.h | 1 + test/common/stream_info/stream_info_impl_test.cc | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/source/common/stream_info/utility.cc b/source/common/stream_info/utility.cc index d8ad2e5f1d..35b370602a 100644 --- a/source/common/stream_info/utility.cc +++ b/source/common/stream_info/utility.cc @@ -200,6 +200,14 @@ absl::optional TimingUtility::lastDownstreamTxByteSent return duration(timing.value().get().lastDownstreamTxByteSent(), stream_info_); } +absl::optional TimingUtility::lastDownstreamHeaderRxByteReceived() { + OptRef timing = stream_info_.downstreamTiming(); + if (!timing) { + return absl::nullopt; + } + return duration(timing.value().get().lastDownstreamHeaderRxByteReceived(), stream_info_); +} + absl::optional TimingUtility::lastDownstreamRxByteReceived() { OptRef timing = stream_info_.downstreamTiming(); if (!timing) { diff --git a/source/common/stream_info/utility.h b/source/common/stream_info/utility.h index 8c8a3d7408..d95e6993ff 100644 --- a/source/common/stream_info/utility.h +++ b/source/common/stream_info/utility.h @@ -219,6 +219,7 @@ class TimingUtility { absl::optional upstreamHandshakeComplete(); absl::optional firstDownstreamTxByteSent(); absl::optional lastDownstreamTxByteSent(); + absl::optional lastDownstreamHeaderRxByteReceived(); absl::optional lastDownstreamRxByteReceived(); absl::optional downstreamHandshakeComplete(); absl::optional lastDownstreamAckReceived(); diff --git a/test/common/stream_info/stream_info_impl_test.cc b/test/common/stream_info/stream_info_impl_test.cc index ca305401b1..6e8358c531 100644 --- a/test/common/stream_info/stream_info_impl_test.cc +++ b/test/common/stream_info/stream_info_impl_test.cc @@ -88,6 +88,10 @@ TEST_F(StreamInfoImplTest, TimingTest) { upstream_timing.onLastUpstreamRxByteReceived(test_time_.timeSystem()); dur = checkDuration(dur, timing.lastUpstreamRxByteReceived()); + EXPECT_FALSE(timing.lastDownstreamHeaderRxByteReceived()); + info.downstreamTiming().onLastDownstreamHeaderRxByteReceived(test_time_.timeSystem()); + dur = checkDuration(dur, timing.lastDownstreamHeaderRxByteReceived()); + EXPECT_FALSE(timing.firstDownstreamTxByteSent()); info.downstreamTiming().onFirstDownstreamTxByteSent(test_time_.timeSystem()); dur = checkDuration(dur, timing.firstDownstreamTxByteSent()); From 80b39e42e433778a5416b6a421bdcd70b19db09b Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 8 Oct 2024 09:44:58 -0400 Subject: [PATCH 29/77] test: fixing a Flake (#36475) Occasionally we do a third resolution --------- Signed-off-by: Alyssa Wilk --- mobile/test/common/integration/client_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index d428ebf4e8..7565359814 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -694,7 +694,7 @@ TEST_P(ClientIntegrationTest, InvalidDomainReresolveWithNoAddresses) { true); Network::TestResolver::unblockResolve(); terminal_callback_.waitReady(); - EXPECT_EQ(2, getCounterValue("dns_cache.base_dns_cache.dns_query_attempt")); + EXPECT_LE(2, getCounterValue("dns_cache.base_dns_cache.dns_query_attempt")); } TEST_P(ClientIntegrationTest, ReresolveAndDrain) { From a9ce6867a7da1ec1b609af8788031fc35139afa4 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 8 Oct 2024 09:52:51 -0400 Subject: [PATCH 30/77] mobile: fixing a flow control bug for multiple large uploads (#36474) Fixing two bugs in the send window code. One where the callback could be called late, after stream completion. One where the callback was per-client not per-stream so streams could overwrite each others upcalls. Risk Level: low Testing: first is tested by existing tests no longer flaking. second TBTested Docs Changes: n/a Release Notes: n/a Fixes https://github.com/envoyproxy/envoy/issues/36493 Signed-off-by: Alyssa Wilk --- mobile/library/common/http/client.cc | 6 +- mobile/library/common/http/client.h | 2 +- mobile/test/common/http/client_test.cc | 79 ++++++++++++++++++++++++++ mobile/test/java/integration/BUILD | 1 - 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index 3a3a8f039e..f1613aed88 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -656,9 +656,9 @@ void Client::sendData(envoy_stream_t stream, Buffer::InstancePtr buffer, bool en direct_stream->wants_write_notification_ = false; // A new callback must be scheduled each time to capture any changes to the // DirectStream's callbacks from call to call. - scheduled_callback_ = dispatcher_.createSchedulableCallback( + direct_stream->scheduled_callback_ = dispatcher_.createSchedulableCallback( [direct_stream] { direct_stream->callbacks_->onSendWindowAvailable(); }); - scheduled_callback_->scheduleCallbackNextIteration(); + direct_stream->scheduled_callback_->scheduleCallbackNextIteration(); } else { // Otherwise, make sure the stack will send a notification when the // buffers are drained. @@ -699,7 +699,6 @@ void Client::cancelStream(envoy_stream_t stream) { // whether it was closed or not. Client::DirectStreamSharedPtr direct_stream = getStream(stream, GetStreamFilters::AllowForAllStreams); - scheduled_callback_ = nullptr; if (direct_stream) { // Attempt to latch the latest stream info. This will be a no-op if the stream // is already complete. @@ -759,6 +758,7 @@ void Client::removeStream(envoy_stream_t stream_handle) { "[S{}] removeStream is a private method that is only called with stream ids that exist", stream_handle)); + direct_stream->scheduled_callback_ = nullptr; // The DirectStream should live through synchronous code that already has a reference to it. // Hence why it is scheduled for deferred deletion. If this was all that was needed then it // would be sufficient to return a shared_ptr in getStream. However, deferred deletion is still diff --git a/mobile/library/common/http/client.h b/mobile/library/common/http/client.h index 046588e53f..6c425747c9 100644 --- a/mobile/library/common/http/client.h +++ b/mobile/library/common/http/client.h @@ -331,6 +331,7 @@ class Client : public Logger::Loggable { // Set true in explicit flow control mode if the library has sent body data and may want to // send more when buffer is available. bool wants_write_notification_{}; + Event::SchedulableCallbackPtr scheduled_callback_; // True if the bridge should operate in explicit flow control mode. // // In this mode only one callback can be sent to the bridge until more is @@ -383,7 +384,6 @@ class Client : public Logger::Loggable { ApiListenerPtr api_listener_; Event::ProvisionalDispatcher& dispatcher_; - Event::SchedulableCallbackPtr scheduled_callback_; HttpClientStats stats_; // The set of open streams, which can safely have request data sent on them // or response data received. diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index bc7d4a61c7..0e15da98cc 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -498,6 +498,85 @@ TEST_P(ClientTest, MultipleStreams) { ASSERT_EQ(callbacks_called1.on_complete_calls_, 1); } +TEST_P(ClientTest, MultipleUploads) { + envoy_stream_t stream1 = 1; + envoy_stream_t stream2 = 2; + auto request_data1 = std::make_unique("request body1"); + auto request_data2 = std::make_unique("request body2"); + + // Create a stream, and set up request_decoder_ and response_encoder_ + StreamCallbacksCalled callbacks_called1; + auto stream_callbacks1 = createDefaultStreamCallbacks(callbacks_called1); + createStream(std::move(stream_callbacks1)); + + // Send request headers. + EXPECT_CALL(*request_decoder_, decodeHeaders_(_, false)); + http_client_.sendHeaders(stream1, createDefaultRequestHeaders(), false); + http_client_.sendData(stream1, std::move(request_data1), true); + + // Start stream2. + // Setup EnvoyStreamCallbacks to handle the response headers. + NiceMock request_decoder2; + ON_CALL(request_decoder2, streamInfo()).WillByDefault(ReturnRef(stream_info_)); + ResponseEncoder* response_encoder2{}; + StreamCallbacksCalled callbacks_called2; + auto stream_callbacks2 = createDefaultStreamCallbacks(callbacks_called1); + stream_callbacks2.on_headers_ = [&](const ResponseHeaderMap& headers, bool end_stream, + envoy_stream_intel) -> void { + EXPECT_TRUE(end_stream); + EXPECT_EQ(headers.Status()->value().getStringView(), "200"); + callbacks_called2.on_headers_calls_ = true; + }; + stream_callbacks2.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) -> void { + callbacks_called2.on_complete_calls_++; + }; + + std::vector window_callbacks; + ON_CALL(dispatcher_, createSchedulableCallback).WillByDefault([&](std::function cb) { + Event::SchedulableCallbackPtr scheduler = + dispatcher_.Event::ProvisionalDispatcher::createSchedulableCallback(cb); + window_callbacks.push_back(scheduler.get()); + return scheduler; + }); + + // Create a stream. + ON_CALL(dispatcher_, isThreadSafe()).WillByDefault(Return(true)); + + // Grab the response encoder in order to dispatch responses on the stream. + // Return the request decoder to make sure calls are dispatched to the decoder via the dispatcher + // API. + EXPECT_CALL(*api_listener_, newStreamHandle(_, _)) + .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoderHandlePtr { + response_encoder2 = &encoder; + return std::make_unique(request_decoder2); + })); + http_client_.startStream(stream2, std::move(stream_callbacks2), explicit_flow_control_); + + // Send request headers. + EXPECT_CALL(request_decoder2, decodeHeaders_(_, false)); + http_client_.sendHeaders(stream2, createDefaultRequestHeaders(), false); + http_client_.sendData(stream2, std::move(request_data2), true); + + for (auto* callback : window_callbacks) { + EXPECT_TRUE(callback->enabled()); + } + + // Finish stream 2. + EXPECT_CALL(dispatcher_, deferredDelete_(_)); + TestResponseHeaderMapImpl response_headers2{{":status", "200"}}; + response_encoder2->encodeHeaders(response_headers2, true); + ASSERT_EQ(callbacks_called2.on_headers_calls_, 1); + // Ensure that the on_headers on the EnvoyStreamCallbacks was called. + ASSERT_EQ(callbacks_called2.on_complete_calls_, 1); + + // Finish stream 1. + EXPECT_CALL(dispatcher_, deferredDelete_(_)); + TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, true); + ASSERT_EQ(callbacks_called1.on_headers_calls_, 1); + ASSERT_EQ(callbacks_called1.on_complete_calls_, 1); +} + TEST_P(ClientTest, EnvoyLocalError) { // Override the on_error default with some custom checks. StreamCallbacksCalled callbacks_called; diff --git a/mobile/test/java/integration/BUILD b/mobile/test/java/integration/BUILD index e987ab6456..7154488b71 100644 --- a/mobile/test/java/integration/BUILD +++ b/mobile/test/java/integration/BUILD @@ -51,7 +51,6 @@ envoy_mobile_android_test( srcs = [ "AndroidEngineExplicitFlowTest.java", ], - flaky = True, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ From 2386bd53eff348198dae27e3c77fa0e50a4ae264 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 8 Oct 2024 11:41:21 -0400 Subject: [PATCH 31/77] listener manager: removing exceptions (#36314) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a https://github.com/envoyproxy/envoy-mobile/issues/176 Signed-off-by: Alyssa Wilk --- envoy/common/exception.h | 4 +- .../common/listener_manager/listener_impl.cc | 107 ++++++++++++------ .../common/listener_manager/listener_impl.h | 38 ++++--- .../listener_manager/listener_manager_impl.cc | 17 ++- .../listener_manager_impl_test.cc | 41 +++++-- tools/code_format/config.yaml | 1 - 6 files changed, 141 insertions(+), 67 deletions(-) diff --git a/envoy/common/exception.h b/envoy/common/exception.h index 5734df1f66..7b74e2094b 100644 --- a/envoy/common/exception.h +++ b/envoy/common/exception.h @@ -30,8 +30,8 @@ class EnvoyException : public std::runtime_error { }; #define SET_AND_RETURN_IF_NOT_OK(check_status, set_status) \ - if (!check_status.ok()) { \ - set_status = check_status; \ + if (const absl::Status temp_status = check_status; !temp_status.ok()) { \ + set_status = temp_status; \ return; \ } diff --git a/source/common/listener_manager/listener_impl.cc b/source/common/listener_manager/listener_impl.cc index 23350a0c22..ed1585afe4 100644 --- a/source/common/listener_manager/listener_impl.cc +++ b/source/common/listener_manager/listener_impl.cc @@ -72,12 +72,27 @@ bool shouldBindToPort(const envoy::config::listener::v3::Listener& config) { } } // namespace +absl::StatusOr> ListenSocketFactoryImpl::create( + ListenerComponentFactory& factory, Network::Address::InstanceConstSharedPtr address, + Network::Socket::Type socket_type, const Network::Socket::OptionsSharedPtr& options, + const std::string& listener_name, uint32_t tcp_backlog_size, + ListenerComponentFactory::BindType bind_type, + const Network::SocketCreationOptions& creation_options, uint32_t num_sockets) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr(new ListenSocketFactoryImpl( + factory, address, socket_type, options, listener_name, tcp_backlog_size, bind_type, + creation_options, num_sockets, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; +} + ListenSocketFactoryImpl::ListenSocketFactoryImpl( ListenerComponentFactory& factory, Network::Address::InstanceConstSharedPtr address, Network::Socket::Type socket_type, const Network::Socket::OptionsSharedPtr& options, const std::string& listener_name, uint32_t tcp_backlog_size, ListenerComponentFactory::BindType bind_type, - const Network::SocketCreationOptions& creation_options, uint32_t num_sockets) + const Network::SocketCreationOptions& creation_options, uint32_t num_sockets, + absl::Status& creation_status) : factory_(factory), local_address_(address), socket_type_(socket_type), options_(options), listener_name_(listener_name), tcp_backlog_size_(tcp_backlog_size), bind_type_(bind_type), socket_creation_options_(creation_options) { @@ -100,8 +115,9 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl( } } - sockets_.push_back(THROW_OR_RETURN_VALUE( - createListenSocketAndApplyOptions(factory, socket_type, 0), Network::SocketSharedPtr)); + auto socket_or_error = createListenSocketAndApplyOptions(factory, socket_type, 0); + SET_AND_RETURN_IF_NOT_OK(socket_or_error.status(), creation_status); + sockets_.push_back(*socket_or_error); if (sockets_[0] != nullptr && local_address_->ip() && local_address_->ip()->port() == 0) { local_address_ = sockets_[0]->connectionInfoProvider().localAddress(); @@ -114,8 +130,9 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl( if (bind_type_ != ListenerComponentFactory::BindType::ReusePort && sockets_[0] != nullptr) { sockets_.push_back(sockets_[0]->duplicate()); } else { - sockets_.push_back(THROW_OR_RETURN_VALUE( - createListenSocketAndApplyOptions(factory, socket_type, i), Network::SocketSharedPtr)); + auto socket_or_error = createListenSocketAndApplyOptions(factory, socket_type, i); + SET_AND_RETURN_IF_NOT_OK(socket_or_error.status(), creation_status); + sockets_.push_back(*socket_or_error); } } ASSERT(sockets_.size() == num_sockets); @@ -165,12 +182,7 @@ absl::StatusOr ListenSocketFactoryImpl::createListenSo fmt::format("{}: Setting socket options {}", listener_name_, ok ? "succeeded" : "failed"); if (!ok) { ENVOY_LOG(warn, "{}", message); -#ifdef ENVOY_DISABLE_EXCEPTIONS - PANIC(message); -#else - throw Network::SocketOptionException(message); -#endif - + return absl::InvalidArgumentError(message); } else { ENVOY_LOG(debug, "{}", message); } @@ -236,8 +248,11 @@ std::string listenerStatsScope(const envoy::config::listener::v3::Listener& conf return absl::StrCat("envoy_internal_", config.name()); } auto address_or_error = Network::Address::resolveProtoAddress(config.address()); - THROW_IF_NOT_OK_REF(address_or_error.status()); - return address_or_error.value()->asString(); + if (address_or_error.status().ok()) { + return address_or_error.value()->asString(); + } + // Listener creation will fail shortly when the address is used. + return absl::StrCat("invalid_address_listener"); } } // namespace @@ -254,10 +269,22 @@ Network::DrainDecision& ListenerFactoryContextBaseImpl::drainDecision() { return Server::DrainManager& ListenerFactoryContextBaseImpl::drainManager() { return *drain_manager_; } Init::Manager& ListenerFactoryContextBaseImpl::initManager() { return server_.initManager(); } +absl::StatusOr> +ListenerImpl::create(const envoy::config::listener::v3::Listener& config, + const std::string& version_info, ListenerManagerImpl& parent, + const std::string& name, bool added_via_api, bool workers_started, + uint64_t hash) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr(new ListenerImpl( + config, version_info, parent, name, added_via_api, workers_started, hash, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; +} + ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, const std::string& name, bool added_via_api, bool workers_started, - uint64_t hash) + uint64_t hash, absl::Status& creation_status) : parent_(parent), socket_type_(config.has_internal_listener() ? Network::Socket::Type::Stream @@ -324,25 +351,28 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, // All the addresses should be same socket type, so get the first address's socket type is // enough. auto address_or_error = Network::Address::resolveProtoAddress(config.address()); - THROW_IF_NOT_OK_REF(address_or_error.status()); + SET_AND_RETURN_IF_NOT_OK(address_or_error.status(), creation_status); auto address = std::move(address_or_error.value()); - THROW_IF_NOT_OK(checkIpv4CompatAddress(address, config.address())); + SET_AND_RETURN_IF_NOT_OK(checkIpv4CompatAddress(address, config.address()), creation_status); addresses_.emplace_back(address); address_opts_list.emplace_back(std::ref(config.socket_options())); for (auto i = 0; i < config.additional_addresses_size(); i++) { if (socket_type_ != Network::Utility::protobufAddressSocketType(config.additional_addresses(i).address())) { - throwEnvoyExceptionOrPanic( + creation_status = absl::InvalidArgumentError( fmt::format("listener {}: has different socket type. The listener only " "support same socket type for all the addresses.", name_)); + return; } auto addresses_or_error = Network::Address::resolveProtoAddress(config.additional_addresses(i).address()); - THROW_IF_NOT_OK_REF(addresses_or_error.status()); + SET_AND_RETURN_IF_NOT_OK(addresses_or_error.status(), creation_status); auto additional_address = std::move(addresses_or_error.value()); - THROW_IF_NOT_OK(checkIpv4CompatAddress(address, config.additional_addresses(i).address())); + SET_AND_RETURN_IF_NOT_OK( + checkIpv4CompatAddress(address, config.additional_addresses(i).address()), + creation_status); addresses_.emplace_back(additional_address); if (config.additional_addresses(i).has_socket_options()) { address_opts_list.emplace_back( @@ -367,20 +397,21 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, addresses_, listener_factory_context_->parentFactoryContext(), initManager()), buildAccessLog(config); - THROW_IF_NOT_OK(validateConfig()); + SET_AND_RETURN_IF_NOT_OK(validateConfig(), creation_status); // buildUdpListenerFactory() must come before buildListenSocketOptions() because the UDP // listener factory can provide additional options. - THROW_IF_NOT_OK(buildUdpListenerFactory(config, parent_.server_.options().concurrency())); + SET_AND_RETURN_IF_NOT_OK(buildUdpListenerFactory(config, parent_.server_.options().concurrency()), + creation_status); buildListenSocketOptions(config, address_opts_list); - THROW_IF_NOT_OK(createListenerFilterFactories(config)); - THROW_IF_NOT_OK(validateFilterChains(config)); - THROW_IF_NOT_OK(buildFilterChains(config)); + SET_AND_RETURN_IF_NOT_OK(createListenerFilterFactories(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(validateFilterChains(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(buildFilterChains(config), creation_status); if (socket_type_ != Network::Socket::Type::Datagram) { buildSocketOptions(config); buildOriginalDstListenerFilter(config); buildProxyProtocolListenerFilter(config); - THROW_IF_NOT_OK(buildInternalListener(config)); + SET_AND_RETURN_IF_NOT_OK(buildInternalListener(config), creation_status); } if (!workers_started_) { // Initialize dynamic_init_manager_ from Server's init manager if it's not initialized. @@ -395,7 +426,7 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, const std::string& name, bool added_via_api, bool workers_started, - uint64_t hash) + uint64_t hash, absl::Status& creation_status) : parent_(parent), addresses_(origin.addresses_), socket_type_(origin.socket_type_), bind_to_port_(shouldBindToPort(config)), mptcp_enabled_(config.enable_mptcp()), hand_off_restored_destination_connections_( @@ -443,11 +474,11 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { buildAccessLog(config); - THROW_IF_NOT_OK(validateConfig()); - THROW_IF_NOT_OK(createListenerFilterFactories(config)); - THROW_IF_NOT_OK(validateFilterChains(config)); - THROW_IF_NOT_OK(buildFilterChains(config)); - THROW_IF_NOT_OK(buildInternalListener(config)); + SET_AND_RETURN_IF_NOT_OK(validateConfig(), creation_status); + SET_AND_RETURN_IF_NOT_OK(createListenerFilterFactories(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(validateFilterChains(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(buildFilterChains(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(buildInternalListener(config), creation_status); if (socket_type_ == Network::Socket::Type::Stream) { // Apply the options below only for TCP. buildSocketOptions(config); @@ -1020,14 +1051,18 @@ bool ListenerImpl::supportUpdateFilterChain(const envoy::config::listener::v3::L return false; } -ListenerImplPtr +absl::StatusOr ListenerImpl::newListenerWithFilterChain(const envoy::config::listener::v3::Listener& config, bool workers_started, uint64_t hash) { + + absl::Status creation_status = absl::OkStatus(); // Use WrapUnique since the constructor is private. - return absl::WrapUnique(new ListenerImpl(*this, config, version_info_, parent_, name_, - added_via_api_, - /* new new workers started state */ workers_started, - /* use new hash */ hash)); + auto ret = absl::WrapUnique(new ListenerImpl(*this, config, version_info_, parent_, name_, + added_via_api_, + /* new new workers started state */ workers_started, + /* use new hash */ hash, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; } void ListenerImpl::diffFilterChain(const ListenerImpl& another_listener, diff --git a/source/common/listener_manager/listener_impl.h b/source/common/listener_manager/listener_impl.h index f7bd70f47a..2157b5eac5 100644 --- a/source/common/listener_manager/listener_impl.h +++ b/source/common/listener_manager/listener_impl.h @@ -63,14 +63,12 @@ class ListenerManagerImpl; class ListenSocketFactoryImpl : public Network::ListenSocketFactory, protected Logger::Loggable { public: - ListenSocketFactoryImpl(ListenerComponentFactory& factory, - Network::Address::InstanceConstSharedPtr address, - Network::Socket::Type socket_type, - const Network::Socket::OptionsSharedPtr& options, - const std::string& listener_name, uint32_t tcp_backlog_size, - ListenerComponentFactory::BindType bind_type, - const Network::SocketCreationOptions& creation_options, - uint32_t num_sockets); + static absl::StatusOr> + create(ListenerComponentFactory& factory, Network::Address::InstanceConstSharedPtr address, + Network::Socket::Type socket_type, const Network::Socket::OptionsSharedPtr& options, + const std::string& listener_name, uint32_t tcp_backlog_size, + ListenerComponentFactory::BindType bind_type, + const Network::SocketCreationOptions& creation_options, uint32_t num_sockets); // Network::ListenSocketFactory Network::Socket::Type socketType() const override { return socket_type_; } @@ -89,6 +87,15 @@ class ListenSocketFactoryImpl : public Network::ListenSocketFactory, absl::Status doFinalPreWorkerInit() override; private: + ListenSocketFactoryImpl(ListenerComponentFactory& factory, + Network::Address::InstanceConstSharedPtr address, + Network::Socket::Type socket_type, + const Network::Socket::OptionsSharedPtr& options, + const std::string& listener_name, uint32_t tcp_backlog_size, + ListenerComponentFactory::BindType bind_type, + const Network::SocketCreationOptions& creation_options, + uint32_t num_sockets, absl::Status& creation_status); + ListenSocketFactoryImpl(const ListenSocketFactoryImpl& factory_to_clone); absl::StatusOr @@ -205,9 +212,10 @@ class ListenerImpl final : public Network::ListenerConfig, * @param hash supplies the hash to use for duplicate checking. * @param concurrency is the number of listeners instances to be created. */ - ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, - ListenerManagerImpl& parent, const std::string& name, bool added_via_api, - bool workers_started, uint64_t hash); + static absl::StatusOr> + create(const envoy::config::listener::v3::Listener& config, const std::string& version_info, + ListenerManagerImpl& parent, const std::string& name, bool added_via_api, + bool workers_started, uint64_t hash); ~ListenerImpl() override; // TODO(lambdai): Explore using the same ListenerImpl object to execute in place filter chain @@ -216,7 +224,7 @@ class ListenerImpl final : public Network::ListenerConfig, * Execute in place filter chain update. The filter chain update is less expensive than full * listener update because connections may not need to be drained. */ - std::unique_ptr + absl::StatusOr> newListenerWithFilterChain(const envoy::config::listener::v3::Listener& config, bool workers_started, uint64_t hash); /** @@ -344,6 +352,9 @@ class ListenerImpl final : public Network::ListenerConfig, SystemTime last_updated_; private: + ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, + ListenerManagerImpl& parent, const std::string& name, bool added_via_api, + bool workers_started, uint64_t hash, absl::Status& creation_status); struct UdpListenerConfigImpl : public Network::UdpListenerConfig { UdpListenerConfigImpl(const envoy::config::listener::v3::UdpListenerConfig config) : config_(config) {} @@ -384,7 +395,8 @@ class ListenerImpl final : public Network::ListenerConfig, */ ListenerImpl(ListenerImpl& origin, const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, - const std::string& name, bool added_via_api, bool workers_started, uint64_t hash); + const std::string& name, bool added_via_api, bool workers_started, uint64_t hash, + absl::Status& creation_status); // Helpers for constructor. void buildAccessLog(const envoy::config::listener::v3::Listener& config); absl::Status buildInternalListener(const envoy::config::listener::v3::Listener& config); diff --git a/source/common/listener_manager/listener_manager_impl.cc b/source/common/listener_manager/listener_manager_impl.cc index 1683caf51b..8de7a7da33 100644 --- a/source/common/listener_manager/listener_manager_impl.cc +++ b/source/common/listener_manager/listener_manager_impl.cc @@ -580,13 +580,17 @@ absl::StatusOr ListenerManagerImpl::addOrUpdateListenerInternal( (*existing_active_listener)->supportUpdateFilterChain(config, workers_started_)) { ENVOY_LOG(debug, "use in place update filter chain update path for listener name={} hash={}", name, hash); - new_listener = + auto listener_or_error = (*existing_active_listener)->newListenerWithFilterChain(config, workers_started_, hash); + RETURN_IF_NOT_OK_REF(listener_or_error.status()); + new_listener = std::move(*listener_or_error); stats_.listener_in_place_updated_.inc(); } else { ENVOY_LOG(debug, "use full listener update path for listener name={} hash={}", name, hash); - new_listener = std::make_unique(config, version_info, *this, name, added_via_api, + auto listener_or_error = ListenerImpl::create(config, version_info, *this, name, added_via_api, workers_started_, hash); + RETURN_IF_NOT_OK_REF(listener_or_error.status()); + new_listener = std::move(*listener_or_error); } ListenerImpl& new_listener_ref = *new_listener; @@ -1197,10 +1201,15 @@ absl::Status ListenerManagerImpl::createListenSocketFactory(ListenerImpl& listen creation_options.mptcp_enabled_ = listener.mptcpEnabled(); for (std::vector::size_type i = 0; i < listener.addresses().size(); i++) { - socket_status = listener.addSocketFactory(std::make_unique( + auto factory_or_error = ListenSocketFactoryImpl::create( *factory_, listener.addresses()[i], socket_type, listener.listenSocketOptions(i), listener.name(), listener.tcpBacklogSize(), bind_type, creation_options, - server_.options().concurrency())); + server_.options().concurrency()); + if (!factory_or_error.status().ok()) { + socket_status = factory_or_error.status(); + } else { + socket_status = listener.addSocketFactory(std::move(*factory_or_error)); + } if (!socket_status.ok()) { break; } diff --git a/test/common/listener_manager/listener_manager_impl_test.cc b/test/common/listener_manager/listener_manager_impl_test.cc index d86093344f..cd7a7f658f 100644 --- a/test/common/listener_manager/listener_manager_impl_test.cc +++ b/test/common/listener_manager/listener_manager_impl_test.cc @@ -1008,11 +1008,12 @@ TEST_P(ListenerManagerImplTest, RejectTcpOptionsWithInternalListenerConfig) { for (const auto& f : listener_mutators) { auto new_listener = listener; f(new_listener); - EXPECT_THROW_WITH_MESSAGE(new ListenerImpl(new_listener, "version", *manager_, "foo", true, - false, /*hash=*/static_cast(0)), - EnvoyException, - "error adding listener named 'foo': has " - "unsupported tcp listener feature"); + EXPECT_EQ(ListenerImpl::create(new_listener, "version", *manager_, "foo", true, false, + /*hash=*/static_cast(0)) + .status() + .message(), + "error adding listener named 'foo': has " + "unsupported tcp listener feature"); } { auto new_listener = listener; @@ -7809,6 +7810,24 @@ TEST(ListenerMessageUtilTest, ListenerMessageHaveDifferentFilterChainsAreEquival EXPECT_TRUE(Server::ListenerMessageUtil::filterChainOnlyChange(listener1, listener2)); } +TEST_P(ListenerManagerImplForInPlaceFilterChainUpdateTest, InvalidAddress) { + // Worker is not started yet. + envoy::config::listener::v3::Listener listener_proto; + Protobuf::TextFormat::ParseFromString(R"EOF( + name: "foo" + address: { + socket_address: { + address: "127.0.0.1.0" + port_value: 1234 + } + } + filter_chains: {} + )EOF", + &listener_proto); + EXPECT_EQ(manager_->addOrUpdateListener(listener_proto, "", true).status().message(), + "malformed IP address: 127.0.0.1.0"); +} + TEST_P(ListenerManagerImplForInPlaceFilterChainUpdateTest, TraditionalUpdateIfWorkerNotStarted) { // Worker is not started yet. auto listener_proto = createDefaultListener(); @@ -8034,13 +8053,13 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, InvalidExtendConnectionBalanceCon extend_balance_config->mutable_typed_config()->set_type_url( "type.googleapis.com/google.protobuf.test"); - auto listener_impl = ListenerImpl(listener, "version", *manager_, "foo", true, false, - /*hash=*/static_cast(0)); + auto listener_impl = *ListenerImpl::create(listener, "version", *manager_, "foo", true, false, + /*hash=*/static_cast(0)); auto socket_factory = std::make_unique(); Network::Address::InstanceConstSharedPtr address( new Network::Address::Ipv4Instance("192.168.0.1", 80, nullptr)); EXPECT_CALL(*socket_factory, localAddress()).WillOnce(ReturnRef(address)); - EXPECT_EQ(listener_impl.addSocketFactory(std::move(socket_factory)).message(), + EXPECT_EQ(listener_impl->addSocketFactory(std::move(socket_factory)).message(), "Didn't find a registered implementation for type: 'google.protobuf.test'"); #endif } @@ -8051,13 +8070,13 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, EmptyConnectionBalanceConfig) { auto listener = createIPv4Listener("TCPListener"); listener.mutable_connection_balance_config(); - auto listener_impl = ListenerImpl(listener, "version", *manager_, "foo", true, false, - /*hash=*/static_cast(0)); + auto listener_impl = *ListenerImpl::create(listener, "version", *manager_, "foo", true, false, + /*hash=*/static_cast(0)); auto socket_factory = std::make_unique(); Network::Address::InstanceConstSharedPtr address( new Network::Address::Ipv4Instance("192.168.0.1", 80, nullptr)); EXPECT_CALL(*socket_factory, localAddress()).WillOnce(ReturnRef(address)); - EXPECT_EQ(listener_impl.addSocketFactory(std::move(socket_factory)).message(), + EXPECT_EQ(listener_impl->addSocketFactory(std::move(socket_factory)).message(), "No valid balance type for connection balance"); #endif } diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 3d117d0f63..b453bb1cb8 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -100,7 +100,6 @@ paths: # legacy core files which throw exceptions. We can add to this list but strongly prefer # StausOr where possible. - source/common/router/route_config_update_receiver_impl.cc - - source/common/listener_manager/listener_impl.cc - source/common/upstream/cluster_manager_impl.cc - source/common/upstream/upstream_impl.cc - source/common/network/listen_socket_impl.cc From 7f4bd24a99b927e650250f116e02a3b61d10f0f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Tue, 8 Oct 2024 12:57:21 -0400 Subject: [PATCH 32/77] Disable bazel's layering_check feature during CodeQL build. (#36500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It doesn't work without sandboxing, and this build uses `--spawn_strategy=local`. https://github.com/bazelbuild/bazel/issues/21592 Once we upgrade to a bazel 7.3.0+, this should no longer be needed. Risk Level: Low; only affects CI CodeQL build Signed-off-by: Alejandro R. Sedeño --- .github/workflows/codeql-daily.yml | 1 + .github/workflows/codeql-push.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 78bacf69ee..8a3e2ad85e 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -64,6 +64,7 @@ jobs: --spawn_strategy=local \ --discard_analysis_cache \ --nouse_action_cache \ + --features="-layering_check" \ --config=clang-libc++ \ --config=ci \ //source/common/http/... diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index b2d59ba7c7..f336d11cc8 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -96,6 +96,7 @@ jobs: --spawn_strategy=local \ --discard_analysis_cache \ --nouse_action_cache \ + --features="-layering_check" \ --config=clang-libc++ \ --config=ci \ $BUILD_TARGETS From dbad0cf2e9317b4601d95bd32366caa8f1256a76 Mon Sep 17 00:00:00 2001 From: Renana Yacobi <138735011+renanay@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:08:24 -0700 Subject: [PATCH 33/77] fips build: fixed an issue when tar is running as root (#36476) fips build: fixed an issue when tar is running as root causing 'Cannot change ownership to uid 1000, gid 1000: Invalid argument' error See this thread for issue and solution: https://superuser.com/questions/1435437/how-to-get-around-this-error-when-untarring-an-archive-tar-cannot-change-owner Signed-off-by: Renana Yacobi --- bazel/external/boringssl_fips.genrule_cmd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd index 51e6b72c78..0f79c4e5fc 100755 --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -44,7 +44,7 @@ fi curl -sLO https://github.com/llvm/llvm-project/releases/download/llvmorg-"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check -tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz +tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz --no-same-owner printf "set(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\n" > ${HOME}/toolchain export PATH="$PWD/clang+llvm-$VERSION-$PLATFORM/bin:$PATH" @@ -66,7 +66,7 @@ fi curl -sLO https://dl.google.com/go/go"$VERSION"."$PLATFORM".tar.gz \ && echo "$SHA256" go"$VERSION"."$PLATFORM".tar.gz | sha256sum --check -tar xf go"$VERSION"."$PLATFORM".tar.gz +tar xf go"$VERSION"."$PLATFORM".tar.gz --no-same-owner export GOPATH="$PWD/gopath" export GOROOT="$PWD/go" @@ -82,7 +82,7 @@ VERSION=1.10.2 SHA256=ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed curl -sLO https://github.com/ninja-build/ninja/archive/refs/tags/v"$VERSION".tar.gz \ && echo "$SHA256" v"$VERSION".tar.gz | sha256sum --check -tar -xvf v"$VERSION".tar.gz +tar -xvf v"$VERSION".tar.gz --no-same-owner cd ninja-"$VERSION" python3 ./configure.py --bootstrap @@ -106,7 +106,7 @@ fi curl -sLO https://github.com/Kitware/CMake/releases/download/v"$VERSION"/cmake-"$VERSION"-"$PLATFORM".tar.gz \ && echo "$SHA256" cmake-"$VERSION"-"$PLATFORM".tar.gz | sha256sum --check -tar xf cmake-"$VERSION"-"$PLATFORM".tar.gz +tar xf cmake-"$VERSION"-"$PLATFORM".tar.gz --no-same-owner export PATH="$PWD/cmake-$VERSION-$PLATFORM/bin:$PATH" From 58abcf73b03ddf8600e48401926f268d6aa5dc00 Mon Sep 17 00:00:00 2001 From: bsurber <73970703+bsurber@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:40:33 -0700 Subject: [PATCH 34/77] rlqs: Updated RLQS Response handling to not reset TokenBucket state (#36478) Commit Message: When an RLQS response comes in that would create a duplicate TokenBucket (because the underlying config hasn't changed), treat this as a refresh to the TTL but don't reset the TokenBucket's state by recreating it. Additional Description: Risk Level: Low Testing: Unit testing Docs Changes: Release Notes: Platform Specific Features: --------- Signed-off-by: Brian Surber --- .../http/rate_limit_quota/client_impl.cc | 12 ++- .../http/rate_limit_quota/client_test.cc | 90 ++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/http/rate_limit_quota/client_impl.cc b/source/extensions/filters/http/rate_limit_quota/client_impl.cc index 876060592e..e8f9c49a11 100644 --- a/source/extensions/filters/http/rate_limit_quota/client_impl.cc +++ b/source/extensions/filters/http/rate_limit_quota/client_impl.cc @@ -100,8 +100,18 @@ void RateLimitClientImpl::onReceiveMessage(RateLimitQuotaResponsePtr&& response) switch (action.bucket_action_case()) { case envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse_BucketAction:: kQuotaAssignmentAction: { - quota_buckets_[bucket_id]->cached_action = action; + absl::optional cached_action = quota_buckets_[bucket_id]->cached_action; quota_buckets_[bucket_id]->current_assignment_time = time_source_.monotonicTime(); + + if (cached_action.has_value() && + Protobuf::util::MessageDifferencer::Equals(*cached_action, action)) { + ENVOY_LOG(debug, + "Cached action matches the incoming response so only TTL is updated for bucket " + "id: {}", + bucket_id); + break; + } + quota_buckets_[bucket_id]->cached_action = action; if (quota_buckets_[bucket_id]->cached_action->has_quota_assignment_action()) { auto rate_limit_strategy = quota_buckets_[bucket_id] ->cached_action->quota_assignment_action() diff --git a/test/extensions/filters/http/rate_limit_quota/client_test.cc b/test/extensions/filters/http/rate_limit_quota/client_test.cc index 336b47b7ff..2765f7e53c 100644 --- a/test/extensions/filters/http/rate_limit_quota/client_test.cc +++ b/test/extensions/filters/http/rate_limit_quota/client_test.cc @@ -17,6 +17,8 @@ class RateLimitClientTest : public testing::Test { RateLimitTestClient test_client{}; }; +using envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse; + TEST_F(RateLimitClientTest, OpenAndCloseStream) { EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); EXPECT_CALL(test_client.stream_, closeStream()); @@ -53,7 +55,7 @@ TEST_F(RateLimitClientTest, SendRequestAndReceiveResponse) { // `onQuotaResponse` callback is expected to be called. EXPECT_CALL(test_client.callbacks_, onQuotaResponse); - envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse resp; + RateLimitQuotaResponse resp; auto response_buf = Grpc::Common::serializeMessage(resp); EXPECT_TRUE(test_client.stream_callbacks_->onReceiveMessageRaw(std::move(response_buf))); @@ -93,6 +95,92 @@ TEST_F(RateLimitClientTest, RestartStreamWhileInUse) { { test_client.client_->sendUsageReport(bucket_id_hash); }); } +TEST_F(RateLimitClientTest, HandlingDuplicateTokenBucketAssignments) { + EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); + ASSERT_NE(test_client.stream_callbacks_, nullptr); + + auto empty_request_headers = Http::RequestHeaderMapImpl::create(); + test_client.stream_callbacks_->onCreateInitialMetadata(*empty_request_headers); + auto empty_response_headers = Http::ResponseHeaderMapImpl::create(); + test_client.stream_callbacks_->onReceiveInitialMetadata(std::move(empty_response_headers)); + + // `onQuotaResponse` callback is expected to be called twice. + EXPECT_CALL(test_client.callbacks_, onQuotaResponse).Times(3); + + ::envoy::type::v3::TokenBucket token_bucket; + token_bucket.set_max_tokens(100); + token_bucket.mutable_tokens_per_fill()->set_value(10); + token_bucket.mutable_fill_interval()->set_seconds(1000); + + ::envoy::service::rate_limit_quota::v3::BucketId bucket_id; + bucket_id.mutable_bucket()->insert({"fairshare_group_id", "mock_group"}); + const size_t bucket_id_hash = MessageUtil::hash(bucket_id); + + Bucket initial_bucket_state; + initial_bucket_state.bucket_id = bucket_id; + test_client.bucket_cache_.insert( + {bucket_id_hash, std::make_unique(std::move(initial_bucket_state))}); + + RateLimitQuotaResponse::BucketAction action; + action.mutable_quota_assignment_action() + ->mutable_rate_limit_strategy() + ->mutable_token_bucket() + ->MergeFrom(token_bucket); + action.mutable_bucket_id()->MergeFrom(bucket_id); + + RateLimitQuotaResponse resp; + resp.add_bucket_action()->MergeFrom(action); + RateLimitQuotaResponse duplicate_resp; + duplicate_resp.add_bucket_action()->MergeFrom(action); + + auto response_buf = Grpc::Common::serializeMessage(resp); + auto duplicate_response_buf = Grpc::Common::serializeMessage(duplicate_resp); + EXPECT_TRUE(test_client.stream_callbacks_->onReceiveMessageRaw(std::move(response_buf))); + + ASSERT_EQ(test_client.bucket_cache_.size(), 1); + ASSERT_TRUE(test_client.bucket_cache_.contains(bucket_id_hash)); + Bucket* first_bucket = test_client.bucket_cache_.at(bucket_id_hash).get(); + TokenBucket* first_token_bucket_limiter = first_bucket->token_bucket_limiter.get(); + EXPECT_TRUE(first_token_bucket_limiter); + + // Send a duplicate response & expect the token bucket to be carried forward + // in the cache to avoid resetting token consumption. + EXPECT_TRUE( + test_client.stream_callbacks_->onReceiveMessageRaw(std::move(duplicate_response_buf))); + + ASSERT_EQ(test_client.bucket_cache_.size(), 1); + ASSERT_TRUE(test_client.bucket_cache_.contains(bucket_id_hash)); + Bucket* second_bucket = test_client.bucket_cache_.at(bucket_id_hash).get(); + TokenBucket* second_token_bucket_limiter = second_bucket->token_bucket_limiter.get(); + EXPECT_TRUE(second_token_bucket_limiter); + EXPECT_EQ(first_token_bucket_limiter, second_token_bucket_limiter); + + // Expect the limiter to be replaced if the config changes. + resp.mutable_bucket_action(0) + ->mutable_quota_assignment_action() + ->mutable_rate_limit_strategy() + ->mutable_token_bucket() + ->set_max_tokens(200); + auto different_response_buf = Grpc::Common::serializeMessage(resp); + EXPECT_TRUE( + test_client.stream_callbacks_->onReceiveMessageRaw(std::move(different_response_buf))); + + ASSERT_EQ(test_client.bucket_cache_.size(), 1); + ASSERT_TRUE(test_client.bucket_cache_.contains(bucket_id_hash)); + Bucket* third_bucket = test_client.bucket_cache_.at(bucket_id_hash).get(); + TokenBucket* third_token_bucket_limiter = third_bucket->token_bucket_limiter.get(); + EXPECT_TRUE(third_token_bucket_limiter); + EXPECT_NE(first_token_bucket_limiter, third_token_bucket_limiter); + + auto empty_response_trailers = Http::ResponseTrailerMapImpl::create(); + test_client.stream_callbacks_->onReceiveTrailingMetadata(std::move(empty_response_trailers)); + + EXPECT_CALL(test_client.stream_, closeStream()); + EXPECT_CALL(test_client.stream_, resetStream()); + test_client.client_->closeStream(); + test_client.client_->onRemoteClose(0, ""); +} + } // namespace } // namespace RateLimitQuota } // namespace HttpFilters From 577330916c9a48753babe3bd2fb4d17b32da9f9a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 8 Oct 2024 15:26:20 -0400 Subject: [PATCH 35/77] docs: updating governance (#36498) Signed-off-by: Alyssa Wilk --- GOVERNANCE.md | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index ddadb310c6..a09984c085 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -1,34 +1,22 @@ # Process for becoming a maintainer -## Your organization is not yet a maintainer +Becoming a maintainer generally means that you are going to be spending substantial time on +Envoy for the foreseeable future. You should have domain expertise and be extremely proficient in C++. -* Express interest to the senior maintainers that your organization is interested in becoming a - maintainer. Becoming a maintainer generally means that you are going to be spending substantial - time (>25%) on Envoy for the foreseeable future. You should have domain expertise and be extremely - proficient in C++. Ultimately your goal is to become a senior maintainer that will represent your - organization. +* Express interest to the + [envoy-maintainers](https://groups.google.com/forum/#!forum/envoy-announce) + that you are interested in becoming a maintainer and, if your company does not have pre-existing maintainers, + that your organization is interested in and willing to sponsoring a maintainer. * We will expect you to start contributing increasingly complicated PRs, under the guidance - of the existing senior maintainers. -* We may ask you to do some PRs from our backlog. + of the existing maintainers. +* We may ask you to fix some issues from our backlog. * As you gain experience with the code base and our standards, we will ask you to do code reviews for incoming PRs (i.e., all maintainers are expected to shoulder a proportional share of community reviews). -* After a period of approximately 2-3 months of working together and making sure we see eye to eye, - the existing senior maintainers will confer and decide whether to grant maintainer status or not. - We make no guarantees on the length of time this will take, but 2-3 months is the approximate - goal. - -## Your organization is currently a maintainer - -* First decide whether your organization really needs more people with maintainer access. Valid - reasons are "blast radius", a large organization that is working on multiple unrelated projects, - etc. -* Contact a senior maintainer for your organization and express interest. -* Start doing PRs and code reviews under the guidance of your senior maintainer. -* After a period of 1-2 months the existing senior maintainers will discuss granting "standard" - maintainer access. -* "Standard" maintainer access can be upgraded to "senior" maintainer access after another 1-2 - months of work and another conference of the existing senior committers. +* After a period of approximately 2-3 months of contributions demonstrating understanding of (at least parts of) + the Envoy code base, reach back out to the maintainers list asking for feedback. At this point, you will either + be granted maintainer status, or be given actionable feedback on any remaining gaps between the contributions + demonstrated and those expected of maintainers, at which point you can close those gaps and reach back out. ## Maintainer responsibilities From 4b07991211eb9262656ee209decac749c57124dc Mon Sep 17 00:00:00 2001 From: RenjieTang Date: Tue, 8 Oct 2024 17:15:50 -0700 Subject: [PATCH 36/77] [mobile]expose onNetworkTypeChanged API to Engine (#36504) Commit Message: expose onNetworkTypeChanged API to Engine Additional Description: This prevents applications from directly depending on InternalEngine. Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Platform Specific Features: mobile only Signed-off-by: Renjie Tang --- mobile/library/cc/engine.cc | 5 +++++ mobile/library/cc/engine.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/mobile/library/cc/engine.cc b/mobile/library/cc/engine.cc index d94ce7480f..b4aa819cb1 100644 --- a/mobile/library/cc/engine.cc +++ b/mobile/library/cc/engine.cc @@ -1,5 +1,6 @@ #include "engine.h" +#include "library/common/engine_types.h" #include "library/common/internal_engine.h" #include "library/common/types/c_types.h" @@ -26,5 +27,9 @@ std::string Engine::dumpStats() { return engine_->dumpStats(); } envoy_status_t Engine::terminate() { return engine_->terminate(); } +void Engine::onDefaultNetworkChanged(NetworkType network) { + engine_->onDefaultNetworkChanged(network); +} + } // namespace Platform } // namespace Envoy diff --git a/mobile/library/cc/engine.h b/mobile/library/cc/engine.h index 172985a08a..12302e1783 100644 --- a/mobile/library/cc/engine.h +++ b/mobile/library/cc/engine.h @@ -3,6 +3,7 @@ #include #include "library/cc/stream_client.h" +#include "library/common/engine_types.h" #include "library/common/types/c_types.h" namespace Envoy { @@ -20,6 +21,7 @@ class Engine : public std::enable_shared_from_this { std::string dumpStats(); StreamClientSharedPtr streamClient(); + void onDefaultNetworkChanged(NetworkType network); envoy_status_t terminate(); Envoy::InternalEngine* engine() { return engine_; } From 3732616bd0f86e3abaec9218621f307dc940be3c Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 9 Oct 2024 00:01:35 -0400 Subject: [PATCH 37/77] substitution formatter: reducing exceptions (#36407) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a https://github.com/envoyproxy/envoy-mobile/issues/176 Signed-off-by: Alyssa Wilk --- .../formatter/http_formatter_context.cc | 4 +- .../common/formatter/http_formatter_context.h | 2 +- .../formatter/substitution_format_string.h | 8 +- .../common/formatter/substitution_formatter.h | 59 +++++++--- source/common/router/header_parser.cc | 2 +- .../common/stream_access_log_common_impl.h | 4 +- .../extensions/access_loggers/file/config.cc | 8 +- .../open_telemetry/substitution_formatter.cc | 3 +- .../extensions/filters/http/oauth2/filter.cc | 10 +- .../filters/network/ratelimit/ratelimit.cc | 3 +- .../network/redis_proxy/router_impl.cc | 2 +- .../substitution_formatter_fuzz_test.cc | 8 +- .../substitution_formatter_speed_test.cc | 51 +-------- .../formatter/substitution_formatter_test.cc | 105 +++++++++--------- .../http/conn_manager_impl_test_base.cc | 2 +- .../network/generic_proxy/access_log_test.cc | 48 ++++---- .../req_without_query_test.cc | 6 +- test/server/admin/admin_instance.cc | 2 +- test/server/admin/admin_test.cc | 4 +- 19 files changed, 164 insertions(+), 167 deletions(-) diff --git a/source/common/formatter/http_formatter_context.cc b/source/common/formatter/http_formatter_context.cc index 82d7095f4b..cf1e967823 100644 --- a/source/common/formatter/http_formatter_context.cc +++ b/source/common/formatter/http_formatter_context.cc @@ -49,10 +49,10 @@ static constexpr absl::string_view DEFAULT_FORMAT = "\"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" " "\"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"; -FormatterPtr HttpSubstitutionFormatUtils::defaultSubstitutionFormatter() { +absl::StatusOr HttpSubstitutionFormatUtils::defaultSubstitutionFormatter() { // It is possible that failed to parse the default format string if the required formatters // are compiled out. - return std::make_unique(DEFAULT_FORMAT, false); + return Envoy::Formatter::FormatterImpl::create(DEFAULT_FORMAT, false); } } // namespace Formatter diff --git a/source/common/formatter/http_formatter_context.h b/source/common/formatter/http_formatter_context.h index 82e08c8754..7009a67f95 100644 --- a/source/common/formatter/http_formatter_context.h +++ b/source/common/formatter/http_formatter_context.h @@ -11,7 +11,7 @@ namespace Formatter { */ class HttpSubstitutionFormatUtils { public: - static FormatterPtr defaultSubstitutionFormatter(); + static absl::StatusOr defaultSubstitutionFormatter(); }; } // namespace Formatter diff --git a/source/common/formatter/substitution_format_string.h b/source/common/formatter/substitution_format_string.h index 3a03ee3cfc..85bbde6750 100644 --- a/source/common/formatter/substitution_format_string.h +++ b/source/common/formatter/substitution_format_string.h @@ -65,8 +65,8 @@ class SubstitutionFormatStringUtils { RETURN_IF_NOT_OK_REF(commands.status()); switch (config.format_case()) { case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormat: - return std::make_unique>( - config.text_format(), config.omit_empty_values(), *commands); + return FormatterBaseImpl::create(config.text_format(), + config.omit_empty_values(), *commands); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kJsonFormat: return createJsonFormatter( config.json_format(), true, config.omit_empty_values(), @@ -76,8 +76,8 @@ class SubstitutionFormatStringUtils { auto data_source_or_error = Config::DataSource::read(config.text_format_source(), true, context.serverFactoryContext().api()); RETURN_IF_NOT_OK(data_source_or_error.status()); - return std::make_unique>( - *data_source_or_error, config.omit_empty_values(), *commands); + return FormatterBaseImpl::create(*data_source_or_error, + config.omit_empty_values(), *commands); } case envoy::config::core::v3::SubstitutionFormatString::FormatCase::FORMAT_NOT_SET: PANIC_DUE_TO_PROTO_UNSET; diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index f33f07471f..3915f58a08 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -100,7 +100,7 @@ class StreamInfoFormatterWrapper : public FormatterProviderBase - static std::vector> + static absl::StatusOr>> parse(absl::string_view format, const std::vector>& command_parsers = {}) { std::string current_token; @@ -137,7 +137,7 @@ class SubstitutionFormatParser { if (!re2::RE2::Consume(&sub_format, commandWithArgsRegex(), &command, &command_arg, &max_len)) { - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( fmt::format("Incorrect configuration: {}. Couldn't find valid command at position {}", format, pos)); } @@ -184,7 +184,8 @@ class SubstitutionFormatParser { } if (!added) { - throwEnvoyExceptionOrPanic(fmt::format("Not supported field in StreamInfo: {}", command)); + return absl::InvalidArgumentError( + fmt::format("Not supported field in StreamInfo: {}", command)); } pos += (sub_format_size - sub_format.size()); @@ -213,16 +214,14 @@ template class FormatterBaseImpl : public FormatterBase public: using CommandParsers = std::vector>; - FormatterBaseImpl(absl::string_view format, bool omit_empty_values = false) - : empty_value_string_(omit_empty_values ? absl::string_view{} - : DefaultUnspecifiedValueStringView) { - providers_ = SubstitutionFormatParser::parse(format); - } - FormatterBaseImpl(absl::string_view format, bool omit_empty_values, - const CommandParsers& command_parsers) - : empty_value_string_(omit_empty_values ? absl::string_view{} - : DefaultUnspecifiedValueStringView) { - providers_ = SubstitutionFormatParser::parse(format, command_parsers); + static absl::StatusOr> + create(absl::string_view format, bool omit_empty_values = false, + const CommandParsers& command_parsers = {}) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr( + new FormatterBaseImpl(creation_status, format, omit_empty_values, command_parsers)); + RETURN_IF_NOT_OK_REF(creation_status); + return ret; } // FormatterBase @@ -239,6 +238,25 @@ template class FormatterBaseImpl : public FormatterBase return log_line; } +protected: + FormatterBaseImpl(absl::Status& creation_status, absl::string_view format, + bool omit_empty_values = false) + : empty_value_string_(omit_empty_values ? absl::string_view{} + : DefaultUnspecifiedValueStringView) { + auto providers_or_error = SubstitutionFormatParser::parse(format); + SET_AND_RETURN_IF_NOT_OK(providers_or_error.status(), creation_status); + providers_ = std::move(*providers_or_error); + } + FormatterBaseImpl(absl::Status& creation_status, absl::string_view format, bool omit_empty_values, + const CommandParsers& command_parsers = {}) + : empty_value_string_(omit_empty_values ? absl::string_view{} + : DefaultUnspecifiedValueStringView) { + auto providers_or_error = + SubstitutionFormatParser::parse(format, command_parsers); + SET_AND_RETURN_IF_NOT_OK(providers_or_error.status(), creation_status); + providers_ = std::move(*providers_or_error); + } + private: const std::string empty_value_string_; std::vector> providers_; @@ -387,8 +405,9 @@ class JsonFormatterImplBase : public FormatterBase { for (JsonFormatBuilder::FormatElement& element : JsonFormatBuilder().fromStruct(struct_format)) { if (element.is_template_) { - parsed_elements_.emplace_back( - SubstitutionFormatParser::parse(element.value_, commands)); + parsed_elements_.emplace_back(THROW_OR_RETURN_VALUE( + SubstitutionFormatParser::parse(element.value_, commands), + std::vector>)); } else { parsed_elements_.emplace_back(std::move(element.value_)); } @@ -565,7 +584,7 @@ template class StructFormatterBase { class FormatBuilder { public: explicit FormatBuilder(const CommandParsers& commands) : commands_(commands) {} - std::vector> + absl::StatusOr>> toFormatStringValue(const std::string& string_format) const { return SubstitutionFormatParser::parse(string_format, commands_); } @@ -580,7 +599,9 @@ template class StructFormatterBase { for (const auto& pair : struct_format.fields()) { switch (pair.second.kind_case()) { case ProtobufWkt::Value::kStringValue: - output->emplace(pair.first, toFormatStringValue(pair.second.string_value())); + output->emplace(pair.first, THROW_OR_RETURN_VALUE( + toFormatStringValue(pair.second.string_value()), + std::vector>)); break; case ProtobufWkt::Value::kStructValue: @@ -608,7 +629,9 @@ template class StructFormatterBase { for (const auto& value : list_value_format.values()) { switch (value.kind_case()) { case ProtobufWkt::Value::kStringValue: - output->emplace_back(toFormatStringValue(value.string_value())); + output->emplace_back( + THROW_OR_RETURN_VALUE(toFormatStringValue(value.string_value()), + std::vector>)); break; case ProtobufWkt::Value::kStructValue: diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index 4d33c0ce45..f7feb18a63 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -45,7 +45,7 @@ parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_valu final_header_value = HeaderParser::translatePerRequestState(final_header_value); // Let the substitution formatter parse the final_header_value. - return std::make_unique(final_header_value, true); + return Envoy::Formatter::FormatterImpl::create(final_header_value, true); } } // namespace diff --git a/source/extensions/access_loggers/common/stream_access_log_common_impl.h b/source/extensions/access_loggers/common/stream_access_log_common_impl.h index 5b07233124..6f7b3582ad 100644 --- a/source/extensions/access_loggers/common/stream_access_log_common_impl.h +++ b/source/extensions/access_loggers/common/stream_access_log_common_impl.h @@ -23,7 +23,9 @@ createStreamAccessLogInstance(const Protobuf::Message& config, AccessLog::Filter Formatter::FormatterBasePtr); } else if (fal_config.access_log_format_case() == T::AccessLogFormatCase::ACCESS_LOG_FORMAT_NOT_SET) { - formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); + formatter = THROW_OR_RETURN_VALUE( + Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + Formatter::FormatterPtr); } Filesystem::FilePathAndType file_info{destination_type, ""}; return std::make_shared( diff --git a/source/extensions/access_loggers/file/config.cc b/source/extensions/access_loggers/file/config.cc index 2761aa5064..33d4eefca2 100644 --- a/source/extensions/access_loggers/file/config.cc +++ b/source/extensions/access_loggers/file/config.cc @@ -31,7 +31,9 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, switch (fal_config.access_log_format_case()) { case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase::kFormat: if (fal_config.format().empty()) { - formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); + formatter = THROW_OR_RETURN_VALUE( + Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + Formatter::FormatterPtr); } else { envoy::config::core::v3::SubstitutionFormatString sff_config; sff_config.mutable_text_format_source()->set_inline_string(fal_config.format()); @@ -60,7 +62,9 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, break; case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase:: ACCESS_LOG_FORMAT_NOT_SET: - formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); + formatter = THROW_OR_RETURN_VALUE( + Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + Formatter::FormatterPtr); break; } diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc index a99b80878d..796915cde9 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc @@ -78,7 +78,8 @@ OpenTelemetryFormatter::FormatBuilder::toFormatListValue( std::vector OpenTelemetryFormatter::FormatBuilder::toFormatStringValue(const std::string& string_format) const { - return Formatter::SubstitutionFormatParser::parse(string_format, commands_); + return THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatParser::parse(string_format, commands_), + std::vector); } ::opentelemetry::proto::common::v1::AnyValue OpenTelemetryFormatter::providersCallback( diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index 12657167f4..4c8bd1da6b 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -422,9 +422,10 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he original_request_url_ = result.original_request_url_; auth_code_ = result.auth_code_; - Formatter::FormatterImpl formatter(config_->redirectUri()); + Formatter::FormatterPtr formatter = THROW_OR_RETURN_VALUE( + Formatter::FormatterImpl::create(config_->redirectUri()), Formatter::FormatterPtr); const auto redirect_uri = - formatter.formatWithContext({&headers}, decoder_callbacks_->streamInfo()); + formatter->formatWithContext({&headers}, decoder_callbacks_->streamInfo()); oauth_client_->asyncGetAccessToken(auth_code_, config_->clientId(), config_->clientSecret(), redirect_uri, config_->authType()); @@ -519,9 +520,10 @@ void OAuth2Filter::redirectToOAuthServer(Http::RequestHeaderMap& headers) const absl::StrCat(stateParamsUrl, "=", escaped_url, "&", stateParamsNonce, "=", nonce); const std::string escaped_state = Http::Utility::PercentEncoding::urlEncodeQueryParameter(state); - Formatter::FormatterImpl formatter(config_->redirectUri()); + Formatter::FormatterPtr formatter = THROW_OR_RETURN_VALUE( + Formatter::FormatterImpl::create(config_->redirectUri()), Formatter::FormatterPtr); const auto redirect_uri = - formatter.formatWithContext({&headers}, decoder_callbacks_->streamInfo()); + formatter->formatWithContext({&headers}, decoder_callbacks_->streamInfo()); const std::string escaped_redirect_uri = Http::Utility::PercentEncoding::urlEncodeQueryParameter(redirect_uri); diff --git a/source/extensions/filters/network/ratelimit/ratelimit.cc b/source/extensions/filters/network/ratelimit/ratelimit.cc index ba9a1aa3a5..426d712a72 100644 --- a/source/extensions/filters/network/ratelimit/ratelimit.cc +++ b/source/extensions/filters/network/ratelimit/ratelimit.cc @@ -30,7 +30,8 @@ Config::Config(const envoy::extensions::filters::network::ratelimit::v3::RateLim for (const auto& entry : descriptor.entries()) { new_descriptor.entries_.push_back({entry.key(), entry.value()}); substitution_formatters_.push_back( - std::make_unique(entry.value(), false)); + THROW_OR_RETURN_VALUE(Formatter::FormatterImpl::create(entry.value(), false), + std::unique_ptr)); } original_descriptors_.push_back(new_descriptor); } diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc index 09621e04fa..72c96b3456 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.cc +++ b/source/extensions/filters/network/redis_proxy/router_impl.cc @@ -136,7 +136,7 @@ void PrefixRoutes::formatKey(std::string& key, std::string redis_key_formatter, redis_key_formatter = absl::StrReplaceAll( redis_key_formatter, {{redis_key_formatter_command_, redis_key_to_be_replaced_}}); } - auto providers = Formatter::SubstitutionFormatParser::parse(redis_key_formatter); + auto providers = *Formatter::SubstitutionFormatParser::parse(redis_key_formatter); std::string formatted_key; for (Formatter::FormatterProviderPtr& provider : providers) { auto provider_formatted_key = provider->formatValueWithContext({}, stream_info); diff --git a/test/common/formatter/substitution_formatter_fuzz_test.cc b/test/common/formatter/substitution_formatter_fuzz_test.cc index 7540c37f93..79f676d810 100644 --- a/test/common/formatter/substitution_formatter_fuzz_test.cc +++ b/test/common/formatter/substitution_formatter_fuzz_test.cc @@ -39,7 +39,13 @@ DEFINE_PROTO_FUZZER(const test::common::substitution::TestCase& input) { { Formatter::FormatterPtr formatter; try { - formatter = std::make_unique(input.format()); + auto formatter_or_error = Formatter::FormatterImpl::create(input.format(), false); + if (!formatter_or_error.status().ok()) { + ENVOY_LOG_MISC(debug, "TEXT formatter failed, EnvoyException: {}", + formatter_or_error.status().message()); + return; + } + formatter = std::move(*formatter_or_error); } catch (const EnvoyException& e) { ENVOY_LOG_MISC(debug, "TEXT formatter failed, EnvoyException: {}", e.what()); return; diff --git a/test/common/formatter/substitution_formatter_speed_test.cc b/test/common/formatter/substitution_formatter_speed_test.cc index 9e39bee306..0c00699348 100644 --- a/test/common/formatter/substitution_formatter_speed_test.cc +++ b/test/common/formatter/substitution_formatter_speed_test.cc @@ -11,25 +11,6 @@ namespace Envoy { namespace { -std::unique_ptr makeLegacyJsonFormatter(bool typed) { - ProtobufWkt::Struct JsonLogFormat; - const std::string format_yaml = R"EOF( - remote_address: '%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%' - start_time: '%START_TIME(%Y/%m/%dT%H:%M:%S%z %s)%' - method: '%REQ(:METHOD)%' - url: '%REQ(X-FORWARDED-PROTO)%://%REQ(:AUTHORITY)%%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%' - protocol: '%PROTOCOL%' - response_code: '%RESPONSE_CODE%' - bytes_sent: '%BYTES_SENT%' - duration: '%DURATION%' - referer: '%REQ(REFERER)%' - user-agent: '%REQ(USER-AGENT)%' - )EOF"; - TestUtility::loadFromYaml(format_yaml, JsonLogFormat); - return std::make_unique(JsonLogFormat, typed, false, - false); -} - std::unique_ptr makeJsonFormatter() { ProtobufWkt::Struct JsonLogFormat; const std::string format_yaml = R"EOF( @@ -87,7 +68,7 @@ static void BM_AccessLogFormatterSetup(benchmark::State& state) { for (auto _ : state) { // NOLINT: Silences warning about dead store std::unique_ptr formatter = - std::make_unique(LogFormat, false); + *Envoy::Formatter::FormatterImpl::create(LogFormat, false); } } BENCHMARK(BM_AccessLogFormatterSetup); @@ -104,7 +85,7 @@ static void BM_AccessLogFormatter(benchmark::State& state) { "s%RESPONSE_CODE% %BYTES_SENT% %DURATION% %REQ(REFERER)% \"%REQ(USER-AGENT)%\" - - -\n"; std::unique_ptr formatter = - std::make_unique(LogFormat, false); + *Envoy::Formatter::FormatterImpl::create(LogFormat, false); size_t output_bytes = 0; for (auto _ : state) { // NOLINT: Silences warning about dead store @@ -145,34 +126,6 @@ static void BM_TypedStructAccessLogFormatter(benchmark::State& state) { } BENCHMARK(BM_TypedStructAccessLogFormatter); -// NOLINTNEXTLINE(readability-identifier-naming) -static void BM_LegacyJsonAccessLogFormatter(benchmark::State& state) { - testing::NiceMock time_system; - std::unique_ptr stream_info = makeStreamInfo(time_system); - auto json_formatter = makeLegacyJsonFormatter(false); - - size_t output_bytes = 0; - for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += json_formatter->formatWithContext({}, *stream_info).length(); - } - benchmark::DoNotOptimize(output_bytes); -} -BENCHMARK(BM_LegacyJsonAccessLogFormatter); - -// NOLINTNEXTLINE(readability-identifier-naming) -static void BM_LegacyTypedJsonAccessLogFormatter(benchmark::State& state) { - testing::NiceMock time_system; - std::unique_ptr stream_info = makeStreamInfo(time_system); - auto json_formatter = makeLegacyJsonFormatter(true); - - size_t output_bytes = 0; - for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += json_formatter->formatWithContext({}, *stream_info).length(); - } - benchmark::DoNotOptimize(output_bytes); -} -BENCHMARK(BM_LegacyTypedJsonAccessLogFormatter); - // NOLINTNEXTLINE(readability-identifier-naming) static void BM_JsonAccessLogFormatter(benchmark::State& state) { testing::NiceMock time_system; diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 11bfd2135c..9c058bdf46 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -4592,21 +4592,21 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "{{%PROTOCOL%}} %RESP(not_exist)%++%RESP(test)% " "%REQ(FIRST?SECOND)% %RESP(FIRST?SECOND)%" "\t@%TRAILER(THIRD)%@\t%TRAILER(TEST?TEST-2)%[]"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); EXPECT_EQ("{{HTTP/1.1}} -++test GET PUT\t@POST@\ttest-2[]", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { NiceMock stream_info; const std::string format = "{}*JUST PLAIN string]"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ(format, formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ(format, formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4614,9 +4614,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%REQ(first):3%|%REQ(first):1%|%RESP(first?second):2%|%REQ(first):" "10%|%TRAILER(second?third):3%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ("GET|G|PU|GET|POS", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("GET|G|PU|GET|POS", formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4627,10 +4627,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:" "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("test_value|{\"inner_key\":\"inner_value\"}|inner_value", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4646,10 +4646,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { StreamInfo::FilterState::LifeSpan::FilterChain); const std::string format = "%FILTER_STATE(testing)%|%FILTER_STATE(serialized)%|" "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("\"test_value\"|-|\"test_va|-", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4661,11 +4661,11 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { time_t expected_time_in_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", expected_time_in_epoch), - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4681,11 +4681,11 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(time)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", expected_time_in_epoch), - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4701,11 +4701,11 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(time)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", expected_time_in_epoch), - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4717,10 +4717,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const time_t test_epoch = 0; const SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("1970/01/01|0|bad_format|1970-01-01T00:00:00.000Z|000000000.0.00.000", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4730,9 +4730,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { "%START_TIME(%s.%3f)%|%START_TIME(%s.%4f)%|%START_TIME(%s.%5f)%|%START_TIME(%s.%6f)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(start_time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("1522796769.123|1522796769.1234|1522796769.12345|1522796769.123456", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4741,10 +4741,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { "%START_TIME(segment1:%s.%3f|segment2:%s.%4f|seg3:%s.%6f|%s-%3f-asdf-%9f|.%7f:segm5:%Y)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(start_time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("segment1:1522796769.123|segment2:1522796769.1234|seg3:1522796769.123456|1522796769-" "123-asdf-123456000|.1234560:segm5:2018", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4754,9 +4754,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%START_TIME(%%%%|%%%%%f|%s%%%%%3f|%1f%%%%%s)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("%%|%%123456000|1522796769%%123|1%%1522796769", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } // The %E formatting option in Absl::FormatTime() behaves differently for non Linux platforms. @@ -4770,8 +4770,8 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%START_TIME(%E4n)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); - FormatterImpl formatter(format, false); - EXPECT_EQ("%E4n", formatter.formatWithContext(formatter_context, stream_info)); + FormatterPtr formatter = *FormatterImpl::create(format, false); + EXPECT_EQ("%E4n", formatter->formatWithContext(formatter_context, stream_info)); } #endif } @@ -4790,22 +4790,22 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { const std::string format = "%PROTOCOL%|%RESP(not_exist)%|" "%REQ(FIRST?SECOND)%|%RESP(FIRST?SECOND)%|" "%TRAILER(THIRD)%|%TRAILER(TEST?TEST-2)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(absl::nullopt)); - EXPECT_EQ("-|-|-|-|-|-", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("-|-|-|-|-|-", formatter->formatWithContext(formatter_context, stream_info)); } { const std::string format = "%PROTOCOL%|%RESP(not_exist)%|" "%REQ(FIRST?SECOND)%%RESP(FIRST?SECOND)%|" "%TRAILER(THIRD)%|%TRAILER(TEST?TEST-2)%"; - FormatterImpl formatter(format, true); + FormatterPtr formatter = *FormatterImpl::create(format, true); EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(absl::nullopt)); - EXPECT_EQ("||||", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("||||", formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4814,9 +4814,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:" "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ("-|-|-", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("-|-|-", formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4825,27 +4825,27 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:" "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; - FormatterImpl formatter(format, true); + FormatterPtr formatter = *FormatterImpl::create(format, true); - EXPECT_EQ("||", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("||", formatter->formatWithContext(formatter_context, stream_info)); } { EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); const std::string format = "%FILTER_STATE(testing)%|%FILTER_STATE(serialized)%|" "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ("-|-|-|-", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("-|-|-|-", formatter->formatWithContext(formatter_context, stream_info)); } { EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); const std::string format = "%FILTER_STATE(testing)%|%FILTER_STATE(serialized)%|" "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; - FormatterImpl formatter(format, true); + FormatterPtr formatter = *FormatterImpl::create(format, true); - EXPECT_EQ("|||", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("|||", formatter->formatWithContext(formatter_context, stream_info)); } } @@ -4895,7 +4895,10 @@ TEST(SubstitutionFormatterTest, ParserFailures) { "%START_TIME(%4On%)%"}; for (const std::string& test_case : test_cases) { - EXPECT_THROW(parser.parse(test_case), EnvoyException) << test_case; + EXPECT_THROW(THROW_OR_RETURN_VALUE(parser.parse(test_case), + std::vector>), + EnvoyException) + << test_case; } } @@ -4906,14 +4909,14 @@ TEST(SubstitutionFormatterTest, ParserSuccesses) { "%DOWNSTREAM_PEER_FINGERPRINT_256%"}; for (const std::string& test_case : test_cases) { - EXPECT_NO_THROW(parser.parse(test_case)); + EXPECT_TRUE(parser.parse(test_case).status().ok()); } } TEST(SubstitutionFormatterTest, EmptyFormatParse) { StreamInfo::MockStreamInfo stream_info; - auto providers = SubstitutionFormatParser::parse(""); + auto providers = *SubstitutionFormatParser::parse(""); EXPECT_EQ(providers.size(), 1); EXPECT_EQ("", providers[0]->formatWithContext({}, stream_info)); @@ -4922,7 +4925,7 @@ TEST(SubstitutionFormatterTest, EmptyFormatParse) { TEST(SubstitutionFormatterTest, EscapingFormatParse) { StreamInfo::MockStreamInfo stream_info; - auto providers = SubstitutionFormatParser::parse("%%"); + auto providers = *SubstitutionFormatParser::parse("%%"); ASSERT_EQ(providers.size(), 1); EXPECT_EQ("%", providers[0]->formatWithContext({}, stream_info)); @@ -4934,7 +4937,7 @@ TEST(SubstitutionFormatterTest, FormatterExtension) { std::vector commands; commands.push_back(std::make_unique()); - auto providers = SubstitutionFormatParser::parse("foo %COMMAND_EXTENSION(x)%", commands); + auto providers = *SubstitutionFormatParser::parse("foo %COMMAND_EXTENSION(x)%", commands); EXPECT_EQ(providers.size(), 2); EXPECT_EQ("TestFormatter", providers[1]->formatWithContext({}, stream_info)); @@ -4952,7 +4955,7 @@ TEST(SubstitutionFormatterTest, PercentEscapingEdgeCase) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - auto providers = SubstitutionFormatParser::parse("%HOSTNAME%%PROTOCOL%"); + auto providers = *SubstitutionFormatParser::parse("%HOSTNAME%%PROTOCOL%"); ASSERT_EQ(providers.size(), 2); EXPECT_EQ("myhostname", providers[0]->formatWithContext({}, stream_info)); @@ -4961,14 +4964,14 @@ TEST(SubstitutionFormatterTest, PercentEscapingEdgeCase) { TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { { - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatParser::parse("%ENVIRONMENT()%"), EnvoyException, - "ENVIRONMENT requires parameters"); + EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatParser::parse("%ENVIRONMENT()%").IgnoreError(), + EnvoyException, "ENVIRONMENT requires parameters"); } { StreamInfo::MockStreamInfo stream_info; - auto providers = SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); + auto providers = *SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); ASSERT_EQ(providers.size(), 1); @@ -4981,7 +4984,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { TestEnvironment::setEnvVar("ENVOY_TEST_ENV", "test", 1); Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar("ENVOY_TEST_ENV"); }); - auto providers = SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); + auto providers = *SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); ASSERT_EQ(providers.size(), 1); @@ -4994,7 +4997,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { TestEnvironment::setEnvVar("ENVOY_TEST_ENV", "test", 1); Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar("ENVOY_TEST_ENV"); }); - auto providers = SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV):2%"); + auto providers = *SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV):2%"); ASSERT_EQ(providers.size(), 1); @@ -5062,7 +5065,7 @@ TEST(SubstitutionFormatterTest, UniqueIdFormatterTest) { StreamInfo::MockStreamInfo stream_info; // Simulate initial parsing of configuration - auto providers1 = SubstitutionFormatParser::parse("%UNIQUE_ID%"); + auto providers1 = *SubstitutionFormatParser::parse("%UNIQUE_ID%"); ASSERT_EQ(providers1.size(), 1); // Generate first unique ID with the initial configuration @@ -5077,7 +5080,7 @@ TEST(SubstitutionFormatterTest, UniqueIdFormatterTest) { EXPECT_NE(id1, id2); // Simulate configuration reload - auto providers2 = SubstitutionFormatParser::parse("%UNIQUE_ID%"); + auto providers2 = *SubstitutionFormatParser::parse("%UNIQUE_ID%"); ASSERT_EQ(providers2.size(), 1); // Generate another unique ID after the simulated reload diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index 0b70e20579..bed8ebe013 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -152,7 +152,7 @@ HttpConnectionManagerImplMixin::HttpConnectionManagerImplMixin() access_log_path_("dummy_path"), access_logs_{AccessLog::InstanceSharedPtr{new Extensions::AccessLoggers::File::FileAccessLog( Filesystem::FilePathAndType{Filesystem::DestinationType::File, access_log_path_}, {}, - Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), log_manager_)}}, + *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), log_manager_)}}, codec_(new NiceMock()), stats_({ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(*fake_stats_.rootScope()), POOL_GAUGE(*fake_stats_.rootScope()), diff --git a/test/extensions/filters/network/generic_proxy/access_log_test.cc b/test/extensions/filters/network/generic_proxy/access_log_test.cc index b0041c6d08..04426bdb06 100644 --- a/test/extensions/filters/network/generic_proxy/access_log_test.cc +++ b/test/extensions/filters/network/generic_proxy/access_log_test.cc @@ -63,112 +63,114 @@ TEST(AccessLogFormatterTest, AccessLogFormatterTest) { { // Test for %METHOD%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%METHOD%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%METHOD%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.method_ = "FAKE_METHOD"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_METHOD"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_METHOD"); } { // Test for %HOST%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%HOST%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%HOST%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.host_ = "FAKE_HOST"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_HOST"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_HOST"); } { // Test for %PATH%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%PATH%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%PATH%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.path_ = "FAKE_PATH"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_PATH"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_PATH"); } { // Test for %PROTOCOL%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%PROTOCOL%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%PROTOCOL%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.protocol_ = "FAKE_PROTOCOL"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_PROTOCOL"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_PROTOCOL"); } { // Test for %REQUEST_PROPERTY%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%REQUEST_PROPERTY(FAKE_KEY)%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create( + "%REQUEST_PROPERTY(FAKE_KEY)%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); request.data_["FAKE_KEY"] = "FAKE_VALUE"; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_VALUE"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_VALUE"); } { // Test for %RESPONSE_PROPERTY%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter( + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create( "%RESPONSE_PROPERTY(FAKE_KEY)%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeResponse response; context.response_ = &response; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); response.data_["FAKE_KEY"] = "FAKE_VALUE"; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_VALUE"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_VALUE"); } { // Test for %GENERIC_RESPONSE_CODE%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%GENERIC_RESPONSE_CODE%"); + auto formatter = + *Envoy::Formatter::FormatterBaseImpl::create("%GENERIC_RESPONSE_CODE%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeResponse response; response.status_ = {-1234, false}; context.response_ = &response; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-1234"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-1234"); } } diff --git a/test/extensions/formatter/req_without_query/req_without_query_test.cc b/test/extensions/formatter/req_without_query/req_without_query_test.cc index e019ef4e91..255f3df184 100644 --- a/test/extensions/formatter/req_without_query/req_without_query_test.cc +++ b/test/extensions/formatter/req_without_query/req_without_query_test.cc @@ -156,9 +156,9 @@ TEST_F(ReqWithoutQueryTest, TestParserNotRecognizingCommand) { )EOF"; TestUtility::loadFromYaml(yaml, config_); - EXPECT_THROW(Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_) - .IgnoreError(), - EnvoyException); + EXPECT_FALSE(Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_) + .status() + .ok()); } } // namespace Formatter diff --git a/test/server/admin/admin_instance.cc b/test/server/admin/admin_instance.cc index a2b4d2f15d..919a1b3083 100644 --- a/test/server/admin/admin_instance.cc +++ b/test/server/admin/admin_instance.cc @@ -14,7 +14,7 @@ AdminInstanceTest::AdminInstanceTest() std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( - file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + file_info, {}, *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); server_.options_.admin_address_path_ = TestEnvironment::temporaryPath("admin.address"); admin_.startHttpListener(access_logs, Network::Test::getCanonicalLoopbackAddress(GetParam()), diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index fb6f64ff6d..997e7900ca 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -82,7 +82,7 @@ TEST_P(AdminInstanceTest, AdminAddress) { std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( - file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + file_info, {}, *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); EXPECT_LOG_CONTAINS( "info", "admin address:", @@ -97,7 +97,7 @@ TEST_P(AdminInstanceTest, AdminBadAddressOutPath) { std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( - file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + file_info, {}, *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); EXPECT_LOG_CONTAINS( "critical", From e0ac5ac31a4f4b88399a9a1efddc590b9b2da3ca Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Wed, 9 Oct 2024 09:05:38 -0400 Subject: [PATCH 38/77] xds: internal refactor using absl::span instead of Protobuf::RepeatedPtrField (#36316) Signed-off-by: Adi Suissa-Peleg --- envoy/config/subscription.h | 8 ++++---- envoy/config/xds_config_tracker.h | 8 ++++---- .../grpc/delta_subscription_state.cc | 6 ++++-- .../config_subscription/grpc/watch_map.cc | 8 ++++---- .../extensions/config_subscription/grpc/watch_map.h | 8 ++++---- .../grpc/xds_mux/delta_subscription_state.cc | 6 ++++-- .../xds_config_tracker_integration_test.cc | 13 ++++++------- test/mocks/config/mocks.h | 9 ++++----- 8 files changed, 34 insertions(+), 32 deletions(-) diff --git a/envoy/config/subscription.h b/envoy/config/subscription.h index 4bcd3e56b4..6ca9dc2403 100644 --- a/envoy/config/subscription.h +++ b/envoy/config/subscription.h @@ -191,10 +191,10 @@ class UntypedConfigUpdateCallbacks { * being updated. Accepted changes have their version_info reflected in subsequent * requests. */ - virtual void onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) PURE; + virtual void + onConfigUpdate(absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) PURE; /** * Called when either the Subscription is unable to fetch a config update or when onConfigUpdate diff --git a/envoy/config/xds_config_tracker.h b/envoy/config/xds_config_tracker.h index e1bbac0e0e..5f0b3ddd74 100644 --- a/envoy/config/xds_config_tracker.h +++ b/envoy/config/xds_config_tracker.h @@ -60,10 +60,10 @@ class XdsConfigTracker { * @param added_resources A list of decoded resources to add to the current state. * @param removed_resources A list of resources to remove from the current state. */ - virtual void onConfigAccepted( - const absl::string_view type_url, - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources) PURE; + virtual void + onConfigAccepted(const absl::string_view type_url, + absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources) PURE; /** * Invoked when xds configs are rejected during xDS ingestion. diff --git a/source/extensions/config_subscription/grpc/delta_subscription_state.cc b/source/extensions/config_subscription/grpc/delta_subscription_state.cc index 7a1f2fd354..1d1b949323 100644 --- a/source/extensions/config_subscription/grpc/delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/delta_subscription_state.cc @@ -218,12 +218,14 @@ void DeltaSubscriptionState::handleGoodResponse( } } - watch_map_.onConfigUpdate(non_heartbeat_resources, message.removed_resources(), + absl::Span non_heartbeat_resources_span = + absl::MakeConstSpan(non_heartbeat_resources.data(), non_heartbeat_resources.size()); + watch_map_.onConfigUpdate(non_heartbeat_resources_span, message.removed_resources(), message.system_version_info()); // Processing point when resources are successfully ingested. if (xds_config_tracker_.has_value()) { - xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources, + xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources_span, message.removed_resources()); } diff --git a/source/extensions/config_subscription/grpc/watch_map.cc b/source/extensions/config_subscription/grpc/watch_map.cc index ffa57508ee..ad6af7e18c 100644 --- a/source/extensions/config_subscription/grpc/watch_map.cc +++ b/source/extensions/config_subscription/grpc/watch_map.cc @@ -216,7 +216,7 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField } void WatchMap::onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, + absl::Span added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { // Track any removals triggered by earlier watch updates. @@ -232,15 +232,15 @@ void WatchMap::onConfigUpdate( // resources the watch map is interested in. Reserve the correct amount of // space for the vector for the good case. decoded_resources.reserve(added_resources.size()); - for (const auto& r : added_resources) { - const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r.name()); + for (const auto* r : added_resources) { + const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r->name()); // If there are no watches, then we don't need to decode. If there are watches, they should all // be for the same resource type, so we can just use the callbacks of the first watch to decode. if (interested_in_r.empty()) { continue; } decoded_resources.emplace_back( - new DecodedResourceImpl((*interested_in_r.begin())->resource_decoder_, r)); + new DecodedResourceImpl((*interested_in_r.begin())->resource_decoder_, *r)); for (const auto& interested_watch : interested_in_r) { per_watch_added[interested_watch].emplace_back(*decoded_resources.back()); } diff --git a/source/extensions/config_subscription/grpc/watch_map.h b/source/extensions/config_subscription/grpc/watch_map.h index 07140804ed..1f824900c2 100644 --- a/source/extensions/config_subscription/grpc/watch_map.h +++ b/source/extensions/config_subscription/grpc/watch_map.h @@ -105,10 +105,10 @@ class WatchMap : public UntypedConfigUpdateCallbacks, public Logger::Loggable& resources, const std::string& version_info) override; - void onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override; + void + onConfigUpdate(absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; WatchMap(const WatchMap&) = delete; diff --git a/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc b/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc index 46f1fb04a6..f274b903bf 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc @@ -188,12 +188,14 @@ void DeltaSubscriptionState::handleGoodResponse( } } - callbacks().onConfigUpdate(non_heartbeat_resources, message.removed_resources(), + absl::Span non_heartbeat_resources_span = + absl::MakeConstSpan(non_heartbeat_resources.data(), non_heartbeat_resources.size()); + callbacks().onConfigUpdate(non_heartbeat_resources_span, message.removed_resources(), message.system_version_info()); // Processing point when resources are successfully ingested. if (xds_config_tracker_.has_value()) { - xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources, + xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources_span, message.removed_resources()); } diff --git a/test/integration/xds_config_tracker_integration_test.cc b/test/integration/xds_config_tracker_integration_test.cc index 9838ef2edf..b80eae8717 100644 --- a/test/integration/xds_config_tracker_integration_test.cc +++ b/test/integration/xds_config_tracker_integration_test.cc @@ -68,15 +68,14 @@ class TestXdsConfigTracker : public Config::XdsConfigTracker { } } - void onConfigAccepted( - const absl::string_view, - const Protobuf::RepeatedPtrField& resources, - const Protobuf::RepeatedPtrField&) override { + void onConfigAccepted(const absl::string_view, + absl::Span resources, + const Protobuf::RepeatedPtrField&) override { stats_.on_config_accepted_.inc(); test::envoy::config::xds::TestTrackerMetadata test_metadata; - for (const auto& resource : resources) { - if (resource.has_metadata()) { - const auto& config_typed_metadata = resource.metadata().typed_filter_metadata(); + for (const auto* resource : resources) { + if (resource->has_metadata()) { + const auto& config_typed_metadata = resource->metadata().typed_filter_metadata(); if (const auto& metadata_it = config_typed_metadata.find(kTestKey); metadata_it != config_typed_metadata.end()) { const auto status = Envoy::MessageUtil::unpackTo(metadata_it->second, test_metadata); diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 2d23f31794..85a10b89a3 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -56,11 +56,10 @@ class MockUntypedConfigUpdateCallbacks : public UntypedConfigUpdateCallbacks { MOCK_METHOD(void, onConfigUpdate, (const std::vector& resources, const std::string& version_info)); - MOCK_METHOD( - void, onConfigUpdate, - (const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info)); + MOCK_METHOD(void, onConfigUpdate, + (absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info)); MOCK_METHOD(void, onConfigUpdateFailed, (Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e)); }; From c6761de5f33e97758471c6b973a57a5c2e5db925 Mon Sep 17 00:00:00 2001 From: Tony Allen Date: Wed, 9 Oct 2024 06:45:39 -0700 Subject: [PATCH 39/77] http_11_proxy: Make inner transport_socket config optional (#36414) http_11_proxy: Make inner transport_socket config optional Given that the top-level [Cluster.transport_socket](https://github.com/envoyproxy/envoy/blob/1a153166a6d1e9336ee8982d1a00ba98655c9d39/api/envoy/config/cluster/v3/cluster.proto#L1099) field is optional and defaults to plaintext, this should also be optional. gRPC is adding support for this transport socket, but they do not have a `raw_buffer` to explicitly configure. See https://github.com/grpc/proposal/pull/455#discussion_r1776143739 for additional context. Risk Level: Low. Testing: Existing tests. Docs Changes: n/a Release Notes: Done. --------- Signed-off-by: Tony Allen --- .../v3/upstream_http_11_connect.proto | 5 ++--- changelogs/current.yaml | 3 +++ .../http_11_proxy/connect_integration_test.cc | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto b/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto index a26a689e22..2c9b5333f4 100644 --- a/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto +++ b/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto @@ -5,7 +5,6 @@ package envoy.extensions.transport_sockets.http_11_proxy.v3; import "envoy/config/core/v3/base.proto"; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.transport_sockets.http_11_proxy.v3"; option java_outer_classname = "UpstreamHttp11ConnectProto"; @@ -34,6 +33,6 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // proxy address in ``config::core::v3::Address`` format. // message Http11ProxyUpstreamTransport { - // The underlying transport socket being wrapped. - config.core.v3.TransportSocket transport_socket = 1 [(validate.rules).message = {required: true}]; + // The underlying transport socket being wrapped. Defaults to plaintext (raw_buffer) if unset. + config.core.v3.TransportSocket transport_socket = 1; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index fdf30064d8..b831842ed7 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -105,6 +105,9 @@ minor_behavior_changes: Connect the QUIC UDP client connection sockets before use and sockets will only bind if the local address is specified. This behavior change can be reverted by setting the ``envoy_reloadable_features_quic_connect_client_udp_sockets`` runtime flag to ``false``. +- area: http_11_proxy + change: | + Make the inner ``transport_socket`` field optional in the proto configuration. - area: conn_handler change: | Enhanced listener filter chain execution to include the case that listener filter has maxReadBytes() of 0, diff --git a/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc b/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc index 37ecf34ad0..3835e801f9 100644 --- a/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc +++ b/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc @@ -74,7 +74,7 @@ name: envoy.clusters.dynamic_forward_proxy bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_transport_socket(); envoy::config::core::v3::TransportSocket inner_socket; inner_socket.CopyFrom(*transport_socket); - if (inner_socket.name().empty()) { + if (set_inner_transport_socket_ && inner_socket.name().empty()) { inner_socket.set_name("envoy.transport_sockets.raw_buffer"); } transport_socket->set_name("envoy.transport_sockets.http_11_proxy"); @@ -135,12 +135,28 @@ name: envoy.clusters.dynamic_forward_proxy } bool use_alpn_ = false; bool try_http3_ = false; + + // If true, we'll explicitly set the inner "transport_socket" field to raw buffer if it is not + // configured. + bool set_inner_transport_socket_ = true; }; INSTANTIATE_TEST_SUITE_P(IpVersions, Http11ConnectHttpIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); +// Test that request/response is successful via plaintext if the inner transport socket is not set. +TEST_P(Http11ConnectHttpIntegrationTest, NoInnerTransportSocketSet) { + set_inner_transport_socket_ = false; + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = + sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 0); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + // Test that with no connect-proxy header, the transport socket is a no-op. TEST_P(Http11ConnectHttpIntegrationTest, NoHeader) { initialize(); From 1f05d19adf0154e93a3d04eba3042c8fa8639406 Mon Sep 17 00:00:00 2001 From: Greg Greenway Date: Wed, 9 Oct 2024 10:38:57 -0700 Subject: [PATCH 40/77] tls: improve validation that context is successfully created (#36512) `createSslClientContext` used to throw on error. Refactor to explictly check success on the StatusOr. Signed-off-by: Greg Greenway --- test/common/tls/context_impl_test.cc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/common/tls/context_impl_test.cc b/test/common/tls/context_impl_test.cc index 206514f173..f1c4b926e1 100644 --- a/test/common/tls/context_impl_test.cc +++ b/test/common/tls/context_impl_test.cc @@ -1298,8 +1298,9 @@ TEST_F(ClientContextConfigImplTest, RSA2048Cert) { *tls_context.mutable_common_tls_context()->add_tls_certificates()); auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that 1024-bit RSA certificates are rejected. @@ -1370,8 +1371,9 @@ TEST_F(ClientContextConfigImplTest, RSA3072Cert) { auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); ContextManagerImpl manager(server_factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that 4096-bit RSA certificates load successfully. @@ -1387,8 +1389,9 @@ TEST_F(ClientContextConfigImplTest, RSA4096Cert) { *tls_context.mutable_common_tls_context()->add_tls_certificates()); auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that P256 ECDSA certs load. @@ -1404,8 +1407,9 @@ TEST_F(ClientContextConfigImplTest, P256EcdsaCert) { *tls_context.mutable_common_tls_context()->add_tls_certificates()); auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that non-P256 ECDSA certs are rejected. From 364e45c65c570caf400b81a13cd10c2943b72bdf Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 9 Oct 2024 14:55:38 -0400 Subject: [PATCH 41/77] Upstream: removing exceptions from hostimp (#35499) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a https://github.com/envoyproxy/envoy-mobile/issues/176 Signed-off-by: Alyssa Wilk From 5024ec7b304a8a305fe0f95473aeac6fd20ddaea Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 9 Oct 2024 16:10:17 -0400 Subject: [PATCH 42/77] upstream: reducing exceptions (#36497) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a https://github.com/envoyproxy/envoy-mobile/issues/176 Signed-off-by: Alyssa Wilk --- .../upstream/health_discovery_service.cc | 10 ++- source/common/upstream/upstream_impl.cc | 75 +++++++++++++------ source/common/upstream/upstream_impl.h | 21 ++++-- 3 files changed, 72 insertions(+), 34 deletions(-) diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index c12503b216..36fa65196d 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -550,10 +550,12 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) factory_context, socket_factory, *scope), std::unique_ptr); - return std::make_unique( - params.server_context_.initManager(), params.server_context_, params.cluster_, - params.bind_config_, params.server_context_.runtime(), std::move(socket_matcher), - std::move(scope), params.added_via_api_, factory_context); + return THROW_OR_RETURN_VALUE( + ClusterInfoImpl::create(params.server_context_.initManager(), params.server_context_, + params.cluster_, params.bind_config_, + params.server_context_.runtime(), std::move(socket_matcher), + std::move(scope), params.added_via_api_, factory_context), + std::unique_ptr); } void HdsCluster::initHealthchecks() { diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 49f7bdaa93..1dafc23d65 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1132,13 +1132,30 @@ LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProto( using ProtocolOptionsHashMap = absl::flat_hash_map; +absl::StatusOr> +ClusterInfoImpl::create(Init::Manager& info, + Server::Configuration::ServerFactoryContext& server_context, + const envoy::config::cluster::v3::Cluster& config, + const absl::optional& bind_config, + Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, + Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, + Server::Configuration::TransportSocketFactoryContext& ctx) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr(new ClusterInfoImpl( + info, server_context, config, bind_config, runtime, std::move(socket_matcher), + std::move(stats_scope), added_via_api, ctx, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; +} + ClusterInfoImpl::ClusterInfoImpl( Init::Manager& init_manager, Server::Configuration::ServerFactoryContext& server_context, const envoy::config::cluster::v3::Cluster& config, const absl::optional& bind_config, Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, - Server::Configuration::TransportSocketFactoryContext& factory_context) + Server::Configuration::TransportSocketFactoryContext& factory_context, + absl::Status& creation_status) : runtime_(runtime), name_(config.name()), observability_name_(!config.alt_stat_name().empty() ? std::make_unique(config.alt_stat_name()) @@ -1253,9 +1270,10 @@ ClusterInfoImpl::ClusterInfoImpl( config.track_cluster_stats().per_endpoint_stats()) { #ifdef WIN32 if (set_local_interface_name_on_upstream_connections_) { - throwEnvoyExceptionOrPanic( + creation_status = absl::InvalidArgumentError( "set_local_interface_name_on_upstream_connections_ cannot be set to true " "on Windows platforms"); + return; } #endif @@ -1264,21 +1282,25 @@ ClusterInfoImpl::ClusterInfoImpl( // TODO(ggreenway): Verify that bypassing virtual dispatch here was intentional if (ClusterInfoImpl::perEndpointStatsEnabled() && server_context.bootstrap().cluster_manager().has_load_stats_config()) { - throwEnvoyExceptionOrPanic("Only one of cluster per_endpoint_stats and cluster manager " - "load_stats_config can be specified"); + creation_status = + absl::InvalidArgumentError("Only one of cluster per_endpoint_stats and cluster manager " + "load_stats_config can be specified"); + return; } if (config.has_max_requests_per_connection() && http_protocol_options_->common_http_protocol_options_.has_max_requests_per_connection()) { - throwEnvoyExceptionOrPanic("Only one of max_requests_per_connection from Cluster or " - "HttpProtocolOptions can be specified"); + creation_status = + absl::InvalidArgumentError("Only one of max_requests_per_connection from Cluster or " + "HttpProtocolOptions can be specified"); + return; } if (config.has_load_balancing_policy() || config.lb_policy() == envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG) { // If load_balancing_policy is set we will use it directly, ignoring lb_policy. - THROW_IF_NOT_OK(configureLbPolicies(config, server_context)); + SET_AND_RETURN_IF_NOT_OK(configureLbPolicies(config, server_context), creation_status); } else { // If load_balancing_policy is not set, we will try to convert legacy lb_policy // to load_balancing_policy and use it. @@ -1286,10 +1308,7 @@ ClusterInfoImpl::ClusterInfoImpl( auto lb_pair = LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProto( lb_factory_context, config, server_context.messageValidationVisitor()); - - if (!lb_pair.ok()) { - throwEnvoyExceptionOrPanic(std::string(lb_pair.status().message())); - } + SET_AND_RETURN_IF_NOT_OK(lb_pair.status(), creation_status); load_balancer_factory_ = lb_pair->factory; ASSERT(load_balancer_factory_ != nullptr, "null load balancer factory"); load_balancer_config_ = std::move(lb_pair->config); @@ -1297,9 +1316,11 @@ ClusterInfoImpl::ClusterInfoImpl( if (config.lb_subset_config().locality_weight_aware() && !config.common_lb_config().has_locality_weighted_lb_config()) { - throwEnvoyExceptionOrPanic(fmt::format("Locality weight aware subset LB requires that a " - "locality_weighted_lb_config be set in {}", - name_)); + creation_status = + absl::InvalidArgumentError(fmt::format("Locality weight aware subset LB requires that a " + "locality_weighted_lb_config be set in {}", + name_)); + return; } // Use default (1h) or configured `idle_timeout`, unless it's set to 0, indicating that no @@ -1345,7 +1366,8 @@ ClusterInfoImpl::ClusterInfoImpl( if (config.has_eds_cluster_config()) { if (config.type() != envoy::config::cluster::v3::Cluster::EDS) { - throwEnvoyExceptionOrPanic("eds_cluster_config set in a non-EDS cluster"); + creation_status = absl::InvalidArgumentError("eds_cluster_config set in a non-EDS cluster"); + return; } } @@ -1365,7 +1387,9 @@ ClusterInfoImpl::ClusterInfoImpl( if (proto_config.has_config_discovery()) { if (proto_config.has_typed_config()) { - throwEnvoyExceptionOrPanic("Only one of typed_config or config_discovery can be used"); + creation_status = + absl::InvalidArgumentError("Only one of typed_config or config_discovery can be used"); + return; } ENVOY_LOG(debug, " dynamic filter name: {}", proto_config.name()); @@ -1405,9 +1429,10 @@ ClusterInfoImpl::ClusterInfoImpl( const auto last_type_url = Config::Utility::getFactoryType(http_filters[http_filters.size() - 1].typed_config()); if (last_type_url != upstream_codec_type_url) { - throwEnvoyExceptionOrPanic(fmt::format( + creation_status = absl::InvalidArgumentError(fmt::format( "The codec filter is the only valid terminal upstream HTTP filter, use '{}'", upstream_codec_type_url)); + return; } } @@ -1416,8 +1441,9 @@ ClusterInfoImpl::ClusterInfoImpl( Server::Configuration::UpstreamHttpFilterConfigFactory> helper(*http_filter_config_provider_manager_, upstream_context_.serverFactoryContext(), factory_context.clusterManager(), upstream_context_, prefix); - THROW_IF_NOT_OK(helper.processFilters(http_filters, "upstream http", "upstream http", - http_filter_factories_)); + SET_AND_RETURN_IF_NOT_OK(helper.processFilters(http_filters, "upstream http", "upstream http", + http_filter_factories_), + creation_status); } } @@ -1595,12 +1621,13 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus auto socket_matcher = std::move(*socket_matcher_or_error); const bool matcher_supports_alpn = socket_matcher->allMatchesSupportAlpn(); auto& dispatcher = server_context.mainThreadDispatcher(); + auto info_or_error = ClusterInfoImpl::create( + init_manager_, server_context, cluster, cluster_context.clusterManager().bindConfig(), + runtime_, std::move(socket_matcher), std::move(stats_scope), cluster_context.addedViaApi(), + *transport_factory_context_); + SET_AND_RETURN_IF_NOT_OK(info_or_error.status(), creation_status); info_ = std::shared_ptr( - new ClusterInfoImpl(init_manager_, server_context, cluster, - cluster_context.clusterManager().bindConfig(), runtime_, - std::move(socket_matcher), std::move(stats_scope), - cluster_context.addedViaApi(), *transport_factory_context_), - [&dispatcher](const ClusterInfoImpl* self) { + (*info_or_error).release(), [&dispatcher](const ClusterInfoImpl* self) { ENVOY_LOG(trace, "Schedule destroy cluster info {}", self->name()); dispatcher.deleteInDispatcherThread( std::unique_ptr(self)); diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index b3c93e451b..6b537c75d9 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -812,12 +812,13 @@ class ClusterInfoImpl : public ClusterInfo, using HttpProtocolOptionsConfigImpl = Envoy::Extensions::Upstreams::Http::ProtocolOptionsConfigImpl; using TcpProtocolOptionsConfigImpl = Envoy::Extensions::Upstreams::Tcp::ProtocolOptionsConfigImpl; - ClusterInfoImpl(Init::Manager& info, Server::Configuration::ServerFactoryContext& server_context, - const envoy::config::cluster::v3::Cluster& config, - const absl::optional& bind_config, - Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, - Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, - Server::Configuration::TransportSocketFactoryContext&); + static absl::StatusOr> + create(Init::Manager& info, Server::Configuration::ServerFactoryContext& server_context, + const envoy::config::cluster::v3::Cluster& config, + const absl::optional& bind_config, + Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, + Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, + Server::Configuration::TransportSocketFactoryContext&); static DeferredCreationCompatibleClusterTrafficStats generateStats(Stats::ScopeSharedPtr scope, const ClusterTrafficStatNames& cluster_stat_names, @@ -1038,6 +1039,14 @@ class ClusterInfoImpl : public ClusterInfo, } protected: + ClusterInfoImpl(Init::Manager& info, Server::Configuration::ServerFactoryContext& server_context, + const envoy::config::cluster::v3::Cluster& config, + const absl::optional& bind_config, + Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, + Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, + Server::Configuration::TransportSocketFactoryContext& context, + absl::Status& creation_status); + // Gets the retry budget percent/concurrency from the circuit breaker thresholds. If the retry // budget message is specified, defaults will be filled in if either params are unspecified. static std::pair, absl::optional> From c72c6e62057f356711bc32a6ab47b7487fe3c5e8 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:12:27 -0400 Subject: [PATCH 43/77] ext_proc: remove unnecessary watermark (#36468) Commit Message: `StopIterationAndWatermark` will raise the watermark when buffered data exceeds the limits, the `requestWatermark` here is redundant and it will also introduce unnecessary stall of Envoy processing and overhead of raise and clear watermark possibly for small bodies Risk Level: LOW Testing: 1.Passed all functional tests (unit test and integration test) 2.This PR performs slightly better in load test. ``` 4KB request body: Without this PR: Memory 183.29MB; Latency P99 : 2716 With this PR: Memory: 160.17MB; Latency P99: 2624 64KB request body: Without this PR: Memory 178.49MB; Latency P99 : 3512 With this PR: Memory: 172.70MB; Latency P99: 3505 ``` Signed-off-by: tyxia --- source/extensions/filters/http/ext_proc/ext_proc.cc | 10 ++++------ .../extensions/filters/http/ext_proc/filter_test.cc | 13 ------------- .../filters/http/ext_proc/ordering_test.cc | 6 ------ 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index e087cdb159..2f3e39e5eb 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -579,13 +579,11 @@ FilterDataStatus Filter::onData(ProcessorState& state, Buffer::Instance& data, b } else { ENVOY_LOG(trace, "Header processing still in progress -- holding body data"); // We don't know what to do with the body until the response comes back. - // We must buffer it in case we need it when that happens. - // Raise a watermark to prevent a buffer overflow until the response comes back. - // When end_stream is true, we need to StopIterationAndWatermark as well to stop the - // ActiveStream from returning error when the last chunk added to stream buffer exceeds the - // buffer limit. + // We must buffer it in case we need it when that happens. Watermark will be raised when the + // buffered data reaches the buffer's watermark limit. When end_stream is true, we need to + // StopIterationAndWatermark as well to stop the ActiveStream from returning error when the + // last chunk added to stream buffer exceeds the buffer limit. state.setPaused(true); - state.requestWatermark(); return FilterDataStatus::StopIterationAndWatermark; } } diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index 148d47e2de..5193c769b5 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -1068,8 +1068,6 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBufferedComesFast) { Buffer::OwnedImpl buffered_data; setUpDecodingBuffering(buffered_data); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); @@ -1082,7 +1080,6 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBufferedComesFast) { EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_4, true)); buffered_data.add(req_data_4); - EXPECT_CALL(decoder_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); processRequestHeaders(true, absl::nullopt); processRequestBody([](const HttpBody& req_body, ProcessingResponse&, BodyResponse&) { @@ -1136,15 +1133,11 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBufferedComesALittleFast) { Buffer::OwnedImpl buffered_data; setUpDecodingBuffering(buffered_data); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_2, false)); buffered_data.add(req_data_2); - // Now the headers response comes in before we get all the data - EXPECT_CALL(decoder_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); processRequestHeaders(true, absl::nullopt); EXPECT_EQ(FilterDataStatus::StopIterationAndBuffer, filter_->decodeData(req_data_3, false)); @@ -1454,15 +1447,11 @@ TEST_F(HttpFilterTest, PostFastRequestPartialBuffering) { Buffer::OwnedImpl buffered_data; setUpDecodingBuffering(buffered_data); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_2, true)); buffered_data.add(req_data_2); - // Now the headers response comes in and we are all done - EXPECT_CALL(decoder_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); processRequestHeaders(true, absl::nullopt); processRequestBody([](const HttpBody& req_body, ProcessingResponse&, BodyResponse&) { @@ -1518,8 +1507,6 @@ TEST_F(HttpFilterTest, PostFastAndBigRequestPartialBuffering) { expected_request_data.add(req_data_1); expected_request_data.add(req_data_2); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_2, false)); diff --git a/test/extensions/filters/http/ext_proc/ordering_test.cc b/test/extensions/filters/http/ext_proc/ordering_test.cc index a12be075b0..b83cb3e3cb 100644 --- a/test/extensions/filters/http/ext_proc/ordering_test.cc +++ b/test/extensions/filters/http/ext_proc/ordering_test.cc @@ -492,13 +492,7 @@ TEST_F(OrderingTest, ResponseSomeDataComesFast) { EXPECT_CALL(stream_delegate_, send(_, false)); sendResponseHeaders(true); - // Some of the data might come back but we should watermark so that we - // don't fill the buffer. - EXPECT_CALL(encoder_callbacks_, onEncoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->encodeData(resp_body_1, false)); - - // When the response does comes back, we should lift the watermark - EXPECT_CALL(encoder_callbacks_, onEncoderFilterBelowWriteBufferLowWatermark()); sendResponseHeadersReply(); EXPECT_CALL(stream_delegate_, send(_, false)); From 9ad582a7a93d7f665aa234140f43c589a14ab0ef Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Wed, 9 Oct 2024 16:14:03 -0400 Subject: [PATCH 44/77] xds-failover: disable moving to primary after fallback responds (#36386) xds-failover: disable moving to primary after fallback responds In #35591 Envoy's implementation was modified to allow Envoy move to the primary xDS source, even after the fallback successfully responded. This PR adds a runtime guard that reverts this behavior. This will be a temporary runtime guard that is introduced to allow testing of the new feature and will be later removed. Note that there's no change to the current Envoy behavior, unless the runtime guard `envoy.reloadable_features.xds_failover_to_primary_enabled` is explicitly set to `false`. Risk Level: low - behind a runtime guard. Testing: Added unit and integration tests. Docs Changes: N/A Release Notes: Added an entry. Platform Specific Features: N/A Runtime guard: `envoy.reloadable_features.xds_failover_to_primary_enabled` was introduced that is set to `true` by default, and keeps the current behavior. --------- Signed-off-by: Adi Suissa-Peleg --- changelogs/current.yaml | 6 + source/common/runtime/runtime_features.cc | 1 + .../grpc/grpc_mux_failover.h | 47 +++++- .../extensions/config_subscription/grpc/BUILD | 1 + .../grpc/grpc_mux_failover_test.cc | 29 ++++ .../grpc/xds_failover_integration_test.cc | 145 ++++++++++++++++++ 6 files changed, 225 insertions(+), 4 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index b831842ed7..2a24f8c539 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -115,6 +115,12 @@ minor_behavior_changes: - area: tracers change: | Set status code based on GRPC status code for OpenTelemetry tracers (previously unset). +- area: xds-failover + change: | + Add the ability to stick with either the primary or the failover xDS sources once Envoy connects to one of them. + This was added behind a runtime guard, to ensure that the move to the primary source can be properly validated, and + will be removed in the future. To allow sticksiyness the runtime flag + ``envoy.reloadable_features.xds_failover_to_primary_enabled`` must be explicitly set to ``false``. - area: http2 change: | Changes the default value of ``envoy.reloadable_features.http2_use_oghttp2`` to ``false``. This changes the codec used for HTTP/2 diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 8306957141..a06e49b182 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -106,6 +106,7 @@ RUNTIME_GUARD(envoy_reloadable_features_use_typed_metadata_in_proxy_protocol_lis RUNTIME_GUARD(envoy_reloadable_features_validate_connect); RUNTIME_GUARD(envoy_reloadable_features_validate_grpc_header_before_log_grpc_status); RUNTIME_GUARD(envoy_reloadable_features_validate_upstream_headers); +RUNTIME_GUARD(envoy_reloadable_features_xds_failover_to_primary_enabled); RUNTIME_GUARD(envoy_reloadable_features_xdstp_path_avoid_colon_encoding); RUNTIME_GUARD(envoy_restart_features_allow_client_socket_creation_failure); RUNTIME_GUARD(envoy_restart_features_allow_slot_destroy_on_worker_threads); diff --git a/source/extensions/config_subscription/grpc/grpc_mux_failover.h b/source/extensions/config_subscription/grpc/grpc_mux_failover.h index 1b7daa8b02..bea808a521 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_failover.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_failover.h @@ -102,6 +102,25 @@ class GrpcMuxFailover : public GrpcStreamInterface, "Already connected to an xDS server, skipping establishNewStream() call"); return; } + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.xds_failover_to_primary_enabled")) { + // Allow stickiness, so if Envoy was ever connected to the primary source only + // retry to reconnect to the primary source, If Envoy was ever connected to the + // failover source then only retry to reconnect to the failover source. + if (previously_connected_to_ == ConnectedTo::Primary) { + ENVOY_LOG_MISC( + info, "Previously connected to the primary xDS source, attempting to reconnect to it"); + connection_state_ = ConnectionState::ConnectingToPrimary; + primary_grpc_stream_->establishNewStream(); + return; + } else if (previously_connected_to_ == ConnectedTo::Failover) { + ENVOY_LOG_MISC( + info, "Previously connected to the failover xDS source, attempting to reconnect to it"); + connection_state_ = ConnectionState::ConnectingToFailover; + failover_grpc_stream_->establishNewStream(); + return; + } + } // connection_state_ is either None, ConnectingToPrimary or // ConnectingToFailover. In the first 2 cases try to connect to the primary // (preferring the primary in the case of None), and in the third case @@ -289,10 +308,30 @@ class GrpcMuxFailover : public GrpcStreamInterface, // This will be called when the failover stream fails to establish a connection, or after the // connection was closed. ASSERT(parent_.connectingToOrConnectedToFailover()); + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.xds_failover_to_primary_enabled")) { + // If previously Envoy was connected to the failover, keep using that. + // Otherwise let the retry mechanism try to access the primary (similar + // to if the runtime flag was not set). + if (parent_.previously_connected_to_ == ConnectedTo::Failover) { + ENVOY_LOG(debug, + "Failover xDS stream disconnected (either after establishing a connection or " + "before). Attempting to reconnect to Failover because Envoy successfully " + "connected to it previously."); + // Not closing the failover stream, allows it to use its retry timer + // to reconnect to the failover source. + // Next attempt will be to the failover after Envoy was already + // connected to it. Allowing to send the initial_resource_versions on reconnect. + parent_.grpc_mux_callbacks_.onEstablishmentFailure(true); + parent_.connection_state_ = ConnectionState::ConnectingToFailover; + return; + } + } // Either this was an intentional disconnection from the failover source, // or unintentional. Either way, try to connect to the primary next. - ENVOY_LOG(debug, "Failover xDS stream diconnected (either after establishing a connection or " - "before). Attempting to connect to the primary stream."); + ENVOY_LOG(debug, + "Failover xDS stream disconnected (either after establishing a connection or " + "before). Attempting to connect to the primary stream."); // This will close the stream and prevent the retry timer from // reconnecting to the failover source. The next attempt will be to the @@ -408,8 +447,8 @@ class GrpcMuxFailover : public GrpcStreamInterface, // exclusive), it can attempt connecting to more than one source at a time. ConnectionState connection_state_; - // A flag that keeps track of whether Envoy successfully connected to either the - // primary or failover source. Envoy is considered successfully connected to a source + // A flag that keeps track of whether Envoy successfully connected to the + // primary source. Envoy is considered successfully connected to a source // once it receives a response from it. bool ever_connected_to_primary_{false}; diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD index 466a14269b..0b41fb7b90 100644 --- a/test/extensions/config_subscription/grpc/BUILD +++ b/test/extensions/config_subscription/grpc/BUILD @@ -273,6 +273,7 @@ envoy_cc_test( "//test/common/stats:stat_test_utility_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc b/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc index 2b611e1c7b..dae34fd62e 100644 --- a/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc @@ -6,6 +6,7 @@ #include "test/extensions/config_subscription/grpc/mocks.h" #include "test/mocks/config/mocks.h" #include "test/mocks/event/mocks.h" +#include "test/test_common/test_runtime.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -446,6 +447,34 @@ TEST_F(GrpcMuxFailoverTest, AlternatingPrimaryAndFailoverAttemptsAfterFailoverAv grpc_mux_failover_->establishNewStream(); } +// Validation that when envoy.reloadable_features.xds_failover_to_primary_enabled is disabled +// and after the failover is available (a response is received), Envoy will only +// try to reconnect to the failover. +// This test will be removed once envoy.reloadable_features.xds_failover_to_primary_enabled +// is deprecated. +TEST_F(GrpcMuxFailoverTest, StickToFailoverAfterFailoverAvailable) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.xds_failover_to_primary_enabled", "false"}}); + connectToFailover(); + + // Emulate 5 disconnects, and ensure the primary reconnection isn't attempted. + for (int attempt = 0; attempt < 5; ++attempt) { + // Emulate a failover source failure that will not result in an attempt to + // connect to the primary. It should not close the failover stream (so + // the retry mechanism will kick in). + EXPECT_CALL(failover_stream_, closeStream()).Times(0); + EXPECT_CALL(grpc_mux_callbacks_, onEstablishmentFailure(true)); + EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); + failover_callbacks_->onEstablishmentFailure(true); + } + + // Emulate a call to establishNewStream() of the failover stream. + EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); + EXPECT_CALL(failover_stream_, establishNewStream()); + grpc_mux_failover_->establishNewStream(); +} + // Validates that multiple calls to establishNewStream when connecting to the // failover are invoked on the failover stream, and not the primary. TEST_F(GrpcMuxFailoverTest, MultipleEstablishFailoverStream) { diff --git a/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc b/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc index de4948c913..a1efe8febd 100644 --- a/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc +++ b/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc @@ -906,4 +906,149 @@ TEST_P(XdsFailoverAdsIntegrationTest, ASSERT_TRUE(failover_xds_connection_->waitForDisconnect()); } } + +// Validation that when envoy.reloadable_features.xds_failover_to_primary_enabled is disabled +// and after the failover responds and then disconnected, Envoy will only +// try to reconnect to the failover. +// This test will be removed once envoy.reloadable_features.xds_failover_to_primary_enabled +// is deprecated. +TEST_P(XdsFailoverAdsIntegrationTest, NoPrimaryUseAfterFailoverResponse) { + // These tests are not executed with GoogleGrpc because they are flaky due to + // the large timeout values for retries. + SKIP_IF_GRPC_CLIENT(Grpc::ClientType::GoogleGrpc); +#ifdef ENVOY_ENABLE_UHV + // With UHV the finishGrpcStream() isn't detected as invalid frame because of + // no ":status" header, unless "envoy.reloadable_features.enable_universal_header_validator" + // is also enabled. + config_helper_.addRuntimeOverride("envoy.reloadable_features.enable_universal_header_validator", + "true"); +#endif + config_helper_.addRuntimeOverride("envoy.reloadable_features.xds_failover_to_primary_enabled", + "false"); + // Set a long LDS initial_fetch_timeout to prevent test flakiness when + // reconnecting to the failover multiple times. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* lds_config = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); + lds_config->mutable_initial_fetch_timeout()->set_seconds(100); + }); + initialize(); + + // 2 consecutive primary failures. + // Expect a connection to the primary. Reject the connection immediately. + primaryConnectionFailure(); + ASSERT_TRUE(xds_connection_->waitForDisconnect()); + // The CDS request fails when the primary disconnects. After that fetch the config + // dump to ensure that the retry timer kicks in. + // Expect another connection attempt to the primary. Reject the stream (gRPC failure) immediately. + // As this is a 2nd consecutive failure, it will trigger failover. + waitForPrimaryXdsRetryTimer(); + primaryConnectionFailure(); + ASSERT_TRUE(xds_connection_->waitForDisconnect()); + + // The CDS request fails when the primary disconnects. + test_server_->waitForCounterGe("cluster_manager.cds.update_failure", 2); + + AssertionResult result = + failover_xds_upstream_->waitForHttpConnection(*dispatcher_, failover_xds_connection_); + RELEASE_ASSERT(result, result.message()); + // Failover is healthy, start the ADS gRPC stream. + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + RELEASE_ASSERT(result, result.message()); + failover_xds_stream_->startGrpcStream(); + + // Ensure basic flow with failover works. + EXPECT_TRUE(compareDiscoveryRequest(CdsTypeUrl, "", {}, {}, {}, true, + Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get())); + sendDiscoveryResponse( + CdsTypeUrl, {ConfigHelper::buildCluster("failover_cluster_0")}, + {ConfigHelper::buildCluster("failover_cluster_0")}, {}, "failover1", {}, + failover_xds_stream_.get()); + // Wait for an EDS request, and send its response. + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + test_server_->waitForGaugeEq("cluster.failover_cluster_0.warming_state", 1); + // Ensure basic flow with failover works. + EXPECT_TRUE(compareDiscoveryRequest( + EdsTypeUrl, "", {"failover_cluster_0"}, {"failover_cluster_0"}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", failover_xds_stream_.get())); + sendDiscoveryResponse( + EdsTypeUrl, {buildClusterLoadAssignment("failover_cluster_0")}, + {buildClusterLoadAssignment("failover_cluster_0")}, {}, "failover1", {}, + failover_xds_stream_.get()); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + test_server_->waitForGaugeGe("cluster_manager.active_clusters", 2); + test_server_->waitForGaugeEq("cluster.failover_cluster_0.warming_state", 0); + EXPECT_EQ(2, test_server_->gauge("control_plane.connected_state")->value()); + EXPECT_TRUE(compareDiscoveryRequest(CdsTypeUrl, "failover1", {}, {}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get())); + EXPECT_TRUE(compareDiscoveryRequest(LdsTypeUrl, "", {}, {}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get())); + + // Envoy has received CDS and EDS responses, it means the failover is available. + // Now disconnect the failover source, this should result in an LDS failure. + // After that add a notification to the main thread to ensure that the retry timer kicks in. + failover_xds_stream_->finishGrpcStream(Grpc::Status::Internal); + test_server_->waitForCounterGe("listener_manager.lds.update_failure", 1); + absl::Notification notification; + test_server_->server().dispatcher().post([&]() { notification.Notify(); }); + notification.WaitForNotification(); + timeSystem().advanceTimeWait(std::chrono::milliseconds(1000)); + + // In this case (received a response), both EnvoyGrpc and GoogleGrpc keep the connection open. + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + RELEASE_ASSERT(result, result.message()); + // Immediately fail the connection. + failover_xds_stream_->finishGrpcStream(Grpc::Status::Internal); + + // Ensure that Envoy still attempts to connect to the failover, + // and keep disconnecting a few times and validate that the primary + // connection isn't attempted. + for (int i = 1; i < 5; ++i) { + ASSERT_TRUE(failover_xds_connection_->waitForDisconnect()); + // Wait longer due to the fixed 5 seconds failover . + waitForPrimaryXdsRetryTimer(i, 6); + // EnvoyGrpc will disconnect if the gRPC stream is immediately closed (as + // done above). + result = failover_xds_upstream_->waitForHttpConnection(*dispatcher_, failover_xds_connection_); + RELEASE_ASSERT(result, result.message()); + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + RELEASE_ASSERT(result, result.message()); + // Immediately fail the connection. + failover_xds_stream_->finishGrpcStream(Grpc::Status::Internal); + } + + // When EnvoyGrpc is used, no new connection to the primary will be attempted. + EXPECT_FALSE( + xds_upstream_->waitForHttpConnection(*dispatcher_, xds_connection_, std::chrono::seconds(1))); + + ASSERT_TRUE(failover_xds_connection_->waitForDisconnect()); + // Wait longer due to the fixed 5 seconds failover . + waitForPrimaryXdsRetryTimer(5, 6); + + // Allow a connection to the failover. + // Expect a connection to the failover when using EnvoyGrpc. + // In case GoogleGrpc is used the current connection will be reused (new stream). + result = failover_xds_upstream_->waitForHttpConnection(*dispatcher_, failover_xds_connection_); + RELEASE_ASSERT(result, result.message()); + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + failover_xds_stream_->startGrpcStream(); + + // Validate that the initial requests with known versions are sent to the + // failover source. + const absl::flat_hash_map cds_eds_initial_resource_versions_map{ + {"failover_cluster_0", "failover1"}}; + const absl::flat_hash_map empty_initial_resource_versions_map; + EXPECT_TRUE(compareDiscoveryRequest( + CdsTypeUrl, "1", {}, {}, {}, true, Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get(), OptRef(cds_eds_initial_resource_versions_map))); + EXPECT_TRUE(compareDiscoveryRequest( + EdsTypeUrl, "1", {"failover_cluster_0"}, {"failover_cluster_0"}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", failover_xds_stream_.get(), + OptRef(cds_eds_initial_resource_versions_map))); + EXPECT_TRUE(compareDiscoveryRequest( + LdsTypeUrl, "", {}, {}, {}, false, Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get(), OptRef(empty_initial_resource_versions_map))); +} } // namespace Envoy From 55b0fc45cfdc2c0df002690606853540cf794fab Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 9 Oct 2024 16:23:02 -0400 Subject: [PATCH 45/77] ci: change googleurl dep (#36515) backing file temporarily deleted and re-updated (with a new hash) Signed-off-by: Alyssa Wilk --- bazel/repository_locations.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b6e02d489b..e42a6e7e2f 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1222,9 +1222,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Chrome URL parsing library", project_desc = "Chrome URL parsing library", project_url = "https://quiche.googlesource.com/googleurl", - # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/dd4080fec0b443296c0ed0036e1e776df8813aa7.tar.gz version = "dd4080fec0b443296c0ed0036e1e776df8813aa7", - sha256 = "59f14d4fb373083b9dc8d389f16bbb817b5f936d1d436aa67e16eb6936028a51", + sha256 = "fc694942e8a7491dcc1dde1bddf48a31370a1f46fef862bc17acf07c34dc6325", + # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/dd4080fec0b443296c0ed0036e1e776df8813aa7.tar.gz urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], extensions = [], From 17d594238f55d918c7828153f05a9c750503cd1d Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Wed, 9 Oct 2024 17:24:53 -0500 Subject: [PATCH 46/77] getaddrinfo: Fix TSAN issue when trace is enabled (#36503) Commit Message: Fixes https://github.com/envoyproxy/envoy/issues/36499 Additional Description: Doing `std::transform` on a shared `std::vector` reference without proper locking isn't thread safe and can cause a UB, especially for the `getaddrinfo` resolver where adding traces is done in a separate thread. This PR fixes by introducing a new `getTraces` API that builds the string traces in an atomic way. Risk Level: low (only enabled with `enable_dfp_dns_trace` runtime flag) Testing: existing tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: getaddrinfo, mobile --------- Signed-off-by: Fredy Wijaya --- envoy/network/BUILD | 1 - envoy/network/dns.h | 4 +- .../dynamic_forward_proxy/dns_cache_impl.cc | 14 +-- .../dns_resolver/apple/apple_dns_impl.h | 2 +- .../network/dns_resolver/cares/dns_impl.h | 2 +- .../dns_resolver/getaddrinfo/getaddrinfo.h | 11 ++- .../dynamic_forward_proxy/test_resolver.h | 2 + .../getaddrinfo/getaddrinfo_test.cc | 95 +++++++++++-------- test/mocks/network/mocks.h | 2 +- 9 files changed, 71 insertions(+), 62 deletions(-) diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 13bbd44570..160559ab8a 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -86,7 +86,6 @@ envoy_cc_library( name = "dns_interface", hdrs = ["dns.h"], deps = [ - "//envoy/common:optref_lib", "//envoy/common:time_interface", "//envoy/network:address_interface", ], diff --git a/envoy/network/dns.h b/envoy/network/dns.h index d2781c853e..3b5fcf893c 100644 --- a/envoy/network/dns.h +++ b/envoy/network/dns.h @@ -6,11 +6,11 @@ #include #include -#include "envoy/common/optref.h" #include "envoy/common/pure.h" #include "envoy/common/time.h" #include "envoy/network/address.h" +#include "absl/types/optional.h" #include "absl/types/variant.h" namespace Envoy { @@ -56,7 +56,7 @@ class ActiveDnsQuery { virtual void addTrace(uint8_t trace) PURE; /** Return the DNS query traces. */ - virtual OptRef> getTraces() PURE; + virtual std::string getTraces() PURE; }; /** diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index 44b2b98e8f..fffd784255 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -421,18 +421,8 @@ void DnsCacheImpl::finishResolve(const std::string& host, std::string details_with_maybe_trace = std::string(details); if (primary_host_info != nullptr && primary_host_info->active_query_ != nullptr) { if (enable_dfp_dns_trace_) { - OptRef> traces = - primary_host_info->active_query_->getTraces(); - if (traces.has_value()) { - std::vector string_traces; - string_traces.reserve(traces.ref().size()); - std::transform(traces.ref().begin(), traces.ref().end(), std::back_inserter(string_traces), - [](const auto& trace) { - return absl::StrCat(trace.trace_, "=", - trace.time_.time_since_epoch().count()); - }); - details_with_maybe_trace = absl::StrCat(details, ":", absl::StrJoin(string_traces, ",")); - } + std::string traces = primary_host_info->active_query_->getTraces(); + details_with_maybe_trace = absl::StrCat(details, ":", traces); } // `cancel` must be called last because the `ActiveQuery` will be destroyed afterward. if (is_timeout) { diff --git a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h index 448c0a5946..7b32cb4680 100644 --- a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h +++ b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h @@ -101,7 +101,7 @@ class AppleDnsResolverImpl : public DnsResolver, protected Logger::Loggable> getTraces() override { return {}; } + std::string getTraces() override { return {}; } static DnsResponse buildDnsResponse(const struct sockaddr* address, uint32_t ttl); diff --git a/source/extensions/network/dns_resolver/cares/dns_impl.h b/source/extensions/network/dns_resolver/cares/dns_impl.h index e61efc3c52..a95ff3e2b0 100644 --- a/source/extensions/network/dns_resolver/cares/dns_impl.h +++ b/source/extensions/network/dns_resolver/cares/dns_impl.h @@ -77,7 +77,7 @@ class DnsResolverImpl : public DnsResolver, protected Logger::Loggable> getTraces() override { return {}; } + std::string getTraces() override { return {}; } // Does the object own itself? Resource reclamation occurs via self-deleting // on query completion or error. diff --git a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h index 27f29a6c6a..2357fdb4ac 100644 --- a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h +++ b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h @@ -62,9 +62,16 @@ class GetAddrInfoDnsResolver : public DnsResolver, public Logger::Loggable> getTraces() override { + std::string getTraces() override { absl::MutexLock lock(&mutex_); - return traces_; + std::vector string_traces; + string_traces.reserve(traces_.size()); + std::transform(traces_.begin(), traces_.end(), std::back_inserter(string_traces), + [](const Trace& trace) { + return absl::StrCat(trace.trace_, "=", + trace.time_.time_since_epoch().count()); + }); + return absl::StrJoin(string_traces, ","); } bool isCancelled() { diff --git a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h index 41b47d10eb..738c5bc6d9 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h +++ b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h @@ -46,6 +46,8 @@ class TestResolver : public GetAddrInfoDnsResolver { if (dns_override.has_value()) { *const_cast(&query->dns_name_) = dns_override.value(); } + // Add a dummy trace for test coverage. + query->addTrace(100); pending_queries_.push_back(PendingQueryInfo{std::move(query), absl::nullopt}); }); return raw_new_query; diff --git a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc index 840e11c0c1..2dcbc115d9 100644 --- a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc +++ b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc @@ -110,7 +110,9 @@ class GetAddrInfoDnsImplTest : public testing::Test { }; MATCHER_P(HasTrace, expected_trace, "") { - return arg.trace_ == static_cast(expected_trace); + std::vector v = absl::StrSplit(arg, '='); + uint8_t trace = std::stoi(v.at(0)); + return trace == static_cast(expected_trace); } TEST_F(GetAddrInfoDnsImplTest, LocalhostResolve) { @@ -124,11 +126,12 @@ TEST_F(GetAddrInfoDnsImplTest, LocalhostResolve) { [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { verifyRealGaiResponse(status, std::move(response)); - EXPECT_THAT(active_dns_query_->getTraces().ref(), - ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), - HasTrace(GetAddrInfoTrace::Starting), - HasTrace(GetAddrInfoTrace::Success), - HasTrace(GetAddrInfoTrace::Callback))); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); dispatcher_->exit(); }); @@ -152,11 +155,12 @@ TEST_F(GetAddrInfoDnsImplTest, Cancel) { [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { verifyRealGaiResponse(status, std::move(response)); - EXPECT_THAT(active_dns_query_->getTraces().ref(), - ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), - HasTrace(GetAddrInfoTrace::Starting), - HasTrace(GetAddrInfoTrace::Success), - HasTrace(GetAddrInfoTrace::Callback))); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); dispatcher_->exit(); }); @@ -177,11 +181,12 @@ TEST_F(GetAddrInfoDnsImplTest, Failure) { EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); EXPECT_EQ("Non-recoverable failure in name resolution", details); EXPECT_TRUE(response.empty()); - EXPECT_THAT(active_dns_query_->getTraces().ref(), - ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), - HasTrace(GetAddrInfoTrace::Starting), - HasTrace(GetAddrInfoTrace::Failed), - HasTrace(GetAddrInfoTrace::Callback))); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Failed), + HasTrace(GetAddrInfoTrace::Callback))); dispatcher_->exit(); }); @@ -201,11 +206,12 @@ TEST_F(GetAddrInfoDnsImplTest, NoData) { std::list&& response) { EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); - EXPECT_THAT(active_dns_query_->getTraces().ref(), - ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), - HasTrace(GetAddrInfoTrace::Starting), - HasTrace(GetAddrInfoTrace::NoResult), - HasTrace(GetAddrInfoTrace::Callback))); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::NoResult), + HasTrace(GetAddrInfoTrace::Callback))); dispatcher_->exit(); }); @@ -225,11 +231,12 @@ TEST_F(GetAddrInfoDnsImplTest, NoName) { std::list&& response) { EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); - EXPECT_THAT(active_dns_query_->getTraces().ref(), - ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), - HasTrace(GetAddrInfoTrace::Starting), - HasTrace(GetAddrInfoTrace::NoResult), - HasTrace(GetAddrInfoTrace::Callback))); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::NoResult), + HasTrace(GetAddrInfoTrace::Callback))); dispatcher_->exit(); }); @@ -246,20 +253,22 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainIndefinitelyAndSuccess) { .Times(2) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{0, 0})); - active_dns_query_ = resolver_->resolve( - "localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - EXPECT_THAT( - active_dns_query_->getTraces().ref(), - ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), - HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), - HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Success), - HasTrace(GetAddrInfoTrace::Callback))); - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } @@ -304,8 +313,9 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndSuccess) { std::list&& response) { EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); + std::vector traces = absl::StrSplit(active_dns_query_->getTraces(), ','); EXPECT_THAT( - active_dns_query_->getTraces().ref(), + traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), @@ -340,8 +350,9 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndFailure) { EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); EXPECT_FALSE(details.empty()); EXPECT_TRUE(response.empty()); + std::vector traces = absl::StrSplit(active_dns_query_->getTraces(), ','); EXPECT_THAT( - active_dns_query_->getTraces().ref(), + traces, ElementsAre( HasTrace(GetAddrInfoTrace::NotStarted), HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index c3f4bfd5c5..c7260c34a8 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -44,7 +44,7 @@ class MockActiveDnsQuery : public ActiveDnsQuery { // Network::ActiveDnsQuery MOCK_METHOD(void, cancel, (CancelReason reason)); MOCK_METHOD(void, addTrace, (uint8_t)); - MOCK_METHOD(OptRef>, getTraces, ()); + MOCK_METHOD(std::string, getTraces, ()); }; class MockFilterManager : public FilterManager { From 601292a6857f991936e5e98e540506fd4377a53a Mon Sep 17 00:00:00 2001 From: code Date: Thu, 10 Oct 2024 21:06:22 +0800 Subject: [PATCH 47/77] proto util: change the input string ref to string view (#36525) Commit Message: proto util: change the input string ref to string view Additional Description: `absl::string_view` is better than`const string&` in most cases to avoid unnecessary copy. Risk Level: low. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. --------- Signed-off-by: wangbaiping --- source/common/protobuf/utility.h | 6 +++--- source/common/protobuf/yaml_utility.cc | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 2d427e7d9e..b14e3720b6 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -255,7 +255,7 @@ class MessageUtil { static std::size_t hash(const Protobuf::Message& message); #ifdef ENVOY_ENABLE_YAML - static void loadFromJson(const std::string& json, Protobuf::Message& message, + static void loadFromJson(absl::string_view json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor); /** * Return ok only when strict conversion(don't ignore unknown field) succeeds. @@ -264,9 +264,9 @@ class MessageUtil { * Return error status for relaxed conversion and set has_unknown_field to false if relaxed * conversion(ignore unknown field) fails. */ - static absl::Status loadFromJsonNoThrow(const std::string& json, Protobuf::Message& message, + static absl::Status loadFromJsonNoThrow(absl::string_view json, Protobuf::Message& message, bool& has_unknown_fileld); - static void loadFromJson(const std::string& json, ProtobufWkt::Struct& message); + static void loadFromJson(absl::string_view json, ProtobufWkt::Struct& message); static void loadFromYaml(const std::string& yaml, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor); #endif diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index 4409abc33d..741817bc04 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -115,7 +115,7 @@ void jsonConvertInternal(const Protobuf::Message& source, } // namespace -void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& message, +void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor) { bool has_unknown_field; auto load_status = loadFromJsonNoThrow(json, message, has_unknown_field); @@ -124,15 +124,16 @@ void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& messa } if (has_unknown_field) { // If the parsing failure is caused by the unknown fields. - THROW_IF_NOT_OK(validation_visitor.onUnknownField("type " + message.GetTypeName() + " reason " + - load_status.ToString())); + THROW_IF_NOT_OK(validation_visitor.onUnknownField( + fmt::format("type {} reason {}", message.GetTypeName(), load_status.ToString()))); } else { // If the error has nothing to do with unknown field. - throw EnvoyException("Unable to parse JSON as proto (" + load_status.ToString() + "): " + json); + throw EnvoyException( + fmt::format("Unable to parse JSON as proto ({}): {}", load_status.ToString(), json)); } } -absl::Status MessageUtil::loadFromJsonNoThrow(const std::string& json, Protobuf::Message& message, +absl::Status MessageUtil::loadFromJsonNoThrow(absl::string_view json, Protobuf::Message& message, bool& has_unknown_fileld) { has_unknown_fileld = false; Protobuf::util::JsonParseOptions options; @@ -163,7 +164,7 @@ absl::Status MessageUtil::loadFromJsonNoThrow(const std::string& json, Protobuf: return relaxed_status; } -void MessageUtil::loadFromJson(const std::string& json, ProtobufWkt::Struct& message) { +void MessageUtil::loadFromJson(absl::string_view json, ProtobufWkt::Struct& message) { // No need to validate if converting to a Struct, since there are no unknown // fields possible. loadFromJson(json, message, ProtobufMessage::getNullValidationVisitor()); From 2eebf0dc57c2b5393b2deaeb0166cb8a9a6bffa3 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Thu, 10 Oct 2024 11:55:06 -0400 Subject: [PATCH 48/77] ext_proc: skip timeout timer on trailer in async mode. (#36524) Commit Message: Like header and body, timer/timeout should be skipped on trailer as well in async. It is because there is no response/ or response is ignored. Risk Level: Low Testing: Integration test Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A --------- Signed-off-by: tyxia --- .../filters/http/ext_proc/ext_proc.cc | 13 +++++-- .../ext_proc/ext_proc_integration_test.cc | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index 2f3e39e5eb..0d35d05425 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -976,9 +976,16 @@ void Filter::sendTrailers(ProcessorState& state, const Http::HeaderMap& trailers auto* trailers_req = state.mutableTrailers(req); MutationUtils::headersToProto(trailers, config_->allowedHeaders(), config_->disallowedHeaders(), *trailers_req->mutable_trailers()); - state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), config_->messageTimeout(), - ProcessorState::CallbackState::TrailersCallback); - ENVOY_LOG(debug, "Sending trailers message"); + + if (observability_mode) { + ENVOY_LOG(debug, "Sending trailers message in observability mode"); + } else { + state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), + config_->messageTimeout(), + ProcessorState::CallbackState::TrailersCallback); + ENVOY_LOG(debug, "Sending trailers message"); + } + sendRequest(std::move(req), false); stats_.stream_msgs_sent_.inc(); } diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index e229fc88b8..7762d38d5b 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -4334,6 +4334,43 @@ TEST_P(ExtProcIntegrationTest, ObservabilityModeWithFullResponse) { timeSystem().advanceTimeWaitImpl(std::chrono::milliseconds(deferred_close_timeout_ms)); } +TEST_P(ExtProcIntegrationTest, ObservabilityModeWithFullRequestAndTimeout) { + proto_config_.set_observability_mode(true); + uint32_t deferred_close_timeout_ms = 2000; + proto_config_.mutable_deferred_close_timeout()->set_seconds(deferred_close_timeout_ms / 1000); + + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED); + proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SEND); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + + initializeConfig(); + HttpIntegrationTest::initialize(); + auto response = sendDownstreamRequestWithBodyAndTrailer("Hello"); + + processRequestHeadersMessage(*grpc_upstreams_[0], true, + [this](const HttpHeaders&, HeadersResponse&) { + // Advance 400 ms. Default timeout is 200ms + timeSystem().advanceTimeWaitImpl(400ms); + return false; + }); + processRequestBodyMessage(*grpc_upstreams_[0], false, [this](const HttpBody&, BodyResponse&) { + // Advance 400 ms. Default timeout is 200ms + timeSystem().advanceTimeWaitImpl(400ms); + return false; + }); + processRequestTrailersMessage(*grpc_upstreams_[0], false, + [this](const HttpTrailers&, TrailersResponse&) { + // Advance 400 ms. Default timeout is 200ms + timeSystem().advanceTimeWaitImpl(400ms); + return false; + }); + + handleUpstreamRequest(); + verifyDownstreamResponse(*response, 200); + + timeSystem().advanceTimeWaitImpl(std::chrono::milliseconds(deferred_close_timeout_ms - 1200)); +} + TEST_P(ExtProcIntegrationTest, ObservabilityModeWithLogging) { proto_config_.set_observability_mode(true); From 06ea5be1a0919908948dfb04a9262e1ba7bf922c Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 10 Oct 2024 18:42:30 +0100 Subject: [PATCH 49/77] deps: Fix (renamed) ipp-crypto repo hash (#36540) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e42a6e7e2f..29af2efded 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -443,17 +443,17 @@ REPOSITORY_LOCATIONS_SPEC = dict( com_github_intel_ipp_crypto_crypto_mb = dict( project_name = "libipp-crypto", project_desc = "Intel® Integrated Performance Primitives Cryptography", - project_url = "https://github.com/intel/ipp-crypto", + project_url = "https://github.com/intel/cryptography-primitives", version = "2021.11.1", - sha256 = "d785fd8d5245ada79068588e5cc4d721d35c50e7d26fc268306f4aaae28ec6d6", - strip_prefix = "ipp-crypto-ippcp_{version}", - urls = ["https://github.com/intel/ipp-crypto/archive/ippcp_{version}.tar.gz"], + sha256 = "76779724378fb3ec8b59047be02f6ca609c1ee70372b0205d937d451987bdb6c", + strip_prefix = "cryptography-primitives-ippcp_{version}", + urls = ["https://github.com/intel/cryptography-primitives/archive/ippcp_{version}.tar.gz"], release_date = "2024-02-28", use_category = ["dataplane_ext"], extensions = ["envoy.tls.key_providers.cryptomb"], cpe = "cpe:2.3:a:intel:cryptography_for_intel_integrated_performance_primitives:*", license = "Apache-2.0", - license_url = "https://github.com/intel/ipp-crypto/blob/ippcp_{version}/LICENSE", + license_url = "https://github.com/intel/cryptography-primitives/blob/ippcp_{version}/LICENSE", ), com_github_intel_qatlib = dict( project_name = "qatlib", From 965647e4e24c244f8b7f4316035e442a037e2cbe Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Thu, 10 Oct 2024 15:13:07 -0400 Subject: [PATCH 50/77] mobile: Update the Apple proxy settings polling to 10s (#36538) This is the same value that Cronet uses: https://source.chromium.org/chromium/chromium/src/+/main:net/proxy_resolution/proxy_config_service_ios.cc;l=23;drc=0ddf755aab1e6d0f9c21fa09cd098c4ff6d1cac0. Signed-off-by: Ali Beyad --- .../common/network/apple_system_proxy_settings_monitor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/library/common/network/apple_system_proxy_settings_monitor.cc b/mobile/library/common/network/apple_system_proxy_settings_monitor.cc index 5a97b6453d..3514f20c88 100644 --- a/mobile/library/common/network/apple_system_proxy_settings_monitor.cc +++ b/mobile/library/common/network/apple_system_proxy_settings_monitor.cc @@ -10,7 +10,7 @@ namespace Envoy { namespace Network { // The interval at which system proxy settings should be polled at. -CFTimeInterval kProxySettingsRefreshRateSeconds = 7; +CFTimeInterval kProxySettingsRefreshRateSeconds = 10; AppleSystemProxySettingsMonitor::AppleSystemProxySettingsMonitor( SystemProxySettingsReadCallback proxy_settings_read_callback) From 74f71bbf1756a362fb3226569a059d34ee0a15d4 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 10 Oct 2024 15:14:13 -0400 Subject: [PATCH 51/77] coverage: ratcheting (#36518) Signed-off-by: Alyssa Wilk --- test/per_file_coverage.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index a31b15d9fe..74583e415d 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -5,41 +5,39 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common:96.2" "source/common/common/posix:96.2" # flaky due to posix: be careful adjusting -"source/common/config:96.1" +"source/common/config:96.3" "source/common/crypto:95.5" -"source/common/event:95.2" # Emulated edge events guards don't report LCOV +"source/common/event:95.6" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.3" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. "source/common/http/http2:96.0" -"source/common/json:94.4" +"source/common/json:95.2" "source/common/matcher:94.4" "source/common/memory:74.5" # tcmalloc code path is not enabled in coverage build, only gperf tcmalloc, see PR#32589 "source/common/network:94.4" # Flaky, `activateFileEvents`, `startSecureTransport` and `ioctl`, listener_socket do not always report LCOV "source/common/network/dns_resolver:91.4" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts -"source/common/quic:93.5" +"source/common/quic:93.6" "source/common/secret:95.4" "source/common/signal:87.2" # Death tests don't report LCOV "source/common/thread:0.0" # Death tests don't report LCOV -"source/common/tls:94.8" +"source/common/tls:95.3" "source/common/tls/cert_validator:94.4" "source/common/tls/private_key:88.9" "source/common/watchdog:58.6" # Death tests don't report LCOV "source/exe:94.2" # increased by #32346, need coverage for terminate_handler and hot restart failures -"source/extensions/common:93.0" #flaky: be careful adjusting "source/extensions/common/proxy_protocol:93.8" # Adjusted for security patch "source/extensions/common/tap:94.6" -"source/extensions/common/wasm:88.0" # flaky: be careful adjusting +"source/extensions/common/wasm:95.0" # flaky: be careful adjusting "source/extensions/common/wasm/ext:92.0" "source/extensions/filters/common/fault:94.5" "source/extensions/filters/common/rbac:90.8" -"source/extensions/filters/http/cache:95.3" +"source/extensions/filters/http/cache:95.4" "source/extensions/filters/http/grpc_json_transcoder:94.2" # TODO(#28232) "source/extensions/filters/http/ip_tagging:88.2" "source/extensions/filters/http/kill_request:91.7" # Death tests don't report LCOV "source/extensions/filters/listener/original_src:92.1" "source/extensions/filters/network/mongo_proxy:96.1" "source/extensions/filters/network/sni_cluster:88.9" -"source/extensions/filters/network/wasm:77.8" -"source/extensions/http/cache/simple_http_cache:95.9" +"source/extensions/http/cache/simple_http_cache:96.5" "source/extensions/rate_limit_descriptors:95.0" "source/extensions/rate_limit_descriptors/expr:95.0" "source/extensions/stat_sinks/graphite_statsd:82.8" # Death tests don't report LCOV From bfa0e07aab8500f49eb54766be7ae4edd1d15de3 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 10 Oct 2024 21:50:27 +0100 Subject: [PATCH 52/77] ci: Remove redundant AZP gc (#36541) Signed-off-by: Ryan Northey --- .github/workflows/garbage.yml | 40 ----------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/garbage.yml diff --git a/.github/workflows/garbage.yml b/.github/workflows/garbage.yml deleted file mode 100644 index d6e0af919e..0000000000 --- a/.github/workflows/garbage.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Garbage collection - -permissions: - contents: read - -on: - schedule: - - cron: '0 0 * * *' - -concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} - cancel-in-progress: true - - -jobs: - azp-agents: - runs-on: ubuntu-22.04 - if: github.repository == 'envoyproxy/envoy' - strategy: - matrix: - include: - - target: envoy-arm-large - pool-id: 21 - - target: envoy-arm-small - pool-id: 24 - - target: envoy-x64-large - pool-id: 20 - - target: envoy-x64-small - pool-id: 23 - - target: salvo-control - pool-id: 22 - - target: x64-nano - pool-id: 17 - steps: - - name: Remove dead AZP agents (${{ matrix.target }}) - uses: envoyproxy/toolshed/gh-actions/azp/agent-cleanup@actions-v0.2.36 - with: - azp-org: cncf - azp-token: ${{ secrets.AZP_TOKEN }} - pool-id: ${{ matrix.pool-id }} From bb28dda78d462851c1810047a4b1488a5a69279a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Oct 2024 13:27:17 +0100 Subject: [PATCH 53/77] build(deps): bump aiohttp from 3.10.9 to 3.10.10 in /tools/base (#36544) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 184 ++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 99c5b90a40..05ac390bec 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -92,98 +92,98 @@ aiohappyeyeballs==2.4.0 \ --hash=sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2 \ --hash=sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd # via aiohttp -aiohttp==3.10.9 \ - --hash=sha256:02d1d6610588bcd743fae827bd6f2e47e0d09b346f230824b4c6fb85c6065f9c \ - --hash=sha256:03690541e4cc866eef79626cfa1ef4dd729c5c1408600c8cb9e12e1137eed6ab \ - --hash=sha256:0bc059ecbce835630e635879f5f480a742e130d9821fbe3d2f76610a6698ee25 \ - --hash=sha256:0c21c82df33b264216abffff9f8370f303dab65d8eee3767efbbd2734363f677 \ - --hash=sha256:1298b854fd31d0567cbb916091be9d3278168064fca88e70b8468875ef9ff7e7 \ - --hash=sha256:1321658f12b6caffafdc35cfba6c882cb014af86bef4e78c125e7e794dfb927b \ - --hash=sha256:143b0026a9dab07a05ad2dd9e46aa859bffdd6348ddc5967b42161168c24f857 \ - --hash=sha256:16e6a51d8bc96b77f04a6764b4ad03eeef43baa32014fce71e882bd71302c7e4 \ - --hash=sha256:172ad884bb61ad31ed7beed8be776eb17e7fb423f1c1be836d5cb357a096bf12 \ - --hash=sha256:17c272cfe7b07a5bb0c6ad3f234e0c336fb53f3bf17840f66bd77b5815ab3d16 \ - --hash=sha256:1a0ee6c0d590c917f1b9629371fce5f3d3f22c317aa96fbdcce3260754d7ea21 \ - --hash=sha256:2746d8994ebca1bdc55a1e998feff4e94222da709623bb18f6e5cfec8ec01baf \ - --hash=sha256:2914caa46054f3b5ff910468d686742ff8cff54b8a67319d75f5d5945fd0a13d \ - --hash=sha256:2bbf94d4a0447705b7775417ca8bb8086cc5482023a6e17cdc8f96d0b1b5aba6 \ - --hash=sha256:2bd9f3eac515c16c4360a6a00c38119333901b8590fe93c3257a9b536026594d \ - --hash=sha256:2c33fa6e10bb7ed262e3ff03cc69d52869514f16558db0626a7c5c61dde3c29f \ - --hash=sha256:2d37f4718002863b82c6f391c8efd4d3a817da37030a29e2682a94d2716209de \ - --hash=sha256:3668d0c2a4d23fb136a753eba42caa2c0abbd3d9c5c87ee150a716a16c6deec1 \ - --hash=sha256:36d4fba838be5f083f5490ddd281813b44d69685db910907636bc5dca6322316 \ - --hash=sha256:40ff5b7660f903dc587ed36ef08a88d46840182d9d4b5694e7607877ced698a1 \ - --hash=sha256:42775de0ca04f90c10c5c46291535ec08e9bcc4756f1b48f02a0657febe89b10 \ - --hash=sha256:482c85cf3d429844396d939b22bc2a03849cb9ad33344689ad1c85697bcba33a \ - --hash=sha256:4e6cb75f8ddd9c2132d00bc03c9716add57f4beff1263463724f6398b813e7eb \ - --hash=sha256:4edc3fd701e2b9a0d605a7b23d3de4ad23137d23fc0dbab726aa71d92f11aaaf \ - --hash=sha256:4fd16b30567c5b8e167923be6e027eeae0f20cf2b8a26b98a25115f28ad48ee0 \ - --hash=sha256:5002a02c17fcfd796d20bac719981d2fca9c006aac0797eb8f430a58e9d12431 \ - --hash=sha256:51d0a4901b27272ae54e42067bc4b9a90e619a690b4dc43ea5950eb3070afc32 \ - --hash=sha256:558b3d223fd631ad134d89adea876e7fdb4c93c849ef195049c063ada82b7d08 \ - --hash=sha256:5c070430fda1a550a1c3a4c2d7281d3b8cfc0c6715f616e40e3332201a253067 \ - --hash=sha256:5f392ef50e22c31fa49b5a46af7f983fa3f118f3eccb8522063bee8bfa6755f8 \ - --hash=sha256:60555211a006d26e1a389222e3fab8cd379f28e0fbf7472ee55b16c6c529e3a6 \ - --hash=sha256:608cecd8d58d285bfd52dbca5b6251ca8d6ea567022c8a0eaae03c2589cd9af9 \ - --hash=sha256:60ad5b8a7452c0f5645c73d4dad7490afd6119d453d302cd5b72b678a85d6044 \ - --hash=sha256:63649309da83277f06a15bbdc2a54fbe75efb92caa2c25bb57ca37762789c746 \ - --hash=sha256:6ebdc3b3714afe1b134b3bbeb5f745eed3ecbcff92ab25d80e4ef299e83a5465 \ - --hash=sha256:6f3c6648aa123bcd73d6f26607d59967b607b0da8ffcc27d418a4b59f4c98c7c \ - --hash=sha256:7003f33f5f7da1eb02f0446b0f8d2ccf57d253ca6c2e7a5732d25889da82b517 \ - --hash=sha256:776e9f3c9b377fcf097c4a04b241b15691e6662d850168642ff976780609303c \ - --hash=sha256:85711eec2d875cd88c7eb40e734c4ca6d9ae477d6f26bd2b5bb4f7f60e41b156 \ - --hash=sha256:87d1e4185c5d7187684d41ebb50c9aeaaaa06ca1875f4c57593071b0409d2444 \ - --hash=sha256:8a3f063b41cc06e8d0b3fcbbfc9c05b7420f41287e0cd4f75ce0a1f3d80729e6 \ - --hash=sha256:8b3fb28a9ac8f2558760d8e637dbf27aef1e8b7f1d221e8669a1074d1a266bb2 \ - --hash=sha256:8bd9125dd0cc8ebd84bff2be64b10fdba7dc6fd7be431b5eaf67723557de3a31 \ - --hash=sha256:8be1a65487bdfc285bd5e9baf3208c2132ca92a9b4020e9f27df1b16fab998a9 \ - --hash=sha256:8cc0d13b4e3b1362d424ce3f4e8c79e1f7247a00d792823ffd640878abf28e56 \ - --hash=sha256:8d9d10d10ec27c0d46ddaecc3c5598c4db9ce4e6398ca872cdde0525765caa2f \ - --hash=sha256:8debb45545ad95b58cc16c3c1cc19ad82cffcb106db12b437885dbee265f0ab5 \ - --hash=sha256:91aa966858593f64c8a65cdefa3d6dc8fe3c2768b159da84c1ddbbb2c01ab4ef \ - --hash=sha256:9331dd34145ff105177855017920dde140b447049cd62bb589de320fd6ddd582 \ - --hash=sha256:99f9678bf0e2b1b695e8028fedac24ab6770937932eda695815d5a6618c37e04 \ - --hash=sha256:9fdf5c839bf95fc67be5794c780419edb0dbef776edcfc6c2e5e2ffd5ee755fa \ - --hash=sha256:a14e4b672c257a6b94fe934ee62666bacbc8e45b7876f9dd9502d0f0fe69db16 \ - --hash=sha256:a19caae0d670771ea7854ca30df76f676eb47e0fd9b2ee4392d44708f272122d \ - --hash=sha256:a35ed3d03910785f7d9d6f5381f0c24002b2b888b298e6f941b2fc94c5055fcd \ - --hash=sha256:a61df62966ce6507aafab24e124e0c3a1cfbe23c59732987fc0fd0d71daa0b88 \ - --hash=sha256:a6e00c8a92e7663ed2be6fcc08a2997ff06ce73c8080cd0df10cc0321a3168d7 \ - --hash=sha256:ac3196952c673822ebed8871cf8802e17254fff2a2ed4835d9c045d9b88c5ec7 \ - --hash=sha256:ac74e794e3aee92ae8f571bfeaa103a141e409863a100ab63a253b1c53b707eb \ - --hash=sha256:ad3675c126f2a95bde637d162f8231cff6bc0bc9fbe31bd78075f9ff7921e322 \ - --hash=sha256:aeebd3061f6f1747c011e1d0b0b5f04f9f54ad1a2ca183e687e7277bef2e0da2 \ - --hash=sha256:ba1a599255ad6a41022e261e31bc2f6f9355a419575b391f9655c4d9e5df5ff5 \ - --hash=sha256:bbdb8def5268f3f9cd753a265756f49228a20ed14a480d151df727808b4531dd \ - --hash=sha256:c2555e4949c8d8782f18ef20e9d39730d2656e218a6f1a21a4c4c0b56546a02e \ - --hash=sha256:c2695c61cf53a5d4345a43d689f37fc0f6d3a2dc520660aec27ec0f06288d1f9 \ - --hash=sha256:c2b627d3c8982691b06d89d31093cee158c30629fdfebe705a91814d49b554f8 \ - --hash=sha256:c46131c6112b534b178d4e002abe450a0a29840b61413ac25243f1291613806a \ - --hash=sha256:c54dc329cd44f7f7883a9f4baaefe686e8b9662e2c6c184ea15cceee587d8d69 \ - --hash=sha256:c7d7cafc11d70fdd8801abfc2ff276744ae4cb39d8060b6b542c7e44e5f2cfc2 \ - --hash=sha256:cb0b2d5d51f96b6cc19e6ab46a7b684be23240426ae951dcdac9639ab111b45e \ - --hash=sha256:d15a29424e96fad56dc2f3abed10a89c50c099f97d2416520c7a543e8fddf066 \ - --hash=sha256:d1f5c9169e26db6a61276008582d945405b8316aae2bb198220466e68114a0f5 \ - --hash=sha256:d271f770b52e32236d945911b2082f9318e90ff835d45224fa9e28374303f729 \ - --hash=sha256:d646fdd74c25bbdd4a055414f0fe32896c400f38ffbdfc78c68e62812a9e0257 \ - --hash=sha256:d6e395c3d1f773cf0651cd3559e25182eb0c03a2777b53b4575d8adc1149c6e9 \ - --hash=sha256:d7c071235a47d407b0e93aa6262b49422dbe48d7d8566e1158fecc91043dd948 \ - --hash=sha256:d97273a52d7f89a75b11ec386f786d3da7723d7efae3034b4dda79f6f093edc1 \ - --hash=sha256:dcf354661f54e6a49193d0b5653a1b011ba856e0b7a76bda2c33e4c6892f34ea \ - --hash=sha256:e3e7fabedb3fe06933f47f1538df7b3a8d78e13d7167195f51ca47ee12690373 \ - --hash=sha256:e525b69ee8a92c146ae5b4da9ecd15e518df4d40003b01b454ad694a27f498b5 \ - --hash=sha256:e709d6ac598c5416f879bb1bae3fd751366120ac3fa235a01de763537385d036 \ - --hash=sha256:e83dfefb4f7d285c2d6a07a22268344a97d61579b3e0dce482a5be0251d672ab \ - --hash=sha256:e86260b76786c28acf0b5fe31c8dca4c2add95098c709b11e8c35b424ebd4f5b \ - --hash=sha256:e883b61b75ca6efc2541fcd52a5c8ccfe288b24d97e20ac08fdf343b8ac672ea \ - --hash=sha256:f0a44bb40b6aaa4fb9a5c1ee07880570ecda2065433a96ccff409c9c20c1624a \ - --hash=sha256:f82ace0ec57c94aaf5b0e118d4366cff5889097412c75aa14b4fd5fc0c44ee3e \ - --hash=sha256:f9ca09414003c0e96a735daa1f071f7d7ed06962ef4fa29ceb6c80d06696d900 \ - --hash=sha256:fa430b871220dc62572cef9c69b41e0d70fcb9d486a4a207a5de4c1f25d82593 \ - --hash=sha256:fc262c3df78c8ff6020c782d9ce02e4bcffe4900ad71c0ecdad59943cba54442 \ - --hash=sha256:fcd546782d03181b0b1d20b43d612429a90a68779659ba8045114b867971ab71 \ - --hash=sha256:fd4ceeae2fb8cabdd1b71c82bfdd39662473d3433ec95b962200e9e752fb70d0 \ - --hash=sha256:fec5fac7aea6c060f317f07494961236434928e6f4374e170ef50b3001e14581 +aiohttp==3.10.10 \ + --hash=sha256:007ec22fbc573e5eb2fb7dec4198ef8f6bf2fe4ce20020798b2eb5d0abda6138 \ + --hash=sha256:00819de9e45d42584bed046314c40ea7e9aea95411b38971082cad449392b08c \ + --hash=sha256:01948b1d570f83ee7bbf5a60ea2375a89dfb09fd419170e7f5af029510033d24 \ + --hash=sha256:038f514fe39e235e9fef6717fbf944057bfa24f9b3db9ee551a7ecf584b5b480 \ + --hash=sha256:03a42ac7895406220124c88911ebee31ba8b2d24c98507f4a8bf826b2937c7f2 \ + --hash=sha256:05646ebe6b94cc93407b3bf34b9eb26c20722384d068eb7339de802154d61bc5 \ + --hash=sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a \ + --hash=sha256:0b00807e2605f16e1e198f33a53ce3c4523114059b0c09c337209ae55e3823a8 \ + --hash=sha256:0e1b370d8007c4ae31ee6db7f9a2fe801a42b146cec80a86766e7ad5c4a259cf \ + --hash=sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871 \ + --hash=sha256:1b66ccafef7336a1e1f0e389901f60c1d920102315a56df85e49552308fc0486 \ + --hash=sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9 \ + --hash=sha256:1eb89d3d29adaf533588f209768a9c02e44e4baf832b08118749c5fad191781d \ + --hash=sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb \ + --hash=sha256:2609e9ab08474702cc67b7702dbb8a80e392c54613ebe80db7e8dbdb79837c68 \ + --hash=sha256:274cfa632350225ce3fdeb318c23b4a10ec25c0e2c880eff951a3842cf358ac1 \ + --hash=sha256:28529e08fde6f12eba8677f5a8608500ed33c086f974de68cc65ab218713a59d \ + --hash=sha256:2b606353da03edcc71130b52388d25f9a30a126e04caef1fd637e31683033abd \ + --hash=sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1 \ + --hash=sha256:333cf6cf8e65f6a1e06e9eb3e643a0c515bb850d470902274239fea02033e9a8 \ + --hash=sha256:3455522392fb15ff549d92fbf4b73b559d5e43dc522588f7eb3e54c3f38beee7 \ + --hash=sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959 \ + --hash=sha256:3bcd391d083f636c06a68715e69467963d1f9600f85ef556ea82e9ef25f043f7 \ + --hash=sha256:3dffb610a30d643983aeb185ce134f97f290f8935f0abccdd32c77bed9388b42 \ + --hash=sha256:3fe407bf93533a6fa82dece0e74dbcaaf5d684e5a51862887f9eaebe6372cd79 \ + --hash=sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38 \ + --hash=sha256:438cd072f75bb6612f2aca29f8bd7cdf6e35e8f160bc312e49fbecab77c99e3a \ + --hash=sha256:4470c73c12cd9109db8277287d11f9dd98f77fc54155fc71a7738a83ffcc8ea8 \ + --hash=sha256:45c3b868724137f713a38376fef8120c166d1eadd50da1855c112fe97954aed8 \ + --hash=sha256:486f7aabfa292719a2753c016cc3a8f8172965cabb3ea2e7f7436c7f5a22a151 \ + --hash=sha256:4f05e9727ce409358baa615dbeb9b969db94324a79b5a5cea45d39bdb01d82e6 \ + --hash=sha256:50aed5155f819873d23520919e16703fc8925e509abbb1a1491b0087d1cd969e \ + --hash=sha256:50edbcad60d8f0e3eccc68da67f37268b5144ecc34d59f27a02f9611c1d4eec7 \ + --hash=sha256:54ca74df1be3c7ca1cf7f4c971c79c2daf48d9aa65dea1a662ae18926f5bc8ce \ + --hash=sha256:578a4b875af3e0daaf1ac6fa983d93e0bbfec3ead753b6d6f33d467100cdc67b \ + --hash=sha256:597a079284b7ee65ee102bc3a6ea226a37d2b96d0418cc9047490f231dc09fe8 \ + --hash=sha256:59bb3c54aa420521dc4ce3cc2c3fe2ad82adf7b09403fa1f48ae45c0cbde6628 \ + --hash=sha256:5c6a5b8c7926ba5d8545c7dd22961a107526562da31a7a32fa2456baf040939f \ + --hash=sha256:64f6c17757251e2b8d885d728b6433d9d970573586a78b78ba8929b0f41d045a \ + --hash=sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7 \ + --hash=sha256:741a46d58677d8c733175d7e5aa618d277cd9d880301a380fd296975a9cdd7bc \ + --hash=sha256:7789050d9e5d0c309c706953e5e8876e38662d57d45f936902e176d19f1c58ab \ + --hash=sha256:77abf6665ae54000b98b3c742bc6ea1d1fb31c394bcabf8b5d2c1ac3ebfe7f3b \ + --hash=sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911 \ + --hash=sha256:7b06b7843929e41a94ea09eb1ce3927865387e3e23ebe108e0d0d09b08d25be9 \ + --hash=sha256:7e338c0523d024fad378b376a79faff37fafb3c001872a618cde1d322400a572 \ + --hash=sha256:7ea7ffc6d6d6f8a11e6f40091a1040995cdff02cfc9ba4c2f30a516cb2633554 \ + --hash=sha256:8105fd8a890df77b76dd3054cddf01a879fc13e8af576805d667e0fa0224c35d \ + --hash=sha256:84afcdea18eda514c25bc68b9af2a2b1adea7c08899175a51fe7c4fb6d551257 \ + --hash=sha256:9294bbb581f92770e6ed5c19559e1e99255e4ca604a22c5c6397b2f9dd3ee42c \ + --hash=sha256:93429602396f3383a797a2a70e5f1de5df8e35535d7806c9f91df06f297e109b \ + --hash=sha256:9627cc1a10c8c409b5822a92d57a77f383b554463d1884008e051c32ab1b3742 \ + --hash=sha256:998f3bd3cfc95e9424a6acd7840cbdd39e45bc09ef87533c006f94ac47296090 \ + --hash=sha256:9c72109213eb9d3874f7ac8c0c5fa90e072d678e117d9061c06e30c85b4cf0e6 \ + --hash=sha256:9fc1500fd2a952c5c8e3b29aaf7e3cc6e27e9cfc0a8819b3bce48cc1b849e4cc \ + --hash=sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142 \ + --hash=sha256:a45d85cf20b5e0d0aa5a8dca27cce8eddef3292bc29d72dcad1641f4ed50aa16 \ + --hash=sha256:a7d8d14fe962153fc681f6366bdec33d4356f98a3e3567782aac1b6e0e40109a \ + --hash=sha256:a8fa23fe62c436ccf23ff930149c047f060c7126eae3ccea005f0483f27b2e28 \ + --hash=sha256:aa6658732517ddabe22c9036479eabce6036655ba87a0224c612e1ae6af2087e \ + --hash=sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94 \ + --hash=sha256:ab5a5a0c7a7991d90446a198689c0535be89bbd6b410a1f9a66688f0880ec026 \ + --hash=sha256:acd48d5b80ee80f9432a165c0ac8cbf9253eaddb6113269a5e18699b33958dbb \ + --hash=sha256:ad7593bb24b2ab09e65e8a1d385606f0f47c65b5a2ae6c551db67d6653e78c28 \ + --hash=sha256:baa42524a82f75303f714108fea528ccacf0386af429b69fff141ffef1c534f9 \ + --hash=sha256:bdfcf6443637c148c4e1a20c48c566aa694fa5e288d34b20fcdc58507882fed3 \ + --hash=sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f \ + --hash=sha256:c02a30b904282777d872266b87b20ed8cc0d1501855e27f831320f471d54d983 \ + --hash=sha256:c1277cd707c465cd09572a774559a3cc7c7a28802eb3a2a9472588f062097205 \ + --hash=sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f \ + --hash=sha256:c5ce2ce7c997e1971b7184ee37deb6ea9922ef5163c6ee5aa3c274b05f9e12fa \ + --hash=sha256:c823bc3971c44ab93e611ab1a46b1eafeae474c0c844aff4b7474287b75fe49c \ + --hash=sha256:ce0cdc074d540265bfeb31336e678b4e37316849d13b308607efa527e981f5c2 \ + --hash=sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb \ + --hash=sha256:d183cf9c797a5291e8301790ed6d053480ed94070637bfaad914dd38b0981f67 \ + --hash=sha256:d9010c31cd6fa59438da4e58a7f19e4753f7f264300cd152e7f90d4602449762 \ + --hash=sha256:d9e5e4a85bdb56d224f412d9c98ae4cbd032cc4f3161818f692cd81766eee65a \ + --hash=sha256:da1dee8948d2137bb51fbb8a53cce6b1bcc86003c6b42565f008438b806cccd8 \ + --hash=sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a \ + --hash=sha256:e00e3505cd80440f6c98c6d69269dcc2a119f86ad0a9fd70bccc59504bebd68a \ + --hash=sha256:e48d5021a84d341bcaf95c8460b152cfbad770d28e5fe14a768988c461b821bc \ + --hash=sha256:e7f8b04d83483577fd9200461b057c9f14ced334dcb053090cea1da9c8321a91 \ + --hash=sha256:edfe3341033a6b53a5c522c802deb2079eee5cbfbb0af032a55064bd65c73a23 \ + --hash=sha256:ef9c33cc5cbca35808f6c74be11eb7f5f6b14d2311be84a15b594bd3e58b5527 \ + --hash=sha256:f2d4324a98062be0525d16f768a03e0bbb3b9fe301ceee99611dc9a7953124e6 \ + --hash=sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c \ + --hash=sha256:f614ab0c76397661b90b6851a030004dac502e48260ea10f2441abd2207fbcc7 \ + --hash=sha256:f7db54c7914cc99d901d93a34704833568d86c20925b2762f9fa779f9cd2e70f \ + --hash=sha256:fbc6264158392bad9df19537e872d476f7c57adf718944cc1e4495cbabf38e2a \ + --hash=sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092 \ + --hash=sha256:ffe595f10566f8276b76dc3a11ae4bb7eba1aac8ddd75811736a15b0d5311414 # via # -r requirements.in # aio-api-github From bbaca29674e716f711d21e5a152b28975b394ef8 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 14 Oct 2024 09:42:37 +0100 Subject: [PATCH 54/77] ci/bazel: Add CI reporting tool (#36539) a step towards fixing #36326 Signed-off-by: Ryan Northey --- tools/base/requirements.in | 1 + tools/base/requirements.txt | 222 ++++++++++++++++++++---------------- tools/ci/BUILD | 14 +++ 3 files changed, 136 insertions(+), 101 deletions(-) create mode 100644 tools/ci/BUILD diff --git a/tools/base/requirements.in b/tools/base/requirements.in index a252ecfd2c..d6108945bb 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -13,6 +13,7 @@ coloredlogs cryptography>=43.0.1 dependatool>=0.2.2 envoy.base.utils>=0.5.0 +envoy.ci.report>=0.0.3 envoy.code.check>=0.5.12 envoy.dependency.check>=0.1.10 envoy.distribution.release>=0.0.9 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 05ac390bec..71f2230295 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -15,6 +15,7 @@ abstracts==0.0.12 \ # aio-run-runner # dependatool # envoy-base-utils + # envoy-ci-report # envoy-code-check # envoy-dependency-check # envoy-distribution-release @@ -25,20 +26,21 @@ aio-api-bazel==0.0.2 \ --hash=sha256:56e36463d236e477b7e282f2d870185a0b978b50e2c3803c1ebf8b8ac4b18f5b \ --hash=sha256:d3f563b7698e874437d80538a89dd4d79bc37de2e850c846330ae456e3f21dcc # via -r requirements.in -aio-api-github==0.2.6 \ - --hash=sha256:71ca0e572a48eab09f3e54267b374fb3d53e246b83f6f23fe1f29f5560acdaed \ - --hash=sha256:be12d6bf612ce2abc85c695ce74547220636f96fe80d4e64cd2de8670db69c32 +aio-api-github==0.2.10 \ + --hash=sha256:2563ad2cd219352a7933b78190c17ab460b503b437bec348a738c0f804adf45f \ + --hash=sha256:75e17073e9e03386d719d1af355f599c16d68345992eaa97aebfa6159b07ad02 # via # -r requirements.in # envoy-base-utils + # envoy-ci-report # envoy-dependency-check aio-api-nist==0.0.4 \ --hash=sha256:1f2909d60ed4fdb3a3ffc37ad6012666f34078b71648394be91f5e67bbf8b6ca \ --hash=sha256:c948ee597b9e7cda7982e17bc4aca509b8aa68510899b42e2d382c10fb0d6f89 # via envoy-dependency-check -aio-core==0.10.3 \ - --hash=sha256:9f5e87347e396a00829e83bbee3769fccdc3ead24522c429e7dfe53d67a50e07 \ - --hash=sha256:b51d32426247f3c265015c5c4b776f9feef15e01c28ca4e868c2ba7b67c556cc +aio-core==0.10.5 \ + --hash=sha256:ba512132df47514a15930b89835ba1ef13522b7e31ec83a3b0622b7431f3f42e \ + --hash=sha256:d8486a0115ade8e0362783b8e7cd988368766d0d911c62cb73b716bfe1f50a92 # via # -r requirements.in # aio-api-bazel @@ -47,6 +49,7 @@ aio-core==0.10.3 \ # aio-run-runner # dependatool # envoy-base-utils + # envoy-ci-report # envoy-code-check # envoy-dependency-check # envoy-distribution-release @@ -63,13 +66,14 @@ aio-run-checker==0.5.7 \ # envoy-dependency-check # envoy-distribution-distrotest # envoy-distribution-verify -aio-run-runner==0.3.3 \ - --hash=sha256:0a783260838a660b4df085d163781fdb35991febf4818524e8f2a13d9f999e07 \ - --hash=sha256:ce904917303723b5495b951e3b6aaacbac2373d92b7a0bd8f562f1d920cfac22 +aio-run-runner==0.3.4 \ + --hash=sha256:3f7bdb91c9a7a60d547d18c620ce2444b4e81d6e6a64a2e1ef465e54db8708bf \ + --hash=sha256:ffa5693d6452b9fc07674cbea665825959d3ea5adca4e492998d242dadf3a773 # via # aio-api-bazel # aio-run-checker # envoy-base-utils + # envoy-ci-report # envoy-distribution-release # envoy-distribution-repo # envoy-docs-sphinx-runner @@ -92,98 +96,98 @@ aiohappyeyeballs==2.4.0 \ --hash=sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2 \ --hash=sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd # via aiohttp -aiohttp==3.10.10 \ - --hash=sha256:007ec22fbc573e5eb2fb7dec4198ef8f6bf2fe4ce20020798b2eb5d0abda6138 \ - --hash=sha256:00819de9e45d42584bed046314c40ea7e9aea95411b38971082cad449392b08c \ - --hash=sha256:01948b1d570f83ee7bbf5a60ea2375a89dfb09fd419170e7f5af029510033d24 \ - --hash=sha256:038f514fe39e235e9fef6717fbf944057bfa24f9b3db9ee551a7ecf584b5b480 \ - --hash=sha256:03a42ac7895406220124c88911ebee31ba8b2d24c98507f4a8bf826b2937c7f2 \ - --hash=sha256:05646ebe6b94cc93407b3bf34b9eb26c20722384d068eb7339de802154d61bc5 \ - --hash=sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a \ - --hash=sha256:0b00807e2605f16e1e198f33a53ce3c4523114059b0c09c337209ae55e3823a8 \ - --hash=sha256:0e1b370d8007c4ae31ee6db7f9a2fe801a42b146cec80a86766e7ad5c4a259cf \ - --hash=sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871 \ - --hash=sha256:1b66ccafef7336a1e1f0e389901f60c1d920102315a56df85e49552308fc0486 \ - --hash=sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9 \ - --hash=sha256:1eb89d3d29adaf533588f209768a9c02e44e4baf832b08118749c5fad191781d \ - --hash=sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb \ - --hash=sha256:2609e9ab08474702cc67b7702dbb8a80e392c54613ebe80db7e8dbdb79837c68 \ - --hash=sha256:274cfa632350225ce3fdeb318c23b4a10ec25c0e2c880eff951a3842cf358ac1 \ - --hash=sha256:28529e08fde6f12eba8677f5a8608500ed33c086f974de68cc65ab218713a59d \ - --hash=sha256:2b606353da03edcc71130b52388d25f9a30a126e04caef1fd637e31683033abd \ - --hash=sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1 \ - --hash=sha256:333cf6cf8e65f6a1e06e9eb3e643a0c515bb850d470902274239fea02033e9a8 \ - --hash=sha256:3455522392fb15ff549d92fbf4b73b559d5e43dc522588f7eb3e54c3f38beee7 \ - --hash=sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959 \ - --hash=sha256:3bcd391d083f636c06a68715e69467963d1f9600f85ef556ea82e9ef25f043f7 \ - --hash=sha256:3dffb610a30d643983aeb185ce134f97f290f8935f0abccdd32c77bed9388b42 \ - --hash=sha256:3fe407bf93533a6fa82dece0e74dbcaaf5d684e5a51862887f9eaebe6372cd79 \ - --hash=sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38 \ - --hash=sha256:438cd072f75bb6612f2aca29f8bd7cdf6e35e8f160bc312e49fbecab77c99e3a \ - --hash=sha256:4470c73c12cd9109db8277287d11f9dd98f77fc54155fc71a7738a83ffcc8ea8 \ - --hash=sha256:45c3b868724137f713a38376fef8120c166d1eadd50da1855c112fe97954aed8 \ - --hash=sha256:486f7aabfa292719a2753c016cc3a8f8172965cabb3ea2e7f7436c7f5a22a151 \ - --hash=sha256:4f05e9727ce409358baa615dbeb9b969db94324a79b5a5cea45d39bdb01d82e6 \ - --hash=sha256:50aed5155f819873d23520919e16703fc8925e509abbb1a1491b0087d1cd969e \ - --hash=sha256:50edbcad60d8f0e3eccc68da67f37268b5144ecc34d59f27a02f9611c1d4eec7 \ - --hash=sha256:54ca74df1be3c7ca1cf7f4c971c79c2daf48d9aa65dea1a662ae18926f5bc8ce \ - --hash=sha256:578a4b875af3e0daaf1ac6fa983d93e0bbfec3ead753b6d6f33d467100cdc67b \ - --hash=sha256:597a079284b7ee65ee102bc3a6ea226a37d2b96d0418cc9047490f231dc09fe8 \ - --hash=sha256:59bb3c54aa420521dc4ce3cc2c3fe2ad82adf7b09403fa1f48ae45c0cbde6628 \ - --hash=sha256:5c6a5b8c7926ba5d8545c7dd22961a107526562da31a7a32fa2456baf040939f \ - --hash=sha256:64f6c17757251e2b8d885d728b6433d9d970573586a78b78ba8929b0f41d045a \ - --hash=sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7 \ - --hash=sha256:741a46d58677d8c733175d7e5aa618d277cd9d880301a380fd296975a9cdd7bc \ - --hash=sha256:7789050d9e5d0c309c706953e5e8876e38662d57d45f936902e176d19f1c58ab \ - --hash=sha256:77abf6665ae54000b98b3c742bc6ea1d1fb31c394bcabf8b5d2c1ac3ebfe7f3b \ - --hash=sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911 \ - --hash=sha256:7b06b7843929e41a94ea09eb1ce3927865387e3e23ebe108e0d0d09b08d25be9 \ - --hash=sha256:7e338c0523d024fad378b376a79faff37fafb3c001872a618cde1d322400a572 \ - --hash=sha256:7ea7ffc6d6d6f8a11e6f40091a1040995cdff02cfc9ba4c2f30a516cb2633554 \ - --hash=sha256:8105fd8a890df77b76dd3054cddf01a879fc13e8af576805d667e0fa0224c35d \ - --hash=sha256:84afcdea18eda514c25bc68b9af2a2b1adea7c08899175a51fe7c4fb6d551257 \ - --hash=sha256:9294bbb581f92770e6ed5c19559e1e99255e4ca604a22c5c6397b2f9dd3ee42c \ - --hash=sha256:93429602396f3383a797a2a70e5f1de5df8e35535d7806c9f91df06f297e109b \ - --hash=sha256:9627cc1a10c8c409b5822a92d57a77f383b554463d1884008e051c32ab1b3742 \ - --hash=sha256:998f3bd3cfc95e9424a6acd7840cbdd39e45bc09ef87533c006f94ac47296090 \ - --hash=sha256:9c72109213eb9d3874f7ac8c0c5fa90e072d678e117d9061c06e30c85b4cf0e6 \ - --hash=sha256:9fc1500fd2a952c5c8e3b29aaf7e3cc6e27e9cfc0a8819b3bce48cc1b849e4cc \ - --hash=sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142 \ - --hash=sha256:a45d85cf20b5e0d0aa5a8dca27cce8eddef3292bc29d72dcad1641f4ed50aa16 \ - --hash=sha256:a7d8d14fe962153fc681f6366bdec33d4356f98a3e3567782aac1b6e0e40109a \ - --hash=sha256:a8fa23fe62c436ccf23ff930149c047f060c7126eae3ccea005f0483f27b2e28 \ - --hash=sha256:aa6658732517ddabe22c9036479eabce6036655ba87a0224c612e1ae6af2087e \ - --hash=sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94 \ - --hash=sha256:ab5a5a0c7a7991d90446a198689c0535be89bbd6b410a1f9a66688f0880ec026 \ - --hash=sha256:acd48d5b80ee80f9432a165c0ac8cbf9253eaddb6113269a5e18699b33958dbb \ - --hash=sha256:ad7593bb24b2ab09e65e8a1d385606f0f47c65b5a2ae6c551db67d6653e78c28 \ - --hash=sha256:baa42524a82f75303f714108fea528ccacf0386af429b69fff141ffef1c534f9 \ - --hash=sha256:bdfcf6443637c148c4e1a20c48c566aa694fa5e288d34b20fcdc58507882fed3 \ - --hash=sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f \ - --hash=sha256:c02a30b904282777d872266b87b20ed8cc0d1501855e27f831320f471d54d983 \ - --hash=sha256:c1277cd707c465cd09572a774559a3cc7c7a28802eb3a2a9472588f062097205 \ - --hash=sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f \ - --hash=sha256:c5ce2ce7c997e1971b7184ee37deb6ea9922ef5163c6ee5aa3c274b05f9e12fa \ - --hash=sha256:c823bc3971c44ab93e611ab1a46b1eafeae474c0c844aff4b7474287b75fe49c \ - --hash=sha256:ce0cdc074d540265bfeb31336e678b4e37316849d13b308607efa527e981f5c2 \ - --hash=sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb \ - --hash=sha256:d183cf9c797a5291e8301790ed6d053480ed94070637bfaad914dd38b0981f67 \ - --hash=sha256:d9010c31cd6fa59438da4e58a7f19e4753f7f264300cd152e7f90d4602449762 \ - --hash=sha256:d9e5e4a85bdb56d224f412d9c98ae4cbd032cc4f3161818f692cd81766eee65a \ - --hash=sha256:da1dee8948d2137bb51fbb8a53cce6b1bcc86003c6b42565f008438b806cccd8 \ - --hash=sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a \ - --hash=sha256:e00e3505cd80440f6c98c6d69269dcc2a119f86ad0a9fd70bccc59504bebd68a \ - --hash=sha256:e48d5021a84d341bcaf95c8460b152cfbad770d28e5fe14a768988c461b821bc \ - --hash=sha256:e7f8b04d83483577fd9200461b057c9f14ced334dcb053090cea1da9c8321a91 \ - --hash=sha256:edfe3341033a6b53a5c522c802deb2079eee5cbfbb0af032a55064bd65c73a23 \ - --hash=sha256:ef9c33cc5cbca35808f6c74be11eb7f5f6b14d2311be84a15b594bd3e58b5527 \ - --hash=sha256:f2d4324a98062be0525d16f768a03e0bbb3b9fe301ceee99611dc9a7953124e6 \ - --hash=sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c \ - --hash=sha256:f614ab0c76397661b90b6851a030004dac502e48260ea10f2441abd2207fbcc7 \ - --hash=sha256:f7db54c7914cc99d901d93a34704833568d86c20925b2762f9fa779f9cd2e70f \ - --hash=sha256:fbc6264158392bad9df19537e872d476f7c57adf718944cc1e4495cbabf38e2a \ - --hash=sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092 \ - --hash=sha256:ffe595f10566f8276b76dc3a11ae4bb7eba1aac8ddd75811736a15b0d5311414 +aiohttp==3.10.9 \ + --hash=sha256:02d1d6610588bcd743fae827bd6f2e47e0d09b346f230824b4c6fb85c6065f9c \ + --hash=sha256:03690541e4cc866eef79626cfa1ef4dd729c5c1408600c8cb9e12e1137eed6ab \ + --hash=sha256:0bc059ecbce835630e635879f5f480a742e130d9821fbe3d2f76610a6698ee25 \ + --hash=sha256:0c21c82df33b264216abffff9f8370f303dab65d8eee3767efbbd2734363f677 \ + --hash=sha256:1298b854fd31d0567cbb916091be9d3278168064fca88e70b8468875ef9ff7e7 \ + --hash=sha256:1321658f12b6caffafdc35cfba6c882cb014af86bef4e78c125e7e794dfb927b \ + --hash=sha256:143b0026a9dab07a05ad2dd9e46aa859bffdd6348ddc5967b42161168c24f857 \ + --hash=sha256:16e6a51d8bc96b77f04a6764b4ad03eeef43baa32014fce71e882bd71302c7e4 \ + --hash=sha256:172ad884bb61ad31ed7beed8be776eb17e7fb423f1c1be836d5cb357a096bf12 \ + --hash=sha256:17c272cfe7b07a5bb0c6ad3f234e0c336fb53f3bf17840f66bd77b5815ab3d16 \ + --hash=sha256:1a0ee6c0d590c917f1b9629371fce5f3d3f22c317aa96fbdcce3260754d7ea21 \ + --hash=sha256:2746d8994ebca1bdc55a1e998feff4e94222da709623bb18f6e5cfec8ec01baf \ + --hash=sha256:2914caa46054f3b5ff910468d686742ff8cff54b8a67319d75f5d5945fd0a13d \ + --hash=sha256:2bbf94d4a0447705b7775417ca8bb8086cc5482023a6e17cdc8f96d0b1b5aba6 \ + --hash=sha256:2bd9f3eac515c16c4360a6a00c38119333901b8590fe93c3257a9b536026594d \ + --hash=sha256:2c33fa6e10bb7ed262e3ff03cc69d52869514f16558db0626a7c5c61dde3c29f \ + --hash=sha256:2d37f4718002863b82c6f391c8efd4d3a817da37030a29e2682a94d2716209de \ + --hash=sha256:3668d0c2a4d23fb136a753eba42caa2c0abbd3d9c5c87ee150a716a16c6deec1 \ + --hash=sha256:36d4fba838be5f083f5490ddd281813b44d69685db910907636bc5dca6322316 \ + --hash=sha256:40ff5b7660f903dc587ed36ef08a88d46840182d9d4b5694e7607877ced698a1 \ + --hash=sha256:42775de0ca04f90c10c5c46291535ec08e9bcc4756f1b48f02a0657febe89b10 \ + --hash=sha256:482c85cf3d429844396d939b22bc2a03849cb9ad33344689ad1c85697bcba33a \ + --hash=sha256:4e6cb75f8ddd9c2132d00bc03c9716add57f4beff1263463724f6398b813e7eb \ + --hash=sha256:4edc3fd701e2b9a0d605a7b23d3de4ad23137d23fc0dbab726aa71d92f11aaaf \ + --hash=sha256:4fd16b30567c5b8e167923be6e027eeae0f20cf2b8a26b98a25115f28ad48ee0 \ + --hash=sha256:5002a02c17fcfd796d20bac719981d2fca9c006aac0797eb8f430a58e9d12431 \ + --hash=sha256:51d0a4901b27272ae54e42067bc4b9a90e619a690b4dc43ea5950eb3070afc32 \ + --hash=sha256:558b3d223fd631ad134d89adea876e7fdb4c93c849ef195049c063ada82b7d08 \ + --hash=sha256:5c070430fda1a550a1c3a4c2d7281d3b8cfc0c6715f616e40e3332201a253067 \ + --hash=sha256:5f392ef50e22c31fa49b5a46af7f983fa3f118f3eccb8522063bee8bfa6755f8 \ + --hash=sha256:60555211a006d26e1a389222e3fab8cd379f28e0fbf7472ee55b16c6c529e3a6 \ + --hash=sha256:608cecd8d58d285bfd52dbca5b6251ca8d6ea567022c8a0eaae03c2589cd9af9 \ + --hash=sha256:60ad5b8a7452c0f5645c73d4dad7490afd6119d453d302cd5b72b678a85d6044 \ + --hash=sha256:63649309da83277f06a15bbdc2a54fbe75efb92caa2c25bb57ca37762789c746 \ + --hash=sha256:6ebdc3b3714afe1b134b3bbeb5f745eed3ecbcff92ab25d80e4ef299e83a5465 \ + --hash=sha256:6f3c6648aa123bcd73d6f26607d59967b607b0da8ffcc27d418a4b59f4c98c7c \ + --hash=sha256:7003f33f5f7da1eb02f0446b0f8d2ccf57d253ca6c2e7a5732d25889da82b517 \ + --hash=sha256:776e9f3c9b377fcf097c4a04b241b15691e6662d850168642ff976780609303c \ + --hash=sha256:85711eec2d875cd88c7eb40e734c4ca6d9ae477d6f26bd2b5bb4f7f60e41b156 \ + --hash=sha256:87d1e4185c5d7187684d41ebb50c9aeaaaa06ca1875f4c57593071b0409d2444 \ + --hash=sha256:8a3f063b41cc06e8d0b3fcbbfc9c05b7420f41287e0cd4f75ce0a1f3d80729e6 \ + --hash=sha256:8b3fb28a9ac8f2558760d8e637dbf27aef1e8b7f1d221e8669a1074d1a266bb2 \ + --hash=sha256:8bd9125dd0cc8ebd84bff2be64b10fdba7dc6fd7be431b5eaf67723557de3a31 \ + --hash=sha256:8be1a65487bdfc285bd5e9baf3208c2132ca92a9b4020e9f27df1b16fab998a9 \ + --hash=sha256:8cc0d13b4e3b1362d424ce3f4e8c79e1f7247a00d792823ffd640878abf28e56 \ + --hash=sha256:8d9d10d10ec27c0d46ddaecc3c5598c4db9ce4e6398ca872cdde0525765caa2f \ + --hash=sha256:8debb45545ad95b58cc16c3c1cc19ad82cffcb106db12b437885dbee265f0ab5 \ + --hash=sha256:91aa966858593f64c8a65cdefa3d6dc8fe3c2768b159da84c1ddbbb2c01ab4ef \ + --hash=sha256:9331dd34145ff105177855017920dde140b447049cd62bb589de320fd6ddd582 \ + --hash=sha256:99f9678bf0e2b1b695e8028fedac24ab6770937932eda695815d5a6618c37e04 \ + --hash=sha256:9fdf5c839bf95fc67be5794c780419edb0dbef776edcfc6c2e5e2ffd5ee755fa \ + --hash=sha256:a14e4b672c257a6b94fe934ee62666bacbc8e45b7876f9dd9502d0f0fe69db16 \ + --hash=sha256:a19caae0d670771ea7854ca30df76f676eb47e0fd9b2ee4392d44708f272122d \ + --hash=sha256:a35ed3d03910785f7d9d6f5381f0c24002b2b888b298e6f941b2fc94c5055fcd \ + --hash=sha256:a61df62966ce6507aafab24e124e0c3a1cfbe23c59732987fc0fd0d71daa0b88 \ + --hash=sha256:a6e00c8a92e7663ed2be6fcc08a2997ff06ce73c8080cd0df10cc0321a3168d7 \ + --hash=sha256:ac3196952c673822ebed8871cf8802e17254fff2a2ed4835d9c045d9b88c5ec7 \ + --hash=sha256:ac74e794e3aee92ae8f571bfeaa103a141e409863a100ab63a253b1c53b707eb \ + --hash=sha256:ad3675c126f2a95bde637d162f8231cff6bc0bc9fbe31bd78075f9ff7921e322 \ + --hash=sha256:aeebd3061f6f1747c011e1d0b0b5f04f9f54ad1a2ca183e687e7277bef2e0da2 \ + --hash=sha256:ba1a599255ad6a41022e261e31bc2f6f9355a419575b391f9655c4d9e5df5ff5 \ + --hash=sha256:bbdb8def5268f3f9cd753a265756f49228a20ed14a480d151df727808b4531dd \ + --hash=sha256:c2555e4949c8d8782f18ef20e9d39730d2656e218a6f1a21a4c4c0b56546a02e \ + --hash=sha256:c2695c61cf53a5d4345a43d689f37fc0f6d3a2dc520660aec27ec0f06288d1f9 \ + --hash=sha256:c2b627d3c8982691b06d89d31093cee158c30629fdfebe705a91814d49b554f8 \ + --hash=sha256:c46131c6112b534b178d4e002abe450a0a29840b61413ac25243f1291613806a \ + --hash=sha256:c54dc329cd44f7f7883a9f4baaefe686e8b9662e2c6c184ea15cceee587d8d69 \ + --hash=sha256:c7d7cafc11d70fdd8801abfc2ff276744ae4cb39d8060b6b542c7e44e5f2cfc2 \ + --hash=sha256:cb0b2d5d51f96b6cc19e6ab46a7b684be23240426ae951dcdac9639ab111b45e \ + --hash=sha256:d15a29424e96fad56dc2f3abed10a89c50c099f97d2416520c7a543e8fddf066 \ + --hash=sha256:d1f5c9169e26db6a61276008582d945405b8316aae2bb198220466e68114a0f5 \ + --hash=sha256:d271f770b52e32236d945911b2082f9318e90ff835d45224fa9e28374303f729 \ + --hash=sha256:d646fdd74c25bbdd4a055414f0fe32896c400f38ffbdfc78c68e62812a9e0257 \ + --hash=sha256:d6e395c3d1f773cf0651cd3559e25182eb0c03a2777b53b4575d8adc1149c6e9 \ + --hash=sha256:d7c071235a47d407b0e93aa6262b49422dbe48d7d8566e1158fecc91043dd948 \ + --hash=sha256:d97273a52d7f89a75b11ec386f786d3da7723d7efae3034b4dda79f6f093edc1 \ + --hash=sha256:dcf354661f54e6a49193d0b5653a1b011ba856e0b7a76bda2c33e4c6892f34ea \ + --hash=sha256:e3e7fabedb3fe06933f47f1538df7b3a8d78e13d7167195f51ca47ee12690373 \ + --hash=sha256:e525b69ee8a92c146ae5b4da9ecd15e518df4d40003b01b454ad694a27f498b5 \ + --hash=sha256:e709d6ac598c5416f879bb1bae3fd751366120ac3fa235a01de763537385d036 \ + --hash=sha256:e83dfefb4f7d285c2d6a07a22268344a97d61579b3e0dce482a5be0251d672ab \ + --hash=sha256:e86260b76786c28acf0b5fe31c8dca4c2add95098c709b11e8c35b424ebd4f5b \ + --hash=sha256:e883b61b75ca6efc2541fcd52a5c8ccfe288b24d97e20ac08fdf343b8ac672ea \ + --hash=sha256:f0a44bb40b6aaa4fb9a5c1ee07880570ecda2065433a96ccff409c9c20c1624a \ + --hash=sha256:f82ace0ec57c94aaf5b0e118d4366cff5889097412c75aa14b4fd5fc0c44ee3e \ + --hash=sha256:f9ca09414003c0e96a735daa1f071f7d7ed06962ef4fa29ceb6c80d06696d900 \ + --hash=sha256:fa430b871220dc62572cef9c69b41e0d70fcb9d486a4a207a5de4c1f25d82593 \ + --hash=sha256:fc262c3df78c8ff6020c782d9ce02e4bcffe4900ad71c0ecdad59943cba54442 \ + --hash=sha256:fcd546782d03181b0b1d20b43d612429a90a68779659ba8045114b867971ab71 \ + --hash=sha256:fd4ceeae2fb8cabdd1b71c82bfdd39662473d3433ec95b962200e9e752fb70d0 \ + --hash=sha256:fec5fac7aea6c060f317f07494961236434928e6f4374e170ef50b3001e14581 # via # -r requirements.in # aio-api-github @@ -522,6 +526,10 @@ envoy-base-utils==0.5.5 \ # envoy-docs-sphinx-runner # envoy-github-release # envoy-gpg-sign +envoy-ci-report==0.0.3 \ + --hash=sha256:1c4bf3cb9f92117eb482186ea4fa58ff15e26641cdaeeae45bb826ae8454859e \ + --hash=sha256:cc59d0980330a4b73cfcaba2a1b5050348c1d6530baffeb354dccea3e46672d2 + # via -r requirements.in envoy-code-check==0.5.13 \ --hash=sha256:58c31be3ba1a3273eec8a76d1dcfe1a3ae5eae4730ca9d70a85fec0d641846c4 \ --hash=sha256:6c568d477642abdf7b41a0b6a5bb21fd480d92e500c53120837a01d4436d8591 @@ -1380,6 +1388,18 @@ trycast==1.2.0 \ # via # aio-core # envoy-base-utils +types-orjson==3.6.2 \ + --hash=sha256:22ee9a79236b6b0bfb35a0684eded62ad930a88a56797fa3c449b026cf7dbfe4 \ + --hash=sha256:cf9afcc79a86325c7aff251790338109ed6f6b1bab09d2d4262dd18c85a3c638 + # via aio-core +types-pytz==2024.2.0.20241003 \ + --hash=sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7 \ + --hash=sha256:575dc38f385a922a212bac00a7d6d2e16e141132a3c955078f4a4fd13ed6cb44 + # via aio-core +types-pyyaml==6.0.12.20240917 \ + --hash=sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570 \ + --hash=sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587 + # via aio-core typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 diff --git a/tools/ci/BUILD b/tools/ci/BUILD new file mode 100644 index 0000000000..209a597755 --- /dev/null +++ b/tools/ci/BUILD @@ -0,0 +1,14 @@ +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/python:namespace.bzl", "envoy_py_namespace") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_py_namespace() + +envoy_entry_point( + name = "report", + pkg = "envoy.ci.report", +) From b0f19e88455d713ebb159985324ff11f3d5241b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:51:57 +0100 Subject: [PATCH 55/77] build(deps): bump frozendict from 2.4.5 to 2.4.6 in /tools/base (#36561) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 79 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 71f2230295..dac71d6512 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -594,45 +594,46 @@ flake8==7.1.1 \ # -r requirements.in # envoy-code-check # pep8-naming -frozendict==2.4.5 \ - --hash=sha256:043bce9f5e7e8df8313d0c961b2da3346c0b002311d88c74cadae2b1f8fd2904 \ - --hash=sha256:0483776fa25f0c3f9ff75a89b5276a582231f83baea85b091dfefde99b95ac5c \ - --hash=sha256:05786d9cff61ee95149c15cdfc10cf4b4cb10e1eab1d4648b0c99ed58fedb86d \ - --hash=sha256:08e8d4ebc950916b5cdeedcb697cf893c5903d9cb8ab27dffe41072a08e1cfc1 \ - --hash=sha256:14755127b988b428f95f11fa626374e247f8a40316697218f1e2f44c107c5027 \ - --hash=sha256:1f364cfe5ef97523a4434e9f458bb4821594d3531d898621e5acae43463dcb5e \ - --hash=sha256:20cf1db30ca0c8f76a05ec1256132f505cf6f93eb382a5961c738377037b5b9d \ - --hash=sha256:24953a1cfe344415e7557413e493e21fa9c4cecbc13284b6f334837aa5601c9f \ - --hash=sha256:2b51e0aae859fc8d56138eac4be121a76647fa66ea620af8a62d9d6938729441 \ - --hash=sha256:35730ff269b8a6eb07c3d91d5fd6a7f5c32e08bef665fe1ef51012bdb2702196 \ - --hash=sha256:3702438d81deab127963ab41dca1738a0fbf11038a50a25c5f814943a2d25de5 \ - --hash=sha256:3af1b09bad761d5500b096b757c346591b5dfdc8443aa9642c2c02b4885dc09e \ - --hash=sha256:4b3eea1678607174c468fe4e3b903625bccfb3215ae2b0138220dc1f6ef71f37 \ - --hash=sha256:4f073c926b1f88fa85ed85222101f61f6c4b2180c95d1528ca6ecc7cef835442 \ - --hash=sha256:6be054ba76e8a49c6846a7369db3eaaa0593c29a644026c25923f13fdea06483 \ - --hash=sha256:6d10ad91ded20fae8737fddcfed1b6358207f95292a335f96c4cd365d97f5e30 \ - --hash=sha256:73ff80ce3f95c3ec60b38ef4ddf5cb8e20ea3edce2f56e89af7c4c45013684bf \ - --hash=sha256:754924d53b1fd2dae7b15f9c86b0610334a840ae728cc717f24aa040fc94136f \ - --hash=sha256:76c7af5f9db9ae01531edaf38fd3e8faae1bb6b94abbf8fa5bd0326f52e777df \ - --hash=sha256:8582f26ca862bb1e40ed790d934adf6afcfc904b0425dcfe01aed2103bed27fb \ - --hash=sha256:95bf788d65e3a50d9deef5c68e568c59acf473fea8a7a4d7e405a7e63ac85cdd \ - --hash=sha256:98b1483980c47bb74ef777ab748031a69eb41c43ef44494e9db79111638fdb84 \ - --hash=sha256:a5448639058157ccd7f26ba83e441066c6ae515beb68b9b99d5b2dbb0bb36b19 \ - --hash=sha256:a81d80ded07f0e34a3fa2b78235a48259b886ad20a96da5526a9404d192c6eb3 \ - --hash=sha256:a967730a115cb8b5d028e0d07cd680c04324e913ee3d1caef9e039ca740343c4 \ - --hash=sha256:af4bdffecff350a68f1a966dec84459d3ea8e2606c14fb3ebe937b8c6c9e9129 \ - --hash=sha256:b8481d83a7219e9e14c719113ea2d8321d7840af35af58c72b3864b7123e7de3 \ - --hash=sha256:bbf9dcb69a44519308c775bcf72b8417c3102f415f77df19b8eddb97b8f88d78 \ - --hash=sha256:d2b2a9c809ebb54c8c0fe2acc38d45299f3e9a9d90061773468896221cab1eaa \ - --hash=sha256:de1126d6dd39d7939be388161b973ffadb9c6e97f8f9fdcb52ba790828621191 \ - --hash=sha256:dee64c44e1a146171bdd2bf4501210f273a5ab73cc2e924a01f81b14e4612f94 \ - --hash=sha256:e591124818f2d5d8f2d284a7b78604f4180e73d770c33252370edccff5969293 \ - --hash=sha256:eb58e0c41bc6bb60a8b0852d50da8836380fb78311608dc26791e3e9aa25bcfe \ - --hash=sha256:ee3e6f4013446d17c580ce0bac41a1f81371c21dd6e34a85c0f26bae94fdfd9b \ - --hash=sha256:f1385852cb05a31ae76c972caa3a07679414b01a664848bde2bae71c9910f045 \ - --hash=sha256:f2ff03ed21657cde7ea04fffc76f0c1736100a03671e4682722685e63486cd96 \ - --hash=sha256:f8d677a6066ca3289181028fccabf77413addddaa7a6629485d52aeaa4f4899c \ - --hash=sha256:fd7add309789595c044c0155a0bddfa9d20c77f65de1e33a14aa3033b936ef63 +frozendict==2.4.6 \ + --hash=sha256:02331541611f3897f260900a1815b63389654951126e6e65545e529b63c08361 \ + --hash=sha256:0aaa11e7c472150efe65adbcd6c17ac0f586896096ab3963775e1c5c58ac0098 \ + --hash=sha256:18d50a2598350b89189da9150058191f55057581e40533e470db46c942373acf \ + --hash=sha256:1b4a3f8f6dd51bee74a50995c39b5a606b612847862203dd5483b9cd91b0d36a \ + --hash=sha256:1f42e6b75254ea2afe428ad6d095b62f95a7ae6d4f8272f0bd44a25dddd20f67 \ + --hash=sha256:2d69418479bfb834ba75b0e764f058af46ceee3d655deb6a0dd0c0c1a5e82f09 \ + --hash=sha256:323f1b674a2cc18f86ab81698e22aba8145d7a755e0ac2cccf142ee2db58620d \ + --hash=sha256:377a65be0a700188fc21e669c07de60f4f6d35fae8071c292b7df04776a1c27b \ + --hash=sha256:49344abe90fb75f0f9fdefe6d4ef6d4894e640fadab71f11009d52ad97f370b9 \ + --hash=sha256:49ffaf09241bc1417daa19362a2241a4aa435f758fd4375c39ce9790443a39cd \ + --hash=sha256:622301b1c29c4f9bba633667d592a3a2b093cb408ba3ce578b8901ace3931ef3 \ + --hash=sha256:665fad3f0f815aa41294e561d98dbedba4b483b3968e7e8cab7d728d64b96e33 \ + --hash=sha256:669237c571856be575eca28a69e92a3d18f8490511eff184937283dc6093bd67 \ + --hash=sha256:7088102345d1606450bd1801a61139bbaa2cb0d805b9b692f8d81918ea835da6 \ + --hash=sha256:7134a2bb95d4a16556bb5f2b9736dceb6ea848fa5b6f3f6c2d6dba93b44b4757 \ + --hash=sha256:7291abacf51798d5ffe632771a69c14fb423ab98d63c4ccd1aa382619afe2f89 \ + --hash=sha256:74b6b26c15dddfefddeb89813e455b00ebf78d0a3662b89506b4d55c6445a9f4 \ + --hash=sha256:7730f8ebe791d147a1586cbf6a42629351d4597773317002181b66a2da0d509e \ + --hash=sha256:807862e14b0e9665042458fde692c4431d660c4219b9bb240817f5b918182222 \ + --hash=sha256:94321e646cc39bebc66954a31edd1847d3a2a3483cf52ff051cd0996e7db07db \ + --hash=sha256:9647c74efe3d845faa666d4853cfeabbaee403b53270cabfc635b321f770e6b8 \ + --hash=sha256:9a8a43036754a941601635ea9c788ebd7a7efbed2becba01b54a887b41b175b9 \ + --hash=sha256:a4e3737cb99ed03200cd303bdcd5514c9f34b29ee48f405c1184141bd68611c9 \ + --hash=sha256:a76cee5c4be2a5d1ff063188232fffcce05dde6fd5edd6afe7b75b247526490e \ + --hash=sha256:b8f2829048f29fe115da4a60409be2130e69402e29029339663fac39c90e6e2b \ + --hash=sha256:ba5ef7328706db857a2bdb2c2a17b4cd37c32a19c017cff1bb7eeebc86b0f411 \ + --hash=sha256:c131f10c4d3906866454c4e89b87a7e0027d533cce8f4652aa5255112c4d6677 \ + --hash=sha256:c3a05c0a50cab96b4bb0ea25aa752efbfceed5ccb24c007612bc63e51299336f \ + --hash=sha256:c9905dcf7aa659e6a11b8051114c9fa76dfde3a6e50e6dc129d5aece75b449a2 \ + --hash=sha256:ce1e9217b85eec6ba9560d520d5089c82dbb15f977906eb345d81459723dd7e3 \ + --hash=sha256:d065db6a44db2e2375c23eac816f1a022feb2fa98cbb50df44a9e83700accbea \ + --hash=sha256:da6a10164c8a50b34b9ab508a9420df38f4edf286b9ca7b7df8a91767baecb34 \ + --hash=sha256:df7cd16470fbd26fc4969a208efadc46319334eb97def1ddf48919b351192b8e \ + --hash=sha256:e72fb86e48811957d66ffb3e95580af7b1af1e6fbd760ad63d7bd79b2c9a07f8 \ + --hash=sha256:eabd21d8e5db0c58b60d26b4bb9839cac13132e88277e1376970172a85ee04b3 \ + --hash=sha256:eddabeb769fab1e122d3a6872982c78179b5bcc909fdc769f3cf1964f55a6d20 \ + --hash=sha256:f4c789fd70879ccb6289a603cdebdc4953e7e5dea047d30c1b180529b28257b5 \ + --hash=sha256:f5b94d5b07c00986f9e37a38dd83c13f5fe3bf3f1ccc8e88edea8fe15d6cd88c \ + --hash=sha256:fc67cbb3c96af7a798fab53d52589752c1673027e516b702ab355510ddf6bdff # via # -r requirements.in # aio-run-runner From 3e03b6963370deebf1a4da3ff90301763fadd992 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:52:24 +0100 Subject: [PATCH 56/77] build(deps): bump actions/upload-artifact from 4.4.0 to 4.4.3 (#36528) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 88324734fb..05c18a27ad 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -33,7 +33,7 @@ jobs: publish_results: true - name: "Upload artifact" - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif From d4a72e04cee55dbd8f1e6c7c91ef1853046069d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:52:45 +0000 Subject: [PATCH 57/77] build(deps): bump yarl from 1.13.1 to 1.15.2 in /tools/base (#36559) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 292 ++++++++++++++++++++++++------------ 1 file changed, 199 insertions(+), 93 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index dac71d6512..cbdbc10bce 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1071,6 +1071,106 @@ ply==3.11 \ --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 \ --hash=sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce # via -r requirements.in +propcache==0.2.0 \ + --hash=sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9 \ + --hash=sha256:0e53cb83fdd61cbd67202735e6a6687a7b491c8742dfc39c9e01e80354956763 \ + --hash=sha256:1235c01ddaa80da8235741e80815ce381c5267f96cc49b1477fdcf8c047ef325 \ + --hash=sha256:140fbf08ab3588b3468932974a9331aff43c0ab8a2ec2c608b6d7d1756dbb6cb \ + --hash=sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b \ + --hash=sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09 \ + --hash=sha256:1ec43d76b9677637a89d6ab86e1fef70d739217fefa208c65352ecf0282be957 \ + --hash=sha256:20a617c776f520c3875cf4511e0d1db847a076d720714ae35ffe0df3e440be68 \ + --hash=sha256:218db2a3c297a3768c11a34812e63b3ac1c3234c3a086def9c0fee50d35add1f \ + --hash=sha256:22aa8f2272d81d9317ff5756bb108021a056805ce63dd3630e27d042c8092798 \ + --hash=sha256:25a1f88b471b3bc911d18b935ecb7115dff3a192b6fef46f0bfaf71ff4f12418 \ + --hash=sha256:25c8d773a62ce0451b020c7b29a35cfbc05de8b291163a7a0f3b7904f27253e6 \ + --hash=sha256:2a60ad3e2553a74168d275a0ef35e8c0a965448ffbc3b300ab3a5bb9956c2162 \ + --hash=sha256:2a66df3d4992bc1d725b9aa803e8c5a66c010c65c741ad901e260ece77f58d2f \ + --hash=sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036 \ + --hash=sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8 \ + --hash=sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2 \ + --hash=sha256:33ac8f098df0585c0b53009f039dfd913b38c1d2edafed0cedcc0c32a05aa110 \ + --hash=sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23 \ + --hash=sha256:363ea8cd3c5cb6679f1c2f5f1f9669587361c062e4899fce56758efa928728f8 \ + --hash=sha256:375a12d7556d462dc64d70475a9ee5982465fbb3d2b364f16b86ba9135793638 \ + --hash=sha256:388f3217649d6d59292b722d940d4d2e1e6a7003259eb835724092a1cca0203a \ + --hash=sha256:3947483a381259c06921612550867b37d22e1df6d6d7e8361264b6d037595f44 \ + --hash=sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2 \ + --hash=sha256:3c997f8c44ec9b9b0bcbf2d422cc00a1d9b9c681f56efa6ca149a941e5560da2 \ + --hash=sha256:3dfafb44f7bb35c0c06eda6b2ab4bfd58f02729e7c4045e179f9a861b07c9850 \ + --hash=sha256:3ebbcf2a07621f29638799828b8d8668c421bfb94c6cb04269130d8de4fb7136 \ + --hash=sha256:3f88a4095e913f98988f5b338c1d4d5d07dbb0b6bad19892fd447484e483ba6b \ + --hash=sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887 \ + --hash=sha256:4569158070180c3855e9c0791c56be3ceeb192defa2cdf6a3f39e54319e56b89 \ + --hash=sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87 \ + --hash=sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348 \ + --hash=sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4 \ + --hash=sha256:53d1bd3f979ed529f0805dd35ddaca330f80a9a6d90bc0121d2ff398f8ed8861 \ + --hash=sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e \ + --hash=sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c \ + --hash=sha256:56bb5c98f058a41bb58eead194b4db8c05b088c93d94d5161728515bd52b052b \ + --hash=sha256:5a5b3bb545ead161be780ee85a2b54fdf7092815995661947812dde94a40f6fb \ + --hash=sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1 \ + --hash=sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de \ + --hash=sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354 \ + --hash=sha256:662dd62358bdeaca0aee5761de8727cfd6861432e3bb828dc2a693aa0471a563 \ + --hash=sha256:676135dcf3262c9c5081cc8f19ad55c8a64e3f7282a21266d05544450bffc3a5 \ + --hash=sha256:67aeb72e0f482709991aa91345a831d0b707d16b0257e8ef88a2ad246a7280bf \ + --hash=sha256:67b69535c870670c9f9b14a75d28baa32221d06f6b6fa6f77a0a13c5a7b0a5b9 \ + --hash=sha256:682a7c79a2fbf40f5dbb1eb6bfe2cd865376deeac65acf9beb607505dced9e12 \ + --hash=sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4 \ + --hash=sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5 \ + --hash=sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71 \ + --hash=sha256:73e4b40ea0eda421b115248d7e79b59214411109a5bc47d0d48e4c73e3b8fcf9 \ + --hash=sha256:74acd6e291f885678631b7ebc85d2d4aec458dd849b8c841b57ef04047833bed \ + --hash=sha256:7665f04d0c7f26ff8bb534e1c65068409bf4687aa2534faf7104d7182debb336 \ + --hash=sha256:7735e82e3498c27bcb2d17cb65d62c14f1100b71723b68362872bca7d0913d90 \ + --hash=sha256:77a86c261679ea5f3896ec060be9dc8e365788248cc1e049632a1be682442063 \ + --hash=sha256:7cf18abf9764746b9c8704774d8b06714bcb0a63641518a3a89c7f85cc02c2ad \ + --hash=sha256:83928404adf8fb3d26793665633ea79b7361efa0287dfbd372a7e74311d51ee6 \ + --hash=sha256:8e40876731f99b6f3c897b66b803c9e1c07a989b366c6b5b475fafd1f7ba3fb8 \ + --hash=sha256:8f188cfcc64fb1266f4684206c9de0e80f54622c3f22a910cbd200478aeae61e \ + --hash=sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2 \ + --hash=sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7 \ + --hash=sha256:92fe151145a990c22cbccf9ae15cae8ae9eddabfc949a219c9f667877e40853d \ + --hash=sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d \ + --hash=sha256:9517d5e9e0731957468c29dbfd0f976736a0e55afaea843726e887f36fe017df \ + --hash=sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b \ + --hash=sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178 \ + --hash=sha256:97e48e8875e6c13909c800fa344cd54cc4b2b0db1d5f911f840458a500fde2c2 \ + --hash=sha256:9e0f07b42d2a50c7dd2d8675d50f7343d998c64008f1da5fef888396b7f84630 \ + --hash=sha256:a3dc1a4b165283bd865e8f8cb5f0c64c05001e0718ed06250d8cac9bec115b48 \ + --hash=sha256:a3ebe9a75be7ab0b7da2464a77bb27febcb4fab46a34f9288f39d74833db7f61 \ + --hash=sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89 \ + --hash=sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb \ + --hash=sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3 \ + --hash=sha256:b33d7a286c0dc1a15f5fc864cc48ae92a846df287ceac2dd499926c3801054a6 \ + --hash=sha256:bc092ba439d91df90aea38168e11f75c655880c12782facf5cf9c00f3d42b562 \ + --hash=sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b \ + --hash=sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58 \ + --hash=sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db \ + --hash=sha256:cfac69017ef97db2438efb854edf24f5a29fd09a536ff3a992b75990720cdc99 \ + --hash=sha256:d2f0d0f976985f85dfb5f3d685697ef769faa6b71993b46b295cdbbd6be8cc37 \ + --hash=sha256:d5bed7f9805cc29c780f3aee05de3262ee7ce1f47083cfe9f77471e9d6777e83 \ + --hash=sha256:d6a21ef516d36909931a2967621eecb256018aeb11fc48656e3257e73e2e247a \ + --hash=sha256:d9b6ddac6408194e934002a69bcaadbc88c10b5f38fb9307779d1c629181815d \ + --hash=sha256:db47514ffdbd91ccdc7e6f8407aac4ee94cc871b15b577c1c324236b013ddd04 \ + --hash=sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70 \ + --hash=sha256:e4a91d44379f45f5e540971d41e4626dacd7f01004826a18cb048e7da7e96544 \ + --hash=sha256:e63e3e1e0271f374ed489ff5ee73d4b6e7c60710e1f76af5f0e1a6117cd26394 \ + --hash=sha256:e70fac33e8b4ac63dfc4c956fd7d85a0b1139adcfc0d964ce288b7c527537fea \ + --hash=sha256:ecddc221a077a8132cf7c747d5352a15ed763b674c0448d811f408bf803d9ad7 \ + --hash=sha256:f45eec587dafd4b2d41ac189c2156461ebd0c1082d2fe7013571598abb8505d1 \ + --hash=sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793 \ + --hash=sha256:f571aea50ba5623c308aa146eb650eebf7dbe0fd8c5d946e28343cb3b5aad577 \ + --hash=sha256:f60f0ac7005b9f5a6091009b09a419ace1610e163fa5deaba5ce3484341840e7 \ + --hash=sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57 \ + --hash=sha256:f6d5749fdd33d90e34c2efb174c7e236829147a2713334d708746e94c4bde40d \ + --hash=sha256:f902804113e032e2cdf8c71015651c97af6418363bea8d78dc0911d56c335032 \ + --hash=sha256:fa1076244f54bb76e65e22cb6910365779d5c3d71d1f18b275f1dfc7b0d71b4d \ + --hash=sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016 \ + --hash=sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504 + # via yarl protobuf==5.28.2 \ --hash=sha256:2c69461a7fcc8e24be697624c09a839976d82ae75062b11a0972e41fd2cd9132 \ --hash=sha256:35cfcb15f213449af7ff6198d6eb5f739c37d7e4f1c09b5d0641babf2cc0c68f \ @@ -1544,99 +1644,105 @@ yapf==0.40.2 \ # via # -r requirements.in # envoy-code-check -yarl==1.13.1 \ - --hash=sha256:08d7148ff11cb8e886d86dadbfd2e466a76d5dd38c7ea8ebd9b0e07946e76e4b \ - --hash=sha256:098b870c18f1341786f290b4d699504e18f1cd050ed179af8123fd8232513424 \ - --hash=sha256:11b3ca8b42a024513adce810385fcabdd682772411d95bbbda3b9ed1a4257644 \ - --hash=sha256:1891d69a6ba16e89473909665cd355d783a8a31bc84720902c5911dbb6373465 \ - --hash=sha256:1bbb418f46c7f7355084833051701b2301092e4611d9e392360c3ba2e3e69f88 \ - --hash=sha256:1d0828e17fa701b557c6eaed5edbd9098eb62d8838344486248489ff233998b8 \ - --hash=sha256:1d8e3ca29f643dd121f264a7c89f329f0fcb2e4461833f02de6e39fef80f89da \ - --hash=sha256:1fa56f34b2236f5192cb5fceba7bbb09620e5337e0b6dfe2ea0ddbd19dd5b154 \ - --hash=sha256:216a6785f296169ed52cd7dcdc2612f82c20f8c9634bf7446327f50398732a51 \ - --hash=sha256:22b739f99c7e4787922903f27a892744189482125cc7b95b747f04dd5c83aa9f \ - --hash=sha256:2430cf996113abe5aee387d39ee19529327205cda975d2b82c0e7e96e5fdabdc \ - --hash=sha256:269c201bbc01d2cbba5b86997a1e0f73ba5e2f471cfa6e226bcaa7fd664b598d \ - --hash=sha256:298c1eecfd3257aa16c0cb0bdffb54411e3e831351cd69e6b0739be16b1bdaa8 \ - --hash=sha256:2a93a4557f7fc74a38ca5a404abb443a242217b91cd0c4840b1ebedaad8919d4 \ - --hash=sha256:2b2442a415a5f4c55ced0fade7b72123210d579f7d950e0b5527fc598866e62c \ - --hash=sha256:2db874dd1d22d4c2c657807562411ffdfabec38ce4c5ce48b4c654be552759dc \ - --hash=sha256:309c104ecf67626c033845b860d31594a41343766a46fa58c3309c538a1e22b2 \ - --hash=sha256:31497aefd68036d8e31bfbacef915826ca2e741dbb97a8d6c7eac66deda3b606 \ - --hash=sha256:373f16f38721c680316a6a00ae21cc178e3a8ef43c0227f88356a24c5193abd6 \ - --hash=sha256:396e59b8de7e4d59ff5507fb4322d2329865b909f29a7ed7ca37e63ade7f835c \ - --hash=sha256:3bb83a0f12701c0b91112a11148b5217617982e1e466069d0555be9b372f2734 \ - --hash=sha256:3de86547c820e4f4da4606d1c8ab5765dd633189791f15247706a2eeabc783ae \ - --hash=sha256:3fdbf0418489525231723cdb6c79e7738b3cbacbaed2b750cb033e4ea208f220 \ - --hash=sha256:40c6e73c03a6befb85b72da213638b8aaa80fe4136ec8691560cf98b11b8ae6e \ - --hash=sha256:44a4c40a6f84e4d5955b63462a0e2a988f8982fba245cf885ce3be7618f6aa7d \ - --hash=sha256:44b07e1690f010c3c01d353b5790ec73b2f59b4eae5b0000593199766b3f7a5c \ - --hash=sha256:45d23c4668d4925688e2ea251b53f36a498e9ea860913ce43b52d9605d3d8177 \ - --hash=sha256:45f209fb4bbfe8630e3d2e2052535ca5b53d4ce2d2026bed4d0637b0416830da \ - --hash=sha256:4afdf84610ca44dcffe8b6c22c68f309aff96be55f5ea2fa31c0c225d6b83e23 \ - --hash=sha256:4feaaa4742517eaceafcbe74595ed335a494c84634d33961214b278126ec1485 \ - --hash=sha256:576365c9f7469e1f6124d67b001639b77113cfd05e85ce0310f5f318fd02fe85 \ - --hash=sha256:5820bd4178e6a639b3ef1db8b18500a82ceab6d8b89309e121a6859f56585b05 \ - --hash=sha256:5989a38ba1281e43e4663931a53fbf356f78a0325251fd6af09dd03b1d676a09 \ - --hash=sha256:5a9bacedbb99685a75ad033fd4de37129449e69808e50e08034034c0bf063f99 \ - --hash=sha256:5b66c87da3c6da8f8e8b648878903ca54589038a0b1e08dde2c86d9cd92d4ac9 \ - --hash=sha256:5c5e32fef09ce101fe14acd0f498232b5710effe13abac14cd95de9c274e689e \ - --hash=sha256:658e8449b84b92a4373f99305de042b6bd0d19bf2080c093881e0516557474a5 \ - --hash=sha256:6a2acde25be0cf9be23a8f6cbd31734536a264723fca860af3ae5e89d771cd71 \ - --hash=sha256:6a5185ad722ab4dd52d5fb1f30dcc73282eb1ed494906a92d1a228d3f89607b0 \ - --hash=sha256:6b7f6e699304717fdc265a7e1922561b02a93ceffdaefdc877acaf9b9f3080b8 \ - --hash=sha256:703b0f584fcf157ef87816a3c0ff868e8c9f3c370009a8b23b56255885528f10 \ - --hash=sha256:7055bbade838d68af73aea13f8c86588e4bcc00c2235b4b6d6edb0dbd174e246 \ - --hash=sha256:78f271722423b2d4851cf1f4fa1a1c4833a128d020062721ba35e1a87154a049 \ - --hash=sha256:7addd26594e588503bdef03908fc207206adac5bd90b6d4bc3e3cf33a829f57d \ - --hash=sha256:81bad32c8f8b5897c909bf3468bf601f1b855d12f53b6af0271963ee67fff0d2 \ - --hash=sha256:82e692fb325013a18a5b73a4fed5a1edaa7c58144dc67ad9ef3d604eccd451ad \ - --hash=sha256:84bbcdcf393139f0abc9f642bf03f00cac31010f3034faa03224a9ef0bb74323 \ - --hash=sha256:86c438ce920e089c8c2388c7dcc8ab30dfe13c09b8af3d306bcabb46a053d6f7 \ - --hash=sha256:8be8cdfe20787e6a5fcbd010f8066227e2bb9058331a4eccddec6c0db2bb85b2 \ - --hash=sha256:8c723c91c94a3bc8033dd2696a0f53e5d5f8496186013167bddc3fb5d9df46a3 \ - --hash=sha256:8ca53632007c69ddcdefe1e8cbc3920dd88825e618153795b57e6ebcc92e752a \ - --hash=sha256:8f722f30366474a99745533cc4015b1781ee54b08de73260b2bbe13316079851 \ - --hash=sha256:942c80a832a79c3707cca46bd12ab8aa58fddb34b1626d42b05aa8f0bcefc206 \ - --hash=sha256:94a993f976cdcb2dc1b855d8b89b792893220db8862d1a619efa7451817c836b \ - --hash=sha256:95c6737f28069153c399d875317f226bbdea939fd48a6349a3b03da6829fb550 \ - --hash=sha256:9915300fe5a0aa663c01363db37e4ae8e7c15996ebe2c6cce995e7033ff6457f \ - --hash=sha256:9a18595e6a2ee0826bf7dfdee823b6ab55c9b70e8f80f8b77c37e694288f5de1 \ - --hash=sha256:9c8854b9f80693d20cec797d8e48a848c2fb273eb6f2587b57763ccba3f3bd4b \ - --hash=sha256:9cec42a20eae8bebf81e9ce23fb0d0c729fc54cf00643eb251ce7c0215ad49fe \ - --hash=sha256:9d2e1626be8712333a9f71270366f4a132f476ffbe83b689dd6dc0d114796c74 \ - --hash=sha256:9d74f3c335cfe9c21ea78988e67f18eb9822f5d31f88b41aec3a1ec5ecd32da5 \ - --hash=sha256:9fb4134cc6e005b99fa29dbc86f1ea0a298440ab6b07c6b3ee09232a3b48f495 \ - --hash=sha256:a0ae6637b173d0c40b9c1462e12a7a2000a71a3258fa88756a34c7d38926911c \ - --hash=sha256:a31d21089894942f7d9a8df166b495101b7258ff11ae0abec58e32daf8088813 \ - --hash=sha256:a3442c31c11088e462d44a644a454d48110f0588de830921fd201060ff19612a \ - --hash=sha256:ab9524e45ee809a083338a749af3b53cc7efec458c3ad084361c1dbf7aaf82a2 \ - --hash=sha256:b1481c048fe787f65e34cb06f7d6824376d5d99f1231eae4778bbe5c3831076d \ - --hash=sha256:b8c837ab90c455f3ea8e68bee143472ee87828bff19ba19776e16ff961425b57 \ - --hash=sha256:bbf2c3f04ff50f16404ce70f822cdc59760e5e2d7965905f0e700270feb2bbfc \ - --hash=sha256:bbf9c2a589be7414ac4a534d54e4517d03f1cbb142c0041191b729c2fa23f320 \ - --hash=sha256:bcd5bf4132e6a8d3eb54b8d56885f3d3a38ecd7ecae8426ecf7d9673b270de43 \ - --hash=sha256:c14c16831b565707149c742d87a6203eb5597f4329278446d5c0ae7a1a43928e \ - --hash=sha256:c49f3e379177f4477f929097f7ed4b0622a586b0aa40c07ac8c0f8e40659a1ac \ - --hash=sha256:c92b89bffc660f1274779cb6fbb290ec1f90d6dfe14492523a0667f10170de26 \ - --hash=sha256:cd66152561632ed4b2a9192e7f8e5a1d41e28f58120b4761622e0355f0fe034c \ - --hash=sha256:cf1ad338620249f8dd6d4b6a91a69d1f265387df3697ad5dc996305cf6c26fb2 \ - --hash=sha256:d07b52c8c450f9366c34aa205754355e933922c79135125541daae6cbf31c799 \ - --hash=sha256:d0d12fe78dcf60efa205e9a63f395b5d343e801cf31e5e1dda0d2c1fb618073d \ - --hash=sha256:d4ee1d240b84e2f213565f0ec08caef27a0e657d4c42859809155cf3a29d1735 \ - --hash=sha256:d959fe96e5c2712c1876d69af0507d98f0b0e8d81bee14cfb3f6737470205419 \ - --hash=sha256:dcaef817e13eafa547cdfdc5284fe77970b891f731266545aae08d6cce52161e \ - --hash=sha256:df4e82e68f43a07735ae70a2d84c0353e58e20add20ec0af611f32cd5ba43fb4 \ - --hash=sha256:ec8cfe2295f3e5e44c51f57272afbd69414ae629ec7c6b27f5a410efc78b70a0 \ - --hash=sha256:ec9dd328016d8d25702a24ee274932aebf6be9787ed1c28d021945d264235b3c \ - --hash=sha256:ef9b85fa1bc91c4db24407e7c4da93a5822a73dd4513d67b454ca7064e8dc6a3 \ - --hash=sha256:f3bf60444269345d712838bb11cc4eadaf51ff1a364ae39ce87a5ca8ad3bb2c8 \ - --hash=sha256:f452cc1436151387d3d50533523291d5f77c6bc7913c116eb985304abdbd9ec9 \ - --hash=sha256:f7917697bcaa3bc3e83db91aa3a0e448bf5cde43c84b7fc1ae2427d2417c0224 \ - --hash=sha256:f90575e9fe3aae2c1e686393a9689c724cd00045275407f71771ae5d690ccf38 \ - --hash=sha256:fb382fd7b4377363cc9f13ba7c819c3c78ed97c36a82f16f3f92f108c787cbbf \ - --hash=sha256:fb9f59f3848edf186a76446eb8bcf4c900fe147cb756fbbd730ef43b2e67c6a7 \ - --hash=sha256:fc2931ac9ce9c61c9968989ec831d3a5e6fcaaff9474e7cfa8de80b7aff5a093 +yarl==1.15.2 \ + --hash=sha256:0545de8c688fbbf3088f9e8b801157923be4bf8e7b03e97c2ecd4dfa39e48e0e \ + --hash=sha256:076b1ed2ac819933895b1a000904f62d615fe4533a5cf3e052ff9a1da560575c \ + --hash=sha256:0afad2cd484908f472c8fe2e8ef499facee54a0a6978be0e0cff67b1254fd747 \ + --hash=sha256:0ccaa1bc98751fbfcf53dc8dfdb90d96e98838010fc254180dd6707a6e8bb179 \ + --hash=sha256:0d3105efab7c5c091609abacad33afff33bdff0035bece164c98bcf5a85ef90a \ + --hash=sha256:0e1af74a9529a1137c67c887ed9cde62cff53aa4d84a3adbec329f9ec47a3936 \ + --hash=sha256:136f9db0f53c0206db38b8cd0c985c78ded5fd596c9a86ce5c0b92afb91c3a19 \ + --hash=sha256:156ececdf636143f508770bf8a3a0498de64da5abd890c7dbb42ca9e3b6c05b8 \ + --hash=sha256:15c87339490100c63472a76d87fe7097a0835c705eb5ae79fd96e343473629ed \ + --hash=sha256:1695497bb2a02a6de60064c9f077a4ae9c25c73624e0d43e3aa9d16d983073c2 \ + --hash=sha256:173563f3696124372831007e3d4b9821746964a95968628f7075d9231ac6bb33 \ + --hash=sha256:173866d9f7409c0fb514cf6e78952e65816600cb888c68b37b41147349fe0057 \ + --hash=sha256:23ec1d3c31882b2a8a69c801ef58ebf7bae2553211ebbddf04235be275a38548 \ + --hash=sha256:243fbbbf003754fe41b5bdf10ce1e7f80bcc70732b5b54222c124d6b4c2ab31c \ + --hash=sha256:28c6cf1d92edf936ceedc7afa61b07e9d78a27b15244aa46bbcd534c7458ee1b \ + --hash=sha256:2aa738e0282be54eede1e3f36b81f1e46aee7ec7602aa563e81e0e8d7b67963f \ + --hash=sha256:2cf441c4b6e538ba0d2591574f95d3fdd33f1efafa864faa077d9636ecc0c4e9 \ + --hash=sha256:30c3ff305f6e06650a761c4393666f77384f1cc6c5c0251965d6bfa5fbc88f7f \ + --hash=sha256:31561a5b4d8dbef1559b3600b045607cf804bae040f64b5f5bca77da38084a8a \ + --hash=sha256:32b66be100ac5739065496c74c4b7f3015cef792c3174982809274d7e51b3e04 \ + --hash=sha256:3433da95b51a75692dcf6cc8117a31410447c75a9a8187888f02ad45c0a86c50 \ + --hash=sha256:34a2d76a1984cac04ff8b1bfc939ec9dc0914821264d4a9c8fd0ed6aa8d4cfd2 \ + --hash=sha256:353665775be69bbfc6d54c8d134bfc533e332149faeddd631b0bc79df0897f46 \ + --hash=sha256:38d0124fa992dbacd0c48b1b755d3ee0a9f924f427f95b0ef376556a24debf01 \ + --hash=sha256:3c56ec1eacd0a5d35b8a29f468659c47f4fe61b2cab948ca756c39b7617f0aa5 \ + --hash=sha256:3db817b4e95eb05c362e3b45dafe7144b18603e1211f4a5b36eb9522ecc62bcf \ + --hash=sha256:3e52474256a7db9dcf3c5f4ca0b300fdea6c21cca0148c8891d03a025649d935 \ + --hash=sha256:416f2e3beaeae81e2f7a45dc711258be5bdc79c940a9a270b266c0bec038fb84 \ + --hash=sha256:435aca062444a7f0c884861d2e3ea79883bd1cd19d0a381928b69ae1b85bc51d \ + --hash=sha256:4388c72174868884f76affcdd3656544c426407e0043c89b684d22fb265e04a5 \ + --hash=sha256:43ebdcc120e2ca679dba01a779333a8ea76b50547b55e812b8b92818d604662c \ + --hash=sha256:458c0c65802d816a6b955cf3603186de79e8fdb46d4f19abaec4ef0a906f50a7 \ + --hash=sha256:533a28754e7f7439f217550a497bb026c54072dbe16402b183fdbca2431935a9 \ + --hash=sha256:553dad9af802a9ad1a6525e7528152a015b85fb8dbf764ebfc755c695f488367 \ + --hash=sha256:5838f2b79dc8f96fdc44077c9e4e2e33d7089b10788464609df788eb97d03aad \ + --hash=sha256:5b48388ded01f6f2429a8c55012bdbd1c2a0c3735b3e73e221649e524c34a58d \ + --hash=sha256:5bc0df728e4def5e15a754521e8882ba5a5121bd6b5a3a0ff7efda5d6558ab3d \ + --hash=sha256:63eab904f8630aed5a68f2d0aeab565dcfc595dc1bf0b91b71d9ddd43dea3aea \ + --hash=sha256:66f629632220a4e7858b58e4857927dd01a850a4cef2fb4044c8662787165cf7 \ + --hash=sha256:670eb11325ed3a6209339974b276811867defe52f4188fe18dc49855774fa9cf \ + --hash=sha256:69d5856d526802cbda768d3e6246cd0d77450fa2a4bc2ea0ea14f0d972c2894b \ + --hash=sha256:6e840553c9c494a35e449a987ca2c4f8372668ee954a03a9a9685075228e5036 \ + --hash=sha256:711bdfae4e699a6d4f371137cbe9e740dc958530cb920eb6f43ff9551e17cfbc \ + --hash=sha256:74abb8709ea54cc483c4fb57fb17bb66f8e0f04438cff6ded322074dbd17c7ec \ + --hash=sha256:75119badf45f7183e10e348edff5a76a94dc19ba9287d94001ff05e81475967b \ + --hash=sha256:766dcc00b943c089349d4060b935c76281f6be225e39994c2ccec3a2a36ad627 \ + --hash=sha256:78e6fdc976ec966b99e4daa3812fac0274cc28cd2b24b0d92462e2e5ef90d368 \ + --hash=sha256:81dadafb3aa124f86dc267a2168f71bbd2bfb163663661ab0038f6e4b8edb810 \ + --hash=sha256:82d5161e8cb8f36ec778fd7ac4d740415d84030f5b9ef8fe4da54784a1f46c94 \ + --hash=sha256:833547179c31f9bec39b49601d282d6f0ea1633620701288934c5f66d88c3e50 \ + --hash=sha256:856b7f1a7b98a8c31823285786bd566cf06226ac4f38b3ef462f593c608a9bd6 \ + --hash=sha256:8657d3f37f781d987037f9cc20bbc8b40425fa14380c87da0cb8dfce7c92d0fb \ + --hash=sha256:93bed8a8084544c6efe8856c362af08a23e959340c87a95687fdbe9c9f280c8b \ + --hash=sha256:954dde77c404084c2544e572f342aef384240b3e434e06cecc71597e95fd1ce7 \ + --hash=sha256:98f68df80ec6ca3015186b2677c208c096d646ef37bbf8b49764ab4a38183931 \ + --hash=sha256:99e12d2bf587b44deb74e0d6170fec37adb489964dbca656ec41a7cd8f2ff178 \ + --hash=sha256:9a13a07532e8e1c4a5a3afff0ca4553da23409fad65def1b71186fb867eeae8d \ + --hash=sha256:9c1e3ff4b89cdd2e1a24c214f141e848b9e0451f08d7d4963cb4108d4d798f1f \ + --hash=sha256:9ce2e0f6123a60bd1a7f5ae3b2c49b240c12c132847f17aa990b841a417598a2 \ + --hash=sha256:9fcda20b2de7042cc35cf911702fa3d8311bd40055a14446c1e62403684afdc5 \ + --hash=sha256:a32d58f4b521bb98b2c0aa9da407f8bd57ca81f34362bcb090e4a79e9924fefc \ + --hash=sha256:a39c36f4218a5bb668b4f06874d676d35a035ee668e6e7e3538835c703634b84 \ + --hash=sha256:a5cafb02cf097a82d74403f7e0b6b9df3ffbfe8edf9415ea816314711764a27b \ + --hash=sha256:a7cf963a357c5f00cb55b1955df8bbe68d2f2f65de065160a1c26b85a1e44172 \ + --hash=sha256:a880372e2e5dbb9258a4e8ff43f13888039abb9dd6d515f28611c54361bc5644 \ + --hash=sha256:ace4cad790f3bf872c082366c9edd7f8f8f77afe3992b134cfc810332206884f \ + --hash=sha256:af8ff8d7dc07ce873f643de6dfbcd45dc3db2c87462e5c387267197f59e6d776 \ + --hash=sha256:b47a6000a7e833ebfe5886b56a31cb2ff12120b1efd4578a6fcc38df16cc77bd \ + --hash=sha256:b71862a652f50babab4a43a487f157d26b464b1dedbcc0afda02fd64f3809d04 \ + --hash=sha256:b7f227ca6db5a9fda0a2b935a2ea34a7267589ffc63c8045f0e4edb8d8dcf956 \ + --hash=sha256:bc8936d06cd53fddd4892677d65e98af514c8d78c79864f418bbf78a4a2edde4 \ + --hash=sha256:bed1b5dbf90bad3bfc19439258c97873eab453c71d8b6869c136346acfe497e7 \ + --hash=sha256:c45817e3e6972109d1a2c65091504a537e257bc3c885b4e78a95baa96df6a3f8 \ + --hash=sha256:c68e820879ff39992c7f148113b46efcd6ec765a4865581f2902b3c43a5f4bbb \ + --hash=sha256:c77494a2f2282d9bbbbcab7c227a4d1b4bb829875c96251f66fb5f3bae4fb053 \ + --hash=sha256:c998d0558805860503bc3a595994895ca0f7835e00668dadc673bbf7f5fbfcbe \ + --hash=sha256:ccad2800dfdff34392448c4bf834be124f10a5bc102f254521d931c1c53c455a \ + --hash=sha256:cd126498171f752dd85737ab1544329a4520c53eed3997f9b08aefbafb1cc53b \ + --hash=sha256:ce44217ad99ffad8027d2fde0269ae368c86db66ea0571c62a000798d69401fb \ + --hash=sha256:d1ac2bc069f4a458634c26b101c2341b18da85cb96afe0015990507efec2e417 \ + --hash=sha256:d417a4f6943112fae3924bae2af7112562285848d9bcee737fc4ff7cbd450e6c \ + --hash=sha256:d538df442c0d9665664ab6dd5fccd0110fa3b364914f9c85b3ef9b7b2e157980 \ + --hash=sha256:ded1b1803151dd0f20a8945508786d57c2f97a50289b16f2629f85433e546d47 \ + --hash=sha256:e2e93b88ecc8f74074012e18d679fb2e9c746f2a56f79cd5e2b1afcf2a8a786b \ + --hash=sha256:e4ca3b9f370f218cc2a0309542cab8d0acdfd66667e7c37d04d617012485f904 \ + --hash=sha256:e4ee8b8639070ff246ad3649294336b06db37a94bdea0d09ea491603e0be73b8 \ + --hash=sha256:e52f77a0cd246086afde8815039f3e16f8d2be51786c0a39b57104c563c5cbb0 \ + --hash=sha256:eaea112aed589131f73d50d570a6864728bd7c0c66ef6c9154ed7b59f24da611 \ + --hash=sha256:ed20a4bdc635f36cb19e630bfc644181dd075839b6fc84cac51c0f381ac472e2 \ + --hash=sha256:eedc3f247ee7b3808ea07205f3e7d7879bc19ad3e6222195cd5fbf9988853e4d \ + --hash=sha256:f0e1844ad47c7bd5d6fa784f1d4accc5f4168b48999303a868fe0f8597bde715 \ + --hash=sha256:f4fe99ce44128c71233d0d72152db31ca119711dfc5f2c82385ad611d8d7f897 \ + --hash=sha256:f8cfd847e6b9ecf9f2f2531c8427035f291ec286c0a4944b0a9fce58c6446046 \ + --hash=sha256:f9ca0e6ce7774dc7830dc0cc4bb6b3eec769db667f230e7c770a628c1aa5681b \ + --hash=sha256:fa2bea05ff0a8fb4d8124498e00e02398f06d23cdadd0fe027d84a3f7afde31e \ + --hash=sha256:fbbb63bed5fcd70cd3dd23a087cd78e4675fb5a2963b8af53f945cbbca79ae16 \ + --hash=sha256:fbda058a9a68bec347962595f50546a8a4a34fd7b0654a7b9697917dc2bf810d \ + --hash=sha256:ffd591e22b22f9cb48e472529db6a47203c41c2c5911ff0a52e85723196c0d75 # via # -r requirements.in # aiohttp From de4fd589d9b64b93446c369259bc1416d7e418f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:52:53 +0000 Subject: [PATCH 58/77] build(deps): bump icalendar from 6.0.0 to 6.0.1 in /tools/base (#36560) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index cbdbc10bce..bbec6c57a1 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -778,9 +778,9 @@ humanfriendly==10.0 \ --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc # via coloredlogs -icalendar==6.0.0 \ - --hash=sha256:567e718551d800362db04ca09777295336e1803f6fc6bc0a7a5e258917fa8ed0 \ - --hash=sha256:7ddf60d343f3c1f716de9b62f6e80ffd95d03cab62464894a0539feab7b5c76e +icalendar==6.0.1 \ + --hash=sha256:1ff44825d7b41c3f77eac9e09cc67a770dd3c2377430c23b0eb7d91603088892 \ + --hash=sha256:9bf3d69203bd0366a9a29a8b0e220574580b86d7918afcb628fc6920287922f3 # via -r requirements.in idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ From 706c4222c538f62ba4ab27421282003835da2a4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:56:19 +0100 Subject: [PATCH 59/77] build(deps): bump google.golang.org/protobuf from 1.34.2 to 1.35.1 in /contrib/golang/filters/http/test/test_data/access_log in the contrib-golang group (#36479) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- contrib/golang/filters/http/test/test_data/access_log/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/golang/filters/http/test/test_data/access_log/go.mod b/contrib/golang/filters/http/test/test_data/access_log/go.mod index b1e07cfaf9..4cf78bab42 100644 --- a/contrib/golang/filters/http/test/test_data/access_log/go.mod +++ b/contrib/golang/filters/http/test/test_data/access_log/go.mod @@ -4,6 +4,6 @@ go 1.22 require github.com/envoyproxy/envoy v1.24.0 -require google.golang.org/protobuf v1.34.2 +require google.golang.org/protobuf v1.35.1 replace github.com/envoyproxy/envoy => ../../../../../../../ From e742e85f33a796b05b6c1516b41fbe3bc7bd9d68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:56:39 +0100 Subject: [PATCH 60/77] build(deps): bump google.golang.org/protobuf from 1.34.2 to 1.35.1 in /contrib/golang/filters/http/test/test_data/echo in the contrib-golang group (#36481) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- contrib/golang/filters/http/test/test_data/echo/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/golang/filters/http/test/test_data/echo/go.mod b/contrib/golang/filters/http/test/test_data/echo/go.mod index d77fa5a01c..585f4386da 100644 --- a/contrib/golang/filters/http/test/test_data/echo/go.mod +++ b/contrib/golang/filters/http/test/test_data/echo/go.mod @@ -15,7 +15,7 @@ require ( require ( github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.1 ) replace github.com/envoyproxy/envoy => ../../../../../../../ From 906a491a13b0b0abe945500f56032acfbcffa355 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:56:55 +0100 Subject: [PATCH 61/77] build(deps): bump google.golang.org/protobuf from 1.34.2 to 1.35.1 in /contrib/golang/filters/http/test/test_data/metric in the contrib-golang group (#36485) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- contrib/golang/filters/http/test/test_data/metric/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/golang/filters/http/test/test_data/metric/go.mod b/contrib/golang/filters/http/test/test_data/metric/go.mod index 470e4da15e..fb8dc3f95b 100644 --- a/contrib/golang/filters/http/test/test_data/metric/go.mod +++ b/contrib/golang/filters/http/test/test_data/metric/go.mod @@ -4,6 +4,6 @@ go 1.22 require github.com/envoyproxy/envoy v1.24.0 -require google.golang.org/protobuf v1.34.2 +require google.golang.org/protobuf v1.35.1 replace github.com/envoyproxy/envoy => ../../../../../../../ From ddfc68846a2a581aa7b4fff2411cc5a5573c3407 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:57:10 +0100 Subject: [PATCH 62/77] build(deps): bump github/codeql-action from 3.26.11 to 3.26.12 (#36484) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 8a3e2ad85e..409b30eb4c 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -34,7 +34,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 + uses: github/codeql-action/init@c36620d31ac7c881962c3d9dd939c40ec9434f2b # codeql-bundle-v3.26.12 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -74,4 +74,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 + uses: github/codeql-action/analyze@c36620d31ac7c881962c3d9dd939c40ec9434f2b # codeql-bundle-v3.26.12 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index f336d11cc8..7c9fa2ebcc 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -65,7 +65,7 @@ jobs: - name: Initialize CodeQL if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 + uses: github/codeql-action/init@c36620d31ac7c881962c3d9dd939c40ec9434f2b # codeql-bundle-v3.26.12 with: languages: cpp @@ -109,4 +109,4 @@ jobs: - name: Perform CodeQL Analysis if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 + uses: github/codeql-action/analyze@c36620d31ac7c881962c3d9dd939c40ec9434f2b # codeql-bundle-v3.26.12 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 05c18a27ad..ef45128864 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -40,6 +40,6 @@ jobs: retention-days: 5 - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11 + uses: github/codeql-action/upload-sarif@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 with: sarif_file: results.sarif From b52271435f9d224be5e28665d80ada2e7e43f94e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:57:20 +0100 Subject: [PATCH 63/77] build(deps): bump actions/checkout from 4.2.0 to 4.2.1 (#36483) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/_precheck_deps.yml | 2 +- .github/workflows/codeql-daily.yml | 2 +- .github/workflows/codeql-push.yml | 2 +- .github/workflows/envoy-dependency.yml | 4 ++-- .github/workflows/mobile-release.yml | 2 +- .github/workflows/mobile-traffic_director.yml | 2 +- .github/workflows/pr_notifier.yml | 2 +- .github/workflows/scorecard.yml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_precheck_deps.yml b/.github/workflows/_precheck_deps.yml index 5254d1633a..1623b42b06 100644 --- a/.github/workflows/_precheck_deps.yml +++ b/.github/workflows/_precheck_deps.yml @@ -51,7 +51,7 @@ jobs: if: ${{ inputs.dependency-review }} steps: - name: Checkout Repository - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ fromJSON(inputs.request).request.sha }} persist-credentials: false diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 409b30eb4c..ae500fc245 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Free disk space uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.2.36 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 7c9fa2ebcc..92b612837b 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -32,7 +32,7 @@ jobs: if: github.repository == 'envoyproxy/envoy' steps: - name: Checkout repository - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: fetch-depth: 2 diff --git a/.github/workflows/envoy-dependency.yml b/.github/workflows/envoy-dependency.yml index 933dad4218..9d98995b7b 100644 --- a/.github/workflows/envoy-dependency.yml +++ b/.github/workflows/envoy-dependency.yml @@ -146,7 +146,7 @@ jobs: path: envoy fetch-depth: 0 token: ${{ steps.appauth.outputs.token }} - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 name: Checkout Envoy build tools repository with: repository: envoyproxy/envoy-build-tools @@ -238,7 +238,7 @@ jobs: issues: write steps: - name: Checkout repository - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Run dependency checker run: | TODAY_DATE=$(date -u -I"date") diff --git a/.github/workflows/mobile-release.yml b/.github/workflows/mobile-release.yml index 35d5f6ba29..4f6610eac5 100644 --- a/.github/workflows/mobile-release.yml +++ b/.github/workflows/mobile-release.yml @@ -86,7 +86,7 @@ jobs: include: - output: envoy steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: fetch-depth: 0 - name: Add safe directory diff --git a/.github/workflows/mobile-traffic_director.yml b/.github/workflows/mobile-traffic_director.yml index 6405c96b78..e3fc6d55be 100644 --- a/.github/workflows/mobile-traffic_director.yml +++ b/.github/workflows/mobile-traffic_director.yml @@ -30,7 +30,7 @@ jobs: timeout-minutes: 120 steps: - name: Checkout repository - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - name: 'Run GcpTrafficDirectorIntegrationTest' diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml index 9172af147e..2024bee400 100644 --- a/.github/workflows/pr_notifier.yml +++ b/.github/workflows/pr_notifier.yml @@ -22,7 +22,7 @@ jobs: || !contains(github.actor, '[bot]')) }} steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Notify about PRs run: | ARGS=() diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index ef45128864..987f7c2b62 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -21,7 +21,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: persist-credentials: false From a2210f405f24fdd2166e4508209e140b6ec26221 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:57:34 +0100 Subject: [PATCH 64/77] build(deps): bump google.golang.org/protobuf from 1.34.2 to 1.35.1 (#36487) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 396033361a..c03a700495 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,6 @@ module github.com/envoyproxy/envoy go 1.22 -require google.golang.org/protobuf v1.34.2 +require google.golang.org/protobuf v1.35.1 require github.com/google/go-cmp v0.5.9 // indirect diff --git a/go.sum b/go.sum index 2fc4d88980..a4b3f4ab14 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= From d1ec84ce95ad25f607da21c6af5be3f92360dabf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:57:44 +0100 Subject: [PATCH 65/77] build(deps): bump google.golang.org/protobuf from 1.34.2 to 1.35.1 in /contrib/golang/filters/http/test/test_data/routeconfig in the contrib-golang group (#36488) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- contrib/golang/filters/http/test/test_data/routeconfig/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/go.mod b/contrib/golang/filters/http/test/test_data/routeconfig/go.mod index 7e51fdbaf5..971599f7db 100644 --- a/contrib/golang/filters/http/test/test_data/routeconfig/go.mod +++ b/contrib/golang/filters/http/test/test_data/routeconfig/go.mod @@ -15,7 +15,7 @@ require ( require ( github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.1 ) replace github.com/envoyproxy/envoy => ../../../../../../../ From 5916b89e0dbb3f03532f155aee852b432eb41531 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:58:41 +0100 Subject: [PATCH 66/77] build(deps): bump google.golang.org/protobuf from 1.34.2 to 1.35.1 in /contrib/golang/router/cluster_specifier/test/test_data/simple in the contrib-golang group (#36480) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../router/cluster_specifier/test/test_data/simple/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod b/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod index 82f1b15caf..c173dd68bb 100644 --- a/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod +++ b/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod @@ -15,7 +15,7 @@ require ( require ( github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.1 ) replace github.com/envoyproxy/envoy => ../../../../../../../ From 844c7c873faed3a6c492e57390cbcd1344b8a2ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:12:30 +0000 Subject: [PATCH 67/77] build(deps): bump envoy-code-check from 0.5.13 to 0.5.14 in /tools/base (#36558) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index bbec6c57a1..946096ea22 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -57,9 +57,9 @@ aio-core==0.10.5 \ # envoy-github-abstract # envoy-github-release # envoy-gpg-identity -aio-run-checker==0.5.7 \ - --hash=sha256:19ce85bc48800c5e0049430346a92d5b2ae98dda95b12ae5332ba1b28e7450e8 \ - --hash=sha256:5b9c5296c824206aebb10ade175eb07e89497761da6749b6ba09507ce03f64af +aio-run-checker==0.5.8 \ + --hash=sha256:9bf5e20773f83113b2eee39277f0ec3e014aeecfd33deb44103b0ed2482d3ff5 \ + --hash=sha256:b5fcb91651da986b13eefd340a45b402100a2967e1eb1a1b941ccfbc36fe0eda # via # dependatool # envoy-code-check @@ -530,9 +530,9 @@ envoy-ci-report==0.0.3 \ --hash=sha256:1c4bf3cb9f92117eb482186ea4fa58ff15e26641cdaeeae45bb826ae8454859e \ --hash=sha256:cc59d0980330a4b73cfcaba2a1b5050348c1d6530baffeb354dccea3e46672d2 # via -r requirements.in -envoy-code-check==0.5.13 \ - --hash=sha256:58c31be3ba1a3273eec8a76d1dcfe1a3ae5eae4730ca9d70a85fec0d641846c4 \ - --hash=sha256:6c568d477642abdf7b41a0b6a5bb21fd480d92e500c53120837a01d4436d8591 +envoy-code-check==0.5.14 \ + --hash=sha256:56ed152ba633b8d6846509424a4dc94996ed0eb3d656495ec2b32a94144ed904 \ + --hash=sha256:83cce34e3c2ee3d1b9a375922ee0d08fedbd8d3621f72d3ee55dd0a7d26a0e0e # via -r requirements.in envoy-dependency-check==0.1.13 \ --hash=sha256:4337b9c4129ae723dc92f70733b167a8dde187368d873687c6c54732d6fb5e48 \ @@ -1500,7 +1500,9 @@ types-pytz==2024.2.0.20241003 \ types-pyyaml==6.0.12.20240917 \ --hash=sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570 \ --hash=sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587 - # via aio-core + # via + # aio-core + # envoy-code-check typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 From df48c7d10e72a18445c15396e84815add1757d8a Mon Sep 17 00:00:00 2001 From: Nandan B N Date: Mon, 14 Oct 2024 18:48:19 +0530 Subject: [PATCH 68/77] dynamic_forward_proxy: fix sub_cluster_confg stuck with warm up due to DFPCluster removed (#35848) Commit Message: - Add an optional argument called `ignore_removal` to `ClusterManagerImpl::addOrUpdateCluster` and `ClusterManagerImpl::loadCluster`. This argument defaults to `false`, so it won't affect any existing flows. We'll be setting this in Cluster Data. - If the `ignore_removal` value is `true`, then the cluster won't be removed when `ClusterManagerImpl::removeCluster` is called. - To remove a cluster that has been added with `ignore_removal` set to `true`, the `remove_ignored` argument must be set to `true` when `ClusterManagerImpl::removeCluster` is called. This helps to manage clusters whose lifecycle is managed by custom implementations similar to DFP clusters. Additional Description: - Currently clusters that are dynamically added with custom implementations from filters, etc. are removed when a CDS event is triggered. This is because these dynamically created clusters will come in diff of CDS and will be removed by the cluster manager. Risk Level: low Testing: unit test Docs Changes: no Release Notes: no --- changelogs/current.yaml | 4 + envoy/upstream/cluster_manager.h | 12 +- source/common/runtime/runtime_features.cc | 1 + .../common/upstream/cluster_manager_impl.cc | 27 ++-- source/common/upstream/cluster_manager_impl.h | 16 ++- .../clusters/dynamic_forward_proxy/cluster.cc | 8 +- .../dynamic_forward_proxy/proxy_filter.cc | 9 +- test/common/upstream/cds_api_impl_test.cc | 11 +- .../upstream/cluster_manager_impl_test.cc | 50 +++++++ .../proxy_filter_integration_test.cc | 123 ++++++++++++++++++ test/mocks/upstream/cluster_manager.h | 6 +- 11 files changed, 236 insertions(+), 31 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 2a24f8c539..6937b4594a 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -216,6 +216,10 @@ bug_fixes: - area: tracing change: | Fixed a bug where the OpenTelemetry tracer exports the OTLP request even when no spans are present. +- area: dynamic_forward_proxy + change: | + Fixed a bug where DFP sub-cluster gets removed due to CDS update and doesn't gets recreated. This behavior can be reverted by + setting the runtime guard ``envoy.reloadable_features.avoid_dfp_cluster_removal_on_cds_update`` to false. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index 2c62af7930..2649307a9e 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -245,10 +245,15 @@ class ClusterManager { * * @param cluster supplies the cluster configuration. * @param version_info supplies the xDS version of the cluster. + * @param avoid_cds_removal If set to true, the cluster will be ignored from removal during CDS + * update. It can be overridden by setting `remove_ignored` to true while + * calling removeCluster(). This is useful for clusters whose lifecycle + * is managed with custom implementation, e.g., DFP clusters. * @return true if the action results in an add/update of a cluster. */ virtual bool addOrUpdateCluster(const envoy::config::cluster::v3::Cluster& cluster, - const std::string& version_info) PURE; + const std::string& version_info, + const bool avoid_cds_removal = false) PURE; /** * Set a callback that will be invoked when all primary clusters have been initialized. @@ -331,10 +336,11 @@ class ClusterManager { * Remove a cluster via API. Only clusters added via addOrUpdateCluster() can * be removed in this manner. Statically defined clusters present when Envoy starts cannot be * removed. - * + * Cluster created using `addOrUpdateCluster()` with `avoid_cds_removal` set to true. + * can be removed by setting `remove_ignored` to true while removeCluster(). * @return true if the action results in the removal of a cluster. */ - virtual bool removeCluster(const std::string& cluster) PURE; + virtual bool removeCluster(const std::string& cluster, const bool remove_ignored = false) PURE; /** * Shutdown the cluster manager prior to destroying connection pools and other thread local data. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index a06e49b182..babfb11c2a 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -30,6 +30,7 @@ // ASAP by filing a bug on github. Overriding non-buggy code is strongly discouraged to avoid the // problem of the bugs being found after the old code path has been removed. RUNTIME_GUARD(envoy_reloadable_features_allow_alt_svc_for_ips); +RUNTIME_GUARD(envoy_reloadable_features_avoid_dfp_cluster_removal_on_cds_update); RUNTIME_GUARD(envoy_reloadable_features_boolean_to_string_fix); RUNTIME_GUARD(envoy_reloadable_features_check_switch_protocol_websocket_handshake); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index c65cdb5f15..367a99bab4 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -793,7 +793,8 @@ void ClusterManagerImpl::applyUpdates(ClusterManagerCluster& cluster, uint32_t p } bool ClusterManagerImpl::addOrUpdateCluster(const envoy::config::cluster::v3::Cluster& cluster, - const std::string& version_info) { + const std::string& version_info, + const bool avoid_cds_removal) { // First we need to see if this new config is new or an update to an existing dynamic cluster. // We don't allow updates to statically configured clusters in the main configuration. We check // both the warming clusters and the active clusters to see if we need an update or the update @@ -842,8 +843,9 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::config::cluster::v3::Cl init_helper_.state() == ClusterManagerInitHelper::State::AllClustersInitialized; // Preserve the previous cluster data to avoid early destroy. The same cluster should be added // before destroy to avoid early initialization complete. - auto status_or_cluster = loadCluster(cluster, new_hash, version_info, /*added_via_api=*/true, - /*required_for_ads=*/false, warming_clusters_); + auto status_or_cluster = + loadCluster(cluster, new_hash, version_info, /*added_via_api=*/true, + /*required_for_ads=*/false, warming_clusters_, avoid_cds_removal); THROW_IF_NOT_OK_REF(status_or_cluster.status()); const ClusterDataPtr previous_cluster = std::move(status_or_cluster.value()); auto& cluster_entry = warming_clusters_.at(cluster_name); @@ -877,11 +879,12 @@ void ClusterManagerImpl::clusterWarmingToActive(const std::string& cluster_name) warming_clusters_.erase(warming_it); } -bool ClusterManagerImpl::removeCluster(const std::string& cluster_name) { +bool ClusterManagerImpl::removeCluster(const std::string& cluster_name, const bool remove_ignored) { bool removed = false; auto existing_active_cluster = active_clusters_.find(cluster_name); if (existing_active_cluster != active_clusters_.end() && - existing_active_cluster->second->added_via_api_) { + existing_active_cluster->second->added_via_api_ && + (!existing_active_cluster->second->avoid_cds_removal_ || remove_ignored)) { removed = true; init_helper_.removeCluster(*existing_active_cluster->second); active_clusters_.erase(existing_active_cluster); @@ -909,7 +912,8 @@ bool ClusterManagerImpl::removeCluster(const std::string& cluster_name) { auto existing_warming_cluster = warming_clusters_.find(cluster_name); if (existing_warming_cluster != warming_clusters_.end() && - existing_warming_cluster->second->added_via_api_) { + existing_warming_cluster->second->added_via_api_ && + (!existing_warming_cluster->second->avoid_cds_removal_ || remove_ignored)) { removed = true; init_helper_.removeCluster(*existing_warming_cluster->second); warming_clusters_.erase(existing_warming_cluster); @@ -930,7 +934,7 @@ absl::StatusOr ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& cluster, const uint64_t cluster_hash, const std::string& version_info, bool added_via_api, const bool required_for_ads, - ClusterMap& cluster_map) { + ClusterMap& cluster_map, const bool avoid_cds_removal) { absl::StatusOr> new_cluster_pair_or_error = factory_.clusterFromProto(cluster, *this, outlier_event_logger_, added_via_api); @@ -994,15 +998,16 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust auto cluster_entry_it = cluster_map.find(cluster_info->name()); if (cluster_entry_it != cluster_map.end()) { result = std::exchange(cluster_entry_it->second, - std::make_unique(cluster, cluster_hash, version_info, - added_via_api, required_for_ads, - std::move(new_cluster), time_source_)); + std::make_unique( + cluster, cluster_hash, version_info, added_via_api, required_for_ads, + std::move(new_cluster), time_source_, avoid_cds_removal)); } else { bool inserted = false; std::tie(cluster_entry_it, inserted) = cluster_map.emplace( cluster_info->name(), std::make_unique(cluster, cluster_hash, version_info, added_via_api, - required_for_ads, std::move(new_cluster), time_source_)); + required_for_ads, std::move(new_cluster), time_source_, + avoid_cds_removal)); ASSERT(inserted); } diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 3d4bfa1656..fdaeb71129 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -253,7 +253,8 @@ class ClusterManagerImpl : public ClusterManager, // Upstream::ClusterManager bool addOrUpdateCluster(const envoy::config::cluster::v3::Cluster& cluster, - const std::string& version_info) override; + const std::string& version_info, + const bool avoid_cds_removal = false) override; void setPrimaryClustersInitializedCb(PrimaryClustersReadyCallback callback) override { init_helper_.setPrimaryClustersInitializedCb(callback); @@ -287,7 +288,7 @@ class ClusterManagerImpl : public ClusterManager, const ClusterSet& primaryClusters() override { return primary_clusters_; } ThreadLocalCluster* getThreadLocalCluster(absl::string_view cluster) override; - bool removeCluster(const std::string& cluster) override; + bool removeCluster(const std::string& cluster, const bool remove_ignored = false) override; void shutdown() override { shutdown_ = true; if (resume_cds_ != nullptr) { @@ -752,11 +753,12 @@ class ClusterManagerImpl : public ClusterManager, ClusterData(const envoy::config::cluster::v3::Cluster& cluster_config, const uint64_t cluster_config_hash, const std::string& version_info, bool added_via_api, bool required_for_ads, ClusterSharedPtr&& cluster, - TimeSource& time_source) + TimeSource& time_source, const bool avoid_cds_removal = false) : cluster_config_(cluster_config), config_hash_(cluster_config_hash), version_info_(version_info), cluster_(std::move(cluster)), - last_updated_(time_source.systemTime()), - added_via_api_(added_via_api), added_or_updated_{}, required_for_ads_(required_for_ads) {} + last_updated_(time_source.systemTime()), added_via_api_(added_via_api), + avoid_cds_removal_(avoid_cds_removal), added_or_updated_{}, + required_for_ads_(required_for_ads) {} bool blockUpdate(uint64_t hash) { return !added_via_api_ || config_hash_ == hash; } @@ -789,6 +791,7 @@ class ClusterManagerImpl : public ClusterManager, Common::CallbackHandlePtr priority_update_cb_; // Keep smaller fields near the end to reduce padding const bool added_via_api_ : 1; + const bool avoid_cds_removal_ : 1; bool added_or_updated_ : 1; const bool required_for_ads_ : 1; }; @@ -866,7 +869,8 @@ class ClusterManagerImpl : public ClusterManager, absl::StatusOr loadCluster(const envoy::config::cluster::v3::Cluster& cluster, const uint64_t cluster_hash, const std::string& version_info, bool added_via_api, - bool required_for_ads, ClusterMap& cluster_map); + bool required_for_ads, ClusterMap& cluster_map, + bool avoid_cds_removal = false); void onClusterInit(ClusterManagerCluster& cluster); void postThreadLocalHealthFailure(const HostSharedPtr& host); void updateClusterCounts(); diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index 4ec0a8298a..6962d2dd15 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -93,7 +93,9 @@ Cluster::~Cluster() { auto cluster_name = it->first; ENVOY_LOG(debug, "cluster='{}' removing from cluster_map & cluster manager", cluster_name); cluster_map_.erase(it++); - cm_.removeCluster(cluster_name); + cm_.removeCluster(cluster_name, + Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.avoid_dfp_cluster_removal_on_cds_update")); } } @@ -131,7 +133,9 @@ void Cluster::checkIdleSubCluster() { auto cluster_name = it->first; ENVOY_LOG(debug, "cluster='{}' removing from cluster_map & cluster manager", cluster_name); cluster_map_.erase(it++); - cm_.removeCluster(cluster_name); + cm_.removeCluster(cluster_name, + Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.avoid_dfp_cluster_removal_on_cds_update")); } else { ++it; } diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc index 030047f172..f3513483dc 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc @@ -87,7 +87,14 @@ LoadClusterEntryHandlePtr ProxyFilterConfig::addDynamicCluster( ENVOY_LOG(debug, "deliver dynamic cluster {} creation to main thread", cluster_name); main_thread_dispatcher_.post([this, cluster, version_info]() { ENVOY_LOG(debug, "initializing dynamic cluster {} creation in main thread", cluster.name()); - cluster_manager_.addOrUpdateCluster(cluster, version_info); + + // Set avoid_cds_removal to true to prevent the cluster from being removed during a CDS + // update. As this cluster lifecycle is managed by DFP cluster, it should not be removed by + // CDS. https://github.com/envoyproxy/envoy/issues/35171 + cluster_manager_.addOrUpdateCluster( + cluster, version_info, + Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.avoid_dfp_cluster_removal_on_cds_update")); }); } else { ENVOY_LOG(debug, "cluster='{}' already created, waiting it warming", cluster_name); diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index 0d31f950ad..f21f62425e 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -46,11 +46,12 @@ class CdsApiImplTest : public testing::Test { } void expectAdd(const std::string& cluster_name, const std::string& version = std::string("")) { - EXPECT_CALL(cm_, addOrUpdateCluster(WithName(cluster_name), version)).WillOnce(Return(true)); + EXPECT_CALL(cm_, addOrUpdateCluster(WithName(cluster_name), version, false)) + .WillOnce(Return(true)); } void expectAddToThrow(const std::string& cluster_name, const std::string& exception_msg) { - EXPECT_CALL(cm_, addOrUpdateCluster(WithName(cluster_name), _)) + EXPECT_CALL(cm_, addOrUpdateCluster(WithName(cluster_name), _, false)) .WillOnce(Throw(EnvoyException(exception_msg))); } @@ -115,7 +116,7 @@ version_info: '1' auto response2 = TestUtility::parseYaml(response2_yaml); EXPECT_CALL(cm_, clusters()).WillOnce(Return(makeClusterInfoMaps({"cluster1"}))); - EXPECT_CALL(cm_, removeCluster("cluster1")).WillOnce(Return(true)); + EXPECT_CALL(cm_, removeCluster("cluster1", false)).WillOnce(Return(true)); const auto decoded_resources_2 = TestUtility::decodeResources(response2); EXPECT_TRUE( @@ -217,7 +218,7 @@ TEST_F(CdsApiImplTest, DeltaConfigUpdate) { } Protobuf::RepeatedPtrField removed; *removed.Add() = "cluster_1"; - EXPECT_CALL(cm_, removeCluster(StrEq("cluster_1"))).WillOnce(Return(true)); + EXPECT_CALL(cm_, removeCluster(StrEq("cluster_1"), false)).WillOnce(Return(true)); const auto decoded_resources = TestUtility::decodeResources(resources); EXPECT_TRUE(cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, removed, "v2").ok()); @@ -312,7 +313,7 @@ version_info: '1' EXPECT_CALL(cm_, clusters()).WillOnce(Return(makeClusterInfoMaps({"cluster1", "cluster2"}))); expectAdd("cluster1", "1"); expectAdd("cluster3", "1"); - EXPECT_CALL(cm_, removeCluster("cluster2")); + EXPECT_CALL(cm_, removeCluster("cluster2", false)); const auto decoded_resources_2 = TestUtility::decodeResources(response2); EXPECT_TRUE( diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 8d9d2ca7b8..946db5738e 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -6585,6 +6585,56 @@ TEST_F(ClusterManagerImplTest, CheckActiveStaticCluster) { "gRPC client cluster 'added_via_api' is not static"); } +TEST_F(ClusterManagerImplTest, ClusterIgnoreRemoval) { + const std::string yaml = R"EOF( + static_resources: + clusters: + - name: good + connect_timeout: 0.250s + lb_policy: ROUND_ROBIN + type: STATIC + load_assignment: + cluster_name: good + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + create(parseBootstrapFromV3Yaml(yaml)); + const std::string added_via_api_yaml = R"EOF( + name: added_via_api + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: added_via_api + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + auto cluster = parseClusterFromV3Yaml(added_via_api_yaml); + + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(cluster, "v1", true)); + + EXPECT_EQ(2, cluster_manager_->clusters().active_clusters_.size()); + EXPECT_TRUE(cluster_manager_->checkActiveStaticCluster("good").ok()); + + // This should not remove the cluster as remove_ignored is set to false + EXPECT_FALSE(cluster_manager_->removeCluster("added_via_api")); + EXPECT_EQ(2, cluster_manager_->clusters().active_clusters_.size()); + + // This should remove the cluster as remove_ignored is set to true + EXPECT_TRUE(cluster_manager_->removeCluster("added_via_api", true)); + EXPECT_EQ(1, cluster_manager_->clusters().active_clusters_.size()); +} + #ifdef WIN32 TEST_F(ClusterManagerImplTest, LocalInterfaceNameForUpstreamConnectionThrowsInWin32) { const std::string yaml = fmt::format(R"EOF( diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 81d4bcee54..be55dddd3d 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -7,6 +7,7 @@ #include "source/common/tls/context_config_impl.h" #include "source/common/tls/ssl_socket.h" +#include "test/common/upstream/utility.h" #include "test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h" #include "test/integration/http_integration.h" #include "test/integration/ssl_utility.h" @@ -1294,6 +1295,68 @@ TEST_P(ProxyFilterIntegrationTest, SubClusterReloadCluster) { test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); } +// Verify that we expire sub clusters and not remove on CDS. +TEST_P(ProxyFilterWithSimtimeIntegrationTest, RemoveViaTTLAndDFPUpdateWithoutAvoidCDSRemoval) { + const std::string cluster_yaml = R"EOF( + name: fake_cluster + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: fake_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + auto cluster = Upstream::parseClusterFromV3Yaml(cluster_yaml); + // make runtime guard false + config_helper_.addRuntimeOverride( + "envoy.reloadable_features.avoid_dfp_cluster_removal_on_cds_update", "false"); + initializeWithArgs(1024, 1024, "", "", true); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + // one more cluster + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 0); + cleanupUpstreamAndDownstream(); + + // Sub cluster expected to be removed after ttl + // > 5m + simTime().advanceTimeWait(std::chrono::milliseconds(300001)); + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 1); + + codec_client_ = makeHttpConnection(lookupPort("http")); + response = sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + + // sub cluster added again + test_server_->waitForCounterEq("cluster_manager.cluster_added", 3); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 1); + cleanupUpstreamAndDownstream(); + + // Make update to DFP cluster + cluster_.mutable_circuit_breakers()->add_thresholds()->mutable_max_connections()->set_value(100); + cds_helper_.setCds({cluster_}); + + // sub cluster removed due to dfp cluster update + test_server_->waitForCounterEq("cluster_manager.cluster_added", 3); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 2); +} + // Verify that we expire sub clusters. TEST_P(ProxyFilterWithSimtimeIntegrationTest, RemoveSubClusterViaTTL) { initializeWithArgs(1024, 1024, "", "", true); @@ -1370,5 +1433,65 @@ TEST_P(ProxyFilterIntegrationTest, SubClusterWithIpHost) { checkSimpleRequestSuccess(0, 0, response.get()); } +// Verify that no DFP clusters are removed when CDS Reload is triggered. +TEST_P(ProxyFilterIntegrationTest, CDSReloadNotRemoveDFPCluster) { + config_helper_.addRuntimeOverride( + "envoy.reloadable_features.avoid_dfp_cluster_removal_on_cds_update", "true"); + const std::string cluster_yaml = R"EOF( + name: fake_cluster + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: fake_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + auto cluster = Upstream::parseClusterFromV3Yaml(cluster_yaml); + + initializeWithArgs(1024, 1024, "", "", true); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + // one more sub cluster + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + test_server_->waitForCounterEq("cluster_manager.cluster_modified", 0); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 0); + + // Cause a cluster reload via CDS. + cds_helper_.setCds({cluster_, cluster}); + // a new cluster is added and no dfp cluster is removed + test_server_->waitForCounterEq("cluster_manager.cluster_added", 3); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + test_server_->waitForCounterEq("cluster_manager.cluster_modified", 0); + // No DFP cluster should be removed. + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 0); + + // The fake upstream connection should stay connected + ASSERT_TRUE(fake_upstream_connection_->connected()); + + // Now send another request. This should not create new sub cluster. + response = sendRequestAndWaitForResponse(request_headers, 512, default_response_headers_, 512); + checkSimpleRequestSuccess(512, 512, response.get()); + + // No new cluster should be added as DFP cluster already exists + test_server_->waitForCounterEq("cluster_manager.cluster_added", 3); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); +} + } // namespace } // namespace Envoy diff --git a/test/mocks/upstream/cluster_manager.h b/test/mocks/upstream/cluster_manager.h index 05c58a99fb..18eea3e87d 100644 --- a/test/mocks/upstream/cluster_manager.h +++ b/test/mocks/upstream/cluster_manager.h @@ -38,8 +38,8 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD(absl::Status, initialize, (const envoy::config::bootstrap::v3::Bootstrap& bootstrap)); MOCK_METHOD(bool, initialized, ()); MOCK_METHOD(bool, addOrUpdateCluster, - (const envoy::config::cluster::v3::Cluster& cluster, - const std::string& version_info)); + (const envoy::config::cluster::v3::Cluster& cluster, const std::string& version_info, + const bool avoid_cds_removal)); MOCK_METHOD(void, setPrimaryClustersInitializedCb, (PrimaryClustersReadyCallback)); MOCK_METHOD(void, setInitializedCb, (InitializationCompleteCallback)); MOCK_METHOD(absl::Status, initializeSecondaryClusters, @@ -48,7 +48,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD(const ClusterSet&, primaryClusters, ()); MOCK_METHOD(ThreadLocalCluster*, getThreadLocalCluster, (absl::string_view cluster)); - MOCK_METHOD(bool, removeCluster, (const std::string& cluster)); + MOCK_METHOD(bool, removeCluster, (const std::string& cluster, const bool remove_ignored)); MOCK_METHOD(void, shutdown, ()); MOCK_METHOD(bool, isShutdown, ()); MOCK_METHOD(const absl::optional&, bindConfig, (), (const)); From 2898e4dc52e8d4ec94c34470974dbaafdae8bfe0 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 14 Oct 2024 09:22:10 -0400 Subject: [PATCH 69/77] map matcher: removing exceptions (#36514) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a https://github.com/envoyproxy/envoy-mobile/issues/176 Signed-off-by: Alyssa Wilk --- source/common/matcher/exact_map_matcher.h | 17 +++++- source/common/matcher/map_matcher.h | 22 +++---- source/common/matcher/matcher.h | 17 ++++-- source/common/matcher/prefix_map_matcher.h | 15 ++++- test/common/matcher/exact_map_matcher_test.cc | 41 ++++++------- .../common/matcher/prefix_map_matcher_test.cc | 57 ++++++++++--------- .../http/match_delegate/config_test.cc | 6 +- tools/code_format/config.yaml | 1 - 8 files changed, 103 insertions(+), 73 deletions(-) diff --git a/source/common/matcher/exact_map_matcher.h b/source/common/matcher/exact_map_matcher.h index ecd56113be..a2e09dfbff 100644 --- a/source/common/matcher/exact_map_matcher.h +++ b/source/common/matcher/exact_map_matcher.h @@ -11,9 +11,14 @@ namespace Matcher { */ template class ExactMapMatcher : public MapMatcher { public: - ExactMapMatcher(DataInputPtr&& data_input, - absl::optional> on_no_match) - : MapMatcher(std::move(data_input), std::move(on_no_match)) {} + static absl::StatusOr> + create(DataInputPtr&& data_input, absl::optional> on_no_match) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr>( + new ExactMapMatcher(std::move(data_input), on_no_match, creation_status)); + RETURN_IF_NOT_OK_REF(creation_status); + return ret; + } void addChild(std::string value, OnMatch&& on_match) override { const auto itr_and_exists = children_.emplace(value, std::move(on_match)); @@ -21,6 +26,12 @@ template class ExactMapMatcher : public MapMatcher { } protected: + template friend class MatchTreeFactory; + + ExactMapMatcher(DataInputPtr&& data_input, + absl::optional> on_no_match, absl::Status& creation_status) + : MapMatcher(std::move(data_input), std::move(on_no_match), creation_status) {} + absl::optional> doMatch(const std::string& data) override { const auto itr = children_.find(data); if (itr != children_.end()) { diff --git a/source/common/matcher/map_matcher.h b/source/common/matcher/map_matcher.h index 1e2c73bfd6..828b74b07d 100644 --- a/source/common/matcher/map_matcher.h +++ b/source/common/matcher/map_matcher.h @@ -16,16 +16,6 @@ namespace Matcher { template class MapMatcher : public MatchTree, Logger::Loggable { public: - MapMatcher(DataInputPtr&& data_input, absl::optional> on_no_match) - : data_input_(std::move(data_input)), on_no_match_(std::move(on_no_match)) { - auto input_type = data_input_->dataInputType(); - if (input_type != DefaultMatchingDataType) { - throwEnvoyExceptionOrPanic( - absl::StrCat("Unsupported data input type: ", input_type, - ", currently only string type is supported in map matcher")); - } - } - // Adds a child to the map. virtual void addChild(std::string value, OnMatch&& on_match) PURE; @@ -59,6 +49,18 @@ class MapMatcher : public MatchTree, Logger::Loggable friend class MatchTreeFactory; + MapMatcher(DataInputPtr&& data_input, absl::optional> on_no_match, + absl::Status& creation_status) + : data_input_(std::move(data_input)), on_no_match_(std::move(on_no_match)) { + auto input_type = data_input_->dataInputType(); + if (input_type != DefaultMatchingDataType) { + creation_status = absl::InvalidArgumentError( + absl::StrCat("Unsupported data input type: ", input_type, + ", currently only string type is supported in map matcher")); + } + } + const DataInputPtr data_input_; const absl::optional> on_no_match_; diff --git a/source/common/matcher/matcher.h b/source/common/matcher/matcher.h index f2174062fc..00332c631f 100644 --- a/source/common/matcher/matcher.h +++ b/source/common/matcher/matcher.h @@ -273,11 +273,12 @@ class MatchTreeFactory : public OnMatchFactory { switch (matcher.matcher_tree().tree_type_case()) { case MatcherType::MatcherTree::kExactMatchMap: { return createMapMatcher(matcher.matcher_tree().exact_match_map(), data_input, - on_no_match); + on_no_match, &ExactMapMatcher::create); } case MatcherType::MatcherTree::kPrefixMatchMap: { return createMapMatcher(matcher.matcher_tree().prefix_match_map(), - data_input, on_no_match); + data_input, on_no_match, + &PrefixMapMatcher::create); } case MatcherType::MatcherTree::TREE_TYPE_NOT_SET: PANIC("unexpected matcher type"); @@ -294,10 +295,14 @@ class MatchTreeFactory : public OnMatchFactory { PANIC_DUE_TO_CORRUPT_ENUM; } + using MapCreationFunction = std::function>>( + DataInputPtr&& data_input, absl::optional> on_no_match)>; + template