Backport flaky test and tsan fixes to releases/v1.15 branch (#13337)

* hds: fix integration test flakes (#12214)

Part of #12184

Signed-off-by: Matt Klein <mklein@lyft.com>
Signed-off-by: Antonio Vicente <avd@google.com>

* Switch to a tsan-instrumented libc++ for tsan tests (#12134)

This fixes #9784 and re-enables vhds_integration_test

Risk Level: Low, but will most likely increase memory usage

Signed-off-by: Dmitri Dolguikh <ddolguik@redhat.com>

Signed-off-by: Antonio Vicente <avd@google.com>

* test: shard hds_integration_test (#12482)

This should avoid TSAN timeout flakes.

Signed-off-by: Matt Klein <mklein@lyft.com>
Signed-off-by: Antonio Vicente <avd@google.com>

* test: shard http2_integration_test (#11939)

This should mitigate TSAN timeout.

Signed-off-by: Lizan Zhou <lizan@tetrate.io>
Signed-off-by: Antonio Vicente <avd@google.com>

* test: fix http2_integration_test flake (#12450)

Fixes #12442

Signed-off-by: Matt Klein <mklein@lyft.com>
Signed-off-by: Antonio Vicente <avd@google.com>

* Kick CI

Signed-off-by: Antonio Vicente <avd@google.com>

Co-authored-by: Matt Klein <mklein@lyft.com>
Co-authored-by: Dmitri Dolguikh <ddolguik@redhat.com>
Co-authored-by: Lizan Zhou <lizan@tetrate.io>

.bazelrc

@@ -155,6 +155,10 @@ build:rbe-toolchain-msan --linkopt=-L/opt/libcxx_msan/lib
 build:rbe-toolchain-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
 build:rbe-toolchain-msan --config=clang-msan
 
+build:rbe-toolchain-tsan --linkopt=-L/opt/libcxx_tsan/lib
+build:rbe-toolchain-tsan --linkopt=-Wl,-rpath,/opt/libcxx_tsan/lib
+build:rbe-toolchain-tsan --config=clang-tsan
+
 build:rbe-toolchain-gcc --config=rbe-toolchain
 build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain
 build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain
@@ -221,6 +225,10 @@ build:docker-msan --config=docker-sandbox
 build:docker-msan --config=rbe-toolchain-clang-libc++
 build:docker-msan --config=rbe-toolchain-msan
 
+build:docker-tsan --config=docker-sandbox
+build:docker-tsan --config=rbe-toolchain-clang-libc++
+build:docker-tsan --config=rbe-toolchain-tsan
+
 # CI configurations
 build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com

ci/do_ci.sh

@@ -185,7 +185,7 @@ elif [[ "$CI_TARGET" == "bazel.tsan" ]]; then
   setup_clang_toolchain
   echo "bazel TSAN debug build with tests"
   echo "Building and testing envoy tests ${TEST_TARGETS}"
-  bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-tsan --build_tests_only ${TEST_TARGETS}
+  bazel_with_collection test --config=rbe-toolchain-tsan ${BAZEL_BUILD_OPTIONS} -c dbg --build_tests_only ${TEST_TARGETS}
   if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then
     echo "Building and testing envoy-filter-example tests..."
     pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}"

source/common/filesystem/posix/filesystem_impl.cc

@@ -96,8 +96,6 @@ std::string InstanceImplPosix::fileReadToEnd(const std::string& path) {
     throw EnvoyException(absl::StrCat("Invalid path: ", path));
   }
 
-  std::ios::sync_with_stdio(false);
-
   std::ifstream file(path);
   if (file.fail()) {
     throw EnvoyException(absl::StrCat("unable to read file: ", path));

test/integration/BUILD

@@ -216,7 +216,9 @@ envoy_cc_test(
     data = [
         "//test/config/integration/certs",
     ],
-    tags = ["fails_on_windows"],
+    tags = [
+        "fails_on_windows",
+    ],
     deps = [
         ":http_integration_lib",
         "//source/common/config:protobuf_link_hacks",
@@ -329,6 +331,7 @@ envoy_cc_test(
         "http2_integration_test.cc",
         "http2_integration_test.h",
     ],
+    shard_count = 4,
     tags = ["fails_on_windows"],
     deps = [
         ":http_integration_lib",
@@ -779,6 +782,7 @@ envoy_cc_test(
 envoy_cc_test(
     name = "hds_integration_test",
     srcs = ["hds_integration_test.cc"],
+    shard_count = 2,
     tags = ["fails_on_windows"],
     deps = [
         ":http_integration_lib",

test/integration/hds_integration_test.cc

@@ -61,8 +61,10 @@ class HdsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest,
     // Endpoint connections
     host_upstream_ =
         std::make_unique<FakeUpstream>(0, FakeHttpConnection::Type::HTTP1, version_, timeSystem());
+    host_upstream_->set_allow_unexpected_disconnects(true);
     host2_upstream_ =
         std::make_unique<FakeUpstream>(0, FakeHttpConnection::Type::HTTP1, version_, timeSystem());
+    host2_upstream_->set_allow_unexpected_disconnects(true);
   }
 
   // Sets up a connection between Envoy and the management server.
@@ -80,7 +82,6 @@ class HdsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest,
     ASSERT_TRUE(host_fake_connection_->waitForNewStream(*dispatcher_, host_stream_));
     ASSERT_TRUE(host_stream_->waitForEndStream(*dispatcher_));
 
-    host_upstream_->set_allow_unexpected_disconnects(true);
     EXPECT_EQ(host_stream_->headers().getPathValue(), "/healthcheck");
     EXPECT_EQ(host_stream_->headers().getMethodValue(), "GET");
     EXPECT_EQ(host_stream_->headers().getHostValue(), "anna");
@@ -90,7 +91,6 @@ class HdsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationParamTest,
       ASSERT_TRUE(host2_fake_connection_->waitForNewStream(*dispatcher_, host2_stream_));
       ASSERT_TRUE(host2_stream_->waitForEndStream(*dispatcher_));
 
-      host2_upstream_->set_allow_unexpected_disconnects(true);
       EXPECT_EQ(host2_stream_->headers().getPathValue(), "/healthcheck");
       EXPECT_EQ(host2_stream_->headers().getMethodValue(), "GET");
       EXPECT_EQ(host2_stream_->headers().getHostValue(), cluster2);
@@ -314,9 +314,10 @@ TEST_P(HdsIntegrationTest, SingleEndpointTimeoutHttp) {
   test_server_->waitForCounterGe("hds_delegate.requests", ++hds_requests_);
 
   // Envoy sends a health check message to an endpoint
-  healthcheckEndpoints();
+  ASSERT_TRUE(host_upstream_->waitForRawConnection(host_fake_raw_connection_));
 
   // Endpoint doesn't respond to the health check
+  ASSERT_TRUE(host_fake_raw_connection_->waitForDisconnect(true));
 
   // Receive updates until the one we expect arrives
   waitForEndpointHealthResponse(envoy::config::core::v3::TIMEOUT);
@@ -380,19 +381,17 @@ TEST_P(HdsIntegrationTest, SingleEndpointTimeoutTcp) {
   server_health_check_specifier_.mutable_cluster_health_checks(0)
       ->mutable_health_checks(0)
       ->mutable_timeout()
-      ->set_nanos(500000000); // 0.5 seconds
+      ->set_nanos(100000000); // 0.1 seconds
 
   hds_stream_->startGrpcStream();
   hds_stream_->sendGrpcMessage(server_health_check_specifier_);
   test_server_->waitForCounterGe("hds_delegate.requests", ++hds_requests_);
 
   // Envoys asks the endpoint if it's healthy
-  host_upstream_->set_allow_unexpected_disconnects(true);
   ASSERT_TRUE(host_upstream_->waitForRawConnection(host_fake_raw_connection_));
-  ASSERT_TRUE(
-      host_fake_raw_connection_->waitForData(FakeRawConnection::waitForInexactMatch("Ping")));
 
   // No response from the endpoint
+  ASSERT_TRUE(host_fake_raw_connection_->waitForDisconnect(true));
 
   // Receive updates until the one we expect arrives
   waitForEndpointHealthResponse(envoy::config::core::v3::TIMEOUT);
@@ -418,7 +417,6 @@ TEST_P(HdsIntegrationTest, SingleEndpointHealthyTcp) {
   test_server_->waitForCounterGe("hds_delegate.requests", ++hds_requests_);
 
   // Envoy asks the endpoint if it's healthy
-  host_upstream_->set_allow_unexpected_disconnects(true);
   ASSERT_TRUE(host_upstream_->waitForRawConnection(host_fake_raw_connection_));
   ASSERT_TRUE(
       host_fake_raw_connection_->waitForData(FakeRawConnection::waitForInexactMatch("Ping")));
@@ -453,7 +451,6 @@ TEST_P(HdsIntegrationTest, SingleEndpointUnhealthyTcp) {
   test_server_->waitForCounterGe("hds_delegate.requests", ++hds_requests_);
 
   // Envoy asks the endpoint if it's healthy
-  host_upstream_->set_allow_unexpected_disconnects(true);
   ASSERT_TRUE(host_upstream_->waitForRawConnection(host_fake_raw_connection_));
   ASSERT_TRUE(
       host_fake_raw_connection_->waitForData(FakeRawConnection::waitForInexactMatch("Ping")));
@@ -691,7 +688,6 @@ TEST_P(HdsIntegrationTest, TestUpdateMessage) {
   test_server_->waitForCounterGe("hds_delegate.requests", ++hds_requests_);
 
   // Envoy sends a health check message to an endpoint
-  host2_upstream_->set_allow_unexpected_disconnects(true);
   ASSERT_TRUE(host2_upstream_->waitForHttpConnection(*dispatcher_, host2_fake_connection_));
   ASSERT_TRUE(host2_fake_connection_->waitForNewStream(*dispatcher_, host2_stream_));
   ASSERT_TRUE(host2_stream_->waitForEndStream(*dispatcher_));

test/integration/http2_integration_test.cc

@@ -829,6 +829,7 @@ TEST_P(Http2IntegrationTest, GrpcRetry) { testGrpcRetry(); }
 // Verify the case where there is an HTTP/2 codec/protocol error with an active stream.
 TEST_P(Http2IntegrationTest, CodecErrorAfterStreamStart) {
   initialize();
+  fake_upstreams_[0]->set_allow_unexpected_disconnects(true);
   codec_client_ = makeHttpConnection(lookupPort("http"));
 
   // Sends a request.