build: link C++ stdlib dynamically in sanitizer runs (#8019)

Description:
Sanitizers doesn't support static link, reverts #7929 and link lib(std)c++ dynamically in sanitizer runs. Addresses test issue for #4251. Added workaround in ASAN for #7647.

Risk Level: Low (test only)
Testing: CI, local libc++ runs
Docs Changes: N/A
Release Notes: N/A
Fixes #7928

.bazelrc

@@ -16,7 +16,7 @@ build --experimental_local_memory_estimate
 build --experimental_strict_action_env=true
 build --host_force_python=PY2
 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a
-build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc
+build --action_env=BAZEL_LINKOPTS=-lm
 build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11
 build --javabase=@bazel_tools//tools/jdk:remote_jdk11
 
@@ -28,19 +28,21 @@ build --action_env=CC
 build --action_env=CXX
 build --action_env=PATH
 
+# Common flags for sanitizers
+build:sanitizer --define tcmalloc=disabled
+build:sanitizer --linkopt -ldl
+build:sanitizer --build_tag_filters=-no_san
+build:sanitizer --test_tag_filters=-no_san
+
 # Basic ASAN/UBSAN that works for gcc
-build:asan --action_env=BAZEL_LINKLIBS=
-build:asan --action_env=BAZEL_LINKOPTS=-lstdc++:-lm
+build:asan --config=sanitizer
+# ASAN install its signal handler, disable ours so the stacktrace will be printed by ASAN
+build:asan --define signal_trace=disabled
 build:asan --define ENVOY_CONFIG_ASAN=1
 build:asan --copt -fsanitize=address,undefined
 build:asan --linkopt -fsanitize=address,undefined
 build:asan --copt -fno-sanitize=vptr
 build:asan --linkopt -fno-sanitize=vptr
-build:asan --linkopt -ldl
-build:asan --define tcmalloc=disabled
-build:asan --build_tag_filters=-no_asan
-build:asan --test_tag_filters=-no_asan
-build:asan --define signal_trace=disabled
 build:asan --copt -DADDRESS_SANITIZER=1
 build:asan --copt -D__SANITIZE_ADDRESS__
 build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1
@@ -62,21 +64,20 @@ build:macos-asan --copt -DGRPC_BAZEL_BUILD
 build:macos-asan --dynamic_mode=off
 
 # Clang TSAN
+build:clang-tsan --config=sanitizer
 build:clang-tsan --define ENVOY_CONFIG_TSAN=1
 build:clang-tsan --copt -fsanitize=thread
 build:clang-tsan --linkopt -fsanitize=thread
 build:clang-tsan --linkopt -fuse-ld=lld
-build:clang-tsan --linkopt -static-libsan
-build:clang-tsan --define tcmalloc=disabled
 # Needed due to https://github.com/libevent/libevent/issues/777
 build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE
 
 # Clang MSAN - broken today since we need to rebuild lib[std]c++ and external deps with MSAN
 # support (see https://github.com/envoyproxy/envoy/issues/443).
+build:clang-msan --config=sanitizer
 build:clang-msan --define ENVOY_CONFIG_MSAN=1
 build:clang-msan --copt -fsanitize=memory
 build:clang-msan --linkopt -fsanitize=memory
-build:clang-msan --define tcmalloc=disabled
 build:clang-msan --copt -fsanitize-memory-track-origins=2
 
 # Clang with libc++
@@ -108,6 +109,7 @@ build:rbe-toolchain-clang-libc++ --config=rbe-toolchain
 build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain
 build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain
 build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled
 
 build:rbe-toolchain-gcc --config=rbe-toolchain
 build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain

bazel/BUILD

@@ -1,11 +1,14 @@
 licenses(["notice"])  # Apache 2
 
-package(default_visibility = ["//visibility:public"])
+load("//bazel:envoy_build_system.bzl", "envoy_package")
+
+envoy_package()
+
+load("//bazel:envoy_internal.bzl", "envoy_select_force_libcpp")
 
 exports_files([
     "gen_sh_test_runner.sh",
     "sh_test_wrapper.sh",
-    "cc_wrapper.py",
 ])
 
 genrule(
@@ -37,6 +40,25 @@ genrule(
     stamp = 1,
 )
 
+# A target to optionally link C++ standard library dynamically in sanitizer runs.
+# TSAN doesn't support libc/libstdc++ static linking per doc:
+#   http://releases.llvm.org/8.0.1/tools/clang/docs/ThreadSanitizer.html
+cc_library(
+    name = "dynamic_stdlib",
+    linkopts = envoy_select_force_libcpp(
+        ["-lc++"],
+        ["-lstdc++"],
+    ),
+)
+
+cc_library(
+    name = "static_stdlib",
+    linkopts = select({
+        "//bazel:linux": ["-static-libgcc"],
+        "//conditions:default": [],
+    }),
+)
+
 config_setting(
     name = "windows_opt_build",
     values = {
@@ -81,6 +103,11 @@ config_setting(
     values = {"define": "ENVOY_CONFIG_ASAN=1"},
 )
 
+config_setting(
+    name = "tsan_build",
+    values = {"define": "ENVOY_CONFIG_TSAN=1"},
+)
+
 config_setting(
     name = "coverage_build",
     values = {"define": "ENVOY_CONFIG_COVERAGE=1"},

bazel/envoy_binary.bzl

@@ -4,6 +4,7 @@ load(
     ":envoy_internal.bzl",
     "envoy_copts",
     "envoy_external_dep_path",
+    "envoy_stdlib_deps",
     "tcmalloc_external_dep",
 )
 
@@ -24,7 +25,7 @@ def envoy_cc_binary(
     if stamped:
         linkopts = linkopts + _envoy_stamped_linkopts()
         deps = deps + _envoy_stamped_deps()
-    deps = deps + [envoy_external_dep_path(dep) for dep in external_deps]
+    deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + envoy_stdlib_deps()
     native.cc_binary(
         name = name,
         srcs = srcs,

bazel/envoy_internal.bzl

@@ -89,6 +89,13 @@ def envoy_select_force_libcpp(if_libcpp, default = None):
         "//conditions:default": default or [],
     })
 
+def envoy_stdlib_deps():
+    return select({
+        "@envoy//bazel:asan_build": ["@envoy//bazel:dynamic_stdlib"],
+        "@envoy//bazel:tsan_build": ["@envoy//bazel:dynamic_stdlib"],
+        "//conditions:default": ["@envoy//bazel:static_stdlib"],
+    })
+
 # Dependencies on tcmalloc_and_profiler should be wrapped with this function.
 def tcmalloc_external_dep(repository):
     return select({

bazel/envoy_test.bzl

@@ -8,6 +8,7 @@ load(
     "envoy_external_dep_path",
     "envoy_linkstatic",
     "envoy_select_force_libcpp",
+    "envoy_stdlib_deps",
     "tcmalloc_external_dep",
 )
 
@@ -80,7 +81,7 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs):
     test_lib_name = name + "_lib"
     envoy_cc_test_library(
         name = test_lib_name,
-        deps = deps + ["//test/fuzz:fuzz_runner_lib"],
+        deps = deps + ["//test/fuzz:fuzz_runner_lib", "//bazel:dynamic_stdlib"],
         **kwargs
     )
     native.cc_test(
@@ -163,7 +164,7 @@ def envoy_cc_test(
         linkopts = _envoy_test_linkopts(),
         linkstatic = envoy_linkstatic(),
         malloc = tcmalloc_external_dep(repository),
-        deps = [
+        deps = envoy_stdlib_deps() + [
             ":" + name + "_lib_internal_only",
             repository + "//test:main",
         ],

bazel/io_opentracing_cpp.patch

@@ -0,0 +1,17 @@
+diff --git a/src/dynamic_load_unix.cpp b/src/dynamic_load_unix.cpp
+index 17e08fd..d25e0c8 100644
+--- a/src/dynamic_load_unix.cpp
++++ b/src/dynamic_load_unix.cpp
+@@ -35,7 +35,11 @@ DynamicallyLoadTracingLibrary(const char* shared_library,
+                               std::string& error_message) noexcept try {
+   dlerror();  // Clear any existing error.
+
+-  const auto handle = dlopen(shared_library, RTLD_NOW | RTLD_LOCAL);
++  const auto handle = dlopen(shared_library, RTLD_NOW | RTLD_LOCAL
++#ifdef __SANITIZE_ADDRESS__
++      | RTLD_NODELETE
++#endif
++  );
+   if (handle == nullptr) {
+     error_message = dlerror();
+     return make_unexpected(dynamic_load_failure_error);

bazel/repositories.bzl

@@ -335,7 +335,12 @@ def _com_github_nghttp2_nghttp2():
     )
 
 def _io_opentracing_cpp():
-    _repository_impl("io_opentracing_cpp")
+    _repository_impl(
+        name = "io_opentracing_cpp",
+        patch_args = ["-p1"],
+        # Workaround for LSAN false positive in https://github.com/envoyproxy/envoy/issues/7647
+        patches = ["@envoy//bazel:io_opentracing_cpp.patch"],
+    )
     native.bind(
         name = "opentracing",
         actual = "@io_opentracing_cpp//:opentracing",

bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD

@@ -116,7 +116,6 @@ cc_toolchain_config(
     "-Wl,-z,relro,-z,now",
     "-B/usr/lib/llvm-8/bin",
     "-lm",
-    "-static-libgcc",
     "-fuse-ld=lld"],
     link_libs = ["-l:libstdc++.a"],
     opt_link_flags = ["-Wl,--gc-sections"],

bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD

@@ -114,7 +114,6 @@ cc_toolchain_config(
     "-Wl,-z,relro,-z,now",
     "-B/usr/lib/llvm-8/bin",
     "-lm",
-    "-static-libgcc",
     "-pthread",
     "-fuse-ld=lld"],
     link_libs = ["-l:libc++.a",

bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD

@@ -115,8 +115,7 @@ cc_toolchain_config(
     "-Wl,-z,relro,-z,now",
     "-B/usr/bin",
     "-pass-exit-codes",
-    "-lm",
-    "-static-libgcc"],
+    "-lm"],
     link_libs = ["-l:libstdc++.a"],
     opt_link_flags = ["-Wl,--gc-sections"],
     unfiltered_compile_flags = ["-fno-canonical-system-headers",

bazel/toolchains/configs/versions.bzl

@@ -1,9 +1,9 @@
 # Generated file, do not modify by hand
 # Generated by 'rbe_ubuntu_gcc_gen' rbe_autoconfig rule
 """Definitions to be used in rbe_repo attr of an rbe_autoconf rule  """
-toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang")
-toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx")
-toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc")
+toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang")
+toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx")
+toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc")
 _TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2]
 _BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]}
 LATEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b"

bazel/toolchains/rbe_toolchains_config.bzl

@@ -10,7 +10,7 @@ _CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs"
 _CLANG_ENV = {
     "BAZEL_COMPILER": "clang",
     "BAZEL_LINKLIBS": "-l%:libstdc++.a",
-    "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld",
+    "BAZEL_LINKOPTS": "-lm:-fuse-ld=lld",
     "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1",
     "GCOV": "llvm-profdata",
     "CC": "clang",
@@ -20,15 +20,15 @@ _CLANG_ENV = {
 
 _CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, {
     "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a",
-    "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld",
+    "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld",
     "BAZEL_CXXOPTS": "-stdlib=libc++",
     "CXXFLAGS": "-stdlib=libc++",
 })
 
 _GCC_ENV = {
     "BAZEL_COMPILER": "gcc",
     "BAZEL_LINKLIBS": "-l%:libstdc++.a",
-    "BAZEL_LINKOPTS": "-lm:-static-libgcc",
+    "BAZEL_LINKOPTS": "-lm",
     "CC": "gcc",
     "CXX": "g++",
     "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin",

test/exe/BUILD

@@ -24,9 +24,8 @@ envoy_sh_test(
     srcs = ["envoy_static_test.sh"],
     coverage = False,
     data = ["//source/exe:envoy-static"],
-    # NOTE: In some environments, ASAN causes dynamic linking no matter what, so don't run this
-    # test when doing ASAN.
-    tags = ["no_asan"],
+    # Sanitizers doesn't like statically linked lib(std)c++ and libgcc, skip this test in that context.
+    tags = ["no_san"],
 )
 
 envoy_sh_test(

test/integration/fake_upstream.cc

@@ -73,30 +73,22 @@ void FakeStream::decodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr) {
 }
 
 void FakeStream::encode100ContinueHeaders(const Http::HeaderMapImpl& headers) {
-  // TSan complains about thread-safety of std::shared_ptr when linked against libc++.
-  // See: https://github.com/envoyproxy/envoy/pull/7929
-  std::unique_ptr<Http::HeaderMapImpl> headers_copy(
+  std::shared_ptr<Http::HeaderMapImpl> headers_copy(
       new Http::HeaderMapImpl(static_cast<const Http::HeaderMap&>(headers)));
-  parent_.connection().dispatcher().post([this, headers = headers_copy.release()]() -> void {
-    encoder_.encode100ContinueHeaders(*headers);
-    delete headers;
-  });
+  parent_.connection().dispatcher().post(
+      [this, headers_copy]() -> void { encoder_.encode100ContinueHeaders(*headers_copy); });
 }
 
 void FakeStream::encodeHeaders(const Http::HeaderMapImpl& headers, bool end_stream) {
-  // TSan complains about thread-safety of std::shared_ptr when linked against libc++.
-  // See: https://github.com/envoyproxy/envoy/pull/7929
-  std::unique_ptr<Http::HeaderMapImpl> headers_copy(
+  std::shared_ptr<Http::HeaderMapImpl> headers_copy(
       new Http::HeaderMapImpl(static_cast<const Http::HeaderMap&>(headers)));
   if (add_served_by_header_) {
     headers_copy->addCopy(Http::LowerCaseString("x-served-by"),
                           parent_.connection().localAddress()->asString());
   }
-  parent_.connection().dispatcher().post(
-      [this, headers = headers_copy.release(), end_stream]() -> void {
-        encoder_.encodeHeaders(*headers, end_stream);
-        delete headers;
-      });
+  parent_.connection().dispatcher().post([this, headers_copy, end_stream]() -> void {
+    encoder_.encodeHeaders(*headers_copy, end_stream);
+  });
 }
 
 void FakeStream::encodeData(absl::string_view data, bool end_stream) {
@@ -114,24 +106,16 @@ void FakeStream::encodeData(uint64_t size, bool end_stream) {
 }
 
 void FakeStream::encodeData(Buffer::Instance& data, bool end_stream) {
-  // TSan complains about thread-safety of std::shared_ptr when linked against libc++.
-  // See: https://github.com/envoyproxy/envoy/pull/7929
-  std::unique_ptr<Buffer::Instance> data_copy(new Buffer::OwnedImpl(data));
-  parent_.connection().dispatcher().post([this, data = data_copy.release(), end_stream]() -> void {
-    encoder_.encodeData(*data, end_stream);
-    delete data;
-  });
+  std::shared_ptr<Buffer::Instance> data_copy(new Buffer::OwnedImpl(data));
+  parent_.connection().dispatcher().post(
+      [this, data_copy, end_stream]() -> void { encoder_.encodeData(*data_copy, end_stream); });
 }
 
 void FakeStream::encodeTrailers(const Http::HeaderMapImpl& trailers) {
-  // TSan complains about thread-safety of std::shared_ptr when linked against libc++.
-  // See: https://github.com/envoyproxy/envoy/pull/7929
-  std::unique_ptr<Http::HeaderMapImpl> trailers_copy(
+  std::shared_ptr<Http::HeaderMapImpl> trailers_copy(
       new Http::HeaderMapImpl(static_cast<const Http::HeaderMap&>(trailers)));
-  parent_.connection().dispatcher().post([this, trailers = trailers_copy.release()]() -> void {
-    encoder_.encodeTrailers(*trailers);
-    delete trailers;
-  });
+  parent_.connection().dispatcher().post(
+      [this, trailers_copy]() -> void { encoder_.encodeTrailers(*trailers_copy); });
 }
 
 void FakeStream::encodeResetStream() {