Rollup merge of rust-lang#38699 - japaric:lsan, r=alexcrichton

LeakSanitizer, ThreadSanitizer, AddressSanitizer and MemorySanitizer support

```
$ cargo new --bin leak && cd $_

$ edit Cargo.toml && tail -n3 $_
```

``` toml
[profile.dev]
opt-level = 1
```

```
$ edit src/main.rs && cat $_
```

``` rust
use std::mem;

fn main() {
    let xs = vec![0, 1, 2, 3];
    mem::forget(xs);
}
```

```
$ RUSTFLAGS="-Z sanitizer=leak" cargo run --target x86_64-unknown-linux-gnu; echo $?
    Finished dev [optimized + debuginfo] target(s) in 0.0 secs
     Running `target/debug/leak`

=================================================================
==10848==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 16 byte(s) in 1 object(s) allocated from:
    #0 0x557c3488db1f in __interceptor_malloc /shared/rust/checkouts/lsan/src/compiler-rt/lib/lsan/lsan_interceptors.cc:55
    #1 0x557c34888aaa in alloc::heap::exchange_malloc::h68f3f8b376a0da42 /shared/rust/checkouts/lsan/src/liballoc/heap.rs:138
    #2 0x557c34888afc in leak::main::hc56ab767de6d653a $PWD/src/main.rs:4
    #3 0x557c348c0806 in __rust_maybe_catch_panic ($PWD/target/debug/leak+0x3d806)

SUMMARY: LeakSanitizer: 16 byte(s) leaked in 1 allocation(s).
23
```

```
$ cargo new --bin racy && cd $_

$ edit src/main.rs && cat $_
```

``` rust
use std::thread;

static mut ANSWER: i32 = 0;

fn main() {
    let t1 = thread::spawn(|| unsafe { ANSWER = 42 });
    unsafe {
        ANSWER = 24;
    }
    t1.join().ok();
}
```

```
$ RUSTFLAGS="-Z sanitizer=thread" cargo run --target x86_64-unknown-linux-gnu; echo $?
==================
WARNING: ThreadSanitizer: data race (pid=12019)
  Write of size 4 at 0x562105989bb4 by thread T1:
    #0 racy::main::_$u7b$$u7b$closure$u7d$$u7d$::hbe13ea9e8ac73f7e $PWD/src/main.rs:6 (racy+0x000000010e3f)
    #1 _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2e466a92accacc78 /shared/rust/checkouts/lsan/src/libstd/panic.rs:296 (racy+0x000000010cc5)
    #2 std::panicking::try::do_call::h7f4d2b38069e4042 /shared/rust/checkouts/lsan/src/libstd/panicking.rs:460 (racy+0x00000000c8f2)
    #3 __rust_maybe_catch_panic <null> (racy+0x0000000b4e56)
    #4 std::panic::catch_unwind::h31ca45621ad66d5a /shared/rust/checkouts/lsan/src/libstd/panic.rs:361 (racy+0x00000000b517)
    #5 std::thread::Builder::spawn::_$u7b$$u7b$closure$u7d$$u7d$::hccfc37175dea0b01 /shared/rust/checkouts/lsan/src/libstd/thread/mod.rs:357 (racy+0x00000000c226)
    #6 _$LT$F$u20$as$u20$alloc..boxed..FnBox$LT$A$GT$$GT$::call_box::hd880bbf91561e033 /shared/rust/checkouts/lsan/src/liballoc/boxed.rs:605 (racy+0x00000000f27e)
    #7 std::sys::imp::thread::Thread::new::thread_start::hebdfc4b3d17afc85 <null> (racy+0x0000000abd40)

  Previous write of size 4 at 0x562105989bb4 by main thread:
    #0 racy::main::h23e6e5ca46d085c3 $PWD/src/main.rs:8 (racy+0x000000010d7c)
    #1 __rust_maybe_catch_panic <null> (racy+0x0000000b4e56)
    #2 __libc_start_main <null> (libc.so.6+0x000000020290)

  Location is global 'racy::ANSWER::h543d2b139f819b19' of size 4 at 0x562105989bb4 (racy+0x0000002f8bb4)

  Thread T1 (tid=12028, running) created by main thread at:
    #0 pthread_create /shared/rust/checkouts/lsan/src/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (racy+0x00000001aedb)
    #1 std::sys::imp::thread::Thread::new::hce44187bf4a36222 <null> (racy+0x0000000ab9ae)
    #2 std::thread::spawn::he382608373eb667e /shared/rust/checkouts/lsan/src/libstd/thread/mod.rs:412 (racy+0x00000000b5aa)
    #3 racy::main::h23e6e5ca46d085c3 $PWD/src/main.rs:6 (racy+0x000000010d5c)
    #4 __rust_maybe_catch_panic <null> (racy+0x0000000b4e56)
    #5 __libc_start_main <null> (libc.so.6+0x000000020290)

SUMMARY: ThreadSanitizer: data race $PWD/src/main.rs:6 in racy::main::_$u7b$$u7b$closure$u7d$$u7d$::hbe13ea9e8ac73f7e
==================
ThreadSanitizer: reported 1 warnings
66
```

```
$ cargo new --bin oob && cd $_

$ edit src/main.rs && cat $_
```

``` rust
fn main() {
    let xs = [0, 1, 2, 3];
    let y = unsafe { *xs.as_ptr().offset(4) };
}
```

```
$ RUSTFLAGS="-Z sanitizer=address" cargo run --target x86_64-unknown-linux-gnu; echo $?
=================================================================
==13328==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x7fff29f3ecd0 at pc 0x55802dc6bf7e bp 0x7fff29f3ec90 sp 0x7fff29f3ec88
READ of size 4 at 0x7fff29f3ecd0 thread T0
    #0 0x55802dc6bf7d in oob::main::h0adc7b67e5feb2e7 $PWD/src/main.rs:3
    #1 0x55802dd60426 in __rust_maybe_catch_panic ($PWD/target/debug/oob+0xfe426)
    #2 0x55802dd58dd9 in std::rt::lang_start::hb2951fc8a59d62a7 ($PWD/target/debug/oob+0xf6dd9)
    #3 0x55802dc6c002 in main ($PWD/target/debug/oob+0xa002)
    #4 0x7fad8c3b3290 in __libc_start_main (/usr/lib/libc.so.6+0x20290)
    #5 0x55802dc6b719 in _start ($PWD/target/debug/oob+0x9719)

Address 0x7fff29f3ecd0 is located in stack of thread T0 at offset 48 in frame
    #0 0x55802dc6bd5f in oob::main::h0adc7b67e5feb2e7 $PWD/src/main.rs:1

  This frame has 1 object(s):
    [32, 48) 'xs' <== Memory access at offset 48 overflows this variable
HINT: this may be a false positive if your program uses some custom stack unwind mechanism or swapcontext
      (longjmp and C++ exceptions *are* supported)
SUMMARY: AddressSanitizer: stack-buffer-overflow $PWD/src/main.rs:3 in oob::main::h0adc7b67e5feb2e7
Shadow bytes around the buggy address:
  0x1000653dfd40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfd50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfd60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfd70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfd80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x1000653dfd90: 00 00 00 00 f1 f1 f1 f1 00 00[f3]f3 00 00 00 00
  0x1000653dfda0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfdb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfdc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfdd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x1000653dfde0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Heap right redzone:      fb
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack partial redzone:   f4
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
==13328==ABORTING
1
```

```
$ cargo new --bin uninit && cd $_

$ edit src/main.rs && cat $_
```

``` rust
use std::mem;

fn main() {
    let xs: [u8; 4] = unsafe { mem::uninitialized() };
    let y = xs[0] + xs[1];
}
```

```
$ RUSTFLAGS="-Z sanitizer=memory" cargo run; echo $?
==30198==WARNING: MemorySanitizer: use-of-uninitialized-value
    #0 0x563f4b6867da in uninit::main::hc2731cd4f2ed48f8 $PWD/src/main.rs:5
    #1 0x563f4b7033b6 in __rust_maybe_catch_panic ($PWD/target/debug/uninit+0x873b6)
    #2 0x563f4b6fbd69 in std::rt::lang_start::hb2951fc8a59d62a7 ($PWD/target/debug/uninit+0x7fd69)
    #3 0x563f4b6868a9 in main ($PWD/target/debug/uninit+0xa8a9)
    #4 0x7fe844354290 in __libc_start_main (/usr/lib/libc.so.6+0x20290)
    #5 0x563f4b6864f9 in _start ($PWD/target/debug/uninit+0xa4f9)

SUMMARY: MemorySanitizer: use-of-uninitialized-value $PWD/src/main.rs:5 in uninit::main::hc2731cd4f2ed48f8
Exiting
77
```

configure

@@ -649,6 +649,7 @@ opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
 opt vendor 0 "enable usage of vendored Rust crates"
+opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"

src/bootstrap/check.rs

@@ -242,6 +242,10 @@ pub fn compiletest(build: &Build,
     cmd.env("RUSTC_BOOTSTRAP", "1");
     build.add_rust_test_threads(&mut cmd);
 
+    if build.config.sanitizers {
+        cmd.env("SANITIZER_SUPPORT", "1");
+    }
+
     cmd.arg("--adb-path").arg("adb");
     cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR);
     if target.contains("android") {

src/bootstrap/compile.rs

@@ -51,6 +51,17 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
     if compiler.stage == 0 && build.local_rebuild && !build.config.use_jemalloc {
         features.push_str(" force_alloc_system");
     }
+
+    if compiler.stage != 0 && build.config.sanitizers {
+        // This variable is used by the sanitizer runtime crates, e.g.
+        // rustc_lsan, to build the sanitizer runtime from C code
+        // When this variable is missing, those crates won't compile the C code,
+        // so we don't set this variable during stage0 where llvm-config is
+        // missing
+        // We also only build the runtimes when --enable-sanitizers (or its
+        // config.toml equivalent) is used
+        cargo.env("LLVM_CONFIG", build.llvm_config(target));
+    }
     cargo.arg("--features").arg(features)
          .arg("--manifest-path")
          .arg(build.src.join("src/rustc/std_shim/Cargo.toml"));

src/bootstrap/config.rs

@@ -48,6 +48,7 @@ pub struct Config {
     pub target_config: HashMap<String, Target>,
     pub full_bootstrap: bool,
     pub extended: bool,
+    pub sanitizers: bool,
 
     // llvm codegen options
     pub llvm_assertions: bool,
@@ -149,6 +150,7 @@ struct Build {
     python: Option<String>,
     full_bootstrap: Option<bool>,
     extended: Option<bool>,
+    sanitizers: Option<bool>,
 }
 
 /// TOML representation of various global install decisions.
@@ -294,6 +296,7 @@ impl Config {
         set(&mut config.vendor, build.vendor);
         set(&mut config.full_bootstrap, build.full_bootstrap);
         set(&mut config.extended, build.extended);
+        set(&mut config.sanitizers, build.sanitizers);
 
         if let Some(ref install) = toml.install {
             config.prefix = install.prefix.clone().map(PathBuf::from);
@@ -438,6 +441,7 @@ impl Config {
                 ("VENDOR", self.vendor),
                 ("FULL_BOOTSTRAP", self.full_bootstrap),
                 ("EXTENDED", self.extended),
+                ("SANITIZERS", self.sanitizers),
             }
 
             match key {

src/bootstrap/config.toml.example

@@ -124,6 +124,9 @@
 # disabled by default.
 #extended = false
 
+# Build the sanitizer runtimes
+#sanitizers = false
+
 # =============================================================================
 # General install configuration options
 # =============================================================================

src/bootstrap/lib.rs

@@ -599,7 +599,8 @@ impl Build {
     /// Get the space-separated set of activated features for the standard
     /// library.
     fn std_features(&self) -> String {
-        let mut features = "panic-unwind".to_string();
+        let mut features = "panic-unwind asan lsan msan tsan".to_string();
+
         if self.config.debug_jemalloc {
             features.push_str(" debug-jemalloc");
         }

src/ci/docker/dist-x86-linux/Dockerfile

@@ -63,6 +63,11 @@ RUN ./build-git.sh
 COPY build-cmake.sh /tmp/
 RUN ./build-cmake.sh
 
+# for sanitizers, we need kernel headers files newer than the ones CentOS ships
+# with so we install newer ones here
+COPY build-headers.sh /tmp/
+RUN ./build-headers.sh
+
 RUN curl -Lo /rustroot/dumb-init \
       https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 && \
       chmod +x /rustroot/dumb-init
@@ -76,5 +81,5 @@ RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST |
 ENV HOSTS=i686-unknown-linux-gnu
 ENV HOSTS=$HOSTS,x86_64-unknown-linux-gnu
 
-ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended --enable-sanitizers
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-x86-linux/build-headers.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+source shared.sh
+
+curl https://cdn.kernel.org/pub/linux/kernel/v3.x/linux-3.2.84.tar.xz | unxz | tar x
+
+cd linux-3.2.84
+hide_output make mrproper
+hide_output make INSTALL_HDR_PATH=dest headers_install
+
+find dest/include \( -name .install -o -name ..install.cmd \) -delete
+yes | cp -fr dest/include/* /usr/include
+
+cd ..
+rm -rf linux-3.2.84

src/ci/docker/x86_64-gnu/Dockerfile

@@ -22,5 +22,5 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
+ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-sanitizers
 ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist

src/librustc/middle/cstore.rs

@@ -229,6 +229,7 @@ pub trait CrateStore<'tcx> {
     fn is_allocator(&self, cnum: CrateNum) -> bool;
     fn is_panic_runtime(&self, cnum: CrateNum) -> bool;
     fn is_compiler_builtins(&self, cnum: CrateNum) -> bool;
+    fn is_sanitizer_runtime(&self, cnum: CrateNum) -> bool;
     fn panic_strategy(&self, cnum: CrateNum) -> PanicStrategy;
     fn extern_crate(&self, cnum: CrateNum) -> Option<ExternCrate>;
     /// The name of the crate as it is referred to in source code of the current
@@ -390,6 +391,7 @@ impl<'tcx> CrateStore<'tcx> for DummyCrateStore {
     fn is_allocator(&self, cnum: CrateNum) -> bool { bug!("is_allocator") }
     fn is_panic_runtime(&self, cnum: CrateNum) -> bool { bug!("is_panic_runtime") }
     fn is_compiler_builtins(&self, cnum: CrateNum) -> bool { bug!("is_compiler_builtins") }
+    fn is_sanitizer_runtime(&self, cnum: CrateNum) -> bool { bug!("is_sanitizer_runtime") }
     fn panic_strategy(&self, cnum: CrateNum) -> PanicStrategy {
         bug!("panic_strategy")
     }

src/librustc/session/config.rs

@@ -51,6 +51,14 @@ pub struct Config {
     pub uint_type: UintTy,
 }
 
+#[derive(Clone)]
+pub enum Sanitizer {
+    Address,
+    Leak,
+    Memory,
+    Thread,
+}
+
 #[derive(Clone, Copy, PartialEq, Hash)]
 pub enum OptLevel {
     No, // -O0
@@ -626,11 +634,13 @@ macro_rules! options {
             Some("a number");
         pub const parse_panic_strategy: Option<&'static str> =
             Some("either `panic` or `abort`");
+        pub const parse_sanitizer: Option<&'static str> =
+            Some("one of: `address`, `leak`, `memory` or `thread`");
     }
 
     #[allow(dead_code)]
     mod $mod_set {
-        use super::{$struct_name, Passes, SomePasses, AllPasses};
+        use super::{$struct_name, Passes, SomePasses, AllPasses, Sanitizer};
         use rustc_back::PanicStrategy;
 
         $(
@@ -751,6 +761,17 @@ macro_rules! options {
             }
             true
         }
+
+        fn parse_sanitizer(slote: &mut Option<Sanitizer>, v: Option<&str>) -> bool {
+            match v {
+                Some("address") => *slote = Some(Sanitizer::Address),
+                Some("leak") => *slote = Some(Sanitizer::Leak),
+                Some("memory") => *slote = Some(Sanitizer::Memory),
+                Some("thread") => *slote = Some(Sanitizer::Thread),
+                _ => return false,
+            }
+            true
+        }
     }
 ) }
 
@@ -949,6 +970,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
           "encode MIR of all functions into the crate metadata"),
     osx_rpath_install_name: bool = (false, parse_bool, [TRACKED],
           "pass `-install_name @rpath/...` to the OSX linker"),
+    sanitizer: Option<Sanitizer> = (None, parse_sanitizer, [UNTRACKED],
+                                   "Use a sanitizer"),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc_asan/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+authors = ["The Rust Project Developers"]
+build = "build.rs"
+name = "rustc_asan"
+version = "0.0.0"
+
+[lib]
+name = "rustc_asan"
+path = "lib.rs"
+
+[build-dependencies]
+build_helper = { path = "../build_helper" }
+cmake = "0.1.18"
+
+[dependencies]
+alloc_system = { path = "../liballoc_system" }
+core = { path = "../libcore" }

src/librustc_asan/build.rs

@@ -0,0 +1,39 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate build_helper;
+extern crate cmake;
+
+use std::path::PathBuf;
+use std::env;
+
+use cmake::Config;
+
+fn main() {
+    if let Some(llvm_config) = env::var_os("LLVM_CONFIG") {
+        let dst = Config::new("../compiler-rt")
+            .define("COMPILER_RT_BUILD_SANITIZERS", "ON")
+            .define("COMPILER_RT_BUILD_BUILTINS", "OFF")
+            .define("COMPILER_RT_BUILD_XRAY", "OFF")
+            .define("LLVM_CONFIG_PATH", llvm_config)
+            .build_target("asan")
+            .build();
+
+        println!("cargo:rustc-link-search=native={}",
+                 dst.join("build/lib/linux").display());
+        println!("cargo:rustc-link-lib=static=clang_rt.asan-x86_64");
+
+        build_helper::rerun_if_changed_anything_in_dir(&PathBuf::from(env::var("CARGO_MANIFEST_DIR")
+                .unwrap())
+            .join("../compiler-rt"));
+    }
+
+    println!("cargo:rerun-if-changed=build.rs");
+}

src/librustc_asan/lib.rs

@@ -0,0 +1,20 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![cfg_attr(not(stage0), feature(sanitizer_runtime))]
+#![cfg_attr(not(stage0), sanitizer_runtime)]
+#![feature(alloc_system)]
+#![feature(staged_api)]
+#![no_std]
+#![unstable(feature = "sanitizer_runtime_lib",
+            reason = "internal implementation detail of sanitizers",
+            issue = "0")]
+
+extern crate alloc_system;

src/librustc_llvm/ffi.rs

@@ -126,6 +126,9 @@ pub enum Attribute {
     UWTable         = 17,
     ZExt            = 18,
     InReg           = 19,
+    SanitizeThread  = 20,
+    SanitizeAddress = 21,
+    SanitizeMemory  = 22,
 }
 
 /// LLVMIntPredicate