Add LLVM CFI support to the Rust compiler

This commit adds LLVM Control Flow Integrity (CFI) support to the Rust
compiler. It initially provides forward-edge control flow protection for
Rust-compiled code only by aggregating function pointers in groups
identified by their number of arguments.

Forward-edge control flow protection for C or C++ and Rust -compiled
code "mixed binaries" (i.e., for when C or C++ and Rust -compiled code
share the same virtual address space) will be provided in later work as
part of this project by defining and using compatible type identifiers
(see Type metadata in the design document in the tracking issue #89653).

LLVM CFI can be enabled with -Zsanitizer=cfi and requires LTO (i.e.,
-Clto).

compiler/rustc_codegen_gcc/src/builder.rs

@@ -915,6 +915,16 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // TODO(antoyo)
     }
 
+    fn type_metadata(&mut self, _function: RValue<'gcc>, _typeid: String) {
+        // Unsupported.
+    }
+
+    fn typeid_metadata(&mut self, _typeid: String) -> RValue<'gcc> {
+        // Unsupported.
+        self.context.new_rvalue_from_int(self.int_type, 0)
+    }
+
+
     fn store(&mut self, val: RValue<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
         self.store_with_flags(val, ptr, align, MemFlags::empty())
     }

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -367,6 +367,11 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // TODO(antoyo)
     }
 
+    fn type_test(&mut self, _pointer: Self::Value, _typeid: Self::Value) -> Self::Value {
+        // Unsupported.
+        self.context.new_rvalue_from_int(self.int_type, 0)
+    }
+
     fn va_start(&mut self, _va_list: RValue<'gcc>) -> RValue<'gcc> {
         unimplemented!();
     }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -604,6 +604,32 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn type_metadata(&mut self, function: &'ll Value, typeid: String) {
+        let typeid_metadata = self.typeid_metadata(typeid);
+        let v = [self.const_usize(0), typeid_metadata];
+        unsafe {
+            llvm::LLVMGlobalSetMetadata(
+                function,
+                llvm::MD_type as c_uint,
+                llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext(
+                    self.cx.llcx,
+                    v.as_ptr(),
+                    v.len() as c_uint,
+                )),
+            )
+        }
+    }
+
+    fn typeid_metadata(&mut self, typeid: String) -> Self::Value {
+        unsafe {
+            llvm::LLVMMDStringInContext(
+                self.cx.llcx,
+                typeid.as_ptr() as *const c_char,
+                typeid.as_bytes().len() as c_uint,
+            )
+        }
+    }
+
     fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
         self.store_with_flags(val, ptr, align, MemFlags::empty())
     }

compiler/rustc_codegen_llvm/src/context.rs

@@ -221,6 +221,15 @@ pub unsafe fn create_module(
         llvm::LLVMRustAddModuleFlag(llmod, avoid_plt, 1);
     }
 
+    if sess.is_sanitizer_cfi_enabled() {
+        // FIXME(rcvalle): Add support for non canonical jump tables.
+        let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast();
+        // FIXME(rcvalle): Add it with Override behavior flag--LLVMRustAddModuleFlag adds it with
+        // Warning behavior flag. Add support for specifying the behavior flag to
+        // LLVMRustAddModuleFlag.
+        llvm::LLVMRustAddModuleFlag(llmod, canonical_jump_tables, 1);
+    }
+
     // Control Flow Guard is currently only supported by the MSVC linker on Windows.
     if sess.target.is_like_msvc {
         match sess.opts.cg.control_flow_guard {
@@ -779,6 +788,8 @@ impl CodegenCx<'b, 'tcx> {
             ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void);
         }
 
+        ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1);
+
         if self.sess().opts.debuginfo != DebugInfo::None {
             ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void);
             ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void);

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -401,6 +401,14 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value {
+        // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time
+        // optimization pass replaces calls to this intrinsic with code to test type membership.
+        let i8p_ty = self.type_i8p();
+        let bitcast = self.bitcast(pointer, i8p_ty);
+        self.call_intrinsic("llvm.type.test", &[bitcast, typeid])
+    }
+
     fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
         self.call_intrinsic("llvm.va_start", &[va_list])
     }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -389,6 +389,7 @@ pub enum MetadataType {
     MD_nontemporal = 9,
     MD_mem_parallel_loop_access = 10,
     MD_nonnull = 11,
+    MD_type = 19,
 }
 
 /// LLVMRustAsmDialect
@@ -975,6 +976,8 @@ extern "C" {
     pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t);
     pub fn LLVMReplaceAllUsesWith(OldVal: &'a Value, NewVal: &'a Value);
     pub fn LLVMSetMetadata(Val: &'a Value, KindID: c_uint, Node: &'a Value);
+    pub fn LLVMGlobalSetMetadata(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
+    pub fn LLVMValueAsMetadata(Node: &'a Value) -> &Metadata;
 
     // Operations on constants of any type
     pub fn LLVMConstNull(Ty: &Type) -> &Value;

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -19,6 +19,7 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty, TypeFoldable};
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
+use rustc_symbol_mangling::typeid_for_fnabi;
 use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
 use rustc_target::abi::{self, HasDataLayout, WrappingRange};
 use rustc_target::spec::abi::Abi;
@@ -818,12 +819,43 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             self.codegen_argument(&mut bx, location, &mut llargs, last_arg);
         }
 
-        let fn_ptr = match (llfn, instance) {
-            (Some(llfn), _) => llfn,
-            (None, Some(instance)) => bx.get_fn_addr(instance),
+        let (is_indirect_call, fn_ptr) = match (llfn, instance) {
+            (Some(llfn), _) => (true, llfn),
+            (None, Some(instance)) => (false, bx.get_fn_addr(instance)),
             _ => span_bug!(span, "no llfn for call"),
         };
 
+        // For backends that support CFI using type membership (i.e., testing whether a given
+        // pointer is associated with a type identifier).
+        if bx.tcx().sess.is_sanitizer_cfi_enabled() && is_indirect_call {
+            // Emit type metadata and checks.
+            // FIXME(rcvalle): Add support for generalized identifiers.
+            // FIXME(rcvalle): Create distinct unnamed MDNodes for internal identifiers.
+            let typeid = typeid_for_fnabi(bx.tcx(), fn_abi);
+            let typeid_metadata = bx.typeid_metadata(typeid.clone());
+
+            // Test whether the function pointer is associated with the type identifier.
+            let cond = bx.type_test(fn_ptr, typeid_metadata);
+            let mut bx_pass = bx.build_sibling_block("type_test.pass");
+            let mut bx_fail = bx.build_sibling_block("type_test.fail");
+            bx.cond_br(cond, bx_pass.llbb(), bx_fail.llbb());
+
+            helper.do_call(
+                self,
+                &mut bx_pass,
+                fn_abi,
+                fn_ptr,
+                &llargs,
+                destination.as_ref().map(|&(_, target)| (ret_dest, target)),
+                cleanup,
+            );
+
+            bx_fail.abort();
+            bx_fail.unreachable();
+
+            return;
+        }
+
         helper.do_call(
             self,
             &mut bx,

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -4,6 +4,7 @@ use rustc_middle::mir;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TypeFoldable};
+use rustc_symbol_mangling::typeid_for_fnabi;
 use rustc_target::abi::call::{FnAbi, PassMode};
 
 use std::iter;
@@ -244,6 +245,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     for (bb, _) in traversal::reverse_postorder(&mir) {
         fx.codegen_block(bb);
     }
+
+    // For backends that support CFI using type membership (i.e., testing whether a given  pointer
+    // is associated with a type identifier).
+    if cx.tcx().sess.is_sanitizer_cfi_enabled() {
+        let typeid = typeid_for_fnabi(cx.tcx(), fn_abi);
+        bx.type_metadata(llfn, typeid.clone());
+    }
 }
 
 /// Produces, for each argument, a `Value` pointing at the

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -158,6 +158,8 @@ pub trait BuilderMethods<'a, 'tcx>:
 
     fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
     fn nonnull_metadata(&mut self, load: Self::Value);
+    fn type_metadata(&mut self, function: Self::Function, typeid: String);
+    fn typeid_metadata(&mut self, typeid: String) -> Self::Value;
 
     fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) -> Self::Value;
     fn store_with_flags(

compiler/rustc_codegen_ssa/src/traits/intrinsic.rs

@@ -24,6 +24,8 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes {
     ///
     /// Currently has any effect only when LLVM versions prior to 12.0 are used as the backend.
     fn sideeffect(&mut self);
+    /// Trait method used to test whether a given pointer is associated with a type identifier.
+    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value;
     /// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in
     /// Rust defined C-variadic functions.
     fn va_start(&mut self, val: Self::Value) -> Self::Value;

compiler/rustc_session/src/options.rs

@@ -351,8 +351,7 @@ mod desc {
     pub const parse_panic_strategy: &str = "either `unwind` or `abort`";
     pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
     pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
-    pub const parse_sanitizers: &str =
-        "comma separated list of sanitizers: `address`, `hwaddress`, `leak`, `memory` or `thread`";
+    pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `leak`, `memory` or `thread`";
     pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2";
     pub const parse_cfguard: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
@@ -584,6 +583,7 @@ mod parse {
             for s in v.split(',') {
                 *slot |= match s {
                     "address" => SanitizerSet::ADDRESS,
+                    "cfi" => SanitizerSet::CFI,
                     "leak" => SanitizerSet::LEAK,
                     "memory" => SanitizerSet::MEMORY,
                     "thread" => SanitizerSet::THREAD,

compiler/rustc_session/src/session.rs

@@ -672,6 +672,9 @@ impl Session {
     pub fn is_nightly_build(&self) -> bool {
         self.opts.unstable_features.is_nightly_build()
     }
+    pub fn is_sanitizer_cfi_enabled(&self) -> bool {
+        self.opts.debugging_opts.sanitizer.contains(SanitizerSet::CFI)
+    }
     pub fn overflow_checks(&self) -> bool {
         self.opts
             .cg
@@ -1398,6 +1401,16 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
                                 disable it using `-C target-feature=-crt-static`",
         );
     }
+
+    // LLVM CFI requires LTO.
+    if sess.is_sanitizer_cfi_enabled() {
+        if sess.opts.cg.lto == config::LtoCli::Unspecified
+            || sess.opts.cg.lto == config::LtoCli::No
+            || sess.opts.cg.lto == config::LtoCli::Thin
+        {
+            sess.err("`-Zsanitizer=cfi` requires `-Clto`");
+        }
+    }
 }
 
 /// Holds data on the current incremental compilation session, if there is one.

compiler/rustc_span/src/symbol.rs

@@ -408,6 +408,7 @@ symbols! {
         cfg_target_thread_local,
         cfg_target_vendor,
         cfg_version,
+        cfi,
         char,
         client,
         clippy,

compiler/rustc_symbol_mangling/src/lib.rs

@@ -103,8 +103,9 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::mono::{InstantiationMode, MonoItem};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, Instance, TyCtxt};
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_session::config::SymbolManglingVersion;
+use rustc_target::abi::call::FnAbi;
 
 use tracing::debug;
 
@@ -150,6 +151,11 @@ fn symbol_name_provider(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::Symb
     ty::SymbolName::new(tcx, &symbol_name)
 }
 
+/// This function computes the typeid for the given function ABI.
+pub fn typeid_for_fnabi(tcx: TyCtxt<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> String {
+    v0::mangle_typeid_for_fnabi(tcx, fn_abi)
+}
+
 /// Computes the symbol name for the given instance. This function will call
 /// `compute_instantiating_crate` if it needs to factor the instantiating crate
 /// into the symbol name.

compiler/rustc_symbol_mangling/src/v0.rs

@@ -9,6 +9,7 @@ use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::print::{Print, Printer};
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind, Subst};
 use rustc_middle::ty::{self, FloatTy, Instance, IntTy, Ty, TyCtxt, TypeFoldable, UintTy};
+use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::Integer;
 use rustc_target::spec::abi::Abi;
 
@@ -55,6 +56,41 @@ pub(super) fn mangle(
     std::mem::take(&mut cx.out)
 }
 
+pub(super) fn mangle_typeid_for_fnabi(
+    _tcx: TyCtxt<'tcx>,
+    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+) -> String {
+    // LLVM uses type metadata to allow IR modules to aggregate pointers by their types.[1] This
+    // type metadata is used by LLVM Control Flow Integrity to test whether a given pointer is
+    // associated with a type identifier (i.e., test type membership).
+    //
+    // Clang uses the Itanium C++ ABI's[2] virtual tables and RTTI typeinfo structure name[3] as
+    // type metadata identifiers for function pointers. The typeinfo name encoding is a
+    // two-character code (i.e., “TS”) prefixed to the type encoding for the function.
+    //
+    // For cross-language LLVM CFI support, a compatible encoding must be used by either
+    //
+    //  a. Using a superset of types that encompasses types used by Clang (i.e., Itanium C++ ABI's
+    //     type encodings[4]), or at least types used at the FFI boundary.
+    //  b. Reducing the types to the least common denominator between types used by Clang (or at
+    //     least types used at the FFI boundary) and Rust compilers (if even possible).
+    //  c. Creating a new ABI for cross-language CFI and using it for Clang and Rust compilers (and
+    //     possibly other compilers).
+    //
+    // Option (b) may weaken the protection for Rust-compiled only code, so it should be provided
+    // as an alternative to a Rust-specific encoding for when mixing Rust and C and C++ -compiled
+    // code. Option (c) would require changes to Clang to use the new ABI.
+    //
+    // [1] https://llvm.org/docs/TypeMetadata.html
+    // [2] https://itanium-cxx-abi.github.io/cxx-abi/abi.html
+    // [3] https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling-special-vtables
+    // [4] https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling-type
+    //
+    // FIXME(rcvalle): See comment above.
+    let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
+    format!("typeid{}", arg_count)
+}
+
 struct BinderLevel {
     /// The range of distances from the root of what's
     /// being printed, to the lifetimes in a binder.

compiler/rustc_target/src/spec/aarch64_apple_darwin.rs

@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(128);
 
     // FIXME: The leak sanitizer currently fails the tests, see #88132.
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;
 
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-arch".to_string(), "arm64".to_string()]);
     base.link_env_remove.extend(super::apple_base::macos_link_env_remove());

compiler/rustc_target/src/spec/aarch64_fuchsia.rs

@@ -8,7 +8,7 @@ pub fn target() -> Target {
         arch: "aarch64".to_string(),
         options: TargetOptions {
             max_atomic_width: Some(128),
-            supported_sanitizers: SanitizerSet::ADDRESS,
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::CFI,
             ..super::fuchsia_base::opts()
         },
     }

compiler/rustc_target/src/spec/aarch64_linux_android.rs

@@ -14,7 +14,7 @@ pub fn target() -> Target {
             // As documented in https://developer.android.com/ndk/guides/cpu-features.html
             // the neon (ASIMD) and FP must exist on all android aarch64 targets.
             features: "+neon,+fp-armv8".to_string(),
-            supported_sanitizers: SanitizerSet::HWADDRESS,
+            supported_sanitizers: SanitizerSet::CFI | SanitizerSet::HWADDRESS,
             ..super::android_base::opts()
         },
     }

compiler/rustc_target/src/spec/aarch64_unknown_freebsd.rs

@@ -9,6 +9,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             max_atomic_width: Some(128),
             supported_sanitizers: SanitizerSet::ADDRESS
+                | SanitizerSet::CFI
                 | SanitizerSet::MEMORY
                 | SanitizerSet::THREAD,
             ..super::freebsd_base::opts()

compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs

@@ -10,6 +10,7 @@ pub fn target() -> Target {
             mcount: "\u{1}_mcount".to_string(),
             max_atomic_width: Some(128),
             supported_sanitizers: SanitizerSet::ADDRESS
+                | SanitizerSet::CFI
                 | SanitizerSet::LEAK
                 | SanitizerSet::MEMORY
                 | SanitizerSet::THREAD