Rollup merge of #132125 - Zalathar:coverage-intrinsics, r=jieyouxu

coverage: Emit LLVM intrinsics using the normal helper method

Codegen already has convenient ways to declare and emit LLVM intrinsics, so there's no need for coverage instrumentation to jump through hoops to emit them manually.

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1725,16 +1725,6 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     fn fptosi_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
         self.fptoint_sat(true, val, dest_ty)
     }
-
-    fn instrprof_increment(
-        &mut self,
-        _fn_name: RValue<'gcc>,
-        _hash: RValue<'gcc>,
-        _num_counters: RValue<'gcc>,
-        _index: RValue<'gcc>,
-    ) {
-        unimplemented!();
-    }
 }
 
 impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1165,39 +1165,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
     }
 
-    fn instrprof_increment(
-        &mut self,
-        fn_name: &'ll Value,
-        hash: &'ll Value,
-        num_counters: &'ll Value,
-        index: &'ll Value,
-    ) {
-        debug!(
-            "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, num_counters, index
-        );
-
-        let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) };
-        let llty = self.cx.type_func(
-            &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()],
-            self.cx.type_void(),
-        );
-        let args = &[fn_name, hash, num_counters, index];
-        let args = self.check_call("call", llty, llfn, args);
-
-        unsafe {
-            let _ = llvm::LLVMRustBuildCall(
-                self.llbuilder,
-                llty,
-                llfn,
-                args.as_ptr() as *const &llvm::Value,
-                args.len() as c_uint,
-                [].as_ptr(),
-                0 as c_uint,
-            );
-        }
-    }
-
     fn call(
         &mut self,
         llty: &'ll Type,
@@ -1667,6 +1634,18 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         kcfi_bundle
     }
 
+    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
+    #[instrument(level = "debug", skip(self))]
+    pub(crate) fn instrprof_increment(
+        &mut self,
+        fn_name: &'ll Value,
+        hash: &'ll Value,
+        num_counters: &'ll Value,
+        index: &'ll Value,
+    ) {
+        self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]);
+    }
+
     /// Emits a call to `llvm.instrprof.mcdc.parameters`.
     ///
     /// This doesn't produce any code directly, but is used as input by
@@ -1676,80 +1655,43 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     ///
     /// [`CodeGenPGO::emitMCDCParameters`]:
     ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
+    #[instrument(level = "debug", skip(self))]
     pub(crate) fn mcdc_parameters(
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
         bitmap_bits: &'ll Value,
     ) {
-        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits);
-
         assert!(
             crate::llvm_util::get_version() >= (19, 0, 0),
             "MCDC intrinsics require LLVM 19 or later"
         );
-
-        let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) };
-        let llty = self.cx.type_func(
-            &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()],
-            self.cx.type_void(),
-        );
-        let args = &[fn_name, hash, bitmap_bits];
-        let args = self.check_call("call", llty, llfn, args);
-
-        unsafe {
-            let _ = llvm::LLVMRustBuildCall(
-                self.llbuilder,
-                llty,
-                llfn,
-                args.as_ptr() as *const &llvm::Value,
-                args.len() as c_uint,
-                [].as_ptr(),
-                0 as c_uint,
-            );
-        }
+        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]);
     }
 
+    #[instrument(level = "debug", skip(self))]
     pub(crate) fn mcdc_tvbitmap_update(
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
         bitmap_index: &'ll Value,
         mcdc_temp: &'ll Value,
     ) {
-        debug!(
-            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, bitmap_index, mcdc_temp
-        );
         assert!(
             crate::llvm_util::get_version() >= (19, 0, 0),
             "MCDC intrinsics require LLVM 19 or later"
         );
-
-        let llfn =
-            unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) };
-        let llty = self.cx.type_func(
-            &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()],
-            self.cx.type_void(),
-        );
         let args = &[fn_name, hash, bitmap_index, mcdc_temp];
-        let args = self.check_call("call", llty, llfn, args);
-        unsafe {
-            let _ = llvm::LLVMRustBuildCall(
-                self.llbuilder,
-                llty,
-                llfn,
-                args.as_ptr() as *const &llvm::Value,
-                args.len() as c_uint,
-                [].as_ptr(),
-                0 as c_uint,
-            );
-        }
+        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args);
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
         self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
     }
 
+    #[instrument(level = "debug", skip(self))]
     pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
-        debug!("mcdc_condbitmap_update() with args ({:?}, {:?})", cond_index, mcdc_temp);
         assert!(
             crate::llvm_util::get_version() >= (19, 0, 0),
             "MCDC intrinsics require LLVM 19 or later"

compiler/rustc_codegen_llvm/src/context.rs

@@ -1099,6 +1099,10 @@ impl<'ll> CodegenCx<'ll, '_> {
 
         if self.sess().instrument_coverage() {
             ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
+            if crate::llvm_util::get_version() >= (19, 0, 0) {
+                ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void);
+                ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void);
+            }
         }
 
         ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1);

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -208,6 +208,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
                 let bitmap_index = bx.const_u32(bitmap_idx);
                 bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap);
+                bx.mcdc_condbitmap_reset(cond_bitmap);
             }
         }
     }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1615,10 +1615,6 @@ unsafe extern "C" {
     pub fn LLVMRustSetAllowReassoc(Instr: &Value);
 
     // Miscellaneous instructions
-    pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;
-    pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value;
-    pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value;
-
     pub fn LLVMRustBuildCall<'a>(
         B: &Builder<'a>,
         Ty: &'a Type,

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -437,14 +437,6 @@ pub trait BuilderMethods<'a, 'tcx>:
     /// Called for `StorageDead`
     fn lifetime_end(&mut self, ptr: Self::Value, size: Size);
 
-    fn instrprof_increment(
-        &mut self,
-        fn_name: Self::Value,
-        hash: Self::Value,
-        num_counters: Self::Value,
-        index: Self::Value,
-    );
-
     fn call(
         &mut self,
         llty: Self::Type,

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1531,45 +1531,6 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty,
                                     ArrayRef<OperandBundleDef>(OpBundles)));
 }
 
-extern "C" LLVMValueRef
-LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
-#if LLVM_VERSION_GE(20, 0)
-  return wrap(llvm::Intrinsic::getOrInsertDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_increment));
-#else
-  return wrap(llvm::Intrinsic::getDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_increment));
-#endif
-}
-
-extern "C" LLVMValueRef
-LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) {
-#if LLVM_VERSION_LT(19, 0)
-  report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions");
-#endif
-#if LLVM_VERSION_GE(20, 0)
-  return wrap(llvm::Intrinsic::getOrInsertDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters));
-#else
-  return wrap(llvm::Intrinsic::getDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters));
-#endif
-}
-
-extern "C" LLVMValueRef
-LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(LLVMModuleRef M) {
-#if LLVM_VERSION_LT(19, 0)
-  report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions");
-#endif
-#if LLVM_VERSION_GE(20, 0)
-  return wrap(llvm::Intrinsic::getOrInsertDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update));
-#else
-  return wrap(llvm::Intrinsic::getDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update));
-#endif
-}
-
 extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst,
                                             unsigned DstAlign, LLVMValueRef Src,
                                             unsigned SrcAlign,