diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index f13c8214af1b7..fc9e18378d5a2 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -223,19 +223,29 @@ impl<'tcx> Inliner<'tcx> { return Err("failed to normalize return type"); } if callsite.fn_sig.abi() == Abi::RustCall { - let (arg_tuple, skipped_args) = match &args[..] { - [arg_tuple] => (arg_tuple, 0), - [_, arg_tuple] => (arg_tuple, 1), + // FIXME: Don't inline user-written `extern "rust-call"` functions, + // since this is generally perf-negative on rustc, and we hope that + // LLVM will inline these functions instead. + if callee_body.spread_arg.is_some() { + return Err("do not inline user-written rust-call functions"); + } + + let (self_arg, arg_tuple) = match &args[..] { + [arg_tuple] => (None, arg_tuple), + [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), _ => bug!("Expected `rust-call` to have 1 or 2 args"), }; + let self_arg_ty = + self_arg.map(|self_arg| self_arg.ty(&caller_body.local_decls, self.tcx)); + let arg_tuple_ty = arg_tuple.ty(&caller_body.local_decls, self.tcx); - let ty::Tuple(arg_tuple_tys) = arg_tuple_ty.kind() else { + let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { bug!("Closure arguments are not passed as a tuple"); }; for (arg_ty, input) in - arg_tuple_tys.iter().zip(callee_body.args_iter().skip(skipped_args)) + self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) { let input_type = callee_body.local_decls[input].ty; if !util::is_subtype(self.tcx, self.param_env, input_type, arg_ty) { diff --git a/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff b/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff index f3f4d895ae238..8f2baf4a3b61c 100644 --- a/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff @@ -5,13 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {main}; -+ let mut _5: (); + scope 1 (inlined f::) { + debug g => _2; + let mut _3: &fn() {main}; + let _4: (); -+ scope 2 (inlined >::call - shim(fn() {main})) { -+ } + } bb0: { @@ -22,9 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ _4 = move (*_3)() -> [return: bb2, unwind unreachable]; ++ _4 = >::call(move _3, const ()) -> [return: bb2, unwind unreachable]; } bb1: { @@ -36,7 +31,6 @@ + } + + bb2: { -+ StorageDead(_5); + StorageDead(_3); + drop(_2) -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff b/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff index 3ce8d9acf3681..ad801fd280ac7 100644 --- a/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff @@ -5,13 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {main}; -+ let mut _5: (); + scope 1 (inlined f::) { + debug g => _2; + let mut _3: &fn() {main}; + let _4: (); -+ scope 2 (inlined >::call - shim(fn() {main})) { -+ } + } bb0: { @@ -22,9 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ _4 = move (*_3)() -> [return: bb4, unwind: bb2]; ++ _4 = >::call(move _3, const ()) -> [return: bb2, unwind: bb3]; } bb1: { @@ -35,18 +30,17 @@ return; + } + -+ bb2 (cleanup): { -+ drop(_2) -> [return: bb3, unwind terminate]; ++ bb2: { ++ StorageDead(_3); ++ drop(_2) -> [return: bb1, unwind continue]; + } + + bb3 (cleanup): { -+ resume; ++ drop(_2) -> [return: bb4, unwind terminate]; + } + -+ bb4: { -+ StorageDead(_5); -+ StorageDead(_3); -+ drop(_2) -> [return: bb1, unwind continue]; ++ bb4 (cleanup): { ++ resume; } } diff --git a/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff b/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff index eb00763541691..fd1f698c60df8 100644 --- a/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff @@ -5,21 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {g}; -+ let mut _5: (); + scope 1 (inlined f::) { + debug g => _2; + let mut _3: &fn() {g}; + let _4: (); -+ scope 2 (inlined >::call - shim(fn() {g})) { -+ scope 3 (inlined g) { -+ scope 4 (inlined f::) { -+ debug g => main; -+ let _6: (); -+ scope 5 (inlined >::call - shim(fn() {main})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -30,10 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ StorageLive(_6); -+ _6 = main() -> [return: bb2, unwind unreachable]; ++ _4 = >::call(move _3, const ()) -> [return: bb2, unwind unreachable]; } bb1: { @@ -45,8 +31,6 @@ + } + + bb2: { -+ StorageDead(_6); -+ StorageDead(_5); + StorageDead(_3); + drop(_2) -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff index 198a232261840..99dc64115a984 100644 --- a/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff @@ -5,21 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {g}; -+ let mut _5: (); + scope 1 (inlined f::) { + debug g => _2; + let mut _3: &fn() {g}; + let _4: (); -+ scope 2 (inlined >::call - shim(fn() {g})) { -+ scope 3 (inlined g) { -+ scope 4 (inlined f::) { -+ debug g => main; -+ let _6: (); -+ scope 5 (inlined >::call - shim(fn() {main})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -30,10 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ StorageLive(_6); -+ _6 = main() -> [return: bb4, unwind: bb2]; ++ _4 = >::call(move _3, const ()) -> [return: bb2, unwind: bb3]; } bb1: { @@ -44,19 +30,17 @@ return; + } + -+ bb2 (cleanup): { -+ drop(_2) -> [return: bb3, unwind terminate]; ++ bb2: { ++ StorageDead(_3); ++ drop(_2) -> [return: bb1, unwind continue]; + } + + bb3 (cleanup): { -+ resume; ++ drop(_2) -> [return: bb4, unwind terminate]; + } + -+ bb4: { -+ StorageDead(_6); -+ StorageDead(_5); -+ StorageDead(_3); -+ drop(_2) -> [return: bb1, unwind continue]; ++ bb4 (cleanup): { ++ resume; } } diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff new file mode 100644 index 0000000000000..757617e594021 --- /dev/null +++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff @@ -0,0 +1,29 @@ +- // MIR for `call` before Inline ++ // MIR for `call` after Inline + + fn call(_1: Box>, _2: I) -> () { + debug mock => _1; + debug input => _2; + let mut _0: (); + let mut _3: &mut std::boxed::Box>; + let mut _4: I; + + bb0: { + StorageLive(_3); + _3 = &mut _1; + StorageLive(_4); + _4 = move _2; + _0 = > as FnMut>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb2, unwind unreachable]; + } + + bb2: { + return; + } + } + diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff new file mode 100644 index 0000000000000..ef85e075eeb37 --- /dev/null +++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff @@ -0,0 +1,37 @@ +- // MIR for `call` before Inline ++ // MIR for `call` after Inline + + fn call(_1: Box>, _2: I) -> () { + debug mock => _1; + debug input => _2; + let mut _0: (); + let mut _3: &mut std::boxed::Box>; + let mut _4: I; + + bb0: { + StorageLive(_3); + _3 = &mut _1; + StorageLive(_4); + _4 = move _2; + _0 = > as FnMut>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb2, unwind: bb4]; + } + + bb2: { + return; + } + + bb3 (cleanup): { + drop(_1) -> [return: bb4, unwind terminate]; + } + + bb4 (cleanup): { + resume; + } + } + diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs new file mode 100644 index 0000000000000..971223c72ca23 --- /dev/null +++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs @@ -0,0 +1,11 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +// compile-flags: -Zmir-enable-passes=+Inline --crate-type=lib + +#![feature(fn_traits, tuple_trait, unboxed_closures)] + +use std::marker::Tuple; + +// EMIT_MIR dont_ice_on_generic_rust_call.call.Inline.diff +pub fn call(mut mock: Box>, input: I) { + mock.call_mut(input) +} diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff new file mode 100644 index 0000000000000..4fa04b05e8993 --- /dev/null +++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff @@ -0,0 +1,32 @@ +- // MIR for `call` before Inline ++ // MIR for `call` after Inline + + fn call(_1: Box) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: &std::boxed::Box; + let mut _4: (i32,); + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = &_1; + StorageLive(_4); + _4 = (const 1_i32,); + _2 = as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + drop(_1) -> [return: bb2, unwind unreachable]; + } + + bb2: { + return; + } + } + diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff new file mode 100644 index 0000000000000..5df730a99302a --- /dev/null +++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff @@ -0,0 +1,40 @@ +- // MIR for `call` before Inline ++ // MIR for `call` after Inline + + fn call(_1: Box) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: &std::boxed::Box; + let mut _4: (i32,); + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = &_1; + StorageLive(_4); + _4 = (const 1_i32,); + _2 = as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + drop(_1) -> [return: bb2, unwind: bb4]; + } + + bb2: { + return; + } + + bb3 (cleanup): { + drop(_1) -> [return: bb4, unwind terminate]; + } + + bb4 (cleanup): { + resume; + } + } + diff --git a/tests/mir-opt/inline/inline_box_fn.rs b/tests/mir-opt/inline/inline_box_fn.rs new file mode 100644 index 0000000000000..348f0e77f92a4 --- /dev/null +++ b/tests/mir-opt/inline/inline_box_fn.rs @@ -0,0 +1,8 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +// unit-test: Inline +// compile-flags: --crate-type=lib + +// EMIT_MIR inline_box_fn.call.Inline.diff +fn call(x: Box) { + x(1); +} diff --git a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff index d83c8d585ea61..8a6eec3352a34 100644 --- a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff @@ -5,20 +5,9 @@ let mut _0: (); let _1: (); + let mut _2: fn() {f}; -+ let mut _4: (); + scope 1 (inlined call::) { + debug f => _2; + let _3: (); -+ scope 2 (inlined >::call_once - shim(fn() {f})) { -+ scope 3 (inlined f) { -+ scope 4 (inlined call::) { -+ debug f => f; -+ let _5: (); -+ scope 5 (inlined >::call_once - shim(fn() {f})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -27,15 +16,10 @@ + StorageLive(_2); + _2 = f; + StorageLive(_3); -+ StorageLive(_4); -+ _4 = const (); -+ StorageLive(_5); -+ _5 = f() -> [return: bb1, unwind unreachable]; ++ _3 = >::call_once(move _2, const ()) -> [return: bb1, unwind unreachable]; } bb1: { -+ StorageDead(_5); -+ StorageDead(_4); + StorageDead(_3); + StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff index a08662959dd8e..a24649c1ebd57 100644 --- a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff @@ -5,20 +5,9 @@ let mut _0: (); let _1: (); + let mut _2: fn() {f}; -+ let mut _4: (); + scope 1 (inlined call::) { + debug f => _2; + let _3: (); -+ scope 2 (inlined >::call_once - shim(fn() {f})) { -+ scope 3 (inlined f) { -+ scope 4 (inlined call::) { -+ debug f => f; -+ let _5: (); -+ scope 5 (inlined >::call_once - shim(fn() {f})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -27,15 +16,10 @@ + StorageLive(_2); + _2 = f; + StorageLive(_3); -+ StorageLive(_4); -+ _4 = const (); -+ StorageLive(_5); -+ _5 = f() -> [return: bb1, unwind continue]; ++ _3 = >::call_once(move _2, const ()) -> [return: bb1, unwind continue]; } bb1: { -+ StorageDead(_5); -+ StorageDead(_4); + StorageDead(_3); + StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff index 0dcd5fae88d3b..7d5553b2f37b2 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff @@ -5,7 +5,6 @@ let mut _0: (); let _1: (!, !); + let mut _2: fn() -> ! {sleep}; -+ let mut _7: (); + scope 1 (inlined call_twice:: ! {sleep}>) { + debug f => _2; + let mut _3: &fn() -> ! {sleep}; @@ -18,10 +17,6 @@ + debug b => _6; + } + } -+ scope 4 (inlined ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { -+ scope 5 (inlined sleep) { -+ } -+ } + } bb0: { @@ -33,13 +28,24 @@ + StorageLive(_6); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_7); -+ _7 = const (); -+ goto -> bb1; ++ _4 = ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind unreachable]; + } + + bb1: { -+ goto -> bb1; ++ StorageDead(_3); ++ StorageLive(_5); ++ _5 = &_2; ++ _6 = ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind unreachable]; ++ } ++ ++ bb2: { ++ StorageDead(_5); ++ _1 = (move _4, move _6); ++ drop(_2) -> [return: bb3, unwind unreachable]; ++ } ++ ++ bb3: { ++ unreachable; } } diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff index dfc12db12a888..073ddeff7cab7 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff @@ -5,7 +5,6 @@ let mut _0: (); let _1: (!, !); + let mut _2: fn() -> ! {sleep}; -+ let mut _8: (); + scope 1 (inlined call_twice:: ! {sleep}>) { + debug f => _2; + let mut _3: &fn() -> ! {sleep}; @@ -19,10 +18,6 @@ + debug b => _6; + } + } -+ scope 4 (inlined ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { -+ scope 5 (inlined sleep) { -+ } -+ } + } bb0: { @@ -34,13 +29,40 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_8); -+ _8 = const (); -+ goto -> bb1; ++ _4 = ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind: bb5]; + } + + bb1: { -+ goto -> bb1; ++ StorageDead(_3); ++ StorageLive(_5); ++ _5 = &_2; ++ _6 = ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind: bb4]; ++ } ++ ++ bb2: { ++ StorageDead(_5); ++ StorageLive(_7); ++ _7 = move _4; ++ _1 = (move _7, move _6); ++ StorageDead(_7); ++ StorageDead(_4); ++ drop(_2) -> [return: bb3, unwind continue]; ++ } ++ ++ bb3: { ++ unreachable; ++ } ++ ++ bb4 (cleanup): { ++ drop(_4) -> [return: bb5, unwind terminate]; ++ } ++ ++ bb5 (cleanup): { ++ drop(_2) -> [return: bb6, unwind terminate]; ++ } ++ ++ bb6 (cleanup): { ++ resume; } } diff --git a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff index b86eb5f35c95e..bee01a5f97b9a 100644 --- a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff @@ -8,8 +8,6 @@ let mut _3: &fn() {foo}; let _4: fn() {foo}; let mut _5: (); -+ scope 1 (inlined >::call - shim(fn() {foo})) { -+ } bb0: { StorageLive(_2); @@ -22,26 +20,20 @@ _3 = &_4; StorageLive(_5); _5 = (); -- _2 = >::call(move _3, move _5) -> [return: bb2, unwind unreachable]; -+ _2 = move (*_3)() -> [return: bb3, unwind unreachable]; + _2 = >::call(move _3, move _5) -> [return: bb2, unwind unreachable]; } bb2: { -+ return; -+ } -+ -+ bb3: { StorageDead(_5); StorageDead(_3); StorageDead(_4); StorageDead(_2); _0 = const (); -- drop(_1) -> [return: bb3, unwind unreachable]; -- } -- -- bb3: { -- return; -+ drop(_1) -> [return: bb2, unwind unreachable]; + drop(_1) -> [return: bb3, unwind unreachable]; + } + + bb3: { + return; } } diff --git a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff index c67babba23e80..b750330df9272 100644 --- a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff @@ -8,55 +8,40 @@ let mut _3: &fn() {foo}; let _4: fn() {foo}; let mut _5: (); -+ scope 1 (inlined >::call - shim(fn() {foo})) { -+ } bb0: { StorageLive(_2); StorageLive(_3); StorageLive(_4); -- _4 = hide_foo() -> [return: bb1, unwind: bb4]; -+ _4 = hide_foo() -> [return: bb1, unwind: bb3]; + _4 = hide_foo() -> [return: bb1, unwind: bb4]; } bb1: { _3 = &_4; StorageLive(_5); _5 = (); -- _2 = >::call(move _3, move _5) -> [return: bb2, unwind: bb4]; -+ _2 = move (*_3)() -> [return: bb5, unwind: bb3]; + _2 = >::call(move _3, move _5) -> [return: bb2, unwind: bb4]; } bb2: { -- StorageDead(_5); -- StorageDead(_3); -- StorageDead(_4); -- StorageDead(_2); -- _0 = const (); -- drop(_1) -> [return: bb3, unwind: bb5]; -+ return; + StorageDead(_5); + StorageDead(_3); + StorageDead(_4); + StorageDead(_2); + _0 = const (); + drop(_1) -> [return: bb3, unwind: bb5]; } -- bb3: { -- return; -+ bb3 (cleanup): { -+ drop(_1) -> [return: bb4, unwind terminate]; + bb3: { + return; } bb4 (cleanup): { -- drop(_1) -> [return: bb5, unwind terminate]; -+ resume; + drop(_1) -> [return: bb5, unwind terminate]; } -- bb5 (cleanup): { -- resume; -+ bb5: { -+ StorageDead(_5); -+ StorageDead(_3); -+ StorageDead(_4); -+ StorageDead(_2); -+ _0 = const (); -+ drop(_1) -> [return: bb2, unwind: bb4]; + bb5 (cleanup): { + resume; } }