From 77f74ed070b648c513be3b1795514168fe7a9ebc Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 27 Dec 2020 10:30:38 +0100 Subject: [PATCH 01/48] Merge commit 'dbee13661efa269cb4cd57bb4c6b99a19732b484' into sync_cg_clif-2020-12-27 --- .vscode/settings.json | 1 + Cargo.lock | 55 +++++----- Cargo.toml | 6 +- Readme.md | 14 ++- build_sysroot/Cargo.lock | 8 +- build_sysroot/Cargo.toml | 3 +- example/std_example.rs | 2 + rust-toolchain | 2 +- scripts/cargo.sh | 4 +- scripts/filter_profile.rs | 2 +- scripts/tests.sh | 10 +- src/backend.rs | 4 +- src/base.rs | 36 +++++-- src/bin/cg_clif.rs | 19 +--- src/bin/cg_clif_build_sysroot.rs | 4 +- src/constant.rs | 8 +- src/debuginfo/emit.rs | 5 +- src/debuginfo/unwind.rs | 6 +- src/driver/aot.rs | 31 +++++- src/driver/jit.rs | 166 ++++++++++++++++++++++++++++--- src/driver/mod.rs | 56 ++++------- src/intrinsics/llvm.rs | 4 +- src/intrinsics/mod.rs | 69 +++---------- src/intrinsics/simd.rs | 25 ++--- src/lib.rs | 79 +++++++++++---- src/optimize/peephole.rs | 39 +++++++- src/pretty_clif.rs | 102 ++++++++++--------- src/vtable.rs | 3 +- 28 files changed, 489 insertions(+), 274 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 04ab5085c196c..7618251acd5c2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"], + "rust-analyzer.assist.importMergeBehaviour": "last", "rust-analyzer.cargo.loadOutDirsFromCheck": true, "rust-analyzer.linkedProjects": [ "./Cargo.toml", diff --git a/Cargo.lock b/Cargo.lock index 67ed41e765231..0382835269d1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-entity", ] @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "byteorder", "cranelift-bforest", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -85,17 +85,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" [[package]] name = "cranelift-entity" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" [[package]] name = "cranelift-frontend" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen", "log", @@ -103,10 +103,28 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "cranelift-jit" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-entity", + "cranelift-module", + "cranelift-native", + "errno", + "libc", + "log", + "region", + "target-lexicon", + "winapi", +] + [[package]] name = "cranelift-module" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "anyhow", "cranelift-codegen", @@ -118,7 +136,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -128,7 +146,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "anyhow", "cranelift-codegen", @@ -138,23 +156,6 @@ dependencies = [ "target-lexicon", ] -[[package]] -name = "cranelift-simplejit" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" -dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-module", - "cranelift-native", - "errno", - "libc", - "log", - "region", - "target-lexicon", - "winapi", -] - [[package]] name = "crc32fast" version = "1.2.1" @@ -325,9 +326,9 @@ dependencies = [ "ar", "cranelift-codegen", "cranelift-frontend", + "cranelift-jit", "cranelift-module", "cranelift-object", - "cranelift-simplejit", "gimli", "indexmap", "libloading", diff --git a/Cargo.toml b/Cargo.toml index cbff06749d3e9..8e1933bb14e7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["dylib"] cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } +cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } target-lexicon = "0.11.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} @@ -27,7 +27,7 @@ libloading = { version = "0.6.0", optional = true } #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } -#cranelift-simplejit = { path = "../wasmtime/cranelift/simplejit" } +#cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } #[patch.crates-io] @@ -35,7 +35,7 @@ libloading = { version = "0.6.0", optional = true } [features] default = ["jit", "inline_asm"] -jit = ["cranelift-simplejit", "libloading"] +jit = ["cranelift-jit", "libloading"] inline_asm = [] [profile.dev] diff --git a/Readme.md b/Readme.md index de54bf67f4a19..22d9e00923f00 100644 --- a/Readme.md +++ b/Readme.md @@ -2,7 +2,7 @@ > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠ -The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). +The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue. @@ -68,7 +68,15 @@ $ $cg_clif_dir/build/cargo.sh jit or ```bash -$ $cg_clif_dir/build/bin/cg_clif --jit my_crate.rs +$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs +``` + +There is also an experimental lazy jit mode. In this mode functions are only compiled once they are +first called. It currently does not work with multi-threaded programs. When a not yet compiled +function is called from another thread than the main thread, you will get an ICE. + +```bash +$ $cg_clif_dir/build/cargo.sh lazy-jit ``` ### Shell @@ -77,7 +85,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/build/bin/cg_clif - --jit + echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic } function jit() { diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index a2b8f449f00ff..990557694ead4 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "cc" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" dependencies = [ "rustc-std-workspace-core", ] diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index e562dedb5324b..3dbd28c286a24 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -5,13 +5,14 @@ version = "0.0.0" [dependencies] core = { path = "./sysroot_src/library/core" } -compiler_builtins = "0.1" alloc = { path = "./sysroot_src/library/alloc" } std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] } test = { path = "./sysroot_src/library/test" } alloc_system = { path = "./alloc_system" } +compiler_builtins = { version = "=0.1.36", default-features = false } + [patch.crates-io] rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } diff --git a/example/std_example.rs b/example/std_example.rs index b38e25328a4ee..015bbdfed4648 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -15,6 +15,8 @@ fn main() { let stderr = ::std::io::stderr(); let mut stderr = stderr.lock(); + // FIXME support lazy jit when multi threading + #[cfg(not(lazy_jit))] std::thread::spawn(move || { println!("Hello from another thread!"); }); diff --git a/rust-toolchain b/rust-toolchain index ed1e64f45db08..d6ad24bcf26dd 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-11-27 +nightly-2020-12-23 diff --git a/scripts/cargo.sh b/scripts/cargo.sh index dcd40acc02a53..a3d6d303057b8 100755 --- a/scripts/cargo.sh +++ b/scripts/cargo.sh @@ -10,7 +10,9 @@ cmd=$1 shift || true if [[ "$cmd" = "jit" ]]; then -cargo "+${TOOLCHAIN}" rustc "$@" -- --jit +cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -Cprefer-dynamic +elif [[ "$cmd" = "lazy-jit" ]]; then +cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy -Cprefer-dynamic else cargo "+${TOOLCHAIN}" "$cmd" "$@" fi diff --git a/scripts/filter_profile.rs b/scripts/filter_profile.rs index 3327c10089d9b..15388926ec9ec 100755 --- a/scripts/filter_profile.rs +++ b/scripts/filter_profile.rs @@ -4,7 +4,7 @@ pushd $(dirname "$0")/../ source build/config.sh popd -PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0 +PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0 #*/ //! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse diff --git a/scripts/tests.sh b/scripts/tests.sh index 114b6f30a4a91..a61774f479ec7 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -15,7 +15,10 @@ function no_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] mini_core_hello_world" - CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" + + echo "[JIT-lazy] mini_core_hello_world" + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" else echo "[JIT] mini_core_hello_world (skipped)" fi @@ -37,7 +40,10 @@ function base_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] std_example" - $MY_RUSTC --jit example/std_example.rs --target "$HOST_TRIPLE" + $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE" + + echo "[JIT-lazy] std_example" + $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --cfg lazy_jit --target "$HOST_TRIPLE" else echo "[JIT] std_example (skipped)" fi diff --git a/src/backend.rs b/src/backend.rs index 9e32259716f51..0ce34c904bdcc 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -162,7 +162,7 @@ impl AddConstructor for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { - let triple = crate::build_isa(sess, true).triple().clone(); + let triple = crate::build_isa(sess).triple().clone(); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -193,7 +193,7 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { let mut builder = ObjectBuilder::new( - crate::build_isa(sess, true), + crate::build_isa(sess), name + ".o", cranelift_module::default_libcall_names(), ) diff --git a/src/base.rs b/src/base.rs index 72073896a723b..34c9561d67622 100644 --- a/src/base.rs +++ b/src/base.rs @@ -118,6 +118,8 @@ pub(crate) fn codegen_fn<'tcx>( context.eliminate_unreachable_code(cx.module.isa()).unwrap(); context.dce(cx.module.isa()).unwrap(); + context.want_disasm = crate::pretty_clif::should_write_ir(tcx); + // Define function let module = &mut cx.module; tcx.sess.time("define function", || { @@ -140,6 +142,16 @@ pub(crate) fn codegen_fn<'tcx>( &clif_comments, ); + if let Some(mach_compile_result) = &context.mach_compile_result { + if let Some(disasm) = &mach_compile_result.disasm { + crate::pretty_clif::write_ir_file( + tcx, + &format!("{}.vcode", tcx.symbol_name(instance).name), + |file| file.write_all(disasm.as_bytes()), + ) + } + } + // Define debuginfo for function let isa = cx.module.isa(); let debug_context = &mut cx.debug_context; @@ -307,7 +319,9 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { } => { let discr = codegen_operand(fx, discr).load_scalar(fx); - if switch_ty.kind() == fx.tcx.types.bool.kind() { + let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() + || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); + if use_bool_opt { assert_eq!(targets.iter().count(), 1); let (then_value, then_block) = targets.iter().next().unwrap(); let then_block = fx.get_block(then_block); @@ -325,12 +339,22 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); let discr = crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr); - if test_zero { - fx.bcx.ins().brz(discr, then_block, &[]); - fx.bcx.ins().jump(else_block, &[]); + if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( + &fx.bcx, discr, test_zero, + ) { + if taken { + fx.bcx.ins().jump(then_block, &[]); + } else { + fx.bcx.ins().jump(else_block, &[]); + } } else { - fx.bcx.ins().brnz(discr, then_block, &[]); - fx.bcx.ins().jump(else_block, &[]); + if test_zero { + fx.bcx.ins().brz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } else { + fx.bcx.ins().brnz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } } } else { let mut switch = ::cranelift_frontend::Switch::new(); diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs index f4d23ebcf4e4d..58e45b4e9b972 100644 --- a/src/bin/cg_clif.rs +++ b/src/bin/cg_clif.rs @@ -44,9 +44,7 @@ fn main() { let mut callbacks = CraneliftPassesCallbacks::default(); rustc_driver::install_ice_hook(); let exit_code = rustc_driver::catch_with_exit_code(|| { - let mut use_jit = false; - - let mut args = std::env::args_os() + let args = std::env::args_os() .enumerate() .map(|(i, arg)| { arg.into_string().unwrap_or_else(|arg| { @@ -56,23 +54,10 @@ fn main() { ) }) }) - .filter(|arg| { - if arg == "--jit" { - use_jit = true; - false - } else { - true - } - }) .collect::>(); - if use_jit { - args.push("-Cprefer-dynamic".to_string()); - } let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { - Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { - config: rustc_codegen_cranelift::BackendConfig { use_jit }, - }) + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None }) }))); run_compiler.run() }); diff --git a/src/bin/cg_clif_build_sysroot.rs b/src/bin/cg_clif_build_sysroot.rs index 165d33dcfb509..8ee4cd46c94e0 100644 --- a/src/bin/cg_clif_build_sysroot.rs +++ b/src/bin/cg_clif_build_sysroot.rs @@ -92,9 +92,7 @@ fn main() { let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); if use_clif { run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { - Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { - config: rustc_codegen_cranelift::BackendConfig { use_jit: false }, - }) + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None }) }))); } run_compiler.run() diff --git a/src/constant.rs b/src/constant.rs index 544b020b71190..beff84fb2e217 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -100,7 +100,10 @@ fn codegen_static_ref<'tcx>( let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); assert!(!layout.is_unsized(), "unsized statics aren't supported"); assert!( - matches!(fx.bcx.func.global_values[local_data_id], GlobalValueData::Symbol { tls: false, ..}), + matches!( + fx.bcx.func.global_values[local_data_id], + GlobalValueData::Symbol { tls: false, .. } + ), "tls static referenced without Rvalue::ThreadLocalRef" ); CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) @@ -447,7 +450,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - module.define_data(data_id, &data_ctx).unwrap(); + // FIXME don't duplicate definitions in lazy jit mode + let _ = module.define_data(data_id, &data_ctx); cx.done.insert(data_id); } diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs index c21835b1fc3aa..6160f9b78d8b3 100644 --- a/src/debuginfo/emit.rs +++ b/src/debuginfo/emit.rs @@ -74,10 +74,7 @@ impl WriterRelocate { /// Perform the collected relocations to be usable for JIT usage. #[cfg(feature = "jit")] - pub(super) fn relocate_for_jit( - mut self, - jit_module: &cranelift_simplejit::SimpleJITModule, - ) -> Vec { + pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec { use std::convert::TryInto; for reloc in self.relocs.drain(..) { diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index e0f62b64e6bbb..49de927cdba05 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -15,11 +15,11 @@ pub(crate) struct UnwindContext<'tcx> { } impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self { + pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { - if isa.flags().is_pic() { + if pic_eh_frame { cie.fde_address_encoding = gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); } @@ -80,7 +80,7 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(feature = "jit")] pub(crate) unsafe fn register_jit( self, - jit_module: &cranelift_simplejit::SimpleJITModule, + jit_module: &cranelift_jit::JITModule, ) -> Option { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian( self.tcx, diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 78d6ff0cb001c..16f9bfc99189f 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::cstore::EncodedMetadata; -use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{DebugInfo, OutputType}; @@ -146,11 +146,34 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege } } - let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None); + let mut cx = crate::CodegenCx::new( + tcx, + module, + tcx.sess.opts.debuginfo != DebugInfo::None, + true, + ); super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + match mono_item { + MonoItem::Fn(inst) => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id) + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { + cx.global_asm.push_str(&*asm.as_str()); + cx.global_asm.push_str("\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } + } + } } let (mut module, global_asm, debug, mut unwind_context) = tcx.sess.time("finalize CodegenCx", || cx.finalize()); @@ -236,7 +259,7 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); - let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa()); + let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 5a844841c2ce5..9a42c675cc144 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -1,16 +1,23 @@ //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object //! files. +use std::cell::RefCell; use std::ffi::CString; use std::os::raw::{c_char, c_int}; use rustc_codegen_ssa::CrateInfo; +use rustc_middle::mir::mono::MonoItem; -use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; +use cranelift_jit::{JITBuilder, JITModule}; use crate::prelude::*; +use crate::{CodegenCx, CodegenMode}; -pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { +thread_local! { + pub static CURRENT_MODULE: RefCell> = RefCell::new(None); +} + +pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); } @@ -35,12 +42,13 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = SimpleJITBuilder::with_isa( - crate::build_isa(tcx.sess, false), + let mut jit_builder = JITBuilder::with_isa( + crate::build_isa(tcx.sess), cranelift_module::default_libcall_names(), ); + jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy)); jit_builder.symbols(imported_symbols); - let mut jit_module = SimpleJITModule::new(jit_builder); + let mut jit_module = JITModule::new(jit_builder); assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); let sig = Signature { @@ -66,20 +74,42 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, jit_module, false); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); - let (mut jit_module, global_asm, _debug, mut unwind_context) = - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + super::time(tcx, "codegen mono items", || { + super::predefine_mono_items(&mut cx, &mono_items); + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + match mono_item { + MonoItem::Fn(inst) => match codegen_mode { + CodegenMode::Aot => unreachable!(), + CodegenMode::Jit => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + }, + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id); + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + tcx.sess + .span_fatal(item.span, "Global asm is not supported in JIT mode"); + } } - tcx.sess.time("finalize CodegenCx", || cx.finalize()) - }); + } + }); + + let (mut jit_module, global_asm, _debug, mut unwind_context) = + tcx.sess.time("finalize CodegenCx", || cx.finalize()); + jit_module.finalize_definitions(); + if !global_asm.is_empty() { - tcx.sess.fatal("Global asm is not supported in JIT mode"); + tcx.sess.fatal("Inline asm is not supported in JIT mode"); } + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); @@ -91,7 +121,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed"); + println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_main) }; @@ -107,11 +137,50 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); + CURRENT_MODULE + .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); + let ret = f(args.len() as c_int, argv.as_ptr()); std::process::exit(ret); } +#[no_mangle] +extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { + rustc_middle::ty::tls::with(|tcx| { + // lift is used to ensure the correct lifetime for instance. + let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); + + CURRENT_MODULE.with(|jit_module| { + let mut jit_module = jit_module.borrow_mut(); + let jit_module = jit_module.as_mut().unwrap(); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); + + let (name, sig) = crate::abi::get_function_name_and_sig( + tcx, + cx.module.isa().triple(), + instance, + true, + ); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + cx.module.prepare_for_function_redefine(func_id).unwrap(); + + tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, instance, Linkage::Export) + }); + + let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize(); + assert!(global_asm.is_empty()); + jit_module.finalize_definitions(); + std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + jit_module.get_finalized_function(func_id) + }) + }) +} + fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { use rustc_middle::middle::dependency_format::Linkage; @@ -171,3 +240,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } + +pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) { + let tcx = cx.tcx; + + let pointer_type = cx.module.target_config().pointer_type(); + + let (name, sig) = + crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + + let instance_ptr = Box::into_raw(Box::new(inst)); + + let jit_fn = cx + .module + .declare_function( + "__clif_jit_fn", + Linkage::Import, + &Signature { + call_conv: cx.module.target_config().default_call_conv, + params: vec![AbiParam::new(pointer_type)], + returns: vec![AbiParam::new(pointer_type)], + }, + ) + .unwrap(); + + let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + let mut builder_ctx = FunctionBuilderContext::new(); + let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + + let jit_fn = cx + .module + .declare_func_in_func(jit_fn, trampoline_builder.func); + let sig_ref = trampoline_builder.func.import_signature(sig); + + let entry_block = trampoline_builder.create_block(); + trampoline_builder.append_block_params_for_function_params(entry_block); + let fn_args = trampoline_builder + .func + .dfg + .block_params(entry_block) + .to_vec(); + + trampoline_builder.switch_to_block(entry_block); + let instance_ptr = trampoline_builder + .ins() + .iconst(pointer_type, instance_ptr as u64 as i64); + let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); + let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; + let call_inst = trampoline_builder + .ins() + .call_indirect(sig_ref, jitted_fn, &fn_args); + let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); + trampoline_builder.ins().return_(&ret_vals); + + cx.module + .define_function( + func_id, + &mut Context::for_function(trampoline), + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); +} diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 7b8cc2ddd48d6..9f4ea9a386551 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; +use crate::CodegenMode; mod aot; #[cfg(feature = "jit")] @@ -20,24 +21,25 @@ pub(crate) fn codegen_crate( ) -> Box { tcx.sess.abort_if_errors(); - if config.use_jit { - let is_executable = tcx - .sess - .crate_types() - .contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } + match config.codegen_mode { + CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + let is_executable = tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::Executable); + if !is_executable { + tcx.sess.fatal("can't jit non-executable crate"); + } - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx); + #[cfg(feature = "jit")] + let _: ! = jit::run_jit(tcx, config.codegen_mode); - #[cfg(not(feature = "jit"))] - tcx.sess - .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + #[cfg(not(feature = "jit"))] + tcx.sess + .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } } - - aot::run_aot(tcx, metadata, need_metadata_module) } fn predefine_mono_items<'tcx>( @@ -63,30 +65,6 @@ fn predefine_mono_items<'tcx>( }); } -fn codegen_mono_item<'tcx, M: Module>( - cx: &mut crate::CodegenCx<'tcx, M>, - mono_item: MonoItem<'tcx>, - linkage: Linkage, -) { - match mono_item { - MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage)); - } - MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id), - MonoItem::GlobalAsm(hir_id) => { - let item = cx.tcx.hir().expect_item(hir_id); - if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { - cx.global_asm.push_str(&*asm.as_str()); - cx.global_asm.push_str("\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); - } - } - } -} - fn time(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { if std::env::var("CG_CLIF_DISPLAY_CG_TIME") .as_ref() diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 171445f2d71b6..d58e4d4995842 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -23,8 +23,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout()); - let lane_ty = fx.clif_type(lane_layout.ty).unwrap(); + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); let mut res = fx.bcx.ins().iconst(types::I32, 0); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index df8aa1b3e6983..be5b247bb9f0b 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -171,27 +171,6 @@ macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { } } -fn lane_type_and_count<'tcx>( - tcx: TyCtxt<'tcx>, - layout: TyAndLayout<'tcx>, -) -> (TyAndLayout<'tcx>, u16) { - assert!(layout.ty.is_simd()); - let lane_count = match layout.fields { - rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(), - _ => unreachable!("lane_type_and_count({:?})", layout), - }; - let lane_layout = layout - .field( - &ty::layout::LayoutCx { - tcx, - param_env: ParamEnv::reveal_all(), - }, - 0, - ) - .unwrap(); - (lane_layout, lane_count) -} - pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option { let (element, count) = match &layout.abi { Abi::Vector { element, count } => (element.clone(), *count), @@ -218,8 +197,10 @@ fn simd_for_each_lane<'tcx, M: Module>( ) { let layout = val.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane_idx in 0..lane_count { @@ -248,8 +229,10 @@ fn simd_pair_for_each_lane<'tcx, M: Module>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane in 0..lane_count { @@ -269,13 +252,14 @@ fn simd_reduce<'tcx, M: Module>( ret: CPlace<'tcx>, f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value, ) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); assert_eq!(lane_layout, ret.layout()); let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); for lane_idx in 1..lane_count { let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) + .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())) .load_scalar(fx); res_val = f(fx, lane_layout, res_val, lane); } @@ -289,14 +273,14 @@ fn simd_reduce_bool<'tcx, M: Module>( ret: CPlace<'tcx>, f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value, ) { - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); assert!(ret.layout().ty.is_bool()); let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean for lane_idx in 1..lane_count { let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) + .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())) .load_scalar(fx); let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean res_val = f(fx, res_val, lane); @@ -460,9 +444,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( "abort" => { trap_abort(fx, "Called intrinsic::abort."); } - "unreachable" => { - trap_unreachable(fx, "[corruption] Called intrinsic::unreachable."); - } "transmute" => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } @@ -575,12 +556,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); } }; - discriminant_value, (c ptr) { - let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout); - let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout()); - ret.write_cvalue(fx, discr); - }; size_of_val, (c ptr) { let layout = fx.layout_of(T); let size = if layout.is_unsized() { @@ -641,22 +616,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("wrapping_"), (c x, c y) { - assert_eq!(x.layout().ty, y.layout().ty); - let bin_op = match intrinsic { - "wrapping_add" => BinOp::Add, - "wrapping_sub" => BinOp::Sub, - "wrapping_mul" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), - }; - let res = crate::num::codegen_int_binop( - fx, - bin_op, - x, - y, - ); - ret.write_cvalue(fx, res); - }; _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { @@ -916,7 +875,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( dest.write_cvalue(fx, val); }; - size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { + pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { let const_val = fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); let val = crate::constant::codegen_const_value( diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 2b32e866e5ef6..e0eb5c59590ff 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -73,11 +73,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); - assert_eq!(lane_type, ret_lane_type); - assert_eq!(n, ret_lane_count); + assert_eq!(lane_ty, ret_lane_ty); + assert_eq!(u64::from(n), ret_lane_count); let total_len = lane_count * 2; @@ -105,14 +105,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; for &idx in &indexes { - assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len); + assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); } for (out_idx, in_idx) in indexes.into_iter().enumerate() { - let in_lane = if in_idx < lane_count { + let in_lane = if u64::from(in_idx) < lane_count { x.value_field(fx, mir::Field::new(in_idx.into())) } else { - y.value_field(fx, mir::Field::new((in_idx - lane_count).into())) + y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap())) }; let out_lane = ret.place_field(fx, mir::Field::new(out_idx)); out_lane.write_cvalue(fx, in_lane); @@ -131,7 +131,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); - let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout()); + let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); } @@ -160,7 +160,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); - let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout()); + let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); } @@ -212,12 +212,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(a.layout(), c.layout()); let layout = a.layout(); - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); assert_eq!(lane_count, ret_lane_count); + let ret_lane_layout = fx.layout_of(ret_lane_ty); for lane in 0..lane_count { - let lane = mir::Field::new(lane.into()); + let lane = mir::Field::new(lane.try_into().unwrap()); let a_lane = a.value_field(fx, lane).load_scalar(fx); let b_lane = b.value_field(fx, lane).load_scalar(fx); let c_lane = c.value_field(fx, lane).load_scalar(fx); diff --git a/src/lib.rs b/src/lib.rs index ba9ee0d450ee6..6e4f3bf2898d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,8 @@ associated_type_bounds, never_type, try_blocks, - hash_drain_filter + hash_drain_filter, + str_split_once )] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] @@ -34,6 +35,7 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; +use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -141,8 +143,8 @@ struct CodegenCx<'tcx, M: Module> { } impl<'tcx, M: Module> CodegenCx<'tcx, M> { - fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool) -> Self { - let unwind_context = UnwindContext::new(tcx, module.isa()); + fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self { + let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame); let debug_context = if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { @@ -172,12 +174,55 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> { } #[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + Aot, + Jit, + JitLazy, +} + +impl Default for CodegenMode { + fn default() -> Self { + CodegenMode::Aot + } +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +#[derive(Copy, Clone, Debug, Default)] pub struct BackendConfig { - pub use_jit: bool, + pub codegen_mode: CodegenMode, +} + +impl BackendConfig { + fn from_opts(opts: &[String]) -> Result { + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + Ok(config) + } } pub struct CraneliftCodegenBackend { - pub config: BackendConfig, + pub config: Option, } impl CodegenBackend for CraneliftCodegenBackend { @@ -204,7 +249,13 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { - let res = driver::codegen_crate(tcx, metadata, need_metadata_module, self.config); + let config = if let Some(config) = self.config { + config + } else { + BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) + .unwrap_or_else(|err| tcx.sess.fatal(&err)) + }; + let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config); rustc_symbol_mangling::test::report_symbol_names(tcx); @@ -250,17 +301,13 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session, enable_pic: bool) -> Box { +fn build_isa(sess: &Session) -> Box { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); let mut flags_builder = settings::builder(); - if enable_pic { - flags_builder.enable("is_pic").unwrap(); - } else { - flags_builder.set("is_pic", "false").unwrap(); - } + flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided flags_builder .set( @@ -283,8 +330,6 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box { @@ -297,7 +342,7 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box { sess.warn("Optimizing for size is not supported. Just ignoring the request"); } - }*/ + } let flags = settings::Flags::new(flags_builder); @@ -311,7 +356,5 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box Box { - Box::new(CraneliftCodegenBackend { - config: BackendConfig { use_jit: false }, - }) + Box::new(CraneliftCodegenBackend { config: None }) } diff --git a/src/optimize/peephole.rs b/src/optimize/peephole.rs index f8e0f3af3d0ad..a575ed8dc35f8 100644 --- a/src/optimize/peephole.rs +++ b/src/optimize/peephole.rs @@ -73,7 +73,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) - })() .unwrap_or_else(|| { match bcx.func.dfg.value_type(arg) { - types::I8 | types::I32 => { + types::I8 | types::I16 => { // WORKAROUND for brz.i8 and brnz.i8 not yet being implemented bcx.ins().uextend(types::I32, arg) } @@ -81,3 +81,40 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) - } }) } + +/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known. +pub(crate) fn maybe_known_branch_taken( + bcx: &FunctionBuilder<'_>, + arg: Value, + test_zero: bool, +) -> Option { + let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + arg_inst + } else { + return None; + }; + + match bcx.func.dfg[arg_inst] { + InstructionData::UnaryBool { + opcode: Opcode::Bconst, + imm, + } => { + if test_zero { + Some(!imm) + } else { + Some(imm) + } + } + InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm, + } => { + if test_zero { + Some(imm.bits() == 0) + } else { + Some(imm.bits() != 0) + } + } + _ => None, + } +} diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index a9f060e51d8f8..22c94fec82fc1 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -53,6 +53,7 @@ //! ``` use std::fmt; +use std::io::Write; use cranelift_codegen::{ entity::SecondaryMap, @@ -200,32 +201,24 @@ impl FunctionCx<'_, '_, M> { } } -pub(crate) fn write_clif_file<'tcx>( - tcx: TyCtxt<'tcx>, - postfix: &str, - isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, - instance: Instance<'tcx>, - context: &cranelift_codegen::Context, - mut clif_comments: &CommentWriter, -) { - use std::io::Write; - - if !cfg!(debug_assertions) - && !tcx +pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { + cfg!(debug_assertions) + || tcx .sess .opts .output_types .contains_key(&OutputType::LlvmAssembly) - { +} + +pub(crate) fn write_ir_file<'tcx>( + tcx: TyCtxt<'tcx>, + name: &str, + write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, +) { + if !should_write_ir(tcx) { return; } - let value_ranges = isa.map(|isa| { - context - .build_value_labels_ranges(isa) - .expect("value location ranges") - }); - let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif"); match std::fs::create_dir(&clif_output_dir) { @@ -234,41 +227,58 @@ pub(crate) fn write_clif_file<'tcx>( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(format!( - "{}.{}.clif", - tcx.symbol_name(instance).name, - postfix - )); - - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa( - tcx.sess, true, /* PIC doesn't matter here */ - )), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let clif_file_name = clif_output_dir.join(name); let res: std::io::Result<()> = try { let mut file = std::fs::File::create(clif_file_name)?; - let target_triple = crate::target_triple(tcx.sess); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", target_triple)?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; + write(&mut file)?; }; if let Err(err) = res { - tcx.sess.warn(&format!("err writing clif file: {}", err)); + tcx.sess.warn(&format!("error writing ir file: {}", err)); } } +pub(crate) fn write_clif_file<'tcx>( + tcx: TyCtxt<'tcx>, + postfix: &str, + isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, + instance: Instance<'tcx>, + context: &cranelift_codegen::Context, + mut clif_comments: &CommentWriter, +) { + write_ir_file( + tcx, + &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa.map(|isa| { + context + .build_value_labels_ranges(isa) + .expect("value location ranges") + }); + + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { + isa: Some(&*crate::build_isa(tcx.sess)), + value_ranges: value_ranges.as_ref(), + }, + ) + .unwrap(); + + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); +} + impl fmt::Debug for FunctionCx<'_, '_, M> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f, "{:?}", self.instance.substs)?; diff --git a/src/vtable.rs b/src/vtable.rs index 238abc0d8bdfa..8f15586a9dc06 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -158,7 +158,8 @@ fn build_vtable<'tcx>( ) .unwrap(); - fx.cx.module.define_data(data_id, &data_ctx).unwrap(); + // FIXME don't duplicate definitions in lazy jit mode + let _ = fx.cx.module.define_data(data_id, &data_ctx); data_id } From 2c41a6953b77bfd3db2e68b6b2f0f7dfba3be6d2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 1 Jan 2021 17:15:04 +0100 Subject: [PATCH 02/48] Rustup to rustc 1.51.0-nightly (44e3daf5e 2020-12-31) --- build_sysroot/Cargo.lock | 4 ++-- rust-toolchain | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 990557694ead4..f3f29957ecd63 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" dependencies = [ "compiler_builtins", "gimli", diff --git a/rust-toolchain b/rust-toolchain index d6ad24bcf26dd..a2b82fb1f4fb3 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-12-23 +nightly-2021-01-01 From 1a1cdac93054a850d2ebb647d045a33989aa812d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 2 Jan 2021 16:59:23 +0100 Subject: [PATCH 03/48] Remove code that was moved from the backend to rustc_incremental --- src/driver/aot.rs | 7 ------- src/intrinsics/mod.rs | 2 +- src/lib.rs | 23 ++++++++--------------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 16f9bfc99189f..df89883f0bbb7 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -281,9 +281,6 @@ pub(super) fn run_aot( None }; - rustc_incremental::assert_dep_graph(tcx); - rustc_incremental::save_dep_graph(tcx); - let metadata_module = if need_metadata_module { let _timer = tcx.prof.generic_activity("codegen crate metadata"); let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || { @@ -322,10 +319,6 @@ pub(super) fn run_aot( None }; - if tcx.sess.opts.output_types.should_codegen() { - rustc_incremental::assert_module_sources::assert_module_sources(tcx); - } - Box::new(( CodegenResults { crate_name: tcx.crate_name(LOCAL_CRATE), diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index be5b247bb9f0b..8946ac43bc65a 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -824,7 +824,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } ty => unreachable!("bswap {}", ty), } - }; + } let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T)); ret.write_cvalue(fx, res); }; diff --git a/src/lib.rs b/src/lib.rs index 6e4f3bf2898d8..4b6431e42b53f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,7 +27,6 @@ extern crate rustc_incremental; extern crate rustc_index; extern crate rustc_session; extern crate rustc_span; -extern crate rustc_symbol_mangling; extern crate rustc_target; // This prevents duplicating functions and statics that are already part of the host rustc process. @@ -257,8 +256,6 @@ impl CodegenBackend for CraneliftCodegenBackend { }; let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config); - rustc_symbol_mangling::test::report_symbol_names(tcx); - res } @@ -280,18 +277,14 @@ impl CodegenBackend for CraneliftCodegenBackend { ) -> Result<(), ErrorReported> { use rustc_codegen_ssa::back::link::link_binary; - let _timer = sess.prof.generic_activity("link_crate"); - - sess.time("linking", || { - let target_cpu = crate::target_triple(sess).to_string(); - link_binary::>( - sess, - &codegen_results, - outputs, - &codegen_results.crate_name.as_str(), - &target_cpu, - ); - }); + let target_cpu = crate::target_triple(sess).to_string(); + link_binary::>( + sess, + &codegen_results, + outputs, + &codegen_results.crate_name.as_str(), + &target_cpu, + ); Ok(()) } From 3ea8915d4a247b5b3c4cfb3424c230ccd2645b17 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 6 Jan 2021 14:54:19 +0100 Subject: [PATCH 04/48] Disable timings by default for cargo.sh Fixes #1121 --- build_sysroot/build_sysroot.sh | 1 + scripts/config.sh | 1 - test.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index d7a72df2eb283..fff5a08bf11f2 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -12,6 +12,7 @@ dir=$(pwd) # build scripts are still compiled using cg_llvm. export RUSTC=$dir"/bin/cg_clif_build_sysroot" export RUSTFLAGS=$RUSTFLAGS" --clif" +export CG_CLIF_DISPLAY_CG_TIME=1 cd "$(dirname "$0")" diff --git a/scripts/config.sh b/scripts/config.sh index dea037e2bc002..d3c0885b3fa6d 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -56,4 +56,3 @@ fi export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -export CG_CLIF_DISPLAY_CG_TIME=1 diff --git a/test.sh b/test.sh index c6c4956e48174..ffd795b83ef93 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -export RUSTFLAGS="-Zrun_dsymutil=no" +export CG_CLIF_DISPLAY_CG_TIME=1 ./build.sh --without-sysroot "$@" From ea73caa3b77ac88df3d87d72826dc12831acb9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Sat, 16 Jan 2021 07:17:13 +0300 Subject: [PATCH 05/48] codegen_cranelift: Fix redundant semicolon warn --- src/intrinsics/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index be5b247bb9f0b..8946ac43bc65a 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -824,7 +824,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } ty => unreachable!("bswap {}", ty), } - }; + } let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T)); ret.write_cvalue(fx, res); }; From c34e165c666a8905acc827309d1e099d7630fee1 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 16 Jan 2021 11:46:21 +0100 Subject: [PATCH 06/48] Update Cranelift --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0382835269d1f..643446a68f125 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,16 +49,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "byteorder", "cranelift-bforest", @@ -75,8 +75,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -84,18 +84,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" [[package]] name = "cranelift-entity" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" [[package]] name = "cranelift-frontend" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "cranelift-codegen", "log", @@ -105,8 +105,8 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "anyhow", "cranelift-codegen", @@ -123,8 +123,8 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "anyhow", "cranelift-codegen", @@ -135,8 +135,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -145,8 +145,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +version = "0.69.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" dependencies = [ "anyhow", "cranelift-codegen", @@ -362,9 +362,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "smallvec" -version = "1.4.2" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "syn" From 1952b1ffaee277a6d10ba6aabf7fa9a3f87383c5 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 16 Jan 2021 11:50:58 +0100 Subject: [PATCH 07/48] Update dependencies --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 643446a68f125..c0f901b27f013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "anyhow" -version = "1.0.34" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "ar" @@ -25,15 +25,15 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "byteorder" -version = "1.3.4" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "cc" -version = "1.0.62" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -209,9 +209,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" [[package]] name = "indexmap" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" dependencies = [ "autocfg", "hashbrown", @@ -219,15 +219,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.80" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" [[package]] name = "libloading" -version = "0.6.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1090080fe06ec2648d0da3881d9453d97e71a45f00eb179af7fdd7e3f686fdb0" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" dependencies = [ "cfg-if 1.0.0", "winapi", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" dependencies = [ "cfg-if 0.1.10", ] @@ -272,9 +272,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" dependencies = [ "proc-macro2", ] @@ -368,9 +368,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "syn" -version = "1.0.48" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" dependencies = [ "proc-macro2", "quote", @@ -385,18 +385,18 @@ checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9" [[package]] name = "thiserror" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" dependencies = [ "proc-macro2", "quote", From cfedad1f75bf22468fce59f754daf1501fa2827d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 16 Jan 2021 13:05:23 +0100 Subject: [PATCH 08/48] Clear domtree after cg_clif optimizations --- src/base.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/base.rs b/src/base.rs index 34c9561d67622..757915ba99a36 100644 --- a/src/base.rs +++ b/src/base.rs @@ -117,6 +117,9 @@ pub(crate) fn codegen_fn<'tcx>( context.compute_domtree(); context.eliminate_unreachable_code(cx.module.isa()).unwrap(); context.dce(cx.module.isa()).unwrap(); + // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't + // invalidate it when it would change. + context.domtree.clear(); context.want_disasm = crate::pretty_clif::should_write_ir(tcx); From c3d7dc93dc24b044aafc521c5ee3d3f81c87a6a9 Mon Sep 17 00:00:00 2001 From: LeSeulArtichaut Date: Sat, 12 Dec 2020 15:32:30 +0100 Subject: [PATCH 09/48] Use ty::{IntTy,UintTy,FloatTy} in rustc --- src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6e4f3bf2898d8..f31c58b92e407 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,7 +83,6 @@ mod vtable; mod prelude { pub(crate) use std::convert::{TryFrom, TryInto}; - pub(crate) use rustc_ast::ast::{FloatTy, IntTy, UintTy}; pub(crate) use rustc_span::Span; pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE}; @@ -91,7 +90,7 @@ mod prelude { pub(crate) use rustc_middle::mir::{self, *}; pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout}; pub(crate) use rustc_middle::ty::{ - self, FnSig, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, + self, FloatTy, FnSig, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, UintTy, }; pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx}; From c0aefeb5b76bbb536f235d5c5d6674181b262537 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 21 Jan 2021 14:51:28 +0100 Subject: [PATCH 10/48] Rustup to rustc 1.51.0-nightly (a4cbb44ae 2021-01-20) --- build_sysroot/Cargo.lock | 12 +++---- build_sysroot/Cargo.toml | 3 +- build_sysroot/prepare_sysroot_src.sh | 9 ++++- ...builtins-Remove-rotate_left-from-Int.patch | 35 +++++++++++++++++++ rust-toolchain | 2 +- 5 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index f3f29957ecd63..b49518ee39a76 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -63,9 +63,7 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd0782e0a7da7598164153173e5a5d4d9b1da094473c98dce0ff91406112369" +version = "0.1.39" dependencies = [ "rustc-std-workspace-core", ] @@ -130,9 +128,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "compiler_builtins", "libc", @@ -141,9 +139,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" dependencies = [ "rustc-std-workspace-core", ] diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index 3dbd28c286a24..82516c98af2a2 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -11,12 +11,13 @@ test = { path = "./sysroot_src/library/test" } alloc_system = { path = "./alloc_system" } -compiler_builtins = { version = "=0.1.36", default-features = false } +compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] } [patch.crates-io] rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" } +compiler_builtins = { path = "./compiler-builtins" } [profile.dev] lto = "off" diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index 40fbaf646a2f6..d3b87e02ba891 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -29,4 +29,11 @@ git commit --no-gpg-sign -m "Patch $file" done popd -echo "Successfully prepared libcore for building" +git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" +pushd compiler-builtins +git checkout -- . +git checkout 0.1.39 +git apply ../../crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch +popd + +echo "Successfully prepared sysroot source for building" diff --git a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch new file mode 100644 index 0000000000000..e14768910a9ac --- /dev/null +++ b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch @@ -0,0 +1,35 @@ +From 7078cca3cb614e1e82da428380b4e16fc3afef46 Mon Sep 17 00:00:00 2001 +From: bjorn3 +Date: Thu, 21 Jan 2021 14:46:36 +0100 +Subject: [PATCH] Remove rotate_left from Int + +--- + src/int/mod.rs | 5 ----- + 1 file changed, 5 deletions(-) + +diff --git a/src/int/mod.rs b/src/int/mod.rs +index 06054c8..3bea17b 100644 +--- a/src/int/mod.rs ++++ b/src/int/mod.rs +@@ -85,7 +85,6 @@ pub trait Int: + fn wrapping_sub(self, other: Self) -> Self; + fn wrapping_shl(self, other: u32) -> Self; + fn wrapping_shr(self, other: u32) -> Self; +- fn rotate_left(self, other: u32) -> Self; + fn overflowing_add(self, other: Self) -> (Self, bool); + fn aborting_div(self, other: Self) -> Self; + fn aborting_rem(self, other: Self) -> Self; +@@ -209,10 +208,6 @@ macro_rules! int_impl_common { + ::wrapping_shr(self, other) + } + +- fn rotate_left(self, other: u32) -> Self { +- ::rotate_left(self, other) +- } +- + fn overflowing_add(self, other: Self) -> (Self, bool) { + ::overflowing_add(self, other) + } +-- +2.26.2.7.g19db9cfb68 + diff --git a/rust-toolchain b/rust-toolchain index a2b82fb1f4fb3..ff530ab260ed6 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2021-01-01 +nightly-2021-01-21 From 7f60301a7a2f851736ea47954a30d3ef1b723a6e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 21 Jan 2021 15:19:31 +0100 Subject: [PATCH 11/48] Disable new failing libcore tests --- .../0022-core-Disable-not-compiling-tests.patch | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/patches/0022-core-Disable-not-compiling-tests.patch b/patches/0022-core-Disable-not-compiling-tests.patch index 8cfffe580a1f0..3eb10069adad6 100644 --- a/patches/0022-core-Disable-not-compiling-tests.patch +++ b/patches/0022-core-Disable-not-compiling-tests.patch @@ -119,5 +119,21 @@ index 6609bc3..241b497 100644 #[test] #[should_panic(expected = "index 0 greater than length of slice")] +diff --git a/library/core/tests/num/ops.rs b/library/core/tests/num/ops.rs +index 9979cc8..d5d1d83 100644 +--- a/library/core/tests/num/ops.rs ++++ b/library/core/tests/num/ops.rs +@@ -238,7 +238,7 @@ macro_rules! test_shift_assign { + } + }; + } +-test_shift!(test_shl_defined, Shl::shl); +-test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign); +-test_shift!(test_shr_defined, Shr::shr); +-test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign); ++//test_shift!(test_shl_defined, Shl::shl); ++//test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign); ++//test_shift!(test_shr_defined, Shr::shr); ++//test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign); -- 2.21.0 (Apple Git-122) From 9bf5cb43aba913b18d08efaf451a7f3974cc65c8 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 21 Jan 2021 15:20:48 +0100 Subject: [PATCH 12/48] Fix indirectly linking to libstd.so with the JIT --- build.sh | 1 + build_sysroot/build_sysroot.sh | 1 + scripts/config.sh | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 26041b59cca18..48fb5819a1931 100755 --- a/build.sh +++ b/build.sh @@ -57,4 +57,5 @@ if [[ "$build_sysroot" == "1" ]]; then dir=$(pwd) cd "$target_dir" time "$dir/build_sysroot/build_sysroot.sh" + cp lib/rustlib/*/lib/libstd-* lib/ fi diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index fff5a08bf11f2..3d0a94ad3cbec 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -25,6 +25,7 @@ export CARGO_TARGET_DIR=target # Build libs export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort" +export __CARGO_DEFAULT_LIB_METADATA="cg_clif" if [[ "$1" != "--debug" ]]; then sysroot_channel='release' # FIXME Enable incremental again once rust-lang/rust#74946 is fixed diff --git a/scripts/config.sh b/scripts/config.sh index d3c0885b3fa6d..fe6d86ea6b489 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -53,6 +53,6 @@ if [[ $(uname) == 'Darwin' ]]; then export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" fi -export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib" +export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH From a19ef67f146df8da634cbab1e5ff5f3c2f3176d1 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 18 Jan 2021 14:48:30 +0100 Subject: [PATCH 13/48] Update Cranelift This implements everything in the new backend necessary for usage with cg_clif --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0f901b27f013..431e806869646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "cranelift-entity", ] @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "byteorder", "cranelift-bforest", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -85,17 +85,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" [[package]] name = "cranelift-entity" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" [[package]] name = "cranelift-frontend" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "cranelift-codegen", "log", @@ -106,7 +106,7 @@ dependencies = [ [[package]] name = "cranelift-jit" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "anyhow", "cranelift-codegen", @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "anyhow", "cranelift-codegen", @@ -136,7 +136,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -146,7 +146,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.69.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8640025d8b3b2ab5188ffc3f1a4b3976d49af3aa" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca" dependencies = [ "anyhow", "cranelift-codegen", From 173ec34e3db3bbbbe0117e2f831053a183d99a40 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 18 Jan 2021 14:48:50 +0100 Subject: [PATCH 14/48] Add a feature flag to switch between oldbe and newbe --- Cargo.toml | 3 ++- build.sh | 10 +++++++--- src/lib.rs | 7 ++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8e1933bb14e7c..eb9d4e09ebc66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } +cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x86", "x64"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } @@ -37,6 +37,7 @@ libloading = { version = "0.6.0", optional = true } default = ["jit", "inline_asm"] jit = ["cranelift-jit", "libloading"] inline_asm = [] +newbe = [] [profile.dev] # By compiling dependencies with optimizations, performing tests gets much faster. diff --git a/build.sh b/build.sh index 48fb5819a1931..a00c01a179cce 100755 --- a/build.sh +++ b/build.sh @@ -5,6 +5,7 @@ set -e export CHANNEL="release" build_sysroot=1 target_dir='build' +newbe='' while [[ $# != 0 ]]; do case $1 in "--debug") @@ -17,9 +18,12 @@ while [[ $# != 0 ]]; do target_dir=$2 shift ;; + "--newbe") + newbe='--features newbe' + ;; *) echo "Unknown flag '$1'" - echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR]" + echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR] [--newbe]" ;; esac shift @@ -39,9 +43,9 @@ else exit 1 fi if [[ "$CHANNEL" == "release" ]]; then - cargo build --release + cargo build $newbe --release else - cargo build + cargo build $newbe fi rm -rf "$target_dir" diff --git a/src/lib.rs b/src/lib.rs index 4b6431e42b53f..2152d2ebdef53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -339,7 +339,12 @@ fn build_isa(sess: &Session) -> Box { let flags = settings::Flags::new(flags_builder); - let mut isa_builder = cranelift_codegen::isa::lookup(target_triple).unwrap(); + let variant = if cfg!(feature = "newbe") { + cranelift_codegen::isa::BackendVariant::MachInst + } else { + cranelift_codegen::isa::BackendVariant::Legacy + }; + let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt` // is interpreted as `bsr`. isa_builder.enable("nehalem").unwrap(); From da4aa92df1cc1ea8b9b5b975f0f3d490f5300cfd Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 21 Jan 2021 19:33:19 +0100 Subject: [PATCH 15/48] Test both oldbe and newbe on CI --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e6d3375fb1bab..9eeca7c2819bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] + env: + - BACKEND: "" + - BACKEND: --features newbe steps: - uses: actions/checkout@v2 @@ -51,7 +54,7 @@ jobs: export COMPILE_RUNS=2 export RUN_RUNS=2 - ./test.sh + ./test.sh $BACKEND - name: Package prebuilt cg_clif run: tar cvfJ cg_clif.tar.xz build From 2ced8c0eb0df6ae52cfc63c88b92c70c46049178 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 21 Jan 2021 21:22:19 +0100 Subject: [PATCH 16/48] Fix rust-analyzer pref name --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7618251acd5c2..19ea41563dfd6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"], - "rust-analyzer.assist.importMergeBehaviour": "last", + "rust-analyzer.assist.importMergeBehavior": "last", "rust-analyzer.cargo.loadOutDirsFromCheck": true, "rust-analyzer.linkedProjects": [ "./Cargo.toml", From ae6daf77ba9762f4f118b2342298ce5fa2e32a7a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 22 Jan 2021 12:28:32 +0100 Subject: [PATCH 17/48] Default to Cranelift newBE --- .github/workflows/main.yml | 2 +- Cargo.toml | 2 +- build.sh | 13 +++++++------ src/lib.rs | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9eeca7c2819bb..20c58423a0c50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: os: [ubuntu-latest, macos-latest] env: - BACKEND: "" - - BACKEND: --features newbe + - BACKEND: --oldbe steps: - uses: actions/checkout@v2 diff --git a/Cargo.toml b/Cargo.toml index eb9d4e09ebc66..4558da2de73ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ libloading = { version = "0.6.0", optional = true } default = ["jit", "inline_asm"] jit = ["cranelift-jit", "libloading"] inline_asm = [] -newbe = [] +oldbe = [] [profile.dev] # By compiling dependencies with optimizations, performing tests gets much faster. diff --git a/build.sh b/build.sh index a00c01a179cce..522b3b3f5923e 100755 --- a/build.sh +++ b/build.sh @@ -5,7 +5,7 @@ set -e export CHANNEL="release" build_sysroot=1 target_dir='build' -newbe='' +oldbe='' while [[ $# != 0 ]]; do case $1 in "--debug") @@ -18,12 +18,13 @@ while [[ $# != 0 ]]; do target_dir=$2 shift ;; - "--newbe") - newbe='--features newbe' + "--oldbe") + oldbe='--features oldbe' ;; *) echo "Unknown flag '$1'" - echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR] [--newbe]" + echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR] [--oldbe]" + exit 1 ;; esac shift @@ -43,9 +44,9 @@ else exit 1 fi if [[ "$CHANNEL" == "release" ]]; then - cargo build $newbe --release + cargo build $oldbe --release else - cargo build $newbe + cargo build $oldbe fi rm -rf "$target_dir" diff --git a/src/lib.rs b/src/lib.rs index 2152d2ebdef53..9b5b7d8051c72 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -339,10 +339,10 @@ fn build_isa(sess: &Session) -> Box { let flags = settings::Flags::new(flags_builder); - let variant = if cfg!(feature = "newbe") { - cranelift_codegen::isa::BackendVariant::MachInst - } else { + let variant = if cfg!(feature = "oldbe") { cranelift_codegen::isa::BackendVariant::Legacy + } else { + cranelift_codegen::isa::BackendVariant::MachInst }; let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt` From d6eb4f571e53612adda69f50441c21841a39c4c7 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 24 Jan 2021 12:50:30 +0100 Subject: [PATCH 18/48] clean up some const error reporting around promoteds --- src/constant.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/constant.rs b/src/constant.rs index beff84fb2e217..5702832bcb67d 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -134,11 +134,9 @@ pub(crate) fn codegen_constant<'tcx>( { Ok(const_val) => const_val, Err(_) => { - if promoted.is_none() { - fx.tcx - .sess - .span_err(constant.span, "erroneous constant encountered"); - } + fx.tcx + .sess + .span_err(constant.span, "erroneous constant encountered"); return crate::trap::trap_unreachable_ret_value( fx, fx.layout_of(const_.ty), From d39b4411296585895b1111d14c1459ea3e9bf9be Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 10:45:01 +0100 Subject: [PATCH 19/48] Start using ArgAbi --- src/abi/pass_mode.rs | 80 +++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 8e3682c86c5fb..957beab745bc3 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -2,6 +2,7 @@ use crate::prelude::*; +use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; pub(super) use EmptySinglePair::*; #[derive(Copy, Clone, Debug)] @@ -83,39 +84,64 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer PassMode::NoPass } else { - match &layout.abi { - Abi::Uninhabited => PassMode::NoPass, - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - if a == types::I128 && b == types::I128 { - // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are - // available on x86_64. Cranelift gets confused when too many return params - // are used. - PassMode::ByRef { - size: Some(layout.size), + let arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); + match arg_abi.mode { + RustcPassMode::Ignore => PassMode::NoPass, + RustcPassMode::Direct(_) => match &arg_abi.layout.abi { + Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout) { + PassMode::ByVal(vector_ty) + } else { + PassMode::ByRef { + size: Some(arg_abi.layout.size), + } } - } else { - PassMode::ByValPair(a, b) } - } - - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, layout) { - PassMode::ByVal(vector_ty) - } else { - PassMode::ByRef { - size: Some(layout.size), + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + if a == types::I128 && b == types::I128 { + // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are + // available on x86_64. Cranelift gets confused when too many return params + // are used. + PassMode::ByRef { + size: Some(arg_abi.layout.size), + } + } else { + PassMode::ByValPair(a, b) } } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Cast(_) | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: false, + } => PassMode::ByRef { + size: Some(arg_abi.layout.size), + }, + RustcPassMode::Indirect { + attrs: _, + extra_attrs, + on_stack: true, + } => { + assert!(extra_attrs.is_none()); + PassMode::ByRef { + size: Some(arg_abi.layout.size) + } } - - Abi::Aggregate { sized: true } => PassMode::ByRef { - size: Some(layout.size), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: false, + } => PassMode::ByRef { + size: None, }, - Abi::Aggregate { sized: false } => PassMode::ByRef { size: None }, } } } From 6170fc617ef61698705e3c3d8847ce72d92991ca Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 10:57:59 +0100 Subject: [PATCH 20/48] Split abi adjustments out --- src/abi/pass_mode.rs | 131 ++++++++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 57 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 957beab745bc3..2d2410f2d6ca1 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -80,69 +80,86 @@ impl PassMode { } pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode { + let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); if layout.is_zst() { // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer - PassMode::NoPass - } else { - let arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); - match arg_abi.mode { - RustcPassMode::Ignore => PassMode::NoPass, - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout) { - PassMode::ByVal(vector_ty) - } else { - PassMode::ByRef { - size: Some(arg_abi.layout.size), - } - } - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - if a == types::I128 && b == types::I128 { - // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are - // available on x86_64. Cranelift gets confused when too many return params - // are used. - PassMode::ByRef { - size: Some(arg_abi.layout.size), - } - } else { - PassMode::ByValPair(a, b) - } + arg_abi.mode = RustcPassMode::Ignore; + } + match arg_abi.mode { + RustcPassMode::Ignore => {} + RustcPassMode::Direct(_) => match &arg_abi.layout.abi { + Abi::Scalar(_) => {}, + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { + arg_abi.mode = RustcPassMode::Indirect { + attrs: ArgAttributes::new(), + extra_attrs: None, + on_stack: false, + }; } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Cast(_) | RustcPassMode::Indirect { - attrs: _, - extra_attrs: None, - on_stack: false, - } => PassMode::ByRef { - size: Some(arg_abi.layout.size), - }, - RustcPassMode::Indirect { - attrs: _, - extra_attrs, - on_stack: true, - } => { - assert!(extra_attrs.is_none()); - PassMode::ByRef { - size: Some(arg_abi.layout.size) + } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + if a == types::I128 && b == types::I128 { + arg_abi.mode = RustcPassMode::Indirect { + attrs: ArgAttributes::new(), + extra_attrs: None, + on_stack: false, + }; } } - RustcPassMode::Indirect { - attrs: _, - extra_attrs: Some(_), - on_stack: false, - } => PassMode::ByRef { - size: None, - }, + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + _ => {} + } + match arg_abi.mode { + RustcPassMode::Ignore => PassMode::NoPass, + RustcPassMode::Direct(_) => match &arg_abi.layout.abi { + Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).unwrap(); + PassMode::ByVal(vector_ty) + } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + PassMode::ByValPair(a, b) + } + _ => unreachable!("{:?}", arg_abi.layout.abi) + }, + RustcPassMode::Cast(_) | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: false, + } => PassMode::ByRef { + size: Some(arg_abi.layout.size), + }, + RustcPassMode::Indirect { + attrs: _, + extra_attrs, + on_stack: true, + } => { + assert!(extra_attrs.is_none()); + PassMode::ByRef { + size: Some(arg_abi.layout.size) + } } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: false, + } => PassMode::ByRef { + size: None, + }, } } From ff3304285a41c85486249c5db337e4561ef970c5 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 11:40:26 +0100 Subject: [PATCH 21/48] Rustup to rustc 1.51.0-nightly (1d0d76f8d 2021-01-24) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index ff530ab260ed6..55ac079c0a9b5 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2021-01-21 +nightly-2021-01-25 From de713a80cac60164f435a4e6a7ca710f5fdccd45 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 15:37:49 +0100 Subject: [PATCH 22/48] Replace all uses of PassMode with ArgAbi --- src/abi/comments.rs | 5 +- src/abi/mod.rs | 61 +++--------- src/abi/pass_mode.rs | 230 +++++++++++++++++++++++++++---------------- src/abi/returning.rs | 95 ++++++++++++------ 4 files changed, 232 insertions(+), 159 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 01073d26e832a..af42e54451b16 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use rustc_middle::mir; +use rustc_target::abi::call::ArgAbi; use cranelift_codegen::entity::EntityRef; @@ -22,7 +23,7 @@ pub(super) fn add_arg_comment<'tcx>( local: Option, local_field: Option, params: EmptySinglePair, - pass_mode: PassMode, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ty: Ty<'tcx>, ) { let local = if let Some(local) = local { @@ -42,7 +43,7 @@ pub(super) fn add_arg_comment<'tcx>( Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), }; - let pass_mode = format!("{:?}", pass_mode); + let pass_mode = format!("{:?}", arg_abi.mode); fx.add_global_comment(format!( "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", kind = kind, diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 76e1987459f87..ddbef5eadfecf 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -6,9 +6,10 @@ mod pass_mode; mod returning; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_target::abi::call::PassMode as RustcPassMode; use rustc_target::spec::abi::Abi; -use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; +use cranelift_codegen::ir::AbiParam; use self::pass_mode::*; use crate::prelude::*; @@ -96,7 +97,6 @@ fn clif_sig_from_fn_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, sig: FnSig<'tcx>, - span: Span, is_vtable_fn: bool, requires_caller_location: bool, ) -> Signature { @@ -147,54 +147,26 @@ fn clif_sig_from_fn_sig<'tcx>( .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))) .unwrap(); } - let pass_mode = get_pass_mode(tcx, layout); + let mut arg_abi = get_arg_abi(tcx, layout); if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic { - match pass_mode { - PassMode::NoPass | PassMode::ByVal(_) => {} - PassMode::ByRef { size: Some(size) } => { - let purpose = ArgumentPurpose::StructArgument(u32::try_from(size.bytes()).expect("struct too big to pass on stack")); - return EmptySinglePair::Single(AbiParam::special(pointer_ty(tcx), purpose)).into_iter(); - } - PassMode::ByValPair(_, _) | PassMode::ByRef { size: None } => { - tcx.sess.span_warn( - span, - &format!( - "Argument of type `{:?}` with pass mode `{:?}` is not yet supported \ - for non-rust abi `{}`. Calling this function may result in a crash.", - layout.ty, - pass_mode, - abi, - ), - ); - } + match arg_abi.mode { + RustcPassMode::Indirect { + ref mut on_stack, .. + } => *on_stack = true, + _ => {} } } - pass_mode.get_param_ty(tcx).map(AbiParam::new).into_iter() + arg_abi.get_abi_param(tcx).into_iter() }) .flatten(); - let (mut params, returns): (Vec<_>, Vec<_>) = match get_pass_mode( + let return_arg_abi = get_arg_abi( tcx, tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(), - ) { - PassMode::NoPass => (inputs.collect(), vec![]), - PassMode::ByVal(ret_ty) => (inputs.collect(), vec![AbiParam::new(ret_ty)]), - PassMode::ByValPair(ret_ty_a, ret_ty_b) => ( - inputs.collect(), - vec![AbiParam::new(ret_ty_a), AbiParam::new(ret_ty_b)], - ), - PassMode::ByRef { size: Some(_) } => { - ( - Some(pointer_ty(tcx)) // First param is place to put return val - .into_iter() - .map(|ty| AbiParam::special(ty, ArgumentPurpose::StructReturn)) - .chain(inputs) - .collect(), - vec![], - ) - } - PassMode::ByRef { size: None } => todo!(), - }; + ); + let (return_ptr, returns) = return_arg_abi.get_abi_return(tcx); + // Sometimes the first param is an pointer to the place where the return value needs to be stored. + let mut params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); if requires_caller_location { params.push(AbiParam::new(pointer_ty(tcx))); @@ -226,7 +198,6 @@ pub(crate) fn get_function_name_and_sig<'tcx>( tcx, triple, fn_sig, - tcx.def_span(inst.def_id()), false, inst.def.requires_caller_location(tcx), ); @@ -584,7 +555,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( nop_inst, format!( "virtual call; self arg pass mode: {:?}", - get_pass_mode(fx.tcx, args[0].layout()) + get_arg_abi(fx.tcx, args[0].layout()).mode, ), ); } @@ -647,7 +618,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( fx.tcx, fx.triple(), fn_sig, - span, is_virtual_call, false, // calls through function pointers never pass the caller location ); @@ -723,7 +693,6 @@ pub(crate) fn codegen_drop<'tcx>( fx.tcx, fx.triple(), fn_sig, - span, true, false, // `drop_in_place` is never `#[track_caller]` ); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 2d2410f2d6ca1..51fc4ecd1ef09 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -2,17 +2,10 @@ use crate::prelude::*; +use cranelift_codegen::ir::ArgumentPurpose; use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; pub(super) use EmptySinglePair::*; -#[derive(Copy, Clone, Debug)] -pub(super) enum PassMode { - NoPass, - ByVal(Type), - ByValPair(Type, Type), - ByRef { size: Option }, -} - #[derive(Copy, Clone, Debug)] pub(super) enum EmptySinglePair { Empty, @@ -67,19 +60,126 @@ impl EmptySinglePair { } } -impl PassMode { - pub(super) fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair { - match self { - PassMode::NoPass => Empty, - PassMode::ByVal(clif_type) => Single(clif_type), - PassMode::ByValPair(a, b) => Pair(a, b), - PassMode::ByRef { size: Some(_) } => Single(pointer_ty(tcx)), - PassMode::ByRef { size: None } => Pair(pointer_ty(tcx), pointer_ty(tcx)), +pub(super) trait ArgAbiExt<'tcx> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair; + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); +} + +impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { + match self.mode { + RustcPassMode::Ignore => EmptySinglePair::Empty, + RustcPassMode::Direct(_) => match &self.layout.abi { + Abi::Scalar(scalar) => { + EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) + } + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + EmptySinglePair::Single(AbiParam::new(vector_ty)) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Pair(_, _) => match &self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + EmptySinglePair::Pair(AbiParam::new(a), AbiParam::new(b)) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack, + } => { + if on_stack { + let size = u32::try_from(self.layout.size.bytes()).unwrap(); + EmptySinglePair::Single(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructArgument(size), + )) + } else { + EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) + } + } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack, + } => { + assert!(!on_stack); + EmptySinglePair::Pair( + AbiParam::new(pointer_ty(tcx)), + AbiParam::new(pointer_ty(tcx)), + ) + } + } + } + + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec) { + match self.mode { + RustcPassMode::Ignore => (None, vec![]), + RustcPassMode::Direct(_) => match &self.layout.abi { + Abi::Scalar(scalar) => ( + None, + vec![AbiParam::new(scalar_to_clif_type( + tcx, + scalar.clone(), + ))], + ), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + (None, vec![AbiParam::new(vector_ty)]) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Pair(_, _) => match &self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + ( + None, + vec![AbiParam::new(a), AbiParam::new(b)], + ) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Cast(_) => ( + Some(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructReturn, + )), + vec![], + ), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack, + } => { + assert!(!on_stack); + ( + Some(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructReturn, + )), + vec![], + ) + } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), } } } -pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode { +pub(super) fn get_arg_abi<'tcx>( + tcx: TyCtxt<'tcx>, + layout: TyAndLayout<'tcx>, +) -> ArgAbi<'tcx, Ty<'tcx>> { let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); if layout.is_zst() { // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer @@ -88,7 +188,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) match arg_abi.mode { RustcPassMode::Ignore => {} RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(_) => {}, + Abi::Scalar(_) => {} // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { @@ -99,7 +199,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) }; } } - _ => unreachable!("{:?}", arg_abi.layout.abi) + _ => unreachable!("{:?}", arg_abi.layout.abi), }, RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { Abi::ScalarPair(a, b) => { @@ -113,54 +213,11 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) }; } } - _ => unreachable!("{:?}", arg_abi.layout.abi) + _ => unreachable!("{:?}", arg_abi.layout.abi), }, _ => {} } - match arg_abi.mode { - RustcPassMode::Ignore => PassMode::NoPass, - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - let vector_ty = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).unwrap(); - PassMode::ByVal(vector_ty) - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - PassMode::ByValPair(a, b) - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Cast(_) | RustcPassMode::Indirect { - attrs: _, - extra_attrs: None, - on_stack: false, - } => PassMode::ByRef { - size: Some(arg_abi.layout.size), - }, - RustcPassMode::Indirect { - attrs: _, - extra_attrs, - on_stack: true, - } => { - assert!(extra_attrs.is_none()); - PassMode::ByRef { - size: Some(arg_abi.layout.size) - } - } - RustcPassMode::Indirect { - attrs: _, - extra_attrs: Some(_), - on_stack: false, - } => PassMode::ByRef { - size: None, - }, - } + arg_abi } /// Get a set of values to be passed as function arguments. @@ -168,14 +225,15 @@ pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, ) -> EmptySinglePair { - match get_pass_mode(fx.tcx, arg.layout()) { - PassMode::NoPass => Empty, - PassMode::ByVal(_) => Single(arg.load_scalar(fx)), - PassMode::ByValPair(_, _) => { + let arg_abi = get_arg_abi(fx.tcx, arg.layout()); + match arg_abi.mode { + RustcPassMode::Ignore => Empty, + RustcPassMode::Direct(_) => Single(arg.load_scalar(fx)), + RustcPassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); Pair(a, b) } - PassMode::ByRef { size: _ } => match arg.force_stack(fx) { + RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => Single(ptr.get_addr(fx)), (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), }, @@ -192,14 +250,11 @@ pub(super) fn cvalue_for_param<'tcx>( arg_ty: Ty<'tcx>, ) -> Option> { let layout = fx.layout_of(arg_ty); - let pass_mode = get_pass_mode(fx.tcx, layout); + let arg_abi = get_arg_abi(fx.tcx, layout); - if let PassMode::NoPass = pass_mode { - return None; - } - - let clif_types = pass_mode.get_param_ty(fx.tcx); - let block_params = clif_types.map(|t| fx.bcx.append_block_param(start_block, t)); + let clif_types = arg_abi.get_abi_param(fx.tcx); + let block_params = + clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( @@ -208,22 +263,31 @@ pub(super) fn cvalue_for_param<'tcx>( local, local_field, block_params, - pass_mode, + &arg_abi, arg_ty, ); - match pass_mode { - PassMode::NoPass => unreachable!(), - PassMode::ByVal(_) => Some(CValue::by_val(block_params.assert_single(), layout)), - PassMode::ByValPair(_, _) => { + match arg_abi.mode { + RustcPassMode::Ignore => None, + RustcPassMode::Direct(_) => Some(CValue::by_val(block_params.assert_single(), layout)), + RustcPassMode::Pair(_, _) => { let (a, b) = block_params.assert_pair(); Some(CValue::by_val_pair(a, b, layout)) } - PassMode::ByRef { size: Some(_) } => Some(CValue::by_ref( + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => Some(CValue::by_ref( Pointer::new(block_params.assert_single()), layout, )), - PassMode::ByRef { size: None } => { + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => { let (ptr, meta) = block_params.assert_pair(); Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout)) } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index f6d40c880d094..9edaa1dd879fc 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -3,6 +3,8 @@ use crate::abi::pass_mode::*; use crate::prelude::*; +use rustc_target::abi::call::PassMode as RustcPassMode; + fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> { fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty)) } @@ -12,10 +14,10 @@ pub(crate) fn can_return_to_ssa_var<'tcx>( tcx: TyCtxt<'tcx>, dest_layout: TyAndLayout<'tcx>, ) -> bool { - match get_pass_mode(tcx, dest_layout) { - PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => true, - // FIXME Make it possible to return ByRef to an ssa var. - PassMode::ByRef { size: _ } => false, + match get_arg_abi(tcx, dest_layout).mode { + RustcPassMode::Ignore | RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => true, + // FIXME Make it possible to return Cast and Indirect to an ssa var. + RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => false, } } @@ -27,24 +29,33 @@ pub(super) fn codegen_return_param<'tcx>( start_block: Block, ) -> CPlace<'tcx> { let ret_layout = return_layout(fx); - let ret_pass_mode = get_pass_mode(fx.tcx, ret_layout); - let (ret_place, ret_param) = match ret_pass_mode { - PassMode::NoPass => (CPlace::no_place(ret_layout), Empty), - PassMode::ByVal(_) | PassMode::ByValPair(_, _) => { + let ret_arg_abi = get_arg_abi(fx.tcx, ret_layout); + let (ret_place, ret_param) = match ret_arg_abi.mode { + RustcPassMode::Ignore => (CPlace::no_place(ret_layout), Empty), + RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa), Empty, ) } - PassMode::ByRef { size: Some(_) } => { + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => { let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); ( CPlace::for_ptr(Pointer::new(ret_param), ret_layout), Single(ret_param), ) } - PassMode::ByRef { size: None } => todo!(), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), }; #[cfg(not(debug_assertions))] @@ -57,7 +68,7 @@ pub(super) fn codegen_return_param<'tcx>( Some(RETURN_PLACE), None, ret_param, - ret_pass_mode, + &ret_arg_abi, ret_layout.ty, ); @@ -74,36 +85,54 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ) -> (Inst, T) { let ret_layout = fx.layout_of(fn_sig.output()); - let output_pass_mode = get_pass_mode(fx.tcx, ret_layout); - let return_ptr = match output_pass_mode { - PassMode::NoPass => None, - PassMode::ByRef { size: Some(_) } => match ret_place { + let output_arg_abi = get_arg_abi(fx.tcx, ret_layout); + let return_ptr = match output_arg_abi.mode { + RustcPassMode::Ignore => None, + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => match ret_place { Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)), None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot }, - PassMode::ByRef { size: None } => todo!(), - PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None, + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), + RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => None, }; let (call_inst, meta) = f(fx, return_ptr); - match output_pass_mode { - PassMode::NoPass => {} - PassMode::ByVal(_) => { + match output_arg_abi.mode { + RustcPassMode::Ignore => {} + RustcPassMode::Direct(_) => { if let Some(ret_place) = ret_place { let ret_val = fx.bcx.inst_results(call_inst)[0]; ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout)); } } - PassMode::ByValPair(_, _) => { + RustcPassMode::Pair(_, _) => { if let Some(ret_place) = ret_place { let ret_val_a = fx.bcx.inst_results(call_inst)[0]; let ret_val_b = fx.bcx.inst_results(call_inst)[1]; ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout)); } } - PassMode::ByRef { size: Some(_) } => {} - PassMode::ByRef { size: None } => todo!(), + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => {} + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), } (call_inst, meta) @@ -111,17 +140,27 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { - match get_pass_mode(fx.tcx, return_layout(fx)) { - PassMode::NoPass | PassMode::ByRef { size: Some(_) } => { + match get_arg_abi(fx.tcx, return_layout(fx)).mode { + RustcPassMode::Ignore + | RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => { fx.bcx.ins().return_(&[]); } - PassMode::ByRef { size: None } => todo!(), - PassMode::ByVal(_) => { + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), + RustcPassMode::Direct(_) => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx).load_scalar(fx); fx.bcx.ins().return_(&[ret_val]); } - PassMode::ByValPair(_, _) => { + RustcPassMode::Pair(_, _) => { let place = fx.get_local_place(RETURN_PLACE); let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); From 2b58d8c187936567d92ab45be8a34be087071e05 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 17:12:16 +0100 Subject: [PATCH 23/48] Misc cleanups --- src/abi/comments.rs | 3 +-- src/abi/pass_mode.rs | 32 +++++--------------------------- src/abi/returning.rs | 1 - 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index af42e54451b16..4847b007a3669 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -24,7 +24,6 @@ pub(super) fn add_arg_comment<'tcx>( local_field: Option, params: EmptySinglePair, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - ty: Ty<'tcx>, ) { let local = if let Some(local) = local { Cow::Owned(format!("{:?}", local)) @@ -51,7 +50,7 @@ pub(super) fn add_arg_comment<'tcx>( local_field = local_field, params = params, pass_mode = pass_mode, - ty = ty, + ty = arg_abi.layout.ty, )); } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 51fc4ecd1ef09..aec321bd4a040 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -123,10 +123,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { RustcPassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => ( None, - vec![AbiParam::new(scalar_to_clif_type( - tcx, - scalar.clone(), - ))], + vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))], ), // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { @@ -139,10 +136,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); - ( - None, - vec![AbiParam::new(a), AbiParam::new(b)], - ) + (None, vec![AbiParam::new(a), AbiParam::new(b)]) } _ => unreachable!("{:?}", self.layout.abi), }, @@ -192,11 +186,7 @@ pub(super) fn get_arg_abi<'tcx>( // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { - arg_abi.mode = RustcPassMode::Indirect { - attrs: ArgAttributes::new(), - extra_attrs: None, - on_stack: false, - }; + arg_abi.make_indirect(); } } _ => unreachable!("{:?}", arg_abi.layout.abi), @@ -206,11 +196,7 @@ pub(super) fn get_arg_abi<'tcx>( let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); if a == types::I128 && b == types::I128 { - arg_abi.mode = RustcPassMode::Indirect { - attrs: ArgAttributes::new(), - extra_attrs: None, - on_stack: false, - }; + arg_abi.make_indirect(); } } _ => unreachable!("{:?}", arg_abi.layout.abi), @@ -257,15 +243,7 @@ pub(super) fn cvalue_for_param<'tcx>( clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] - crate::abi::comments::add_arg_comment( - fx, - "arg", - local, - local_field, - block_params, - &arg_abi, - arg_ty, - ); + crate::abi::comments::add_arg_comment(fx, "arg", local, local_field, block_params, &arg_abi); match arg_abi.mode { RustcPassMode::Ignore => None, diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 9edaa1dd879fc..3a5f61315f832 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -69,7 +69,6 @@ pub(super) fn codegen_return_param<'tcx>( None, ret_param, &ret_arg_abi, - ret_layout.ty, ); ret_place From 4555737152c0f68df5596b16d6e996d19caf2a6a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 26 Jan 2021 15:11:03 +0100 Subject: [PATCH 24/48] Split symbol name and signature calculation --- src/abi/mod.rs | 12 ++++++------ src/base.rs | 3 ++- src/driver/jit.rs | 12 ++++-------- src/driver/mod.rs | 3 ++- src/main_shim.rs | 4 ++-- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index ddbef5eadfecf..bc2111726d254 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -179,12 +179,12 @@ fn clif_sig_from_fn_sig<'tcx>( } } -pub(crate) fn get_function_name_and_sig<'tcx>( +pub(crate) fn get_function_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, inst: Instance<'tcx>, support_vararg: bool, -) -> (String, Signature) { +) -> Signature { assert!(!inst.substs.needs_infer()); let fn_sig = tcx .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_sig_for_fn_abi(tcx, inst)); @@ -194,14 +194,13 @@ pub(crate) fn get_function_name_and_sig<'tcx>( "Variadic function definitions are not yet supported", ); } - let sig = clif_sig_from_fn_sig( + clif_sig_from_fn_sig( tcx, triple, fn_sig, false, inst.def.requires_caller_location(tcx), - ); - (tcx.symbol_name(inst).name.to_string(), sig) + ) } /// Instance must be monomorphized @@ -210,7 +209,8 @@ pub(crate) fn import_function<'tcx>( module: &mut impl Module, inst: Instance<'tcx>, ) -> FuncId { - let (name, sig) = get_function_name_and_sig(tcx, module.isa().triple(), inst, true); + let name = tcx.symbol_name(inst).name.to_string(); + let sig = get_function_sig(tcx, module.isa().triple(), inst, true); module .declare_function(&name, Linkage::Import, &sig) .unwrap() diff --git a/src/base.rs b/src/base.rs index 757915ba99a36..1fafc1215975e 100644 --- a/src/base.rs +++ b/src/base.rs @@ -19,7 +19,8 @@ pub(crate) fn codegen_fn<'tcx>( let mir = tcx.instance_mir(instance.def); // Declare function - let (name, sig) = get_function_name_and_sig(tcx, cx.module.isa().triple(), instance, false); + let name = tcx.symbol_name(instance).name.to_string(); + let sig = get_function_sig(tcx, cx.module.isa().triple(), instance, false); let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); cx.cached_context.clear(); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 9a42c675cc144..6a87925927707 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -156,12 +156,8 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 let jit_module = jit_module.as_mut().unwrap(); let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); - let (name, sig) = crate::abi::get_function_name_and_sig( - tcx, - cx.module.isa().triple(), - instance, - true, - ); + let name = tcx.symbol_name(instance).name.to_string(); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance, true); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) @@ -246,8 +242,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In let pointer_type = cx.module.target_config().pointer_type(); - let (name, sig) = - crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true); + let name = tcx.symbol_name(inst).name.to_string(); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst, true); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 9f4ea9a386551..e462f34a04f99 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -50,7 +50,8 @@ fn predefine_mono_items<'tcx>( for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let (name, sig) = get_function_name_and_sig( + let name = cx.tcx.symbol_name(instance).name.to_string(); + let sig= get_function_sig( cx.tcx, cx.module.isa().triple(), instance, diff --git a/src/main_shim.rs b/src/main_shim.rs index 6c472e6774fe7..7900abb32a3fc 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -69,8 +69,8 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); - let (main_name, main_sig) = - get_function_name_and_sig(tcx, m.isa().triple(), instance, false); + let main_name = tcx.symbol_name(instance).name.to_string(); + let main_sig = get_function_sig(tcx, m.isa().triple(), instance, false); let main_func_id = m .declare_function(&main_name, Linkage::Import, &main_sig) .unwrap(); From fc595f1a555d7f43802679511e9fdf1f64f2c49a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 26 Jan 2021 21:41:20 +0100 Subject: [PATCH 25/48] [WIP] Use FnAbi everywhere instead of our own abi calculations --- src/abi/comments.rs | 9 +- src/abi/mod.rs | 227 ++++++++++++++++++----------------------- src/abi/pass_mode.rs | 113 ++++++++------------ src/abi/returning.rs | 138 ++++++++++++++++--------- src/analyze.rs | 6 +- src/base.rs | 3 + src/common.rs | 57 +++++++++-- src/lib.rs | 2 +- src/value_and_place.rs | 117 ++++++++++----------- 9 files changed, 355 insertions(+), 317 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 4847b007a3669..41cb4c627f899 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -4,7 +4,7 @@ use std::borrow::Cow; use rustc_middle::mir; -use rustc_target::abi::call::ArgAbi; +use rustc_target::abi::call::PassMode; use cranelift_codegen::entity::EntityRef; @@ -23,7 +23,8 @@ pub(super) fn add_arg_comment<'tcx>( local: Option, local_field: Option, params: EmptySinglePair, - arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + arg_abi_mode: PassMode, + arg_layout: TyAndLayout<'tcx>, ) { let local = if let Some(local) = local { Cow::Owned(format!("{:?}", local)) @@ -42,7 +43,7 @@ pub(super) fn add_arg_comment<'tcx>( Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), }; - let pass_mode = format!("{:?}", arg_abi.mode); + let pass_mode = format!("{:?}", arg_abi_mode); fx.add_global_comment(format!( "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", kind = kind, @@ -50,7 +51,7 @@ pub(super) fn add_arg_comment<'tcx>( local_field = local_field, params = params, pass_mode = pass_mode, - ty = arg_abi.layout.ty, + ty = arg_layout.ty, )); } diff --git a/src/abi/mod.rs b/src/abi/mod.rs index bc2111726d254..55ebd39e3f132 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -6,7 +6,8 @@ mod pass_mode; mod returning; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_target::abi::call::PassMode as RustcPassMode; +use rustc_middle::ty::layout::FnAbiExt; +use rustc_target::abi::call::{Conv, FnAbi}; use rustc_target::spec::abi::Abi; use cranelift_codegen::ir::AbiParam; @@ -16,6 +17,7 @@ use crate::prelude::*; pub(crate) use self::returning::{can_return_to_ssa_var, codegen_return}; +// FIXME remove // Copied from https://github.com/rust-lang/rust/blob/f52c72948aa1dd718cc1f168d21c91c584c0a662/src/librustc_middle/ty/layout.rs#L2301 #[rustfmt::skip] pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::PolyFnSig<'tcx> { @@ -93,84 +95,38 @@ pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx } } -fn clif_sig_from_fn_sig<'tcx>( +fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, - sig: FnSig<'tcx>, - is_vtable_fn: bool, - requires_caller_location: bool, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> Signature { - let abi = match sig.abi { - Abi::System => Abi::C, - abi => abi, - }; - let (call_conv, inputs, output): (CallConv, Vec>, Ty<'tcx>) = match abi { - Abi::Rust => ( - CallConv::triple_default(triple), - sig.inputs().to_vec(), - sig.output(), - ), - Abi::C | Abi::Unadjusted => ( - CallConv::triple_default(triple), - sig.inputs().to_vec(), - sig.output(), - ), - Abi::SysV64 => (CallConv::SystemV, sig.inputs().to_vec(), sig.output()), - Abi::RustCall => { - assert_eq!(sig.inputs().len(), 2); - let extra_args = match sig.inputs().last().unwrap().kind() { - ty::Tuple(ref tupled_arguments) => tupled_arguments, - _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"), - }; - let mut inputs: Vec> = vec![sig.inputs()[0]]; - inputs.extend(extra_args.types()); - (CallConv::triple_default(triple), inputs, sig.output()) + let call_conv = match fn_abi.conv { + Conv::Rust | Conv::C => CallConv::triple_default(triple), + Conv::X86_64SysV => CallConv::SystemV, + Conv::X86_64Win64 => CallConv::WindowsFastcall, + Conv::ArmAapcs + | Conv::Msp430Intr + | Conv::PtxKernel + | Conv::X86Fastcall + | Conv::X86Intr + | Conv::X86Stdcall + | Conv::X86ThisCall + | Conv::X86VectorCall + | Conv::AmdGpuKernel + | Conv::AvrInterrupt + | Conv::AvrNonBlockingInterrupt => { + todo!("{:?}", fn_abi.conv) } - Abi::System => unreachable!(), - Abi::RustIntrinsic => ( - CallConv::triple_default(triple), - sig.inputs().to_vec(), - sig.output(), - ), - _ => unimplemented!("unsupported abi {:?}", sig.abi), }; - - let inputs = inputs - .into_iter() - .enumerate() - .map(|(i, ty)| { - let mut layout = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap(); - if i == 0 && is_vtable_fn { - // Virtual calls turn their self param into a thin pointer. - // See https://github.com/rust-lang/rust/blob/37b6a5e5e82497caf5353d9d856e4eb5d14cbe06/src/librustc/ty/layout.rs#L2519-L2572 for more info - layout = tcx - .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))) - .unwrap(); - } - let mut arg_abi = get_arg_abi(tcx, layout); - if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic { - match arg_abi.mode { - RustcPassMode::Indirect { - ref mut on_stack, .. - } => *on_stack = true, - _ => {} - } - } - arg_abi.get_abi_param(tcx).into_iter() - }) + let inputs = fn_abi + .args + .iter() + .map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()) .flatten(); - let return_arg_abi = get_arg_abi( - tcx, - tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(), - ); - let (return_ptr, returns) = return_arg_abi.get_abi_return(tcx); + let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx); // Sometimes the first param is an pointer to the place where the return value needs to be stored. - let mut params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); - - if requires_caller_location { - params.push(AbiParam::new(pointer_ty(tcx))); - } + let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); Signature { params, @@ -194,12 +150,11 @@ pub(crate) fn get_function_sig<'tcx>( "Variadic function definitions are not yet supported", ); } - clif_sig_from_fn_sig( + + clif_sig_from_fn_abi( tcx, triple, - fn_sig, - false, - inst.def.requires_caller_location(tcx), + &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]), ) } @@ -337,6 +292,9 @@ pub(crate) fn codegen_fn_prelude<'tcx>( Spread(Vec>>), } + let fn_abi = fx.fn_abi.take().unwrap(); + let mut arg_abis_iter = fn_abi.args.iter(); + let func_params = fx .mir .args_iter() @@ -356,14 +314,16 @@ pub(crate) fn codegen_fn_prelude<'tcx>( }; let mut params = Vec::new(); - for (i, arg_ty) in tupled_arg_tys.types().enumerate() { - let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_ty); + for (i, _arg_ty) in tupled_arg_tys.types().enumerate() { + let arg_abi = arg_abis_iter.next().unwrap(); + let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_abi); params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { - let param = cvalue_for_param(fx, start_block, Some(local), None, arg_ty); + let arg_abi = arg_abis_iter.next().unwrap(); + let param = cvalue_for_param(fx, start_block, Some(local), None, arg_abi); (local, ArgKind::Normal(param), arg_ty) } }) @@ -372,11 +332,13 @@ pub(crate) fn codegen_fn_prelude<'tcx>( assert!(fx.caller_location.is_none()); if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. - fx.caller_location = Some( - cvalue_for_param(fx, start_block, None, None, fx.tcx.caller_location_ty()).unwrap(), - ); + let arg_abi = arg_abis_iter.next().unwrap(); + fx.caller_location = Some(cvalue_for_param(fx, start_block, None, None, arg_abi).unwrap()); } + assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind"); + fx.fn_abi = Some(fn_abi); + fx.bcx.switch_to_block(start_block); fx.bcx.ins().nop(); @@ -504,6 +466,21 @@ pub(crate) fn codegen_terminator_call<'tcx>( None }; + let extra_args = &args[fn_sig.inputs().len()..]; + let extra_args = extra_args + .iter() + .map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))) + .collect::>(); + let fn_abi = if let Some(instance) = instance { + FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args) + } else { + FnAbi::of_fn_ptr( + &RevealAllLayoutCx(fx.tcx), + fn_ty.fn_sig(fx.tcx), + &extra_args, + ) + }; + let is_cold = instance .map(|inst| { fx.tcx @@ -541,8 +518,8 @@ pub(crate) fn codegen_terminator_call<'tcx>( // | indirect call target // | | the first argument to be passed - // v v v virtual calls are special cased below - let (func_ref, first_arg, is_virtual_call) = match instance { + // v v + let (func_ref, first_arg) = match instance { // Trait object call Some(Instance { def: InstanceDef::Virtual(_, idx), @@ -553,23 +530,19 @@ pub(crate) fn codegen_terminator_call<'tcx>( let nop_inst = fx.bcx.ins().nop(); fx.add_comment( nop_inst, - format!( - "virtual call; self arg pass mode: {:?}", - get_arg_abi(fx.tcx, args[0].layout()).mode, - ), + format!("virtual call; self arg pass mode: {:?}", &fn_abi.args[0],), ); } let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0], idx); - (Some(method), Single(ptr), true) + (Some(method), Single(ptr)) } // Normal call Some(_) => ( None, args.get(0) - .map(|arg| adjust_arg_for_abi(fx, *arg)) + .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) .unwrap_or(Empty), - false, ), // Indirect call @@ -583,23 +556,27 @@ pub(crate) fn codegen_terminator_call<'tcx>( ( Some(func), args.get(0) - .map(|arg| adjust_arg_for_abi(fx, *arg)) + .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) .unwrap_or(Empty), - false, ) } }; let ret_place = destination.map(|(place, _)| place); - let (call_inst, call_args) = - self::returning::codegen_with_call_return_arg(fx, fn_sig, ret_place, |fx, return_ptr| { + let (call_inst, call_args) = self::returning::codegen_with_call_return_arg( + fx, + &fn_abi.ret, + ret_place, + |fx, return_ptr| { + let regular_args_count = args.len(); let mut call_args: Vec = return_ptr .into_iter() .chain(first_arg.into_iter()) .chain( args.into_iter() + .enumerate() .skip(1) - .map(|arg| adjust_arg_for_abi(fx, arg).into_iter()) + .map(|(i, arg)| adjust_arg_for_abi(fx, arg, &fn_abi.args[i]).into_iter()) .flatten(), ) .collect::>(); @@ -610,17 +587,17 @@ pub(crate) fn codegen_terminator_call<'tcx>( { // Pass the caller location for `#[track_caller]`. let caller_location = fx.get_caller_location(span); - call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter()); + call_args.extend( + adjust_arg_for_abi(fx, caller_location, &fn_abi.args[regular_args_count]) + .into_iter(), + ); + assert_eq!(fn_abi.args.len(), regular_args_count + 1); + } else { + assert_eq!(fn_abi.args.len(), regular_args_count); } let call_inst = if let Some(func_ref) = func_ref { - let sig = clif_sig_from_fn_sig( - fx.tcx, - fx.triple(), - fn_sig, - is_virtual_call, - false, // calls through function pointers never pass the caller location - ); + let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi); let sig = fx.bcx.import_signature(sig); fx.bcx.ins().call_indirect(sig, func_ref, &call_args) } else { @@ -630,7 +607,8 @@ pub(crate) fn codegen_terminator_call<'tcx>( }; (call_inst, call_args) - }); + }, + ); // FIXME find a cleaner way to support varargs if fn_sig.c_variadic { @@ -671,36 +649,33 @@ pub(crate) fn codegen_drop<'tcx>( drop_place: CPlace<'tcx>, ) { let ty = drop_place.layout().ty; - let drop_fn = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); + let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); - if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def { + if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def { // we don't actually need to drop anything } else { - let drop_fn_ty = drop_fn.ty(fx.tcx, ParamEnv::reveal_all()); - let fn_sig = fx.tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - drop_fn_ty.fn_sig(fx.tcx), - ); - assert_eq!(fn_sig.output(), fx.tcx.mk_unit()); - match ty.kind() { ty::Dynamic(..) => { let (ptr, vtable) = drop_place.to_ptr_maybe_unsized(); let ptr = ptr.get_addr(fx); let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap()); - let sig = clif_sig_from_fn_sig( - fx.tcx, - fx.triple(), - fn_sig, - true, - false, // `drop_in_place` is never `#[track_caller]` - ); + // FIXME(eddyb) perhaps move some of this logic into + // `Instance::resolve_drop_in_place`? + let virtual_drop = Instance { + def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0), + substs: drop_instance.substs, + }; + let fn_abi = FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), virtual_drop, &[]); + + let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi); let sig = fx.bcx.import_signature(sig); fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); } _ => { - assert!(!matches!(drop_fn.def, InstanceDef::Virtual(_, _))); + assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _))); + + let fn_abi = FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), drop_instance, &[]); let arg_value = drop_place.place_ref( fx, @@ -712,17 +687,19 @@ pub(crate) fn codegen_drop<'tcx>( }, )), ); - let arg_value = adjust_arg_for_abi(fx, arg_value); + let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0]); let mut call_args: Vec = arg_value.into_iter().collect::>(); - if drop_fn.def.requires_caller_location(fx.tcx) { + if drop_instance.def.requires_caller_location(fx.tcx) { // Pass the caller location for `#[track_caller]`. let caller_location = fx.get_caller_location(span); - call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter()); + call_args.extend( + adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1]).into_iter(), + ); } - let func_ref = fx.get_function_ref(drop_fn); + let func_ref = fx.get_function_ref(drop_instance); fx.bcx.ins().call(func_ref, &call_args); } } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index aec321bd4a040..e2b78bfeac0ba 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -1,9 +1,10 @@ //! Argument passing use crate::prelude::*; +use crate::value_and_place::assert_assignable; use cranelift_codegen::ir::ArgumentPurpose; -use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; +use rustc_target::abi::call::{ArgAbi, PassMode}; pub(super) use EmptySinglePair::*; #[derive(Copy, Clone, Debug)] @@ -68,8 +69,8 @@ pub(super) trait ArgAbiExt<'tcx> { impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { match self.mode { - RustcPassMode::Ignore => EmptySinglePair::Empty, - RustcPassMode::Direct(_) => match &self.layout.abi { + PassMode::Ignore => EmptySinglePair::Empty, + PassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => { EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) } @@ -79,7 +80,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Pair(_, _) => match &self.layout.abi { + PassMode::Pair(_, _) => match &self.layout.abi { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); @@ -87,8 +88,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), - RustcPassMode::Indirect { + PassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack, @@ -103,7 +104,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) } } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack, @@ -119,8 +120,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec) { match self.mode { - RustcPassMode::Ignore => (None, vec![]), - RustcPassMode::Direct(_) => match &self.layout.abi { + PassMode::Ignore => (None, vec![]), + PassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => ( None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))], @@ -132,7 +133,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Pair(_, _) => match &self.layout.abi { + PassMode::Pair(_, _) => match &self.layout.abi { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); @@ -140,14 +141,14 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - RustcPassMode::Cast(_) => ( + PassMode::Cast(_) => ( Some(AbiParam::special( pointer_ty(tcx), ArgumentPurpose::StructReturn, )), vec![], ), - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack, @@ -161,7 +162,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { vec![], ) } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, @@ -170,56 +171,21 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } } -pub(super) fn get_arg_abi<'tcx>( - tcx: TyCtxt<'tcx>, - layout: TyAndLayout<'tcx>, -) -> ArgAbi<'tcx, Ty<'tcx>> { - let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); - if layout.is_zst() { - // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer - arg_abi.mode = RustcPassMode::Ignore; - } - match arg_abi.mode { - RustcPassMode::Ignore => {} - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(_) => {} - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { - arg_abi.make_indirect(); - } - } - _ => unreachable!("{:?}", arg_abi.layout.abi), - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - if a == types::I128 && b == types::I128 { - arg_abi.make_indirect(); - } - } - _ => unreachable!("{:?}", arg_abi.layout.abi), - }, - _ => {} - } - arg_abi -} - /// Get a set of values to be passed as function arguments. pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> EmptySinglePair { - let arg_abi = get_arg_abi(fx.tcx, arg.layout()); + assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty); match arg_abi.mode { - RustcPassMode::Ignore => Empty, - RustcPassMode::Direct(_) => Single(arg.load_scalar(fx)), - RustcPassMode::Pair(_, _) => { + PassMode::Ignore => Empty, + PassMode::Direct(_) => Single(arg.load_scalar(fx)), + PassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); Pair(a, b) } - RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => match arg.force_stack(fx) { + PassMode::Cast(_) | PassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => Single(ptr.get_addr(fx)), (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), }, @@ -233,41 +199,52 @@ pub(super) fn cvalue_for_param<'tcx>( start_block: Block, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option, - arg_ty: Ty<'tcx>, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> Option> { - let layout = fx.layout_of(arg_ty); - let arg_abi = get_arg_abi(fx.tcx, layout); - let clif_types = arg_abi.get_abi_param(fx.tcx); let block_params = clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] - crate::abi::comments::add_arg_comment(fx, "arg", local, local_field, block_params, &arg_abi); + crate::abi::comments::add_arg_comment( + fx, + "arg", + local, + local_field, + block_params, + arg_abi.mode, + arg_abi.layout, + ); match arg_abi.mode { - RustcPassMode::Ignore => None, - RustcPassMode::Direct(_) => Some(CValue::by_val(block_params.assert_single(), layout)), - RustcPassMode::Pair(_, _) => { + PassMode::Ignore => None, + PassMode::Direct(_) => { + Some(CValue::by_val(block_params.assert_single(), arg_abi.layout)) + } + PassMode::Pair(_, _) => { let (a, b) = block_params.assert_pair(); - Some(CValue::by_val_pair(a, b, layout)) + Some(CValue::by_val_pair(a, b, arg_abi.layout)) } - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => Some(CValue::by_ref( Pointer::new(block_params.assert_single()), - layout, + arg_abi.layout, )), - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => { let (ptr, meta) = block_params.assert_pair(); - Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout)) + Some(CValue::by_ref_unsized( + Pointer::new(ptr), + meta, + arg_abi.layout, + )) } } } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 3a5f61315f832..d7a82e0c37703 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -3,21 +3,55 @@ use crate::abi::pass_mode::*; use crate::prelude::*; -use rustc_target::abi::call::PassMode as RustcPassMode; - -fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> { - fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty)) -} +use rustc_middle::ty::layout::FnAbiExt; +use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; /// Can the given type be returned into an ssa var or does it need to be returned on the stack. pub(crate) fn can_return_to_ssa_var<'tcx>( - tcx: TyCtxt<'tcx>, - dest_layout: TyAndLayout<'tcx>, + fx: &FunctionCx<'_, 'tcx, impl Module>, + func: &mir::Operand<'tcx>, + args: &[mir::Operand<'tcx>], ) -> bool { - match get_arg_abi(tcx, dest_layout).mode { - RustcPassMode::Ignore | RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => true, + let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx)); + let fn_sig = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx)); + + // Handle special calls like instrinsics and empty drop glue. + let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() { + let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs) + .unwrap() + .unwrap() + .polymorphize(fx.tcx); + + match instance.def { + InstanceDef::Intrinsic(_) | InstanceDef::DropGlue(_, _) => { + return true; + } + _ => Some(instance), + } + } else { + None + }; + + let extra_args = &args[fn_sig.inputs().len()..]; + let extra_args = extra_args + .iter() + .map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))) + .collect::>(); + let fn_abi = if let Some(instance) = instance { + FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args) + } else { + FnAbi::of_fn_ptr( + &RevealAllLayoutCx(fx.tcx), + fn_ty.fn_sig(fx.tcx), + &extra_args, + ) + }; + match fn_abi.ret.mode { + PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => true, // FIXME Make it possible to return Cast and Indirect to an ssa var. - RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => false, + PassMode::Cast(_) | PassMode::Indirect { .. } => false, } } @@ -28,30 +62,39 @@ pub(super) fn codegen_return_param<'tcx>( ssa_analyzed: &rustc_index::vec::IndexVec, start_block: Block, ) -> CPlace<'tcx> { - let ret_layout = return_layout(fx); - let ret_arg_abi = get_arg_abi(fx.tcx, ret_layout); - let (ret_place, ret_param) = match ret_arg_abi.mode { - RustcPassMode::Ignore => (CPlace::no_place(ret_layout), Empty), - RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => { + let (ret_place, ret_param) = match fx.fn_abi.as_ref().unwrap().ret.mode { + PassMode::Ignore => ( + CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), + Empty, + ), + PassMode::Direct(_) | PassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( - super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa), + super::make_local_place( + fx, + RETURN_PLACE, + fx.fn_abi.as_ref().unwrap().ret.layout, + is_ssa, + ), Empty, ) } - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => { let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); ( - CPlace::for_ptr(Pointer::new(ret_param), ret_layout), + CPlace::for_ptr( + Pointer::new(ret_param), + fx.fn_abi.as_ref().unwrap().ret.layout, + ), Single(ret_param), ) } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, @@ -68,7 +111,8 @@ pub(super) fn codegen_return_param<'tcx>( Some(RETURN_PLACE), None, ret_param, - &ret_arg_abi, + fx.fn_abi.as_ref().unwrap().ret.mode, + fx.fn_abi.as_ref().unwrap().ret.layout, ); ret_place @@ -78,17 +122,14 @@ pub(super) fn codegen_return_param<'tcx>( /// returns the call return value(s) if any are written to the correct place. pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( fx: &mut FunctionCx<'_, 'tcx, M>, - fn_sig: FnSig<'tcx>, + ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ret_place: Option>, f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option) -> (Inst, T), ) -> (Inst, T) { - let ret_layout = fx.layout_of(fn_sig.output()); - - let output_arg_abi = get_arg_abi(fx.tcx, ret_layout); - let return_ptr = match output_arg_abi.mode { - RustcPassMode::Ignore => None, - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + let return_ptr = match ret_arg_abi.mode { + PassMode::Ignore => None, + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, @@ -96,38 +137,41 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)), None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot }, - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => unreachable!("unsized return value"), - RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => None, + PassMode::Direct(_) | PassMode::Pair(_, _) => None, }; let (call_inst, meta) = f(fx, return_ptr); - match output_arg_abi.mode { - RustcPassMode::Ignore => {} - RustcPassMode::Direct(_) => { + match ret_arg_abi.mode { + PassMode::Ignore => {} + PassMode::Direct(_) => { if let Some(ret_place) = ret_place { let ret_val = fx.bcx.inst_results(call_inst)[0]; - ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout)); + ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout)); } } - RustcPassMode::Pair(_, _) => { + PassMode::Pair(_, _) => { if let Some(ret_place) = ret_place { let ret_val_a = fx.bcx.inst_results(call_inst)[0]; let ret_val_b = fx.bcx.inst_results(call_inst)[1]; - ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout)); + ret_place.write_cvalue( + fx, + CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout), + ); } } - RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => {} - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, @@ -139,27 +183,27 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { - match get_arg_abi(fx.tcx, return_layout(fx)).mode { - RustcPassMode::Ignore - | RustcPassMode::Cast(_) - | RustcPassMode::Indirect { + match fx.fn_abi.as_ref().unwrap().ret.mode { + PassMode::Ignore + | PassMode::Cast(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => { fx.bcx.ins().return_(&[]); } - RustcPassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => unreachable!("unsized return value"), - RustcPassMode::Direct(_) => { + PassMode::Direct(_) => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx).load_scalar(fx); fx.bcx.ins().return_(&[ret_val]); } - RustcPassMode::Pair(_, _) => { + PassMode::Pair(_, _) => { let place = fx.get_local_place(RETURN_PLACE); let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); diff --git a/src/analyze.rs b/src/analyze.rs index adf5c7ac4fee7..dc5e8a7e30498 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -40,11 +40,9 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec { + TerminatorKind::Call { destination, func, args, .. } => { if let Some((dest_place, _dest_bb)) = destination { - let dest_layout = fx - .layout_of(fx.monomorphize(&dest_place.ty(&fx.mir.local_decls, fx.tcx).ty)); - if !crate::abi::can_return_to_ssa_var(fx.tcx, dest_layout) { + if !crate::abi::can_return_to_ssa_var(fx, func, args) { not_ssa(&mut flag_map, dest_place.local) } } diff --git a/src/base.rs b/src/base.rs index 1fafc1215975e..1eff0d4f5167b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -2,6 +2,8 @@ use rustc_index::vec::IndexVec; use rustc_middle::ty::adjustment::PointerCast; +use rustc_middle::ty::layout::FnAbiExt; +use rustc_target::abi::call::FnAbi; use crate::prelude::*; @@ -51,6 +53,7 @@ pub(crate) fn codegen_fn<'tcx>( instance, mir, + fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), bcx, block_map, diff --git a/src/common.rs b/src/common.rs index 1485d4451b815..fbee84e09f7a6 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,4 +1,5 @@ use rustc_index::vec::IndexVec; +use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; @@ -294,6 +295,7 @@ pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> { pub(crate) instance: Instance<'tcx>, pub(crate) mir: &'tcx Body<'tcx>, + pub(crate) fn_abi: Option>>, pub(crate) bcx: FunctionBuilder<'clif>, pub(crate) block_map: IndexVec, @@ -319,16 +321,7 @@ impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> { type TyAndLayout = TyAndLayout<'tcx>; fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> { - assert!(!ty.still_further_specializable()); - self.tcx - .layout_of(ParamEnv::reveal_all().and(&ty)) - .unwrap_or_else(|e| { - if let layout::LayoutError::SizeOverflow(_) = e { - self.tcx.sess.fatal(&e.to_string()) - } else { - bug!("failed to get layout for `{}`: {}", ty, e) - } - }) + RevealAllLayoutCx(self.tcx).layout_of(ty) } } @@ -442,3 +435,47 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { self.bcx.ins().global_value(self.pointer_type, local_msg_id) } } + +pub(crate) struct RevealAllLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>); + +impl<'tcx> LayoutOf for RevealAllLayoutCx<'tcx> { + type Ty = Ty<'tcx>; + type TyAndLayout = TyAndLayout<'tcx>; + + fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> { + assert!(!ty.still_further_specializable()); + self.0 + .layout_of(ParamEnv::reveal_all().and(&ty)) + .unwrap_or_else(|e| { + if let layout::LayoutError::SizeOverflow(_) = e { + self.0.sess.fatal(&e.to_string()) + } else { + bug!("failed to get layout for `{}`: {}", ty, e) + } + }) + } +} + +impl<'tcx> layout::HasTyCtxt<'tcx> for RevealAllLayoutCx<'tcx> { + fn tcx<'b>(&'b self) -> TyCtxt<'tcx> { + self.0 + } +} + +impl<'tcx> rustc_target::abi::HasDataLayout for RevealAllLayoutCx<'tcx> { + fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout { + &self.0.data_layout + } +} + +impl<'tcx> layout::HasParamEnv<'tcx> for RevealAllLayoutCx<'tcx> { + fn param_env(&self) -> ParamEnv<'tcx> { + ParamEnv::reveal_all() + } +} + +impl<'tcx> HasTargetSpec for RevealAllLayoutCx<'tcx> { + fn target_spec(&self) -> &Target { + &self.0.sess.target + } +} diff --git a/src/lib.rs b/src/lib.rs index 9b5b7d8051c72..ed7ee3b536534 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,7 +90,7 @@ mod prelude { pub(crate) use rustc_middle::mir::{self, *}; pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout}; pub(crate) use rustc_middle::ty::{ - self, FnSig, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, + self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, }; pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx}; diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 5bcb11fd515a0..17cb09d558707 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -450,64 +450,6 @@ impl<'tcx> CPlace<'tcx> { fx: &mut FunctionCx<'_, 'tcx, impl Module>, from: CValue<'tcx>, ) { - fn assert_assignable<'tcx>( - fx: &FunctionCx<'_, 'tcx, impl Module>, - from_ty: Ty<'tcx>, - to_ty: Ty<'tcx>, - ) { - match (from_ty.kind(), to_ty.kind()) { - (ty::Ref(_, a, _), ty::Ref(_, b, _)) - | ( - ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), - ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), - ) => { - assert_assignable(fx, a, b); - } - (ty::FnPtr(_), ty::FnPtr(_)) => { - let from_sig = fx.tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - from_ty.fn_sig(fx.tcx), - ); - let to_sig = fx.tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - to_ty.fn_sig(fx.tcx), - ); - assert_eq!( - from_sig, to_sig, - "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", - from_sig, to_sig, fx, - ); - // fn(&T) -> for<'l> fn(&'l T) is allowed - } - (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { - for (from, to) in from_traits.iter().zip(to_traits) { - let from = fx - .tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); - let to = fx - .tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); - assert_eq!( - from, to, - "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", - from_traits, to_traits, fx, - ); - } - // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed - } - _ => { - assert_eq!( - from_ty, - to_ty, - "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", - from_ty, - to_ty, - fx, - ); - } - } - } - assert_assignable(fx, from.layout().ty, self.layout().ty); self.write_cvalue_maybe_transmute(fx, from, "write_cvalue"); @@ -794,3 +736,62 @@ impl<'tcx> CPlace<'tcx> { } } } + +#[track_caller] +pub(crate) fn assert_assignable<'tcx>( + fx: &FunctionCx<'_, 'tcx, impl Module>, + from_ty: Ty<'tcx>, + to_ty: Ty<'tcx>, +) { + match (from_ty.kind(), to_ty.kind()) { + (ty::Ref(_, a, _), ty::Ref(_, b, _)) + | ( + ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), + ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), + ) => { + assert_assignable(fx, a, b); + } + (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ })) + | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => { + assert_assignable(fx, a, b); + } + (ty::FnPtr(_), ty::FnPtr(_)) => { + let from_sig = fx.tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + from_ty.fn_sig(fx.tcx), + ); + let to_sig = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx)); + assert_eq!( + from_sig, to_sig, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + // fn(&T) -> for<'l> fn(&'l T) is allowed + } + (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { + for (from, to) in from_traits.iter().zip(to_traits) { + let from = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); + let to = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); + assert_eq!( + from, to, + "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", + from_traits, to_traits, fx, + ); + } + // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed + } + _ => { + assert_eq!( + from_ty, to_ty, + "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", + from_ty, to_ty, fx, + ); + } + } +} From d2634478781be741c872f33c7793797c081dc16f Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:24:31 +0100 Subject: [PATCH 26/48] Replace EmptySinglePair with SmallVec --- Cargo.lock | 1 + Cargo.toml | 1 + src/abi/comments.rs | 17 ++++-- src/abi/mod.rs | 7 ++- src/abi/pass_mode.rs | 128 ++++++++++++++----------------------------- src/abi/returning.rs | 12 ++-- 6 files changed, 66 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 431e806869646..5495cfa5eaa0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -333,6 +333,7 @@ dependencies = [ "indexmap", "libloading", "object", + "smallvec", "target-lexicon", ] diff --git a/Cargo.toml b/Cargo.toml index 4558da2de73ea..3820fce6d1e0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ object = { version = "0.22.0", default-features = false, features = ["std", "rea ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.0.2" libloading = { version = "0.6.0", optional = true } +smallvec = "1.6.1" # Uncomment to use local checkout of cranelift #[patch."https://github.com/bytecodealliance/wasmtime/"] diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 41cb4c627f899..9aab45b62e211 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -8,7 +8,6 @@ use rustc_target::abi::call::PassMode; use cranelift_codegen::entity::EntityRef; -use crate::abi::pass_mode::*; use crate::prelude::*; pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) { @@ -22,7 +21,7 @@ pub(super) fn add_arg_comment<'tcx>( kind: &str, local: Option, local_field: Option, - params: EmptySinglePair, + params: &[Value], arg_abi_mode: PassMode, arg_layout: TyAndLayout<'tcx>, ) { @@ -38,9 +37,17 @@ pub(super) fn add_arg_comment<'tcx>( }; let params = match params { - Empty => Cow::Borrowed("-"), - Single(param) => Cow::Owned(format!("= {:?}", param)), - Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), + [] => Cow::Borrowed("-"), + [param] => Cow::Owned(format!("= {:?}", param)), + [param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)), + params => Cow::Owned(format!( + "= {}", + params + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + )), }; let pass_mode = format!("{:?}", arg_abi_mode); diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 55ebd39e3f132..bc35ca2de40f6 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -11,6 +11,7 @@ use rustc_target::abi::call::{Conv, FnAbi}; use rustc_target::spec::abi::Abi; use cranelift_codegen::ir::AbiParam; +use smallvec::smallvec; use self::pass_mode::*; use crate::prelude::*; @@ -534,7 +535,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( ); } let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0], idx); - (Some(method), Single(ptr)) + (Some(method), smallvec![ptr]) } // Normal call @@ -542,7 +543,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( None, args.get(0) .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) - .unwrap_or(Empty), + .unwrap_or(smallvec![]), ), // Indirect call @@ -557,7 +558,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( Some(func), args.get(0) .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0])) - .unwrap_or(Empty), + .unwrap_or(smallvec![]), ) } }; diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index e2b78bfeac0ba..e047ddcebc98b 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -5,78 +5,24 @@ use crate::value_and_place::assert_assignable; use cranelift_codegen::ir::ArgumentPurpose; use rustc_target::abi::call::{ArgAbi, PassMode}; -pub(super) use EmptySinglePair::*; - -#[derive(Copy, Clone, Debug)] -pub(super) enum EmptySinglePair { - Empty, - Single(T), - Pair(T, T), -} - -impl EmptySinglePair { - pub(super) fn into_iter(self) -> EmptySinglePairIter { - EmptySinglePairIter(self) - } - - pub(super) fn map(self, mut f: impl FnMut(T) -> U) -> EmptySinglePair { - match self { - Empty => Empty, - Single(v) => Single(f(v)), - Pair(a, b) => Pair(f(a), f(b)), - } - } -} - -pub(super) struct EmptySinglePairIter(EmptySinglePair); - -impl Iterator for EmptySinglePairIter { - type Item = T; - - fn next(&mut self) -> Option { - match std::mem::replace(&mut self.0, Empty) { - Empty => None, - Single(v) => Some(v), - Pair(a, b) => { - self.0 = Single(b); - Some(a) - } - } - } -} - -impl EmptySinglePair { - pub(super) fn assert_single(self) -> T { - match self { - Single(v) => v, - _ => panic!("Called assert_single on {:?}", self), - } - } - - pub(super) fn assert_pair(self) -> (T, T) { - match self { - Pair(a, b) => (a, b), - _ => panic!("Called assert_pair on {:?}", self), - } - } -} +use smallvec::{smallvec, SmallVec}; pub(super) trait ArgAbiExt<'tcx> { - fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair; + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>; fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); } impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { - fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { match self.mode { - PassMode::Ignore => EmptySinglePair::Empty, + PassMode::Ignore => smallvec![], PassMode::Direct(_) => match &self.layout.abi { Abi::Scalar(scalar) => { - EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) + smallvec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))] } Abi::Vector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); - EmptySinglePair::Single(AbiParam::new(vector_ty)) + smallvec![AbiParam::new(vector_ty)] } _ => unreachable!("{:?}", self.layout.abi), }, @@ -84,11 +30,11 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); - EmptySinglePair::Pair(AbiParam::new(a), AbiParam::new(b)) + smallvec![AbiParam::new(a), AbiParam::new(b)] } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + PassMode::Cast(_) => smallvec![AbiParam::new(pointer_ty(tcx))], PassMode::Indirect { attrs: _, extra_attrs: None, @@ -96,12 +42,12 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } => { if on_stack { let size = u32::try_from(self.layout.size.bytes()).unwrap(); - EmptySinglePair::Single(AbiParam::special( + smallvec![AbiParam::special( pointer_ty(tcx), ArgumentPurpose::StructArgument(size), - )) + )] } else { - EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) + smallvec![AbiParam::new(pointer_ty(tcx))] } } PassMode::Indirect { @@ -110,10 +56,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { on_stack, } => { assert!(!on_stack); - EmptySinglePair::Pair( + smallvec![ AbiParam::new(pointer_ty(tcx)), AbiParam::new(pointer_ty(tcx)), - ) + ] } } } @@ -176,18 +122,18 @@ pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, -) -> EmptySinglePair { +) -> SmallVec<[Value; 2]> { assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty); match arg_abi.mode { - PassMode::Ignore => Empty, - PassMode::Direct(_) => Single(arg.load_scalar(fx)), + PassMode::Ignore => smallvec![], + PassMode::Direct(_) => smallvec![arg.load_scalar(fx)], PassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); - Pair(a, b) + smallvec![a, b] } PassMode::Cast(_) | PassMode::Indirect { .. } => match arg.force_stack(fx) { - (ptr, None) => Single(ptr.get_addr(fx)), - (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), + (ptr, None) => smallvec![ptr.get_addr(fx)], + (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], }, } } @@ -202,8 +148,10 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> Option> { let clif_types = arg_abi.get_abi_param(fx.tcx); - let block_params = - clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); + let block_params = clif_types + .into_iter() + .map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)) + .collect::>(); #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( @@ -211,7 +159,7 @@ pub(super) fn cvalue_for_param<'tcx>( "arg", local, local_field, - block_params, + &block_params, arg_abi.mode, arg_abi.layout, ); @@ -219,30 +167,38 @@ pub(super) fn cvalue_for_param<'tcx>( match arg_abi.mode { PassMode::Ignore => None, PassMode::Direct(_) => { - Some(CValue::by_val(block_params.assert_single(), arg_abi.layout)) + assert_eq!(block_params.len(), 1, "{:?}", block_params); + Some(CValue::by_val(block_params[0], arg_abi.layout)) } PassMode::Pair(_, _) => { - let (a, b) = block_params.assert_pair(); - Some(CValue::by_val_pair(a, b, arg_abi.layout)) + assert_eq!(block_params.len(), 2, "{:?}", block_params); + Some(CValue::by_val_pair( + block_params[0], + block_params[1], + arg_abi.layout, + )) } PassMode::Cast(_) | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, - } => Some(CValue::by_ref( - Pointer::new(block_params.assert_single()), - arg_abi.layout, - )), + } => { + assert_eq!(block_params.len(), 1, "{:?}", block_params); + Some(CValue::by_ref( + Pointer::new(block_params[0]), + arg_abi.layout, + )) + } PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _, } => { - let (ptr, meta) = block_params.assert_pair(); + assert_eq!(block_params.len(), 2, "{:?}", block_params); Some(CValue::by_ref_unsized( - Pointer::new(ptr), - meta, + Pointer::new(block_params[0]), + block_params[1], arg_abi.layout, )) } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index d7a82e0c37703..8376f845734ba 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -1,10 +1,10 @@ //! Return value handling -use crate::abi::pass_mode::*; use crate::prelude::*; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; +use smallvec::{SmallVec, smallvec}; /// Can the given type be returned into an ssa var or does it need to be returned on the stack. pub(crate) fn can_return_to_ssa_var<'tcx>( @@ -62,10 +62,10 @@ pub(super) fn codegen_return_param<'tcx>( ssa_analyzed: &rustc_index::vec::IndexVec, start_block: Block, ) -> CPlace<'tcx> { - let (ret_place, ret_param) = match fx.fn_abi.as_ref().unwrap().ret.mode { + let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { PassMode::Ignore => ( CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), - Empty, + smallvec![], ), PassMode::Direct(_) | PassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; @@ -76,7 +76,7 @@ pub(super) fn codegen_return_param<'tcx>( fx.fn_abi.as_ref().unwrap().ret.layout, is_ssa, ), - Empty, + smallvec![], ) } PassMode::Cast(_) @@ -91,7 +91,7 @@ pub(super) fn codegen_return_param<'tcx>( Pointer::new(ret_param), fx.fn_abi.as_ref().unwrap().ret.layout, ), - Single(ret_param), + smallvec![ret_param], ) } PassMode::Indirect { @@ -110,7 +110,7 @@ pub(super) fn codegen_return_param<'tcx>( "ret", Some(RETURN_PLACE), None, - ret_param, + &ret_param, fx.fn_abi.as_ref().unwrap().ret.mode, fx.fn_abi.as_ref().unwrap().ret.layout, ); From aa23f862dc020a568278dc4ad92c455a0a3ced46 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:32:56 +0100 Subject: [PATCH 27/48] Remove vararg support check This check wasn't very useful and removing it simplifies the code. --- src/abi/mod.rs | 12 +----------- src/base.rs | 2 +- src/driver/jit.rs | 4 ++-- src/driver/mod.rs | 7 +------ src/main_shim.rs | 2 +- 5 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index bc35ca2de40f6..a27d5b8ab02a2 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -140,18 +140,8 @@ pub(crate) fn get_function_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, inst: Instance<'tcx>, - support_vararg: bool, ) -> Signature { assert!(!inst.substs.needs_infer()); - let fn_sig = tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_sig_for_fn_abi(tcx, inst)); - if fn_sig.c_variadic && !support_vararg { - tcx.sess.span_fatal( - tcx.def_span(inst.def_id()), - "Variadic function definitions are not yet supported", - ); - } - clif_sig_from_fn_abi( tcx, triple, @@ -166,7 +156,7 @@ pub(crate) fn import_function<'tcx>( inst: Instance<'tcx>, ) -> FuncId { let name = tcx.symbol_name(inst).name.to_string(); - let sig = get_function_sig(tcx, module.isa().triple(), inst, true); + let sig = get_function_sig(tcx, module.isa().triple(), inst); module .declare_function(&name, Linkage::Import, &sig) .unwrap() diff --git a/src/base.rs b/src/base.rs index 1eff0d4f5167b..e81aa52780b3b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -22,7 +22,7 @@ pub(crate) fn codegen_fn<'tcx>( // Declare function let name = tcx.symbol_name(instance).name.to_string(); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance, false); + let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); cx.cached_context.clear(); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 6a87925927707..2d14ff2c0221d 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -157,7 +157,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); let name = tcx.symbol_name(instance).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance, true); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) @@ -243,7 +243,7 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In let pointer_type = cx.module.target_config().pointer_type(); let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst, true); + let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); let func_id = cx .module .declare_function(&name, Linkage::Export, &sig) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index e462f34a04f99..752c3f747d5f7 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -51,12 +51,7 @@ fn predefine_mono_items<'tcx>( match mono_item { MonoItem::Fn(instance) => { let name = cx.tcx.symbol_name(instance).name.to_string(); - let sig= get_function_sig( - cx.tcx, - cx.module.isa().triple(), - instance, - false, - ); + let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); cx.module.declare_function(&name, linkage, &sig).unwrap(); } diff --git a/src/main_shim.rs b/src/main_shim.rs index 7900abb32a3fc..b193cea877dad 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -70,7 +70,7 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); let main_name = tcx.symbol_name(instance).name.to_string(); - let main_sig = get_function_sig(tcx, m.isa().triple(), instance, false); + let main_sig = get_function_sig(tcx, m.isa().triple(), instance); let main_func_id = m .declare_function(&main_name, Linkage::Import, &main_sig) .unwrap(); From e564a0ad319c8fabddc3e62616be0dbfd761ecec Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:33:06 +0100 Subject: [PATCH 28/48] Rustfmt --- src/abi/returning.rs | 2 +- src/analyze.rs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 8376f845734ba..3acfae3e1e52d 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -4,7 +4,7 @@ use crate::prelude::*; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; -use smallvec::{SmallVec, smallvec}; +use smallvec::{smallvec, SmallVec}; /// Can the given type be returned into an ssa var or does it need to be returned on the stack. pub(crate) fn can_return_to_ssa_var<'tcx>( diff --git a/src/analyze.rs b/src/analyze.rs index dc5e8a7e30498..62fbcfe3f7a5d 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -40,7 +40,12 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec { + TerminatorKind::Call { + destination, + func, + args, + .. + } => { if let Some((dest_place, _dest_bb)) = destination { if !crate::abi::can_return_to_ssa_var(fx, func, args) { not_ssa(&mut flag_map, dest_place.local) From 268d7bc459d436d28171e37050edec287f950bfe Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 27 Jan 2021 10:36:40 +0100 Subject: [PATCH 29/48] Remove fn_sig_for_fn_abi --- src/abi/mod.rs | 78 ---------------------------------------------- src/pretty_clif.rs | 9 +++--- 2 files changed, 4 insertions(+), 83 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index a27d5b8ab02a2..92d6b3897538b 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -18,84 +18,6 @@ use crate::prelude::*; pub(crate) use self::returning::{can_return_to_ssa_var, codegen_return}; -// FIXME remove -// Copied from https://github.com/rust-lang/rust/blob/f52c72948aa1dd718cc1f168d21c91c584c0a662/src/librustc_middle/ty/layout.rs#L2301 -#[rustfmt::skip] -pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::PolyFnSig<'tcx> { - use rustc_middle::ty::subst::Subst; - - // FIXME(davidtwco,eddyb): A `ParamEnv` should be passed through to this function. - let ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); - match *ty.kind() { - ty::FnDef(..) => { - // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering - // parameters unused if they show up in the signature, but not in the `mir::Body` - // (i.e. due to being inside a projection that got normalized, see - // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping - // track of a polymorphization `ParamEnv` to allow normalizing later. - let mut sig = match *ty.kind() { - ty::FnDef(def_id, substs) => tcx - .normalize_erasing_regions(tcx.param_env(def_id), tcx.fn_sig(def_id)) - .subst(tcx, substs), - _ => unreachable!(), - }; - - if let ty::InstanceDef::VtableShim(..) = instance.def { - // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`. - sig = sig.map_bound(|mut sig| { - let mut inputs_and_output = sig.inputs_and_output.to_vec(); - inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]); - sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output); - sig - }); - } - sig - } - ty::Closure(def_id, substs) => { - let sig = substs.as_closure().sig(); - - let env_ty = tcx.closure_env_ty(def_id, substs).unwrap(); - sig.map_bound(|sig| { - tcx.mk_fn_sig( - std::iter::once(env_ty.skip_binder()).chain(sig.inputs().iter().cloned()), - sig.output(), - sig.c_variadic, - sig.unsafety, - sig.abi, - ) - }) - } - ty::Generator(_, substs, _) => { - let sig = substs.as_generator().poly_sig(); - - let env_region = ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind: ty::BrEnv }); - let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty); - - let pin_did = tcx.require_lang_item(rustc_hir::LangItem::Pin, None); - let pin_adt_ref = tcx.adt_def(pin_did); - let pin_substs = tcx.intern_substs(&[env_ty.into()]); - let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs); - - sig.map_bound(|sig| { - let state_did = tcx.require_lang_item(rustc_hir::LangItem::GeneratorState, None); - let state_adt_ref = tcx.adt_def(state_did); - let state_substs = - tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]); - let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); - - tcx.mk_fn_sig( - [env_ty, sig.resume_ty].iter(), - &ret_ty, - false, - rustc_hir::Unsafety::Normal, - rustc_target::spec::abi::Abi::Rust, - ) - }) - } - _ => bug!("unexpected type {:?} in Instance::fn_sig", ty), - } -} - fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 22c94fec82fc1..f4a15ab12d511 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -61,7 +61,9 @@ use cranelift_codegen::{ write::{FuncWriter, PlainWriter}, }; +use rustc_middle::ty::layout::FnAbiExt; use rustc_session::config::OutputType; +use rustc_target::abi::call::FnAbi; use crate::prelude::*; @@ -78,11 +80,8 @@ impl CommentWriter { format!("symbol {}", tcx.symbol_name(instance).name), format!("instance {:?}", instance), format!( - "sig {:?}", - tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - crate::abi::fn_sig_for_fn_abi(tcx, instance) - ) + "abi {:?}", + FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[]) ), String::new(), ] From e8f48e4bae83295816f035474a726a5d92056453 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 29 Jan 2021 22:15:15 +0100 Subject: [PATCH 30/48] [WIP] Implement PassMode::Cast --- src/abi/mod.rs | 25 +++++-- src/abi/pass_mode.rs | 162 ++++++++++++++++++++++++++++++++++++++----- src/abi/returning.rs | 38 +++++++--- 3 files changed, 191 insertions(+), 34 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 92d6b3897538b..c227bdd534f34 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -191,12 +191,24 @@ pub(crate) fn codegen_fn_prelude<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, start_block: Block, ) { + fx.bcx.append_block_params_for_function_params(start_block); + + fx.bcx.switch_to_block(start_block); + fx.bcx.ins().nop(); + let ssa_analyzed = crate::analyze::analyze(fx); #[cfg(debug_assertions)] self::comments::add_args_header_comment(fx); - let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, start_block); + let mut block_params_iter = fx + .bcx + .func + .dfg + .block_params(start_block) + .to_vec() + .into_iter(); + let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); // None means pass_mode == NoPass @@ -229,14 +241,14 @@ pub(crate) fn codegen_fn_prelude<'tcx>( let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.types().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_abi); + let param = cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, start_block, Some(local), None, arg_abi); + let param = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); (local, ArgKind::Normal(param), arg_ty) } }) @@ -246,14 +258,13 @@ pub(crate) fn codegen_fn_prelude<'tcx>( if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); - fx.caller_location = Some(cvalue_for_param(fx, start_block, None, None, arg_abi).unwrap()); + fx.caller_location = + Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); } assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind"); fx.fn_abi = Some(fn_abi); - - fx.bcx.switch_to_block(start_block); - fx.bcx.ins().nop(); + assert!(block_params_iter.next().is_none(), "arg_value left behind"); #[cfg(debug_assertions)] self::comments::add_locals_header_comment(fx); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index e047ddcebc98b..6f27fa52d882c 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -4,7 +4,7 @@ use crate::prelude::*; use crate::value_and_place::assert_assignable; use cranelift_codegen::ir::ArgumentPurpose; -use rustc_target::abi::call::{ArgAbi, PassMode}; +use rustc_target::abi::call::{ArgAbi, CastTarget, PassMode, Reg, RegKind}; use smallvec::{smallvec, SmallVec}; pub(super) trait ArgAbiExt<'tcx> { @@ -12,6 +12,78 @@ pub(super) trait ArgAbiExt<'tcx> { fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); } +fn reg_to_abi_param(reg: Reg) -> AbiParam { + let clif_ty = match (reg.kind, reg.size.bytes()) { + (RegKind::Integer, 1) => types::I8, + (RegKind::Integer, 2) => types::I16, + (RegKind::Integer, 4) => types::I32, + (RegKind::Integer, 8) => types::I64, + (RegKind::Integer, 16) => types::I128, + (RegKind::Float, 4) => types::F32, + (RegKind::Float, 8) => types::F64, + (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), + _ => unreachable!("{:?}", reg), + }; + AbiParam::new(clif_ty) +} + +fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { + let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { + (0, 0) + } else { + ( + cast.rest.total.bytes() / cast.rest.unit.size.bytes(), + cast.rest.total.bytes() % cast.rest.unit.size.bytes(), + ) + }; + + if cast.prefix.iter().all(|x| x.is_none()) { + // Simplify to a single unit when there is no prefix and size <= unit size + if cast.rest.total <= cast.rest.unit.size { + let clif_ty = match (cast.rest.unit.kind, cast.rest.unit.size.bytes()) { + (RegKind::Integer, 1) => types::I8, + (RegKind::Integer, 2) => types::I16, + (RegKind::Integer, 3..=4) => types::I32, + (RegKind::Integer, 5..=8) => types::I64, + (RegKind::Integer, 9..=16) => types::I128, + (RegKind::Float, 4) => types::F32, + (RegKind::Float, 8) => types::F64, + (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), + _ => unreachable!("{:?}", cast.rest.unit), + }; + return smallvec![AbiParam::new(clif_ty)]; + } + } + + // Create list of fields in the main structure + let mut args = cast + .prefix + .iter() + .flatten() + .map(|&kind| { + reg_to_abi_param(Reg { + kind, + size: cast.prefix_chunk_size, + }) + }) + .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))) + .collect::>(); + + // Append final integer + if rem_bytes != 0 { + // Only integers can be really split further. + assert_eq!(cast.rest.unit.kind, RegKind::Integer); + args.push(reg_to_abi_param(Reg { + kind: RegKind::Integer, + size: Size::from_bytes(rem_bytes), + })); + } + + args +} + +// FIXME respect argument extension mode + impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { match self.mode { @@ -34,7 +106,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(_) => smallvec![AbiParam::new(pointer_ty(tcx))], + PassMode::Cast(cast) => cast_target_to_abi_params(cast), PassMode::Indirect { attrs: _, extra_attrs: None, @@ -87,13 +159,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(_) => ( - Some(AbiParam::special( - pointer_ty(tcx), - ArgumentPurpose::StructReturn, - )), - vec![], - ), + PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()), PassMode::Indirect { attrs: _, extra_attrs: None, @@ -117,6 +183,60 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } } +pub(super) fn to_casted_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + arg: CValue<'tcx>, + cast: CastTarget, +) -> SmallVec<[Value; 2]> { + let (ptr, meta) = arg.force_stack(fx); + assert!(meta.is_none()); + let mut offset = 0; + cast_target_to_abi_params(cast) + .into_iter() + .map(|param| { + let val = ptr + .offset_i64(fx, offset) + .load(fx, param.value_type, MemFlags::new()); + offset += i64::from(param.value_type.bytes()); + val + }) + .collect() +} + +pub(super) fn from_casted_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + block_params: &[Value], + layout: TyAndLayout<'tcx>, + cast: CastTarget, +) -> CValue<'tcx> { + let abi_params = cast_target_to_abi_params(cast); + let size = abi_params + .iter() + .map(|param| param.value_type.bytes()) + .sum(); + // Stack slot size may be bigger for for example `[u8; 3]` which is packed into an `i32`. + assert!(u64::from(size) >= layout.size.bytes()); + let stack_slot = fx.bcx.create_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + size, + offset: None, + }); + let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); + let mut offset = 0; + let mut block_params_iter = block_params.into_iter().copied(); + for param in abi_params { + let val = ptr.offset_i64(fx, offset).store( + fx, + block_params_iter.next().unwrap(), + MemFlags::new(), + ); + offset += i64::from(param.value_type.bytes()); + val + } + assert_eq!(block_params_iter.next(), None, "Leftover block param"); + CValue::by_ref(ptr, layout) +} + /// Get a set of values to be passed as function arguments. pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, @@ -131,7 +251,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>( let (a, b) = arg.load_scalar_pair(fx); smallvec![a, b] } - PassMode::Cast(_) | PassMode::Indirect { .. } => match arg.force_stack(fx) { + PassMode::Cast(cast) => to_casted_value(fx, arg, cast), + PassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => smallvec![ptr.get_addr(fx)], (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], }, @@ -142,15 +263,22 @@ pub(super) fn adjust_arg_for_abi<'tcx>( /// as necessary. pub(super) fn cvalue_for_param<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, - start_block: Block, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option, #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + block_params_iter: &mut impl Iterator, ) -> Option> { - let clif_types = arg_abi.get_abi_param(fx.tcx); - let block_params = clif_types + let block_params = arg_abi + .get_abi_param(fx.tcx) .into_iter() - .map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)) + .map(|abi_param| { + let block_param = block_params_iter.next().unwrap(); + assert_eq!( + fx.bcx.func.dfg.value_type(block_param), + abi_param.value_type + ); + block_param + }) .collect::>(); #[cfg(debug_assertions)] @@ -178,8 +306,10 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, )) } - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Cast(cast) => { + Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + } + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 3acfae3e1e52d..a382963bf1ed7 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -60,14 +60,14 @@ pub(crate) fn can_return_to_ssa_var<'tcx>( pub(super) fn codegen_return_param<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, ssa_analyzed: &rustc_index::vec::IndexVec, - start_block: Block, + block_params_iter: &mut impl Iterator, ) -> CPlace<'tcx> { let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { PassMode::Ignore => ( CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), smallvec![], ), - PassMode::Direct(_) | PassMode::Pair(_, _) => { + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( super::make_local_place( @@ -79,13 +79,13 @@ pub(super) fn codegen_return_param<'tcx>( smallvec![], ) } - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, } => { - let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); + let ret_param = block_params_iter.next().unwrap(); + assert_eq!(fx.bcx.func.dfg.value_type(ret_param), pointer_ty(fx.tcx)); ( CPlace::for_ptr( Pointer::new(ret_param), @@ -128,8 +128,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ) -> (Inst, T) { let return_ptr = match ret_arg_abi.mode { PassMode::Ignore => None, - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, @@ -142,7 +141,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( extra_attrs: Some(_), on_stack: _, } => unreachable!("unsized return value"), - PassMode::Direct(_) | PassMode::Pair(_, _) => None, + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => None, }; let (call_inst, meta) = f(fx, return_ptr); @@ -165,8 +164,20 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ); } } - PassMode::Cast(_) - | PassMode::Indirect { + PassMode::Cast(cast) => { + if let Some(ret_place) = ret_place { + let results = fx + .bcx + .inst_results(call_inst) + .into_iter() + .copied() + .collect::>(); + let result = + super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); + ret_place.write_cvalue(fx, result); + } + } + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _, @@ -185,7 +196,6 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { match fx.fn_abi.as_ref().unwrap().ret.mode { PassMode::Ignore - | PassMode::Cast(_) | PassMode::Indirect { attrs: _, extra_attrs: None, @@ -208,5 +218,11 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); } + PassMode::Cast(cast) => { + let place = fx.get_local_place(RETURN_PLACE); + let ret_val = place.to_cvalue(fx); + let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast); + fx.bcx.ins().return_(&ret_vals); + } } } From 139a6d12de65f81a69e345731f0547c824730a19 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 11:02:24 +0100 Subject: [PATCH 31/48] Fix 128bit checked math intrinsic calls --- src/abi/mod.rs | 25 ++++++++++-------- src/base.rs | 6 ++++- src/codegen_i128.rs | 62 +++++++++++++++++++++++---------------------- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index c227bdd534f34..6a025f2e88ae3 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -102,13 +102,13 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { pub(crate) fn lib_call( &mut self, name: &str, - input_tys: Vec, - output_tys: Vec, + params: Vec, + returns: Vec, args: &[Value], ) -> &[Value] { let sig = Signature { - params: input_tys.iter().cloned().map(AbiParam::new).collect(), - returns: output_tys.iter().cloned().map(AbiParam::new).collect(), + params, + returns, call_conv: CallConv::triple_default(self.triple()), }; let func_id = self @@ -140,16 +140,18 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { .iter() .map(|arg| { ( - self.clif_type(arg.layout().ty).unwrap(), + AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self), ) }) .unzip(); let return_layout = self.layout_of(return_ty); let return_tys = if let ty::Tuple(tup) = return_ty.kind() { - tup.types().map(|ty| self.clif_type(ty).unwrap()).collect() + tup.types() + .map(|ty| AbiParam::new(self.clif_type(ty).unwrap())) + .collect() } else { - vec![self.clif_type(return_ty).unwrap()] + vec![AbiParam::new(self.clif_type(return_ty).unwrap())] }; let ret_vals = self.lib_call(name, input_tys, return_tys, &args); match *ret_vals { @@ -208,7 +210,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>( .block_params(start_block) .to_vec() .into_iter(); - let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); + let ret_place = + self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); // None means pass_mode == NoPass @@ -241,14 +244,16 @@ pub(crate) fn codegen_fn_prelude<'tcx>( let mut params = Vec::new(); for (i, _arg_ty) in tupled_arg_tys.types().enumerate() { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); + let param = + cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); params.push(param); } (local, ArgKind::Spread(params), arg_ty) } else { let arg_abi = arg_abis_iter.next().unwrap(); - let param = cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); + let param = + cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); (local, ArgKind::Normal(param), arg_ty) } }) diff --git a/src/base.rs b/src/base.rs index e81aa52780b3b..4842628a99da7 100644 --- a/src/base.rs +++ b/src/base.rs @@ -1060,7 +1060,11 @@ pub(crate) fn codegen_panic_inner<'tcx>( fx.lib_call( &*symbol_name, - vec![fx.pointer_type, fx.pointer_type, fx.pointer_type], + vec![ + AbiParam::new(fx.pointer_type), + AbiParam::new(fx.pointer_type), + AbiParam::new(fx.pointer_type), + ], vec![], args, ); diff --git a/src/codegen_i128.rs b/src/codegen_i128.rs index d6a38bdafc9ba..866ba90e4ae4b 100644 --- a/src/codegen_i128.rs +++ b/src/codegen_i128.rs @@ -1,5 +1,7 @@ //! Replaces 128-bit operators with lang item calls where necessary +use cranelift_codegen::ir::ArgumentPurpose; + use crate::prelude::*; pub(crate) fn maybe_codegen<'tcx>( @@ -24,41 +26,41 @@ pub(crate) fn maybe_codegen<'tcx>( None } BinOp::Add | BinOp::Sub if !checked => None, - BinOp::Add => { - let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); - return Some(if is_signed { - fx.easy_call("__rust_i128_addo", &[lhs, rhs], out_ty) + BinOp::Mul if !checked => { + let val_ty = if is_signed { + fx.tcx.types.i128 } else { - fx.easy_call("__rust_u128_addo", &[lhs, rhs], out_ty) - }); + fx.tcx.types.u128 + }; + Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty)) } - BinOp::Sub => { + BinOp::Add | BinOp::Sub | BinOp::Mul => { + assert!(checked); let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); - return Some(if is_signed { - fx.easy_call("__rust_i128_subo", &[lhs, rhs], out_ty) - } else { - fx.easy_call("__rust_u128_subo", &[lhs, rhs], out_ty) - }); - } - BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), - BinOp::Mul => { - let res = if checked { - let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); - if is_signed { - fx.easy_call("__rust_i128_mulo", &[lhs, rhs], out_ty) - } else { - fx.easy_call("__rust_u128_mulo", &[lhs, rhs], out_ty) - } - } else { - let val_ty = if is_signed { - fx.tcx.types.i128 - } else { - fx.tcx.types.u128 - }; - fx.easy_call("__multi3", &[lhs, rhs], val_ty) + let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); + let param_types = vec![ + AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn), + AbiParam::new(types::I128), + AbiParam::new(types::I128), + ]; + let args = [ + out_place.to_ptr().get_addr(fx), + lhs.load_scalar(fx), + rhs.load_scalar(fx), + ]; + let name = match (bin_op, is_signed) { + (BinOp::Add, false) => "__rust_u128_addo", + (BinOp::Add, true) => "__rust_i128_addo", + (BinOp::Sub, false) => "__rust_u128_subo", + (BinOp::Sub, true) => "__rust_i128_subo", + (BinOp::Mul, false) => "__rust_u128_mulo", + (BinOp::Mul, true) => "__rust_i128_mulo", + _ => unreachable!(), }; - Some(res) + fx.lib_call(name, param_types, vec![], &args); + Some(out_place.to_cvalue(fx)) } + BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), BinOp::Div => { assert!(!checked); if is_signed { From b5ddb76f84f3e6df242e8c766b5015a124a50c15 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 12:11:40 +0100 Subject: [PATCH 32/48] Force stack slot size to be a multiple of 16 This ensures that all stack slots are aligned to 16 bytes. Without this linking against crates compiled with cg_llvm may cause a crash due to simd instructions requiring a 16 byte alignment. --- src/abi/pass_mode.rs | 11 +++++------ src/value_and_place.rs | 8 ++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 6f27fa52d882c..b3231e94a33b9 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -144,7 +144,6 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))], ), - // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); (None, vec![AbiParam::new(vector_ty)]) @@ -210,7 +209,7 @@ pub(super) fn from_casted_value<'tcx>( cast: CastTarget, ) -> CValue<'tcx> { let abi_params = cast_target_to_abi_params(cast); - let size = abi_params + let size: u32 = abi_params .iter() .map(|param| param.value_type.bytes()) .sum(); @@ -218,7 +217,9 @@ pub(super) fn from_casted_value<'tcx>( assert!(u64::from(size) >= layout.size.bytes()); let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, - size, + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (size + 15) / 16 * 16, offset: None, }); let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); @@ -306,9 +307,7 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, )) } - PassMode::Cast(cast) => { - Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) - } + PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)), PassMode::Indirect { attrs: _, extra_attrs: None, diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 17cb09d558707..765604e0f984e 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -334,7 +334,9 @@ impl<'tcx> CPlace<'tcx> { let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, - size: u32::try_from(layout.size.bytes()).unwrap(), + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16, offset: None, }); CPlace { @@ -498,7 +500,9 @@ impl<'tcx> CPlace<'tcx> { // FIXME do something more efficient for transmutes between vectors and integers. let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, - size: src_ty.bytes(), + // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to + // specify stack slot alignment. + size: (src_ty.bytes() + 15) / 16 * 16, offset: None, }); let ptr = Pointer::stack_slot(stack_slot); From 2d6981756da4a21cb54c2b76378a66b326df209e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 12:25:20 +0100 Subject: [PATCH 33/48] Handle argument extension mode --- src/abi/pass_mode.rs | 52 +++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index b3231e94a33b9..2f91e83386ceb 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -3,8 +3,10 @@ use crate::prelude::*; use crate::value_and_place::assert_assignable; -use cranelift_codegen::ir::ArgumentPurpose; -use rustc_target::abi::call::{ArgAbi, CastTarget, PassMode, Reg, RegKind}; +use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; +use rustc_target::abi::call::{ + ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind, +}; use smallvec::{smallvec, SmallVec}; pub(super) trait ArgAbiExt<'tcx> { @@ -27,6 +29,15 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam { AbiParam::new(clif_ty) } +fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { + match arg_attrs.arg_ext { + RustcArgExtension::None => {} + RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext, + RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext, + } + param +} + fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { (0, 0) @@ -82,15 +93,16 @@ fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { args } -// FIXME respect argument extension mode - impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { match self.mode { PassMode::Ignore => smallvec![], - PassMode::Direct(_) => match &self.layout.abi { + PassMode::Direct(attrs) => match &self.layout.abi { Abi::Scalar(scalar) => { - smallvec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))] + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())), + attrs + )] } Abi::Vector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); @@ -98,39 +110,45 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Pair(_, _) => match &self.layout.abi { + PassMode::Pair(attrs_a, attrs_b) => match &self.layout.abi { Abi::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a.clone()); let b = scalar_to_clif_type(tcx, b.clone()); - smallvec![AbiParam::new(a), AbiParam::new(b)] + smallvec![ + apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a), + apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b), + ] } _ => unreachable!("{:?}", self.layout.abi), }, PassMode::Cast(cast) => cast_target_to_abi_params(cast), PassMode::Indirect { - attrs: _, + attrs, extra_attrs: None, on_stack, } => { if on_stack { let size = u32::try_from(self.layout.size.bytes()).unwrap(); - smallvec![AbiParam::special( - pointer_ty(tcx), - ArgumentPurpose::StructArgument(size), + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),), + attrs )] } else { - smallvec![AbiParam::new(pointer_ty(tcx))] + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::new(pointer_ty(tcx)), + attrs + )] } } PassMode::Indirect { - attrs: _, - extra_attrs: Some(_), + attrs, + extra_attrs: Some(extra_attrs), on_stack, } => { assert!(!on_stack); smallvec![ - AbiParam::new(pointer_ty(tcx)), - AbiParam::new(pointer_ty(tcx)), + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), extra_attrs), ] } } From 7fcf59f19e711b89285505785f97dd57f80cbe2e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 14:19:13 +0100 Subject: [PATCH 34/48] Add print on panic for predefining of a function --- src/driver/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 752c3f747d5f7..2497f9dfdfbcf 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -51,6 +51,7 @@ fn predefine_mono_items<'tcx>( match mono_item { MonoItem::Fn(instance) => { let name = cx.tcx.symbol_name(instance).name.to_string(); + let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); cx.module.declare_function(&name, linkage, &sig).unwrap(); From 3eb649c63116b63033135d21635caad806dc8e2a Mon Sep 17 00:00:00 2001 From: Paris DOUADY Date: Sat, 30 Jan 2021 16:25:48 +0100 Subject: [PATCH 35/48] dont set rustflags when no specific linker needed --- scripts/config.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/config.sh b/scripts/config.sh index fe6d86ea6b489..e2914b9bee8de 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -44,7 +44,11 @@ fi dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) export RUSTC=$dir"/bin/cg_clif" -export RUSTFLAGS=$linker" "$RUSTFLAGS + +if [ ! -z $linker ]; then + export RUSTFLAGS=$linker" "$RUSTFLAGS +fi + export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\ '-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir From f3447682d0534f03f7b0195ea8e736f152095718 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 17:22:10 +0100 Subject: [PATCH 36/48] Handle PassMode::Cast in combination with #[repr(align)] --- src/abi/pass_mode.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 2f91e83386ceb..1202c23dbe7b3 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -227,17 +227,19 @@ pub(super) fn from_casted_value<'tcx>( cast: CastTarget, ) -> CValue<'tcx> { let abi_params = cast_target_to_abi_params(cast); - let size: u32 = abi_params + let abi_param_size: u32 = abi_params .iter() .map(|param| param.value_type.bytes()) .sum(); - // Stack slot size may be bigger for for example `[u8; 3]` which is packed into an `i32`. - assert!(u64::from(size) >= layout.size.bytes()); + let layout_size = u32::try_from(layout.size.bytes()).unwrap(); let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to // specify stack slot alignment. - size: (size + 15) / 16 * 16, + // Stack slot size may be bigger for for example `[u8; 3]` which is packed into an `i32`. + // It may also be smaller for example when the type is a wrapper around an integer with a + // larger alignment than the integer. + size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16, offset: None, }); let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); From 8faae2992a6fe33f4a2cf6ac0b11cffd48a7604c Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 18:27:28 +0100 Subject: [PATCH 37/48] Add missing dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 18196bce00945..b241bef9d1e7f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ perf.data.old *.string* /build /build_sysroot/sysroot_src +/build_sysroot/compiler-builtins /rust /rand /regex From 356d749952c2ebc66a30c36f45da57f37f7baac0 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 19:38:57 +0100 Subject: [PATCH 38/48] Update Readme.md for the fixed abi compatibility --- Readme.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 22d9e00923f00..00b9dc2267d20 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,4 @@ -# WIP Cranelift codegen backend for rust - -> ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠ +# Cranelift codegen backend for rust The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift). This has the potential to improve compilation times in debug mode. @@ -103,7 +101,6 @@ function jit_calc() { ## Not yet supported -* Good non-rust abi support ([several problems](https://github.com/bjorn3/rustc_codegen_cranelift/issues/10)) * Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041) * On Linux there is support for invoking an external assembler for `global_asm!` and `asm!`. `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You From 7657ebccc86fc6ac18f8fdd79987050a71523b0a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 20:05:38 +0100 Subject: [PATCH 39/48] Fix macOS build --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index d135c938bf585..344e429563694 100755 --- a/build.sh +++ b/build.sh @@ -36,7 +36,7 @@ unamestr=$(uname) if [[ "$unamestr" == 'Linux' ]]; then export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS elif [[ "$unamestr" == 'Darwin' ]]; then - export RUSTFLAGS='-Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS + export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS dylib_ext='dylib' else echo "Unsupported os" From 84f24e530a0f2359916db4b6e2f47ca52df42994 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 30 Jan 2021 21:26:37 +0100 Subject: [PATCH 40/48] Re-enable mir inlining for the sysroot --- build_sysroot/build_sysroot.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index 3d0a94ad3cbec..ad2796f2e37c9 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -29,8 +29,7 @@ export __CARGO_DEFAULT_LIB_METADATA="cg_clif" if [[ "$1" != "--debug" ]]; then sysroot_channel='release' # FIXME Enable incremental again once rust-lang/rust#74946 is fixed - # FIXME Enable -Zmir-opt-level=2 again once it doesn't ice anymore - CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS" cargo build --target "$TARGET_TRIPLE" --release + CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release else sysroot_channel='debug' cargo build --target "$TARGET_TRIPLE" From 1548411ad904844994cde01990dc9570450d419e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 12:43:39 +0100 Subject: [PATCH 41/48] Add missing directory to clean_all.sh --- clean_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean_all.sh b/clean_all.sh index 5a69c862d016d..b47efe72bce03 100755 --- a/clean_all.sh +++ b/clean_all.sh @@ -1,5 +1,5 @@ #!/bin/bash --verbose set -e -rm -rf target/ build/ build_sysroot/{sysroot_src/,target/} perf.data{,.old} +rm -rf target/ build/ build_sysroot/{sysroot_src/,target/,compiler-builtins/} perf.data{,.old} rm -rf rand/ regex/ simple-raytracer/ From 9c943bf5de795c784aa33e29cb25fa51bc76c833 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 12:44:02 +0100 Subject: [PATCH 42/48] Fix sysroot dir for cg_clif_build_sysroot --- src/bin/cg_clif_build_sysroot.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/bin/cg_clif_build_sysroot.rs b/src/bin/cg_clif_build_sysroot.rs index 8ee4cd46c94e0..83e5dc6e6724d 100644 --- a/src/bin/cg_clif_build_sysroot.rs +++ b/src/bin/cg_clif_build_sysroot.rs @@ -53,10 +53,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { .unwrap() .parent() .unwrap() - .parent() - .unwrap() - .join("build_sysroot") - .join("sysroot"), + .to_owned(), ); } } From 9384af41986da8c1697acde643b3f899d7becaf2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 12:46:17 +0100 Subject: [PATCH 43/48] Link examples to the correct libc when targeting Windows --- example/alloc_example.rs | 3 ++- example/mini_core.rs | 4 ++-- example/mod_bench.rs | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/example/alloc_example.rs b/example/alloc_example.rs index dc2ad4c676e80..f59600ebb330c 100644 --- a/example/alloc_example.rs +++ b/example/alloc_example.rs @@ -11,7 +11,8 @@ use alloc_system::System; #[global_allocator] static ALLOC: System = System; -#[link(name = "c")] +#[cfg_attr(unix, link(name = "c"))] +#[cfg_attr(target_env = "msvc", link(name = "msvcrt"))] extern "C" { fn puts(s: *const u8) -> i32; } diff --git a/example/mini_core.rs b/example/mini_core.rs index 10cba99205629..002ec7e2e3d7a 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -532,8 +532,8 @@ pub mod intrinsics { } pub mod libc { - #[cfg_attr(not(windows), link(name = "c"))] - #[cfg_attr(windows, link(name = "msvcrt"))] + #[cfg_attr(unix, link(name = "c"))] + #[cfg_attr(target_env = "msvc", link(name = "msvcrt"))] extern "C" { pub fn puts(s: *const i8) -> i32; pub fn printf(format: *const i8, ...) -> i32; diff --git a/example/mod_bench.rs b/example/mod_bench.rs index bc65221362346..152041aa9ed0b 100644 --- a/example/mod_bench.rs +++ b/example/mod_bench.rs @@ -1,7 +1,8 @@ #![feature(start, box_syntax, core_intrinsics, lang_items)] #![no_std] -#[link(name = "c")] +#[cfg_attr(unix, link(name = "c"))] +#[cfg_attr(target_env = "msvc", link(name = "msvcrt"))] extern {} #[panic_handler] From 9164653c10b7aa7cfea2b0556ca7c332e6df4a0b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 13:08:07 +0100 Subject: [PATCH 44/48] Split config.sh and add support for copying the mingw runtime objects --- build.sh | 7 +++++++ build_sysroot/build_sysroot.sh | 1 - scripts/config.sh | 29 +---------------------------- scripts/ext_config.sh | 26 ++++++++++++++++++++++++++ test.sh | 2 +- 5 files changed, 35 insertions(+), 30 deletions(-) create mode 100644 scripts/ext_config.sh diff --git a/build.sh b/build.sh index 344e429563694..d688d365f16d6 100755 --- a/build.sh +++ b/build.sh @@ -48,6 +48,8 @@ else cargo build $oldbe fi +source scripts/ext_config.sh + rm -rf "$target_dir" mkdir "$target_dir" mkdir "$target_dir"/bin "$target_dir"/lib @@ -55,6 +57,11 @@ ln target/$CHANNEL/cg_clif{,_build_sysroot} "$target_dir"/bin ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir" +mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" +if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then + cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" +fi + if [[ "$build_sysroot" == "1" ]]; then echo "[BUILD] sysroot" export CG_CLIF_INCR_CACHE_DISABLED=1 diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index ad2796f2e37c9..2785a289d950a 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -36,6 +36,5 @@ else fi # Copy files to sysroot -mkdir -p "$dir/lib/rustlib/$TARGET_TRIPLE/lib/" ln "target/$TARGET_TRIPLE/$sysroot_channel/deps/"* "$dir/lib/rustlib/$TARGET_TRIPLE/lib/" rm "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"*.{rmeta,d} diff --git a/scripts/config.sh b/scripts/config.sh index e2914b9bee8de..834708aa9a6fb 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -12,28 +12,6 @@ else exit 1 fi -HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") -TARGET_TRIPLE=$HOST_TRIPLE -#TARGET_TRIPLE="x86_64-pc-windows-gnu" -#TARGET_TRIPLE="aarch64-unknown-linux-gnu" - -linker='' -RUN_WRAPPER='' -export JIT_SUPPORTED=1 -if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - export JIT_SUPPORTED=0 - if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then - # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. - linker='-Clinker=aarch64-linux-gnu-gcc' - RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu' - elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then - # We are cross-compiling for Windows. Run tests in wine. - RUN_WRAPPER='wine' - else - echo "Unknown non-native platform" - fi -fi - if echo "$RUSTC_WRAPPER" | grep sccache; then echo echo -e "\x1b[1;93m=== Warning: Unset RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m" @@ -45,18 +23,13 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) export RUSTC=$dir"/bin/cg_clif" -if [ ! -z $linker ]; then - export RUSTFLAGS=$linker" "$RUSTFLAGS -fi - export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\ '-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir # FIXME remove once the atomic shim is gone -if [[ $(uname) == 'Darwin' ]]; then +if [[ "$unamestr" == 'Darwin' ]]; then export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" fi export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH - diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh new file mode 100644 index 0000000000000..eab41631e2bda --- /dev/null +++ b/scripts/ext_config.sh @@ -0,0 +1,26 @@ +# Note to people running shellcheck: this file should only be sourced, not executed directly. + +# Various env vars that should only be set for the build system but not for cargo.sh + +set -e + +export CG_CLIF_DISPLAY_CG_TIME=1 + +export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") +export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE} + +export RUN_WRAPPER='' +export JIT_SUPPORTED=1 +if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then + export JIT_SUPPORTED=0 + if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then + # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. + export RUSTFLAGS='-Clinker=aarch64-linux-gnu-gcc '$RUSTFLAGS + export RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu' + elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then + # We are cross-compiling for Windows. Run tests in wine. + export RUN_WRAPPER='wine' + else + echo "Unknown non-native platform" + fi +fi diff --git a/test.sh b/test.sh index ffd795b83ef93..4f3be0765281b 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -export CG_CLIF_DISPLAY_CG_TIME=1 +source scripts/ext_config.sh ./build.sh --without-sysroot "$@" From 8c7ae41f621a2e524bb6585882828314142da9e8 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 14:04:19 +0100 Subject: [PATCH 45/48] Remove code comment --- src/num.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/num.rs b/src/num.rs index 41f4a9b9662bc..d1d2b3b872a4b 100644 --- a/src/num.rs +++ b/src/num.rs @@ -280,7 +280,6 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( (val, fx.bcx.ins().bor(has_underflow, has_overflow)) } types::I64 => { - //let val = fx.easy_call("__mulodi4", &[lhs, rhs, overflow_ptr], types::I64); let val = fx.bcx.ins().imul(lhs, rhs); let has_overflow = if !signed { let val_hi = fx.bcx.ins().umulhi(lhs, rhs); From 0929e379944f3e783092749381f48865b8488875 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 14:15:50 +0100 Subject: [PATCH 46/48] Fix a small error in the readme --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 00b9dc2267d20..6fa5eebdc2f3d 100644 --- a/Readme.md +++ b/Readme.md @@ -101,7 +101,7 @@ function jit_calc() { ## Not yet supported -* Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041) +* Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041)) * On Linux there is support for invoking an external assembler for `global_asm!` and `asm!`. `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You have to specify specific registers instead. From 17cffc1757dc6d8a89e2537e90c6cc5b336a66bf Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 14:56:10 +0100 Subject: [PATCH 47/48] Move some env var definitions around to fix cross-compilation --- build.sh | 1 - build_sysroot/build_sysroot.sh | 1 - scripts/ext_config.sh | 1 + scripts/tests.sh | 2 +- test.sh | 2 -- 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index d688d365f16d6..2f8e0bbc46062 100755 --- a/build.sh +++ b/build.sh @@ -64,7 +64,6 @@ fi if [[ "$build_sysroot" == "1" ]]; then echo "[BUILD] sysroot" - export CG_CLIF_INCR_CACHE_DISABLED=1 dir=$(pwd) cd "$target_dir" time "$dir/build_sysroot/build_sysroot.sh" diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index 2785a289d950a..282ce4a582c4b 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -12,7 +12,6 @@ dir=$(pwd) # build scripts are still compiled using cg_llvm. export RUSTC=$dir"/bin/cg_clif_build_sysroot" export RUSTFLAGS=$RUSTFLAGS" --clif" -export CG_CLIF_DISPLAY_CG_TIME=1 cd "$(dirname "$0")" diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh index eab41631e2bda..7971f620df14b 100644 --- a/scripts/ext_config.sh +++ b/scripts/ext_config.sh @@ -5,6 +5,7 @@ set -e export CG_CLIF_DISPLAY_CG_TIME=1 +export CG_CLIF_INCR_CACHE_DISABLED=1 export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE} diff --git a/scripts/tests.sh b/scripts/tests.sh index a61774f479ec7..d37b57babe612 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -3,7 +3,7 @@ set -e source build/config.sh -export CG_CLIF_INCR_CACHE_DISABLED=1 +source scripts/ext_config.sh MY_RUSTC="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2" function no_sysroot_tests() { diff --git a/test.sh b/test.sh index 4f3be0765281b..97d82c18d977f 100755 --- a/test.sh +++ b/test.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -source scripts/ext_config.sh - ./build.sh --without-sysroot "$@" rm -r target/out || true From d556c56f792756dd7cfec742b9f2e07612dc10f4 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 31 Jan 2021 18:31:33 +0100 Subject: [PATCH 48/48] Add option to build using the sysroot of rustc --- build.sh | 37 ++++++++++++++++++++++++++----------- test.sh | 2 +- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/build.sh b/build.sh index 2f8e0bbc46062..598ce35eceaac 100755 --- a/build.sh +++ b/build.sh @@ -3,7 +3,7 @@ set -e # Settings export CHANNEL="release" -build_sysroot=1 +build_sysroot="clif" target_dir='build' oldbe='' while [[ $# != 0 ]]; do @@ -11,8 +11,9 @@ while [[ $# != 0 ]]; do "--debug") export CHANNEL="debug" ;; - "--without-sysroot") - build_sysroot=0 + "--sysroot") + build_sysroot=$2 + shift ;; "--target-dir") target_dir=$2 @@ -23,7 +24,7 @@ while [[ $# != 0 ]]; do ;; *) echo "Unknown flag '$1'" - echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR] [--oldbe]" + echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--oldbe]" exit 1 ;; esac @@ -62,10 +63,24 @@ if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" fi -if [[ "$build_sysroot" == "1" ]]; then - echo "[BUILD] sysroot" - dir=$(pwd) - cd "$target_dir" - time "$dir/build_sysroot/build_sysroot.sh" - cp lib/rustlib/*/lib/libstd-* lib/ -fi +case "$build_sysroot" in + "none") + ;; + "llvm") + cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/" + ;; + "clif") + echo "[BUILD] sysroot" + dir=$(pwd) + cd "$target_dir" + time "$dir/build_sysroot/build_sysroot.sh" + cp lib/rustlib/*/lib/libstd-* lib/ + ;; + *) + echo "Unknown sysroot kind \`$build_sysroot\`." + echo "The allowed values are:" + echo " none A sysroot that doesn't contain the standard library" + echo " llvm Copy the sysroot from rustc compiled by cg_llvm" + echo " clif Build a new sysroot using cg_clif" + exit 1 +esac diff --git a/test.sh b/test.sh index 97d82c18d977f..5ab10e0e905c7 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -./build.sh --without-sysroot "$@" +./build.sh --sysroot none "$@" rm -r target/out || true