From 0ef11587b0130607627414f5b92bf657d01d7b63 Mon Sep 17 00:00:00 2001
From: Gary Guo <gary@garyguo.net>
Date: Sun, 26 Sep 2021 17:38:05 +0100
Subject: [PATCH 01/84] Remove NullOp::Box

---
 src/base.rs | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/src/base.rs b/src/base.rs
index 1b30edd293862..a17d10cff665b 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -708,30 +708,6 @@ fn codegen_stmt<'tcx>(
                     let operand = operand.load_scalar(fx);
                     lval.write_cvalue(fx, CValue::by_val(operand, box_layout));
                 }
-                Rvalue::NullaryOp(NullOp::Box, content_ty) => {
-                    let usize_type = fx.clif_type(fx.tcx.types.usize).unwrap();
-                    let content_ty = fx.monomorphize(content_ty);
-                    let layout = fx.layout_of(content_ty);
-                    let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64);
-                    let llalign = fx.bcx.ins().iconst(usize_type, layout.align.abi.bytes() as i64);
-                    let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
-
-                    // Allocate space:
-                    let def_id =
-                        match fx.tcx.lang_items().require(rustc_hir::LangItem::ExchangeMalloc) {
-                            Ok(id) => id,
-                            Err(s) => {
-                                fx.tcx
-                                    .sess
-                                    .fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
-                            }
-                        };
-                    let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
-                    let func_ref = fx.get_function_ref(instance);
-                    let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]);
-                    let ptr = fx.bcx.inst_results(call)[0];
-                    lval.write_cvalue(fx, CValue::by_val(ptr, box_layout));
-                }
                 Rvalue::NullaryOp(null_op, ty) => {
                     assert!(
                         lval.layout()
@@ -742,7 +718,6 @@ fn codegen_stmt<'tcx>(
                     let val = match null_op {
                         NullOp::SizeOf => layout.size.bytes(),
                         NullOp::AlignOf => layout.align.abi.bytes(),
-                        NullOp::Box => unreachable!(),
                     };
                     let val =
                         CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());

From a5a14258e5c2ef32f05621595740595676d54177 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Thu, 30 Dec 2021 14:53:41 +0100
Subject: [PATCH 02/84] Merge commit '40b00f4200fbdeefd11815398cb46394b8cb0a5e'
 into sync_cg_clif-2021-12-30

---
 .github/workflows/main.yml              | 15 +++++++++++
 .github/workflows/nightly-cranelift.yml |  2 +-
 .gitignore                              |  1 +
 Cargo.toml                              | 19 -------------
 Readme.md                               |  2 +-
 build_system/build_backend.rs           | 31 ++++++++++++---------
 build_system/build_sysroot.rs           |  4 +--
 docs/usage.md                           |  6 ++---
 example/mini_core_hello_world.rs        |  1 +
 rust-toolchain                          |  2 +-
 scripts/{cargo.rs => cargo-clif.rs}     |  0
 scripts/setup_rust_fork.sh              |  2 +-
 scripts/test_rustc_tests.sh             | 16 ++++++-----
 scripts/tests.sh                        | 36 ++++++++++++-------------
 src/common.rs                           |  2 +-
 src/debuginfo/emit.rs                   |  2 +-
 src/debuginfo/mod.rs                    | 18 +++++--------
 src/debuginfo/unwind.rs                 |  8 ++++--
 src/driver/aot.rs                       |  2 +-
 src/lib.rs                              |  2 +-
 20 files changed, 90 insertions(+), 81 deletions(-)
 rename scripts/{cargo.rs => cargo-clif.rs} (100%)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7b73d3c00e60e..3aba528abfd6d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -5,6 +5,21 @@ on:
   - pull_request
 
 jobs:
+  rustfmt:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install rustfmt
+      run: |
+        rustup component add rustfmt
+
+    - name: Rustfmt
+      run: |
+        cargo fmt --check
+
   build:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
diff --git a/.github/workflows/nightly-cranelift.yml b/.github/workflows/nightly-cranelift.yml
index c5b96a4782804..a019793edd8d2 100644
--- a/.github/workflows/nightly-cranelift.yml
+++ b/.github/workflows/nightly-cranelift.yml
@@ -3,7 +3,7 @@ name: Test nightly Cranelift
 on:
   push:
   schedule:
-    - cron: '1 17 * * *' # At 01:17 UTC every day.
+    - cron: '17 1 * * *' # At 01:17 UTC every day.
 
 jobs:
   build:
diff --git a/.gitignore b/.gitignore
index b6567aca78679..5aeaf3a178804 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@ perf.data.old
 *.events
 *.string*
 /y.bin
+/y.bin.dSYM
 /build
 /build_sysroot/sysroot_src
 /build_sysroot/compiler-builtins
diff --git a/Cargo.toml b/Cargo.toml
index 900411286b52e..3be4250296e77 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,31 +40,12 @@ unstable-features = ["jit", "inline_asm"]
 jit = ["cranelift-jit", "libloading"]
 inline_asm = []
 
-[profile.dev]
-# By compiling dependencies with optimizations, performing tests gets much faster.
-opt-level = 3
-
-[profile.dev.package.rustc_codegen_cranelift]
-# Disabling optimizations for cg_clif itself makes compilation after a change faster.
-opt-level = 0
-
-[profile.release.package.rustc_codegen_cranelift]
-incremental = true
-
 # Disable optimizations and debuginfo of build scripts and some of the heavy build deps, as the
 # execution time of build scripts is so fast that optimizing them slows down the total build time.
-[profile.dev.build-override]
-opt-level = 0
-debug = false
-
 [profile.release.build-override]
 opt-level = 0
 debug = false
 
-[profile.dev.package.cranelift-codegen-meta]
-opt-level = 0
-debug = false
-
 [profile.release.package.cranelift-codegen-meta]
 opt-level = 0
 debug = false
diff --git a/Readme.md b/Readme.md
index dad8ed90b53b8..8a2db5a43ecbf 100644
--- a/Readme.md
+++ b/Readme.md
@@ -37,7 +37,7 @@ Assuming `$cg_clif_dir` is the directory you cloned this repo into and you follo
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/build/cargo build
+$ $cg_clif_dir/build/cargo-clif build
 ```
 
 This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs
index ccc50ee4a59bf..1382c7e53793e 100644
--- a/build_system/build_backend.rs
+++ b/build_system/build_backend.rs
@@ -10,6 +10,18 @@ pub(crate) fn build_backend(
     let mut cmd = Command::new("cargo");
     cmd.arg("build").arg("--target").arg(host_triple);
 
+    cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
+
+    let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();
+
+    if env::var("CI").as_ref().map(|val| &**val) == Ok("true") {
+        // Deny warnings on CI
+        rustflags += " -Dwarnings";
+
+        // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
+        cmd.env("CARGO_BUILD_INCREMENTAL", "false");
+    }
+
     if use_unstable_features {
         cmd.arg("--features").arg("unstable-features");
     }
@@ -22,25 +34,20 @@ pub(crate) fn build_backend(
         _ => unreachable!(),
     }
 
+    // Set the rpath to make the cg_clif executable find librustc_codegen_cranelift without changing
+    // LD_LIBRARY_PATH
     if cfg!(unix) {
         if cfg!(target_os = "macos") {
-            cmd.env(
-                "RUSTFLAGS",
-                "-Csplit-debuginfo=unpacked \
+            rustflags += " -Csplit-debuginfo=unpacked \
                 -Clink-arg=-Wl,-rpath,@loader_path/../lib \
-                -Zosx-rpath-install-name"
-                    .to_string()
-                    + env::var("RUSTFLAGS").as_deref().unwrap_or(""),
-            );
+                -Zosx-rpath-install-name";
         } else {
-            cmd.env(
-                "RUSTFLAGS",
-                "-Clink-arg=-Wl,-rpath=$ORIGIN/../lib ".to_string()
-                    + env::var("RUSTFLAGS").as_deref().unwrap_or(""),
-            );
+            rustflags += " -Clink-arg=-Wl,-rpath=$ORIGIN/../lib ";
         }
     }
 
+    cmd.env("RUSTFLAGS", rustflags);
+
     eprintln!("[BUILD] rustc_codegen_cranelift");
     crate::utils::spawn_and_wait(cmd);
 
diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs
index 1c78e7b5171ee..2956fb698e175 100644
--- a/build_system/build_sysroot.rs
+++ b/build_system/build_sysroot.rs
@@ -46,9 +46,9 @@ pub(crate) fn build_sysroot(
     // Build and copy cargo wrapper
     let mut build_cargo_wrapper_cmd = Command::new("rustc");
     build_cargo_wrapper_cmd
-        .arg("scripts/cargo.rs")
+        .arg("scripts/cargo-clif.rs")
         .arg("-o")
-        .arg(target_dir.join("cargo"))
+        .arg(target_dir.join("cargo-clif"))
         .arg("-g");
     spawn_and_wait(build_cargo_wrapper_cmd);
 
diff --git a/docs/usage.md b/docs/usage.md
index bcc5745d9d197..785c738378374 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -9,7 +9,7 @@ Assuming `$cg_clif_dir` is the directory you cloned this repo into and you follo
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/build/cargo build
+$ $cg_clif_dir/build/cargo-clif build
 ```
 
 This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
@@ -32,7 +32,7 @@ In jit mode cg_clif will immediately execute your code without creating an execu
 > The jit mode will probably need cargo integration to make this possible.
 
 ```bash
-$ $cg_clif_dir/build/cargo jit
+$ $cg_clif_dir/build/cargo-clif jit
 ```
 
 or
@@ -45,7 +45,7 @@ There is also an experimental lazy jit mode. In this mode functions are only com
 first called.
 
 ```bash
-$ $cg_clif_dir/build/cargo lazy-jit
+$ $cg_clif_dir/build/cargo-clif lazy-jit
 ```
 
 ## Shell
diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs
index cbfdb3c44f33e..ef3b575d39314 100644
--- a/example/mini_core_hello_world.rs
+++ b/example/mini_core_hello_world.rs
@@ -129,6 +129,7 @@ fn call_return_u128_pair() {
     return_u128_pair();
 }
 
+#[allow(unreachable_code)] // FIXME false positive
 fn main() {
     take_unique(Unique {
         pointer: 0 as *const (),
diff --git a/rust-toolchain b/rust-toolchain
index 7b5db307a2dc2..cab94c0b8cfa7 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2021-12-20"
+channel = "nightly-2021-12-30"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/scripts/cargo.rs b/scripts/cargo-clif.rs
similarity index 100%
rename from scripts/cargo.rs
rename to scripts/cargo-clif.rs
diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh
index 46c3b5b7f11ad..73600faa1e9c2 100644
--- a/scripts/setup_rust_fork.sh
+++ b/scripts/setup_rust_fork.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-./y.rs build
+./y.rs build --no-unstable-features
 source scripts/config.sh
 
 echo "[SETUP] Rust fork"
diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh
index 99fddf5361e43..6bcc3049ecc4e 100755
--- a/scripts/test_rustc_tests.sh
+++ b/scripts/test_rustc_tests.sh
@@ -47,6 +47,8 @@ rm src/test/ui/codegen/init-large-type.rs # same
 rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
 rm src/test/ui/issues/issue-33992.rs # unsupported linkages
 rm src/test/ui/issues/issue-51947.rs # same
+rm src/test/incremental/hashes/function_interfaces.rs # same
+rm src/test/incremental/hashes/statics.rs # same
 rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
 rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
 rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
@@ -60,18 +62,14 @@ rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and n
 
 rm src/test/incremental/hashes/inline_asm.rs # inline asm
 rm src/test/incremental/issue-72386.rs # same
-rm src/test/incremental/issue-49482.rs # same
-rm src/test/incremental/issue-54059.rs # same
 rm src/test/incremental/lto.rs # requires lto
+rm src/test/incremental/dirty_clean.rs # TODO
 
 rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/
 rm -r src/test/run-make/unstable-flag-required # same
 rm -r src/test/run-make/rustdoc-* # same
 rm -r src/test/run-make/emit-named-files # requires full --emit support
 
-rm src/test/pretty/asm.rs # inline asm
-rm src/test/pretty/raw-str-nonexpr.rs # same
-
 rm -r src/test/run-pass-valgrind/unsized-locals
 
 rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
@@ -97,6 +95,12 @@ rm src/test/ui/command/command-current-dir.rs # can't find libstd.so
 
 rm src/test/ui/abi/stack-protector.rs # requires stack protector support
 
+rm src/test/incremental/issue-80691-bad-eval-cache.rs # wrong exit code
+rm src/test/incremental/spike-neg1.rs # errors out for some reason
+rm src/test/incremental/spike-neg2.rs # same
+
+rm src/test/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM
+
 echo "[TEST] rustc test suite"
-RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui,incremental}
 popd
diff --git a/scripts/tests.sh b/scripts/tests.sh
index fd2b3761ff036..bdb3de0936dc9 100755
--- a/scripts/tests.sh
+++ b/scripts/tests.sh
@@ -80,73 +80,73 @@ function base_sysroot_tests() {
 
 function extended_sysroot_tests() {
     pushd rand
-    ../build/cargo clean
+    ../build/cargo-clif clean
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[TEST] rust-random/rand"
-        ../build/cargo test --workspace
+        ../build/cargo-clif test --workspace
     else
         echo "[AOT] rust-random/rand"
-        ../build/cargo build --workspace --target $TARGET_TRIPLE --tests
+        ../build/cargo-clif build --workspace --target $TARGET_TRIPLE --tests
     fi
     popd
 
     pushd simple-raytracer
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
-        hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo clean" \
+        hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo-clif clean" \
         "RUSTC=rustc RUSTFLAGS='' cargo build" \
-        "../build/cargo build"
+        "../build/cargo-clif build"
 
         echo "[BENCH RUN] ebobby/simple-raytracer"
         cp ./target/debug/main ./raytracer_cg_clif
         hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_clif
     else
-        ../build/cargo clean
+        ../build/cargo-clif clean
         echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
         echo "[COMPILE] ebobby/simple-raytracer"
-        ../build/cargo build --target $TARGET_TRIPLE
+        ../build/cargo-clif build --target $TARGET_TRIPLE
         echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
     fi
     popd
 
     pushd build_sysroot/sysroot_src/library/core/tests
     echo "[TEST] libcore"
-    ../../../../../build/cargo clean
+    ../../../../../build/cargo-clif clean
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        ../../../../../build/cargo test
+        ../../../../../build/cargo-clif test
     else
-        ../../../../../build/cargo build --target $TARGET_TRIPLE --tests
+        ../../../../../build/cargo-clif build --target $TARGET_TRIPLE --tests
     fi
     popd
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo clean
+    ../build/cargo-clif clean
     export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning
     # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo build --example shootout-regex-dna --target $TARGET_TRIPLE
+    ../build/cargo-clif build --example shootout-regex-dna --target $TARGET_TRIPLE
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         cat examples/regexdna-input.txt \
-            | ../build/cargo run --example shootout-regex-dna --target $TARGET_TRIPLE \
+            | ../build/cargo-clif run --example shootout-regex-dna --target $TARGET_TRIPLE \
             | grep -v "Spawned thread" > res.txt
         diff -u res.txt examples/regexdna-output.txt
     fi
 
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[TEST] rust-lang/regex tests"
-        ../build/cargo test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+        ../build/cargo-clif test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
     else
         echo "[AOT] rust-lang/regex tests"
-        ../build/cargo build --tests --target $TARGET_TRIPLE
+        ../build/cargo-clif build --tests --target $TARGET_TRIPLE
     fi
     popd
 
     pushd portable-simd
     echo "[TEST] rust-lang/portable-simd"
-    ../build/cargo clean
-    ../build/cargo build --all-targets --target $TARGET_TRIPLE
+    ../build/cargo-clif clean
+    ../build/cargo-clif build --all-targets --target $TARGET_TRIPLE
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        ../build/cargo test -q
+        ../build/cargo-clif test -q
     fi
     popd
 }
diff --git a/src/common.rs b/src/common.rs
index 644204d10b8ed..3b6025c73d10b 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -237,7 +237,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) module: &'m mut dyn Module,
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) target_config: TargetFrontendConfig, // Cached from module
-    pub(crate) pointer_type: Type, // Cached from module
+    pub(crate) pointer_type: Type,                  // Cached from module
     pub(crate) constants_cx: ConstantCx,
 
     pub(crate) instance: Instance<'tcx>,
diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs
index 4120ba6e53352..589910ede9688 100644
--- a/src/debuginfo/emit.rs
+++ b/src/debuginfo/emit.rs
@@ -67,7 +67,7 @@ impl WriterRelocate {
     }
 
     /// Perform the collected relocations to be usable for JIT usage.
-    #[cfg(feature = "jit")]
+    #[cfg(all(feature = "jit", not(windows)))]
     pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
         for reloc in self.relocs.drain(..) {
             match reloc.name {
diff --git a/src/debuginfo/mod.rs b/src/debuginfo/mod.rs
index dd19dd5d2b91c..638b025be229d 100644
--- a/src/debuginfo/mod.rs
+++ b/src/debuginfo/mod.rs
@@ -10,7 +10,7 @@ use crate::prelude::*;
 use rustc_index::vec::IndexVec;
 
 use cranelift_codegen::entity::EntityRef;
-use cranelift_codegen::ir::{LabelValueLoc, ValueLabel};
+use cranelift_codegen::ir::{Endianness, LabelValueLoc, ValueLabel};
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::ValueLocRange;
 
@@ -23,15 +23,6 @@ use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, X86_64};
 pub(crate) use emit::{DebugReloc, DebugRelocName};
 pub(crate) use unwind::UnwindContext;
 
-fn target_endian(tcx: TyCtxt<'_>) -> RunTimeEndian {
-    use rustc_target::abi::Endian;
-
-    match tcx.data_layout.endian {
-        Endian::Big => RunTimeEndian::Big,
-        Endian::Little => RunTimeEndian::Little,
-    }
-}
-
 pub(crate) struct DebugContext<'tcx> {
     tcx: TyCtxt<'tcx>,
 
@@ -60,6 +51,11 @@ impl<'tcx> DebugContext<'tcx> {
             address_size: isa.frontend_config().pointer_bytes(),
         };
 
+        let endian = match isa.endianness() {
+            Endianness::Little => RunTimeEndian::Little,
+            Endianness::Big => RunTimeEndian::Big,
+        };
+
         let mut dwarf = DwarfUnit::new(encoding);
 
         let producer = format!(
@@ -108,7 +104,7 @@ impl<'tcx> DebugContext<'tcx> {
         DebugContext {
             tcx,
 
-            endian: target_endian(tcx),
+            endian,
 
             dwarf,
             unit_range_list: RangeList(Vec::new()),
diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs
index f0896ea0e167f..e4f28338096e1 100644
--- a/src/debuginfo/unwind.rs
+++ b/src/debuginfo/unwind.rs
@@ -2,6 +2,7 @@
 
 use crate::prelude::*;
 
+use cranelift_codegen::ir::Endianness;
 use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa};
 
 use cranelift_object::ObjectProduct;
@@ -17,8 +18,11 @@ pub(crate) struct UnwindContext {
 }
 
 impl UnwindContext {
-    pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
-        let endian = super::target_endian(tcx);
+    pub(crate) fn new(isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
+        let endian = match isa.endianness() {
+            Endianness::Little => RunTimeEndian::Little,
+            Endianness::Big => RunTimeEndian::Big,
+        };
         let mut frame_table = FrameTable::default();
 
         let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
diff --git a/src/driver/aot.rs b/src/driver/aot.rs
index 7f888c80464d4..046e4393a68d6 100644
--- a/src/driver/aot.rs
+++ b/src/driver/aot.rs
@@ -243,7 +243,7 @@ pub(crate) fn run_aot(
     let isa = crate::build_isa(tcx.sess, &backend_config);
     let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string());
     assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
-    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
+    let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
     let created_alloc_shim =
         crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
 
diff --git a/src/lib.rs b/src/lib.rs
index 3f2884748272a..cb18f42f741d8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -141,7 +141,7 @@ impl<'tcx> CodegenCx<'tcx> {
         assert_eq!(pointer_ty(tcx), isa.pointer_type());
 
         let unwind_context =
-            UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
+            UnwindContext::new(isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
         let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None };
         CodegenCx {
             tcx,

From e4fff03d40ca602d447f3965e1afed2262978640 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Wed, 5 Jan 2022 12:22:23 +0100
Subject: [PATCH 03/84] Update Cranelift to 0.79.1

This version fixes the regressions in 0.79.0
---
 Cargo.lock | 51 ++++++++++++++++++++++++---------------------------
 Cargo.toml | 14 +++++++-------
 2 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 65e142a00f88e..4e2106e3fbe25 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -33,18 +33,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc0cb7df82c8cf8f2e6a8dd394a0932a71369c160cc9b027dca414fced242513"
+checksum = "ebddaa5d12cb299b0bc7c930aff12c5591d4ba9aa84eea637807e07283b900aa"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe4463c15fa42eee909e61e5eac4866b7c6d22d0d8c621e57a0c5380753bfa8c"
+checksum = "da1daf5641177162644b521b64418564b8ed5deb126275a4d91472d13e7c72df"
 dependencies = [
  "cranelift-bforest",
  "cranelift-codegen-meta",
@@ -59,31 +59,30 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793f6a94a053a55404ea16e1700202a88101672b8cd6b4df63e13cde950852bf"
+checksum = "001c1e9e540940c81596e547e732f99c2146c21ea7e82da99be961a1e86feefa"
 dependencies = [
  "cranelift-codegen-shared",
- "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44aa1846df275bce5eb30379d65964c7afc63c05a117076e62a119c25fe174be"
+checksum = "0ebaf07b5d7501cc606f41c81333bd63a5a17eb501362ccb10bc8ff5c03d0232"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3a45d8d6318bf8fc518154d9298eab2a8154ec068a8885ff113f6db8d69bb3a"
+checksum = "a27ada0e3ffe5325179fc750252c18d614fa5470d595ce5c8a794c495434d80a"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e07339bd461766deb7605169de039e01954768ff730fa1254e149001884a8525"
+checksum = "2912c0eec9fd3df2dcf82b02b642caaa85d762b84ac5a3b27bc93a07eeeb64e2"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -93,9 +92,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e8f0d60fb5d67f7a1e5c49db38ba96d1c846921faef02085fc5590b74781747"
+checksum = "bb1bb6937aefbd4c84ec16bcbf4e4138e0c182a965e9b45b59c8ad0c5a716fd5"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -111,21 +110,19 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "825ac7e0959cbe7ddc9cc21209f0319e611a57f9fcb2b723861fe7ef2017e651"
+checksum = "ff8c96b8b2fad9879207762ec6fe2f0d32d1f98b215a9dc360f52f7d2b832600"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
- "cranelift-entity",
- "log",
 ]
 
 [[package]]
 name = "cranelift-native"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03e2fca76ff57e0532936a71e3fc267eae6a19a86656716479c66e7f912e3d7b"
+checksum = "9fd20f78f378f55a70738a2eb9815dcd7e8455ff091b70701cfd086dd44927da"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -134,9 +131,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.78.0"
+version = "0.79.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55500d0fc9bb05c0944fc4506649249d28f55bd4fe95b87f0e55bf41058f0e6d"
+checksum = "34a2db23dcee236e25bccb9a6339aa555c1c0e27573c198607d869530453fb4a"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -157,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.25.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7"
+checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
 dependencies = [
  "indexmap",
 ]
@@ -233,9 +230,9 @@ dependencies = [
 
 [[package]]
 name = "regalloc"
-version = "0.0.32"
+version = "0.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6304468554ed921da3d32c355ea107b8d13d7b8996c3adfb7aab48d3bc321f4"
+checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a"
 dependencies = [
  "log",
  "rustc-hash",
diff --git a/Cargo.toml b/Cargo.toml
index 3be4250296e77..7bd3012ff863f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,14 +8,14 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.78.0", features = ["unwind", "all-arch"] }
-cranelift-frontend = "0.78.0"
-cranelift-module = "0.78.0"
-cranelift-native = "0.78.0"
-cranelift-jit = { version = "0.78.0", optional = true }
-cranelift-object = "0.78.0"
+cranelift-codegen = { version = "0.79.1", features = ["unwind", "all-arch"] }
+cranelift-frontend = "0.79.1"
+cranelift-module = "0.79.1"
+cranelift-native = "0.79.1"
+cranelift-jit = { version = "0.79.1", optional = true }
+cranelift-object = "0.79.1"
 target-lexicon = "0.12.0"
-gimli = { version = "0.25.0", default-features = false, features = ["write"]}
+gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }

From 99136301583c6c88e41ac517b9b4b37dadf1ec83 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 14:54:43 +0100
Subject: [PATCH 04/84] Reduce usage of subst types in the intrinsic code

Using the arguments often saves a layout_of call
---
 src/intrinsics/mod.rs | 114 ++++++++++++++++++++----------------------
 1 file changed, 55 insertions(+), 59 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index f4703b22ecbcf..da9aa45069b83 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -41,19 +41,11 @@ macro intrinsic_arg {
     }
 }
 
-macro intrinsic_substs {
-    ($substs:expr, $index:expr,) => {},
-    ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
-        let $first = $substs.type_at($index);
-        intrinsic_substs!($substs, $index+1, $($rest),*);
-    }
-}
-
 macro intrinsic_match {
     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
     _ => $unknown:block;
     $(
-        $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
+        $($($name:tt).*)|+ $(if $cond:expr)?, ($($a:ident $arg:ident),*) $content:block;
     )*) => {
         let _ = $substs; // Silence warning when substs is unused.
         match $intrinsic {
@@ -61,9 +53,6 @@ macro intrinsic_match {
                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
                     #[allow(unused_parens, non_snake_case)]
                     {
-                        $(
-                            intrinsic_substs!($substs, 0, $($subst),*);
-                        )?
                         if let [$($arg),*] = $args {
                             let ($($arg,)*) = (
                                 $(intrinsic_arg!($a $fx, $arg),)*
@@ -492,7 +481,8 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         breakpoint, () {
             fx.bcx.ins().debugtrap();
         };
-        copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
+        copy | copy_nonoverlapping, (v src, v dst, v count) {
+            let elem_ty = substs.type_at(0);
             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
             assert_eq!(args.len(), 3);
             let byte_amount = if elem_size != 1 {
@@ -510,7 +500,8 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             }
         };
         // NOTE: the volatile variants have src and dst swapped
-        volatile_copy_memory | volatile_copy_nonoverlapping_memory, <elem_ty> (v dst, v src, v count) {
+        volatile_copy_memory | volatile_copy_nonoverlapping_memory, (v dst, v src, v count) {
+            let elem_ty = substs.type_at(0);
             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
             assert_eq!(args.len(), 3);
             let byte_amount = if elem_size != 1 {
@@ -528,8 +519,8 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
             }
         };
-        size_of_val, <T> (c ptr) {
-            let layout = fx.layout_of(T);
+        size_of_val, (c ptr) {
+            let layout = fx.layout_of(substs.type_at(0));
             let size = if layout.is_unsized() {
                 let (_ptr, info) = ptr.load_scalar_pair(fx);
                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
@@ -542,8 +533,8 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             };
             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
         };
-        min_align_of_val, <T> (c ptr) {
-            let layout = fx.layout_of(T);
+        min_align_of_val, (c ptr) {
+            let layout = fx.layout_of(substs.type_at(0));
             let align = if layout.is_unsized() {
                 let (_ptr, info) = ptr.load_scalar_pair(fx);
                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
@@ -589,7 +580,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             );
             ret.write_cvalue(fx, res);
         };
-        saturating_add | saturating_sub, <T> (c lhs, c rhs) {
+        saturating_add | saturating_sub, (c lhs, c rhs) {
             assert_eq!(lhs.layout().ty, rhs.layout().ty);
             let bin_op = match intrinsic {
                 sym::saturating_add => BinOp::Add,
@@ -597,7 +588,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 _ => unreachable!(),
             };
 
-            let signed = type_sign(T);
+            let signed = type_sign(lhs.layout().ty);
 
             let checked_res = crate::num::codegen_checked_int_binop(
                 fx,
@@ -607,7 +598,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             );
 
             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
-            let clif_ty = fx.clif_type(T).unwrap();
+            let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
 
             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 
@@ -629,17 +620,19 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 _ => unreachable!(),
             };
 
-            let res = CValue::by_val(val, fx.layout_of(T));
+            let res = CValue::by_val(val, lhs.layout());
 
             ret.write_cvalue(fx, res);
         };
-        rotate_left, <T>(v x, v y) {
-            let layout = fx.layout_of(T);
+        rotate_left, (c x, v y) {
+            let layout = x.layout();
+            let x = x.load_scalar(fx);
             let res = fx.bcx.ins().rotl(x, y);
             ret.write_cvalue(fx, CValue::by_val(res, layout));
         };
-        rotate_right, <T>(v x, v y) {
-            let layout = fx.layout_of(T);
+        rotate_right, (c x, v y) {
+            let layout = x.layout();
+            let x = x.load_scalar(fx);
             let res = fx.bcx.ins().rotr(x, y);
             ret.write_cvalue(fx, CValue::by_val(res, layout));
         };
@@ -675,29 +668,33 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             // FIXME use emit_small_memset
             fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
         };
-        ctlz | ctlz_nonzero, <T> (v arg) {
+        ctlz | ctlz_nonzero, (c arg) {
+            let val = arg.load_scalar(fx);
             // FIXME trap on `ctlz_nonzero` with zero arg.
-            let res = fx.bcx.ins().clz(arg);
-            let res = CValue::by_val(res, fx.layout_of(T));
+            let res = fx.bcx.ins().clz(val);
+            let res = CValue::by_val(res, arg.layout());
             ret.write_cvalue(fx, res);
         };
-        cttz | cttz_nonzero, <T> (v arg) {
+        cttz | cttz_nonzero, (c arg) {
+            let val = arg.load_scalar(fx);
             // FIXME trap on `cttz_nonzero` with zero arg.
-            let res = fx.bcx.ins().ctz(arg);
-            let res = CValue::by_val(res, fx.layout_of(T));
+            let res = fx.bcx.ins().ctz(val);
+            let res = CValue::by_val(res, arg.layout());
             ret.write_cvalue(fx, res);
         };
-        ctpop, <T> (v arg) {
-            let res = fx.bcx.ins().popcnt(arg);
-            let res = CValue::by_val(res, fx.layout_of(T));
+        ctpop, (c arg) {
+            let val = arg.load_scalar(fx);
+            let res = fx.bcx.ins().popcnt(val);
+            let res = CValue::by_val(res, arg.layout());
             ret.write_cvalue(fx, res);
         };
-        bitreverse, <T> (v arg) {
-            let res = fx.bcx.ins().bitrev(arg);
-            let res = CValue::by_val(res, fx.layout_of(T));
+        bitreverse, (c arg) {
+            let val = arg.load_scalar(fx);
+            let res = fx.bcx.ins().bitrev(val);
+            let res = CValue::by_val(res, arg.layout());
             ret.write_cvalue(fx, res);
         };
-        bswap, <T> (v arg) {
+        bswap, (c arg) {
             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
                 match bcx.func.dfg.value_type(v) {
@@ -773,15 +770,16 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                     ty => unreachable!("bswap {}", ty),
                 }
             }
-            let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
+            let val = arg.load_scalar(fx);
+            let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout());
             ret.write_cvalue(fx, res);
         };
-        assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () {
-            let layout = fx.layout_of(T);
+        assert_inhabited | assert_zero_valid | assert_uninit_valid, () {
+            let layout = fx.layout_of(substs.type_at(0));
             if layout.abi.is_uninhabited() {
                 with_no_trimmed_paths(|| crate::base::codegen_panic(
                     fx,
-                    &format!("attempted to instantiate uninhabited type `{}`", T),
+                    &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
                     span,
                 ));
                 return;
@@ -790,7 +788,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true) {
                 with_no_trimmed_paths(|| crate::base::codegen_panic(
                     fx,
-                    &format!("attempted to zero-initialize type `{}`, which is invalid", T),
+                    &format!("attempted to zero-initialize type `{}`, which is invalid", layout.ty),
                     span,
                 ));
                 return;
@@ -799,7 +797,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false) {
                 with_no_trimmed_paths(|| crate::base::codegen_panic(
                     fx,
-                    &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
+                    &format!("attempted to leave type `{}` uninitialized, which is invalid", layout.ty),
                     span,
                 ));
                 return;
@@ -832,10 +830,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, val);
         };
 
-        ptr_offset_from, <T> (v ptr, v base) {
+        ptr_offset_from, (v ptr, v base) {
+            let ty = substs.type_at(0);
             let isize_layout = fx.layout_of(fx.tcx.types.isize);
 
-            let pointee_size: u64 = fx.layout_of(T).size.bytes();
+            let pointee_size: u64 = fx.layout_of(ty).size.bytes();
             let diff = fx.bcx.ins().isub(ptr, base);
             // FIXME this can be an exact division.
             let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout);
@@ -864,13 +863,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             // FIXME use a compiler fence once Cranelift supports it
             fx.bcx.ins().fence();
         };
-        _ if intrinsic.as_str().starts_with("atomic_load"), <T> (v ptr) {
-            validate_atomic_type!(fx, intrinsic, span, T);
-            let ty = fx.clif_type(T).unwrap();
+        _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) {
+            let ty = substs.type_at(0);
+            validate_atomic_type!(fx, intrinsic, span, ty);
+            let clif_ty = fx.clif_type(ty).unwrap();
 
-            let val = fx.bcx.ins().atomic_load(ty, MemFlags::trusted(), ptr);
+            let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
 
-            let val = CValue::by_val(val, fx.layout_of(T));
+            let val = CValue::by_val(val, fx.layout_of(ty));
             ret.write_cvalue(fx, val);
         };
         _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
@@ -1101,18 +1101,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
         };
 
-        raw_eq, <T>(v lhs_ref, v rhs_ref) {
-            fn type_by_size(size: Size) -> Option<Type> {
-                Type::int(size.bits().try_into().ok()?)
-            }
-
-            let size = fx.layout_of(T).layout.size;
+        raw_eq, (v lhs_ref, v rhs_ref) {
+            let size = fx.layout_of(substs.type_at(0)).layout.size;
             // FIXME add and use emit_small_memcmp
             let is_eq_value =
                 if size == Size::ZERO {
                     // No bytes means they're trivially equal
                     fx.bcx.ins().iconst(types::I8, 1)
-                } else if let Some(clty) = type_by_size(size) {
+                } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
                     // Can't use `trusted` for these loads; they could be unaligned.
                     let mut flags = MemFlags::new();
                     flags.set_notrap();

From c5b969583ff12cfd0cd85e2923753101024b72ad Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 15:17:42 +0100
Subject: [PATCH 05/84] Split codegen_intrinsic_call function

This should reduce compile times of cg_clif
---
 src/intrinsics/mod.rs | 49 +++++++++++++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index da9aa45069b83..517deba580505 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -73,7 +73,7 @@ macro intrinsic_match {
 }
 
 macro call_intrinsic_match {
-    ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
+    ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $args:expr, $(
         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
     )*) => {
         match $intrinsic {
@@ -87,19 +87,13 @@ macro call_intrinsic_match {
                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
                         $ret.write_cvalue($fx, res);
 
-                        if let Some((_, dest)) = $destination {
-                            let ret_block = $fx.get_block(dest);
-                            $fx.bcx.ins().jump(ret_block, &[]);
-                            return;
-                        } else {
-                            unreachable!();
-                        }
+                        return true;
                     } else {
                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
                     }
                 }
             )*
-            _ => {}
+            _ => false,
         }
     }
 }
@@ -397,7 +391,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     span: Span,
 ) {
     let intrinsic = fx.tcx.item_name(instance.def_id());
-    let substs = instance.substs;
 
     let ret = match destination {
         Some((place, _)) => place,
@@ -420,13 +413,27 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
-        return;
+    } else if codegen_float_intrinsic_call(fx, instance, args, ret) {
+        let ret_block = fx.get_block(destination.expect("Float intrinsics don't diverge").1);
+        fx.bcx.ins().jump(ret_block, &[]);
+    } else {
+        codegen_regular_intrinsic_call(fx, instance, args, ret, span, destination);
     }
+}
 
-    let usize_layout = fx.layout_of(fx.tcx.types.usize);
+fn codegen_float_intrinsic_call<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    instance: Instance<'tcx>,
+    args: &[mir::Operand<'tcx>],
+    ret: CPlace<'tcx>,
+) -> bool {
+    let def_id = instance.def_id();
+    let substs = instance.substs;
+
+    let intrinsic = fx.tcx.item_name(def_id);
 
     call_intrinsic_match! {
-        fx, intrinsic, substs, ret, destination, args,
+        fx, intrinsic, substs, ret, args,
         expf32(flt) -> f32 => expf,
         expf64(flt) -> f64 => exp,
         exp2f32(flt) -> f32 => exp2f,
@@ -467,6 +474,22 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         cosf32(flt) -> f32 => cosf,
         cosf64(flt) -> f64 => cos,
     }
+}
+
+fn codegen_regular_intrinsic_call<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    instance: Instance<'tcx>,
+    args: &[mir::Operand<'tcx>],
+    ret: CPlace<'tcx>,
+    span: Span,
+    destination: Option<(CPlace<'tcx>, BasicBlock)>,
+) {
+    let def_id = instance.def_id();
+    let substs = instance.substs;
+
+    let intrinsic = fx.tcx.item_name(def_id);
+
+    let usize_layout = fx.layout_of(fx.tcx.types.usize);
 
     intrinsic_match! {
         fx, intrinsic, substs, args,

From 70cc24254500f783ddd08e65e6abcf21e27c0c27 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 15:22:46 +0100
Subject: [PATCH 06/84] Remove a couple of duplicate calls

---
 src/intrinsics/mod.rs  | 25 ++++++++++---------------
 src/intrinsics/simd.rs | 11 +++++------
 2 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 517deba580505..cfe3e7bb92019 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -9,7 +9,8 @@ pub(crate) use cpuid::codegen_cpuid_call;
 pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_span::symbol::{kw, sym};
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_span::symbol::{kw, sym, Symbol};
 
 use crate::prelude::*;
 use cranelift_codegen::ir::AtomicRmwOp;
@@ -391,6 +392,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     span: Span,
 ) {
     let intrinsic = fx.tcx.item_name(instance.def_id());
+    let substs = instance.substs;
 
     let ret = match destination {
         Some((place, _)) => place,
@@ -410,28 +412,24 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     };
 
     if intrinsic.as_str().starts_with("simd_") {
-        self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
+        self::simd::codegen_simd_intrinsic_call(fx, intrinsic, substs, args, ret, span);
         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
-    } else if codegen_float_intrinsic_call(fx, instance, args, ret) {
+    } else if codegen_float_intrinsic_call(fx, intrinsic, substs, args, ret) {
         let ret_block = fx.get_block(destination.expect("Float intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
     } else {
-        codegen_regular_intrinsic_call(fx, instance, args, ret, span, destination);
+        codegen_regular_intrinsic_call(fx, instance, intrinsic, substs, args, ret, span, destination);
     }
 }
 
 fn codegen_float_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    instance: Instance<'tcx>,
+    intrinsic: Symbol,
+    substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
 ) -> bool {
-    let def_id = instance.def_id();
-    let substs = instance.substs;
-
-    let intrinsic = fx.tcx.item_name(def_id);
-
     call_intrinsic_match! {
         fx, intrinsic, substs, ret, args,
         expf32(flt) -> f32 => expf,
@@ -479,16 +477,13 @@ fn codegen_float_intrinsic_call<'tcx>(
 fn codegen_regular_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     instance: Instance<'tcx>,
+    intrinsic: Symbol,
+    substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
     span: Span,
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 ) {
-    let def_id = instance.def_id();
-    let substs = instance.substs;
-
-    let intrinsic = fx.tcx.item_name(def_id);
-
     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 
     intrinsic_match! {
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 6c0631d9ecbd0..d8dcf5d0ab981 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -1,20 +1,19 @@
 //! Codegen `extern "platform-intrinsic"` intrinsics.
 
+use rustc_middle::ty::subst::SubstsRef;
+use rustc_span::Symbol;
+
 use super::*;
 use crate::prelude::*;
 
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    instance: Instance<'tcx>,
+    intrinsic: Symbol,
+    substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
     span: Span,
 ) {
-    let def_id = instance.def_id();
-    let substs = instance.substs;
-
-    let intrinsic = fx.tcx.item_name(def_id);
-
     intrinsic_match! {
         fx, intrinsic, substs, args,
         _ => {

From 409e3eb2cbc7a18e1a9e6bc607766ef18cd79dfc Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 15:24:10 +0100
Subject: [PATCH 07/84] Remove unnecessary argument

---
 src/intrinsics/mod.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index cfe3e7bb92019..24e9ed338223b 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -74,13 +74,12 @@ macro intrinsic_match {
 }
 
 macro call_intrinsic_match {
-    ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $args:expr, $(
+    ($fx:expr, $intrinsic:expr, $ret:expr, $args:expr, $(
         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
     )*) => {
         match $intrinsic {
             $(
                 sym::$name => {
-                    assert!($substs.is_noop());
                     if let [$(ref $arg),*] = *$args {
                         let ($($arg,)*) = (
                             $(codegen_operand($fx, $arg),)*
@@ -415,7 +414,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         self::simd::codegen_simd_intrinsic_call(fx, intrinsic, substs, args, ret, span);
         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
-    } else if codegen_float_intrinsic_call(fx, intrinsic, substs, args, ret) {
+    } else if codegen_float_intrinsic_call(fx, intrinsic, args, ret) {
         let ret_block = fx.get_block(destination.expect("Float intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
     } else {
@@ -426,12 +425,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
 fn codegen_float_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: Symbol,
-    substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
 ) -> bool {
     call_intrinsic_match! {
-        fx, intrinsic, substs, ret, args,
+        fx, intrinsic, ret, args,
         expf32(flt) -> f32 => expf,
         expf64(flt) -> f64 => exp,
         exp2f32(flt) -> f32 => exp2f,

From 046e094842b4c1b046aad66750838304c017796b Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 15:31:44 +0100
Subject: [PATCH 08/84] Only use a single bug!() invocation in
 call_intrinsic_match

This reduces code size
---
 src/intrinsics/mod.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 24e9ed338223b..29b30631d0fcf 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -88,13 +88,13 @@ macro call_intrinsic_match {
                         $ret.write_cvalue($fx, res);
 
                         return true;
-                    } else {
-                        bug!("wrong number of args for intrinsic {:?}", $intrinsic);
                     }
                 }
             )*
-            _ => false,
+            _ => return false,
         }
+
+        bug!("wrong number of args for intrinsic {:?}", $intrinsic);
     }
 }
 

From a1a164083ea9cdf8f3d6f053cdfb6b3355787c44 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 16:44:54 +0100
Subject: [PATCH 09/84] Move call_intrinsic_match macro into
 codegen_float_intrinsic_call

---
 src/intrinsics/mod.rs | 50 +++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 29b30631d0fcf..27e3b1b11f1de 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -73,31 +73,6 @@ macro intrinsic_match {
     }
 }
 
-macro call_intrinsic_match {
-    ($fx:expr, $intrinsic:expr, $ret:expr, $args:expr, $(
-        $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
-    )*) => {
-        match $intrinsic {
-            $(
-                sym::$name => {
-                    if let [$(ref $arg),*] = *$args {
-                        let ($($arg,)*) = (
-                            $(codegen_operand($fx, $arg),)*
-                        );
-                        let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
-                        $ret.write_cvalue($fx, res);
-
-                        return true;
-                    }
-                }
-            )*
-            _ => return false,
-        }
-
-        bug!("wrong number of args for intrinsic {:?}", $intrinsic);
-    }
-}
-
 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     match $ty.kind() {
         ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@@ -428,6 +403,31 @@ fn codegen_float_intrinsic_call<'tcx>(
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
 ) -> bool {
+    macro call_intrinsic_match {
+        ($fx:expr, $intrinsic:expr, $ret:expr, $args:expr, $(
+            $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
+        )*) => {
+            match $intrinsic {
+                $(
+                    sym::$name => {
+                        if let [$(ref $arg),*] = *$args {
+                            let ($($arg,)*) = (
+                                $(codegen_operand($fx, $arg),)*
+                            );
+                            let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
+                            $ret.write_cvalue($fx, res);
+
+                            return true;
+                        }
+                    }
+                )*
+                _ => return false,
+            }
+
+            bug!("wrong number of args for intrinsic {:?}", $intrinsic);
+        }
+    }
+
     call_intrinsic_match! {
         fx, intrinsic, ret, args,
         expf32(flt) -> f32 => expf,

From 300974714c96524806b44e36c5d6a7d0e854fc3e Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:11:28 +0100
Subject: [PATCH 10/84] Dedup write_cvalue calls in
 codegen_float_intrinsic_call

Also directly use an array instead of going through a tuple. This
reduces the amount of llvm ir lines for this function by almost half
from 3086 to 1662.
---
 src/intrinsics/mod.rs | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 27e3b1b11f1de..0d667847b9aab 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -393,7 +393,16 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         let ret_block = fx.get_block(destination.expect("Float intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
     } else {
-        codegen_regular_intrinsic_call(fx, instance, intrinsic, substs, args, ret, span, destination);
+        codegen_regular_intrinsic_call(
+            fx,
+            instance,
+            intrinsic,
+            substs,
+            args,
+            ret,
+            span,
+            destination,
+        );
     }
 }
 
@@ -407,24 +416,27 @@ fn codegen_float_intrinsic_call<'tcx>(
         ($fx:expr, $intrinsic:expr, $ret:expr, $args:expr, $(
             $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
         )*) => {
-            match $intrinsic {
+            let res = match $intrinsic {
                 $(
                     sym::$name => {
                         if let [$(ref $arg),*] = *$args {
-                            let ($($arg,)*) = (
-                                $(codegen_operand($fx, $arg),)*
-                            );
-                            let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
-                            $ret.write_cvalue($fx, res);
-
-                            return true;
+                            let args = [$(codegen_operand($fx, $arg),)*];
+                            Some($fx.easy_call(stringify!($func), &args, $fx.tcx.types.$ty))
+                        } else {
+                            None
                         }
                     }
                 )*
                 _ => return false,
+            };
+
+            if let Some(res) = res {
+                $ret.write_cvalue($fx, res);
+            } else {
+                bug!("wrong number of args for intrinsic {:?}", $intrinsic);
             }
 
-            bug!("wrong number of args for intrinsic {:?}", $intrinsic);
+            true
         }
     }
 

From baad993daead9ddc127dd897035f6616f4e367f6 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:19:11 +0100
Subject: [PATCH 11/84] Dedup codegen_operand calls in
 codegen_float_intrinsic_call

This reduces the amount of llvm ir lines for this function by a little
over half from 1662 to 781.
---
 src/intrinsics/mod.rs | 105 +++++++++++++++++++++++-------------------
 1 file changed, 58 insertions(+), 47 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 0d667847b9aab..8da6c7ae9eb8e 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -414,73 +414,84 @@ fn codegen_float_intrinsic_call<'tcx>(
 ) -> bool {
     macro call_intrinsic_match {
         ($fx:expr, $intrinsic:expr, $ret:expr, $args:expr, $(
-            $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
+            $name:ident($arg_count:literal) -> $ty:ident => $func:ident,
         )*) => {
-            let res = match $intrinsic {
+            let (name, arg_count, ty) = match $intrinsic {
                 $(
-                    sym::$name => {
-                        if let [$(ref $arg),*] = *$args {
-                            let args = [$(codegen_operand($fx, $arg),)*];
-                            Some($fx.easy_call(stringify!($func), &args, $fx.tcx.types.$ty))
-                        } else {
-                            None
-                        }
-                    }
+                    sym::$name => (stringify!($func), $arg_count, $fx.tcx.types.$ty),
                 )*
                 _ => return false,
             };
 
-            if let Some(res) = res {
-                $ret.write_cvalue($fx, res);
-            } else {
+            if $args.len() != arg_count {
                 bug!("wrong number of args for intrinsic {:?}", $intrinsic);
             }
 
+            let (a, b, c);
+            let args = match $args {
+                [x] => {
+                    a = [codegen_operand($fx, x)];
+                    &a as &[_]
+                }
+                [x, y] => {
+                    b = [codegen_operand($fx, x), codegen_operand($fx, y)];
+                    &b
+                }
+                [x, y, z] => {
+                    c = [codegen_operand($fx, x), codegen_operand($fx, y), codegen_operand($fx, z)];
+                    &c
+                }
+                _ => unreachable!(),
+            };
+
+            let res = $fx.easy_call(name, &args, ty);
+            $ret.write_cvalue($fx, res);
+
             true
         }
     }
 
     call_intrinsic_match! {
         fx, intrinsic, ret, args,
-        expf32(flt) -> f32 => expf,
-        expf64(flt) -> f64 => exp,
-        exp2f32(flt) -> f32 => exp2f,
-        exp2f64(flt) -> f64 => exp2,
-        sqrtf32(flt) -> f32 => sqrtf,
-        sqrtf64(flt) -> f64 => sqrt,
-        powif32(a, x) -> f32 => __powisf2, // compiler-builtins
-        powif64(a, x) -> f64 => __powidf2, // compiler-builtins
-        powf32(a, x) -> f32 => powf,
-        powf64(a, x) -> f64 => pow,
-        logf32(flt) -> f32 => logf,
-        logf64(flt) -> f64 => log,
-        log2f32(flt) -> f32 => log2f,
-        log2f64(flt) -> f64 => log2,
-        log10f32(flt) -> f32 => log10f,
-        log10f64(flt) -> f64 => log10,
-        fabsf32(flt) -> f32 => fabsf,
-        fabsf64(flt) -> f64 => fabs,
-        fmaf32(x, y, z) -> f32 => fmaf,
-        fmaf64(x, y, z) -> f64 => fma,
-        copysignf32(x, y) -> f32 => copysignf,
-        copysignf64(x, y) -> f64 => copysign,
+        expf32(1) -> f32 => expf,
+        expf64(1) -> f64 => exp,
+        exp2f32(1) -> f32 => exp2f,
+        exp2f64(1) -> f64 => exp2,
+        sqrtf32(1) -> f32 => sqrtf,
+        sqrtf64(1) -> f64 => sqrt,
+        powif32(2) -> f32 => __powisf2, // compiler-builtins
+        powif64(2) -> f64 => __powidf2, // compiler-builtins
+        powf32(2) -> f32 => powf,
+        powf64(2) -> f64 => pow,
+        logf32(1) -> f32 => logf,
+        logf64(1) -> f64 => log,
+        log2f32(1) -> f32 => log2f,
+        log2f64(1) -> f64 => log2,
+        log10f32(1) -> f32 => log10f,
+        log10f64(1) -> f64 => log10,
+        fabsf32(1) -> f32 => fabsf,
+        fabsf64(1) -> f64 => fabs,
+        fmaf32(3) -> f32 => fmaf,
+        fmaf64(3) -> f64 => fma,
+        copysignf32(2) -> f32 => copysignf,
+        copysignf64(2) -> f64 => copysign,
 
         // rounding variants
         // FIXME use clif insts
-        floorf32(flt) -> f32 => floorf,
-        floorf64(flt) -> f64 => floor,
-        ceilf32(flt) -> f32 => ceilf,
-        ceilf64(flt) -> f64 => ceil,
-        truncf32(flt) -> f32 => truncf,
-        truncf64(flt) -> f64 => trunc,
-        roundf32(flt) -> f32 => roundf,
-        roundf64(flt) -> f64 => round,
+        floorf32(1) -> f32 => floorf,
+        floorf64(1) -> f64 => floor,
+        ceilf32(1) -> f32 => ceilf,
+        ceilf64(1) -> f64 => ceil,
+        truncf32(1) -> f32 => truncf,
+        truncf64(1) -> f64 => trunc,
+        roundf32(1) -> f32 => roundf,
+        roundf64(1) -> f64 => round,
 
         // trigonometry
-        sinf32(flt) -> f32 => sinf,
-        sinf64(flt) -> f64 => sin,
-        cosf32(flt) -> f32 => cosf,
-        cosf64(flt) -> f64 => cos,
+        sinf32(1) -> f32 => sinf,
+        sinf64(1) -> f64 => sin,
+        cosf32(1) -> f32 => cosf,
+        cosf64(1) -> f64 => cos,
     }
 }
 

From 9e6d8c1b244213c0a7677504ffeced9cc9c97e27 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:22:23 +0100
Subject: [PATCH 12/84] Remove the call_intrinsic_match macro

---
 src/intrinsics/mod.rs | 136 ++++++++++++++++++------------------------
 1 file changed, 59 insertions(+), 77 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 8da6c7ae9eb8e..bd6ef41ef66c0 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -412,87 +412,69 @@ fn codegen_float_intrinsic_call<'tcx>(
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
 ) -> bool {
-    macro call_intrinsic_match {
-        ($fx:expr, $intrinsic:expr, $ret:expr, $args:expr, $(
-            $name:ident($arg_count:literal) -> $ty:ident => $func:ident,
-        )*) => {
-            let (name, arg_count, ty) = match $intrinsic {
-                $(
-                    sym::$name => (stringify!($func), $arg_count, $fx.tcx.types.$ty),
-                )*
-                _ => return false,
-            };
-
-            if $args.len() != arg_count {
-                bug!("wrong number of args for intrinsic {:?}", $intrinsic);
-            }
-
-            let (a, b, c);
-            let args = match $args {
-                [x] => {
-                    a = [codegen_operand($fx, x)];
-                    &a as &[_]
-                }
-                [x, y] => {
-                    b = [codegen_operand($fx, x), codegen_operand($fx, y)];
-                    &b
-                }
-                [x, y, z] => {
-                    c = [codegen_operand($fx, x), codegen_operand($fx, y), codegen_operand($fx, z)];
-                    &c
-                }
-                _ => unreachable!(),
-            };
+    let (name, arg_count, ty) = match intrinsic {
+        sym::expf32 => ("expf", 1, fx.tcx.types.f32),
+        sym::expf64 => ("exp", 1, fx.tcx.types.f64),
+        sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
+        sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
+        sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
+        sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
+        sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
+        sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
+        sym::powf32 => ("powf", 2, fx.tcx.types.f32),
+        sym::powf64 => ("pow", 2, fx.tcx.types.f64),
+        sym::logf32 => ("logf", 1, fx.tcx.types.f32),
+        sym::logf64 => ("log", 1, fx.tcx.types.f64),
+        sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
+        sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
+        sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
+        sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
+        sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
+        sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
+        sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
+        sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
+        sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
+        sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
+        sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
+        sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
+        sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
+        sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
+        sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
+        sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
+        sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
+        sym::roundf64 => ("round", 1, fx.tcx.types.f64),
+        sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
+        sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
+        sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
+        sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
+        _ => return false,
+    };
 
-            let res = $fx.easy_call(name, &args, ty);
-            $ret.write_cvalue($fx, res);
+    if args.len() != arg_count {
+        bug!("wrong number of args for intrinsic {:?}", intrinsic);
+    }
 
-            true
+    let (a, b, c);
+    let args = match args {
+        [x] => {
+            a = [codegen_operand(fx, x)];
+            &a as &[_]
         }
-    }
+        [x, y] => {
+            b = [codegen_operand(fx, x), codegen_operand(fx, y)];
+            &b
+        }
+        [x, y, z] => {
+            c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
+            &c
+        }
+        _ => unreachable!(),
+    };
 
-    call_intrinsic_match! {
-        fx, intrinsic, ret, args,
-        expf32(1) -> f32 => expf,
-        expf64(1) -> f64 => exp,
-        exp2f32(1) -> f32 => exp2f,
-        exp2f64(1) -> f64 => exp2,
-        sqrtf32(1) -> f32 => sqrtf,
-        sqrtf64(1) -> f64 => sqrt,
-        powif32(2) -> f32 => __powisf2, // compiler-builtins
-        powif64(2) -> f64 => __powidf2, // compiler-builtins
-        powf32(2) -> f32 => powf,
-        powf64(2) -> f64 => pow,
-        logf32(1) -> f32 => logf,
-        logf64(1) -> f64 => log,
-        log2f32(1) -> f32 => log2f,
-        log2f64(1) -> f64 => log2,
-        log10f32(1) -> f32 => log10f,
-        log10f64(1) -> f64 => log10,
-        fabsf32(1) -> f32 => fabsf,
-        fabsf64(1) -> f64 => fabs,
-        fmaf32(3) -> f32 => fmaf,
-        fmaf64(3) -> f64 => fma,
-        copysignf32(2) -> f32 => copysignf,
-        copysignf64(2) -> f64 => copysign,
-
-        // rounding variants
-        // FIXME use clif insts
-        floorf32(1) -> f32 => floorf,
-        floorf64(1) -> f64 => floor,
-        ceilf32(1) -> f32 => ceilf,
-        ceilf64(1) -> f64 => ceil,
-        truncf32(1) -> f32 => truncf,
-        truncf64(1) -> f64 => trunc,
-        roundf32(1) -> f32 => roundf,
-        roundf64(1) -> f64 => round,
-
-        // trigonometry
-        sinf32(1) -> f32 => sinf,
-        sinf64(1) -> f64 => sin,
-        cosf32(1) -> f32 => cosf,
-        cosf64(1) -> f64 => cos,
-    }
+    let res = fx.easy_call(name, &args, ty);
+    ret.write_cvalue(fx, res);
+
+    true
 }
 
 fn codegen_regular_intrinsic_call<'tcx>(

From 9295b086f6e2a42a7739bd522051060b0c12f885 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:29:16 +0100
Subject: [PATCH 13/84] Turn validate_simd_type into a function

This effectively outlines it, significantly reducing the size of
the codegen_simd_intrinsic_call llvm ir from 10419 lines to 6378 lines.
---
 src/intrinsics/mod.rs  |  8 ++---
 src/intrinsics/simd.rs | 80 +++++++++++++++++++++---------------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index bd6ef41ef66c0..f305942a8c3b3 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -91,11 +91,11 @@ macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     }
 }
 
-macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
-    if !$ty.is_simd() {
-        $fx.tcx.sess.span_err($span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", $intrinsic, $ty));
+fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span: Span, ty: Ty<'_>) {
+    if !ty.is_simd() {
+        fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
         // Prevent verifier error
-        crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
+        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
         return;
     }
 }
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index d8dcf5d0ab981..9e42ff587bd74 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -21,7 +21,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_cast, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
                 let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
 
@@ -34,27 +34,27 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_eq, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(fx, Equal|Equal(x, y) -> ret);
         };
         simd_ne, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret);
         };
         simd_lt, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
         };
         simd_le, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
         };
         simd_gt, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
         };
         simd_ge, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(
                 fx,
                 UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
@@ -64,7 +64,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
         _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
 
             // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
             // If there is no suffix, use the index array length.
@@ -166,7 +166,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_extract, (c v, o idx) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                 idx_const
             } else {
@@ -194,7 +194,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_neg, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
                 let ret_lane = match lane_layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().ineg(lane),
@@ -206,7 +206,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fabs, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
                 let ret_lane = fx.bcx.ins().fabs(lane);
                 CValue::by_val(ret_lane, ret_lane_layout)
@@ -214,7 +214,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fsqrt, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
                 let ret_lane = fx.bcx.ins().sqrt(lane);
                 CValue::by_val(ret_lane, ret_lane_layout)
@@ -222,23 +222,23 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_add, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
         };
         simd_sub, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
         };
         simd_mul, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
         };
         simd_div, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
         };
         simd_rem, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
                 let res_lane = match lane_layout.ty.kind() {
                     ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
@@ -261,28 +261,28 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
         simd_shl, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_binop!(fx, ishl(x, y) -> ret);
         };
         simd_shr, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
         };
         simd_and, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_binop!(fx, band(x, y) -> ret);
         };
         simd_or, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_binop!(fx, bor(x, y) -> ret);
         };
         simd_xor, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_int_binop!(fx, bxor(x, y) -> ret);
         };
 
         simd_fma, (c a, c b, c c) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             assert_eq!(a.layout(), b.layout());
             assert_eq!(a.layout(), c.layout());
             let layout = a.layout();
@@ -305,16 +305,16 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fmin, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_flt_binop!(fx, fmin(x, y) -> ret);
         };
         simd_fmax, (c x, c y) {
-            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
+            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_flt_binop!(fx, fmax(x, y) -> ret);
         };
 
         simd_round, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
                 let res_lane = match lane_layout.ty.kind() {
                     ty::Float(FloatTy::F32) => fx.lib_call(
@@ -335,21 +335,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
         simd_ceil, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
                 let ret_lane = fx.bcx.ins().ceil(lane);
                 CValue::by_val(ret_lane, ret_lane_layout)
             });
         };
         simd_floor, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
                 let ret_lane = fx.bcx.ins().floor(lane);
                 CValue::by_val(ret_lane, ret_lane_layout)
             });
         };
         simd_trunc, (c a) {
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
                 let ret_lane = fx.bcx.ins().trunc(lane);
                 CValue::by_val(ret_lane, ret_lane_layout)
@@ -357,7 +357,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| {
                 if lane_layout.ty.is_floating_point() {
                     fx.bcx.ins().fadd(a, b)
@@ -368,7 +368,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| {
                 if lane_layout.ty.is_floating_point() {
                     fx.bcx.ins().fmul(a, b)
@@ -379,32 +379,32 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_all, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_any, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_and, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_or, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_xor, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bxor(a, b));
         };
 
         simd_reduce_min, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
                 let lt = match layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
@@ -417,7 +417,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_max, (c v) {
-            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
+            validate_simd_type(fx, intrinsic, span, v.layout().ty);
             simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
                 let gt = match layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
@@ -430,8 +430,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_select, (c m, c a, c b) {
-            validate_simd_type!(fx, intrinsic, span, m.layout().ty);
-            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
+            validate_simd_type(fx, intrinsic, span, m.layout().ty);
+            validate_simd_type(fx, intrinsic, span, a.layout().ty);
             assert_eq!(a.layout(), b.layout());
 
             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);

From 4e3a8d5fb90dca1dd8462ea45b73d23ba0603f76 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:30:01 +0100
Subject: [PATCH 14/84] Move validate_simd_type from intrinsics to
 intrinsics::simd

---
 src/intrinsics/mod.rs  | 9 ---------
 src/intrinsics/simd.rs | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index f305942a8c3b3..36490c370009d 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -91,15 +91,6 @@ macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     }
 }
 
-fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span: Span, ty: Ty<'_>) {
-    if !ty.is_simd() {
-        fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
-        // Prevent verifier error
-        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
-        return;
-    }
-}
-
 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
     let (element, count) = match layout.abi {
         Abi::Vector { element, count } => (element, count),
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 9e42ff587bd74..8bc4dd8e615b1 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -6,6 +6,15 @@ use rustc_span::Symbol;
 use super::*;
 use crate::prelude::*;
 
+fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span: Span, ty: Ty<'_>) {
+    if !ty.is_simd() {
+        fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
+        // Prevent verifier error
+        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+        return;
+    }
+}
+
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: Symbol,

From 8ace43e65012a5e4a3e07d399a2a5832e18cf917 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:34:55 +0100
Subject: [PATCH 15/84] Move a couple of macros to intrinsics::simd

---
 src/intrinsics/mod.rs  | 115 -----------------------------------------
 src/intrinsics/simd.rs | 115 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+), 115 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 36490c370009d..d6b35bba9e7ff 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -234,121 +234,6 @@ fn bool_to_zero_or_max_uint<'tcx>(
     CValue::by_val(res, layout)
 }
 
-macro simd_cmp {
-    ($fx:expr, $cc:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        let vector_ty = clif_vector_type($fx.tcx, $x.layout());
-
-        if let Some(vector_ty) = vector_ty {
-            let x = $x.load_scalar($fx);
-            let y = $y.load_scalar($fx);
-            let val = if vector_ty.lane_type().is_float() {
-                $fx.bcx.ins().fcmp(FloatCC::$cc_f, x, y)
-            } else {
-                $fx.bcx.ins().icmp(IntCC::$cc, x, y)
-            };
-
-            // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
-            let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
-
-            $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
-        } else {
-            simd_pair_for_each_lane(
-                $fx,
-                $x,
-                $y,
-                $ret,
-                |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-                    let res_lane = match lane_layout.ty.kind() {
-                        ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
-                        ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-                        _ => unreachable!("{:?}", lane_layout.ty),
-                    };
-                    bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
-                },
-            );
-        }
-    },
-    ($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        // FIXME use vector icmp when possible
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
-                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
-            },
-        );
-    },
-}
-
-macro simd_int_binop {
-    ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
-    },
-    ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                CValue::by_val(res_lane, ret_lane_layout)
-            },
-        );
-    },
-}
-
-macro simd_int_flt_binop {
-    ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
-    },
-    ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-                    ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                CValue::by_val(res_lane, ret_lane_layout)
-            },
-        );
-    },
-}
-
-macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
-    simd_pair_for_each_lane(
-        $fx,
-        $x,
-        $y,
-        $ret,
-        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-            let res_lane = match lane_layout.ty.kind() {
-                ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
-                _ => unreachable!("{:?}", lane_layout.ty),
-            };
-            CValue::by_val(res_lane, ret_lane_layout)
-        },
-    );
-}
-
 pub(crate) fn codegen_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     instance: Instance<'tcx>,
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 8bc4dd8e615b1..181b45a874097 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -15,6 +15,121 @@ fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span:
     }
 }
 
+macro simd_cmp {
+    ($fx:expr, $cc:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
+        let vector_ty = clif_vector_type($fx.tcx, $x.layout());
+
+        if let Some(vector_ty) = vector_ty {
+            let x = $x.load_scalar($fx);
+            let y = $y.load_scalar($fx);
+            let val = if vector_ty.lane_type().is_float() {
+                $fx.bcx.ins().fcmp(FloatCC::$cc_f, x, y)
+            } else {
+                $fx.bcx.ins().icmp(IntCC::$cc, x, y)
+            };
+
+            // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
+            let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
+
+            $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
+        } else {
+            simd_pair_for_each_lane(
+                $fx,
+                $x,
+                $y,
+                $ret,
+                |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
+                    let res_lane = match lane_layout.ty.kind() {
+                        ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
+                        ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
+                        _ => unreachable!("{:?}", lane_layout.ty),
+                    };
+                    bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
+                },
+            );
+        }
+    },
+    ($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
+        // FIXME use vector icmp when possible
+        simd_pair_for_each_lane(
+            $fx,
+            $x,
+            $y,
+            $ret,
+            |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
+                let res_lane = match lane_layout.ty.kind() {
+                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
+                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
+                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
+                    _ => unreachable!("{:?}", lane_layout.ty),
+                };
+                bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
+            },
+        );
+    },
+}
+
+macro simd_int_binop {
+    ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
+        simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
+    },
+    ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
+        simd_pair_for_each_lane(
+            $fx,
+            $x,
+            $y,
+            $ret,
+            |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
+                let res_lane = match lane_layout.ty.kind() {
+                    ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
+                    ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
+                    _ => unreachable!("{:?}", lane_layout.ty),
+                };
+                CValue::by_val(res_lane, ret_lane_layout)
+            },
+        );
+    },
+}
+
+macro simd_int_flt_binop {
+    ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
+        simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
+    },
+    ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
+        simd_pair_for_each_lane(
+            $fx,
+            $x,
+            $y,
+            $ret,
+            |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
+                let res_lane = match lane_layout.ty.kind() {
+                    ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
+                    ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
+                    ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
+                    _ => unreachable!("{:?}", lane_layout.ty),
+                };
+                CValue::by_val(res_lane, ret_lane_layout)
+            },
+        );
+    },
+}
+
+macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
+    simd_pair_for_each_lane(
+        $fx,
+        $x,
+        $y,
+        $ret,
+        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
+            let res_lane = match lane_layout.ty.kind() {
+                ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
+                _ => unreachable!("{:?}", lane_layout.ty),
+            };
+            CValue::by_val(res_lane, ret_lane_layout)
+        },
+    );
+}
+
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: Symbol,

From 78e2d4a275caec70a0b64bdc97084bacc3610076 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:39:00 +0100
Subject: [PATCH 16/84] Remove support for vector icmp for now

Real simd support will need an overhaul in the future anyway. For now it
only complicates the code.
---
 src/intrinsics/simd.rs | 46 ++++++++++++++----------------------------
 1 file changed, 15 insertions(+), 31 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 181b45a874097..443e2954e5117 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -17,37 +17,21 @@ fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span:
 
 macro simd_cmp {
     ($fx:expr, $cc:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        let vector_ty = clif_vector_type($fx.tcx, $x.layout());
-
-        if let Some(vector_ty) = vector_ty {
-            let x = $x.load_scalar($fx);
-            let y = $y.load_scalar($fx);
-            let val = if vector_ty.lane_type().is_float() {
-                $fx.bcx.ins().fcmp(FloatCC::$cc_f, x, y)
-            } else {
-                $fx.bcx.ins().icmp(IntCC::$cc, x, y)
-            };
-
-            // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
-            let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
-
-            $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
-        } else {
-            simd_pair_for_each_lane(
-                $fx,
-                $x,
-                $y,
-                $ret,
-                |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-                    let res_lane = match lane_layout.ty.kind() {
-                        ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
-                        ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-                        _ => unreachable!("{:?}", lane_layout.ty),
-                    };
-                    bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
-                },
-            );
-        }
+        // FIXME use vector icmp when possible
+        simd_pair_for_each_lane(
+            $fx,
+            $x,
+            $y,
+            $ret,
+            |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
+                let res_lane = match lane_layout.ty.kind() {
+                    ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
+                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
+                    _ => unreachable!("{:?}", lane_layout.ty),
+                };
+                bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
+            },
+        );
     },
     ($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
         // FIXME use vector icmp when possible

From d4d2b24d5530c50aa80985938fe13e51e6db8750 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 17:44:55 +0100
Subject: [PATCH 17/84] Slightly simplify some macros by removing an extra case
 for when signedness doesn't matter

This is slightly more verbose when invoking the macro.
---
 src/intrinsics/simd.rs | 146 +++++++++++++++++------------------------
 1 file changed, 60 insertions(+), 86 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 443e2954e5117..bea99346b0a8d 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -15,90 +15,64 @@ fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span:
     }
 }
 
-macro simd_cmp {
-    ($fx:expr, $cc:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        // FIXME use vector icmp when possible
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
-                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
-            },
-        );
-    },
-    ($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        // FIXME use vector icmp when possible
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
-                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
-            },
-        );
-    },
+macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
+    // FIXME use vector instructions when possible
+    simd_pair_for_each_lane(
+        $fx,
+        $x,
+        $y,
+        $ret,
+        |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
+            let res_lane = match lane_layout.ty.kind() {
+                ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
+                ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
+                ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
+                _ => unreachable!("{:?}", lane_layout.ty),
+            };
+            bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
+        },
+    );
 }
 
-macro simd_int_binop {
-    ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
-    },
-    ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                CValue::by_val(res_lane, ret_lane_layout)
-            },
-        );
-    },
+macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
+    // FIXME use vector instructions when possible
+    simd_pair_for_each_lane(
+        $fx,
+        $x,
+        $y,
+        $ret,
+        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
+            let res_lane = match lane_layout.ty.kind() {
+                ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
+                ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
+                _ => unreachable!("{:?}", lane_layout.ty),
+            };
+            CValue::by_val(res_lane, ret_lane_layout)
+        },
+    );
 }
 
-macro simd_int_flt_binop {
-    ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
-    },
-    ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
-        simd_pair_for_each_lane(
-            $fx,
-            $x,
-            $y,
-            $ret,
-            |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-                    ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                CValue::by_val(res_lane, ret_lane_layout)
-            },
-        );
-    },
+macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
+    // FIXME use vector instructions when possible
+    simd_pair_for_each_lane(
+        $fx,
+        $x,
+        $y,
+        $ret,
+        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
+            let res_lane = match lane_layout.ty.kind() {
+                ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
+                ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
+                ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
+                _ => unreachable!("{:?}", lane_layout.ty),
+            };
+            CValue::by_val(res_lane, ret_lane_layout)
+        },
+    );
 }
 
 macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
+    // FIXME use vector instructions when possible
     simd_pair_for_each_lane(
         $fx,
         $x,
@@ -143,11 +117,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_eq, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, Equal|Equal(x, y) -> ret);
+            simd_cmp!(fx, Equal|Equal|Equal(x, y) -> ret);
         };
         simd_ne, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret);
+            simd_cmp!(fx, NotEqual|NotEqual|NotEqual(x, y) -> ret);
         };
         simd_lt, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
@@ -331,15 +305,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_add, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
+            simd_int_flt_binop!(fx, iadd|iadd|fadd(x, y) -> ret);
         };
         simd_sub, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
+            simd_int_flt_binop!(fx, isub|isub|fsub(x, y) -> ret);
         };
         simd_mul, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
+            simd_int_flt_binop!(fx, imul|imul|fmul(x, y) -> ret);
         };
         simd_div, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
@@ -370,7 +344,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
         simd_shl, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, ishl(x, y) -> ret);
+            simd_int_binop!(fx, ishl|ishl(x, y) -> ret);
         };
         simd_shr, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
@@ -378,15 +352,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
         simd_and, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, band(x, y) -> ret);
+            simd_int_binop!(fx, band|band(x, y) -> ret);
         };
         simd_or, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, bor(x, y) -> ret);
+            simd_int_binop!(fx, bor|bor(x, y) -> ret);
         };
         simd_xor, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, bxor(x, y) -> ret);
+            simd_int_binop!(fx, bxor|bxor(x, y) -> ret);
         };
 
         simd_fma, (c a, c b, c c) {

From 57d25ef60e7237e18092aea4081a11a4d5a28c1c Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 18:32:27 +0100
Subject: [PATCH 18/84] Use simplified version of bool_to_zero_or_max_uint in
 simd_cmp

---
 src/intrinsics/simd.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index bea99346b0a8d..f38a30011d395 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -29,7 +29,13 @@ macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident)
                 ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
                 _ => unreachable!("{:?}", lane_layout.ty),
             };
-            bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
+
+            let ty = fx.clif_type(res_lane_layout.ty).unwrap();
+
+            let res_lane = fx.bcx.ins().bint(ty, res_lane);
+            let res_lane = fx.bcx.ins().ineg(res_lane);
+
+            CValue::by_val(res_lane, res_lane_layout)
         },
     );
 }

From b60eced4057791f9a5c94ac5c9a1c26d015dd2b7 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 18:43:08 +0100
Subject: [PATCH 19/84] Return Value instead of CValue from the
 simd_for_each_lane closure

---
 src/intrinsics/llvm.rs | 14 ++++----
 src/intrinsics/mod.rs  | 10 +++---
 src/intrinsics/simd.rs | 74 +++++++++++++++++-------------------------
 3 files changed, 42 insertions(+), 56 deletions(-)

diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs
index be3704ca2768e..13c7cf677edd7 100644
--- a/src/intrinsics/llvm.rs
+++ b/src/intrinsics/llvm.rs
@@ -83,22 +83,20 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
         "llvm.x86.sse2.psrli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _res_lane_layout, lane| {
+                match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
-                };
-                CValue::by_val(res_lane, res_lane_layout)
+                }
             });
         };
         "llvm.x86.sse2.pslli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _res_lane_layout, lane| {
+                match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
-                };
-                CValue::by_val(res_lane, res_lane_layout)
+                }
             });
         };
         "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) {
diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index d6b35bba9e7ff..dee192a69af32 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -113,7 +113,7 @@ fn simd_for_each_lane<'tcx>(
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
         Value,
-    ) -> CValue<'tcx>,
+    ) -> Value,
 ) {
     let layout = val.layout();
 
@@ -127,6 +127,7 @@ fn simd_for_each_lane<'tcx>(
         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 
         let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
+        let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 
         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
     }
@@ -143,7 +144,7 @@ fn simd_pair_for_each_lane<'tcx>(
         TyAndLayout<'tcx>,
         Value,
         Value,
-    ) -> CValue<'tcx>,
+    ) -> Value,
 ) {
     assert_eq!(x.layout(), y.layout());
     let layout = x.layout();
@@ -159,6 +160,7 @@ fn simd_pair_for_each_lane<'tcx>(
         let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
 
         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
+        let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 
         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
     }
@@ -215,7 +217,7 @@ fn bool_to_zero_or_max_uint<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     val: Value,
-) -> CValue<'tcx> {
+) -> Value {
     let ty = fx.clif_type(layout.ty).unwrap();
 
     let int_ty = match ty {
@@ -231,7 +233,7 @@ fn bool_to_zero_or_max_uint<'tcx>(
         res = fx.bcx.ins().bitcast(ty, res);
     }
 
-    CValue::by_val(res, layout)
+    res
 }
 
 pub(crate) fn codegen_intrinsic_call<'tcx>(
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index f38a30011d395..0ab48aaea1ad1 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -33,9 +33,7 @@ macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident)
             let ty = fx.clif_type(res_lane_layout.ty).unwrap();
 
             let res_lane = fx.bcx.ins().bint(ty, res_lane);
-            let res_lane = fx.bcx.ins().ineg(res_lane);
-
-            CValue::by_val(res_lane, res_lane_layout)
+            fx.bcx.ins().ineg(res_lane)
         },
     );
 }
@@ -47,13 +45,12 @@ macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $r
         $x,
         $y,
         $ret,
-        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-            let res_lane = match lane_layout.ty.kind() {
+        |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+            match lane_layout.ty.kind() {
                 ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
                 ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
                 _ => unreachable!("{:?}", lane_layout.ty),
-            };
-            CValue::by_val(res_lane, ret_lane_layout)
+            }
         },
     );
 }
@@ -65,14 +62,13 @@ macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident,
         $x,
         $y,
         $ret,
-        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-            let res_lane = match lane_layout.ty.kind() {
+        |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+            match lane_layout.ty.kind() {
                 ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
                 ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
                 ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
                 _ => unreachable!("{:?}", lane_layout.ty),
-            };
-            CValue::by_val(res_lane, ret_lane_layout)
+            }
         },
     );
 }
@@ -84,12 +80,11 @@ macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
         $x,
         $y,
         $ret,
-        |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-            let res_lane = match lane_layout.ty.kind() {
+        |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+            match lane_layout.ty.kind() {
                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
                 _ => unreachable!("{:?}", lane_layout.ty),
-            };
-            CValue::by_val(res_lane, ret_lane_layout)
+            }
         },
     );
 }
@@ -116,8 +111,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let from_signed = type_sign(lane_layout.ty);
                 let to_signed = type_sign(ret_lane_layout.ty);
 
-                let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
-                CValue::by_val(ret_lane, ret_lane_layout)
+                clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed)
             });
         };
 
@@ -283,29 +277,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_neg, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
-                let ret_lane = match lane_layout.ty.kind() {
+            simd_for_each_lane(fx, a, ret, |fx, lane_layout, _ret_lane_layout, lane| {
+                match lane_layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().ineg(lane),
                     ty::Float(_) => fx.bcx.ins().fneg(lane),
                     _ => unreachable!(),
-                };
-                CValue::by_val(ret_lane, ret_lane_layout)
+                }
             });
         };
 
         simd_fabs, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
-                let ret_lane = fx.bcx.ins().fabs(lane);
-                CValue::by_val(ret_lane, ret_lane_layout)
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+                fx.bcx.ins().fabs(lane)
             });
         };
 
         simd_fsqrt, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
-                let ret_lane = fx.bcx.ins().sqrt(lane);
-                CValue::by_val(ret_lane, ret_lane_layout)
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+                fx.bcx.ins().sqrt(lane)
             });
         };
 
@@ -327,8 +318,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
         simd_rem, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
+            simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+                match lane_layout.ty.kind() {
                     ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
                     ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane),
                     ty::Float(FloatTy::F32) => fx.lib_call(
@@ -344,8 +335,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                         &[x_lane, y_lane],
                     )[0],
                     _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                CValue::by_val(res_lane, ret_lane_layout)
+                }
             });
         };
         simd_shl, (c x, c y) {
@@ -403,8 +393,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_round, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
-                let res_lane = match lane_layout.ty.kind() {
+            simd_for_each_lane(fx, a, ret, |fx, lane_layout, _ret_lane_layout, lane| {
+                match lane_layout.ty.kind() {
                     ty::Float(FloatTy::F32) => fx.lib_call(
                         "roundf",
                         vec![AbiParam::new(types::F32)],
@@ -418,29 +408,25 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                         &[lane],
                     )[0],
                     _ => unreachable!("{:?}", lane_layout.ty),
-                };
-                CValue::by_val(res_lane, ret_lane_layout)
+                }
             });
         };
         simd_ceil, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
-                let ret_lane = fx.bcx.ins().ceil(lane);
-                CValue::by_val(ret_lane, ret_lane_layout)
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+                fx.bcx.ins().ceil(lane)
             });
         };
         simd_floor, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
-                let ret_lane = fx.bcx.ins().floor(lane);
-                CValue::by_val(ret_lane, ret_lane_layout)
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+                fx.bcx.ins().floor(lane)
             });
         };
         simd_trunc, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
-                let ret_lane = fx.bcx.ins().trunc(lane);
-                CValue::by_val(ret_lane, ret_lane_layout)
+            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+                fx.bcx.ins().trunc(lane)
             });
         };
 

From 2633024850e9b7fa8aa9a856953312bccc3740bc Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 18:55:57 +0100
Subject: [PATCH 20/84] Don't monomorphize the simd helpers for each closure

This halves the total amount of llvm ir lines for simd related functions
from 18227 to 9604.
---
 src/intrinsics/llvm.rs |  6 +++---
 src/intrinsics/mod.rs  |  8 ++++----
 src/intrinsics/simd.rs | 44 +++++++++++++++++++++---------------------
 3 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs
index 13c7cf677edd7..8bcfbc945fbf5 100644
--- a/src/intrinsics/llvm.rs
+++ b/src/intrinsics/llvm.rs
@@ -73,7 +73,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
                 kind => unreachable!("kind {:?}", kind),
             };
 
-            simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_layout, res_lane_layout, x_lane, y_lane| {
                 let res_lane = match lane_layout.ty.kind() {
                     ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
                     _ => unreachable!("{:?}", lane_layout.ty),
@@ -83,7 +83,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
         "llvm.x86.sse2.psrli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _res_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _res_lane_layout, lane| {
                 match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
@@ -92,7 +92,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
         "llvm.x86.sse2.pslli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _res_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _res_lane_layout, lane| {
                 match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index dee192a69af32..473afd1682794 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -108,7 +108,7 @@ fn simd_for_each_lane<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(
+    f: &dyn Fn(
         &mut FunctionCx<'_, '_, 'tcx>,
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
@@ -138,7 +138,7 @@ fn simd_pair_for_each_lane<'tcx>(
     x: CValue<'tcx>,
     y: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(
+    f: &dyn Fn(
         &mut FunctionCx<'_, '_, 'tcx>,
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
@@ -171,7 +171,7 @@ fn simd_reduce<'tcx>(
     val: CValue<'tcx>,
     acc: Option<Value>,
     ret: CPlace<'tcx>,
-    f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
+    f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     let lane_layout = fx.layout_of(lane_ty);
@@ -192,7 +192,7 @@ fn simd_reduce_bool<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
+    f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     assert!(ret.layout().ty.is_bool());
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 0ab48aaea1ad1..dc04c7643b2a4 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -22,7 +22,7 @@ macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident)
         $x,
         $y,
         $ret,
-        |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
+        &|fx, lane_layout, res_lane_layout, x_lane, y_lane| {
             let res_lane = match lane_layout.ty.kind() {
                 ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
                 ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
@@ -45,7 +45,7 @@ macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $r
         $x,
         $y,
         $ret,
-        |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+        &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
             match lane_layout.ty.kind() {
                 ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
                 ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
@@ -62,7 +62,7 @@ macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident,
         $x,
         $y,
         $ret,
-        |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+        &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
             match lane_layout.ty.kind() {
                 ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
                 ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
@@ -80,7 +80,7 @@ macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
         $x,
         $y,
         $ret,
-        |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+        &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
             match lane_layout.ty.kind() {
                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
                 _ => unreachable!("{:?}", lane_layout.ty),
@@ -105,7 +105,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_cast, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, lane_layout, ret_lane_layout, lane| {
                 let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
 
                 let from_signed = type_sign(lane_layout.ty);
@@ -277,7 +277,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_neg, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, lane_layout, _ret_lane_layout, lane| {
                 match lane_layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().ineg(lane),
                     ty::Float(_) => fx.bcx.ins().fneg(lane),
@@ -288,14 +288,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_fabs, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
                 fx.bcx.ins().fabs(lane)
             });
         };
 
         simd_fsqrt, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
                 fx.bcx.ins().sqrt(lane)
             });
         };
@@ -318,7 +318,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
         simd_rem, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
                 match lane_layout.ty.kind() {
                     ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
                     ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane),
@@ -393,7 +393,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_round, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, lane_layout, _ret_lane_layout, lane| {
                 match lane_layout.ty.kind() {
                     ty::Float(FloatTy::F32) => fx.lib_call(
                         "roundf",
@@ -413,26 +413,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
         simd_ceil, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
                 fx.bcx.ins().ceil(lane)
             });
         };
         simd_floor, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
                 fx.bcx.ins().floor(lane)
             });
         };
         simd_trunc, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, |fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
                 fx.bcx.ins().trunc(lane)
             });
         };
 
         simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| {
+            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_layout, a, b| {
                 if lane_layout.ty.is_floating_point() {
                     fx.bcx.ins().fadd(a, b)
                 } else {
@@ -443,7 +443,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| {
+            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_layout, a, b| {
                 if lane_layout.ty.is_floating_point() {
                     fx.bcx.ins().fmul(a, b)
                 } else {
@@ -454,32 +454,32 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_reduce_all, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b));
+            simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_any, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b));
+            simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_and, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().band(a, b));
+            simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_or, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bor(a, b));
+            simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_xor, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bxor(a, b));
+            simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().bxor(a, b));
         };
 
         simd_reduce_min, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
+            simd_reduce(fx, v, None, ret, &|fx, layout, a, b| {
                 let lt = match layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
@@ -492,7 +492,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_reduce_max, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
+            simd_reduce(fx, v, None, ret, &|fx, layout, a, b| {
                 let gt = match layout.ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),

From b7cda373d585d024b120401b2b796181567e5ae9 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 19:07:15 +0100
Subject: [PATCH 21/84] Pass Ty instead of TyAndLayout to the closure of
 various simd helpers

This reduces the total amount of llvm ir lines for simd related
functions from 9604 to 9467.
---
 src/intrinsics/llvm.rs |  12 ++--
 src/intrinsics/mod.rs  |  27 +++-----
 src/intrinsics/simd.rs | 146 +++++++++++++++++------------------------
 3 files changed, 75 insertions(+), 110 deletions(-)

diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs
index 8bcfbc945fbf5..20f8699d12abd 100644
--- a/src/intrinsics/llvm.rs
+++ b/src/intrinsics/llvm.rs
@@ -73,17 +73,17 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
                 kind => unreachable!("kind {:?}", kind),
             };
 
-            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-                let res_lane = match lane_layout.ty.kind() {
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+                let res_lane = match lane_ty.kind() {
                     ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
-                    _ => unreachable!("{:?}", lane_layout.ty),
+                    _ => unreachable!("{:?}", lane_ty),
                 };
-                bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
+                bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
             });
         };
         "llvm.x86.sse2.psrli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _res_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
                 match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
@@ -92,7 +92,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
         "llvm.x86.sse2.pslli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _res_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
                 match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 473afd1682794..1e384668fc72e 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -108,12 +108,7 @@ fn simd_for_each_lane<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: &dyn Fn(
-        &mut FunctionCx<'_, '_, 'tcx>,
-        TyAndLayout<'tcx>,
-        TyAndLayout<'tcx>,
-        Value,
-    ) -> Value,
+    f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
 ) {
     let layout = val.layout();
 
@@ -126,7 +121,7 @@ fn simd_for_each_lane<'tcx>(
     for lane_idx in 0..lane_count {
         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 
-        let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
+        let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 
         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
@@ -138,13 +133,7 @@ fn simd_pair_for_each_lane<'tcx>(
     x: CValue<'tcx>,
     y: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: &dyn Fn(
-        &mut FunctionCx<'_, '_, 'tcx>,
-        TyAndLayout<'tcx>,
-        TyAndLayout<'tcx>,
-        Value,
-        Value,
-    ) -> Value,
+    f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
 ) {
     assert_eq!(x.layout(), y.layout());
     let layout = x.layout();
@@ -159,7 +148,7 @@ fn simd_pair_for_each_lane<'tcx>(
         let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
         let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
 
-        let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
+        let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 
         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
@@ -171,7 +160,7 @@ fn simd_reduce<'tcx>(
     val: CValue<'tcx>,
     acc: Option<Value>,
     ret: CPlace<'tcx>,
-    f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
+    f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     let lane_layout = fx.layout_of(lane_ty);
@@ -181,7 +170,7 @@ fn simd_reduce<'tcx>(
         if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
     for lane_idx in start_lane..lane_count {
         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
-        res_val = f(fx, lane_layout, res_val, lane);
+        res_val = f(fx, lane_layout.ty, res_val, lane);
     }
     let res = CValue::by_val(res_val, lane_layout);
     ret.write_cvalue(fx, res);
@@ -215,10 +204,10 @@ fn simd_reduce_bool<'tcx>(
 
 fn bool_to_zero_or_max_uint<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    layout: TyAndLayout<'tcx>,
+    ty: Ty<'tcx>,
     val: Value,
 ) -> Value {
-    let ty = fx.clif_type(layout.ty).unwrap();
+    let ty = fx.clif_type(ty).unwrap();
 
     let int_ty = match ty {
         types::F32 => types::I32,
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index dc04c7643b2a4..106a190096dba 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -17,76 +17,52 @@ fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span:
 
 macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
     // FIXME use vector instructions when possible
-    simd_pair_for_each_lane(
-        $fx,
-        $x,
-        $y,
-        $ret,
-        &|fx, lane_layout, res_lane_layout, x_lane, y_lane| {
-            let res_lane = match lane_layout.ty.kind() {
-                ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
-                ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
-                ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-                _ => unreachable!("{:?}", lane_layout.ty),
-            };
+    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+        let res_lane = match lane_ty.kind() {
+            ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
+            ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
+            ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
+            _ => unreachable!("{:?}", lane_ty),
+        };
 
-            let ty = fx.clif_type(res_lane_layout.ty).unwrap();
+        let ty = fx.clif_type(res_lane_ty).unwrap();
 
-            let res_lane = fx.bcx.ins().bint(ty, res_lane);
-            fx.bcx.ins().ineg(res_lane)
-        },
-    );
+        let res_lane = fx.bcx.ins().bint(ty, res_lane);
+        fx.bcx.ins().ineg(res_lane)
+    });
 }
 
 macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
     // FIXME use vector instructions when possible
-    simd_pair_for_each_lane(
-        $fx,
-        $x,
-        $y,
-        $ret,
-        &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
-            match lane_layout.ty.kind() {
-                ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-                ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-                _ => unreachable!("{:?}", lane_layout.ty),
-            }
-        },
-    );
+    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+        match lane_ty.kind() {
+            ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
+            ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
+            _ => unreachable!("{:?}", lane_ty),
+        }
+    });
 }
 
 macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
     // FIXME use vector instructions when possible
-    simd_pair_for_each_lane(
-        $fx,
-        $x,
-        $y,
-        $ret,
-        &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
-            match lane_layout.ty.kind() {
-                ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-                ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-                ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
-                _ => unreachable!("{:?}", lane_layout.ty),
-            }
-        },
-    );
+    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+        match lane_ty.kind() {
+            ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
+            ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
+            ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
+            _ => unreachable!("{:?}", lane_ty),
+        }
+    });
 }
 
 macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
     // FIXME use vector instructions when possible
-    simd_pair_for_each_lane(
-        $fx,
-        $x,
-        $y,
-        $ret,
-        &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
-            match lane_layout.ty.kind() {
-                ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
-                _ => unreachable!("{:?}", lane_layout.ty),
-            }
-        },
-    );
+    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+        match lane_ty.kind() {
+            ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
+            _ => unreachable!("{:?}", lane_ty),
+        }
+    });
 }
 
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
@@ -105,13 +81,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_cast, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, lane_layout, ret_lane_layout, lane| {
-                let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
+            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
+                let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
 
-                let from_signed = type_sign(lane_layout.ty);
-                let to_signed = type_sign(ret_lane_layout.ty);
+                let from_signed = type_sign(lane_ty);
+                let to_signed = type_sign(ret_lane_ty);
 
-                clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed)
+                clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
             });
         };
 
@@ -277,8 +253,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_neg, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, lane_layout, _ret_lane_layout, lane| {
-                match lane_layout.ty.kind() {
+            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+                match lane_ty.kind() {
                     ty::Int(_) => fx.bcx.ins().ineg(lane),
                     ty::Float(_) => fx.bcx.ins().fneg(lane),
                     _ => unreachable!(),
@@ -288,14 +264,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_fabs, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().fabs(lane)
             });
         };
 
         simd_fsqrt, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().sqrt(lane)
             });
         };
@@ -318,8 +294,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
         simd_rem, (c x, c y) {
             validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
-                match lane_layout.ty.kind() {
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+                match lane_ty.kind() {
                     ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
                     ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane),
                     ty::Float(FloatTy::F32) => fx.lib_call(
@@ -334,7 +310,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                         vec![AbiParam::new(types::F64)],
                         &[x_lane, y_lane],
                     )[0],
-                    _ => unreachable!("{:?}", lane_layout.ty),
+                    _ => unreachable!("{:?}", lane_ty),
                 }
             });
         };
@@ -393,8 +369,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_round, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, lane_layout, _ret_lane_layout, lane| {
-                match lane_layout.ty.kind() {
+            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+                match lane_ty.kind() {
                     ty::Float(FloatTy::F32) => fx.lib_call(
                         "roundf",
                         vec![AbiParam::new(types::F32)],
@@ -407,33 +383,33 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                         vec![AbiParam::new(types::F64)],
                         &[lane],
                     )[0],
-                    _ => unreachable!("{:?}", lane_layout.ty),
+                    _ => unreachable!("{:?}", lane_ty),
                 }
             });
         };
         simd_ceil, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().ceil(lane)
             });
         };
         simd_floor, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().floor(lane)
             });
         };
         simd_trunc, (c a) {
             validate_simd_type(fx, intrinsic, span, a.layout().ty);
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().trunc(lane)
             });
         };
 
         simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_layout, a, b| {
-                if lane_layout.ty.is_floating_point() {
+            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
+                if lane_ty.is_floating_point() {
                     fx.bcx.ins().fadd(a, b)
                 } else {
                     fx.bcx.ins().iadd(a, b)
@@ -443,8 +419,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_layout, a, b| {
-                if lane_layout.ty.is_floating_point() {
+            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
+                if lane_ty.is_floating_point() {
                     fx.bcx.ins().fmul(a, b)
                 } else {
                     fx.bcx.ins().imul(a, b)
@@ -464,23 +440,23 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_reduce_and, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().band(a, b));
+            simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_or, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().bor(a, b));
+            simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_xor, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().bxor(a, b));
+            simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
         };
 
         simd_reduce_min, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, &|fx, layout, a, b| {
-                let lt = match layout.ty.kind() {
+            simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
+                let lt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
                     ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
@@ -492,8 +468,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_reduce_max, (c v) {
             validate_simd_type(fx, intrinsic, span, v.layout().ty);
-            simd_reduce(fx, v, None, ret, &|fx, layout, a, b| {
-                let gt = match layout.ty.kind() {
+            simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
+                let gt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
                     ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),

From 4dbd3196f9c810b7b985e579cb6f13ce907a8286 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 9 Jan 2022 19:30:07 +0100
Subject: [PATCH 22/84] Move most code from y.rs to build_system/mod.rs

y.rs can't be rustfmt'ed without making it no longer be a valid bash
script.
---
 build_system/build_backend.rs |   2 +-
 build_system/build_sysroot.rs |  10 +--
 build_system/mod.rs           | 127 ++++++++++++++++++++++++++++++++
 build_system/prepare.rs       |   4 +-
 y.rs                          | 133 +---------------------------------
 5 files changed, 138 insertions(+), 138 deletions(-)
 create mode 100644 build_system/mod.rs

diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs
index 1382c7e53793e..0a56eb131ed30 100644
--- a/build_system/build_backend.rs
+++ b/build_system/build_backend.rs
@@ -49,7 +49,7 @@ pub(crate) fn build_backend(
     cmd.env("RUSTFLAGS", rustflags);
 
     eprintln!("[BUILD] rustc_codegen_cranelift");
-    crate::utils::spawn_and_wait(cmd);
+    super::utils::spawn_and_wait(cmd);
 
     Path::new("target").join(host_triple).join(channel)
 }
diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs
index 2956fb698e175..c9c003d461095 100644
--- a/build_system/build_sysroot.rs
+++ b/build_system/build_sysroot.rs
@@ -3,9 +3,9 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::{self, Command};
 
-use crate::rustc_info::{get_file_name, get_rustc_version};
-use crate::utils::{spawn_and_wait, try_hard_link};
-use crate::SysrootKind;
+use super::rustc_info::{get_file_name, get_rustc_version};
+use super::utils::{spawn_and_wait, try_hard_link};
+use super::SysrootKind;
 
 pub(crate) fn build_sysroot(
     channel: &str,
@@ -52,7 +52,7 @@ pub(crate) fn build_sysroot(
         .arg("-g");
     spawn_and_wait(build_cargo_wrapper_cmd);
 
-    let default_sysroot = crate::rustc_info::get_default_sysroot();
+    let default_sysroot = super::rustc_info::get_default_sysroot();
 
     let rustlib = target_dir.join("lib").join("rustlib");
     let host_rustlib_lib = rustlib.join(host_triple).join("lib");
@@ -167,7 +167,7 @@ fn build_clif_sysroot_for_triple(
 
     let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel);
 
-    if !crate::config::get_bool("keep_sysroot") {
+    if !super::config::get_bool("keep_sysroot") {
         // Cleanup the target dir with the exception of build scripts and the incremental cache
         for dir in ["build", "deps", "examples", "native"] {
             if build_dir.join(dir).exists() {
diff --git a/build_system/mod.rs b/build_system/mod.rs
new file mode 100644
index 0000000000000..b228da3981fdb
--- /dev/null
+++ b/build_system/mod.rs
@@ -0,0 +1,127 @@
+use std::env;
+use std::path::PathBuf;
+use std::process;
+
+mod build_backend;
+mod build_sysroot;
+mod config;
+mod prepare;
+mod rustc_info;
+mod utils;
+
+fn usage() {
+    eprintln!("Usage:");
+    eprintln!("  ./y.rs prepare");
+    eprintln!(
+        "  ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--no-unstable-features]"
+    );
+}
+
+macro_rules! arg_error {
+    ($($err:tt)*) => {{
+        eprintln!($($err)*);
+        usage();
+        std::process::exit(1);
+    }};
+}
+
+enum Command {
+    Build,
+}
+
+#[derive(Copy, Clone)]
+pub(crate) enum SysrootKind {
+    None,
+    Clif,
+    Llvm,
+}
+
+pub fn main() {
+    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
+    env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
+    // The target dir is expected in the default location. Guard against the user changing it.
+    env::set_var("CARGO_TARGET_DIR", "target");
+
+    let mut args = env::args().skip(1);
+    let command = match args.next().as_deref() {
+        Some("prepare") => {
+            if args.next().is_some() {
+                arg_error!("./x.rs prepare doesn't expect arguments");
+            }
+            prepare::prepare();
+            process::exit(0);
+        }
+        Some("build") => Command::Build,
+        Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
+        Some(command) => arg_error!("Unknown command {}", command),
+        None => {
+            usage();
+            process::exit(0);
+        }
+    };
+
+    let mut target_dir = PathBuf::from("build");
+    let mut channel = "release";
+    let mut sysroot_kind = SysrootKind::Clif;
+    let mut use_unstable_features = true;
+    while let Some(arg) = args.next().as_deref() {
+        match arg {
+            "--target-dir" => {
+                target_dir = PathBuf::from(args.next().unwrap_or_else(|| {
+                    arg_error!("--target-dir requires argument");
+                }))
+            }
+            "--debug" => channel = "debug",
+            "--sysroot" => {
+                sysroot_kind = match args.next().as_deref() {
+                    Some("none") => SysrootKind::None,
+                    Some("clif") => SysrootKind::Clif,
+                    Some("llvm") => SysrootKind::Llvm,
+                    Some(arg) => arg_error!("Unknown sysroot kind {}", arg),
+                    None => arg_error!("--sysroot requires argument"),
+                }
+            }
+            "--no-unstable-features" => use_unstable_features = false,
+            flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
+            arg => arg_error!("Unexpected argument {}", arg),
+        }
+    }
+
+    let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {
+        host_triple
+    } else if let Some(host_triple) = config::get_value("host") {
+        host_triple
+    } else {
+        rustc_info::get_host_triple()
+    };
+    let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") {
+        if target_triple != "" {
+            target_triple
+        } else {
+            host_triple.clone() // Empty target triple can happen on GHA
+        }
+    } else if let Some(target_triple) = config::get_value("target") {
+        target_triple
+    } else {
+        host_triple.clone()
+    };
+
+    if target_triple.ends_with("-msvc") {
+        eprintln!("The MSVC toolchain is not yet supported by rustc_codegen_cranelift.");
+        eprintln!("Switch to the MinGW toolchain for Windows support.");
+        eprintln!("Hint: You can use `rustup set default-host x86_64-pc-windows-gnu` to");
+        eprintln!("set the global default target to MinGW");
+        process::exit(1);
+    }
+
+    let cg_clif_build_dir =
+        build_backend::build_backend(channel, &host_triple, use_unstable_features);
+    build_sysroot::build_sysroot(
+        channel,
+        sysroot_kind,
+        &target_dir,
+        cg_clif_build_dir,
+        &host_triple,
+        &target_triple,
+    );
+}
diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 561e2ed7b0017..76ea506b6ac4f 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -5,8 +5,8 @@ use std::fs;
 use std::path::Path;
 use std::process::Command;
 
-use crate::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
-use crate::utils::{copy_dir_recursively, spawn_and_wait};
+use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
+use super::utils::{copy_dir_recursively, spawn_and_wait};
 
 pub(crate) fn prepare() {
     prepare_sysroot();
diff --git a/y.rs b/y.rs
index 98b114de91078..18528d5429729 100755
--- a/y.rs
+++ b/y.rs
@@ -23,136 +23,9 @@ exec ${0/.rs/.bin} $@
 //!
 //! The name `y.rs` was chosen to not conflict with rustc's `x.py`.
 
-use std::env;
-use std::path::PathBuf;
-use std::process;
-
-#[path = "build_system/build_backend.rs"]
-mod build_backend;
-#[path = "build_system/build_sysroot.rs"]
-mod build_sysroot;
-#[path = "build_system/config.rs"]
-mod config;
-#[path = "build_system/prepare.rs"]
-mod prepare;
-#[path = "build_system/rustc_info.rs"]
-mod rustc_info;
-#[path = "build_system/utils.rs"]
-mod utils;
-
-fn usage() {
-    eprintln!("Usage:");
-    eprintln!("  ./y.rs prepare");
-    eprintln!(
-        "  ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--no-unstable-features]"
-    );
-}
-
-macro_rules! arg_error {
-    ($($err:tt)*) => {{
-        eprintln!($($err)*);
-        usage();
-        std::process::exit(1);
-    }};
-}
-
-enum Command {
-    Build,
-}
-
-#[derive(Copy, Clone)]
-enum SysrootKind {
-    None,
-    Clif,
-    Llvm,
-}
+#[path = "build_system/mod.rs"]
+mod build_system;
 
 fn main() {
-    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
-    env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
-    // The target dir is expected in the default location. Guard against the user changing it.
-    env::set_var("CARGO_TARGET_DIR", "target");
-
-    let mut args = env::args().skip(1);
-    let command = match args.next().as_deref() {
-        Some("prepare") => {
-            if args.next().is_some() {
-                arg_error!("./x.rs prepare doesn't expect arguments");
-            }
-            prepare::prepare();
-            process::exit(0);
-        }
-        Some("build") => Command::Build,
-        Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
-        Some(command) => arg_error!("Unknown command {}", command),
-        None => {
-            usage();
-            process::exit(0);
-        }
-    };
-
-    let mut target_dir = PathBuf::from("build");
-    let mut channel = "release";
-    let mut sysroot_kind = SysrootKind::Clif;
-    let mut use_unstable_features = true;
-    while let Some(arg) = args.next().as_deref() {
-        match arg {
-            "--target-dir" => {
-                target_dir = PathBuf::from(args.next().unwrap_or_else(|| {
-                    arg_error!("--target-dir requires argument");
-                }))
-            }
-            "--debug" => channel = "debug",
-            "--sysroot" => {
-                sysroot_kind = match args.next().as_deref() {
-                    Some("none") => SysrootKind::None,
-                    Some("clif") => SysrootKind::Clif,
-                    Some("llvm") => SysrootKind::Llvm,
-                    Some(arg) => arg_error!("Unknown sysroot kind {}", arg),
-                    None => arg_error!("--sysroot requires argument"),
-                }
-            }
-            "--no-unstable-features" => use_unstable_features = false,
-            flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
-            arg => arg_error!("Unexpected argument {}", arg),
-        }
-    }
-
-    let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {
-        host_triple
-    } else if let Some(host_triple) = crate::config::get_value("host") {
-        host_triple
-    } else {
-        rustc_info::get_host_triple()
-    };
-    let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") {
-        if target_triple != "" {
-            target_triple
-        } else {
-            host_triple.clone() // Empty target triple can happen on GHA
-        }
-    } else if let Some(target_triple) = crate::config::get_value("target") {
-        target_triple
-    } else {
-        host_triple.clone()
-    };
-
-    if target_triple.ends_with("-msvc") {
-        eprintln!("The MSVC toolchain is not yet supported by rustc_codegen_cranelift.");
-        eprintln!("Switch to the MinGW toolchain for Windows support.");
-        eprintln!("Hint: You can use `rustup set default-host x86_64-pc-windows-gnu` to");
-        eprintln!("set the global default target to MinGW");
-        process::exit(1);
-    }
-
-    let cg_clif_build_dir =
-        build_backend::build_backend(channel, &host_triple, use_unstable_features);
-    build_sysroot::build_sysroot(
-        channel,
-        sysroot_kind,
-        &target_dir,
-        cg_clif_build_dir,
-        &host_triple,
-        &target_triple,
-    );
+    build_system::main();
 }

From 759c4ac08f653423cf5105400256b4702a5bbcd6 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Mon, 10 Jan 2022 12:26:03 +0100
Subject: [PATCH 23/84] Update to Cranelift 0.80.0

---
 Cargo.lock | 40 ++++++++++++++++++++--------------------
 Cargo.toml | 12 ++++++------
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4e2106e3fbe25..bd36a48599390 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -33,18 +33,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebddaa5d12cb299b0bc7c930aff12c5591d4ba9aa84eea637807e07283b900aa"
+checksum = "9516ba6b2ba47b4cbf63b713f75b432fafa0a0e0464ec8381ec76e6efe931ab3"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da1daf5641177162644b521b64418564b8ed5deb126275a4d91472d13e7c72df"
+checksum = "489e5d0081f7edff6be12d71282a8bf387b5df64d5592454b75d662397f2d642"
 dependencies = [
  "cranelift-bforest",
  "cranelift-codegen-meta",
@@ -59,30 +59,30 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "001c1e9e540940c81596e547e732f99c2146c21ea7e82da99be961a1e86feefa"
+checksum = "d36ee1140371bb0f69100e734b30400157a4adf7b86148dee8b0a438763ead48"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ebaf07b5d7501cc606f41c81333bd63a5a17eb501362ccb10bc8ff5c03d0232"
+checksum = "981da52d8f746af1feb96290c83977ff8d41071a7499e991d8abae0d4869f564"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a27ada0e3ffe5325179fc750252c18d614fa5470d595ce5c8a794c495434d80a"
+checksum = "a2906740053dd3bcf95ce53df0fd9b5649c68ae4bd9adada92b406f059eae461"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2912c0eec9fd3df2dcf82b02b642caaa85d762b84ac5a3b27bc93a07eeeb64e2"
+checksum = "b7cb156de1097f567d46bf57a0cd720a72c3e15e1a2bd8b1041ba2fc894471b7"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -92,9 +92,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb1bb6937aefbd4c84ec16bcbf4e4138e0c182a965e9b45b59c8ad0c5a716fd5"
+checksum = "8e0f0e20dbcac1e6c3caef955e004598a9a6a5f310e591e2c629ec15c7fa6bfa"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -110,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff8c96b8b2fad9879207762ec6fe2f0d32d1f98b215a9dc360f52f7d2b832600"
+checksum = "93460fc789770f2a63163bfb5f2b851635ce29d91526f2e96854bcc4ed53a778"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -120,9 +120,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd20f78f378f55a70738a2eb9815dcd7e8455ff091b70701cfd086dd44927da"
+checksum = "166028ca0343a6ee7bddac0e70084e142b23f99c701bd6f6ea9123afac1a7a46"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -131,9 +131,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.79.1"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34a2db23dcee236e25bccb9a6339aa555c1c0e27573c198607d869530453fb4a"
+checksum = "263618594523ad29349fd43d93bfeb04d2bbdfd5df9fcf04190582dbdcff3220"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
diff --git a/Cargo.toml b/Cargo.toml
index 7bd3012ff863f..879c1256e9d33 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.79.1", features = ["unwind", "all-arch"] }
-cranelift-frontend = "0.79.1"
-cranelift-module = "0.79.1"
-cranelift-native = "0.79.1"
-cranelift-jit = { version = "0.79.1", optional = true }
-cranelift-object = "0.79.1"
+cranelift-codegen = { version = "0.80.0", features = ["unwind", "all-arch"] }
+cranelift-frontend = "0.80.0"
+cranelift-module = "0.80.0"
+cranelift-native = "0.80.0"
+cranelift-jit = { version = "0.80.0", optional = true }
+cranelift-object = "0.80.0"
 target-lexicon = "0.12.0"
 gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

From 8f02c4e2ee4a5e86bc14a3fe4d1bee9c3f2fc253 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Mon, 10 Jan 2022 12:26:23 +0100
Subject: [PATCH 24/84] Add y.bin to clean_all.sh

---
 clean_all.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clean_all.sh b/clean_all.sh
index 865de7d234f14..ea1f8c1e8920a 100755
--- a/clean_all.sh
+++ b/clean_all.sh
@@ -2,5 +2,5 @@
 set -e
 
 rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
-rm -rf target/ build/ perf.data{,.old}
+rm -rf target/ build/ perf.data{,.old} y.bin
 rm -rf rand/ regex/ simple-raytracer/ portable-simd/

From 409276cf55a666932524d9d9dd9e0a0329364db3 Mon Sep 17 00:00:00 2001
From: Aaron Hill <aa1ronham@gmail.com>
Date: Sun, 2 Jan 2022 22:37:05 -0500
Subject: [PATCH 25/84] Store a `Symbol` instead of an `Ident` in
 `VariantDef`/`FieldDef`

The field is also renamed from `ident` to `name. In most cases,
we don't actually need the `Span`. A new `ident` method is added
to `VariantDef` and `FieldDef`, which constructs the full `Ident`
using `tcx.def_ident_span()`. This method is used in the cases
where we actually need an `Ident`.

This makes incremental compilation properly track changes
to the `Span`, without all of the invalidations caused by storing
a `Span` directly via an `Ident`.
---
 src/debuginfo/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/debuginfo/mod.rs b/src/debuginfo/mod.rs
index 638b025be229d..8e203b8cfa063 100644
--- a/src/debuginfo/mod.rs
+++ b/src/debuginfo/mod.rs
@@ -174,7 +174,7 @@ impl<'tcx> DebugContext<'tcx> {
 
                     field_entry.set(
                         gimli::DW_AT_name,
-                        AttributeValue::String(field_def.ident.as_str().to_string().into_bytes()),
+                        AttributeValue::String(field_def.name.as_str().to_string().into_bytes()),
                     );
                     field_entry.set(
                         gimli::DW_AT_data_member_location,

From 7ec4de3ab8b2ed5113503e0ebb8f63ce306337ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= <tomasz.miasko@gmail.com>
Date: Wed, 12 Jan 2022 00:00:00 +0000
Subject: [PATCH 26/84] Remove deprecated LLVM-style inline assembly

---
 src/base.rs     | 12 ------------
 src/constant.rs |  2 +-
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/src/base.rs b/src/base.rs
index b16f5af66f249..5a889734f215b 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -749,18 +749,6 @@ fn codegen_stmt<'tcx>(
         | StatementKind::Retag { .. }
         | StatementKind::AscribeUserType(..) => {}
 
-        StatementKind::LlvmInlineAsm(asm) => {
-            match asm.asm.asm.as_str().trim() {
-                "" => {
-                    // Black box
-                }
-                _ => fx.tcx.sess.span_fatal(
-                    stmt.source_info.span,
-                    "Legacy `llvm_asm!` inline assembly is not supported. \
-                    Try using the new `asm!` instead.",
-                ),
-            }
-        }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
         StatementKind::CopyNonOverlapping(inner) => {
             let dst = codegen_operand(fx, &inner.dst);
diff --git a/src/constant.rs b/src/constant.rs
index 9a6c45ae98d5f..7ef09a1a6141d 100644
--- a/src/constant.rs
+++ b/src/constant.rs
@@ -506,7 +506,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         {
                             return None;
                         }
-                        StatementKind::LlvmInlineAsm(_) | StatementKind::CopyNonOverlapping(_) => {
+                        StatementKind::CopyNonOverlapping(_) => {
                             return None;
                         } // conservative handling
                         StatementKind::Assign(_)

From eef35308efcd493587d9b1d23b2363d99246a220 Mon Sep 17 00:00:00 2001
From: Ellen <supbscripter@gmail.com>
Date: Wed, 12 Jan 2022 03:19:52 +0000
Subject: [PATCH 27/84] initial revert

---
 src/constant.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/constant.rs b/src/constant.rs
index 9a6c45ae98d5f..0c06c77472b91 100644
--- a/src/constant.rs
+++ b/src/constant.rs
@@ -129,11 +129,13 @@ pub(crate) fn codegen_constant<'tcx>(
     };
     let const_val = match const_.val {
         ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(uv) if fx.tcx.is_static(uv.def.did) => {
-            assert!(uv.substs(fx.tcx).is_empty());
-            assert!(uv.promoted.is_none());
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
+            assert!(substs.is_empty());
+            assert!(promoted.is_none());
 
-            return codegen_static_ref(fx, uv.def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
+            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
         }
         ConstKind::Unevaluated(unevaluated) => {
             match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {

From 48abe3c6fa29dbedc611e22c6acde7d0022219f4 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Mon, 10 Jan 2022 15:32:45 +0100
Subject: [PATCH 28/84] Use Symbol for target features in asm handling

This saves a couple of Symbol::intern calls
---
 src/inline_asm.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index 93384bc551101..be39dbd2e2acb 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -6,7 +6,7 @@ use std::fmt::Write;
 
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir::InlineAsmOperand;
-use rustc_span::Symbol;
+use rustc_span::sym;
 use rustc_target::asm::*;
 
 pub(crate) fn codegen_inline_asm<'tcx>(
@@ -184,7 +184,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         let sess = self.tcx.sess;
         let map = allocatable_registers(
             self.arch,
-            |feature| sess.target_features.contains(&Symbol::intern(feature)),
+            |feature| sess.target_features.contains(&feature),
             &sess.target,
         );
         let mut allocated = FxHashMap::<_, (bool, bool)>::default();
@@ -319,9 +319,9 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for saving clobbered registers
         let abi_clobber = InlineAsmClobberAbi::parse(
             self.arch,
-            |feature| self.tcx.sess.target_features.contains(&Symbol::intern(feature)),
+            |feature| self.tcx.sess.target_features.contains(&feature),
             &self.tcx.sess.target,
-            Symbol::intern("C"),
+            sym::C,
         )
         .unwrap()
         .clobbered_regs();

From e59b024e025795ecf7e3a57471451996be637656 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Mon, 10 Jan 2022 15:48:05 +0100
Subject: [PATCH 29/84] Pass target_features set instead of has_feature closure

This avoids unnecessary monomorphizations in codegen backends
---
 src/inline_asm.rs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index be39dbd2e2acb..c242c75ed18ff 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -182,11 +182,7 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
 impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
     fn allocate_registers(&mut self) {
         let sess = self.tcx.sess;
-        let map = allocatable_registers(
-            self.arch,
-            |feature| sess.target_features.contains(&feature),
-            &sess.target,
-        );
+        let map = allocatable_registers(self.arch, &sess.target_features, &sess.target);
         let mut allocated = FxHashMap::<_, (bool, bool)>::default();
         let mut regs = vec![None; self.operands.len()];
 
@@ -319,7 +315,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for saving clobbered registers
         let abi_clobber = InlineAsmClobberAbi::parse(
             self.arch,
-            |feature| self.tcx.sess.target_features.contains(&feature),
+            &self.tcx.sess.target_features,
             &self.tcx.sess.target,
             sym::C,
         )

From b85cbddf751b056ff1a7f925ccdcf9d04e554725 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 18 Jan 2022 13:02:02 +0100
Subject: [PATCH 30/84] Rustup to rustc 1.60.0-nightly (ee5d8d37b 2022-01-17)

---
 rust-toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain b/rust-toolchain
index cab94c0b8cfa7..f1bc748284d71 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2021-12-30"
+channel = "nightly-2022-01-18"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From 0472f6c91ce7afc7215a4e1e1be312c2c1ccba8c Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 18 Jan 2022 13:41:01 +0100
Subject: [PATCH 31/84] Use 2021 edition for libcore tests

---
 patches/0022-sysroot-Disable-not-compiling-tests.patch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/patches/0022-sysroot-Disable-not-compiling-tests.patch b/patches/0022-sysroot-Disable-not-compiling-tests.patch
index 25a315f666e27..ff8d5e8742bbb 100644
--- a/patches/0022-sysroot-Disable-not-compiling-tests.patch
+++ b/patches/0022-sysroot-Disable-not-compiling-tests.patch
@@ -22,7 +22,7 @@ index 0000000..46fd999
 +[package]
 +name = "core"
 +version = "0.0.0"
-+edition = "2018"
++edition = "2021"
 +
 +[lib]
 +name = "coretests"

From b91ecc78bf619f3d5ad1b9b590e25f77f3ab11f6 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 18 Jan 2022 13:59:40 +0100
Subject: [PATCH 32/84] Update rustc test ignore list

---
 scripts/test_rustc_tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh
index 6bcc3049ecc4e..a9a44d2c654a6 100755
--- a/scripts/test_rustc_tests.sh
+++ b/scripts/test_rustc_tests.sh
@@ -98,6 +98,7 @@ rm src/test/ui/abi/stack-protector.rs # requires stack protector support
 rm src/test/incremental/issue-80691-bad-eval-cache.rs # wrong exit code
 rm src/test/incremental/spike-neg1.rs # errors out for some reason
 rm src/test/incremental/spike-neg2.rs # same
+rm src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs # gives a stackoverflow before the backend runs
 
 rm src/test/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM
 

From f3b2d37e7754458a14ac99cee970ede383eddc2f Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 18 Jan 2022 15:24:11 +0100
Subject: [PATCH 33/84] Record object file artifact size in self profile data

---
 src/driver/aot.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/driver/aot.rs b/src/driver/aot.rs
index 046e4393a68d6..2e047c7eea14e 100644
--- a/src/driver/aot.rs
+++ b/src/driver/aot.rs
@@ -56,6 +56,9 @@ fn emit_module(
 
     let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name));
     let obj = product.object.write().unwrap();
+
+    tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap());
+
     if let Err(err) = std::fs::write(&tmp_file, obj) {
         tcx.sess.fatal(&format!("error writing object file: {}", err));
     }

From 5f6c59e63faf0705d4c6e1fbd7a66ffc59b9ae1f Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 18 Jan 2022 18:58:37 +0100
Subject: [PATCH 34/84] Pass only the Function to write_clif_file

---
 src/base.rs         | 12 ++++++------
 src/optimize/mod.rs |  2 +-
 src/pretty_clif.rs  |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/base.rs b/src/base.rs
index 5a889734f215b..c1c9a08dc5f6e 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -105,19 +105,19 @@ pub(crate) fn codegen_fn<'tcx>(
 
     fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
 
-    // Store function in context
-    let context = &mut cx.cached_context;
-    context.func = func;
-
     crate::pretty_clif::write_clif_file(
         tcx,
         "unopt",
         module.isa(),
         instance,
-        &context,
+        &func,
         &clif_comments,
     );
 
+    // Store function in context
+    let context = &mut cx.cached_context;
+    context.func = func;
+
     // Verify function
     verify_func(tcx, &clif_comments, &context.func);
 
@@ -156,7 +156,7 @@ pub(crate) fn codegen_fn<'tcx>(
         "opt",
         module.isa(),
         instance,
-        &context,
+        &context.func,
         &clif_comments,
     );
 
diff --git a/src/optimize/mod.rs b/src/optimize/mod.rs
index 61033d85a1274..d1f89adb3bb91 100644
--- a/src/optimize/mod.rs
+++ b/src/optimize/mod.rs
@@ -15,6 +15,6 @@ pub(crate) fn optimize_function<'tcx>(
 ) {
     // FIXME classify optimizations over opt levels once we have more
 
-    crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx, &*clif_comments);
+    crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx.func, &*clif_comments);
     crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
 }
diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs
index 4dffb89e10570..5b77f4ef0c9f1 100644
--- a/src/pretty_clif.rs
+++ b/src/pretty_clif.rs
@@ -234,7 +234,7 @@ pub(crate) fn write_clif_file<'tcx>(
     postfix: &str,
     isa: &dyn cranelift_codegen::isa::TargetIsa,
     instance: Instance<'tcx>,
-    context: &cranelift_codegen::Context,
+    func: &cranelift_codegen::ir::Function,
     mut clif_comments: &CommentWriter,
 ) {
     write_ir_file(
@@ -245,7 +245,7 @@ pub(crate) fn write_clif_file<'tcx>(
             cranelift_codegen::write::decorate_function(
                 &mut clif_comments,
                 &mut clif,
-                &context.func,
+                func,
             )
             .unwrap();
 

From 78b65718bce8d7f8b2e1d1af74141cebbd78cf5f Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 18 Jan 2022 19:07:26 +0100
Subject: [PATCH 35/84] Split compile_fn out of codegen_fn

---
 src/base.rs | 43 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/src/base.rs b/src/base.rs
index c1c9a08dc5f6e..ad009b5a3f2fb 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -6,8 +6,11 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiOf;
 
+use indexmap::IndexSet;
+
 use crate::constant::ConstantCx;
 use crate::prelude::*;
+use crate::pretty_clif::CommentWriter;
 
 pub(crate) fn codegen_fn<'tcx>(
     cx: &mut crate::CodegenCx<'tcx>,
@@ -99,7 +102,7 @@ pub(crate) fn codegen_fn<'tcx>(
 
     // Recover all necessary data from fx, before accessing func will prevent future access to it.
     let instance = fx.instance;
-    let mut clif_comments = fx.clif_comments;
+    let clif_comments = fx.clif_comments;
     let source_info_set = fx.source_info_set;
     let local_map = fx.local_map;
 
@@ -114,13 +117,40 @@ pub(crate) fn codegen_fn<'tcx>(
         &clif_comments,
     );
 
+    // Verify function
+    verify_func(tcx, &clif_comments, &func);
+
+    compile_fn(
+        cx,
+        module,
+        instance,
+        symbol_name.name,
+        func_id,
+        func,
+        clif_comments,
+        source_info_set,
+        local_map,
+    );
+}
+
+fn compile_fn<'tcx>(
+    cx: &mut crate::CodegenCx<'tcx>,
+    module: &mut dyn Module,
+    instance: Instance<'tcx>,
+    symbol_name: &str,
+    func_id: FuncId,
+    func: Function,
+    mut clif_comments: CommentWriter,
+    source_info_set: IndexSet<SourceInfo>,
+    local_map: IndexVec<mir::Local, CPlace<'tcx>>,
+) {
+    let tcx = cx.tcx;
+
     // Store function in context
     let context = &mut cx.cached_context;
+    context.clear();
     context.func = func;
 
-    // Verify function
-    verify_func(tcx, &clif_comments, &context.func);
-
     // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
     // instruction, which doesn't have an encoding.
     context.compute_cfg();
@@ -177,7 +207,7 @@ pub(crate) fn codegen_fn<'tcx>(
             debug_context.define_function(
                 instance,
                 func_id,
-                symbol_name.name,
+                symbol_name,
                 isa,
                 context,
                 &source_info_set,
@@ -186,9 +216,6 @@ pub(crate) fn codegen_fn<'tcx>(
         }
         unwind_context.add_function(func_id, &context, isa);
     });
-
-    // Clear context to make it usable for the next function
-    context.clear();
 }
 
 pub(crate) fn verify_func(

From 513d72d007619cef8fc60565b1bde7924362a442 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Wed, 19 Jan 2022 10:33:23 +0100
Subject: [PATCH 36/84] remove `is_noop`

---
 src/intrinsics/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index f4703b22ecbcf..55c9b4d9ba12d 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -90,7 +90,7 @@ macro call_intrinsic_match {
         match $intrinsic {
             $(
                 sym::$name => {
-                    assert!($substs.is_noop());
+                    assert!($substs.is_empty());
                     if let [$(ref $arg),*] = *$args {
                         let ($($arg,)*) = (
                             $(codegen_operand($fx, $arg),)*

From 7a8227ecf954f61b4e2b6f9635ee0eab060d811b Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Wed, 19 Jan 2022 15:36:14 +0100
Subject: [PATCH 37/84] Rustup to rustc 1.60.0-nightly (9ad5d82f8 2022-01-18)

---
 rust-toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain b/rust-toolchain
index f1bc748284d71..c42f8e5f45569 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-01-18"
+channel = "nightly-2022-01-19"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From c6e607a94761c3ec18b8ce2274bc3150573ca4bb Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Wed, 19 Jan 2022 15:45:04 +0100
Subject: [PATCH 38/84] Implement unchecked_mul intrinsic

---
 src/intrinsics/mod.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 1e384668fc72e..1756cf78c2d88 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -436,12 +436,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
         };
 
-        unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem
+        unchecked_add | unchecked_sub | unchecked_mul | unchecked_div | exact_div | unchecked_rem
         | unchecked_shl | unchecked_shr, (c x, c y) {
             // FIXME trap on overflow
             let bin_op = match intrinsic {
                 sym::unchecked_add => BinOp::Add,
                 sym::unchecked_sub => BinOp::Sub,
+                sym::unchecked_mul => BinOp::Mul,
                 sym::unchecked_div | sym::exact_div => BinOp::Div,
                 sym::unchecked_rem => BinOp::Rem,
                 sym::unchecked_shl => BinOp::Shl,

From cc24cea10123d4de65f155051d2e8b554cb718fd Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Wed, 19 Jan 2022 16:38:58 +0100
Subject: [PATCH 39/84] Rustfmt

---
 src/pretty_clif.rs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs
index 5b77f4ef0c9f1..ca7116b887d5a 100644
--- a/src/pretty_clif.rs
+++ b/src/pretty_clif.rs
@@ -242,12 +242,8 @@ pub(crate) fn write_clif_file<'tcx>(
         || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
         |file| {
             let mut clif = String::new();
-            cranelift_codegen::write::decorate_function(
-                &mut clif_comments,
-                &mut clif,
-                func,
-            )
-            .unwrap();
+            cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func)
+                .unwrap();
 
             for flag in isa.flags().iter() {
                 writeln!(file, "set {}", flag)?;

From 2c26139bbcff3afef42e746f9c8fb61d7daae9b4 Mon Sep 17 00:00:00 2001
From: pierwill <pierwill@users.noreply.github.com>
Date: Thu, 6 Jan 2022 13:27:59 -0600
Subject: [PATCH 40/84] Use an `indexmap` to avoid sorting `LocalDefId`s

Update `indexmap` to 1.8.0.

Bless test
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 65e142a00f88e..faed52727c828 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -172,9 +172,9 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 
 [[package]]
 name = "indexmap"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
 dependencies = [
  "autocfg",
  "hashbrown",
diff --git a/Cargo.toml b/Cargo.toml
index 3be4250296e77..2d19040b5091c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,7 +19,7 @@ gimli = { version = "0.25.0", default-features = false, features = ["write"]}
 object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
-indexmap = "1.0.2"
+indexmap = "1.8.0"
 libloading = { version = "0.6.0", optional = true }
 smallvec = "1.6.1"
 

From 96dc846a7d6e5827477ded238afef7c01af9e6f6 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 16:52:18 +0100
Subject: [PATCH 41/84] Rustup to rustc 1.60.0-nightly (a00e130da 2022-01-29)

---
 build_sysroot/Cargo.lock                             | 12 ++++++------
 .../0022-sysroot-Disable-not-compiling-tests.patch   |  2 +-
 rust-toolchain                                       |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index dd09656248022..aa69e032fa8a3 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -110,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.11.2"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.112"
+version = "0.2.116"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
+checksum = "565dbd88872dbe4cc8a46e527f26483c1d1f7afa6b884a3bd6cd893d4f98da74"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -319,9 +319,9 @@ dependencies = [
 
 [[package]]
 name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
+version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
diff --git a/patches/0022-sysroot-Disable-not-compiling-tests.patch b/patches/0022-sysroot-Disable-not-compiling-tests.patch
index ff8d5e8742bbb..1c45c7573c813 100644
--- a/patches/0022-sysroot-Disable-not-compiling-tests.patch
+++ b/patches/0022-sysroot-Disable-not-compiling-tests.patch
@@ -44,7 +44,7 @@ index 1a6be3a..42dbd59 100644
 --- a/library/core/tests/ptr.rs
 +++ b/library/core/tests/ptr.rs
 @@ -250,6 +250,7 @@ fn test_unsized_nonnull() {
-     assert!(ys == zs);
+     };
  }
  
 +/*
diff --git a/rust-toolchain b/rust-toolchain
index c42f8e5f45569..bf316efc324b9 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-01-19"
+channel = "nightly-2022-01-30"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From 3ff158e3ed06f9d16d7a0067952642751eb176e1 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 16:57:30 +0100
Subject: [PATCH 42/84] Update dependencies

---
 Cargo.lock | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c2ff264319207..77516aeaaff31 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "anyhow"
-version = "1.0.51"
+version = "1.0.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
+checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0"
 
 [[package]]
 name = "ar"
@@ -145,9 +145,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.3.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836"
+checksum = "a2209c310e29876f7f0b2721e7e26b84aff178aa3da5d091f9bfbf47669e60e3"
 dependencies = [
  "cfg-if",
 ]
@@ -179,9 +179,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.112"
+version = "0.2.116"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
+checksum = "565dbd88872dbe4cc8a46e527f26483c1d1f7afa6b884a3bd6cd893d4f98da74"
 
 [[package]]
 name = "libloading"
@@ -278,9 +278,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
+checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
 
 [[package]]
 name = "target-lexicon"

From 1b8ea0705e235828e86d85e9707cea14a034fb19 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 17:26:53 +0100
Subject: [PATCH 43/84] Add const_allocate and const_deallocate intrinsics

---
 src/intrinsics/mod.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 1756cf78c2d88..693b0e21444df 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -1020,6 +1020,16 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
         };
 
+        const_allocate, (c _size, c _align) {
+            // returns a null pointer at runtime.
+            let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
+            ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
+        };
+
+        const_deallocate, (c _ptr, c _size, c _align) {
+            // nop at runtime.
+        };
+
         black_box, (c a) {
             // FIXME implement black_box semantics
             ret.write_cvalue(fx, a);

From 246998f5ecd6e5b57a6a2707f0e899dea6352dee Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 17:43:09 +0100
Subject: [PATCH 44/84] Remove some unused lint allows

---
 src/intrinsics/llvm.rs |  4 ++--
 src/intrinsics/mod.rs  | 25 +++++++++----------------
 src/intrinsics/simd.rs |  4 ++--
 3 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs
index 20f8699d12abd..5bbbc18b3f432 100644
--- a/src/intrinsics/llvm.rs
+++ b/src/intrinsics/llvm.rs
@@ -8,14 +8,14 @@ use rustc_middle::ty::subst::SubstsRef;
 pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: &str,
-    substs: SubstsRef<'tcx>,
+    _substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 ) {
     let ret = destination.unwrap().0;
 
     intrinsic_match! {
-        fx, intrinsic, substs, args,
+        fx, intrinsic, args,
         _ => {
             fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
             crate::trap::trap_unimplemented(fx, intrinsic);
diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 693b0e21444df..825b317d131a6 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -43,28 +43,21 @@ macro intrinsic_arg {
 }
 
 macro intrinsic_match {
-    ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
+    ($fx:expr, $intrinsic:expr, $args:expr,
     _ => $unknown:block;
     $(
         $($($name:tt).*)|+ $(if $cond:expr)?, ($($a:ident $arg:ident),*) $content:block;
     )*) => {
-        let _ = $substs; // Silence warning when substs is unused.
         match $intrinsic {
             $(
                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
-                    #[allow(unused_parens, non_snake_case)]
-                    {
-                        if let [$($arg),*] = $args {
-                            let ($($arg,)*) = (
-                                $(intrinsic_arg!($a $fx, $arg),)*
-                            );
-                            #[warn(unused_parens, non_snake_case)]
-                            {
-                                $content
-                            }
-                        } else {
-                            bug!("wrong number of args for intrinsic {:?}", $intrinsic);
-                        }
+                    if let [$($arg),*] = $args {
+                        let ($($arg,)*) = (
+                            $(intrinsic_arg!($a $fx, $arg),)*
+                        );
+                        $content
+                    } else {
+                        bug!("wrong number of args for intrinsic {:?}", $intrinsic);
                     }
                 }
             )*
@@ -357,7 +350,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 
     intrinsic_match! {
-        fx, intrinsic, substs, args,
+        fx, intrinsic, args,
         _ => {
             fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
         };
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 106a190096dba..292d3ca45f184 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -68,13 +68,13 @@ macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: Symbol,
-    substs: SubstsRef<'tcx>,
+    _substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
     span: Span,
 ) {
     intrinsic_match! {
-        fx, intrinsic, substs, args,
+        fx, intrinsic, args,
         _ => {
             fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
         };

From 5efd7782b33e396a8206fb7c919bede2a3a03a75 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 18:00:36 +0100
Subject: [PATCH 45/84] Don't generate unnecessary let $arg=$arg for intrinsics

---
 src/intrinsics/llvm.rs |  4 ++--
 src/intrinsics/mod.rs  | 12 ++++--------
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs
index 5bbbc18b3f432..098862b0662f3 100644
--- a/src/intrinsics/llvm.rs
+++ b/src/intrinsics/llvm.rs
@@ -52,8 +52,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, res);
         };
         "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) {
-            let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
-            let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
+            let kind = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
+            let flt_cc = match kind.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) {
                 0 => FloatCC::Equal,
                 1 => FloatCC::LessThan,
                 2 => FloatCC::LessThanOrEqual,
diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 825b317d131a6..3429a7a03aad5 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -31,14 +31,12 @@ macro intrinsic_pat {
 }
 
 macro intrinsic_arg {
-    (o $fx:expr, $arg:ident) => {
-        $arg
-    },
+    (o $fx:expr, $arg:ident) => {},
     (c $fx:expr, $arg:ident) => {
-        codegen_operand($fx, $arg)
+        let $arg = codegen_operand($fx, $arg);
     },
     (v $fx:expr, $arg:ident) => {
-        codegen_operand($fx, $arg).load_scalar($fx)
+        let $arg = codegen_operand($fx, $arg).load_scalar($fx);
     }
 }
 
@@ -52,9 +50,7 @@ macro intrinsic_match {
             $(
                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
                     if let [$($arg),*] = $args {
-                        let ($($arg,)*) = (
-                            $(intrinsic_arg!($a $fx, $arg),)*
-                        );
+                        $(intrinsic_arg!($a $fx, $arg);)*
                         $content
                     } else {
                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);

From bb1b5cdde26cc5055fb6b3f1a5ff22a1e16db72b Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 18:23:01 +0100
Subject: [PATCH 46/84] Remove validate_atomic_type

By expanding it in place. Also extract a common
report_atomic_type_validation_error function to reduce code duplication.
---
 src/intrinsics/mod.rs | 144 +++++++++++++++++++++++++++++++++---------
 1 file changed, 114 insertions(+), 30 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 3429a7a03aad5..a93887a76052d 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -62,22 +62,21 @@ macro intrinsic_match {
     }
 }
 
-macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
-    match $ty.kind() {
-        ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
-        _ => {
-            $fx.tcx.sess.span_err(
-                $span,
-                &format!(
-                    "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
-                    $intrinsic, $ty
-                ),
-            );
-            // Prevent verifier error
-            crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
-            return;
-        }
-    }
+fn report_atomic_type_validation_error<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    intrinsic: Symbol,
+    span: Span,
+    ty: Ty<'tcx>,
+) {
+    fx.tcx.sess.span_err(
+        span,
+        &format!(
+            "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
+            intrinsic, ty
+        ),
+    );
+    // Prevent verifier error
+    crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
 }
 
 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
@@ -743,7 +742,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) {
             let ty = substs.type_at(0);
-            validate_atomic_type!(fx, intrinsic, span, ty);
+            match ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, ty);
+                    return;
+                }
+            }
             let clif_ty = fx.clif_type(ty).unwrap();
 
             let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
@@ -752,7 +757,14 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, val);
         };
         _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
-            validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
+            let ty = substs.type_at(0);
+            match ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, ty);
+                    return;
+                }
+            }
 
             let val = val.load_scalar(fx);
 
@@ -760,7 +772,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) {
             let layout = new.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let new = new.load_scalar(fx);
@@ -772,7 +790,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
             let layout = new.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
 
             let test_old = test_old.load_scalar(fx);
             let new = new.load_scalar(fx);
@@ -786,7 +810,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
         _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) {
             let layout = amount.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let amount = amount.load_scalar(fx);
@@ -798,7 +828,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) {
             let layout = amount.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let amount = amount.load_scalar(fx);
@@ -810,7 +846,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -822,7 +864,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -834,7 +882,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -846,7 +900,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -858,7 +918,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -870,7 +936,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -882,7 +954,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);
@@ -894,7 +972,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
         _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) {
             let layout = src.layout();
-            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            match layout.ty.kind() {
+                ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
+                _ => {
+                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    return;
+                }
+            }
             let ty = fx.clif_type(layout.ty).unwrap();
 
             let src = src.load_scalar(fx);

From 037aafbbaf2ee41a11807a1abdec24eb23f505c2 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 18:35:08 +0100
Subject: [PATCH 47/84] Fix simd type validation

---
 src/intrinsics/simd.rs | 227 +++++++++++++++++++++++++++++------------
 1 file changed, 161 insertions(+), 66 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 292d3ca45f184..644d379d0fad4 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -6,16 +6,23 @@ use rustc_span::Symbol;
 use super::*;
 use crate::prelude::*;
 
-fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span: Span, ty: Ty<'_>) {
-    if !ty.is_simd() {
-        fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
-        // Prevent verifier error
-        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+fn report_simd_type_validation_error(
+    fx: &mut FunctionCx<'_, '_, '_>,
+    intrinsic: Symbol,
+    span: Span,
+    ty: Ty<'_>,
+) {
+    fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
+    // Prevent verifier error
+    crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+}
+
+macro simd_cmp($fx:expr, $intrinsic:ident, $span:ident, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
         return;
     }
-}
 
-macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
         let res_lane = match lane_ty.kind() {
@@ -32,7 +39,12 @@ macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident)
     });
 }
 
-macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
+macro simd_int_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
+        return;
+    }
+
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
         match lane_ty.kind() {
@@ -43,7 +55,12 @@ macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $r
     });
 }
 
-macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
+macro simd_int_flt_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
+        return;
+    }
+
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
         match lane_ty.kind() {
@@ -55,7 +72,12 @@ macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident,
     });
 }
 
-macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
+macro simd_flt_binop($fx:expr, $intrinsic:ident, $span:ident, $op:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
+        return;
+    }
+
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
         match lane_ty.kind() {
@@ -80,7 +102,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_cast, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
                 let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
 
@@ -92,29 +118,25 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_eq, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, Equal|Equal|Equal(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, Equal|Equal|Equal(x, y) -> ret);
         };
         simd_ne, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, NotEqual|NotEqual|NotEqual(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, NotEqual|NotEqual|NotEqual(x, y) -> ret);
         };
         simd_lt, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
         };
         simd_le, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
         };
         simd_gt, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
         };
         simd_ge, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(
                 fx,
+                intrinsic,
+                span,
                 UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
                 (x, y) -> ret
             );
@@ -122,7 +144,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
         _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
 
             // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
             // If there is no suffix, use the index array length.
@@ -224,7 +249,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_extract, (c v, o idx) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                 idx_const
             } else {
@@ -252,7 +281,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_neg, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                 match lane_ty.kind() {
                     ty::Int(_) => fx.bcx.ins().ineg(lane),
@@ -263,37 +296,45 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fabs, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().fabs(lane)
             });
         };
 
         simd_fsqrt, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().sqrt(lane)
             });
         };
 
         simd_add, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, iadd|iadd|fadd(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, iadd|iadd|fadd(x, y) -> ret);
         };
         simd_sub, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, isub|isub|fsub(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, isub|isub|fsub(x, y) -> ret);
         };
         simd_mul, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, imul|imul|fmul(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, imul|imul|fmul(x, y) -> ret);
         };
         simd_div, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, udiv|sdiv|fdiv(x, y) -> ret);
         };
         simd_rem, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
+
             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
                 match lane_ty.kind() {
                     ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
@@ -315,28 +356,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
         simd_shl, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, ishl|ishl(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, ishl|ishl(x, y) -> ret);
         };
         simd_shr, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, ushr|sshr(x, y) -> ret);
         };
         simd_and, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, band|band(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, band|band(x, y) -> ret);
         };
         simd_or, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, bor|bor(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, bor|bor(x, y) -> ret);
         };
         simd_xor, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, bxor|bxor(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, bxor|bxor(x, y) -> ret);
         };
 
         simd_fma, (c a, c b, c c) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
             assert_eq!(a.layout(), b.layout());
             assert_eq!(a.layout(), c.layout());
             let layout = a.layout();
@@ -359,16 +398,18 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fmin, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_flt_binop!(fx, fmin(x, y) -> ret);
+            simd_flt_binop!(fx, intrinsic, span, fmin(x, y) -> ret);
         };
         simd_fmax, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_flt_binop!(fx, fmax(x, y) -> ret);
+            simd_flt_binop!(fx, intrinsic, span, fmax(x, y) -> ret);
         };
 
         simd_round, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                 match lane_ty.kind() {
                     ty::Float(FloatTy::F32) => fx.lib_call(
@@ -388,26 +429,42 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
         simd_ceil, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().ceil(lane)
             });
         };
         simd_floor, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().floor(lane)
             });
         };
         simd_trunc, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().trunc(lane)
             });
         };
 
         simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
                 if lane_ty.is_floating_point() {
                     fx.bcx.ins().fadd(a, b)
@@ -418,7 +475,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
                 if lane_ty.is_floating_point() {
                     fx.bcx.ins().fmul(a, b)
@@ -429,32 +490,56 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_all, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_any, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_and, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_or, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_xor, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
         };
 
         simd_reduce_min, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
                 let lt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
@@ -467,7 +552,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_max, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
                 let gt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
@@ -480,8 +569,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_select, (c m, c a, c b) {
-            validate_simd_type(fx, intrinsic, span, m.layout().ty);
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !m.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
+                return;
+            }
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
             assert_eq!(a.layout(), b.layout());
 
             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);

From 1ae27ea6aaf68ddc1e2b7812cbe1f356d13711c7 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 19:19:54 +0100
Subject: [PATCH 48/84] Remove simd_cmp macro

This reduces duplication in the codegened source file
---
 src/intrinsics/simd.rs | 100 ++++++++++++++++++++++-------------------
 1 file changed, 55 insertions(+), 45 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 644d379d0fad4..e0687b1d4eff2 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -17,28 +17,6 @@ fn report_simd_type_validation_error(
     crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
 }
 
-macro simd_cmp($fx:expr, $intrinsic:ident, $span:ident, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
-    if !$x.layout().ty.is_simd() {
-        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
-        return;
-    }
-
-    // FIXME use vector instructions when possible
-    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
-        let res_lane = match lane_ty.kind() {
-            ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
-            ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
-            ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
-            _ => unreachable!("{:?}", lane_ty),
-        };
-
-        let ty = fx.clif_type(res_lane_ty).unwrap();
-
-        let res_lane = fx.bcx.ins().bint(ty, res_lane);
-        fx.bcx.ins().ineg(res_lane)
-    });
-}
-
 macro simd_int_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
     if !$x.layout().ty.is_simd() {
         report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
@@ -117,29 +95,61 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
 
-        simd_eq, (c x, c y) {
-            simd_cmp!(fx, intrinsic, span, Equal|Equal|Equal(x, y) -> ret);
-        };
-        simd_ne, (c x, c y) {
-            simd_cmp!(fx, intrinsic, span, NotEqual|NotEqual|NotEqual(x, y) -> ret);
-        };
-        simd_lt, (c x, c y) {
-            simd_cmp!(fx, intrinsic, span, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
-        };
-        simd_le, (c x, c y) {
-            simd_cmp!(fx, intrinsic, span, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
-        };
-        simd_gt, (c x, c y) {
-            simd_cmp!(fx, intrinsic, span, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
-        };
-        simd_ge, (c x, c y) {
-            simd_cmp!(
-                fx,
-                intrinsic,
-                span,
-                UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
-                (x, y) -> ret
-            );
+        simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) {
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
+
+            // FIXME use vector instructions when possible
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
+                let res_lane = match (lane_ty.kind(), intrinsic) {
+                    (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
+                    (ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
+                    (ty::Uint(_), sym::simd_lt) => {
+                        fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
+                    }
+                    (ty::Uint(_), sym::simd_le) => {
+                        fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
+                    }
+                    (ty::Uint(_), sym::simd_gt) => {
+                        fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
+                    }
+                    (ty::Uint(_), sym::simd_ge) => {
+                        fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
+                    }
+
+                    (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
+                    (ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
+                    (ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
+                    (ty::Int(_), sym::simd_le) => {
+                        fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
+                    }
+                    (ty::Int(_), sym::simd_gt) => {
+                        fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
+                    }
+                    (ty::Int(_), sym::simd_ge) => {
+                        fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
+                    }
+
+                    (ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
+                    (ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
+                    (ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
+                    (ty::Float(_), sym::simd_le) => {
+                        fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
+                    }
+                    (ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
+                    (ty::Float(_), sym::simd_ge) => {
+                        fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
+                    }
+                    _ => unreachable!(),
+                };
+
+                let ty = fx.clif_type(res_lane_ty).unwrap();
+
+                let res_lane = fx.bcx.ins().bint(ty, res_lane);
+                fx.bcx.ins().ineg(res_lane)
+            });
         };
 
         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U

From c1d699d37b0d8a9b67c6c02234383c3302517d59 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 19:31:18 +0100
Subject: [PATCH 49/84] Remove the remaining simd intrinsic macros

---
 src/intrinsics/simd.rs | 150 ++++++++++++++++++++---------------------
 1 file changed, 72 insertions(+), 78 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index e0687b1d4eff2..893d809a47e22 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -17,54 +17,6 @@ fn report_simd_type_validation_error(
     crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
 }
 
-macro simd_int_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
-    if !$x.layout().ty.is_simd() {
-        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
-        return;
-    }
-
-    // FIXME use vector instructions when possible
-    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
-        match lane_ty.kind() {
-            ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-            ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-            _ => unreachable!("{:?}", lane_ty),
-        }
-    });
-}
-
-macro simd_int_flt_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
-    if !$x.layout().ty.is_simd() {
-        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
-        return;
-    }
-
-    // FIXME use vector instructions when possible
-    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
-        match lane_ty.kind() {
-            ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
-            ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
-            ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
-            _ => unreachable!("{:?}", lane_ty),
-        }
-    });
-}
-
-macro simd_flt_binop($fx:expr, $intrinsic:ident, $span:ident, $op:ident($x:ident, $y:ident) -> $ret:ident) {
-    if !$x.layout().ty.is_simd() {
-        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
-        return;
-    }
-
-    // FIXME use vector instructions when possible
-    simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
-        match lane_ty.kind() {
-            ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
-            _ => unreachable!("{:?}", lane_ty),
-        }
-    });
-}
-
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: Symbol,
@@ -142,6 +94,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                     (ty::Float(_), sym::simd_ge) => {
                         fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
                     }
+
                     _ => unreachable!(),
                 };
 
@@ -327,17 +280,34 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
 
-        simd_add, (c x, c y) {
-            simd_int_flt_binop!(fx, intrinsic, span, iadd|iadd|fadd(x, y) -> ret);
-        };
-        simd_sub, (c x, c y) {
-            simd_int_flt_binop!(fx, intrinsic, span, isub|isub|fsub(x, y) -> ret);
-        };
-        simd_mul, (c x, c y) {
-            simd_int_flt_binop!(fx, intrinsic, span, imul|imul|fmul(x, y) -> ret);
-        };
-        simd_div, (c x, c y) {
-            simd_int_flt_binop!(fx, intrinsic, span, udiv|sdiv|fdiv(x, y) -> ret);
+        simd_add | simd_sub | simd_mul | simd_div, (c x, c y) {
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
+
+            // FIXME use vector instructions when possible
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
+                lane_ty.kind(),
+                intrinsic,
+            ) {
+                (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
+
+                (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
+                (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
+                (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
+                (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
+
+                (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
+                (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
+                (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
+                (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
+
+                _ => unreachable!(),
+            });
         };
         simd_rem, (c x, c y) {
             if !x.layout().ty.is_simd() {
@@ -365,20 +335,31 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 }
             });
         };
-        simd_shl, (c x, c y) {
-            simd_int_binop!(fx, intrinsic, span, ishl|ishl(x, y) -> ret);
-        };
-        simd_shr, (c x, c y) {
-            simd_int_binop!(fx, intrinsic, span, ushr|sshr(x, y) -> ret);
-        };
-        simd_and, (c x, c y) {
-            simd_int_binop!(fx, intrinsic, span, band|band(x, y) -> ret);
-        };
-        simd_or, (c x, c y) {
-            simd_int_binop!(fx, intrinsic, span, bor|bor(x, y) -> ret);
-        };
-        simd_xor, (c x, c y) {
-            simd_int_binop!(fx, intrinsic, span, bxor|bxor(x, y) -> ret);
+        simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
+
+            // FIXME use vector instructions when possible
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
+                lane_ty.kind(),
+                intrinsic,
+            ) {
+                (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
+
+                (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
+                (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
+                (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
+                (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
+                (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
+
+                _ => unreachable!(),
+            });
         };
 
         simd_fma, (c a, c b, c c) {
@@ -407,11 +388,24 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
         };
 
-        simd_fmin, (c x, c y) {
-            simd_flt_binop!(fx, intrinsic, span, fmin(x, y) -> ret);
-        };
-        simd_fmax, (c x, c y) {
-            simd_flt_binop!(fx, intrinsic, span, fmax(x, y) -> ret);
+        simd_fmin | simd_fmax, (c x, c y) {
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
+
+            // FIXME use vector instructions when possible
+            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
+                match lane_ty.kind() {
+                    ty::Float(_) => {},
+                    _ => unreachable!("{:?}", lane_ty),
+                }
+                match intrinsic {
+                    sym::simd_fmin => fx.bcx.ins().fmin(x_lane, y_lane),
+                    sym::simd_fmax => fx.bcx.ins().fmax(x_lane, y_lane),
+                    _ => unreachable!(),
+                }
+            });
         };
 
         simd_round, (c a) {

From 5a3cfb24d8db7818bf843b5aed316cd1e4b06fda Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 19:37:01 +0100
Subject: [PATCH 50/84] Merge codegen of several simd intrinsics

This reduces code duplication
---
 src/intrinsics/simd.rs | 57 ++++++++++++------------------------------
 1 file changed, 16 insertions(+), 41 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 893d809a47e22..b4aa5c48ccb0a 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -280,7 +280,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
 
-        simd_add | simd_sub | simd_mul | simd_div, (c x, c y) {
+        simd_add | simd_sub | simd_mul | simd_div | simd_rem
+        | simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
             if !x.layout().ty.is_simd() {
                 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
                 return;
@@ -295,57 +296,31 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
                 (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
                 (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
+                (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
 
                 (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
                 (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
                 (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
                 (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
+                (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
 
                 (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
                 (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
                 (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
                 (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
+                (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
+                    "fmodf",
+                    vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
+                    vec![AbiParam::new(types::F32)],
+                    &[x_lane, y_lane],
+                )[0],
+                (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
+                    "fmod",
+                    vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
+                    vec![AbiParam::new(types::F64)],
+                    &[x_lane, y_lane],
+                )[0],
 
-                _ => unreachable!(),
-            });
-        };
-        simd_rem, (c x, c y) {
-            if !x.layout().ty.is_simd() {
-                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
-                return;
-            }
-
-            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
-                match lane_ty.kind() {
-                    ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
-                    ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane),
-                    ty::Float(FloatTy::F32) => fx.lib_call(
-                        "fmodf",
-                        vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
-                        vec![AbiParam::new(types::F32)],
-                        &[x_lane, y_lane],
-                    )[0],
-                    ty::Float(FloatTy::F64) => fx.lib_call(
-                        "fmod",
-                        vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
-                        vec![AbiParam::new(types::F64)],
-                        &[x_lane, y_lane],
-                    )[0],
-                    _ => unreachable!("{:?}", lane_ty),
-                }
-            });
-        };
-        simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
-            if !x.layout().ty.is_simd() {
-                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
-                return;
-            }
-
-            // FIXME use vector instructions when possible
-            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
-                lane_ty.kind(),
-                intrinsic,
-            ) {
                 (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
                 (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
                 (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),

From bccf0a1f8dc1b043ec5c8e6cb0851406b9008fa4 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 30 Jan 2022 19:44:15 +0100
Subject: [PATCH 51/84] Merge codegen of a couple more simd intrinsics

---
 src/intrinsics/simd.rs | 58 ++++++++++--------------------------------
 1 file changed, 14 insertions(+), 44 deletions(-)

diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index b4aa5c48ccb0a..4153bc08e8ad4 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -258,28 +258,6 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
 
-        simd_fabs, (c a) {
-            if !a.layout().ty.is_simd() {
-                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
-                return;
-            }
-
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
-                fx.bcx.ins().fabs(lane)
-            });
-        };
-
-        simd_fsqrt, (c a) {
-            if !a.layout().ty.is_simd() {
-                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
-                return;
-            }
-
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
-                fx.bcx.ins().sqrt(lane)
-            });
-        };
-
         simd_add | simd_sub | simd_mul | simd_div | simd_rem
         | simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
             if !x.layout().ty.is_simd() {
@@ -407,34 +385,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 }
             });
         };
-        simd_ceil, (c a) {
-            if !a.layout().ty.is_simd() {
-                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
-                return;
-            }
 
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
-                fx.bcx.ins().ceil(lane)
-            });
-        };
-        simd_floor, (c a) {
-            if !a.layout().ty.is_simd() {
-                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
-                return;
-            }
-
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
-                fx.bcx.ins().floor(lane)
-            });
-        };
-        simd_trunc, (c a) {
+        simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) {
             if !a.layout().ty.is_simd() {
                 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                 return;
             }
 
-            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
-                fx.bcx.ins().trunc(lane)
+            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
+                match lane_ty.kind() {
+                    ty::Float(_) => {},
+                    _ => unreachable!("{:?}", lane_ty),
+                }
+                match intrinsic {
+                    sym::simd_fabs => fx.bcx.ins().fabs(lane),
+                    sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
+                    sym::simd_ceil => fx.bcx.ins().ceil(lane),
+                    sym::simd_floor => fx.bcx.ins().floor(lane),
+                    sym::simd_trunc => fx.bcx.ins().trunc(lane),
+                    _ => unreachable!(),
+                }
             });
         };
 

From 441e2e67fa47f2495bd1642e546b073724c76704 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 6 Feb 2022 14:50:12 +0100
Subject: [PATCH 52/84] Rustup to rustc 1.60.0-nightly (88fb06a1f 2022-02-05)

---
 build_sysroot/Cargo.lock                         |  6 +++---
 build_system/prepare.rs                          |  2 +-
 ...iltins-Disable-128bit-atomic-operations.patch | 16 ++++++++--------
 rust-toolchain                                   |  2 +-
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index aa69e032fa8a3..4076dc32d552f 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -56,7 +56,7 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.66"
+version = "0.1.68"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.116"
+version = "0.2.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "565dbd88872dbe4cc8a46e527f26483c1d1f7afa6b884a3bd6cd893d4f98da74"
+checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c"
 dependencies = [
  "rustc-std-workspace-core",
 ]
diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 76ea506b6ac4f..2addd58900780 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -92,7 +92,7 @@ fn prepare_sysroot() {
     clone_repo(
         "build_sysroot/compiler-builtins",
         "https://github.com/rust-lang/compiler-builtins.git",
-        "0.1.66",
+        "0.1.68",
     );
     apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
 }
diff --git a/patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch b/patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch
index 7daea99f5794d..460e42d1d8cc3 100644
--- a/patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch
+++ b/patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch
@@ -13,33 +13,33 @@ index 107762c..2d1ae10 100644
 --- a/src/mem/mod.rs
 +++ b/src/mem/mod.rs
 @@ -137,10 +137,6 @@ intrinsics! {
-     pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+     pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
          memcpy_element_unordered_atomic(dest, src, bytes);
      }
 -    #[cfg(target_has_atomic_load_store = "128")]
--    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+-    pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
 -        memcpy_element_unordered_atomic(dest, src, bytes);
 -    }
  
      #[cfg(target_has_atomic_load_store = "8")]
-     pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+     pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
 @@ -158,10 +154,6 @@ intrinsics! {
-     pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+     pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
          memmove_element_unordered_atomic(dest, src, bytes);
      }
 -    #[cfg(target_has_atomic_load_store = "128")]
--    pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+-    pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
 -        memmove_element_unordered_atomic(dest, src, bytes);
 -    }
  
      #[cfg(target_has_atomic_load_store = "8")]
-     pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
+     pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
 @@ -179,8 +171,4 @@ intrinsics! {
-     pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
+     pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
          memset_element_unordered_atomic(s, c, bytes);
      }
 -    #[cfg(target_has_atomic_load_store = "128")]
--    pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
+-    pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
 -        memset_element_unordered_atomic(s, c, bytes);
 -    }
  }
diff --git a/rust-toolchain b/rust-toolchain
index bf316efc324b9..90dbee76e9c0a 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-01-30"
+channel = "nightly-2022-02-06"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From 583333a2b2d50219580a2a0b8553b87b954d2df7 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 6 Feb 2022 15:13:29 +0100
Subject: [PATCH 53/84] Don't try to reinstall ripgrep if it is already
 installed

---
 scripts/test_rustc_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh
index a9a44d2c654a6..efb4236e07253 100755
--- a/scripts/test_rustc_tests.sh
+++ b/scripts/test_rustc_tests.sh
@@ -8,7 +8,7 @@ source ./scripts/setup_rust_fork.sh
 echo "[TEST] Test suite of rustc"
 pushd rust
 
-cargo install ripgrep
+command -v rg >/dev/null 2>&1 || cargo install ripgrep
 
 rm -r src/test/ui/{extern/,panics/,unsized-locals/,lto/,simd*,linkage*,unwind-*.rs} || true
 for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto|// needs-asm-support" src/test/ui); do

From 75a463a583d9b74ec73a8308e77c6adb17fbd0a2 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 8 Feb 2022 18:24:50 +0100
Subject: [PATCH 54/84] Update Cranelift to 0.81.0

---
 Cargo.lock        | 44 ++++++++++++++++++++++----------------------
 Cargo.toml        | 12 ++++++------
 src/allocator.rs  |  9 ++-------
 src/base.rs       |  5 +----
 src/driver/jit.rs | 10 +---------
 src/main_shim.rs  |  4 +---
 6 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 77516aeaaff31..f15e319e3b807 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -33,18 +33,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9516ba6b2ba47b4cbf63b713f75b432fafa0a0e0464ec8381ec76e6efe931ab3"
+checksum = "71447555acc6c875c52c407d572fc1327dc5c34cba72b4b2e7ad048aa4e4fd19"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "489e5d0081f7edff6be12d71282a8bf387b5df64d5592454b75d662397f2d642"
+checksum = "ec9a10261891a7a919b0d4f6aa73582e88441d9a8f6173c88efbe4a5a362ea67"
 dependencies = [
  "cranelift-bforest",
  "cranelift-codegen-meta",
@@ -59,30 +59,30 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d36ee1140371bb0f69100e734b30400157a4adf7b86148dee8b0a438763ead48"
+checksum = "815755d76fcbcf6e17ab888545b28ab775f917cb12ce0797e60cd41a2288692c"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "981da52d8f746af1feb96290c83977ff8d41071a7499e991d8abae0d4869f564"
+checksum = "23ea92f2a67335a2e4d3c9c65624c3b14ae287d595b0650822c41824febab66b"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2906740053dd3bcf95ce53df0fd9b5649c68ae4bd9adada92b406f059eae461"
+checksum = "bd25847875e388c500ad3624b4d2e14067955c93185194a7222246a25b91c975"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7cb156de1097f567d46bf57a0cd720a72c3e15e1a2bd8b1041ba2fc894471b7"
+checksum = "308bcfb7eb47bdf5ff6e1ace262af4ed39ec19f204c751fffb037e0e82a0c8bf"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -92,9 +92,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e0f0e20dbcac1e6c3caef955e004598a9a6a5f310e591e2c629ec15c7fa6bfa"
+checksum = "f560b3a314b8d15facf411e5d29b917c3e787a2bbc3fcdc5183bc0c5b7d4fe01"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -110,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93460fc789770f2a63163bfb5f2b851635ce29d91526f2e96854bcc4ed53a778"
+checksum = "3a57aba9e603d694d1430ff38bd914bae23ef9c2e44b25a65e318905807e654c"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -120,9 +120,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "166028ca0343a6ee7bddac0e70084e142b23f99c701bd6f6ea9123afac1a7a46"
+checksum = "12cdc799aee673be2317e631d4569a1ba0a7e77a07a7ce45557086d2e02e9514"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -131,9 +131,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.80.0"
+version = "0.81.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "263618594523ad29349fd43d93bfeb04d2bbdfd5df9fcf04190582dbdcff3220"
+checksum = "502a7333836052fcdf4425d7f7a21264d99f862d32b9c3a0e47cd920487a9b60"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -230,9 +230,9 @@ dependencies = [
 
 [[package]]
 name = "regalloc"
-version = "0.0.33"
+version = "0.0.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a"
+checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02"
 dependencies = [
  "log",
  "rustc-hash",
diff --git a/Cargo.toml b/Cargo.toml
index 91f74269ea16e..178404af42d67 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.80.0", features = ["unwind", "all-arch"] }
-cranelift-frontend = "0.80.0"
-cranelift-module = "0.80.0"
-cranelift-native = "0.80.0"
-cranelift-jit = { version = "0.80.0", optional = true }
-cranelift-object = "0.80.0"
+cranelift-codegen = { version = "0.81.0", features = ["unwind", "all-arch"] }
+cranelift-frontend = "0.81.0"
+cranelift-module = "0.81.0"
+cranelift-native = "0.81.0"
+cranelift-jit = { version = "0.81.0", optional = true }
+cranelift-object = "0.81.0"
 target-lexicon = "0.12.0"
 gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
diff --git a/src/allocator.rs b/src/allocator.rs
index 637d30f9344f9..82247b47888e7 100644
--- a/src/allocator.rs
+++ b/src/allocator.rs
@@ -3,7 +3,6 @@
 
 use crate::prelude::*;
 
-use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 
 /// Returns whether an allocator shim was created
@@ -91,9 +90,7 @@ fn codegen_inner(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        module
-            .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
-            .unwrap();
+        module.define_function(func_id, &mut ctx).unwrap();
         unwind_context.add_function(func_id, &ctx, module.isa());
     }
 
@@ -130,8 +127,6 @@ fn codegen_inner(
         bcx.seal_all_blocks();
         bcx.finalize();
     }
-    module
-        .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
-        .unwrap();
+    module.define_function(func_id, &mut ctx).unwrap();
     unwind_context.add_function(func_id, &ctx, module.isa());
 }
diff --git a/src/base.rs b/src/base.rs
index ad009b5a3f2fb..1dc9cbda2d5eb 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -1,6 +1,5 @@
 //! Codegen of a single function
 
-use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::InlineAsmOptions;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
@@ -175,9 +174,7 @@ fn compile_fn<'tcx>(
     // Define function
     tcx.sess.time("define function", || {
         context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
-        module
-            .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
-            .unwrap()
+        module.define_function(func_id, context).unwrap()
     });
 
     // Write optimized function to file for debugging
diff --git a/src/driver/jit.rs b/src/driver/jit.rs
index 309d27090b5cb..9e07528313dd8 100644
--- a/src/driver/jit.rs
+++ b/src/driver/jit.rs
@@ -7,7 +7,6 @@ use std::lazy::SyncOnceCell;
 use std::os::raw::{c_char, c_int};
 use std::sync::{mpsc, Mutex};
 
-use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
 use rustc_session::Session;
@@ -381,12 +380,5 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
     trampoline_builder.ins().return_(&ret_vals);
 
-    module
-        .define_function(
-            func_id,
-            &mut cx.cached_context,
-            &mut NullTrapSink {},
-            &mut NullStackMapSink {},
-        )
-        .unwrap();
+    module.define_function(func_id, &mut cx.cached_context).unwrap();
 }
diff --git a/src/main_shim.rs b/src/main_shim.rs
index 8fd1e4f5811f5..9ce727279c27c 100644
--- a/src/main_shim.rs
+++ b/src/main_shim.rs
@@ -1,4 +1,3 @@
-use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
 use rustc_middle::ty::subst::GenericArg;
 use rustc_middle::ty::AssocKind;
@@ -152,8 +151,7 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
-            .unwrap();
+        m.define_function(cmain_func_id, &mut ctx).unwrap();
         unwind_context.add_function(cmain_func_id, &ctx, m.isa());
     }
 }

From d416c68114e11309b4a1a41cd267afa31a5b02c1 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 31 Jan 2022 19:55:34 +0100
Subject: [PATCH 55/84] Ensure that queries only return Copy types.

---
 src/common.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common.rs b/src/common.rs
index 3b6025c73d10b..79d8555451409 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -274,7 +274,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> {
     #[inline]
     fn handle_fn_abi_err(
         &self,
-        err: FnAbiError<'tcx>,
+        err: &'tcx FnAbiError<'tcx>,
         span: Span,
         fn_abi_request: FnAbiRequest<'tcx>,
     ) -> ! {
@@ -396,7 +396,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> {
     #[inline]
     fn handle_fn_abi_err(
         &self,
-        err: FnAbiError<'tcx>,
+        err: &'tcx FnAbiError<'tcx>,
         span: Span,
         fn_abi_request: FnAbiRequest<'tcx>,
     ) -> ! {

From cf9c65bb585a9bf73d32055ceff4d5c809385879 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 1 Feb 2022 18:44:45 +0100
Subject: [PATCH 56/84] Make FnAbiError Copy.

---
 src/common.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common.rs b/src/common.rs
index 79d8555451409..3b6025c73d10b 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -274,7 +274,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> {
     #[inline]
     fn handle_fn_abi_err(
         &self,
-        err: &'tcx FnAbiError<'tcx>,
+        err: FnAbiError<'tcx>,
         span: Span,
         fn_abi_request: FnAbiRequest<'tcx>,
     ) -> ! {
@@ -396,7 +396,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> {
     #[inline]
     fn handle_fn_abi_err(
         &self,
-        err: &'tcx FnAbiError<'tcx>,
+        err: FnAbiError<'tcx>,
         span: Span,
         fn_abi_request: FnAbiRequest<'tcx>,
     ) -> ! {

From 4e39cde7f3584b5bf5017346b53f8493f32de7f8 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Thu, 10 Feb 2022 18:27:18 +0100
Subject: [PATCH 57/84] Unconditionally update symbols

All paths to an ArchiveBuilder::build call update_symbols first.
---
 src/archive.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/archive.rs b/src/archive.rs
index b0eb3864d80c8..a099e8b3a6af3 100644
--- a/src/archive.rs
+++ b/src/archive.rs
@@ -105,8 +105,6 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
         Ok(())
     }
 
-    fn update_symbols(&mut self) {}
-
     fn build(mut self) {
         enum BuilderKind {
             Bsd(ar::Builder<File>),

From d4594457993600987b4370ab5b258d4276e12b37 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 13 Feb 2022 12:24:02 +0100
Subject: [PATCH 58/84] Rustup to rustc 1.60.0-nightly (5d8767cb2 2022-02-12)

---
 build_sysroot/Cargo.lock | 4 ++--
 rust-toolchain           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index 4076dc32d552f..6c453b7e6a214 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -34,9 +34,9 @@ dependencies = [
 
 [[package]]
 name = "autocfg"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "cc"
diff --git a/rust-toolchain b/rust-toolchain
index 90dbee76e9c0a..93b784079bdc0 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-02-06"
+channel = "nightly-2022-02-13"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From 6a20fa93b594a0d166402e811a033c6f1e153cf2 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 25 Jan 2022 14:13:38 +1100
Subject: [PATCH 59/84] Overhaul `TyS` and `Ty`.

Specifically, change `Ty` from this:
```
pub type Ty<'tcx> = &'tcx TyS<'tcx>;
```
to this
```
pub struct Ty<'tcx>(Interned<'tcx, TyS<'tcx>>);
```
There are two benefits to this.
- It's now a first class type, so we can define methods on it. This
  means we can move a lot of methods away from `TyS`, leaving `TyS` as a
  barely-used type, which is appropriate given that it's not meant to
  be used directly.
- The uniqueness requirement is now explicit, via the `Interned` type.
  E.g. the pointer-based `Eq` and `Hash` comes from `Interned`, rather
  than via `TyS`, which wasn't obvious at all.

Much of this commit is boring churn. The interesting changes are in
these files:
- compiler/rustc_middle/src/arena.rs
- compiler/rustc_middle/src/mir/visit.rs
- compiler/rustc_middle/src/ty/context.rs
- compiler/rustc_middle/src/ty/mod.rs

Specifically:
- Most mentions of `TyS` are removed. It's very much a dumb struct now;
  `Ty` has all the smarts.
- `TyS` now has `crate` visibility instead of `pub`.
- `TyS::make_for_test` is removed in favour of the static `BOOL_TY`,
  which just works better with the new structure.
- The `Eq`/`Ord`/`Hash` impls are removed from `TyS`. `Interned`s impls
  of `Eq`/`Hash` now suffice. `Ord` is now partly on `Interned`
  (pointer-based, for the `Equal` case) and partly on `TyS`
  (contents-based, for the other cases).
- There are many tedious sigil adjustments, i.e. adding or removing `*`
  or `&`. They seem to be unavoidable.
---
 src/base.rs            |  8 ++++----
 src/common.rs          |  4 ++--
 src/constant.rs        |  2 +-
 src/debuginfo/mod.rs   |  4 ++--
 src/unsize.rs          |  2 +-
 src/value_and_place.rs | 10 +++++-----
 6 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/base.rs b/src/base.rs
index 5a889734f215b..fe33a1f9b0928 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -79,7 +79,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let arg_uninhabited = fx
         .mir
         .args_iter()
-        .any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
+        .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
 
     if !crate::constant::check_constants(&mut fx) {
         fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
@@ -818,16 +818,16 @@ pub(crate) fn codegen_place<'tcx>(
                 match cplace.layout().ty.kind() {
                     ty::Array(elem_ty, _len) => {
                         assert!(!from_end, "array subslices are never `from_end`");
-                        let elem_layout = fx.layout_of(elem_ty);
+                        let elem_layout = fx.layout_of(*elem_ty);
                         let ptr = cplace.to_ptr();
                         cplace = CPlace::for_ptr(
                             ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)),
-                            fx.layout_of(fx.tcx.mk_array(elem_ty, to - from)),
+                            fx.layout_of(fx.tcx.mk_array(*elem_ty, to - from)),
                         );
                     }
                     ty::Slice(elem_ty) => {
                         assert!(from_end, "slice subslices should be `from_end`");
-                        let elem_layout = fx.layout_of(elem_ty);
+                        let elem_layout = fx.layout_of(*elem_ty);
                         let (ptr, len) = cplace.to_ptr_maybe_unsized();
                         let len = len.unwrap();
                         cplace = CPlace::for_ptr_with_extra(
diff --git a/src/common.rs b/src/common.rs
index 3b6025c73d10b..50f98965ab5d2 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -61,7 +61,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
         },
         ty::FnPtr(_) => pointer_ty(tcx),
         ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
-            if has_ptr_meta(tcx, pointee_ty) {
+            if has_ptr_meta(tcx, *pointee_ty) {
                 return None;
             } else {
                 pointer_ty(tcx)
@@ -100,7 +100,7 @@ fn clif_pair_type_from_ty<'tcx>(
             (a, b)
         }
         ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
-            if has_ptr_meta(tcx, pointee_ty) {
+            if has_ptr_meta(tcx, *pointee_ty) {
                 (pointer_ty(tcx), pointer_ty(tcx))
             } else {
                 return None;
diff --git a/src/constant.rs b/src/constant.rs
index 74571817969d3..9df6c7766c68f 100644
--- a/src/constant.rs
+++ b/src/constant.rs
@@ -490,7 +490,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                                         return None;
                                     }
                                     let const_val = mir_operand_get_const_val(fx, operand)?;
-                                    if fx.layout_of(ty).size
+                                    if fx.layout_of(*ty).size
                                         != const_val.try_to_scalar_int()?.size()
                                     {
                                         return None;
diff --git a/src/debuginfo/mod.rs b/src/debuginfo/mod.rs
index 8e203b8cfa063..693092ba543ea 100644
--- a/src/debuginfo/mod.rs
+++ b/src/debuginfo/mod.rs
@@ -114,7 +114,7 @@ impl<'tcx> DebugContext<'tcx> {
     }
 
     fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
-        if let Some(type_id) = self.types.get(ty) {
+        if let Some(type_id) = self.types.get(&ty) {
             return *type_id;
         }
 
@@ -143,7 +143,7 @@ impl<'tcx> DebugContext<'tcx> {
                 // Ensure that type is inserted before recursing to avoid duplicates
                 self.types.insert(ty, type_id);
 
-                let pointee = self.dwarf_ty(pointee_ty);
+                let pointee = self.dwarf_ty(*pointee_ty);
 
                 let type_entry = self.dwarf.unit.get_mut(type_id);
 
diff --git a/src/unsize.rs b/src/unsize.rs
index fd96858010ea9..8cae506e0cb62 100644
--- a/src/unsize.rs
+++ b/src/unsize.rs
@@ -66,7 +66,7 @@ fn unsize_ptr<'tcx>(
         (&ty::Ref(_, a, _), &ty::Ref(_, b, _))
         | (&ty::Ref(_, a, _), &ty::RawPtr(ty::TypeAndMut { ty: b, .. }))
         | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => {
-            (src, unsized_info(fx, a, b, old_info))
+            (src, unsized_info(fx, *a, *b, old_info))
         }
         (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) if def_a.is_box() && def_b.is_box() => {
             let (a, b) = (src_layout.ty.boxed_ty(), dst_layout.ty.boxed_ty());
diff --git a/src/value_and_place.rs b/src/value_and_place.rs
index f29d13ccabddd..b016af5174e01 100644
--- a/src/value_and_place.rs
+++ b/src/value_and_place.rs
@@ -514,7 +514,7 @@ impl<'tcx> CPlace<'tcx> {
                     // Can only happen for vector types
                     let len =
                         u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
-                    let vector_ty = fx.clif_type(element).unwrap().by(len).unwrap();
+                    let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
 
                     let data = match from.0 {
                         CValueInner::ByRef(ptr, None) => {
@@ -721,8 +721,8 @@ impl<'tcx> CPlace<'tcx> {
         index: Value,
     ) -> CPlace<'tcx> {
         let (elem_layout, ptr) = match self.layout().ty.kind() {
-            ty::Array(elem_ty, _) => (fx.layout_of(elem_ty), self.to_ptr()),
-            ty::Slice(elem_ty) => (fx.layout_of(elem_ty), self.to_ptr_maybe_unsized().0),
+            ty::Array(elem_ty, _) => (fx.layout_of(*elem_ty), self.to_ptr()),
+            ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0),
             _ => bug!("place_index({:?})", self.layout().ty),
         };
 
@@ -781,11 +781,11 @@ pub(crate) fn assert_assignable<'tcx>(
             ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }),
             ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }),
         ) => {
-            assert_assignable(fx, a, b);
+            assert_assignable(fx, *a, *b);
         }
         (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }))
         | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => {
-            assert_assignable(fx, a, b);
+            assert_assignable(fx, *a, *b);
         }
         (ty::FnPtr(_), ty::FnPtr(_)) => {
             let from_sig = fx.tcx.normalize_erasing_late_bound_regions(

From 18e7b7ece129d0393bb739cb4f9557689e184080 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 28 Jan 2022 11:25:15 +1100
Subject: [PATCH 60/84] Overhaul `RegionKind` and `Region`.

Specifically, change `Region` from this:
```
pub type Region<'tcx> = &'tcx RegionKind;
```
to this:
```
pub struct Region<'tcx>(&'tcx Interned<RegionKind>);
```

This now matches `Ty` and `Predicate` more closely.

Things to note
- Regions have always been interned, but we haven't been using pointer-based
  `Eq` and `Hash`. This is now happening.
- I chose to impl `Deref` for `Region` because it makes pattern matching a lot
  nicer, and `Region` can be viewed as just a smart wrapper for `RegionKind`.
- Various methods are moved from `RegionKind` to `Region`.
- There is a lot of tedious sigil changes.
- A couple of types like `HighlightBuilder`, `RegionHighlightMode` now have a
  `'tcx` lifetime because they hold a `Ty<'tcx>`, so they can call `mk_region`.
- A couple of test outputs change slightly, I'm not sure why, but the new
  outputs are a little better.
---
 src/abi/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/abi/mod.rs b/src/abi/mod.rs
index 72ebc84c1a344..a0550860fa545 100644
--- a/src/abi/mod.rs
+++ b/src/abi/mod.rs
@@ -544,7 +544,7 @@ pub(crate) fn codegen_drop<'tcx>(
                 let arg_value = drop_place.place_ref(
                     fx,
                     fx.layout_of(fx.tcx.mk_ref(
-                        &ty::RegionKind::ReErased,
+                        fx.tcx.lifetimes.re_erased,
                         TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut },
                     )),
                 );

From 06bc64df9213e928bf37c565f9c2f6b09391a384 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 2 Feb 2022 14:24:45 +1100
Subject: [PATCH 61/84] Overhaul `Const`.

Specifically, rename the `Const` struct as `ConstS` and re-introduce `Const` as
this:
```
pub struct Const<'tcx>(&'tcx Interned<ConstS>);
```
This now matches `Ty` and `Predicate` more closely, including using
pointer-based `eq` and `hash`.

Notable changes:
- `mk_const` now takes a `ConstS`.
- `Const` was copy, despite being 48 bytes. Now `ConstS` is not, so need a
  we need separate arena for it, because we can't use the `Dropless` one any
  more.
- Many `&'tcx Const<'tcx>`/`&Const<'tcx>` to `Const<'tcx>` changes
- Many `ct.ty` to `ct.ty()` and `ct.val` to `ct.val()` changes.
- Lots of tedious sigil fiddling.
---
 src/base.rs     |  2 +-
 src/constant.rs | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/base.rs b/src/base.rs
index fe33a1f9b0928..917afa4eae06c 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -668,7 +668,7 @@ fn codegen_stmt<'tcx>(
                     let times = fx
                         .monomorphize(times)
                         .eval(fx.tcx, ParamEnv::reveal_all())
-                        .val
+                        .val()
                         .try_to_bits(fx.tcx.data_layout.pointer_size)
                         .unwrap();
                     if operand.layout().size.bytes() == 0 {
diff --git a/src/constant.rs b/src/constant.rs
index 9df6c7766c68f..274fb211b7bbb 100644
--- a/src/constant.rs
+++ b/src/constant.rs
@@ -46,7 +46,7 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
             ConstantKind::Ty(ct) => ct,
             ConstantKind::Val(..) => continue,
         };
-        match const_.val {
+        match const_.val() {
             ConstKind::Value(_) => {}
             ConstKind::Unevaluated(unevaluated) => {
                 if let Err(err) =
@@ -127,7 +127,7 @@ pub(crate) fn codegen_constant<'tcx>(
         ConstantKind::Ty(ct) => ct,
         ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty),
     };
-    let const_val = match const_.val {
+    let const_val = match const_.val() {
         ConstKind::Value(const_val) => const_val,
         ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
             if fx.tcx.is_static(def.did) =>
@@ -135,7 +135,7 @@ pub(crate) fn codegen_constant<'tcx>(
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
-            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
+            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty())).to_cvalue(fx);
         }
         ConstKind::Unevaluated(unevaluated) => {
             match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
@@ -152,7 +152,7 @@ pub(crate) fn codegen_constant<'tcx>(
         | ConstKind::Error(_) => unreachable!("{:?}", const_),
     };
 
-    codegen_const_value(fx, const_val, const_.ty)
+    codegen_const_value(fx, const_val, const_.ty())
 }
 
 pub(crate) fn codegen_const_value<'tcx>(
@@ -465,7 +465,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
     match operand {
         Operand::Constant(const_) => match const_.literal {
             ConstantKind::Ty(const_) => {
-                fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value()
+                fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val().try_to_value()
             }
             ConstantKind::Val(val, _) => Some(val),
         },

From f606a5807cc954dc9f01f5a943f8479bdee96e09 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Wed, 16 Feb 2022 12:43:07 +0100
Subject: [PATCH 62/84] Rustup to rustc 1.60.0-nightly (09cb29c64 2022-02-15)

---
 build_sysroot/Cargo.lock                      |  6 ++--
 build_system/prepare.rs                       |  2 +-
 ...027-sysroot-128bit-atomic-operations.patch | 33 -------------------
 rust-toolchain                                |  2 +-
 4 files changed, 5 insertions(+), 38 deletions(-)

diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index 6c453b7e6a214..37b4f3efc9691 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -56,7 +56,7 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.68"
+version = "0.1.70"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.117"
+version = "0.2.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c"
+checksum = "06e509672465a0504304aa87f9f176f2b2b716ed8fb105ebe5c02dc6dce96a94"
 dependencies = [
  "rustc-std-workspace-core",
 ]
diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 2addd58900780..4a7df2cebbcb6 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -92,7 +92,7 @@ fn prepare_sysroot() {
     clone_repo(
         "build_sysroot/compiler-builtins",
         "https://github.com/rust-lang/compiler-builtins.git",
-        "0.1.68",
+        "0.1.70",
     );
     apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
 }
diff --git a/patches/0027-sysroot-128bit-atomic-operations.patch b/patches/0027-sysroot-128bit-atomic-operations.patch
index ffee641457ab2..8e6652af3747b 100644
--- a/patches/0027-sysroot-128bit-atomic-operations.patch
+++ b/patches/0027-sysroot-128bit-atomic-operations.patch
@@ -8,7 +8,6 @@ Cranelift doesn't support them yet
  library/core/src/panic/unwind_safe.rs |  6 -----
  library/core/src/sync/atomic.rs       | 38 ---------------------------
  library/core/tests/atomic.rs          |  4 ---
- library/std/src/time/monotonic.rs     |  6 +++--
  4 files changed, 4 insertions(+), 50 deletions(-)
 
 diff --git a/library/core/src/panic/unwind_safe.rs b/library/core/src/panic/unwind_safe.rs
@@ -99,38 +98,6 @@ index b735957..ea728b6 100644
      #[cfg(target_has_atomic = "ptr")]
      assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
      #[cfg(target_has_atomic = "ptr")]
-diff --git a/library/std/src/time/monotonic.rs b/library/std/src/time/monotonic.rs
-index fa96b7a..2854f9c 100644
---- a/library/std/src/time/monotonic.rs
-+++ b/library/std/src/time/monotonic.rs
-@@ -5,7 +5,7 @@ pub(super) fn monotonize(raw: time::Instant) -> time::Instant {
-     inner::monotonize(raw)
- }
- 
--#[cfg(any(all(target_has_atomic = "64", not(target_has_atomic = "128")), target_arch = "aarch64"))]
-+#[cfg(target_has_atomic = "64")]
- pub mod inner {
-     use crate::sync::atomic::AtomicU64;
-     use crate::sync::atomic::Ordering::*;
-@@ -70,6 +70,7 @@ pub mod inner {
-     }
- }
- 
-+/*
- #[cfg(all(target_has_atomic = "128", not(target_arch = "aarch64")))]
- pub mod inner {
-     use crate::sync::atomic::AtomicU128;
-@@ -94,8 +95,9 @@ pub mod inner {
-         ZERO.checked_add_duration(&Duration::new(secs, nanos)).unwrap()
-     }
- }
-+*/
- 
--#[cfg(not(any(target_has_atomic = "64", target_has_atomic = "128")))]
-+#[cfg(not(target_has_atomic = "64"))]
- pub mod inner {
-     use crate::cmp;
-     use crate::sys::time;
 -- 
 2.26.2.7.g19db9cfb68
 
diff --git a/rust-toolchain b/rust-toolchain
index 93b784079bdc0..78be383864d48 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-02-13"
+channel = "nightly-2022-02-16"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From 7e80bc3c8d8ab353ae498b10ad91049db5d3dbca Mon Sep 17 00:00:00 2001
From: Mark Rousskov <mark.simulacrum@gmail.com>
Date: Wed, 16 Feb 2022 13:04:48 -0500
Subject: [PATCH 63/84] Move ty::print methods to Drop-based scope guards

---
 src/intrinsics/mod.rs | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index 55c9b4d9ba12d..42717ad0ae0e1 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -779,29 +779,35 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () {
             let layout = fx.layout_of(T);
             if layout.abi.is_uninhabited() {
-                with_no_trimmed_paths(|| crate::base::codegen_panic(
-                    fx,
-                    &format!("attempted to instantiate uninhabited type `{}`", T),
-                    span,
-                ));
+                with_no_trimmed_paths!({
+                    crate::base::codegen_panic(
+                        fx,
+                        &format!("attempted to instantiate uninhabited type `{}`", T),
+                        span,
+                    )
+                });
                 return;
             }
 
             if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true) {
-                with_no_trimmed_paths(|| crate::base::codegen_panic(
-                    fx,
-                    &format!("attempted to zero-initialize type `{}`, which is invalid", T),
-                    span,
-                ));
+                with_no_trimmed_paths!({
+                    crate::base::codegen_panic(
+                        fx,
+                        &format!("attempted to zero-initialize type `{}`, which is invalid", T),
+                        span,
+                    );
+                });
                 return;
             }
 
             if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false) {
-                with_no_trimmed_paths(|| crate::base::codegen_panic(
-                    fx,
-                    &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
-                    span,
-                ));
+                with_no_trimmed_paths!({
+                    crate::base::codegen_panic(
+                        fx,
+                        &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
+                        span,
+                    )
+                });
                 return;
             }
         };

From 997492538b5db29ca3e350f18ab1ca1fab0dadff Mon Sep 17 00:00:00 2001
From: mqy <meng.qingyou@gmail.com>
Date: Sat, 19 Feb 2022 03:47:41 +0800
Subject: [PATCH 64/84] rustdoc: several minor fixes

---
 compiler/rustc_ast/src/ast.rs     | 2 +-
 library/core/src/ops/generator.rs | 2 +-
 library/core/src/pin.rs           | 4 ++--
 src/tools/lld-wrapper/src/main.rs | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index e9135b7163025..e616a09024cec 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -285,7 +285,7 @@ impl ParenthesizedArgs {
 
 pub use crate::node_id::{NodeId, CRATE_NODE_ID, DUMMY_NODE_ID};
 
-/// A modifier on a bound, e.g., `?Sized` or `~const Trait`.
+/// A modifier on a bound, e.g., `?Trait` or `~const Trait`.
 ///
 /// Negative bounds should also be handled here.
 #[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug)]
diff --git a/library/core/src/ops/generator.rs b/library/core/src/ops/generator.rs
index 52a2e464e3a7c..7c3b2a644e8db 100644
--- a/library/core/src/ops/generator.rs
+++ b/library/core/src/ops/generator.rs
@@ -47,7 +47,7 @@ pub enum GeneratorState<Y, R> {
 /// fn main() {
 ///     let mut generator = || {
 ///         yield 1;
-///         return "foo"
+///         "foo"
 ///     };
 ///
 ///     match Pin::new(&mut generator).resume(()) {
diff --git a/library/core/src/pin.rs b/library/core/src/pin.rs
index dec1b5270d58b..d642df4f6a021 100644
--- a/library/core/src/pin.rs
+++ b/library/core/src/pin.rs
@@ -1100,8 +1100,8 @@ pub macro pin($value:expr $(,)?) {
     //     that would break `Pin`'s invariants.
     //   - `{ $value }` is braced, making it a _block expression_, thus **moving**
     //     the given `$value`, and making it _become an **anonymous** temporary_.
-    //     By virtue of being anonynomous, it can no longer be accessed, thus
-    //     preventing any attemps to `mem::replace` it or `mem::forget` it, _etc._
+    //     By virtue of being anonymous, it can no longer be accessed, thus
+    //     preventing any attempts to `mem::replace` it or `mem::forget` it, _etc._
     //
     // This gives us a `pin!` definition that is sound, and which works, but only
     // in certain scenarios:
diff --git a/src/tools/lld-wrapper/src/main.rs b/src/tools/lld-wrapper/src/main.rs
index 1601bf1b34e9c..8d19a054a1d25 100644
--- a/src/tools/lld-wrapper/src/main.rs
+++ b/src/tools/lld-wrapper/src/main.rs
@@ -8,7 +8,7 @@
 //! In Rust with `-Z gcc-ld=lld` we have gcc or clang invoke rust-lld. Since there is no way to
 //! make gcc/clang pass `-flavor <flavor>` as the first two arguments in the linker invocation
 //! and since Windows does not support symbolic links for files this wrapper is used in place of a
-//! symblic link. It execs `../rust-lld -flavor ld` if the feature `ld` is enabled and
+//! symbolic link. It execs `../rust-lld -flavor ld` if the feature `ld` is enabled and
 //! `../rust-lld -flavor ld64` if `ld64` is enabled. On Windows it spawns a `..\rust-lld.exe`
 //! child process.
 

From aa601574a5fe861bffd641beccf59c7be3ed16c9 Mon Sep 17 00:00:00 2001
From: Nixon Enraght-Moony <nixon.emoony@gmail.com>
Date: Fri, 18 Feb 2022 21:58:49 +0000
Subject: [PATCH 65/84] rustdoc-json: Better Header Type

- Make ABI an enum, instead of being stringly typed
- Replace Qualifier HashSet with 3 bools
- Merge ABI field into header, as they always occor together
---
 compiler/rustc_hir/src/hir.rs      |  4 +++
 src/librustdoc/json/conversions.rs | 48 ++++++++++++++++--------------
 src/rustdoc-json-types/lib.rs      | 43 +++++++++++++++++---------
 3 files changed, 58 insertions(+), 37 deletions(-)

diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 0961d0131d07c..5087bdd37e8bb 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -2777,6 +2777,10 @@ impl FnHeader {
     pub fn is_const(&self) -> bool {
         matches!(&self.constness, Constness::Const)
     }
+
+    pub fn is_unsafe(&self) -> bool {
+        matches!(&self.unsafety, Unsafety::Unsafe)
+    }
 }
 
 #[derive(Debug, HashStable_Generic)]
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index e77bd5c922313..d4aedb41ddb18 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -12,6 +12,7 @@ use rustc_hir::{def::CtorKind, def_id::DefId};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::def_id::CRATE_DEF_INDEX;
 use rustc_span::Pos;
+use rustc_target::spec::abi::Abi as RustcAbi;
 
 use rustdoc_json_types::*;
 
@@ -19,7 +20,6 @@ use crate::clean::utils::print_const_expr;
 use crate::clean::{self, ItemId};
 use crate::formats::item_type::ItemType;
 use crate::json::JsonRenderer;
-use std::collections::HashSet;
 
 impl JsonRenderer<'_> {
     pub(super) fn convert_item(&self, item: clean::Item) -> Option<Item> {
@@ -271,22 +271,28 @@ crate fn from_ctor_kind(struct_type: CtorKind) -> StructType {
     }
 }
 
-crate fn from_fn_header(header: &rustc_hir::FnHeader) -> HashSet<Qualifiers> {
-    let mut v = HashSet::new();
-
-    if let rustc_hir::Unsafety::Unsafe = header.unsafety {
-        v.insert(Qualifiers::Unsafe);
-    }
-
-    if let rustc_hir::IsAsync::Async = header.asyncness {
-        v.insert(Qualifiers::Async);
+crate fn from_fn_header(header: &rustc_hir::FnHeader) -> Header {
+    Header {
+        async_: header.is_async(),
+        const_: header.is_const(),
+        unsafe_: header.is_unsafe(),
+        abi: convert_abi(header.abi),
     }
+}
 
-    if let rustc_hir::Constness::Const = header.constness {
-        v.insert(Qualifiers::Const);
+fn convert_abi(a: RustcAbi) -> Abi {
+    match a {
+        RustcAbi::Rust => Abi::Rust,
+        RustcAbi::C { unwind } => Abi::C { unwind },
+        RustcAbi::Cdecl { unwind } => Abi::Cdecl { unwind },
+        RustcAbi::Stdcall { unwind } => Abi::Stdcall { unwind },
+        RustcAbi::Fastcall { unwind } => Abi::Fastcall { unwind },
+        RustcAbi::Aapcs { unwind } => Abi::Aapcs { unwind },
+        RustcAbi::Win64 { unwind } => Abi::Win64 { unwind },
+        RustcAbi::SysV64 { unwind } => Abi::SysV64 { unwind },
+        RustcAbi::System { unwind } => Abi::System { unwind },
+        _ => Abi::Other(a.to_string()),
     }
-
-    v
 }
 
 impl FromWithTcx<clean::Function> for Function {
@@ -296,7 +302,6 @@ impl FromWithTcx<clean::Function> for Function {
             decl: decl.into_tcx(tcx),
             generics: generics.into_tcx(tcx),
             header: from_fn_header(&header),
-            abi: header.abi.to_string(),
         }
     }
 }
@@ -465,16 +470,14 @@ impl FromWithTcx<clean::BareFunctionDecl> for FunctionPointer {
     fn from_tcx(bare_decl: clean::BareFunctionDecl, tcx: TyCtxt<'_>) -> Self {
         let clean::BareFunctionDecl { unsafety, generic_params, decl, abi } = bare_decl;
         FunctionPointer {
-            header: if let rustc_hir::Unsafety::Unsafe = unsafety {
-                let mut hs = HashSet::new();
-                hs.insert(Qualifiers::Unsafe);
-                hs
-            } else {
-                HashSet::new()
+            header: Header {
+                unsafe_: matches!(unsafety, rustc_hir::Unsafety::Unsafe),
+                const_: false,
+                async_: false,
+                abi: convert_abi(abi),
             },
             generic_params: generic_params.into_iter().map(|x| x.into_tcx(tcx)).collect(),
             decl: decl.into_tcx(tcx),
-            abi: abi.to_string(),
         }
     }
 }
@@ -554,7 +557,6 @@ crate fn from_function_method(
         decl: decl.into_tcx(tcx),
         generics: generics.into_tcx(tcx),
         header: from_fn_header(&header),
-        abi: header.abi.to_string(),
         has_body,
     }
 }
diff --git a/src/rustdoc-json-types/lib.rs b/src/rustdoc-json-types/lib.rs
index 600833664be5e..be9bbc7391d75 100644
--- a/src/rustdoc-json-types/lib.rs
+++ b/src/rustdoc-json-types/lib.rs
@@ -3,13 +3,13 @@
 //! These types are the public API exposed through the `--output-format json` flag. The [`Crate`]
 //! struct is the root of the JSON blob and all other items are contained within.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
 
 /// rustdoc format-version.
-pub const FORMAT_VERSION: u32 = 10;
+pub const FORMAT_VERSION: u32 = 11;
 
 /// A `Crate` is the root of the emitted JSON blob. It contains all type/documentation information
 /// about the language items in the local crate, as well as info about external items to allow
@@ -287,29 +287,45 @@ pub enum StructType {
     Unit,
 }
 
-#[non_exhaustive]
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
-#[serde(rename_all = "snake_case")]
-pub enum Qualifiers {
-    Const,
-    Unsafe,
-    Async,
+pub struct Header {
+    #[serde(rename = "const")]
+    pub const_: bool,
+    #[serde(rename = "unsafe")]
+    pub unsafe_: bool,
+    #[serde(rename = "async")]
+    pub async_: bool,
+    pub abi: Abi,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub enum Abi {
+    // We only have a concrete listing here for stable ABI's because their are so many
+    // See rustc_ast_passes::feature_gate::PostExpansionVisitor::check_abi for the list
+    Rust,
+    C { unwind: bool },
+    Cdecl { unwind: bool },
+    Stdcall { unwind: bool },
+    Fastcall { unwind: bool },
+    Aapcs { unwind: bool },
+    Win64 { unwind: bool },
+    SysV64 { unwind: bool },
+    System { unwind: bool },
+    Other(String),
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct Function {
     pub decl: FnDecl,
     pub generics: Generics,
-    pub header: HashSet<Qualifiers>,
-    pub abi: String,
+    pub header: Header,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct Method {
     pub decl: FnDecl,
     pub generics: Generics,
-    pub header: HashSet<Qualifiers>,
-    pub abi: String,
+    pub header: Header,
     pub has_body: bool,
 }
 
@@ -426,8 +442,7 @@ pub enum Type {
 pub struct FunctionPointer {
     pub decl: FnDecl,
     pub generic_params: Vec<GenericParamDef>,
-    pub header: HashSet<Qualifiers>,
-    pub abi: String,
+    pub header: Header,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]

From fd5adefce11c820b5ee1045058a435aa174bccf4 Mon Sep 17 00:00:00 2001
From: Nixon Enraght-Moony <nixon.emoony@gmail.com>
Date: Fri, 18 Feb 2022 23:32:37 +0000
Subject: [PATCH 66/84] rustdoc-json: Add tests for fn qualifiers and ABI

---
 src/test/rustdoc-json/fn_pointer/abi.rs       | 25 +++++++++
 src/test/rustdoc-json/fn_pointer/header.rs    |  5 --
 .../rustdoc-json/fn_pointer/qualifiers.rs     |  9 +++
 src/test/rustdoc-json/fns/abi.rs              | 25 +++++++++
 src/test/rustdoc-json/fns/header.rs           | 22 --------
 src/test/rustdoc-json/fns/qualifiers.rs       | 33 +++++++++++
 src/test/rustdoc-json/method_abi.rs           | 25 ---------
 src/test/rustdoc-json/methods/abi.rs          | 55 +++++++++++++++++++
 src/test/rustdoc-json/methods/header.rs       | 26 ---------
 src/test/rustdoc-json/methods/qualifiers.rs   | 37 +++++++++++++
 10 files changed, 184 insertions(+), 78 deletions(-)
 create mode 100644 src/test/rustdoc-json/fn_pointer/abi.rs
 delete mode 100644 src/test/rustdoc-json/fn_pointer/header.rs
 create mode 100644 src/test/rustdoc-json/fn_pointer/qualifiers.rs
 create mode 100644 src/test/rustdoc-json/fns/abi.rs
 delete mode 100644 src/test/rustdoc-json/fns/header.rs
 create mode 100644 src/test/rustdoc-json/fns/qualifiers.rs
 delete mode 100644 src/test/rustdoc-json/method_abi.rs
 create mode 100644 src/test/rustdoc-json/methods/abi.rs
 delete mode 100644 src/test/rustdoc-json/methods/header.rs
 create mode 100644 src/test/rustdoc-json/methods/qualifiers.rs

diff --git a/src/test/rustdoc-json/fn_pointer/abi.rs b/src/test/rustdoc-json/fn_pointer/abi.rs
new file mode 100644
index 0000000000000..eef20e60a6a46
--- /dev/null
+++ b/src/test/rustdoc-json/fn_pointer/abi.rs
@@ -0,0 +1,25 @@
+// ignore-tidy-linelength
+
+#![feature(abi_vectorcall)]
+#![feature(c_unwind)]
+
+// @is abi.json "$.index[*][?(@.name=='AbiRust')].inner.type.inner.header.abi" \"Rust\"
+pub type AbiRust = fn();
+
+// @is - "$.index[*][?(@.name=='AbiC')].inner.type.inner.header.abi" '{"C": {"unwind": false}}'
+pub type AbiC = extern "C" fn();
+
+// @is - "$.index[*][?(@.name=='AbiSystem')].inner.type.inner.header.abi" '{"System": {"unwind": false}}'
+pub type AbiSystem = extern "system" fn();
+
+// @is - "$.index[*][?(@.name=='AbiCUnwind')].inner.type.inner.header.abi" '{"C": {"unwind": true}}'
+pub type AbiCUnwind = extern "C-unwind" fn();
+
+// @is - "$.index[*][?(@.name=='AbiSystemUnwind')].inner.type.inner.header.abi" '{"System": {"unwind": true}}'
+pub type AbiSystemUnwind = extern "system-unwind" fn();
+
+// @is - "$.index[*][?(@.name=='AbiVecorcall')].inner.type.inner.header.abi.Other" '"\"vectorcall\""'
+pub type AbiVecorcall = extern "vectorcall" fn();
+
+// @is - "$.index[*][?(@.name=='AbiVecorcallUnwind')].inner.type.inner.header.abi.Other" '"\"vectorcall-unwind\""'
+pub type AbiVecorcallUnwind = extern "vectorcall-unwind" fn();
diff --git a/src/test/rustdoc-json/fn_pointer/header.rs b/src/test/rustdoc-json/fn_pointer/header.rs
deleted file mode 100644
index a5038e0cd2aa8..0000000000000
--- a/src/test/rustdoc-json/fn_pointer/header.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-// @has header.json "$.index[*][?(@.name=='FnPointer')].inner.type.inner.header" "[]"
-pub type FnPointer = fn();
-
-// @has - "$.index[*][?(@.name=='UnsafePointer')].inner.type.inner.header" '["unsafe"]'
-pub type UnsafePointer = unsafe fn();
diff --git a/src/test/rustdoc-json/fn_pointer/qualifiers.rs b/src/test/rustdoc-json/fn_pointer/qualifiers.rs
new file mode 100644
index 0000000000000..3819220853639
--- /dev/null
+++ b/src/test/rustdoc-json/fn_pointer/qualifiers.rs
@@ -0,0 +1,9 @@
+// @is qualifiers.json "$.index[*][?(@.name=='FnPointer')].inner.type.inner.header.unsafe" false
+// @is - "$.index[*][?(@.name=='FnPointer')].inner.type.inner.header.const" false
+// @is - "$.index[*][?(@.name=='FnPointer')].inner.type.inner.header.async" false
+pub type FnPointer = fn();
+
+// @is - "$.index[*][?(@.name=='UnsafePointer')].inner.type.inner.header.unsafe" true
+// @is - "$.index[*][?(@.name=='UnsafePointer')].inner.type.inner.header.const" false
+// @is - "$.index[*][?(@.name=='UnsafePointer')].inner.type.inner.header.async" false
+pub type UnsafePointer = unsafe fn();
diff --git a/src/test/rustdoc-json/fns/abi.rs b/src/test/rustdoc-json/fns/abi.rs
new file mode 100644
index 0000000000000..16b579130656a
--- /dev/null
+++ b/src/test/rustdoc-json/fns/abi.rs
@@ -0,0 +1,25 @@
+// ignore-tidy-linelength
+
+#![feature(abi_vectorcall)]
+#![feature(c_unwind)]
+
+// @is abi.json "$.index[*][?(@.name=='abi_rust')].inner.header.abi" \"Rust\"
+pub fn abi_rust() {}
+
+// @is - "$.index[*][?(@.name=='abi_c')].inner.header.abi" '{"C": {"unwind": false}}'
+pub extern "C" fn abi_c() {}
+
+// @is - "$.index[*][?(@.name=='abi_system')].inner.header.abi" '{"System": {"unwind": false}}'
+pub extern "system" fn abi_system() {}
+
+// @is - "$.index[*][?(@.name=='abi_c_unwind')].inner.header.abi" '{"C": {"unwind": true}}'
+pub extern "C-unwind" fn abi_c_unwind() {}
+
+// @is - "$.index[*][?(@.name=='abi_system_unwind')].inner.header.abi" '{"System": {"unwind": true}}'
+pub extern "system-unwind" fn abi_system_unwind() {}
+
+// @is - "$.index[*][?(@.name=='abi_vectorcall')].inner.header.abi.Other" '"\"vectorcall\""'
+pub extern "vectorcall" fn abi_vectorcall() {}
+
+// @is - "$.index[*][?(@.name=='abi_vectorcall_unwind')].inner.header.abi.Other" '"\"vectorcall-unwind\""'
+pub extern "vectorcall-unwind" fn abi_vectorcall_unwind() {}
diff --git a/src/test/rustdoc-json/fns/header.rs b/src/test/rustdoc-json/fns/header.rs
deleted file mode 100644
index 29741dd50dadc..0000000000000
--- a/src/test/rustdoc-json/fns/header.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-// edition:2018
-
-// @has header.json "$.index[*][?(@.name=='nothing_fn')].inner.header" "[]"
-pub fn nothing_fn() {}
-
-// @has - "$.index[*][?(@.name=='const_fn')].inner.header" '["const"]'
-pub const fn const_fn() {}
-
-// @has - "$.index[*][?(@.name=='async_fn')].inner.header" '["async"]'
-pub async fn async_fn() {}
-
-// @count - "$.index[*][?(@.name=='async_unsafe_fn')].inner.header[*]" 2
-// @has - "$.index[*][?(@.name=='async_unsafe_fn')].inner.header[*]" '"async"'
-// @has - "$.index[*][?(@.name=='async_unsafe_fn')].inner.header[*]" '"unsafe"'
-pub async unsafe fn async_unsafe_fn() {}
-
-// @count - "$.index[*][?(@.name=='const_unsafe_fn')].inner.header[*]" 2
-// @has - "$.index[*][?(@.name=='const_unsafe_fn')].inner.header[*]" '"const"'
-// @has - "$.index[*][?(@.name=='const_unsafe_fn')].inner.header[*]" '"unsafe"'
-pub const unsafe fn const_unsafe_fn() {}
-
-// It's impossible for a function to be both const and async, so no test for that
diff --git a/src/test/rustdoc-json/fns/qualifiers.rs b/src/test/rustdoc-json/fns/qualifiers.rs
new file mode 100644
index 0000000000000..5cb3b43e66a96
--- /dev/null
+++ b/src/test/rustdoc-json/fns/qualifiers.rs
@@ -0,0 +1,33 @@
+// edition:2018
+
+// @is qualifiers.json "$.index[*][?(@.name=='nothing_fn')].inner.header.async" false
+// @is - "$.index[*][?(@.name=='nothing_fn')].inner.header.const"  false
+// @is - "$.index[*][?(@.name=='nothing_fn')].inner.header.unsafe" false
+pub fn nothing_fn() {}
+
+// @is - "$.index[*][?(@.name=='unsafe_fn')].inner.header.async"  false
+// @is - "$.index[*][?(@.name=='unsafe_fn')].inner.header.const"  false
+// @is - "$.index[*][?(@.name=='unsafe_fn')].inner.header.unsafe" true
+pub unsafe fn unsafe_fn() {}
+
+// @is - "$.index[*][?(@.name=='const_fn')].inner.header.async"  false
+// @is - "$.index[*][?(@.name=='const_fn')].inner.header.const"  true
+// @is - "$.index[*][?(@.name=='const_fn')].inner.header.unsafe" false
+pub const fn const_fn() {}
+
+// @is - "$.index[*][?(@.name=='async_fn')].inner.header.async"  true
+// @is - "$.index[*][?(@.name=='async_fn')].inner.header.const"  false
+// @is - "$.index[*][?(@.name=='async_fn')].inner.header.unsafe" false
+pub async fn async_fn() {}
+
+// @is - "$.index[*][?(@.name=='async_unsafe_fn')].inner.header.async"  true
+// @is - "$.index[*][?(@.name=='async_unsafe_fn')].inner.header.const"  false
+// @is - "$.index[*][?(@.name=='async_unsafe_fn')].inner.header.unsafe" true
+pub async unsafe fn async_unsafe_fn() {}
+
+// @is - "$.index[*][?(@.name=='const_unsafe_fn')].inner.header.async"  false
+// @is - "$.index[*][?(@.name=='const_unsafe_fn')].inner.header.const"  true
+// @is - "$.index[*][?(@.name=='const_unsafe_fn')].inner.header.unsafe" true
+pub const unsafe fn const_unsafe_fn() {}
+
+// It's impossible for a function to be both const and async, so no test for that
diff --git a/src/test/rustdoc-json/method_abi.rs b/src/test/rustdoc-json/method_abi.rs
deleted file mode 100644
index 6fabbc836117b..0000000000000
--- a/src/test/rustdoc-json/method_abi.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// @has method_abi.json "$.index[*][?(@.name=='Foo')]"
-pub struct Foo;
-
-impl Foo {
-    // @has - "$.index[*][?(@.name=='abi_rust')].inner.abi" '"\"Rust\""'
-    pub fn abi_rust() {}
-
-    // @has - "$.index[*][?(@.name=='abi_c')].inner.abi" '"\"C\""'
-    pub extern "C" fn abi_c() {}
-
-    // @has - "$.index[*][?(@.name=='abi_system')].inner.abi" '"\"system\""'
-    pub extern "system" fn abi_system() {}
-}
-
-// @has method_abi.json "$.index[*][?(@.name=='Bar')]"
-pub trait Bar {
-    // @has - "$.index[*][?(@.name=='trait_abi_rust')].inner.abi" '"\"Rust\""'
-    fn trait_abi_rust();
-
-    // @has - "$.index[*][?(@.name=='trait_abi_c')].inner.abi" '"\"C\""'
-    extern "C" fn trait_abi_c();
-
-    // @has - "$.index[*][?(@.name=='trait_abi_system')].inner.abi" '"\"system\""'
-    extern "system" fn trait_abi_system();
-}
diff --git a/src/test/rustdoc-json/methods/abi.rs b/src/test/rustdoc-json/methods/abi.rs
new file mode 100644
index 0000000000000..07b01d03bf6e3
--- /dev/null
+++ b/src/test/rustdoc-json/methods/abi.rs
@@ -0,0 +1,55 @@
+// ignore-tidy-linelength
+
+#![feature(abi_vectorcall)]
+#![feature(c_unwind)]
+#![feature(no_core)]
+#![no_core]
+
+// @has abi.json "$.index[*][?(@.name=='Foo')]"
+pub struct Foo;
+
+impl Foo {
+    // @is - "$.index[*][?(@.name=='abi_rust')].inner.header.abi" \"Rust\"
+    pub fn abi_rust() {}
+
+    // @is - "$.index[*][?(@.name=='abi_c')].inner.header.abi" '{"C": {"unwind": false}}'
+    pub extern "C" fn abi_c() {}
+
+    // @is - "$.index[*][?(@.name=='abi_system')].inner.header.abi" '{"System": {"unwind": false}}'
+    pub extern "system" fn abi_system() {}
+
+    // @is - "$.index[*][?(@.name=='abi_c_unwind')].inner.header.abi" '{"C": {"unwind": true}}'
+    pub extern "C-unwind" fn abi_c_unwind() {}
+
+    // @is - "$.index[*][?(@.name=='abi_system_unwind')].inner.header.abi" '{"System": {"unwind": true}}'
+    pub extern "system-unwind" fn abi_system_unwind() {}
+
+    // @is - "$.index[*][?(@.name=='abi_vectorcall')].inner.header.abi.Other" '"\"vectorcall\""'
+    pub extern "vectorcall" fn abi_vectorcall() {}
+
+    // @is - "$.index[*][?(@.name=='abi_vectorcall_unwind')].inner.header.abi.Other" '"\"vectorcall-unwind\""'
+    pub extern "vectorcall-unwind" fn abi_vectorcall_unwind() {}
+}
+
+pub trait Bar {
+    // @is - "$.index[*][?(@.name=='trait_abi_rust')].inner.header.abi" \"Rust\"
+    fn trait_abi_rust() {}
+
+    // @is - "$.index[*][?(@.name=='trait_abi_c')].inner.header.abi" '{"C": {"unwind": false}}'
+    extern "C" fn trait_abi_c() {}
+
+    // @is - "$.index[*][?(@.name=='trait_abi_system')].inner.header.abi" '{"System": {"unwind": false}}'
+    extern "system" fn trait_abi_system() {}
+
+    // @is - "$.index[*][?(@.name=='trait_abi_c_unwind')].inner.header.abi" '{"C": {"unwind": true}}'
+    extern "C-unwind" fn trait_abi_c_unwind() {}
+
+    // @is - "$.index[*][?(@.name=='trait_abi_system_unwind')].inner.header.abi" '{"System": {"unwind": true}}'
+    extern "system-unwind" fn trait_abi_system_unwind() {}
+
+    // @is - "$.index[*][?(@.name=='trait_abi_vectorcall')].inner.header.abi.Other" '"\"vectorcall\""'
+    extern "vectorcall" fn trait_abi_vectorcall() {}
+
+    // @is - "$.index[*][?(@.name=='trait_abi_vectorcall_unwind')].inner.header.abi.Other" '"\"vectorcall-unwind\""'
+    extern "vectorcall-unwind" fn trait_abi_vectorcall_unwind() {}
+}
diff --git a/src/test/rustdoc-json/methods/header.rs b/src/test/rustdoc-json/methods/header.rs
deleted file mode 100644
index 50a3db75ef395..0000000000000
--- a/src/test/rustdoc-json/methods/header.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-// edition:2018
-
-pub struct Foo;
-
-impl Foo {
-    // @has header.json "$.index[*][?(@.name=='nothing_meth')].inner.header" "[]"
-    pub fn nothing_meth() {}
-
-    // @has - "$.index[*][?(@.name=='const_meth')].inner.header" '["const"]'
-    pub const fn const_meth() {}
-
-    // @has - "$.index[*][?(@.name=='async_meth')].inner.header" '["async"]'
-    pub async fn async_meth() {}
-
-    // @count - "$.index[*][?(@.name=='async_unsafe_meth')].inner.header[*]" 2
-    // @has - "$.index[*][?(@.name=='async_unsafe_meth')].inner.header[*]" '"async"'
-    // @has - "$.index[*][?(@.name=='async_unsafe_meth')].inner.header[*]" '"unsafe"'
-    pub async unsafe fn async_unsafe_meth() {}
-
-    // @count - "$.index[*][?(@.name=='const_unsafe_meth')].inner.header[*]" 2
-    // @has - "$.index[*][?(@.name=='const_unsafe_meth')].inner.header[*]" '"const"'
-    // @has - "$.index[*][?(@.name=='const_unsafe_meth')].inner.header[*]" '"unsafe"'
-    pub const unsafe fn const_unsafe_meth() {}
-
-    // It's impossible for a method to be both const and async, so no test for that
-}
diff --git a/src/test/rustdoc-json/methods/qualifiers.rs b/src/test/rustdoc-json/methods/qualifiers.rs
new file mode 100644
index 0000000000000..af36d36b6607d
--- /dev/null
+++ b/src/test/rustdoc-json/methods/qualifiers.rs
@@ -0,0 +1,37 @@
+// edition:2018
+
+pub struct Foo;
+
+impl Foo {
+    // @is qualifiers.json "$.index[*][?(@.name=='const_meth')].inner.header.async" false
+    // @is - "$.index[*][?(@.name=='const_meth')].inner.header.const"  true
+    // @is - "$.index[*][?(@.name=='const_meth')].inner.header.unsafe" false
+    pub const fn const_meth() {}
+
+    // @is - "$.index[*][?(@.name=='nothing_meth')].inner.header.async"  false
+    // @is - "$.index[*][?(@.name=='nothing_meth')].inner.header.const"  false
+    // @is - "$.index[*][?(@.name=='nothing_meth')].inner.header.unsafe" false
+    pub fn nothing_meth() {}
+
+    // @is - "$.index[*][?(@.name=='unsafe_meth')].inner.header.async"  false
+    // @is - "$.index[*][?(@.name=='unsafe_meth')].inner.header.const"  false
+    // @is - "$.index[*][?(@.name=='unsafe_meth')].inner.header.unsafe" true
+    pub unsafe fn unsafe_meth() {}
+
+    // @is - "$.index[*][?(@.name=='async_meth')].inner.header.async"  true
+    // @is - "$.index[*][?(@.name=='async_meth')].inner.header.const"  false
+    // @is - "$.index[*][?(@.name=='async_meth')].inner.header.unsafe" false
+    pub async fn async_meth() {}
+
+    // @is - "$.index[*][?(@.name=='async_unsafe_meth')].inner.header.async"  true
+    // @is - "$.index[*][?(@.name=='async_unsafe_meth')].inner.header.const"  false
+    // @is - "$.index[*][?(@.name=='async_unsafe_meth')].inner.header.unsafe" true
+    pub async unsafe fn async_unsafe_meth() {}
+
+    // @is - "$.index[*][?(@.name=='const_unsafe_meth')].inner.header.async"  false
+    // @is - "$.index[*][?(@.name=='const_unsafe_meth')].inner.header.const"  true
+    // @is - "$.index[*][?(@.name=='const_unsafe_meth')].inner.header.unsafe" true
+    pub const unsafe fn const_unsafe_meth() {}
+
+    // It's impossible for a method to be both const and async, so no test for that
+}

From 4a5b069558518a99fe9765d10735451b7be0dbeb Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 20 Feb 2022 16:28:22 +0100
Subject: [PATCH 67/84] Rustup to rustc 1.61.0-nightly (3b348d932 2022-02-19)

---
 build_sysroot/Cargo.lock | 8 ++++----
 rust-toolchain           | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index 37b4f3efc9691..3fe9c1e90697d 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -40,9 +40,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "cc"
-version = "1.0.72"
+version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
 
 [[package]]
 name = "cfg-if"
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.118"
+version = "0.2.119"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06e509672465a0504304aa87f9f176f2b2b716ed8fb105ebe5c02dc6dce96a94"
+checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
 dependencies = [
  "rustc-std-workspace-core",
 ]
diff --git a/rust-toolchain b/rust-toolchain
index 78be383864d48..f1f801c54d778 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-02-16"
+channel = "nightly-2022-02-20"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From 2aad0066ba529ddd195b4f5ab0579879c8e9522f Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 20 Feb 2022 17:03:47 +0100
Subject: [PATCH 68/84] Update ignored rustc tests list

---
 scripts/test_rustc_tests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh
index efb4236e07253..b146ea360376a 100755
--- a/scripts/test_rustc_tests.sh
+++ b/scripts/test_rustc_tests.sh
@@ -78,6 +78,8 @@ rm src/test/ui/intrinsics/const-eval-select-x86_64.rs # same
 rm src/test/ui/match/issue-82392.rs # differing error
 rm src/test/ui/consts/min_const_fn/address_of_const.rs # same
 rm src/test/ui/consts/issue-miri-1910.rs # same
+rm src/test/ui/generic-associated-types/bugs/issue-80626.rs # same
+rm src/test/ui/generic-associated-types/bugs/issue-89008.rs # same
 rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
 
 rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition

From d34bcdd49c1d584487bc69e9b44e8b3fd957f4a7 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Mon, 7 Feb 2022 16:06:31 +0100
Subject: [PATCH 69/84] use `List<Ty<'tcx>>` for tuples

---
 src/abi/mod.rs | 4 ++--
 src/common.rs  | 7 +++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/abi/mod.rs b/src/abi/mod.rs
index a0550860fa545..a249e5fa8ac9c 100644
--- a/src/abi/mod.rs
+++ b/src/abi/mod.rs
@@ -117,7 +117,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
             .unzip();
         let return_layout = self.layout_of(return_ty);
         let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
-            tup.types().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
+            tup.iter().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
         } else {
             vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
         };
@@ -199,7 +199,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
                 };
 
                 let mut params = Vec::new();
-                for (i, _arg_ty) in tupled_arg_tys.types().enumerate() {
+                for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() {
                     let arg_abi = arg_abis_iter.next().unwrap();
                     let param =
                         cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter);
diff --git a/src/common.rs b/src/common.rs
index 50f98965ab5d2..d3e36be3244f1 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -90,10 +90,9 @@ fn clif_pair_type_from_ty<'tcx>(
     ty: Ty<'tcx>,
 ) -> Option<(types::Type, types::Type)> {
     Some(match ty.kind() {
-        ty::Tuple(substs) if substs.len() == 2 => {
-            let mut types = substs.types();
-            let a = clif_type_from_ty(tcx, types.next().unwrap())?;
-            let b = clif_type_from_ty(tcx, types.next().unwrap())?;
+        ty::Tuple(types) if types.len() == 2 => {
+            let a = clif_type_from_ty(tcx, types[0])?;
+            let b = clif_type_from_ty(tcx, types[1])?;
             if a.is_vector() || b.is_vector() {
                 return None;
             }

From c1d75a27b2cb5c936e1551cdf62716d3f39c0880 Mon Sep 17 00:00:00 2001
From: Mario Carneiro <di.gama@gmail.com>
Date: Mon, 21 Feb 2022 01:17:37 -0800
Subject: [PATCH 70/84] fix names in feature(...) suggestion

---
 compiler/rustc_feature/src/builtin_attrs.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 1fb1a38a927d5..53762eef78592 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -453,7 +453,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     // Internal attributes: Stability, deprecation, and unsafe:
     // ==========================================================================
 
-    ungated!(feature, CrateLevel, template!(List: "name1, name1, ..."), DuplicatesOk),
+    ungated!(feature, CrateLevel, template!(List: "name1, name2, ..."), DuplicatesOk),
     // DuplicatesOk since it has its own validation
     ungated!(
         rustc_deprecated, Normal,

From b673c6332ab5435bdb3a03b64fb4e865ba0092d5 Mon Sep 17 00:00:00 2001
From: Mario Carneiro <di.gama@gmail.com>
Date: Mon, 21 Feb 2022 02:58:44 -0800
Subject: [PATCH 71/84] fix test

---
 src/test/ui/feature-gates/gated-bad-feature.stderr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/ui/feature-gates/gated-bad-feature.stderr b/src/test/ui/feature-gates/gated-bad-feature.stderr
index a8ec9391523c7..477bf5e866eef 100644
--- a/src/test/ui/feature-gates/gated-bad-feature.stderr
+++ b/src/test/ui/feature-gates/gated-bad-feature.stderr
@@ -20,13 +20,13 @@ error: malformed `feature` attribute input
   --> $DIR/gated-bad-feature.rs:5:1
    |
 LL | #![feature]
-   | ^^^^^^^^^^^ help: must be of the form: `#![feature(name1, name1, ...)]`
+   | ^^^^^^^^^^^ help: must be of the form: `#![feature(name1, name2, ...)]`
 
 error: malformed `feature` attribute input
   --> $DIR/gated-bad-feature.rs:6:1
    |
 LL | #![feature = "foo"]
-   | ^^^^^^^^^^^^^^^^^^^ help: must be of the form: `#![feature(name1, name1, ...)]`
+   | ^^^^^^^^^^^^^^^^^^^ help: must be of the form: `#![feature(name1, name2, ...)]`
 
 error: aborting due to 5 previous errors
 

From e62c26e39d5a6ffda051aa956efde35226262951 Mon Sep 17 00:00:00 2001
From: Amanieu d'Antras <amanieu@gmail.com>
Date: Thu, 17 Feb 2022 14:16:52 +0000
Subject: [PATCH 72/84] On ARM, use relocation_model to detect whether r9
 should be reserved

The previous approach of checking for the reserve-r9 target feature
didn't actually work because LLVM only sets this feature very late when
initializing the per-function subtarget.
---
 src/inline_asm.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index c242c75ed18ff..10c2f06faf3ee 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -182,7 +182,12 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
 impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
     fn allocate_registers(&mut self) {
         let sess = self.tcx.sess;
-        let map = allocatable_registers(self.arch, &sess.target_features, &sess.target);
+        let map = allocatable_registers(
+            self.arch,
+            sess.relocation_model(),
+            &sess.target_features,
+            &sess.target,
+        );
         let mut allocated = FxHashMap::<_, (bool, bool)>::default();
         let mut regs = vec![None; self.operands.len()];
 
@@ -315,6 +320,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for saving clobbered registers
         let abi_clobber = InlineAsmClobberAbi::parse(
             self.arch,
+            self.tcx.sess.relocation_model(),
             &self.tcx.sess.target_features,
             &self.tcx.sess.target,
             sym::C,

From 73cf3aaa7892d2f5d19324111f79de87978459ee Mon Sep 17 00:00:00 2001
From: Amanieu d'Antras <amanieu@gmail.com>
Date: Thu, 17 Feb 2022 18:16:04 +0000
Subject: [PATCH 73/84] Take CodegenFnAttrs into account when validating asm!
 register operands

Checking of asm! register operands now properly takes function
attributes such as #[target_feature] and #[instruction_set] into
account.
---
 src/inline_asm.rs | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index 10c2f06faf3ee..deac5dfd3ec1a 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -106,6 +106,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     let mut asm_gen = InlineAssemblyGenerator {
         tcx: fx.tcx,
         arch: fx.tcx.sess.asm_arch.unwrap(),
+        enclosing_def_id: fx.instance.def_id(),
         template,
         operands,
         options,
@@ -169,6 +170,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 struct InlineAssemblyGenerator<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     arch: InlineAsmArch,
+    enclosing_def_id: DefId,
     template: &'a [InlineAsmTemplatePiece],
     operands: &'a [InlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
@@ -185,7 +187,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         let map = allocatable_registers(
             self.arch,
             sess.relocation_model(),
-            &sess.target_features,
+            self.tcx.asm_target_features(self.enclosing_def_id),
             &sess.target,
         );
         let mut allocated = FxHashMap::<_, (bool, bool)>::default();
@@ -318,15 +320,9 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
 
         // Allocate stack slots for saving clobbered registers
-        let abi_clobber = InlineAsmClobberAbi::parse(
-            self.arch,
-            self.tcx.sess.relocation_model(),
-            &self.tcx.sess.target_features,
-            &self.tcx.sess.target,
-            sym::C,
-        )
-        .unwrap()
-        .clobbered_regs();
+        let abi_clobber = InlineAsmClobberAbi::parse(self.arch, &self.tcx.sess.target, sym::C)
+            .unwrap()
+            .clobbered_regs();
         for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) {
             let mut need_save = true;
             // If the register overlaps with a register clobbered by function call, then

From a08809ff7badda201cc98fc20d9340a07567e43a Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 21 Feb 2022 16:58:12 -0800
Subject: [PATCH 74/84] Suggest calling .display() on PathBuf too

---
 library/core/src/fmt/mod.rs                 |  2 +-
 src/test/ui/suggestions/path-display.rs     |  6 +++++-
 src/test/ui/suggestions/path-display.stderr | 12 +++++++++++-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs
index e1ea98813031a..90c5719f486cb 100644
--- a/library/core/src/fmt/mod.rs
+++ b/library/core/src/fmt/mod.rs
@@ -728,7 +728,7 @@ pub use macros::Debug;
 /// ```
 #[rustc_on_unimplemented(
     on(
-        _Self = "std::path::Path",
+        any(_Self = "std::path::Path", _Self = "std::path::PathBuf"),
         label = "`{Self}` cannot be formatted with the default formatter; call `.display()` on it",
         note = "call `.display()` or `.to_string_lossy()` to safely print paths, \
                 as they may contain non-Unicode data"
diff --git a/src/test/ui/suggestions/path-display.rs b/src/test/ui/suggestions/path-display.rs
index 62fc9e79f7aa2..3a022e6b02fa4 100644
--- a/src/test/ui/suggestions/path-display.rs
+++ b/src/test/ui/suggestions/path-display.rs
@@ -1,7 +1,11 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 fn main() {
     let path = Path::new("/tmp/foo/bar.txt");
     println!("{}", path);
     //~^ ERROR E0277
+
+    let path = PathBuf::from("/tmp/foo/bar.txt");
+    println!("{}", path);
+    //~^ ERROR E0277
 }
diff --git a/src/test/ui/suggestions/path-display.stderr b/src/test/ui/suggestions/path-display.stderr
index 25c73c4c8741a..5e718d79307a8 100644
--- a/src/test/ui/suggestions/path-display.stderr
+++ b/src/test/ui/suggestions/path-display.stderr
@@ -8,6 +8,16 @@ LL |     println!("{}", path);
    = note: call `.display()` or `.to_string_lossy()` to safely print paths, as they may contain non-Unicode data
    = note: this error originates in the macro `$crate::format_args_nl` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: aborting due to previous error
+error[E0277]: `PathBuf` doesn't implement `std::fmt::Display`
+  --> $DIR/path-display.rs:9:20
+   |
+LL |     println!("{}", path);
+   |                    ^^^^ `PathBuf` cannot be formatted with the default formatter; call `.display()` on it
+   |
+   = help: the trait `std::fmt::Display` is not implemented for `PathBuf`
+   = note: call `.display()` or `.to_string_lossy()` to safely print paths, as they may contain non-Unicode data
+   = note: this error originates in the macro `$crate::format_args_nl` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0277`.

From dcd93bc5b49566b55e2219950ae0ef3c0f4d986a Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 22 Feb 2022 11:36:39 +0100
Subject: [PATCH 75/84] Use 2021 edition in ./x.py fmt

---
 src/bootstrap/format.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs
index 2408344487bb1..30fe6a7a44695 100644
--- a/src/bootstrap/format.rs
+++ b/src/bootstrap/format.rs
@@ -13,7 +13,7 @@ fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl F
     // avoid the submodule config paths from coming into play,
     // we only allow a single global config for the workspace for now
     cmd.arg("--config-path").arg(&src.canonicalize().unwrap());
-    cmd.arg("--edition").arg("2018");
+    cmd.arg("--edition").arg("2021");
     cmd.arg("--unstable-features");
     cmd.arg("--skip-children");
     if check {

From 21987c11aacf9fb170c73c3ad21a828ae913961d Mon Sep 17 00:00:00 2001
From: Krasimir Georgiev <krasimir@google.com>
Date: Tue, 22 Feb 2022 14:47:44 +0100
Subject: [PATCH 76/84] Bump download-ci-llvm-stamp for llvm-nm inclusion

We started using it in https://github.com/rust-lang/rust/pull/94023.
---
 src/bootstrap/download-ci-llvm-stamp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp
index ef635ee26df57..20a98111bebaa 100644
--- a/src/bootstrap/download-ci-llvm-stamp
+++ b/src/bootstrap/download-ci-llvm-stamp
@@ -1,4 +1,4 @@
 Change this file to make users of the `download-ci-llvm` configuration download
 a new version of LLVM from CI, even if the LLVM submodule hasn’t changed.
 
-Last change is for: https://github.com/rust-lang/rust/pull/91229
+Last change is for: https://github.com/rust-lang/rust/pull/94023

From a849857bda0a474c16b4cd36a10c1ef4adf3caed Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Tue, 22 Feb 2022 15:20:57 +0100
Subject: [PATCH 77/84] Prevent generation of infinite redirections

---
 src/librustdoc/html/render/context.rs | 60 +++++++++++++++------------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/src/librustdoc/html/render/context.rs b/src/librustdoc/html/render/context.rs
index a7f852a432c82..3f71a53f963e4 100644
--- a/src/librustdoc/html/render/context.rs
+++ b/src/librustdoc/html/render/context.rs
@@ -201,19 +201,19 @@ impl<'tcx> Context<'tcx> {
         } else {
             tyname.as_str()
         };
-        let page = layout::Page {
-            css_class: tyname_s,
-            root_path: &self.root_path(),
-            static_root_path: self.shared.static_root_path.as_deref(),
-            title: &title,
-            description: &desc,
-            keywords: &keywords,
-            resource_suffix: &self.shared.resource_suffix,
-            extra_scripts: &[],
-            static_extra_scripts: &[],
-        };
 
         if !self.render_redirect_pages {
+            let page = layout::Page {
+                css_class: tyname_s,
+                root_path: &self.root_path(),
+                static_root_path: self.shared.static_root_path.as_deref(),
+                title: &title,
+                description: &desc,
+                keywords: &keywords,
+                resource_suffix: &self.shared.resource_suffix,
+                extra_scripts: &[],
+                static_extra_scripts: &[],
+            };
             layout::render(
                 &self.shared.layout,
                 &page,
@@ -223,23 +223,31 @@ impl<'tcx> Context<'tcx> {
             )
         } else {
             if let Some(&(ref names, ty)) = self.cache().paths.get(&it.def_id.expect_def_id()) {
-                let mut path = String::new();
-                for name in &names[..names.len() - 1] {
-                    path.push_str(&name.as_str());
-                    path.push('/');
-                }
-                path.push_str(&item_path(ty, &names.last().unwrap().as_str()));
-                match self.shared.redirections {
-                    Some(ref redirections) => {
-                        let mut current_path = String::new();
-                        for name in &self.current {
-                            current_path.push_str(&name.as_str());
-                            current_path.push('/');
+                if self.current.len() + 1 != names.len()
+                    || self.current.iter().zip(names.iter()).any(|(a, b)| a != b)
+                {
+                    // We checked that the redirection isn't pointing to the current file,
+                    // preventing an infinite redirection loop in the generated
+                    // documentation.
+
+                    let mut path = String::new();
+                    for name in &names[..names.len() - 1] {
+                        path.push_str(&name.as_str());
+                        path.push('/');
+                    }
+                    path.push_str(&item_path(ty, &names.last().unwrap().as_str()));
+                    match self.shared.redirections {
+                        Some(ref redirections) => {
+                            let mut current_path = String::new();
+                            for name in &self.current {
+                                current_path.push_str(&name.as_str());
+                                current_path.push('/');
+                            }
+                            current_path.push_str(&item_path(ty, &names.last().unwrap().as_str()));
+                            redirections.borrow_mut().insert(current_path, path);
                         }
-                        current_path.push_str(&item_path(ty, &names.last().unwrap().as_str()));
-                        redirections.borrow_mut().insert(current_path, path);
+                        None => return layout::redirect(&format!("{}{}", self.root_path(), path)),
                     }
-                    None => return layout::redirect(&format!("{}{}", self.root_path(), path)),
                 }
             }
             String::new()

From 1d95acba346354fa216dde1bde777ee71ca912b7 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Tue, 22 Feb 2022 15:21:15 +0100
Subject: [PATCH 78/84] Add test for infinite redirection

---
 src/test/rustdoc/infinite-redirection.rs | 29 ++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 src/test/rustdoc/infinite-redirection.rs

diff --git a/src/test/rustdoc/infinite-redirection.rs b/src/test/rustdoc/infinite-redirection.rs
new file mode 100644
index 0000000000000..96a43323ce29d
--- /dev/null
+++ b/src/test/rustdoc/infinite-redirection.rs
@@ -0,0 +1,29 @@
+#![crate_name = "foo"]
+
+// This test ensures that there is no "infinite redirection" file generated (a
+// file which redirects to itself).
+
+// We check it's not a redirection file.
+// @has 'foo/builders/struct.ActionRowBuilder.html'
+// @has - '//*[@id="synthetic-implementations"]' 'Auto Trait Implementations'
+
+// And that the link in the module is targetting it.
+// @has 'foo/builders/index.html'
+// @has - '//a[@href="struct.ActionRowBuilder.html"]' 'ActionRowBuilder'
+
+mod auto {
+    mod action_row {
+        pub struct ActionRowBuilder;
+    }
+
+    #[doc(hidden)]
+    pub mod builders {
+        pub use super::action_row::ActionRowBuilder;
+    }
+}
+
+pub use auto::*;
+
+pub mod builders {
+    pub use crate::auto::builders::*;
+}

From 98bc47ff2d347da172afd73588302507cc6ce4d1 Mon Sep 17 00:00:00 2001
From: Antti Korpi <antti.korpimaki@gmail.com>
Date: Tue, 22 Feb 2022 18:26:41 +0100
Subject: [PATCH 79/84] Typo fix: Close inline-code backtick

---
 .../src/compiler-flags/debug_info_for_profiling.md              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doc/unstable-book/src/compiler-flags/debug_info_for_profiling.md b/src/doc/unstable-book/src/compiler-flags/debug_info_for_profiling.md
index 44bd3baeeedfc..ee72b6adf8e9f 100644
--- a/src/doc/unstable-book/src/compiler-flags/debug_info_for_profiling.md
+++ b/src/doc/unstable-book/src/compiler-flags/debug_info_for_profiling.md
@@ -1,4 +1,4 @@
-# `debug-info-for-profiling
+# `debug-info-for-profiling`
 
 ---
 

From c61d5923f29d7f02f69e4987b6169f3fc3187857 Mon Sep 17 00:00:00 2001
From: NyantasticUwU <NyantasticUwU@gmail.com>
Date: Tue, 22 Feb 2022 11:44:45 -0600
Subject: [PATCH 80/84] Fix typo.

Yeah just a typo (probably some breaking changes in here be careful) :)
---
 library/std/src/thread/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs
index f8d790c37852c..beb606099341e 100644
--- a/library/std/src/thread/mod.rs
+++ b/library/std/src/thread/mod.rs
@@ -1482,7 +1482,7 @@ fn _assert_sync_and_send() {
 ///
 /// Parallelism is a resource. A given machine provides a certain capacity for
 /// parallelism, i.e., a bound on the number of computations it can perform
-/// simultaneously. This number often corresponds to the amount of CPUs or
+/// simultaneously. This number often corresponds to the amount of CPUs a
 /// computer has, but it may diverge in various cases.
 ///
 /// Host environments such as VMs or container orchestrators may want to

From dc973aef2af4db8ddfc4e8e30f93b4e94560062c Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 22 Feb 2022 20:37:22 +0100
Subject: [PATCH 81/84] Rustup to rustc 1.61.0-nightly (03a8cc7df 2022-02-21)

---
 build_sysroot/Cargo.lock | 4 ++--
 rust-toolchain           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index 3fe9c1e90697d..b97863951429c 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -121,9 +121,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+checksum = "1ab7905ea95c6d9af62940f9d7dd9596d54c334ae2c15300c482051292d5637f"
 dependencies = [
  "compiler_builtins",
  "libc",
diff --git a/rust-toolchain b/rust-toolchain
index f1f801c54d778..6492660d65537 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-02-20"
+channel = "nightly-2022-02-22"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From d0192e62914f3ca48314f5f04a1a9e9c486e8f3e Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Tue, 22 Feb 2022 19:09:11 -0500
Subject: [PATCH 82/84] Miri: extend comments on downcast operation

---
 compiler/rustc_const_eval/src/interpret/operand.rs | 4 +++-
 compiler/rustc_const_eval/src/interpret/place.rs   | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index bc4dca4c146f6..cd147b03bcaa2 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -405,10 +405,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         op: &OpTy<'tcx, M::PointerTag>,
         variant: VariantIdx,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
-        // Downcasts only change the layout
         Ok(match op.try_as_mplace() {
             Ok(ref mplace) => self.mplace_downcast(mplace, variant)?.into(),
             Err(..) => {
+                // Downcasts only change the layout.
+                // (In particular, no check about whether this is even the active variant -- that's by design,
+                // see https://github.com/rust-lang/rust/issues/93688#issuecomment-1032929496.)
                 let layout = op.layout.for_variant(self, variant);
                 OpTy { layout, ..*op }
             }
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index e9b2df53a3313..b1784b12c6520 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -479,7 +479,9 @@ where
         base: &MPlaceTy<'tcx, M::PointerTag>,
         variant: VariantIdx,
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
-        // Downcasts only change the layout
+        // Downcasts only change the layout.
+        // (In particular, no check about whether this is even the active variant -- that's by design,
+        // see https://github.com/rust-lang/rust/issues/93688#issuecomment-1032929496.)
         assert!(!base.meta.has_meta());
         Ok(MPlaceTy { layout: base.layout.for_variant(self, variant), ..*base })
     }

From 1113cd5bbe81d4c94dd95e5d376869fb4516335e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= <tomasz.miasko@gmail.com>
Date: Tue, 22 Feb 2022 00:00:00 +0000
Subject: [PATCH 83/84] Rename `region_should_not_be_omitted` to
 `should_print_region`

to avoid double negation
---
 .../src/interpret/intrinsics/type_name.rs                 | 2 +-
 compiler/rustc_middle/src/ty/print/pretty.rs              | 8 ++++----
 compiler/rustc_symbol_mangling/src/legacy.rs              | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs
index e6f243e28dbc5..aa84b16b50281 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs
@@ -149,7 +149,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> {
 }
 
 impl<'tcx> PrettyPrinter<'tcx> for AbsolutePathPrinter<'tcx> {
-    fn region_should_not_be_omitted(&self, _region: ty::Region<'_>) -> bool {
+    fn should_print_region(&self, _region: ty::Region<'_>) -> bool {
         false
     }
     fn comma_sep<T>(mut self, mut elems: impl Iterator<Item = T>) -> Result<Self, Self::Error>
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 359e4f5e58147..403315a7a78f2 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -275,7 +275,7 @@ pub trait PrettyPrinter<'tcx>:
     /// Returns `true` if the region should be printed in
     /// optional positions, e.g., `&'a T` or `dyn Tr + 'b`.
     /// This is typically the case for all non-`'_` regions.
-    fn region_should_not_be_omitted(&self, region: ty::Region<'_>) -> bool;
+    fn should_print_region(&self, region: ty::Region<'_>) -> bool;
 
     // Defaults (should not be overridden):
 
@@ -577,7 +577,7 @@ pub trait PrettyPrinter<'tcx>:
             }
             ty::Ref(r, ty, mutbl) => {
                 p!("&");
-                if self.region_should_not_be_omitted(r) {
+                if self.should_print_region(r) {
                     p!(print(r), " ");
                 }
                 p!(print(ty::TypeAndMut { ty, mutbl }))
@@ -621,7 +621,7 @@ pub trait PrettyPrinter<'tcx>:
                 p!(print_def_path(def.did, substs));
             }
             ty::Dynamic(data, r) => {
-                let print_r = self.region_should_not_be_omitted(r);
+                let print_r = self.should_print_region(r);
                 if print_r {
                     p!("(");
                 }
@@ -1914,7 +1914,7 @@ impl<'tcx, F: fmt::Write> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx, F> {
         Ok(inner)
     }
 
-    fn region_should_not_be_omitted(&self, region: ty::Region<'_>) -> bool {
+    fn should_print_region(&self, region: ty::Region<'_>) -> bool {
         let highlight = self.region_highlight_mode;
         if highlight.region_highlighted(region).is_some() {
             return true;
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index cec1d4bc15717..47bf0fe122156 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -346,7 +346,7 @@ impl<'tcx> Printer<'tcx> for &mut SymbolPrinter<'tcx> {
 }
 
 impl<'tcx> PrettyPrinter<'tcx> for &mut SymbolPrinter<'tcx> {
-    fn region_should_not_be_omitted(&self, _region: ty::Region<'_>) -> bool {
+    fn should_print_region(&self, _region: ty::Region<'_>) -> bool {
         false
     }
     fn comma_sep<T>(mut self, mut elems: impl Iterator<Item = T>) -> Result<Self, Self::Error>

From 35d9c6bf256968e1b40e0d554607928bdf9cebea Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Wed, 23 Feb 2022 11:45:41 +0100
Subject: [PATCH 84/84] Rustup to rustc 1.61.0-nightly (68369a041 2022-02-22)

---
 rust-toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain b/rust-toolchain
index 6492660d65537..1019b1f069e50 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-02-22"
+channel = "nightly-2022-02-23"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]