TransmuteFrom: normalize types, unify confirmation and error reporting

Refactor to share code between `TransmuteFrom`'s trait selection
and error reporting code paths. Additionally normalizes the
source and destination types, and gracefully handles normalization
errors.

Fixes #130413

compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs

@@ -245,6 +245,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                                         span, "silent safe transmute error"
                                     );
                                 }
+                                GetSafeTransmuteErrorAndReason::Default => {
+                                    (err_msg, None)
+                                }
                                 GetSafeTransmuteErrorAndReason::Error {
                                     err_msg,
                                     safe_transmute_explanation,
@@ -2219,38 +2222,25 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         trait_ref: ty::PolyTraitRef<'tcx>,
         span: Span,
     ) -> GetSafeTransmuteErrorAndReason {
-        use rustc_transmute::Answer;
-
-        // Erase regions because layout code doesn't particularly care about regions.
-        let trait_ref =
-            self.tcx.erase_regions(self.tcx.instantiate_bound_regions_with_erased(trait_ref));
-
-        let src_and_dst = rustc_transmute::Types {
-            dst: trait_ref.args.type_at(0),
-            src: trait_ref.args.type_at(1),
-        };
-        let Some(assume) = rustc_transmute::Assume::from_const(
-            self.infcx.tcx,
-            obligation.param_env,
-            trait_ref.args.const_at(2),
-        ) else {
-            self.dcx().span_delayed_bug(
-                span,
-                "Unable to construct rustc_transmute::Assume where it was previously possible",
-            );
-            return GetSafeTransmuteErrorAndReason::Silent;
-        };
-
-        let dst = trait_ref.args.type_at(0);
-        let src = trait_ref.args.type_at(1);
-        let err_msg = format!("`{src}` cannot be safely transmuted into `{dst}`");
-
-        match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable(
-            obligation.cause,
-            src_and_dst,
-            assume,
+        match crate::transmute::confirm_transmutability_candidate(
+            &self.infcx,
+            &obligation,
+            trait_ref,
         ) {
-            Answer::No(reason) => {
+            Ok(None) => {
+                self.dcx().span_delayed_bug(
+                    span,
+                    "unable to analyze transmutability where it was previously possible",
+                );
+                GetSafeTransmuteErrorAndReason::Silent
+            }
+            Ok(Some((src, dst, _, _))) => {
+                let err_msg = format!("`{src}` cannot be safely transmuted into `{dst}`");
+                GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation: None }
+            }
+            Err(None) => GetSafeTransmuteErrorAndReason::Default,
+            Err(Some((src, dst, reason))) => {
+                let err_msg = format!("`{src}` cannot be safely transmuted into `{dst}`");
                 let safe_transmute_explanation = match reason {
                     rustc_transmute::Reason::SrcIsNotYetSupported => {
                         format!("analyzing the transmutability of `{src}` is not yet supported")
@@ -2318,18 +2308,6 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     safe_transmute_explanation: Some(safe_transmute_explanation),
                 }
             }
-            // Should never get a Yes at this point! We already ran it before, and did not get a Yes.
-            Answer::Yes => span_bug!(
-                span,
-                "Inconsistent rustc_transmute::is_transmutable(...) result, got Yes",
-            ),
-            // Reached when a different obligation (namely `Freeze`) causes the
-            // transmutability analysis to fail. In this case, silence the
-            // transmutability error message in favor of that more specific
-            // error.
-            Answer::If(_) => {
-                GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation: None }
-            }
         }
     }
 

compiler/rustc_trait_selection/src/error_reporting/traits/mod.rs

@@ -43,6 +43,7 @@ pub struct ImplCandidate<'tcx> {
 
 enum GetSafeTransmuteErrorAndReason {
     Silent,
+    Default,
     Error { err_msg: String, safe_transmute_explanation: Option<String> },
 }
 

compiler/rustc_trait_selection/src/lib.rs

@@ -41,5 +41,6 @@ pub mod infer;
 pub mod regions;
 pub mod solve;
 pub mod traits;
+mod transmute;
 
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -292,7 +292,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
-        use rustc_transmute::{Answer, Assume, Condition};
+        use rustc_transmute::{Assume, Condition};
 
         /// Generate sub-obligations for reference-to-reference transmutations.
         fn reference_obligations<'tcx>(
@@ -400,35 +400,19 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             }
         }
 
-        let predicate = obligation.predicate.skip_binder();
+        let trait_ref = obligation.predicate.to_poly_trait_ref();
 
-        let Some(assume) = rustc_transmute::Assume::from_const(
-            self.infcx.tcx,
-            obligation.param_env,
-            predicate.trait_ref.args.const_at(2),
-        ) else {
-            return Err(Unimplemented);
-        };
-
-        let dst = predicate.trait_ref.args.type_at(0);
-        let src = predicate.trait_ref.args.type_at(1);
-
-        debug!(?src, ?dst);
-        let mut transmute_env = rustc_transmute::TransmuteTypeEnv::new(self.infcx);
-        let maybe_transmutable = transmute_env.is_transmutable(
-            obligation.cause.clone(),
-            rustc_transmute::Types { dst, src },
-            assume,
-        );
-
-        let fully_flattened = match maybe_transmutable {
-            Answer::No(_) => Err(Unimplemented)?,
-            Answer::If(cond) => flatten_answer_tree(self.tcx(), obligation, cond, assume),
-            Answer::Yes => vec![],
-        };
-
-        debug!(?fully_flattened);
-        Ok(fully_flattened)
+        match crate::transmute::confirm_transmutability_candidate(
+            &self.infcx,
+            obligation,
+            trait_ref,
+        ) {
+            Ok(None) => Ok(vec![]),
+            Ok(Some((_, _, assume, cond))) => {
+                Ok(flatten_answer_tree(self.tcx(), obligation, cond, assume))
+            }
+            Err(_) => Err(SelectionError::Unimplemented),
+        }
     }
 
     /// This handles the case where an `auto trait Foo` impl is being used.

compiler/rustc_trait_selection/src/transmute.rs

@@ -0,0 +1,78 @@
+use rustc_infer::infer::InferCtxt;
+use rustc_infer::traits::{Obligation, ObligationCause};
+use rustc_middle::ty::{Binder, ParamEnv, PolyTraitRef, Ty, TyCtxt};
+use rustc_transmute::layout::rustc::Ref;
+use rustc_transmute::{Answer, Assume, Condition, Reason};
+
+/// Attempts to confirm the given transmutability candidate.
+///
+/// The return type of this routine reflects that it is designed to support the
+/// needs of both trait selection and error reporting; it returns:
+/// - `Ok(None)` if the types are transmutable
+/// - `Ok(Some(src, dst, assumptions, condition))` if the `src` is transmutable
+///   into `dst` when `condition` holds under `assumptions`
+/// - `Err(None)` if transmutability cannot be assessed; e.g., due to a
+///   malformed `Assume`
+/// - `Err(Some((src, dst, reason))` if `src` is not transmutable into `dst`
+///   because of `reason`.
+pub(crate) fn confirm_transmutability_candidate<'tcx, P>(
+    infcx: &InferCtxt<'tcx>,
+    obligation: &Obligation<'tcx, P>,
+    trait_ref: PolyTraitRef<'tcx>,
+) -> Result<
+    Option<(Ty<'tcx>, Ty<'tcx>, Assume, Condition<Ref<'tcx>>)>,
+    Option<(Ty<'tcx>, Ty<'tcx>, Reason<Ref<'tcx>>)>,
+> {
+    /// Attempts to deeply normalize `ty`.
+    fn try_normalize<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        param_env: ParamEnv<'tcx>,
+        ty: Binder<'tcx, Ty<'tcx>>,
+    ) -> Option<Binder<'tcx, Ty<'tcx>>> {
+        use rustc_infer::infer::TyCtxtInferExt;
+
+        use crate::traits::ObligationCtxt;
+        let infcx = tcx.infer_ctxt().with_next_trait_solver(true).build();
+        let ocx = ObligationCtxt::new(&infcx);
+        let cause = ObligationCause::dummy();
+        let Ok(ty) = ocx.deeply_normalize(&cause, param_env, ty) else { return None };
+        let errors = ocx.select_all_or_error();
+        if errors.is_empty() { Some(ty) } else { None }
+    }
+
+    let dst = trait_ref.map_bound(|tr| tr.args.type_at(0));
+    let src = trait_ref.map_bound(|tr| tr.args.type_at(1));
+
+    let Some(dst) = try_normalize(infcx.tcx, obligation.param_env, dst) else {
+        return Err(None);
+    };
+
+    let Some(src) = try_normalize(infcx.tcx, obligation.param_env, src) else {
+        return Err(None);
+    };
+
+    // The immediate layouts of `src` and `dst` do not depend on lifetimes.
+    let dst = dst.skip_binder();
+    let src = src.skip_binder();
+
+    let Some(assume) = rustc_transmute::Assume::from_const(
+        infcx.tcx,
+        obligation.param_env,
+        trait_ref.skip_binder().args.const_at(2),
+    ) else {
+        return Err(None);
+    };
+
+    let mut transmute_env = rustc_transmute::TransmuteTypeEnv::new(infcx);
+    let maybe_transmutable = transmute_env.is_transmutable(
+        obligation.cause.clone(),
+        rustc_transmute::Types { dst, src },
+        assume,
+    );
+
+    match maybe_transmutable {
+        Answer::No(reason) => return Err(Some((src, dst, reason))),
+        Answer::If(cond) => Ok(Some((src, dst, assume, cond))),
+        Answer::Yes => Ok(None),
+    }
+}

compiler/rustc_transmute/src/layout/tree.rs

@@ -195,10 +195,11 @@ pub(crate) mod rustc {
     impl<'tcx> From<&LayoutError<'tcx>> for Err {
         fn from(err: &LayoutError<'tcx>) -> Self {
             match err {
-                LayoutError::Unknown(..) | LayoutError::ReferencesError(..) => Self::UnknownLayout,
+                LayoutError::Unknown(..)
+                | LayoutError::ReferencesError(..)
+                | LayoutError::NormalizationFailure(..) => Self::UnknownLayout,
                 LayoutError::SizeOverflow(..) => Self::SizeOverflow,
                 LayoutError::Cycle(err) => Self::TypeError(*err),
-                err => unimplemented!("{:?}", err),
             }
         }
     }

tests/ui/transmutability/alignment/align-fail.rs

@@ -18,5 +18,5 @@ mod assert {
 }
 
 fn main() {
-    assert::is_maybe_transmutable::<&'static [u8; 0], &'static [u16; 0]>(); //~ ERROR `&[u8; 0]` cannot be safely transmuted into `&[u16; 0]`
+    assert::is_maybe_transmutable::<&'static [u8; 0], &'static [u16; 0]>(); //~ ERROR `&'static [u8; 0]` cannot be safely transmuted into `&'static [u16; 0]`
 }

tests/ui/transmutability/alignment/align-fail.stderr

@@ -1,8 +1,8 @@
-error[E0277]: `&[u8; 0]` cannot be safely transmuted into `&[u16; 0]`
+error[E0277]: `&'static [u8; 0]` cannot be safely transmuted into `&'static [u16; 0]`
   --> $DIR/align-fail.rs:21:55
    |
-LL | ...tatic [u8; 0], &'static [u16; 0]>();
-   |                   ^^^^^^^^^^^^^^^^^ the minimum alignment of `&[u8; 0]` (1) should be greater than that of `&[u16; 0]` (2)
+LL | ...8; 0], &'static [u16; 0]>();
+   |           ^^^^^^^^^^^^^^^^^ the minimum alignment of `&'static [u8; 0]` (1) should be greater than that of `&'static [u16; 0]` (2)
    |
 note: required by a bound in `is_maybe_transmutable`
   --> $DIR/align-fail.rs:9:14

tests/ui/transmutability/assoc-bound.rs

@@ -0,0 +1,16 @@
+#![crate_type = "lib"]
+#![feature(transmutability)]
+trait Aaa {
+    type Y;
+}
+
+trait Bbb {
+    type B: std::mem::TransmuteFrom<()>;
+}
+
+impl<T> Bbb for T
+where
+    T: Aaa,
+{
+    type B = T::Y; //~ERROR: `()` cannot be safely transmuted into `<T as Aaa>::Y`
+}

tests/ui/transmutability/assoc-bound.stderr

@@ -0,0 +1,19 @@
+error[E0277]: `()` cannot be safely transmuted into `<T as Aaa>::Y`
+  --> $DIR/assoc-bound.rs:15:14
+   |
+LL |     type B = T::Y;
+   |              ^^^^ `<T as Aaa>::Y` has an unknown layout
+   |
+note: required by a bound in `Bbb::B`
+  --> $DIR/assoc-bound.rs:8:13
+   |
+LL |     type B: std::mem::TransmuteFrom<()>;
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `Bbb::B`
+help: consider further restricting the associated type
+   |
+LL |     T: Aaa, <T as Aaa>::Y: TransmuteFrom<(), Assume { alignment: false, lifetimes: false, safety: false, validity: false }>
+   |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui/transmutability/references/recursive-wrapper-types-bit-compatible-mut.stderr

@@ -1,8 +1,8 @@
-error[E0277]: `&A` cannot be safely transmuted into `&mut B`
+error[E0277]: `&'static A` cannot be safely transmuted into `&'static mut B`
   --> $DIR/recursive-wrapper-types-bit-compatible-mut.rs:23:49
    |
 LL |     assert::is_maybe_transmutable::<&'static A, &'static mut B>();
-   |                                                 ^^^^^^^^^^^^^^ `&A` is a shared reference, but `&mut B` is a unique reference
+   |                                                 ^^^^^^^^^^^^^^ `&'static A` is a shared reference, but `&'static mut B` is a unique reference
    |
 note: required by a bound in `is_maybe_transmutable`
   --> $DIR/recursive-wrapper-types-bit-compatible-mut.rs:9:14

tests/ui/transmutability/references/unit-to-u8.stderr

@@ -1,4 +1,4 @@
-error[E0277]: `&Unit` cannot be safely transmuted into `&u8`
+error[E0277]: `&'static Unit` cannot be safely transmuted into `&'static u8`
   --> $DIR/unit-to-u8.rs:22:52
    |
 LL |     assert::is_maybe_transmutable::<&'static Unit, &'static u8>();

tests/ui/transmutability/references/unsafecell.stderr

@@ -1,4 +1,4 @@
-error[E0277]: `&u8` cannot be safely transmuted into `&UnsafeCell<u8>`
+error[E0277]: `&'static u8` cannot be safely transmuted into `&'static UnsafeCell<u8>`
   --> $DIR/unsafecell.rs:27:50
    |
 LL |     assert::is_maybe_transmutable::<&'static u8, &'static UnsafeCell<u8>>();
@@ -13,7 +13,7 @@ LL |     where
 LL |         Dst: TransmuteFrom<Src, { Assume::SAFETY }>
    |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `is_maybe_transmutable`
 
-error[E0277]: `&UnsafeCell<u8>` cannot be safely transmuted into `&UnsafeCell<u8>`
+error[E0277]: `&'static UnsafeCell<u8>` cannot be safely transmuted into `&'static UnsafeCell<u8>`
   --> $DIR/unsafecell.rs:29:62
    |
 LL |     assert::is_maybe_transmutable::<&'static UnsafeCell<u8>, &'static UnsafeCell<u8>>();