Rollup merge of #118983 - Urgau:invalid_ref_casting-bigger-layout, r=…

…oli-obk

Warn on references casting to bigger memory layout

This PR extends the [`invalid_reference_casting`](https://doc.rust-lang.org/rustc/lints/listing/deny-by-default.html#invalid-reference-casting) lint (*deny-by-default*) which currently lint on `&T -> &mut T` casting to also lint on `&(mut) A -> &(mut) B` where `size_of::<B>() > size_of::<A>()` (bigger memory layout requirement).

The goal is to detect such cases:

```rust
let u8_ref: &u8 = &0u8;
let u64_ref: &u64 = unsafe { &*(u8_ref as *const u8 as *const u64) };
//~^ ERROR casting references to a bigger memory layout is undefined behavior

let mat3 = Mat3 { a: Vec3(0i32, 0, 0), b: Vec3(0, 0, 0), c: Vec3(0, 0, 0) };
let mat3 = unsafe { &*(&mat3 as *const _ as *const [[i64; 3]; 3]) };
//~^ ERROR casting references to a bigger memory layout is undefined behavior
```

This is added to help people who write unsafe code, especially when people have matrix struct that they cast to simple array of arrays.

EDIT: One caveat, due to the [`&Header`](rust-lang/unsafe-code-guidelines#256) uncertainty the lint only fires when it can find the underline allocation.

~~I have manually tested all the new expressions that warn against Miri, and they all report immediate UB.~~

r? ``@est31``

tests/ui/transmute_ptr_to_ptr.fixed

@@ -35,16 +35,16 @@ fn transmute_ptr_to_ptr() {
         // ref-ref transmutes; bad
         let _: &f32 = &*(&1u32 as *const u32 as *const f32);
         //~^ ERROR: transmute from a reference to a reference
-        let _: &f64 = &*(&1f32 as *const f32 as *const f64);
+        let _: &f32 = &*(&1f64 as *const f64 as *const f32);
         //~^ ERROR: transmute from a reference to a reference
         //:^ this test is here because both f32 and f64 are the same TypeVariant, but they are not
         // the same type
         let _: &mut f32 = &mut *(&mut 1u32 as *mut u32 as *mut f32);
         //~^ ERROR: transmute from a reference to a reference
         let _: &GenericParam<f32> = &*(&GenericParam { t: 1u32 } as *const GenericParam<u32> as *const GenericParam<f32>);
         //~^ ERROR: transmute from a reference to a reference
-        let u8_ref: &u8 = &0u8;
-        let u64_ref: &u64 = unsafe { &*(u8_ref as *const u8 as *const u64) };
+        let u64_ref: &u64 = &0u64;
+        let u8_ref: &u8 = unsafe { &*(u64_ref as *const u64 as *const u8) };
         //~^ ERROR: transmute from a reference to a reference
     }
 

tests/ui/transmute_ptr_to_ptr.rs

@@ -35,16 +35,16 @@ fn transmute_ptr_to_ptr() {
         // ref-ref transmutes; bad
         let _: &f32 = std::mem::transmute(&1u32);
         //~^ ERROR: transmute from a reference to a reference
-        let _: &f64 = std::mem::transmute(&1f32);
+        let _: &f32 = std::mem::transmute(&1f64);
         //~^ ERROR: transmute from a reference to a reference
         //:^ this test is here because both f32 and f64 are the same TypeVariant, but they are not
         // the same type
         let _: &mut f32 = std::mem::transmute(&mut 1u32);
         //~^ ERROR: transmute from a reference to a reference
         let _: &GenericParam<f32> = std::mem::transmute(&GenericParam { t: 1u32 });
         //~^ ERROR: transmute from a reference to a reference
-        let u8_ref: &u8 = &0u8;
-        let u64_ref: &u64 = unsafe { std::mem::transmute(u8_ref) };
+        let u64_ref: &u64 = &0u64;
+        let u8_ref: &u8 = unsafe { std::mem::transmute(u64_ref) };
         //~^ ERROR: transmute from a reference to a reference
     }
 

tests/ui/transmute_ptr_to_ptr.stderr

@@ -22,8 +22,8 @@ LL |         let _: &f32 = std::mem::transmute(&1u32);
 error: transmute from a reference to a reference
   --> $DIR/transmute_ptr_to_ptr.rs:38:23
    |
-LL |         let _: &f64 = std::mem::transmute(&1f32);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(&1f32 as *const f32 as *const f64)`
+LL |         let _: &f32 = std::mem::transmute(&1f64);
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(&1f64 as *const f64 as *const f32)`
 
 error: transmute from a reference to a reference
   --> $DIR/transmute_ptr_to_ptr.rs:42:27
@@ -38,10 +38,10 @@ LL |         let _: &GenericParam<f32> = std::mem::transmute(&GenericParam { t:
    |                                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(&GenericParam { t: 1u32 } as *const GenericParam<u32> as *const GenericParam<f32>)`
 
 error: transmute from a reference to a reference
-  --> $DIR/transmute_ptr_to_ptr.rs:47:38
+  --> $DIR/transmute_ptr_to_ptr.rs:47:36
    |
-LL |         let u64_ref: &u64 = unsafe { std::mem::transmute(u8_ref) };
-   |                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(u8_ref as *const u8 as *const u64)`
+LL |         let u8_ref: &u8 = unsafe { std::mem::transmute(u64_ref) };
+   |                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `&*(u64_ref as *const u64 as *const u8)`
 
 error: aborting due to 7 previous errors