Hide niches under UnsafeCell

src/ci/docker/dist-various-2/Dockerfile

@@ -48,7 +48,7 @@ RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc
 COPY dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/
 # We pass the commit id of the port of LLVM's libunwind to the build script.
 # Any update to the commit id here, should cause the container image to be re-built from this point on.
-RUN /tmp/build-x86_64-fortanix-unknown-sgx-toolchain.sh "53b586346f2c7870e20b170decdc30729d97c42b"
+RUN /tmp/build-x86_64-fortanix-unknown-sgx-toolchain.sh "5125c169b30837208a842f85f7ae44a83533bd0e"
 
 COPY dist-various-2/build-wasi-toolchain.sh /tmp/
 RUN /tmp/build-wasi-toolchain.sh

src/libcore/cell.rs

@@ -1475,6 +1475,7 @@ impl<T: ?Sized + fmt::Display> fmt::Display for RefMut<'_, T> {
 #[lang = "unsafe_cell"]
 #[stable(feature = "rust1", since = "1.0.0")]
 #[repr(transparent)]
+#[cfg_attr(not(bootstrap), repr(no_niche))] // rust-lang/rust#68303.
 pub struct UnsafeCell<T: ?Sized> {
     value: T,
 }

src/libcore/lib.rs

@@ -138,6 +138,7 @@
 #![feature(const_type_id)]
 #![feature(const_caller_location)]
 #![feature(assoc_int_consts)]
+#![cfg_attr(not(bootstrap), feature(no_niche))] // rust-lang/rust#68303
 
 #[prelude_import]
 #[allow(unused)]

src/librustc/ty/layout.rs

@@ -356,12 +356,14 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
             debug!("univariant offset: {:?} field: {:#?}", offset, field);
             offsets[i as usize] = offset;
 
-            if let Some(mut niche) = field.largest_niche.clone() {
-                let available = niche.available(dl);
-                if available > largest_niche_available {
-                    largest_niche_available = available;
-                    niche.offset += offset;
-                    largest_niche = Some(niche);
+            if !repr.hide_niche() {
+                if let Some(mut niche) = field.largest_niche.clone() {
+                    let available = niche.available(dl);
+                    if available > largest_niche_available {
+                        largest_niche_available = available;
+                        niche.offset += offset;
+                        largest_niche = Some(niche);
+                    }
                 }
             }
 
@@ -838,7 +840,11 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                             }
 
                             // Update `largest_niche` if we have introduced a larger niche.
-                            let niche = Niche::from_scalar(dl, Size::ZERO, scalar.clone());
+                            let niche = if def.repr.hide_niche() {
+                                None
+                            } else {
+                                Niche::from_scalar(dl, Size::ZERO, scalar.clone())
+                            };
                             if let Some(niche) = niche {
                                 match &st.largest_niche {
                                     Some(largest_niche) => {
@@ -863,6 +869,11 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                     return Ok(tcx.intern_layout(st));
                 }
 
+                // At this point, we have handled all unions and
+                // structs. (We have also handled univariant enums
+                // that allow representation optimization.)
+                assert!(def.is_enum());
+
                 // The current code for niche-filling relies on variant indices
                 // instead of actual discriminants, so dataful enums with
                 // explicit discriminants (RFC #2363) would misbehave.

src/librustc/ty/mod.rs

@@ -2041,7 +2041,8 @@ bitflags! {
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
         const IS_LINEAR          = 1 << 3;
-
+        // If true, don't expose any niche to type's context.
+        const HIDE_NICHE         = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
         const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits |
                                    ReprFlags::IS_SIMD.bits |
@@ -2078,6 +2079,7 @@ impl ReprOptions {
                         ReprFlags::empty()
                     }
                     attr::ReprTransparent => ReprFlags::IS_TRANSPARENT,
+                    attr::ReprNoNiche => ReprFlags::HIDE_NICHE,
                     attr::ReprSimd => ReprFlags::IS_SIMD,
                     attr::ReprInt(i) => {
                         size = Some(i);
@@ -2118,6 +2120,10 @@ impl ReprOptions {
     pub fn linear(&self) -> bool {
         self.flags.contains(ReprFlags::IS_LINEAR)
     }
+    #[inline]
+    pub fn hide_niche(&self) -> bool {
+        self.flags.contains(ReprFlags::HIDE_NICHE)
+    }
 
     pub fn discr_type(&self) -> attr::IntType {
         self.int.unwrap_or(attr::SignedInt(ast::IntTy::Isize))

src/librustc_attr/builtin.rs

@@ -838,6 +838,7 @@ pub enum ReprAttr {
     ReprSimd,
     ReprTransparent,
     ReprAlign(u32),
+    ReprNoNiche,
 }
 
 #[derive(Eq, PartialEq, Debug, RustcEncodable, RustcDecodable, Copy, Clone, HashStable_Generic)]
@@ -893,6 +894,7 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> {
                         sym::packed => Some(ReprPacked(1)),
                         sym::simd => Some(ReprSimd),
                         sym::transparent => Some(ReprTransparent),
+                        sym::no_niche => Some(ReprNoNiche),
                         name => int_type_of_word(name).map(ReprInt),
                     };
 

src/librustc_builtin_macros/deriving/generic/mod.rs

@@ -825,7 +825,8 @@ fn find_repr_type_name(sess: &ParseSess, type_attrs: &[ast::Attribute]) -> &'sta
                 attr::ReprPacked(_)
                 | attr::ReprSimd
                 | attr::ReprAlign(_)
-                | attr::ReprTransparent => continue,
+                | attr::ReprTransparent
+                | attr::ReprNoNiche => continue,
 
                 attr::ReprC => "i32",
 

src/librustc_feature/active.rs

@@ -204,6 +204,10 @@ declare_features! (
     /// Added for testing E0705; perma-unstable.
     (active, test_2018_feature, "1.31.0", None, Some(Edition::Edition2018)),
 
+    /// Allows `#[repr(no_niche)]` (an implementation detail of `rustc`,
+    /// it is not on path for eventual stabilization).
+    (active, no_niche, "1.42.0", None, None),
+
     // no-tracking-issue-end
 
     // -------------------------------------------------------------------------

src/librustc_passes/check_attr.rs

@@ -16,9 +16,10 @@ use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::DUMMY_HIR_ID;
 use rustc_hir::{self, HirId, Item, ItemKind, TraitItem};
 use rustc_session::lint::builtin::{CONFLICTING_REPR_HINTS, UNUSED_ATTRIBUTES};
+use rustc_session::parse::feature_err;
 use rustc_span::symbol::sym;
 use rustc_span::Span;
-use syntax::ast::Attribute;
+use syntax::ast::{Attribute, NestedMetaItem};
 use syntax::attr;
 
 fn target_from_impl_item<'tcx>(tcx: TyCtxt<'tcx>, impl_item: &hir::ImplItem<'_>) -> Target {
@@ -287,6 +288,21 @@ impl CheckAttrVisitor<'tcx> {
                         _ => ("a", "struct, enum, or union"),
                     }
                 }
+                sym::no_niche => {
+                    if !self.tcx.features().enabled(sym::no_niche) {
+                        feature_err(
+                            &self.tcx.sess.parse_sess,
+                            sym::no_niche,
+                            hint.span(),
+                            "the attribute `repr(no_niche)` is currently unstable",
+                        )
+                        .emit();
+                    }
+                    match target {
+                        Target::Struct | Target::Enum => continue,
+                        _ => ("a", "struct or enum"),
+                    }
+                }
                 sym::i8
                 | sym::u8
                 | sym::i16
@@ -314,8 +330,10 @@ impl CheckAttrVisitor<'tcx> {
         // This is not ideal, but tracking precisely which ones are at fault is a huge hassle.
         let hint_spans = hints.iter().map(|hint| hint.span());
 
-        // Error on repr(transparent, <anything else>).
-        if is_transparent && hints.len() > 1 {
+        // Error on repr(transparent, <anything else apart from no_niche>).
+        let non_no_niche = |hint: &&NestedMetaItem| hint.name_or_empty() != sym::no_niche;
+        let non_no_niche_count = hints.iter().filter(non_no_niche).count();
+        if is_transparent && non_no_niche_count > 1 {
             let hint_spans: Vec<_> = hint_spans.clone().collect();
             struct_span_err!(
                 self.tcx.sess,

src/librustc_span/symbol.rs

@@ -491,6 +491,7 @@ symbols! {
         non_exhaustive,
         non_modrs_mods,
         no_sanitize,
+        no_niche,
         no_stack_check,
         no_start,
         no_std,
@@ -587,6 +588,7 @@ symbols! {
         repr128,
         repr_align,
         repr_align_enum,
+        repr_no_niche,
         repr_packed,
         repr_simd,
         repr_transparent,

src/libstd/sys/sgx/rwlock.rs

@@ -10,10 +10,10 @@ pub struct RWLock {
     writer: SpinMutex<WaitVariable<bool>>,
 }
 
-// Below is to check at compile time, that RWLock has size of 128 bytes.
+// Check at compile time that RWLock size matches C definition (see test_c_rwlock_initializer below)
 #[allow(dead_code)]
 unsafe fn rw_lock_size_assert(r: RWLock) {
-    mem::transmute::<RWLock, [u8; 128]>(r);
+    mem::transmute::<RWLock, [u8; 144]>(r);
 }
 
 impl RWLock {
@@ -210,15 +210,17 @@ mod tests {
     // be changed too.
     #[test]
     fn test_c_rwlock_initializer() {
+        #[rustfmt::skip]
         const RWLOCK_INIT: &[u8] = &[
-            0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-            0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x00 */ 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x10 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x20 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x30 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x40 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x50 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x60 */ 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x70 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            /* 0x80 */ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
         ];
 
         #[inline(never)]
@@ -239,7 +241,7 @@ mod tests {
             zero_stack();
             let mut init = MaybeUninit::<RWLock>::zeroed();
             rwlock_new(&mut init);
-            assert_eq!(mem::transmute::<_, [u8; 128]>(init.assume_init()).as_slice(), RWLOCK_INIT)
+            assert_eq!(mem::transmute::<_, [u8; 144]>(init.assume_init()).as_slice(), RWLOCK_INIT)
         };
     }
 }

src/test/ui/layout/unsafe-cell-hides-niche.rs

@@ -0,0 +1,32 @@
+// For rust-lang/rust#68303: the contents of `UnsafeCell<T>` cannot
+// participate in the niche-optimization for enum discriminants. This
+// test checks that an `Option<UnsafeCell<NonZeroU32>>` has the same
+// size in memory as an `Option<UnsafeCell<u32>>` (namely, 8 bytes).
+
+// run-pass
+
+#![feature(no_niche)]
+
+use std::cell::UnsafeCell;
+use std::mem::size_of;
+use std::num::NonZeroU32 as N32;
+
+struct Wrapper<T>(T);
+
+#[repr(transparent)]
+struct Transparent<T>(T);
+
+#[repr(no_niche)]
+struct NoNiche<T>(T);
+
+fn main() {
+    assert_eq!(size_of::<Option<Wrapper<u32>>>(),     8);
+    assert_eq!(size_of::<Option<Wrapper<N32>>>(),     4);
+    assert_eq!(size_of::<Option<Transparent<u32>>>(), 8);
+    assert_eq!(size_of::<Option<Transparent<N32>>>(), 4);
+    assert_eq!(size_of::<Option<NoNiche<u32>>>(),     8);
+    assert_eq!(size_of::<Option<NoNiche<N32>>>(),     8);
+
+    assert_eq!(size_of::<Option<UnsafeCell<u32>>>(),  8);
+    assert_eq!(size_of::<Option<UnsafeCell<N32>>>(),  8);
+}

src/test/ui/repr/feature-gate-no-niche.rs

@@ -0,0 +1,20 @@
+use std::num::NonZeroU8 as N8;
+use std::num::NonZeroU16 as N16;
+
+#[repr(no_niche)]
+pub struct Cloaked(N16);
+//~^^ ERROR the attribute `repr(no_niche)` is currently unstable [E0658]
+
+#[repr(transparent, no_niche)]
+pub struct Shadowy(N16);
+//~^^ ERROR the attribute `repr(no_niche)` is currently unstable [E0658]
+
+#[repr(no_niche)]
+pub enum Cloaked1 { _A(N16), }
+//~^^ ERROR the attribute `repr(no_niche)` is currently unstable [E0658]
+
+#[repr(no_niche)]
+pub enum Cloaked2 { _A(N16), _B(u8, N8) }
+//~^^ ERROR the attribute `repr(no_niche)` is currently unstable [E0658]
+
+fn main() { }

src/test/ui/repr/feature-gate-no-niche.stderr

@@ -0,0 +1,35 @@
+error[E0658]: the attribute `repr(no_niche)` is currently unstable
+  --> $DIR/feature-gate-no-niche.rs:4:8
+   |
+LL | #[repr(no_niche)]
+   |        ^^^^^^^^
+   |
+   = help: add `#![feature(no_niche)]` to the crate attributes to enable
+
+error[E0658]: the attribute `repr(no_niche)` is currently unstable
+  --> $DIR/feature-gate-no-niche.rs:8:21
+   |
+LL | #[repr(transparent, no_niche)]
+   |                     ^^^^^^^^
+   |
+   = help: add `#![feature(no_niche)]` to the crate attributes to enable
+
+error[E0658]: the attribute `repr(no_niche)` is currently unstable
+  --> $DIR/feature-gate-no-niche.rs:12:8
+   |
+LL | #[repr(no_niche)]
+   |        ^^^^^^^^
+   |
+   = help: add `#![feature(no_niche)]` to the crate attributes to enable
+
+error[E0658]: the attribute `repr(no_niche)` is currently unstable
+  --> $DIR/feature-gate-no-niche.rs:16:8
+   |
+LL | #[repr(no_niche)]
+   |        ^^^^^^^^
+   |
+   = help: add `#![feature(no_niche)]` to the crate attributes to enable
+
+error: aborting due to 4 previous errors
+
+For more information about this error, try `rustc --explain E0658`.

src/test/ui/repr/repr-no-niche-inapplicable-to-unions.rs

@@ -0,0 +1,14 @@
+#![feature(no_niche)]
+
+use std::num::NonZeroU8 as N8;
+use std::num::NonZeroU16 as N16;
+
+#[repr(no_niche)]
+pub union Cloaked1 { _A: N16 }
+//~^^ ERROR attribute should be applied to struct or enum [E0517]
+
+#[repr(no_niche)]
+pub union Cloaked2 { _A: N16, _B: (u8, N8) }
+//~^^ ERROR attribute should be applied to struct or enum [E0517]
+
+fn main() { }

src/test/ui/repr/repr-no-niche-inapplicable-to-unions.stderr

@@ -0,0 +1,19 @@
+error[E0517]: attribute should be applied to struct or enum
+  --> $DIR/repr-no-niche-inapplicable-to-unions.rs:6:8
+   |
+LL | #[repr(no_niche)]
+   |        ^^^^^^^^
+LL | pub union Cloaked1 { _A: N16 }
+   | ------------------------------ not a struct or enum
+
+error[E0517]: attribute should be applied to struct or enum
+  --> $DIR/repr-no-niche-inapplicable-to-unions.rs:10:8
+   |
+LL | #[repr(no_niche)]
+   |        ^^^^^^^^
+LL | pub union Cloaked2 { _A: N16, _B: (u8, N8) }
+   | -------------------------------------------- not a struct or enum
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0517`.