CoVec mvp

Cargo.lock

@@ -303,7 +303,7 @@ dependencies = [
 
 [[package]]
 name = "cargo"
-version = "0.69.0"
+version = "0.68.0"
 dependencies = [
  "anyhow",
  "bytesize",

compiler/rustc_ast/src/ast.rs

@@ -25,6 +25,7 @@ pub use UnsafeSource::*;
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter};
 use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream};
+use core::alloc::GlobalCoAllocMeta;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lrc;
@@ -36,7 +37,6 @@ use rustc_span::{Span, DUMMY_SP};
 use std::fmt;
 use std::mem;
 use thin_vec::{thin_vec, ThinVec};
-
 /// A "Label" is an identifier of some point in sources,
 /// e.g. in the following code:
 ///
@@ -3083,26 +3083,26 @@ mod size_asserts {
     static_assert_size!(AssocItem, 104);
     static_assert_size!(AssocItemKind, 32);
     static_assert_size!(Attribute, 32);
-    static_assert_size!(Block, 48);
-    static_assert_size!(Expr, 72);
-    static_assert_size!(ExprKind, 40);
-    static_assert_size!(Fn, 184);
+    static_assert_size!(Block, 48 + mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Expr, 72 + mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(ExprKind, 40 + mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Fn, 184 + 2 * mem::size_of::<GlobalCoAllocMeta>());
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
-    static_assert_size!(GenericBound, 72);
-    static_assert_size!(Generics, 72);
-    static_assert_size!(Impl, 184);
-    static_assert_size!(Item, 184);
-    static_assert_size!(ItemKind, 112);
+    static_assert_size!(GenericBound, 72 + mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Generics, 72 + 2 * mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Impl, 184 + 3 * mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(Item, 184 + 3 * mem::size_of::<GlobalCoAllocMeta>());
+    static_assert_size!(ItemKind, 112 + 3 * mem::size_of::<GlobalCoAllocMeta>());
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
     static_assert_size!(MetaItemLit, 40);
     static_assert_size!(Param, 40);
-    static_assert_size!(Pat, 88);
+    static_assert_size!(Pat, 88 + mem::size_of::<GlobalCoAllocMeta>());
     static_assert_size!(Path, 24);
     static_assert_size!(PathSegment, 24);
-    static_assert_size!(PatKind, 64);
+    static_assert_size!(PatKind, 64 + mem::size_of::<GlobalCoAllocMeta>());
     static_assert_size!(Stmt, 32);
     static_assert_size!(StmtKind, 16);
     static_assert_size!(Ty, 64);

compiler/rustc_ast/src/lib.rs

@@ -12,6 +12,7 @@
 #![feature(box_patterns)]
 #![feature(const_default_impls)]
 #![feature(const_trait_impl)]
+#![feature(global_co_alloc_meta)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
 #![feature(min_specialization)]

compiler/rustc_middle/src/lib.rs

@@ -32,6 +32,7 @@
 #![feature(exhaustive_patterns)]
 #![feature(generators)]
 #![feature(get_mut_unchecked)]
+#![feature(global_co_alloc_meta)]
 #![feature(if_let_guard)]
 #![feature(iter_from_generator)]
 #![feature(negative_impls)]

compiler/rustc_middle/src/mir/mod.rs

@@ -13,6 +13,7 @@ use crate::ty::visit::{TypeVisitable, TypeVisitor};
 use crate::ty::{self, DefIdTree, List, Ty, TyCtxt};
 use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
+use core::alloc::GlobalCoAllocMeta;
 
 use rustc_data_structures::captures::Captures;
 use rustc_errors::ErrorGuaranteed;
@@ -3060,7 +3061,7 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 144);
+    static_assert_size!(BasicBlockData<'_>, 144 + mem::size_of::<GlobalCoAllocMeta>());
     static_assert_size!(LocalDecl<'_>, 56);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);

compiler/rustc_middle/src/mir/syntax.rs

@@ -4,6 +4,8 @@
 //! The intention is that this file only contains datatype declarations, no code.
 
 use super::{BasicBlock, Constant, Field, Local, SwitchTargets, UserTypeProjection};
+use core::alloc::GlobalCoAllocMeta;
+use core::mem;
 
 use crate::mir::coverage::{CodeRegion, CoverageKind};
 use crate::traits::Reveal;
@@ -1268,6 +1270,6 @@ mod size_asserts {
     static_assert_size!(Operand<'_>, 24);
     static_assert_size!(Place<'_>, 16);
     static_assert_size!(PlaceElem<'_>, 24);
-    static_assert_size!(Rvalue<'_>, 40);
+    static_assert_size!(Rvalue<'_>, 40 + mem::size_of::<GlobalCoAllocMeta>());
     // tidy-alphabetical-end
 }

compiler/rustc_parse/src/lib.rs

@@ -2,6 +2,7 @@
 
 #![feature(array_windows)]
 #![feature(box_patterns)]
+#![feature(global_co_alloc_meta)]
 #![feature(if_let_guard)]
 #![feature(iter_intersperse)]
 #![feature(let_chains)]

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -1,4 +1,6 @@
 use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
+use core::alloc::GlobalCoAllocMeta;
+use core::mem;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{AttrTokenStream, AttributesData, ToAttrTokenStream};
 use rustc_ast::tokenstream::{AttrTokenTree, DelimSpan, LazyAttrTokenStream, Spacing};
@@ -469,6 +471,6 @@ mod size_asserts {
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
     static_assert_size!(AttrWrapper, 16);
-    static_assert_size!(LazyAttrTokenStreamImpl, 144);
+    static_assert_size!(LazyAttrTokenStreamImpl, 144 + mem::size_of::<GlobalCoAllocMeta>());
     // tidy-alphabetical-end
 }

compiler/rustc_parse/src/parser/mod.rs

@@ -12,6 +12,7 @@ mod ty;
 
 use crate::lexer::UnmatchedBrace;
 pub use attr_wrapper::AttrWrapper;
+use core::alloc::GlobalCoAllocMeta;
 pub use diagnostics::AttemptLocalParseRecovery;
 pub(crate) use item::FnParseMode;
 pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
@@ -168,7 +169,10 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Parser<'_>, 336);
+rustc_data_structures::static_assert_size!(
+    Parser<'_>,
+    336 + 4 * mem::size_of::<GlobalCoAllocMeta>()
+);
 
 /// Stores span information about a closure.
 #[derive(Clone)]

compiler/rustc_trait_selection/src/lib.rs

@@ -15,6 +15,7 @@
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
 #![feature(drain_filter)]
+#![feature(global_co_alloc_meta)]
 #![feature(hash_drain_filter)]
 #![feature(let_chains)]
 #![feature(if_let_guard)]

compiler/rustc_trait_selection/src/traits/fulfill.rs

@@ -1,4 +1,6 @@
 use crate::infer::{InferCtxt, TyOrConstInferVar};
+use core::alloc::GlobalCoAllocMeta;
+use core::mem;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::obligation_forest::ProcessResult;
 use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome};
@@ -80,7 +82,7 @@ pub struct PendingPredicateObligation<'tcx> {
 
 // `PendingPredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(PendingPredicateObligation<'_>, 72);
+static_assert_size!(PendingPredicateObligation<'_>, 72 + mem::size_of::<GlobalCoAllocMeta>());
 
 impl<'a, 'tcx> FulfillmentContext<'tcx> {
     /// Creates a new fulfillment context.

library/alloc/src/boxed.rs

@@ -146,6 +146,7 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
+use core::alloc;
 use core::any::Any;
 use core::async_iter::AsyncIterator;
 use core::borrow;
@@ -698,7 +699,12 @@ impl<T> Box<[T]> {
                 Err(_) => return Err(AllocError),
             };
             let ptr = Global.allocate(layout)?;
-            Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len))
+            Ok(RawVec::<T, Global, false>::from_raw_parts_in(
+                ptr.as_mut_ptr() as *mut _,
+                len,
+                Global,
+            )
+            .into_box(len))
         }
     }
 
@@ -730,12 +736,20 @@ impl<T> Box<[T]> {
                 Err(_) => return Err(AllocError),
             };
             let ptr = Global.allocate_zeroed(layout)?;
-            Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len))
+            Ok(RawVec::<T, Global, false>::from_raw_parts_in(
+                ptr.as_mut_ptr() as *mut _,
+                len,
+                Global,
+            )
+            .into_box(len))
         }
     }
 }
 
-impl<T, A: Allocator> Box<[T], A> {
+impl<T, A: Allocator> Box<[T], A>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots::<A>()]:,
+{
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator.
     ///
     /// # Examples
@@ -763,7 +777,10 @@ impl<T, A: Allocator> Box<[T], A> {
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
-        unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) }
+        unsafe {
+            RawVec::<T, A, { alloc::SHORT_TERM_VEC_PREFERS_COOP }>::with_capacity_in(len, alloc)
+                .into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator,
@@ -791,7 +808,12 @@ impl<T, A: Allocator> Box<[T], A> {
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
-        unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) }
+        unsafe {
+            RawVec::<T, A, { alloc::SHORT_TERM_VEC_PREFERS_COOP }>::with_capacity_zeroed_in(
+                len, alloc,
+            )
+            .into_box(len)
+        }
     }
 }
 
@@ -1661,8 +1683,12 @@ impl<T, const N: usize> TryFrom<Box<[T]>> for Box<[T; N]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")]
-impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> {
-    type Error = Vec<T>;
+impl<T, const N: usize, const COOP_PREFERRED: bool> TryFrom<Vec<T, Global, COOP_PREFERRED>>
+    for Box<[T; N]>
+where
+    [(); crate::co_alloc_metadata_num_slots_with_preference_global(COOP_PREFERRED)]:,
+{
+    type Error = Vec<T, Global, COOP_PREFERRED>;
 
     /// Attempts to convert a `Vec<T>` into a `Box<[T; N]>`.
     ///
@@ -1682,7 +1708,7 @@ impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> {
     /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap();
     /// assert_eq!(state.len(), 100);
     /// ```
-    fn try_from(vec: Vec<T>) -> Result<Self, Self::Error> {
+    fn try_from(vec: Vec<T, Global, COOP_PREFERRED>) -> Result<Self, Self::Error> {
         if vec.len() == N {
             let boxed_slice = vec.into_boxed_slice();
             Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) })
@@ -2019,7 +2045,10 @@ impl<I> FromIterator<I> for Box<[I]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "box_slice_clone", since = "1.3.0")]
-impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A> {
+impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots::<A>()]:,
+{
     fn clone(&self) -> Self {
         let alloc = Box::allocator(self).clone();
         self.to_vec_in(alloc).into_boxed_slice()

library/alloc/src/collections/binary_heap.rs

@@ -143,11 +143,13 @@
 #![allow(missing_docs)]
 #![stable(feature = "rust1", since = "1.0.0")]
 
-use core::fmt;
 use core::iter::{FromIterator, FusedIterator, InPlaceIterable, SourceIter, TrustedLen};
 use core::mem::{self, swap, ManuallyDrop};
 use core::ops::{Deref, DerefMut};
 use core::ptr;
+use core::{alloc, fmt};
+
+use crate::alloc::Global;
 
 use crate::collections::TryReserveError;
 use crate::slice;
@@ -1196,7 +1198,7 @@ impl<T> BinaryHeap<T> {
     /// ```
     #[inline]
     #[stable(feature = "drain", since = "1.6.0")]
-    pub fn drain(&mut self) -> Drain<'_, T> {
+    pub fn drain(&mut self) -> Drain<'_, T, { alloc::SHORT_TERM_VEC_PREFERS_COOP }> {
         Drain { iter: self.data.drain(..) }
     }
 
@@ -1476,12 +1478,18 @@ unsafe impl<T: Ord> TrustedLen for IntoIterSorted<T> {}
 /// [`drain`]: BinaryHeap::drain
 #[stable(feature = "drain", since = "1.6.0")]
 #[derive(Debug)]
-pub struct Drain<'a, T: 'a> {
-    iter: vec::Drain<'a, T>,
+pub struct Drain<'a, T: 'a, const COOP_PREFERRED: bool>
+where
+    [(); crate::co_alloc_metadata_num_slots_with_preference_global(COOP_PREFERRED)]:,
+{
+    iter: vec::Drain<'a, T, Global, COOP_PREFERRED>,
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T> Iterator for Drain<'_, T> {
+impl<T, const COOP_PREFERRED: bool> Iterator for Drain<'_, T, COOP_PREFERRED>
+where
+    [(); crate::co_alloc_metadata_num_slots_with_preference_global(COOP_PREFERRED)]:,
+{
     type Item = T;
 
     #[inline]
@@ -1496,22 +1504,22 @@ impl<T> Iterator for Drain<'_, T> {
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T> DoubleEndedIterator for Drain<'_, T> {
+impl<T, const COOP_PREFERRED: bool> DoubleEndedIterator for Drain<'_, T, COOP_PREFERRED> {
     #[inline]
     fn next_back(&mut self) -> Option<T> {
         self.iter.next_back()
     }
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T> ExactSizeIterator for Drain<'_, T> {
+impl<T, const COOP_PREFERRED: bool> ExactSizeIterator for Drain<'_, T, COOP_PREFERRED> {
     fn is_empty(&self) -> bool {
         self.iter.is_empty()
     }
 }
 
 #[stable(feature = "fused", since = "1.26.0")]
-impl<T> FusedIterator for Drain<'_, T> {}
+impl<T, const COOP_PREFERRED: bool> FusedIterator for Drain<'_, T, COOP_PREFERRED> {}
 
 /// A draining iterator over the elements of a `BinaryHeap`.
 ///