Auto merge of #59693 - nnethercote:64-bit-Spans, r=petrochenkov

Increase `Span` from 4 bytes to 8 bytes.

This increases the size of some important types, such as `ast::Expr` and
`mir::Statement`. However, it drastically reduces how much the interner
is used, and the fields are more natural sizes that don't require bit
operations to extract.

As a result, instruction counts drop across a range of workloads, by as
much as 10% for `script-servo` incremental builds.

Peak memory usage goes up a little for some cases, but down by more for
some other cases -- as much as 18% for non-incremental builds of
`packed-simd`.

The commit also:
- removes the `repr(packed)`, because it has negligible effect, but can
  cause undefined behaviour;
- replaces explicit impls of common traits (`Copy`, `PartialEq`, etc.)
  with derived ones.

r? @petrochenkov

src/librustc/mir/mod.rs

@@ -1735,7 +1735,7 @@ pub struct Statement<'tcx> {
 
 // `Statement` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-static_assert!(MEM_SIZE_OF_STATEMENT: mem::size_of::<Statement<'_>>() == 48);
+static_assert!(MEM_SIZE_OF_STATEMENT: mem::size_of::<Statement<'_>>() == 56);
 
 impl<'tcx> Statement<'tcx> {
     /// Changes a statement to a nop. This is both faster than deleting instructions and avoids

src/libsyntax/ast.rs

@@ -946,7 +946,7 @@ pub struct Expr {
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::<Expr>() == 88);
+static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::<Expr>() == 96);
 
 impl Expr {
     /// Whether this expression would be valid somewhere that expects a value; for example, an `if`

src/libsyntax_pos/hygiene.rs

@@ -218,14 +218,17 @@ pub fn clear_markings() {
 }
 
 impl SyntaxContext {
+    #[inline]
     pub const fn empty() -> Self {
         SyntaxContext(0)
     }
 
+    #[inline]
     crate fn as_u32(self) -> u32 {
         self.0
     }
 
+    #[inline]
     crate fn from_u32(raw: u32) -> SyntaxContext {
         SyntaxContext(raw)
     }

src/libsyntax_pos/span_encoding.rs

@@ -9,122 +9,106 @@ use crate::{BytePos, SpanData};
 use crate::hygiene::SyntaxContext;
 
 use rustc_data_structures::fx::FxHashMap;
-use std::hash::{Hash, Hasher};
 
 /// A compressed span.
-/// Contains either fields of `SpanData` inline if they are small, or index into span interner.
-/// The primary goal of `Span` is to be as small as possible and fit into other structures
-/// (that's why it uses `packed` as well). Decoding speed is the second priority.
-/// See `SpanData` for the info on span fields in decoded representation.
-#[repr(packed)]
-pub struct Span(u32);
-
-impl Copy for Span {}
-impl Clone for Span {
-    #[inline]
-    fn clone(&self) -> Span {
-        *self
-    }
-}
-impl PartialEq for Span {
-    #[inline]
-    fn eq(&self, other: &Span) -> bool {
-        let a = self.0;
-        let b = other.0;
-        a == b
-    }
-}
-impl Eq for Span {}
-impl Hash for Span {
-    #[inline]
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let a = self.0;
-        a.hash(state)
-    }
+///
+/// `SpanData` is 12 bytes, which is a bit too big to stick everywhere. `Span`
+/// is a form that only takes up 8 bytes, with less space for the length and
+/// context. The vast majority (99.9%+) of `SpanData` instances will fit within
+/// those 8 bytes; any `SpanData` whose fields don't fit into a `Span` are
+/// stored in a separate interner table, and the `Span` will index into that
+/// table. Interning is rare enough that the cost is low, but common enough
+/// that the code is exercised regularly.
+///
+/// An earlier version of this code used only 4 bytes for `Span`, but that was
+/// slower because only 80--90% of spans could be stored inline (even less in
+/// very large crates) and so the interner was used a lot more.
+///
+/// Inline (compressed) format:
+/// - `span.base_or_index == span_data.lo`
+/// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<= MAX_LEN`)
+/// - `span.ctxt == span_data.ctxt` (must be `<= MAX_CTXT`)
+///
+/// Interned format:
+/// - `span.base_or_index == index` (indexes into the interner table)
+/// - `span.len_or_tag == LEN_TAG` (high bit set, all other bits are zero)
+/// - `span.ctxt == 0`
+///
+/// The inline form uses 0 for the tag value (rather than 1) so that we don't
+/// need to mask out the tag bit when getting the length, and so that the
+/// dummy span can be all zeroes.
+///
+/// Notes about the choice of field sizes:
+/// - `base` is 32 bits in both `Span` and `SpanData`, which means that `base`
+///   values never cause interning. The number of bits needed for `base`
+///   depends on the crate size. 32 bits allows up to 4 GiB of code in a crate.
+///   `script-servo` is the largest crate in `rustc-perf`, requiring 26 bits
+///   for some spans.
+/// - `len` is 15 bits in `Span` (a u16, minus 1 bit for the tag) and 32 bits
+///   in `SpanData`, which means that large `len` values will cause interning.
+///   The number of bits needed for `len` does not depend on the crate size.
+///   The most common number of bits for `len` are 0--7, with a peak usually at
+///   3 or 4, and then it drops off quickly from 8 onwards. 15 bits is enough
+///   for 99.99%+ of cases, but larger values (sometimes 20+ bits) might occur
+///   dozens of times in a typical crate.
+/// - `ctxt` is 16 bits in `Span` and 32 bits in `SpanData`, which means that
+///   large `ctxt` values will cause interning. The number of bits needed for
+///   `ctxt` values depend partly on the crate size and partly on the form of
+///   the code. No crates in `rustc-perf` need more than 15 bits for `ctxt`,
+///   but larger crates might need more than 16 bits.
+///
+#[derive(Clone, Copy, Eq, PartialEq, Hash)]
+pub struct Span {
+    base_or_index: u32,
+    len_or_tag: u16,
+    ctxt_or_zero: u16
 }
 
+const LEN_TAG: u16 = 0b1000_0000_0000_0000;
+const MAX_LEN: u32 = 0b0111_1111_1111_1111;
+const MAX_CTXT: u32 = 0b1111_1111_1111_1111;
+
 /// Dummy span, both position and length are zero, syntax context is zero as well.
-/// This span is kept inline and encoded with format 0.
-pub const DUMMY_SP: Span = Span(0);
+pub const DUMMY_SP: Span = Span { base_or_index: 0, len_or_tag: 0, ctxt_or_zero: 0 };
 
 impl Span {
     #[inline]
-    pub fn new(lo: BytePos, hi: BytePos, ctxt: SyntaxContext) -> Self {
-        encode(&match lo <= hi {
-            true => SpanData { lo, hi, ctxt },
-            false => SpanData { lo: hi, hi: lo, ctxt },
-        })
+    pub fn new(mut lo: BytePos, mut hi: BytePos, ctxt: SyntaxContext) -> Self {
+        if lo > hi {
+            std::mem::swap(&mut lo, &mut hi);
+        }
+
+        let (base, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32());
+
+        if len <= MAX_LEN && ctxt2 <= MAX_CTXT {
+            // Inline format.
+            Span { base_or_index: base, len_or_tag: len as u16, ctxt_or_zero: ctxt2 as u16 }
+        } else {
+            // Interned format.
+            let index = with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt }));
+            Span { base_or_index: index, len_or_tag: LEN_TAG, ctxt_or_zero: 0 }
+        }
     }
 
     #[inline]
     pub fn data(self) -> SpanData {
-        decode(self)
+        if self.len_or_tag != LEN_TAG {
+            // Inline format.
+            debug_assert!(self.len_or_tag as u32 <= MAX_LEN);
+            SpanData {
+                lo: BytePos(self.base_or_index),
+                hi: BytePos(self.base_or_index + self.len_or_tag as u32),
+                ctxt: SyntaxContext::from_u32(self.ctxt_or_zero as u32),
+            }
+        } else {
+            // Interned format.
+            debug_assert!(self.ctxt_or_zero == 0);
+            let index = self.base_or_index;
+            with_span_interner(|interner| *interner.get(index))
+        }
     }
 }
 
-// Tags
-const TAG_INLINE: u32 = 0;
-const TAG_INTERNED: u32 = 1;
-const TAG_MASK: u32 = 1;
-
-// Fields indexes
-const BASE_INDEX: usize = 0;
-const LEN_INDEX: usize = 1;
-const CTXT_INDEX: usize = 2;
-
-// Tag = 0, inline format.
-// -------------------------------------------------------------
-// | base 31:7  | len 6:1  | ctxt (currently 0 bits) | tag 0:0 |
-// -------------------------------------------------------------
-// Since there are zero bits for ctxt, only SpanData with a 0 SyntaxContext
-// can be inline.
-const INLINE_SIZES: [u32; 3] = [25, 6, 0];
-const INLINE_OFFSETS: [u32; 3] = [7, 1, 1];
-
-// Tag = 1, interned format.
-// ------------------------
-// | index 31:1 | tag 0:0 |
-// ------------------------
-const INTERNED_INDEX_SIZE: u32 = 31;
-const INTERNED_INDEX_OFFSET: u32 = 1;
-
-#[inline]
-fn encode(sd: &SpanData) -> Span {
-    let (base, len, ctxt) = (sd.lo.0, sd.hi.0 - sd.lo.0, sd.ctxt.as_u32());
-
-    let val = if (base >> INLINE_SIZES[BASE_INDEX]) == 0 &&
-                 (len >> INLINE_SIZES[LEN_INDEX]) == 0 &&
-                 (ctxt >> INLINE_SIZES[CTXT_INDEX]) == 0 {
-        (base << INLINE_OFFSETS[BASE_INDEX]) | (len << INLINE_OFFSETS[LEN_INDEX]) |
-        (ctxt << INLINE_OFFSETS[CTXT_INDEX]) | TAG_INLINE
-    } else {
-        let index = with_span_interner(|interner| interner.intern(sd));
-        (index << INTERNED_INDEX_OFFSET) | TAG_INTERNED
-    };
-    Span(val)
-}
-
-#[inline]
-fn decode(span: Span) -> SpanData {
-    let val = span.0;
-
-    // Extract a field at position `pos` having size `size`.
-    let extract = |pos: u32, size: u32| {
-        let mask = ((!0u32) as u64 >> (32 - size)) as u32; // Can't shift u32 by 32
-        (val >> pos) & mask
-    };
-
-    let (base, len, ctxt) = if val & TAG_MASK == TAG_INLINE {(
-        extract(INLINE_OFFSETS[BASE_INDEX], INLINE_SIZES[BASE_INDEX]),
-        extract(INLINE_OFFSETS[LEN_INDEX], INLINE_SIZES[LEN_INDEX]),
-        extract(INLINE_OFFSETS[CTXT_INDEX], INLINE_SIZES[CTXT_INDEX]),
-    )} else {
-        let index = extract(INTERNED_INDEX_OFFSET, INTERNED_INDEX_SIZE);
-        return with_span_interner(|interner| *interner.get(index));
-    };
-    SpanData { lo: BytePos(base), hi: BytePos(base + len), ctxt: SyntaxContext::from_u32(ctxt) }
-}
-
 #[derive(Default)]
 pub struct SpanInterner {
     spans: FxHashMap<SpanData, u32>,