Restructure Rc and Arc to support StructAlloc

Author: Julian Orth

library/alloc/src/rc.rs

@@ -257,12 +257,13 @@ use core::hash::{Hash, Hasher};
 use core::intrinsics::abort;
 use core::iter;
 use core::marker::{self, PhantomData, Unpin, Unsize};
-use core::mem::{self, align_of_val_raw, forget, size_of_val};
+use core::mem::{self, forget, size_of_val};
 use core::ops::{CoerceUnsized, Deref, DispatchFromDyn, Receiver};
 use core::pin::Pin;
 use core::ptr::{self, NonNull};
 use core::slice::from_raw_parts_mut;
 
+use crate::alloc::struct_alloc::StructAlloc;
 use crate::alloc::{
     box_free, handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw,
 };
@@ -273,13 +274,31 @@ use crate::vec::Vec;
 #[cfg(test)]
 mod tests;
 
-// This is repr(C) to future-proof against possible field-reordering, which
-// would interfere with otherwise safe [into|from]_raw() of transmutable
-// inner types.
-#[repr(C)]
-struct RcBox<T: ?Sized> {
+struct RcBoxMetadata {
     strong: Cell<usize>,
     weak: Cell<usize>,
+}
+
+impl RcBoxMetadata {
+    // There is an implicit weak pointer owned by all the strong
+    // pointers, which ensures that the weak destructor never frees
+    // the allocation while the strong destructor is running, even
+    // if the weak pointer is stored inside the strong one.
+    #[inline]
+    fn new_strong() -> Self {
+        Self { strong: Cell::new(1), weak: Cell::new(1) }
+    }
+
+    #[inline]
+    fn new_weak() -> Self {
+        Self { strong: Cell::new(0), weak: Cell::new(1) }
+    }
+}
+
+// This is repr(C) to support StructAlloc
+#[repr(C)]
+struct RcBox<T: ?Sized> {
+    meta: RcBoxMetadata,
     value: T,
 }
 
@@ -340,13 +359,7 @@ impl<T> Rc<T> {
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn new(value: T) -> Rc<T> {
-        // There is an implicit weak pointer owned by all the strong
-        // pointers, which ensures that the weak destructor never frees
-        // the allocation while the strong destructor is running, even
-        // if the weak pointer is stored inside the strong one.
-        Self::from_inner(
-            Box::leak(box RcBox { strong: Cell::new(1), weak: Cell::new(1), value }).into(),
-        )
+        Self::from_inner(Box::leak(box RcBox { meta: RcBoxMetadata::new_strong(), value }).into())
     }
 
     /// Constructs a new `Rc<T>` using a weak reference to itself. Attempting
@@ -378,8 +391,7 @@ impl<T> Rc<T> {
         // Construct the inner in the "uninitialized" state with a single
         // weak reference.
         let uninit_ptr: NonNull<_> = Box::leak(box RcBox {
-            strong: Cell::new(0),
-            weak: Cell::new(1),
+            meta: RcBoxMetadata::new_weak(),
             value: mem::MaybeUninit::<T>::uninit(),
         })
         .into();
@@ -400,9 +412,9 @@ impl<T> Rc<T> {
             let inner = init_ptr.as_ptr();
             ptr::write(ptr::addr_of_mut!((*inner).value), data);
 
-            let prev_value = (*inner).strong.get();
+            let prev_value = (*inner).meta.strong.get();
             debug_assert_eq!(prev_value, 0, "No prior strong references should exist");
-            (*inner).strong.set(1);
+            (*inner).meta.strong.set(1);
         }
 
         let strong = Rc::from_inner(init_ptr);
@@ -489,13 +501,8 @@ impl<T> Rc<T> {
     /// ```
     #[unstable(feature = "allocator_api", issue = "32838")]
     pub fn try_new(value: T) -> Result<Rc<T>, AllocError> {
-        // There is an implicit weak pointer owned by all the strong
-        // pointers, which ensures that the weak destructor never frees
-        // the allocation while the strong destructor is running, even
-        // if the weak pointer is stored inside the strong one.
         Ok(Self::from_inner(
-            Box::leak(Box::try_new(RcBox { strong: Cell::new(1), weak: Cell::new(1), value })?)
-                .into(),
+            Box::leak(Box::try_new(RcBox { meta: RcBoxMetadata::new_strong(), value })?).into(),
         ))
     }
 
@@ -1170,8 +1177,8 @@ impl<T: ?Sized> Rc<T> {
         unsafe {
             debug_assert_eq!(Layout::for_value(&*inner), layout);
 
-            ptr::write(&mut (*inner).strong, Cell::new(1));
-            ptr::write(&mut (*inner).weak, Cell::new(1));
+            ptr::write(&mut (*inner).meta.strong, Cell::new(1));
+            ptr::write(&mut (*inner).meta.weak, Cell::new(1));
         }
 
         Ok(inner)
@@ -2087,7 +2094,7 @@ impl<T: ?Sized> Weak<T> {
             // is dropped, the data field will be dropped in-place).
             Some(unsafe {
                 let ptr = self.ptr.as_ptr();
-                WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak }
+                WeakInner { strong: &(*ptr).meta.strong, weak: &(*ptr).meta.weak }
             })
         }
     }
@@ -2296,12 +2303,12 @@ trait RcInnerPtr {
 impl<T: ?Sized> RcInnerPtr for RcBox<T> {
     #[inline(always)]
     fn weak_ref(&self) -> &Cell<usize> {
-        &self.weak
+        &self.meta.weak
     }
 
     #[inline(always)]
     fn strong_ref(&self) -> &Cell<usize> {
-        &self.strong
+        &self.meta.strong
     }
 }
 
@@ -2334,24 +2341,21 @@ impl<T: ?Sized> AsRef<T> for Rc<T> {
 #[stable(feature = "pin", since = "1.33.0")]
 impl<T: ?Sized> Unpin for Rc<T> {}
 
+type RcStructAlloc = StructAlloc<RcBoxMetadata>;
+
 /// Get the offset within an `RcBox` for the payload behind a pointer.
 ///
 /// # Safety
 ///
 /// The pointer must point to (and have valid metadata for) a previously
 /// valid instance of T, but the T is allowed to be dropped.
-unsafe fn data_offset<T: ?Sized>(ptr: *const T) -> isize {
-    // Align the unsized value to the end of the RcBox.
-    // Because RcBox is repr(C), it will always be the last field in memory.
-    // SAFETY: since the only unsized types possible are slices, trait objects,
-    // and extern types, the input safety requirement is currently enough to
-    // satisfy the requirements of align_of_val_raw; this is an implementation
-    // detail of the language that may not be relied upon outside of std.
-    unsafe { data_offset_align(align_of_val_raw(ptr)) }
-}
-
-#[inline]
-fn data_offset_align(align: usize) -> isize {
-    let layout = Layout::new::<RcBox<()>>();
-    (layout.size() + layout.padding_needed_for(align)) as isize
+unsafe fn data_offset<T: ?Sized>(data_ptr: *const T) -> isize {
+    unsafe {
+        // SAFETY: since the only unsized types possible are slices, trait objects,
+        // and extern types, the input safety requirement is currently enough to
+        // satisfy the requirements of for_value_raw; this is an implementation
+        // detail of the language that may not be relied upon outside of std.
+        let data_layout = Layout::for_value_raw(data_ptr);
+        RcStructAlloc::offset_of_data(data_layout) as isize
+    }
 }

library/alloc/src/sync.rs

@@ -14,14 +14,15 @@ use core::hint;
 use core::intrinsics::abort;
 use core::iter;
 use core::marker::{PhantomData, Unpin, Unsize};
-use core::mem::{self, align_of_val_raw, size_of_val};
+use core::mem::{self, size_of_val};
 use core::ops::{CoerceUnsized, Deref, DispatchFromDyn, Receiver};
 use core::pin::Pin;
 use core::ptr::{self, NonNull};
 use core::slice::from_raw_parts_mut;
 use core::sync::atomic;
 use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst};
 
+use crate::alloc::struct_alloc::StructAlloc;
 use crate::alloc::{
     box_free, handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw,
 };
@@ -296,18 +297,33 @@ impl<T: ?Sized + fmt::Debug> fmt::Debug for Weak<T> {
     }
 }
 
-// This is repr(C) to future-proof against possible field-reordering, which
-// would interfere with otherwise safe [into|from]_raw() of transmutable
-// inner types.
-#[repr(C)]
-struct ArcInner<T: ?Sized> {
+struct ArcInnerMetadata {
     strong: atomic::AtomicUsize,
 
     // the value usize::MAX acts as a sentinel for temporarily "locking" the
     // ability to upgrade weak pointers or downgrade strong ones; this is used
     // to avoid races in `make_mut` and `get_mut`.
     weak: atomic::AtomicUsize,
+}
 
+impl ArcInnerMetadata {
+    // Start the weak pointer count as 1 which is the weak pointer that's
+    // held by all the strong pointers (kinda), see std/rc.rs for more info
+    #[inline]
+    fn new_strong() -> Self {
+        Self { strong: atomic::AtomicUsize::new(1), weak: atomic::AtomicUsize::new(1) }
+    }
+
+    #[inline]
+    fn new_weak() -> Self {
+        Self { strong: atomic::AtomicUsize::new(0), weak: atomic::AtomicUsize::new(1) }
+    }
+}
+
+// This is repr(C) to support StructAlloc
+#[repr(C)]
+struct ArcInner<T: ?Sized> {
+    meta: ArcInnerMetadata,
     data: T,
 }
 
@@ -327,13 +343,7 @@ impl<T> Arc<T> {
     #[inline]
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn new(data: T) -> Arc<T> {
-        // Start the weak pointer count as 1 which is the weak pointer that's
-        // held by all the strong pointers (kinda), see std/rc.rs for more info
-        let x: Box<_> = box ArcInner {
-            strong: atomic::AtomicUsize::new(1),
-            weak: atomic::AtomicUsize::new(1),
-            data,
-        };
+        let x: Box<_> = box ArcInner { meta: ArcInnerMetadata::new_strong(), data };
         Self::from_inner(Box::leak(x).into())
     }
 
@@ -363,8 +373,7 @@ impl<T> Arc<T> {
         // Construct the inner in the "uninitialized" state with a single
         // weak reference.
         let uninit_ptr: NonNull<_> = Box::leak(box ArcInner {
-            strong: atomic::AtomicUsize::new(0),
-            weak: atomic::AtomicUsize::new(1),
+            meta: ArcInnerMetadata::new_weak(),
             data: mem::MaybeUninit::<T>::uninit(),
         })
         .into();
@@ -398,7 +407,7 @@ impl<T> Arc<T> {
             //
             // These side effects do not impact us in any way, and no other side effects are
             // possible with safe code alone.
-            let prev_value = (*inner).strong.fetch_add(1, Release);
+            let prev_value = (*inner).meta.strong.fetch_add(1, Release);
             debug_assert_eq!(prev_value, 0, "No prior strong references should exist");
         }
 
@@ -494,13 +503,7 @@ impl<T> Arc<T> {
     #[unstable(feature = "allocator_api", issue = "32838")]
     #[inline]
     pub fn try_new(data: T) -> Result<Arc<T>, AllocError> {
-        // Start the weak pointer count as 1 which is the weak pointer that's
-        // held by all the strong pointers (kinda), see std/rc.rs for more info
-        let x: Box<_> = Box::try_new(ArcInner {
-            strong: atomic::AtomicUsize::new(1),
-            weak: atomic::AtomicUsize::new(1),
-            data,
-        })?;
+        let x: Box<_> = Box::try_new(ArcInner { meta: ArcInnerMetadata::new_strong(), data })?;
         Ok(Self::from_inner(Box::leak(x).into()))
     }
 
@@ -593,11 +596,11 @@ impl<T> Arc<T> {
     #[inline]
     #[stable(feature = "arc_unique", since = "1.4.0")]
     pub fn try_unwrap(this: Self) -> Result<T, Self> {
-        if this.inner().strong.compare_exchange(1, 0, Relaxed, Relaxed).is_err() {
+        if this.inner().meta.strong.compare_exchange(1, 0, Relaxed, Relaxed).is_err() {
             return Err(this);
         }
 
-        acquire!(this.inner().strong);
+        acquire!(this.inner().meta.strong);
 
         unsafe {
             let elem = ptr::read(&this.ptr.as_ref().data);
@@ -867,13 +870,13 @@ impl<T: ?Sized> Arc<T> {
     pub fn downgrade(this: &Self) -> Weak<T> {
         // This Relaxed is OK because we're checking the value in the CAS
         // below.
-        let mut cur = this.inner().weak.load(Relaxed);
+        let mut cur = this.inner().meta.weak.load(Relaxed);
 
         loop {
             // check if the weak counter is currently "locked"; if so, spin.
             if cur == usize::MAX {
                 hint::spin_loop();
-                cur = this.inner().weak.load(Relaxed);
+                cur = this.inner().meta.weak.load(Relaxed);
                 continue;
             }
 
@@ -884,7 +887,7 @@ impl<T: ?Sized> Arc<T> {
             // Unlike with Clone(), we need this to be an Acquire read to
             // synchronize with the write coming from `is_unique`, so that the
             // events prior to that write happen before this read.
-            match this.inner().weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) {
+            match this.inner().meta.weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) {
                 Ok(_) => {
                     // Make sure we do not create a dangling Weak
                     debug_assert!(!is_dangling(this.ptr.as_ptr()));
@@ -918,7 +921,7 @@ impl<T: ?Sized> Arc<T> {
     #[inline]
     #[stable(feature = "arc_counts", since = "1.15.0")]
     pub fn weak_count(this: &Self) -> usize {
-        let cnt = this.inner().weak.load(SeqCst);
+        let cnt = this.inner().meta.weak.load(SeqCst);
         // If the weak count is currently locked, the value of the
         // count was 0 just before taking the lock.
         if cnt == usize::MAX { 0 } else { cnt - 1 }
@@ -947,7 +950,7 @@ impl<T: ?Sized> Arc<T> {
     #[inline]
     #[stable(feature = "arc_counts", since = "1.15.0")]
     pub fn strong_count(this: &Self) -> usize {
-        this.inner().strong.load(SeqCst)
+        this.inner().meta.strong.load(SeqCst)
     }
 
     /// Increments the strong reference count on the `Arc<T>` associated with the
@@ -1112,8 +1115,8 @@ impl<T: ?Sized> Arc<T> {
         debug_assert_eq!(unsafe { Layout::for_value(&*inner) }, layout);
 
         unsafe {
-            ptr::write(&mut (*inner).strong, atomic::AtomicUsize::new(1));
-            ptr::write(&mut (*inner).weak, atomic::AtomicUsize::new(1));
+            ptr::write(&mut (*inner).meta.strong, atomic::AtomicUsize::new(1));
+            ptr::write(&mut (*inner).meta.weak, atomic::AtomicUsize::new(1));
         }
 
         Ok(inner)
@@ -1276,7 +1279,7 @@ impl<T: ?Sized> Clone for Arc<T> {
         // another must already provide any required synchronization.
         //
         // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
-        let old_size = self.inner().strong.fetch_add(1, Relaxed);
+        let old_size = self.inner().meta.strong.fetch_add(1, Relaxed);
 
         // However we need to guard against massive refcounts in case someone
         // is `mem::forget`ing Arcs. If we don't do this the count can overflow
@@ -1352,7 +1355,7 @@ impl<T: Clone> Arc<T> {
         // before release writes (i.e., decrements) to `strong`. Since we hold a
         // weak count, there's no chance the ArcInner itself could be
         // deallocated.
-        if this.inner().strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() {
+        if this.inner().meta.strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() {
             // Another strong pointer exists, so we must clone.
             // Pre-allocate memory to allow writing the cloned value directly.
             let mut arc = Self::new_uninit();
@@ -1361,7 +1364,7 @@ impl<T: Clone> Arc<T> {
                 (**this).write_clone_into_raw(data.as_mut_ptr());
                 *this = arc.assume_init();
             }
-        } else if this.inner().weak.load(Relaxed) != 1 {
+        } else if this.inner().meta.weak.load(Relaxed) != 1 {
             // Relaxed suffices in the above because this is fundamentally an
             // optimization: we are always racing with weak pointers being
             // dropped. Worst case, we end up allocated a new Arc unnecessarily.
@@ -1388,7 +1391,7 @@ impl<T: Clone> Arc<T> {
         } else {
             // We were the sole reference of either kind; bump back up the
             // strong ref count.
-            this.inner().strong.store(1, Release);
+            this.inner().meta.strong.store(1, Release);
         }
 
         // As with `get_mut()`, the unsafety is ok because our reference was
@@ -1484,16 +1487,16 @@ impl<T: ?Sized> Arc<T> {
         // writes to `strong` (in particular in `Weak::upgrade`) prior to decrements
         // of the `weak` count (via `Weak::drop`, which uses release).  If the upgraded
         // weak ref was never dropped, the CAS here will fail so we do not care to synchronize.
-        if self.inner().weak.compare_exchange(1, usize::MAX, Acquire, Relaxed).is_ok() {
+        if self.inner().meta.weak.compare_exchange(1, usize::MAX, Acquire, Relaxed).is_ok() {
             // This needs to be an `Acquire` to synchronize with the decrement of the `strong`
             // counter in `drop` -- the only access that happens when any but the last reference
             // is being dropped.
-            let unique = self.inner().strong.load(Acquire) == 1;
+            let unique = self.inner().meta.strong.load(Acquire) == 1;
 
             // The release write here synchronizes with a read in `downgrade`,
             // effectively preventing the above read of `strong` from happening
             // after the write.
-            self.inner().weak.store(1, Release); // release the lock
+            self.inner().meta.weak.store(1, Release); // release the lock
             unique
         } else {
             false
@@ -1533,7 +1536,7 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Arc<T> {
         // Because `fetch_sub` is already atomic, we do not need to synchronize
         // with other threads unless we are going to delete the object. This
         // same logic applies to the below `fetch_sub` to the `weak` count.
-        if self.inner().strong.fetch_sub(1, Release) != 1 {
+        if self.inner().meta.strong.fetch_sub(1, Release) != 1 {
             return;
         }
 
@@ -1565,7 +1568,7 @@ unsafe impl<#[may_dangle] T: ?Sized> Drop for Arc<T> {
         //
         // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
         // [2]: (https://github.com/rust-lang/rust/pull/41714)
-        acquire!(self.inner().strong);
+        acquire!(self.inner().meta.strong);
 
         unsafe {
             self.drop_slow();
@@ -1883,7 +1886,7 @@ impl<T: ?Sized> Weak<T> {
             // is dropped, the data field will be dropped in-place).
             Some(unsafe {
                 let ptr = self.ptr.as_ptr();
-                WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak }
+                WeakInner { strong: &(*ptr).meta.strong, weak: &(*ptr).meta.weak }
             })
         }
     }
@@ -2455,24 +2458,21 @@ impl<T: ?Sized> AsRef<T> for Arc<T> {
 #[stable(feature = "pin", since = "1.33.0")]
 impl<T: ?Sized> Unpin for Arc<T> {}
 
+type ArcStructAlloc = StructAlloc<ArcInnerMetadata>;
+
 /// Get the offset within an `ArcInner` for the payload behind a pointer.
 ///
 /// # Safety
 ///
 /// The pointer must point to (and have valid metadata for) a previously
 /// valid instance of T, but the T is allowed to be dropped.
-unsafe fn data_offset<T: ?Sized>(ptr: *const T) -> isize {
-    // Align the unsized value to the end of the ArcInner.
-    // Because RcBox is repr(C), it will always be the last field in memory.
-    // SAFETY: since the only unsized types possible are slices, trait objects,
-    // and extern types, the input safety requirement is currently enough to
-    // satisfy the requirements of align_of_val_raw; this is an implementation
-    // detail of the language that may not be relied upon outside of std.
-    unsafe { data_offset_align(align_of_val_raw(ptr)) }
-}
-
-#[inline]
-fn data_offset_align(align: usize) -> isize {
-    let layout = Layout::new::<ArcInner<()>>();
-    (layout.size() + layout.padding_needed_for(align)) as isize
+unsafe fn data_offset<T: ?Sized>(data_ptr: *const T) -> isize {
+    unsafe {
+        // SAFETY: since the only unsized types possible are slices, trait objects,
+        // and extern types, the input safety requirement is currently enough to
+        // satisfy the requirements of for_value_raw; this is an implementation
+        // detail of the language that may not be relied upon outside of std.
+        let data_layout = Layout::for_value_raw(data_ptr);
+        ArcStructAlloc::offset_of_data(data_layout) as isize
+    }
 }

src/etc/gdb_providers.py

@@ -146,8 +146,8 @@ def __init__(self, valobj, is_atomic=False):
         self.is_atomic = is_atomic
         self.ptr = unwrap_unique_or_non_null(valobj["ptr"])
         self.value = self.ptr["data" if is_atomic else "value"]
-        self.strong = self.ptr["strong"]["v" if is_atomic else "value"]["value"]
-        self.weak = self.ptr["weak"]["v" if is_atomic else "value"]["value"] - 1
+        self.strong = self.ptr["meta"]["strong"]["v" if is_atomic else "value"]["value"]
+        self.weak = self.ptr["meta"]["weak"]["v" if is_atomic else "value"]["value"] - 1
 
     def to_string(self):
         if self.is_atomic:

src/etc/lldb_providers.py

@@ -601,10 +601,10 @@ def __init__(self, valobj, dict, is_atomic=False):
 
         self.value = self.ptr.GetChildMemberWithName("data" if is_atomic else "value")
 
-        self.strong = self.ptr.GetChildMemberWithName("strong").GetChildAtIndex(
-            0).GetChildMemberWithName("value")
-        self.weak = self.ptr.GetChildMemberWithName("weak").GetChildAtIndex(
-            0).GetChildMemberWithName("value")
+        self.strong = self.ptr.GetChildMemberWithName("meta")\
+                .GetChildMemberWithName("strong").GetChildAtIndex(0).GetChildMemberWithName("value")
+        self.weak = self.ptr.GetChildMemberWithName("meta")\
+                .GetChildMemberWithName("weak").GetChildAtIndex(0).GetChildMemberWithName("value")
 
         self.value_builder = ValueBuilder(valobj)