From b0e296024403d0daf7f7f1658a26f573f838f8ac Mon Sep 17 00:00:00 2001 From: sarah el kazdadi Date: Mon, 1 Aug 2022 17:36:10 +0200 Subject: [PATCH 1/7] feat: implement rfc240 --- benches/bench.rs | 26 +- src/lib.rs | 2495 ++++++++++++++++++---------------------------- src/tests.rs | 209 ++-- tests/macro.rs | 2 +- 4 files changed, 1103 insertions(+), 1629 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index b52ee15..b677c19 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -6,12 +6,13 @@ extern crate smallvec; extern crate test; use self::test::Bencher; -use smallvec::{ExtendFromSlice, SmallVec}; +use smallvec::SmallVec; const VEC_SIZE: usize = 16; const SPILLED_SIZE: usize = 100; -trait Vector: for<'a> From<&'a [T]> + Extend + ExtendFromSlice { + +trait Vector: for<'a> From<&'a [T]> + Extend { fn new() -> Self; fn push(&mut self, val: T); fn pop(&mut self) -> Option; @@ -19,6 +20,7 @@ trait Vector: for<'a> From<&'a [T]> + Extend + ExtendFromSlice { fn insert(&mut self, n: usize, val: T); fn from_elem(val: T, n: usize) -> Self; fn from_elems(val: &[T]) -> Self; + fn extend_from_slice(&mut self, other: &[T]); } impl Vector for Vec { @@ -49,9 +51,13 @@ impl Vector for Vec { fn from_elems(val: &[T]) -> Self { val.to_owned() } + + fn extend_from_slice(&mut self, other: &[T]) { + Vec::extend_from_slice(self, other) + } } -impl Vector for SmallVec<[T; VEC_SIZE]> { +impl Vector for SmallVec { fn new() -> Self { Self::new() } @@ -79,6 +85,10 @@ impl Vector for SmallVec<[T; VEC_SIZE]> { fn from_elems(val: &[T]) -> Self { SmallVec::from_slice(val) } + + fn extend_from_slice(&mut self, other: &[T]) { + SmallVec::extend_from_slice(self, other) + } } macro_rules! make_benches { @@ -93,7 +103,7 @@ macro_rules! make_benches { } make_benches! { - SmallVec<[u64; VEC_SIZE]> { + SmallVec { bench_push => gen_push(SPILLED_SIZE as _), bench_push_small => gen_push(VEC_SIZE as _), bench_insert_push => gen_insert_push(SPILLED_SIZE as _), @@ -263,7 +273,7 @@ fn gen_from_elem>(n: usize, b: &mut Bencher) { fn bench_insert_many(b: &mut Bencher) { #[inline(never)] fn insert_many_noinline>( - vec: &mut SmallVec<[u64; VEC_SIZE]>, + vec: &mut SmallVec, index: usize, iterable: I, ) { @@ -271,7 +281,7 @@ fn bench_insert_many(b: &mut Bencher) { } b.iter(|| { - let mut vec = SmallVec::<[u64; VEC_SIZE]>::new(); + let mut vec = SmallVec::::new(); insert_many_noinline(&mut vec, 0, 0..SPILLED_SIZE as _); insert_many_noinline(&mut vec, 0, 0..SPILLED_SIZE as _); vec @@ -282,7 +292,7 @@ fn bench_insert_many(b: &mut Bencher) { fn bench_insert_from_slice(b: &mut Bencher) { let v: Vec = (0..SPILLED_SIZE as _).collect(); b.iter(|| { - let mut vec = SmallVec::<[u64; VEC_SIZE]>::new(); + let mut vec = SmallVec::::new(); vec.insert_from_slice(0, &v); vec.insert_from_slice(0, &v); vec @@ -292,7 +302,7 @@ fn bench_insert_from_slice(b: &mut Bencher) { #[bench] fn bench_macro_from_list(b: &mut Bencher) { b.iter(|| { - let vec: SmallVec<[u64; 16]> = smallvec![ + let vec: SmallVec = smallvec![ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 32, 36, 0x40, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, diff --git a/src/lib.rs b/src/lib.rs index 921347a..7b293e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,36 +26,6 @@ //! When this feature is enabled, `SmallVec<[u8; _]>` implements the `std::io::Write` trait. //! This feature is not compatible with `#![no_std]` programs. //! -//! ### `union` -//! -//! **This feature requires Rust 1.49.** -//! -//! When the `union` feature is enabled `smallvec` will track its state (inline or spilled) -//! without the use of an enum tag, reducing the size of the `smallvec` by one machine word. -//! This means that there is potentially no space overhead compared to `Vec`. -//! Note that `smallvec` can still be larger than `Vec` if the inline buffer is larger than two -//! machine words. -//! -//! To use this feature add `features = ["union"]` in the `smallvec` section of Cargo.toml. -//! Note that this feature requires Rust 1.49. -//! -//! Tracking issue: [rust-lang/rust#55149](https://github.com/rust-lang/rust/issues/55149) -//! -//! ### `const_generics` -//! -//! **This feature requires Rust 1.51.** -//! -//! When this feature is enabled, `SmallVec` works with any arrays of any size, not just a fixed -//! list of sizes. -//! -//! ### `const_new` -//! -//! **This feature requires Rust 1.51.** -//! -//! This feature exposes the functions [`SmallVec::new_const`], [`SmallVec::from_const`], and [`smallvec_inline`] which enables the `SmallVec` to be initialized from a const context. -//! For details, see the -//! [Rust Reference](https://doc.rust-lang.org/reference/const_eval.html#const-functions). -//! //! ### `specialization` //! //! **This feature is unstable and requires a nightly build of the Rust toolchain.** @@ -77,202 +47,42 @@ //! Tracking issue: [rust-lang/rust#34761](https://github.com/rust-lang/rust/issues/34761) #![no_std] -#![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(feature = "specialization", allow(incomplete_features))] -#![cfg_attr(feature = "specialization", feature(specialization))] -#![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))] -#![deny(missing_docs)] #[doc(hidden)] pub extern crate alloc; -#[cfg(any(test, feature = "write"))] +// #[cfg(any(test, feature = "write"))] extern crate std; #[cfg(test)] mod tests; -#[allow(deprecated)] -use alloc::alloc::{Layout, LayoutErr}; use alloc::boxed::Box; -use alloc::{vec, vec::Vec}; -use core::borrow::{Borrow, BorrowMut}; -use core::cmp; -use core::fmt; -use core::hash::{Hash, Hasher}; -use core::hint::unreachable_unchecked; -use core::iter::{repeat, FromIterator, FusedIterator, IntoIterator}; -use core::mem; +use alloc::vec; +use alloc::vec::Vec; + +use alloc::alloc::Layout; +use core::borrow::Borrow; +use core::borrow::BorrowMut; +use core::fmt::Debug; +use core::mem::align_of; +use core::mem::size_of; +use core::mem::ManuallyDrop; use core::mem::MaybeUninit; -use core::ops::{self, Range, RangeBounds}; -use core::ptr::{self, NonNull}; -use core::slice::{self, SliceIndex}; +use core::ptr::addr_of; +use core::ptr::addr_of_mut; +use core::ptr::copy_nonoverlapping; +#[cfg(feature = "serde")] +use core::marker::PhantomData; #[cfg(feature = "serde")] use serde::{ de::{Deserialize, Deserializer, SeqAccess, Visitor}, ser::{Serialize, SerializeSeq, Serializer}, }; - -#[cfg(feature = "serde")] -use core::marker::PhantomData; - #[cfg(feature = "write")] use std::io; -/// Creates a [`SmallVec`] containing the arguments. -/// -/// `smallvec!` allows `SmallVec`s to be defined with the same syntax as array expressions. -/// There are two forms of this macro: -/// -/// - Create a [`SmallVec`] containing a given list of elements: -/// -/// ``` -/// # #[macro_use] extern crate smallvec; -/// # use smallvec::SmallVec; -/// # fn main() { -/// let v: SmallVec<[_; 128]> = smallvec![1, 2, 3]; -/// assert_eq!(v[0], 1); -/// assert_eq!(v[1], 2); -/// assert_eq!(v[2], 3); -/// # } -/// ``` -/// -/// - Create a [`SmallVec`] from a given element and size: -/// -/// ``` -/// # #[macro_use] extern crate smallvec; -/// # use smallvec::SmallVec; -/// # fn main() { -/// let v: SmallVec<[_; 0x8000]> = smallvec![1; 3]; -/// assert_eq!(v, SmallVec::from_buf([1, 1, 1])); -/// # } -/// ``` -/// -/// Note that unlike array expressions this syntax supports all elements -/// which implement [`Clone`] and the number of elements doesn't have to be -/// a constant. -/// -/// This will use `clone` to duplicate an expression, so one should be careful -/// using this with types having a nonstandard `Clone` implementation. For -/// example, `smallvec![Rc::new(1); 5]` will create a vector of five references -/// to the same boxed integer value, not five references pointing to independently -/// boxed integers. - -#[macro_export] -macro_rules! smallvec { - // count helper: transform any expression into 1 - (@one $x:expr) => (1usize); - ($elem:expr; $n:expr) => ({ - $crate::SmallVec::from_elem($elem, $n) - }); - ($($x:expr),*$(,)*) => ({ - let count = 0usize $(+ $crate::smallvec!(@one $x))*; - #[allow(unused_mut)] - let mut vec = $crate::SmallVec::new(); - if count <= vec.inline_size() { - $(vec.push($x);)* - vec - } else { - $crate::SmallVec::from_vec($crate::alloc::vec![$($x,)*]) - } - }); -} - -/// Creates an inline [`SmallVec`] containing the arguments. This macro is enabled by the feature `const_new`. -/// -/// `smallvec_inline!` allows `SmallVec`s to be defined with the same syntax as array expressions in `const` contexts. -/// The inline storage `A` will always be an array of the size specified by the arguments. -/// There are two forms of this macro: -/// -/// - Create a [`SmallVec`] containing a given list of elements: -/// -/// ``` -/// # #[macro_use] extern crate smallvec; -/// # use smallvec::SmallVec; -/// # fn main() { -/// const V: SmallVec<[i32; 3]> = smallvec_inline![1, 2, 3]; -/// assert_eq!(V[0], 1); -/// assert_eq!(V[1], 2); -/// assert_eq!(V[2], 3); -/// # } -/// ``` -/// -/// - Create a [`SmallVec`] from a given element and size: -/// -/// ``` -/// # #[macro_use] extern crate smallvec; -/// # use smallvec::SmallVec; -/// # fn main() { -/// const V: SmallVec<[i32; 3]> = smallvec_inline![1; 3]; -/// assert_eq!(V, SmallVec::from_buf([1, 1, 1])); -/// # } -/// ``` -/// -/// Note that the behavior mimics that of array expressions, in contrast to [`smallvec`]. -#[cfg(feature = "const_new")] -#[cfg_attr(docsrs, doc(cfg(feature = "const_new")))] -#[macro_export] -macro_rules! smallvec_inline { - // count helper: transform any expression into 1 - (@one $x:expr) => (1usize); - ($elem:expr; $n:expr) => ({ - $crate::SmallVec::<[_; $n]>::from_const([$elem; $n]) - }); - ($($x:expr),+ $(,)?) => ({ - const N: usize = 0usize $(+ $crate::smallvec_inline!(@one $x))*; - $crate::SmallVec::<[_; N]>::from_const([$($x,)*]) - }); -} - -/// `panic!()` in debug builds, optimization hint in release. -#[cfg(not(feature = "union"))] -macro_rules! debug_unreachable { - () => { - debug_unreachable!("entered unreachable code") - }; - ($e:expr) => { - if cfg!(not(debug_assertions)) { - unreachable_unchecked(); - } else { - panic!($e); - } - }; -} - -/// Trait to be implemented by a collection that can be extended from a slice -/// -/// ## Example -/// -/// ```rust -/// use smallvec::{ExtendFromSlice, SmallVec}; -/// -/// fn initialize>(v: &mut V) { -/// v.extend_from_slice(b"Test!"); -/// } -/// -/// let mut vec = Vec::new(); -/// initialize(&mut vec); -/// assert_eq!(&vec, b"Test!"); -/// -/// let mut small_vec = SmallVec::<[u8; 8]>::new(); -/// initialize(&mut small_vec); -/// assert_eq!(&small_vec as &[_], b"Test!"); -/// ``` -#[doc(hidden)] -#[deprecated] -pub trait ExtendFromSlice { - /// Extends a collection from a slice of its element type - fn extend_from_slice(&mut self, other: &[T]); -} - -#[allow(deprecated)] -impl ExtendFromSlice for Vec { - fn extend_from_slice(&mut self, other: &[T]) { - Vec::extend_from_slice(self, other) - } -} - /// Error type for APIs with fallible heap allocation #[derive(Debug)] pub enum CollectionAllocErr { @@ -284,20 +94,24 @@ pub enum CollectionAllocErr { layout: Layout, }, } - -impl fmt::Display for CollectionAllocErr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl core::fmt::Display for CollectionAllocErr { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "Allocation error: {:?}", self) } } -#[allow(deprecated)] -impl From for CollectionAllocErr { - fn from(_: LayoutErr) -> Self { - CollectionAllocErr::CapacityOverflow - } +/// Either a stack array with `length <= N` or a heap array +/// whose pointer and capacity are stored here. +/// +/// We store a `*const T` instead of a `*mut T` so that the type is covariant +/// with respect to `T`. +#[repr(C)] +pub union RawSmallVec { + inline: ManuallyDrop>, + heap: (*const T, usize), } +#[inline] fn infallible(result: Result) -> T { match result { Ok(x) => x, @@ -306,20 +120,165 @@ fn infallible(result: Result) -> T { } } -/// FIXME: use `Layout::array` when we require a Rust version where it’s stable -/// https://github.com/rust-lang/rust/issues/55724 -fn layout_array(n: usize) -> Result { - let size = mem::size_of::() - .checked_mul(n) - .ok_or(CollectionAllocErr::CapacityOverflow)?; - let align = mem::align_of::(); - Layout::from_size_align(size, align).map_err(|_| CollectionAllocErr::CapacityOverflow) +impl RawSmallVec { + #[inline] + const fn is_zst() -> bool { + size_of::() == 0 + } + + #[inline] + const fn new() -> Self { + Self::new_inline(MaybeUninit::uninit()) + } + #[inline] + const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self { + inline: ManuallyDrop::new(inline), + } + } + #[inline] + const fn new_heap(ptr: *mut T, capacity: usize) -> Self { + Self { + heap: (ptr, capacity), + } + } + + #[inline] + const fn as_ptr_inline(&self) -> *const T { + // SAFETY: This is safe because we don't read the value. We only get a pointer to the data. + // Dereferencing the pointer is unsafe so unsafe code is required to misuse the return + // value. + (unsafe { addr_of!(self.inline) }) as *const T + } + + #[inline] + fn as_mut_ptr_inline(&mut self) -> *mut T { + // SAFETY: See above. + (unsafe { addr_of_mut!(self.inline) }) as *mut T + } + + #[inline] + const unsafe fn as_ptr_heap(&self) -> *const T { + self.heap.0 + } + + #[inline] + unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { + self.heap.0 as *mut T + } + + unsafe fn try_grow( + &mut self, + len: TaggedLen, + new_capacity: usize, + ) -> Result<(), CollectionAllocErr> { + use alloc::alloc::{alloc, realloc}; + debug_assert!(new_capacity > 0); + + if Self::is_zst() { + debug_assert_eq!(len.value(Self::is_zst()), usize::MAX); + Err(CollectionAllocErr::CapacityOverflow) + } else { + let was_on_heap = len.on_heap(Self::is_zst()); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() + } else { + self.as_mut_ptr_inline() + }; + let len = len.value(Self::is_zst()); + + let new_layout = Layout::array::(new_capacity) + .map_err(|_| CollectionAllocErr::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(CollectionAllocErr::CapacityOverflow); + } + + if len == 0 || !was_on_heap { + // get a fresh allocation + + // layout has non zero size + let new_ptr = alloc(new_layout) as *mut T; + if new_ptr.is_null() { + Err(CollectionAllocErr::AllocErr { layout: new_layout }) + } else { + copy_nonoverlapping(ptr, new_ptr, len); + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } + } else { + // use realloc + + // this can't overflow since we already constructed an equivalent layout during + // the previous allocation + let old_layout = Layout::from_size_align_unchecked( + self.heap.1 * size_of::(), + align_of::(), + ); + + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the previous memory block + // new_layout.size() is greater than zero + // does not overflow when rounded up to alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + if new_ptr.is_null() { + Err(CollectionAllocErr::AllocErr { layout: new_layout }) + } else { + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } + } + } + } +} + +/// Vec guarantees that its length is always less than isize::MAX in *bytes*. +/// +/// For a non ZST, this means that the length is less than isize::MAX objects, which implies we +/// have at least one free bit we can use. We use the least significant bit for the tag. And store +/// the length in the `usize::BITS - 1` most significant bits. +/// +/// For a ZST, we never use the heap, so we just store the length directly. +#[repr(transparent)] +#[derive(Clone, Copy)] +struct TaggedLen(usize); + +impl TaggedLen { + #[inline] + pub const fn new(len: usize, on_heap: bool, is_zst: bool) -> Self { + if is_zst { + debug_assert!(!on_heap); + TaggedLen(len) + } else { + debug_assert!(len < isize::MAX as usize); + TaggedLen((len << 1) | on_heap as usize) + } + } + + #[inline] + #[must_use] + pub const fn on_heap(self, is_zst: bool) -> bool { + if is_zst { + false + } else { + (self.0 & 1_usize) == 1 + } + } + + #[inline] + pub const fn value(self, is_zst: bool) -> usize { + if is_zst { + self.0 + } else { + self.0 >> 1 + } + } } -unsafe fn deallocate(ptr: *mut T, capacity: usize) { - // This unwrap should succeed since the same did when allocating. - let layout = layout_array::(capacity).unwrap(); - alloc::alloc::dealloc(ptr as *mut u8, layout) +#[repr(C)] +pub struct SmallVec { + len: TaggedLen, + raw: RawSmallVec, } /// An iterator that removes the items from a `SmallVec` and yields them by value. @@ -327,33 +286,21 @@ unsafe fn deallocate(ptr: *mut T, capacity: usize) { /// Returned from [`SmallVec::drain`][1]. /// /// [1]: struct.SmallVec.html#method.drain -pub struct Drain<'a, T: 'a + Array> { +pub struct Drain<'a, T: 'a, const N: usize> { tail_start: usize, tail_len: usize, - iter: slice::Iter<'a, T::Item>, - vec: NonNull>, -} - -impl<'a, T: 'a + Array> fmt::Debug for Drain<'a, T> -where - T::Item: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() - } + iter: core::slice::Iter<'a, T>, + vec: core::ptr::NonNull>, } -unsafe impl<'a, T: Sync + Array> Sync for Drain<'a, T> {} -unsafe impl<'a, T: Send + Array> Send for Drain<'a, T> {} - -impl<'a, T: 'a + Array> Iterator for Drain<'a, T> { - type Item = T::Item; +impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { + type Item = T; #[inline] - fn next(&mut self) -> Option { + fn next(&mut self) -> Option { self.iter .next() - .map(|reference| unsafe { ptr::read(reference) }) + .map(|reference| unsafe { core::ptr::read(reference) }) } #[inline] @@ -362,25 +309,25 @@ impl<'a, T: 'a + Array> Iterator for Drain<'a, T> { } } -impl<'a, T: 'a + Array> DoubleEndedIterator for Drain<'a, T> { +impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { #[inline] - fn next_back(&mut self) -> Option { + fn next_back(&mut self) -> Option { self.iter .next_back() - .map(|reference| unsafe { ptr::read(reference) }) + .map(|reference| unsafe { core::ptr::read(reference) }) } } -impl<'a, T: Array> ExactSizeIterator for Drain<'a, T> { +impl<'a, T, const N: usize> ExactSizeIterator for Drain<'a, T, N> { #[inline] fn len(&self) -> usize { self.iter.len() } } -impl<'a, T: Array> FusedIterator for Drain<'a, T> {} +impl<'a, T, const N: usize> core::iter::FusedIterator for Drain<'a, T, N> {} -impl<'a, T: 'a + Array> Drop for Drain<'a, T> { +impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { fn drop(&mut self) { self.for_each(drop); @@ -397,7 +344,7 @@ impl<'a, T: 'a + Array> Drop for Drain<'a, T> { let ptr = source_vec.as_mut_ptr(); let src = ptr.add(tail); let dst = ptr.add(start); - ptr::copy(src, dst, self.tail_len); + core::ptr::copy(src, dst, self.tail_len); } source_vec.set_len(start + self.tail_len); } @@ -405,396 +352,260 @@ impl<'a, T: 'a + Array> Drop for Drain<'a, T> { } } -#[cfg(feature = "union")] -union SmallVecData { - inline: core::mem::ManuallyDrop>, - heap: (*mut A::Item, usize), +/// An iterator that consumes a `SmallVec` and yields its items by value. +/// +/// Returned from [`SmallVec::into_iter`][1]. +/// +/// [1]: struct.SmallVec.html#method.into_iter +pub struct IntoIter { + raw: RawSmallVec, + begin: usize, + end: TaggedLen, } -#[cfg(all(feature = "union", feature = "const_new"))] -impl SmallVecData<[T; N]> { - #[cfg_attr(docsrs, doc(cfg(feature = "const_new")))] +impl IntoIter { #[inline] - const fn from_const(inline: MaybeUninit<[T; N]>) -> Self { - SmallVecData { - inline: core::mem::ManuallyDrop::new(inline), - } + const fn is_zst() -> bool { + size_of::() == 0 } -} -#[cfg(feature = "union")] -impl SmallVecData { - #[inline] - unsafe fn inline(&self) -> *const A::Item { - self.inline.as_ptr() as *const A::Item - } #[inline] - unsafe fn inline_mut(&mut self) -> *mut A::Item { - self.inline.as_mut_ptr() as *mut A::Item - } - #[inline] - fn from_inline(inline: MaybeUninit) -> SmallVecData { - SmallVecData { - inline: core::mem::ManuallyDrop::new(inline), + const fn as_ptr(&self) -> *const T { + let on_heap = self.end.on_heap(Self::is_zst()); + if on_heap { + unsafe { self.raw.as_ptr_heap() } + } else { + self.raw.as_ptr_inline() } } + #[inline] - unsafe fn into_inline(self) -> MaybeUninit { - core::mem::ManuallyDrop::into_inner(self.inline) - } - #[inline] - unsafe fn heap(&self) -> (*mut A::Item, usize) { - self.heap + fn as_mut_ptr(&mut self) -> *mut T { + let on_heap = self.end.on_heap(Self::is_zst()); + if on_heap { + unsafe { self.raw.as_mut_ptr_heap() } + } else { + self.raw.as_mut_ptr_inline() + } } + #[inline] - unsafe fn heap_mut(&mut self) -> &mut (*mut A::Item, usize) { - &mut self.heap + pub fn as_slice(&self) -> &[T] { + unsafe { + let ptr = self.as_ptr(); + core::slice::from_raw_parts( + ptr.add(self.begin), + self.end.value(Self::is_zst()) - self.begin, + ) + } } + #[inline] - fn from_heap(ptr: *mut A::Item, len: usize) -> SmallVecData { - SmallVecData { heap: (ptr, len) } + pub fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { + let ptr = self.as_mut_ptr(); + core::slice::from_raw_parts_mut( + ptr.add(self.begin), + self.end.value(Self::is_zst()) - self.begin, + ) + } } } -#[cfg(not(feature = "union"))] -enum SmallVecData { - Inline(MaybeUninit), - Heap((*mut A::Item, usize)), -} +impl Iterator for IntoIter { + type Item = T; -#[cfg(all(not(feature = "union"), feature = "const_new"))] -impl SmallVecData<[T; N]> { - #[cfg_attr(docsrs, doc(cfg(feature = "const_new")))] #[inline] - const fn from_const(inline: MaybeUninit<[T; N]>) -> Self { - SmallVecData::Inline(inline) + fn next(&mut self) -> Option { + if self.begin == self.end.value(Self::is_zst()) { + None + } else { + unsafe { + let ptr = self.as_mut_ptr(); + let value = ptr.add(self.begin).read(); + self.begin += 1; + Some(value) + } + } } -} -#[cfg(not(feature = "union"))] -impl SmallVecData { #[inline] - unsafe fn inline(&self) -> *const A::Item { - match self { - SmallVecData::Inline(a) => a.as_ptr() as *const A::Item, - _ => debug_unreachable!(), - } + fn size_hint(&self) -> (usize, Option) { + let size = self.end.value(Self::is_zst()) - self.begin; + (size, Some(size)) } +} + +impl DoubleEndedIterator for IntoIter { #[inline] - unsafe fn inline_mut(&mut self) -> *mut A::Item { - match self { - SmallVecData::Inline(a) => a.as_mut_ptr() as *mut A::Item, - _ => debug_unreachable!(), + fn next_back(&mut self) -> Option { + let mut end = self.end.value(Self::is_zst()); + if self.begin == end { + None + } else { + unsafe { + let ptr = self.as_mut_ptr(); + let on_heap = self.end.on_heap(Self::is_zst()); + end -= 1; + self.end = TaggedLen::new(end, on_heap, Self::is_zst()); + let value = ptr.add(end).read(); + Some(value) + } } } +} +impl ExactSizeIterator for IntoIter {} +impl core::iter::FusedIterator for IntoIter {} + +impl SmallVec { #[inline] - fn from_inline(inline: MaybeUninit) -> SmallVecData { - SmallVecData::Inline(inline) + const fn is_zst() -> bool { + size_of::() == 0 } + #[inline] - unsafe fn into_inline(self) -> MaybeUninit { - match self { - SmallVecData::Inline(a) => a, - _ => debug_unreachable!(), + pub const fn new() -> SmallVec { + Self { + len: TaggedLen::new(0, false, Self::is_zst()), + raw: RawSmallVec::new(), } } + #[inline] - unsafe fn heap(&self) -> (*mut A::Item, usize) { - match self { - SmallVecData::Heap(data) => *data, - _ => debug_unreachable!(), - } + pub fn with_capacity(capacity: usize) -> Self { + let mut this = Self::new(); + this.reserve_exact(capacity); + this } + #[inline] - unsafe fn heap_mut(&mut self) -> &mut (*mut A::Item, usize) { - match self { - SmallVecData::Heap(data) => data, - _ => debug_unreachable!(), + pub fn from_vec(vec: Vec) -> Self { + if Self::is_zst() { + // "Move" elements to stack buffer. They're ZST so we don't actually have to do + // anything. Just make sure they're not dropped. + // We don't wrap the vector in ManuallyDrop so that when it's dropped, the memory is + // deallocated, if it needs to be. + let mut vec = vec; + let len = vec.len(); + + // SAFETY: `0` is less than the vector's capacity. + // old_len..new_len is an empty range. So there are no uninitialized elements + unsafe { vec.set_len(0) }; + Self { + len: TaggedLen::new(len, false, Self::is_zst()), + raw: RawSmallVec::new(), + } + } else { + let mut vec = ManuallyDrop::new(vec); + let len = vec.len(); + let cap = vec.capacity(); + let ptr = vec.as_mut_ptr(); + + Self { + len: TaggedLen::new(len, true, Self::is_zst()), + raw: RawSmallVec::new_heap(ptr, cap), + } } } - #[inline] - fn from_heap(ptr: *mut A::Item, len: usize) -> SmallVecData { - SmallVecData::Heap((ptr, len)) - } -} - -unsafe impl Send for SmallVecData {} -unsafe impl Sync for SmallVecData {} - -/// A `Vec`-like container that can store a small number of elements inline. -/// -/// `SmallVec` acts like a vector, but can store a limited amount of data inline within the -/// `SmallVec` struct rather than in a separate allocation. If the data exceeds this limit, the -/// `SmallVec` will "spill" its data onto the heap, allocating a new buffer to hold it. -/// -/// The amount of data that a `SmallVec` can store inline depends on its backing store. The backing -/// store can be any type that implements the `Array` trait; usually it is a small fixed-sized -/// array. For example a `SmallVec<[u64; 8]>` can hold up to eight 64-bit integers inline. -/// -/// ## Example -/// -/// ```rust -/// use smallvec::SmallVec; -/// let mut v = SmallVec::<[u8; 4]>::new(); // initialize an empty vector -/// -/// // The vector can hold up to 4 items without spilling onto the heap. -/// v.extend(0..4); -/// assert_eq!(v.len(), 4); -/// assert!(!v.spilled()); -/// -/// // Pushing another element will force the buffer to spill: -/// v.push(4); -/// assert_eq!(v.len(), 5); -/// assert!(v.spilled()); -/// ``` -pub struct SmallVec { - // The capacity field is used to determine which of the storage variants is active: - // If capacity <= Self::inline_capacity() then the inline variant is used and capacity holds the current length of the vector (number of elements actually in use). - // If capacity > Self::inline_capacity() then the heap variant is used and capacity holds the size of the memory allocation. - capacity: usize, - data: SmallVecData, -} -impl SmallVec { - /// Construct an empty vector #[inline] - pub fn new() -> SmallVec { - // Try to detect invalid custom implementations of `Array`. Hopefully, - // this check should be optimized away entirely for valid ones. - assert!( - mem::size_of::() == A::size() * mem::size_of::() - && mem::align_of::() >= mem::align_of::() - ); - SmallVec { - capacity: 0, - data: SmallVecData::from_inline(MaybeUninit::uninit()), + pub const fn from_buf(buf: [T; N]) -> Self { + Self { + len: TaggedLen::new(N, false, Self::is_zst()), + raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), } } - /// Construct an empty vector with enough capacity pre-allocated to store at least `n` - /// elements. - /// - /// Will create a heap allocation only if `n` is larger than the inline capacity. - /// - /// ``` - /// # use smallvec::SmallVec; - /// - /// let v: SmallVec<[u8; 3]> = SmallVec::with_capacity(100); - /// - /// assert!(v.is_empty()); - /// assert!(v.capacity() >= 100); - /// ``` - #[inline] - pub fn with_capacity(n: usize) -> Self { - let mut v = SmallVec::new(); - v.reserve_exact(n); - v - } - - /// Construct a new `SmallVec` from a `Vec`. - /// - /// Elements will be copied to the inline buffer if vec.capacity() <= Self::inline_capacity(). - /// - /// ```rust - /// use smallvec::SmallVec; - /// - /// let vec = vec![1, 2, 3, 4, 5]; - /// let small_vec: SmallVec<[_; 3]> = SmallVec::from_vec(vec); - /// - /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); - /// ``` - #[inline] - pub fn from_vec(mut vec: Vec) -> SmallVec { - if vec.capacity() <= Self::inline_capacity() { - unsafe { - let mut data = SmallVecData::::from_inline(MaybeUninit::uninit()); - let len = vec.len(); - vec.set_len(0); - ptr::copy_nonoverlapping(vec.as_ptr(), data.inline_mut(), len); - - SmallVec { - capacity: len, - data, - } - } - } else { - let (ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len()); - mem::forget(vec); - - SmallVec { - capacity: cap, - data: SmallVecData::from_heap(ptr, len), - } + #[inline] + pub fn from_buf_and_len(buf: [T; N], len: usize) -> Self { + assert!(len <= N); + let mut vec = Self { + len: TaggedLen::new(len, false, Self::is_zst()), + raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), + }; + // Deallocate the remaining elements so no memory is leaked. + unsafe { + // SAFETY: both the input and output pointers are in range of the stack allocation + let remainder_ptr = addr_of_mut!(vec.raw.inline).add(len); + let remainder_len = N - len; + + // SAFETY: the values are initialized, so dropping them here is fine. + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + remainder_ptr, + remainder_len, + )); } + + vec } - /// Constructs a new `SmallVec` on the stack from an `A` without - /// copying elements. - /// - /// ```rust - /// use smallvec::SmallVec; - /// - /// let buf = [1, 2, 3, 4, 5]; - /// let small_vec: SmallVec<_> = SmallVec::from_buf(buf); - /// - /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); - /// ``` #[inline] - pub fn from_buf(buf: A) -> SmallVec { - SmallVec { - capacity: A::size(), - data: SmallVecData::from_inline(MaybeUninit::new(buf)), + pub const unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit<[T; N]>, len: usize) -> Self { + debug_assert!(len <= N); + Self { + len: TaggedLen::new(len, false, Self::is_zst()), + raw: RawSmallVec::new_inline(buf), } } - /// Constructs a new `SmallVec` on the stack from an `A` without - /// copying elements. Also sets the length, which must be less or - /// equal to the size of `buf`. - /// - /// ```rust - /// use smallvec::SmallVec; - /// - /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; - /// let small_vec: SmallVec<_> = SmallVec::from_buf_and_len(buf, 5); - /// - /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); - /// ``` - #[inline] - pub fn from_buf_and_len(buf: A, len: usize) -> SmallVec { - assert!(len <= A::size()); - unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), len) } - } - - /// Constructs a new `SmallVec` on the stack from an `A` without - /// copying elements. Also sets the length. The user is responsible - /// for ensuring that `len <= A::size()`. - /// - /// ```rust - /// use smallvec::SmallVec; - /// use std::mem::MaybeUninit; - /// - /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; - /// let small_vec: SmallVec<_> = unsafe { - /// SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) - /// }; - /// - /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); - /// ``` - #[inline] - pub unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit, len: usize) -> SmallVec { - SmallVec { - capacity: len, - data: SmallVecData::from_inline(buf), - } + #[inline] + unsafe fn set_on_heap(&mut self) { + self.len = TaggedLen::new(self.len(), true, Self::is_zst()); + } + #[inline] + unsafe fn set_inline(&mut self) { + self.len = TaggedLen::new(self.len(), false, Self::is_zst()); } - /// Sets the length of a vector. - /// - /// This will explicitly set the size of the vector, without actually - /// modifying its buffers, so it is up to the caller to ensure that the - /// vector is actually the specified size. + #[inline] pub unsafe fn set_len(&mut self, new_len: usize) { - let (_, len_ptr, _) = self.triple_mut(); - *len_ptr = new_len; + debug_assert!(new_len <= self.capacity()); + let on_heap = self.len.on_heap(Self::is_zst()); + self.len = TaggedLen::new(new_len, on_heap, Self::is_zst()); } - /// The maximum number of elements this vector can hold inline #[inline] - fn inline_capacity() -> usize { - if mem::size_of::() > 0 { - A::size() + const fn inline_capacity() -> usize { + if Self::is_zst() { + usize::MAX } else { - // For zero-size items code like `ptr.add(offset)` always returns the same pointer. - // Therefore all items are at the same address, - // and any array size has capacity for infinitely many items. - // The capacity is limited by the bit width of the length field. - // - // `Vec` also does this: - // https://github.com/rust-lang/rust/blob/1.44.0/src/liballoc/raw_vec.rs#L186 - // - // In our case, this also ensures that a smallvec of zero-size items never spills, - // and we never try to allocate zero bytes which `std::alloc::alloc` disallows. - core::usize::MAX + N } } - /// The maximum number of elements this vector can hold inline #[inline] - pub fn inline_size(&self) -> usize { + pub const fn inline_size(&self) -> usize { Self::inline_capacity() } - /// The number of elements stored in the vector #[inline] - pub fn len(&self) -> usize { - self.triple().1 + pub const fn len(&self) -> usize { + self.len.value(Self::is_zst()) } - /// Returns `true` if the vector is empty + #[must_use] #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } - /// The number of items the vector can hold without reallocating - #[inline] - pub fn capacity(&self) -> usize { - self.triple().2 - } - - /// Returns a tuple with (data ptr, len, capacity) - /// Useful to get all SmallVec properties with a single check of the current storage variant. - #[inline] - fn triple(&self) -> (*const A::Item, usize, usize) { - unsafe { - if self.spilled() { - let (ptr, len) = self.data.heap(); - (ptr, len, self.capacity) - } else { - (self.data.inline(), self.capacity, Self::inline_capacity()) - } - } - } - - /// Returns a tuple with (data ptr, len ptr, capacity) #[inline] - fn triple_mut(&mut self) -> (*mut A::Item, &mut usize, usize) { - unsafe { - if self.spilled() { - let &mut (ptr, ref mut len_ptr) = self.data.heap_mut(); - (ptr, len_ptr, self.capacity) - } else { - ( - self.data.inline_mut(), - &mut self.capacity, - Self::inline_capacity(), - ) - } + pub const fn capacity(&self) -> usize { + if self.len.on_heap(Self::is_zst()) { + unsafe { self.raw.heap.1 } + } else { + self.inline_size() } } - /// Returns `true` if the data has spilled into a separate heap-allocated buffer. #[inline] - pub fn spilled(&self) -> bool { - self.capacity > Self::inline_capacity() + pub const fn spilled(&self) -> bool { + self.len.on_heap(Self::is_zst()) } - /// Creates a draining iterator that removes the specified range in the vector - /// and yields the removed items. - /// - /// Note 1: The element range is removed even if the iterator is only - /// partially consumed or not consumed at all. - /// - /// Note 2: It is unspecified how many elements are removed from the vector - /// if the `Drain` value is leaked. - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - pub fn drain(&mut self, range: R) -> Drain<'_, A> + pub fn drain(&mut self, range: R) -> Drain<'_, T, N> where - R: RangeBounds, + R: core::ops::RangeBounds, { use core::ops::Bound::*; @@ -816,443 +627,376 @@ impl SmallVec { unsafe { self.set_len(start); - let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); + let range_slice = core::slice::from_raw_parts(self.as_ptr().add(start), end - start); Drain { tail_start: end, tail_len: len - end, iter: range_slice.iter(), // Since self is a &mut, passing it to a function would invalidate the slice iterator. - vec: NonNull::new_unchecked(self as *mut _), + vec: core::ptr::NonNull::new_unchecked(self as *mut _), } } } - /// Append an item to the vector. #[inline] - pub fn push(&mut self, value: A::Item) { - unsafe { - let (mut ptr, mut len, cap) = self.triple_mut(); - if *len == cap { - self.reserve(1); - let &mut (heap_ptr, ref mut heap_len) = self.data.heap_mut(); - ptr = heap_ptr; - len = heap_len; - } - ptr::write(ptr.add(*len), value); - *len += 1; + pub fn push(&mut self, value: T) { + let len = self.len(); + if len == self.capacity() { + self.reserve(1); } + // SAFETY: both the input and output are within the allocation + let ptr = unsafe { self.as_mut_ptr().add(len) }; + // SAFETY: we allocated enough space in case it wasn't enough, so the address is valid for + // writes. + unsafe { ptr.write(value) }; + unsafe { self.set_len(len + 1) } } - /// Remove an item from the end of the vector and return it, or None if empty. #[inline] - pub fn pop(&mut self) -> Option { - unsafe { - let (ptr, len_ptr, _) = self.triple_mut(); - if *len_ptr == 0 { - return None; - } - let last_index = *len_ptr - 1; - *len_ptr = last_index; - Some(ptr::read(ptr.add(last_index))) + pub fn pop(&mut self) -> Option { + if self.is_empty() { + None + } else { + let len = self.len() - 1; + // SAFETY: len < old_len since this can't overflow, because the old length is non zero + unsafe { self.set_len(len) }; + let value = unsafe { self.as_mut_ptr().add(len).read() }; + Some(value) } } - /// Moves all the elements of `other` into `self`, leaving `other` empty. - /// - /// # Example - /// - /// ``` - /// # use smallvec::{SmallVec, smallvec}; - /// let mut v0: SmallVec<[u8; 16]> = smallvec![1, 2, 3]; - /// let mut v1: SmallVec<[u8; 32]> = smallvec![4, 5, 6]; - /// v0.append(&mut v1); - /// assert_eq!(*v0, [1, 2, 3, 4, 5, 6]); - /// assert_eq!(*v1, []); - /// ``` - pub fn append(&mut self, other: &mut SmallVec) - where - B: Array, - { - self.extend(other.drain(..)) + #[inline] + pub fn append(&mut self, other: &mut SmallVec) { + // can't overflow since both are smaller than isize::MAX and 2 * isize::MAX < usize::MAX + let len = self.len(); + let other_len = other.len(); + let total_len = len + other_len; + if total_len > self.capacity() { + self.reserve(other_len); + } + + // SAFETY: see `Self::push` + let ptr = unsafe { self.as_mut_ptr().add(len) }; + // SAFETY: we have a mutable reference to each vector and each uniquely owns its memory. + // so the ranges can't overlap + unsafe { copy_nonoverlapping(other.as_ptr(), ptr, other_len) }; + unsafe { other.set_len(0) } } - /// Re-allocate to set the capacity to `max(new_cap, inline_size())`. - /// - /// Panics if `new_cap` is less than the vector's length - /// or if the capacity computation overflows `usize`. - pub fn grow(&mut self, new_cap: usize) { - infallible(self.try_grow(new_cap)) + #[inline] + pub fn grow(&mut self, new_capacity: usize) { + infallible(self.try_grow(new_capacity)); } - /// Re-allocate to set the capacity to `max(new_cap, inline_size())`. - /// - /// Panics if `new_cap` is less than the vector's length - pub fn try_grow(&mut self, new_cap: usize) -> Result<(), CollectionAllocErr> { - unsafe { - let (ptr, &mut len, cap) = self.triple_mut(); - let unspilled = !self.spilled(); - assert!(new_cap >= len); - if new_cap <= self.inline_size() { - if unspilled { - return Ok(()); - } - self.data = SmallVecData::from_inline(MaybeUninit::uninit()); - ptr::copy_nonoverlapping(ptr, self.data.inline_mut(), len); - self.capacity = len; - deallocate(ptr, cap); - } else if new_cap != cap { - let layout = layout_array::(new_cap)?; - debug_assert!(layout.size() > 0); - let new_alloc; - if unspilled { - new_alloc = NonNull::new(alloc::alloc::alloc(layout)) - .ok_or(CollectionAllocErr::AllocErr { layout })? - .cast() - .as_ptr(); - ptr::copy_nonoverlapping(ptr, new_alloc, len); - } else { - // This should never fail since the same succeeded - // when previously allocating `ptr`. - let old_layout = layout_array::(cap)?; - - let new_ptr = alloc::alloc::realloc(ptr as *mut u8, old_layout, layout.size()); - new_alloc = NonNull::new(new_ptr) - .ok_or(CollectionAllocErr::AllocErr { layout })? - .cast() - .as_ptr(); + #[inline] + pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), CollectionAllocErr> { + let len = self.len(); + assert!(new_capacity >= len); + + if new_capacity > self.inline_size() { + let result = unsafe { self.raw.try_grow(self.len, new_capacity) }; + if result.is_ok() { + unsafe { self.set_on_heap() }; + } + result + } else { + if self.spilled() { + unsafe { + let (ptr, old_cap) = self.raw.heap; + copy_nonoverlapping(ptr, self.raw.as_mut_ptr_inline(), len); + { + let _drop_dealloc = DropDealloc { + ptr: ptr as *mut u8, + size_bytes: old_cap * size_of::(), + align: align_of::(), + }; + } + self.set_inline(); } - self.data = SmallVecData::from_heap(new_alloc, len); - self.capacity = new_cap; } Ok(()) } } - /// Reserve capacity for `additional` more elements to be inserted. - /// - /// May reserve more space to avoid frequent reallocations. - /// - /// Panics if the capacity computation overflows `usize`. #[inline] pub fn reserve(&mut self, additional: usize) { - infallible(self.try_reserve(additional)) + // can't overflow since len <= capacity + if additional > self.capacity() - self.len() { + let new_capacity = infallible( + self.len() + .checked_add(additional) + .and_then(usize::checked_next_power_of_two) + .ok_or(CollectionAllocErr::CapacityOverflow), + ); + self.grow(new_capacity); + } } - /// Reserve capacity for `additional` more elements to be inserted. - /// - /// May reserve more space to avoid frequent reallocations. + #[inline] pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { - // prefer triple_mut() even if triple() would work - // so that the optimizer removes duplicated calls to it - // from callers like insert() - let (_, &mut len, cap) = self.triple_mut(); - if cap - len >= additional { - return Ok(()); + if additional > self.capacity() - self.len() { + let new_capacity = self + .len() + .checked_add(additional) + .and_then(usize::checked_next_power_of_two) + .ok_or(CollectionAllocErr::CapacityOverflow)?; + self.try_grow(new_capacity) + } else { + Ok(()) } - let new_cap = len - .checked_add(additional) - .and_then(usize::checked_next_power_of_two) - .ok_or(CollectionAllocErr::CapacityOverflow)?; - self.try_grow(new_cap) } - /// Reserve the minimum capacity for `additional` more elements to be inserted. - /// - /// Panics if the new capacity overflows `usize`. + #[inline] pub fn reserve_exact(&mut self, additional: usize) { - infallible(self.try_reserve_exact(additional)) + // can't overflow since len <= capacity + if additional > self.capacity() - self.len() { + let new_capacity = infallible( + self.len() + .checked_add(additional) + .ok_or(CollectionAllocErr::CapacityOverflow), + ); + self.grow(new_capacity); + } } - /// Reserve the minimum capacity for `additional` more elements to be inserted. + #[inline] pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { - let (_, &mut len, cap) = self.triple_mut(); - if cap - len >= additional { - return Ok(()); + if additional > self.capacity() - self.len() { + let new_capacity = self + .len() + .checked_add(additional) + .ok_or(CollectionAllocErr::CapacityOverflow)?; + self.try_grow(new_capacity) + } else { + Ok(()) } - let new_cap = len - .checked_add(additional) - .ok_or(CollectionAllocErr::CapacityOverflow)?; - self.try_grow(new_cap) } - /// Shrink the capacity of the vector as much as possible. - /// - /// When possible, this will move data from an external heap buffer to the vector's inline - /// storage. + #[inline] pub fn shrink_to_fit(&mut self) { if !self.spilled() { return; } let len = self.len(); - if self.inline_size() >= len { + if len <= self.inline_size() { + let (ptr, capacity) = unsafe { self.raw.heap }; + self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); + unsafe { copy_nonoverlapping(ptr, self.raw.as_mut_ptr_inline(), len) }; + unsafe { self.set_inline() }; unsafe { - let (ptr, len) = self.data.heap(); - self.data = SmallVecData::from_inline(MaybeUninit::uninit()); - ptr::copy_nonoverlapping(ptr, self.data.inline_mut(), len); - deallocate(ptr, self.capacity); - self.capacity = len; - } - } else if self.capacity() > len { + alloc::alloc::dealloc( + ptr as *mut T as *mut u8, + Layout::from_size_align_unchecked(capacity * size_of::(), align_of::()), + ) + }; + } else if len < self.capacity() { self.grow(len); } } - /// Shorten the vector, keeping the first `len` elements and dropping the rest. - /// - /// If `len` is greater than or equal to the vector's current length, this has no - /// effect. - /// - /// This does not re-allocate. If you want the vector's capacity to shrink, call - /// `shrink_to_fit` after truncating. + #[inline] pub fn truncate(&mut self, len: usize) { - unsafe { - let (ptr, len_ptr, _) = self.triple_mut(); - while len < *len_ptr { - let last_index = *len_ptr - 1; - *len_ptr = last_index; - ptr::drop_in_place(ptr.add(last_index)); + let old_len = self.len(); + if len < old_len { + unsafe { + self.set_len(len); + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.as_mut_ptr().add(len), + old_len - len, + )) } } } - /// Extracts a slice containing the entire vector. - /// - /// Equivalent to `&s[..]`. - pub fn as_slice(&self) -> &[A::Item] { - self + #[inline] + pub fn as_slice(&self) -> &[T] { + let len = self.len(); + let ptr = self.as_ptr(); + unsafe { core::slice::from_raw_parts(ptr, len) } } - /// Extracts a mutable slice of the entire vector. - /// - /// Equivalent to `&mut s[..]`. - pub fn as_mut_slice(&mut self) -> &mut [A::Item] { - self + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + let len = self.len(); + let ptr = self.as_mut_ptr(); + unsafe { core::slice::from_raw_parts_mut(ptr, len) } } - /// Remove the element at position `index`, replacing it with the last element. - /// - /// This does not preserve ordering, but is O(1). - /// - /// Panics if `index` is out of bounds. #[inline] - pub fn swap_remove(&mut self, index: usize) -> A::Item { + pub fn swap_remove(&mut self, index: usize) -> T { let len = self.len(); - self.swap(len - 1, index); - self.pop() - .unwrap_or_else(|| unsafe { unreachable_unchecked() }) + assert!(index < len); + let new_len = len - 1; + unsafe { + self.set_len(new_len); + let ptr = self.as_mut_ptr(); + let last = ptr.add(new_len); + let ith = ptr.add(index); + let last_item = last.read(); + let ith_item = ith.read(); + ith.write(last_item); + ith_item + } } - /// Remove all elements from the vector. #[inline] pub fn clear(&mut self) { self.truncate(0); } - /// Remove and return the element at position `index`, shifting all elements after it to the - /// left. - /// - /// Panics if `index` is out of bounds. - pub fn remove(&mut self, index: usize) -> A::Item { + #[inline] + pub fn remove(&mut self, index: usize) -> T { + let len = self.len(); + assert!(index < len); + let new_len = len - 1; unsafe { - let (mut ptr, len_ptr, _) = self.triple_mut(); - let len = *len_ptr; - assert!(index < len); - *len_ptr = len - 1; - ptr = ptr.add(index); - let item = ptr::read(ptr); - ptr::copy(ptr.add(1), ptr, len - index - 1); - item + self.set_len(new_len); + let ptr = self.as_mut_ptr(); + let ith = ptr.add(index); + let ith_item = ith.read(); + core::ptr::copy(ith.add(1), ith, new_len - index); + ith_item } } - /// Insert an element at position `index`, shifting all elements after it to the right. - /// - /// Panics if `index > len`. - pub fn insert(&mut self, index: usize, element: A::Item) { + #[inline] + pub fn insert(&mut self, index: usize, value: T) { + let len = self.len(); + assert!(index <= len); self.reserve(1); - + let ptr = self.as_mut_ptr(); unsafe { - let (mut ptr, len_ptr, _) = self.triple_mut(); - let len = *len_ptr; - ptr = ptr.add(index); if index < len { - ptr::copy(ptr, ptr.add(1), len - index); - } else if index == len { - // No elements need shifting. - } else { - panic!("index exceeds length"); + core::ptr::copy(ptr.add(index), ptr.add(index + 1), len - index); } - *len_ptr = len + 1; - ptr::write(ptr, element); + ptr.add(index).write(value); + self.set_len(len + 1); } } - /// Insert multiple elements at position `index`, shifting all following elements toward the - /// back. - pub fn insert_many>(&mut self, index: usize, iterable: I) { - let mut iter = iterable.into_iter(); - if index == self.len() { + fn insert_many_impl>(&mut self, mut index: usize, mut iter: I) { + let len = self.len(); + if index == len { return self.extend(iter); } - let (lower_size_bound, _) = iter.size_hint(); - assert!(lower_size_bound <= core::isize::MAX as usize); // Ensure offset is indexable - assert!(index + lower_size_bound >= index); // Protect against overflow + let (lower_bound, _) = iter.size_hint(); + self.reserve(lower_bound); - let mut num_added = 0; - let old_len = self.len(); - assert!(index <= old_len); - - unsafe { - // Reserve space for `lower_size_bound` elements. - self.reserve(lower_size_bound); - let start = self.as_mut_ptr(); - let ptr = start.add(index); - - // Move the trailing elements. - ptr::copy(ptr, ptr.add(lower_size_bound), old_len - index); - - // In case the iterator panics, don't double-drop the items we just copied above. - self.set_len(0); - let mut guard = DropOnPanic { - start, - skip: index..(index + lower_size_bound), - len: old_len + lower_size_bound, - }; - - // The set_len above invalidates the previous pointers, so we must re-create them. - let start = self.as_mut_ptr(); - let ptr = start.add(index); - - while num_added < lower_size_bound { - let element = match iter.next() { - Some(x) => x, - None => break, - }; - let cur = ptr.add(num_added); - ptr::write(cur, element); - guard.skip.start += 1; - num_added += 1; - } + let count = unsafe { + let ptr = self.as_mut_ptr(); + let count = insert_many_batch_phase(ptr, index, lower_bound, len, &mut iter); + self.set_len(len + count); + count + }; - if num_added < lower_size_bound { - // Iterator provided fewer elements than the hint. Move the tail backward. - ptr::copy( - ptr.add(lower_size_bound), - ptr.add(num_added), - old_len - index, - ); - } - // There are no more duplicate or uninitialized slots, so the guard is not needed. - self.set_len(old_len + num_added); - mem::forget(guard); - } + index += count; + iter.enumerate() + .for_each(|(i, item)| self.insert(index + i, item)); + } - // Insert any remaining elements one-by-one. - for element in iter { - self.insert(index + num_added, element); - num_added += 1; - } + #[inline] + pub fn insert_many>(&mut self, index: usize, iterable: I) { + self.insert_many_impl(index, iterable.into_iter()); + } - struct DropOnPanic { - start: *mut T, - skip: Range, // Space we copied-out-of, but haven't written-to yet. - len: usize, + #[inline] + pub const fn as_ptr(&self) -> *const T { + if self.len.on_heap(Self::is_zst()) { + unsafe { self.raw.as_ptr_heap() } + } else { + self.raw.as_ptr_inline() } + } - impl Drop for DropOnPanic { - fn drop(&mut self) { - for i in 0..self.len { - if !self.skip.contains(&i) { - unsafe { - ptr::drop_in_place(self.start.add(i)); - } - } - } - } + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + if self.len.on_heap(Self::is_zst()) { + unsafe { self.raw.as_mut_ptr_heap() } + } else { + self.raw.as_mut_ptr_inline() } } - /// Convert a SmallVec to a Vec, without reallocating if the SmallVec has already spilled onto - /// the heap. - pub fn into_vec(self) -> Vec { - if self.spilled() { + #[inline] + pub fn into_vec(self) -> Vec { + let len = self.len(); + if !self.spilled() { + let mut vec = Vec::with_capacity(len); + let this = ManuallyDrop::new(self); unsafe { - let (ptr, len) = self.data.heap(); - let v = Vec::from_raw_parts(ptr, len, self.capacity); - mem::forget(self); - v + copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); + vec.set_len(len); } + vec } else { - self.into_iter().collect() + let this = ManuallyDrop::new(self); + unsafe { + let (ptr, cap) = this.raw.heap; + Vec::from_raw_parts(ptr as *mut T, len, cap) + } } } - /// Converts a `SmallVec` into a `Box<[T]>` without reallocating if the `SmallVec` has already spilled - /// onto the heap. - /// - /// Note that this will drop any excess capacity. - pub fn into_boxed_slice(self) -> Box<[A::Item]> { + #[inline] + pub fn into_boxed_slice(self) -> Box<[T]> { self.into_vec().into_boxed_slice() } - /// Convert the SmallVec into an `A` if possible. Otherwise return `Err(Self)`. - /// - /// This method returns `Err(Self)` if the SmallVec is too short (and the `A` contains uninitialized elements), - /// or if the SmallVec is too long (and all the elements were spilled to the heap). - pub fn into_inner(self) -> Result { - if self.spilled() || self.len() != A::size() { - // Note: A::size, not Self::inline_capacity + #[inline] + pub fn into_inner(self) -> Result<[T; N], Self> { + if self.len() != N { Err(self) } else { - unsafe { - let data = ptr::read(&self.data); - mem::forget(self); - Ok(data.into_inline().assume_init()) - } + let this = ManuallyDrop::new(self); + let ptr = this.as_ptr() as *const [T; N]; + unsafe { Ok(ptr.read()) } } } - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` such that `f(&e)` returns `false`. - /// This method operates in place and preserves the order of the retained - /// elements. - pub fn retain bool>(&mut self, mut f: F) { + pub fn retain bool>(&mut self, mut f: F) { let mut del = 0; let len = self.len(); for i in 0..len { - if !f(&mut self[i]) { - del += 1; - } else if del > 0 { - self.swap(i - del, i); + unsafe { + let ptr = self.as_mut_ptr().add(i); + if !f(&mut *ptr) { + del += 1; + } else if del > 0 { + core::mem::swap(&mut *ptr, &mut *ptr.sub(del)); + } } } self.truncate(len - del); } - /// Retains only the elements specified by the predicate. - /// - /// This method is identical in behaviour to [`retain`]; it is included only - /// to maintain api-compatability with `std::Vec`, where the methods are - /// separate for historical reasons. - pub fn retain_mut bool>(&mut self, f: F) { + #[inline] + pub fn retain_mut bool>(&mut self, f: F) { self.retain(f) } - /// Removes consecutive duplicate elements. + #[inline] pub fn dedup(&mut self) where - A::Item: PartialEq, + T: PartialEq, { self.dedup_by(|a, b| a == b); } - /// Removes consecutive duplicate elements using the given equality relation. + #[inline] + pub fn dedup_by_key(&mut self, mut key: F) + where + F: FnMut(&mut T) -> K, + K: PartialEq, + { + self.dedup_by(|a, b| key(a) == key(b)); + } + + #[inline] pub fn dedup_by(&mut self, mut same_bucket: F) where - F: FnMut(&mut A::Item, &mut A::Item) -> bool, + F: FnMut(&mut T, &mut T) -> bool, { // See the implementation of Vec::dedup_by in the // standard library for an explanation of this algorithm. @@ -1271,7 +1015,7 @@ impl SmallVec { if !same_bucket(&mut *p_r, &mut *p_wm1) { if r != w { let p_w = p_wm1.add(1); - mem::swap(&mut *p_r, &mut *p_w); + core::mem::swap(&mut *p_r, &mut *p_w); } w += 1; } @@ -1281,43 +1025,9 @@ impl SmallVec { self.truncate(w); } - /// Removes consecutive elements that map to the same key. - pub fn dedup_by_key(&mut self, mut key: F) - where - F: FnMut(&mut A::Item) -> K, - K: PartialEq, - { - self.dedup_by(|a, b| key(a) == key(b)); - } - - /// Resizes the `SmallVec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `SmallVec` is extended by the difference, with each - /// additional slot filled with the result of calling the closure `f`. The return values from `f` - //// will end up in the `SmallVec` in the order they have been generated. - /// - /// If `new_len` is less than `len`, the `SmallVec` is simply truncated. - /// - /// This method uses a closure to create new values on every push. If you'd rather `Clone` a given - /// value, use `resize`. If you want to use the `Default` trait to generate values, you can pass - /// `Default::default()` as the second argument. - /// - /// Added for std::vec::Vec compatibility (added in Rust 1.33.0) - /// - /// ``` - /// # use smallvec::{smallvec, SmallVec}; - /// let mut vec : SmallVec<[_; 4]> = smallvec![1, 2, 3]; - /// vec.resize_with(5, Default::default); - /// assert_eq!(&*vec, &[1, 2, 3, 0, 0]); - /// - /// let mut vec : SmallVec<[_; 4]> = smallvec![]; - /// let mut p = 1; - /// vec.resize_with(4, || { p *= 2; p }); - /// assert_eq!(&*vec, &[2, 4, 8, 16]); - /// ``` pub fn resize_with(&mut self, new_len: usize, f: F) where - F: FnMut() -> A::Item, + F: FnMut() -> T, { let old_len = self.len(); if old_len < new_len { @@ -1332,500 +1042,306 @@ impl SmallVec { } } - /// Creates a `SmallVec` directly from the raw components of another - /// `SmallVec`. - /// - /// # Safety - /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` needs to have been previously allocated via `SmallVec` for its - /// spilled storage (at least, it's highly likely to be incorrect if it - /// wasn't). - /// * `ptr`'s `A::Item` type needs to be the same size and alignment that - /// it was allocated with - /// * `length` needs to be less than or equal to `capacity`. - /// * `capacity` needs to be the capacity that the pointer was allocated - /// with. - /// - /// Violating these may cause problems like corrupting the allocator's - /// internal data structures. - /// - /// Additionally, `capacity` must be greater than the amount of inline - /// storage `A` has; that is, the new `SmallVec` must need to spill over - /// into heap allocated storage. This condition is asserted against. - /// - /// The ownership of `ptr` is effectively transferred to the - /// `SmallVec` which may then deallocate, reallocate or change the - /// contents of memory pointed to by the pointer at will. Ensure - /// that nothing else uses the pointer after calling this - /// function. - /// - /// # Examples - /// - /// ``` - /// # #[macro_use] extern crate smallvec; - /// # use smallvec::SmallVec; - /// use std::mem; - /// use std::ptr; - /// - /// fn main() { - /// let mut v: SmallVec<[_; 1]> = smallvec![1, 2, 3]; - /// - /// // Pull out the important parts of `v`. - /// let p = v.as_mut_ptr(); - /// let len = v.len(); - /// let cap = v.capacity(); - /// let spilled = v.spilled(); - /// - /// unsafe { - /// // Forget all about `v`. The heap allocation that stored the - /// // three values won't be deallocated. - /// mem::forget(v); - /// - /// // Overwrite memory with [4, 5, 6]. - /// // - /// // This is only safe if `spilled` is true! Otherwise, we are - /// // writing into the old `SmallVec`'s inline storage on the - /// // stack. - /// assert!(spilled); - /// for i in 0..len { - /// ptr::write(p.add(i), 4 + i); - /// } - /// - /// // Put everything back together into a SmallVec with a different - /// // amount of inline storage, but which is still less than `cap`. - /// let rebuilt = SmallVec::<[_; 2]>::from_raw_parts(p, len, cap); - /// assert_eq!(&*rebuilt, &[4, 5, 6]); - /// } - /// } - #[inline] - pub unsafe fn from_raw_parts(ptr: *mut A::Item, length: usize, capacity: usize) -> SmallVec { - assert!(capacity > Self::inline_capacity()); + #[inline] + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> SmallVec { + assert!(!Self::is_zst()); SmallVec { - capacity, - data: SmallVecData::from_heap(ptr, length), + len: TaggedLen::new(length, true, Self::is_zst()), + raw: RawSmallVec::new_heap(ptr, capacity), } } - /// Returns a raw pointer to the vector's buffer. - pub fn as_ptr(&self) -> *const A::Item { - // We shadow the slice method of the same name to avoid going through - // `deref`, which creates an intermediate reference that may place - // additional safety constraints on the contents of the slice. - self.triple().0 + fn extend_impl>(&mut self, mut iter: I) { + let len = self.len(); + let (lower_bound, _) = iter.size_hint(); + self.reserve(lower_bound); + unsafe { + let ptr = self.as_mut_ptr(); + let count = extend_batch_phase(ptr, lower_bound, len, &mut iter); + self.set_len(len + count); + } + iter.for_each(|item| self.push(item)); } +} - /// Returns a raw mutable pointer to the vector's buffer. - pub fn as_mut_ptr(&mut self) -> *mut A::Item { - // We shadow the slice method of the same name to avoid going through - // `deref_mut`, which creates an intermediate reference that may place - // additional safety constraints on the contents of the slice. - self.triple_mut().0 +impl Default for SmallVec { + #[inline] + fn default() -> Self { + Self::new() } } -impl SmallVec -where - A::Item: Copy, -{ - /// Copy the elements from a slice into a new `SmallVec`. - /// - /// For slices of `Copy` types, this is more efficient than `SmallVec::from(slice)`. - pub fn from_slice(slice: &[A::Item]) -> Self { +impl SmallVec { + #[inline] + pub fn from_slice(slice: &[T]) -> Self { let len = slice.len(); - if len <= Self::inline_capacity() { - SmallVec { - capacity: len, - data: SmallVecData::from_inline(unsafe { - let mut data: MaybeUninit = MaybeUninit::uninit(); - ptr::copy_nonoverlapping( - slice.as_ptr(), - data.as_mut_ptr() as *mut A::Item, - len, - ); - data - }), - } - } else { - let mut b = slice.to_vec(); - let (ptr, cap) = (b.as_mut_ptr(), b.capacity()); - mem::forget(b); - SmallVec { - capacity: cap, - data: SmallVecData::from_heap(ptr, len), - } + + let mut this = Self::with_capacity(len); + let ptr = this.as_mut_ptr(); + unsafe { + copy_nonoverlapping(slice.as_ptr(), ptr, len); + this.set_len(len); } + this } - /// Copy elements from a slice into the vector at position `index`, shifting any following - /// elements toward the back. - /// - /// For slices of `Copy` types, this is more efficient than `insert`. - pub fn insert_from_slice(&mut self, index: usize, slice: &[A::Item]) { - self.reserve(slice.len()); - + #[inline] + pub fn insert_from_slice(&mut self, index: usize, slice: &[T]) { let len = self.len(); + let other_len = slice.len(); assert!(index <= len); - + self.reserve(other_len); unsafe { - let slice_ptr = slice.as_ptr(); - let ptr = self.as_mut_ptr().add(index); - ptr::copy(ptr, ptr.add(slice.len()), len - index); - ptr::copy_nonoverlapping(slice_ptr, ptr, slice.len()); - self.set_len(len + slice.len()); + let base_ptr = self.as_mut_ptr(); + let ith_ptr = base_ptr.add(index); + let shifted_ptr = base_ptr.add(index + other_len); + core::ptr::copy(ith_ptr, shifted_ptr, len - index); + copy_nonoverlapping(slice.as_ptr(), ith_ptr, other_len); + self.set_len(len + other_len); } } - /// Copy elements from a slice and append them to the vector. - /// - /// For slices of `Copy` types, this is more efficient than `extend`. #[inline] - pub fn extend_from_slice(&mut self, slice: &[A::Item]) { + pub fn extend_from_slice(&mut self, slice: &[T]) { let len = self.len(); - self.insert_from_slice(len, slice); + let other_len = slice.len(); + self.reserve(other_len); + unsafe { + let base_ptr = self.as_mut_ptr(); + let end_ptr = base_ptr.add(len); + copy_nonoverlapping(slice.as_ptr(), end_ptr, other_len); + self.set_len(len + other_len); + } } } -impl SmallVec -where - A::Item: Clone, -{ - /// Resizes the vector so that its length is equal to `len`. - /// - /// If `len` is less than the current length, the vector simply truncated. - /// - /// If `len` is greater than the current length, `value` is appended to the - /// vector until its length equals `len`. - pub fn resize(&mut self, len: usize, value: A::Item) { +impl SmallVec { + #[inline] + pub fn resize(&mut self, len: usize, value: T) { let old_len = self.len(); - if len > old_len { - self.extend(repeat(value).take(len - old_len)); + self.extend(core::iter::repeat(value).take(len - old_len)); } else { self.truncate(len); } } - /// Creates a `SmallVec` with `n` copies of `elem`. - /// ``` - /// use smallvec::SmallVec; - /// - /// let v = SmallVec::<[char; 128]>::from_elem('d', 2); - /// assert_eq!(v, SmallVec::from_buf(['d', 'd'])); - /// ``` - pub fn from_elem(elem: A::Item, n: usize) -> Self { + #[inline] + pub fn from_elem(elem: T, n: usize) -> Self { if n > Self::inline_capacity() { - vec![elem; n].into() + Self::from_vec(vec![elem; n]) } else { - let mut v = SmallVec::::new(); + let mut v = Self::new(); + unsafe { - let (ptr, len_ptr, _) = v.triple_mut(); - let mut local_len = SetLenOnDrop::new(len_ptr); + let ptr = v.as_mut_ptr(); + let mut guard = DropGuard { ptr, len: 0 }; + // assume T is expensive to clone for i in 0..n { - ::core::ptr::write(ptr.add(i), elem.clone()); - local_len.increment_len(1); + guard.len = i; + ptr.add(i).write(elem.clone()); } + core::mem::forget(guard); + v.set_len(n); } v } } } -impl ops::Deref for SmallVec { - type Target = [A::Item]; +struct DropShiftGuard { + ptr: *mut T, + len: usize, + shifted_ptr: *const T, + shifted_len: usize, +} +impl Drop for DropShiftGuard { #[inline] - fn deref(&self) -> &[A::Item] { + fn drop(&mut self) { unsafe { - let (ptr, len, _) = self.triple(); - slice::from_raw_parts(ptr, len) + core::ptr::slice_from_raw_parts_mut(self.ptr, self.len).drop_in_place(); + core::ptr::copy(self.shifted_ptr, self.ptr, self.shifted_len); } } } -impl ops::DerefMut for SmallVec { +struct DropGuard { + ptr: *mut T, + len: usize, +} +impl Drop for DropGuard { #[inline] - fn deref_mut(&mut self) -> &mut [A::Item] { + fn drop(&mut self) { unsafe { - let (ptr, &mut len, _) = self.triple_mut(); - slice::from_raw_parts_mut(ptr, len) + core::ptr::slice_from_raw_parts_mut(self.ptr, self.len).drop_in_place(); } } } -impl AsRef<[A::Item]> for SmallVec { - #[inline] - fn as_ref(&self) -> &[A::Item] { - self - } -} +unsafe fn insert_many_batch_phase>( + ptr: *mut T, + index: usize, + lower_bound: usize, + len: usize, + iter: &mut I, +) -> usize { + // shift elements to the right to make space for the initial elements from the iterator + core::ptr::copy(ptr.add(index), ptr.add(index + lower_bound), len - index); + let ptr_ith = ptr.add(index); + let mut guard = DropShiftGuard { + ptr: ptr_ith, + len: 0, + shifted_ptr: ptr_ith.add(lower_bound), + shifted_len: len - index, + }; + iter.take(lower_bound).enumerate().for_each(|(i, item)| { + ptr_ith.add(i).write(item); + guard.len = i + 1; + }); + let count = guard.len; + core::mem::forget(guard); -impl AsMut<[A::Item]> for SmallVec { - #[inline] - fn as_mut(&mut self) -> &mut [A::Item] { - self + if count < lower_bound { + core::ptr::copy(ptr_ith.add(lower_bound), ptr_ith.add(count), len - index); } + count } -impl Borrow<[A::Item]> for SmallVec { - #[inline] - fn borrow(&self) -> &[A::Item] { - self - } +unsafe fn extend_batch_phase>( + ptr: *mut T, + lower_bound: usize, + len: usize, + iter: &mut I, +) -> usize { + let ptr_end = ptr.add(len); + let mut guard = DropGuard { + ptr: ptr_end, + len: 0, + }; + iter.take(lower_bound).enumerate().for_each(|(i, item)| { + ptr_end.add(i).write(item); + guard.len = i + 1; + }); + let count = guard.len; + core::mem::forget(guard); + count } -impl BorrowMut<[A::Item]> for SmallVec { +impl Extend for SmallVec { #[inline] - fn borrow_mut(&mut self) -> &mut [A::Item] { - self + fn extend>(&mut self, iterable: I) { + self.extend_impl(iterable.into_iter()); } } -#[cfg(feature = "write")] -#[cfg_attr(docsrs, doc(cfg(feature = "write")))] -impl> io::Write for SmallVec { - #[inline] - fn write(&mut self, buf: &[u8]) -> io::Result { - self.extend_from_slice(buf); - Ok(buf.len()) - } - - #[inline] - fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { - self.extend_from_slice(buf); - Ok(()) - } - - #[inline] - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } +struct DropDealloc { + ptr: *mut u8, + size_bytes: usize, + align: usize, } -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl Serialize for SmallVec -where - A::Item: Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - let mut state = serializer.serialize_seq(Some(self.len()))?; - for item in self { - state.serialize_element(&item)?; +impl Drop for DropDealloc { + #[inline] + fn drop(&mut self) { + unsafe { + if self.size_bytes > 0 { + alloc::alloc::dealloc( + self.ptr, + Layout::from_size_align_unchecked(self.size_bytes, self.align), + ); + } } - state.end() } } -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl<'de, A: Array> Deserialize<'de> for SmallVec -where - A::Item: Deserialize<'de>, -{ - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_seq(SmallVecVisitor { - phantom: PhantomData, - }) +impl Drop for SmallVec { + fn drop(&mut self) { + let on_heap = self.spilled(); + let len = self.len(); + let ptr = self.as_mut_ptr(); + unsafe { + let _drop_dealloc = if on_heap { + let capacity = self.capacity(); + Some(DropDealloc { + ptr: ptr as *mut u8, + size_bytes: capacity * size_of::(), + align: align_of::(), + }) + } else { + None + }; + core::ptr::slice_from_raw_parts_mut(ptr, len).drop_in_place(); + } } } -#[cfg(feature = "serde")] -struct SmallVecVisitor { - phantom: PhantomData, -} - -#[cfg(feature = "serde")] -impl<'de, A: Array> Visitor<'de> for SmallVecVisitor -where - A::Item: Deserialize<'de>, -{ - type Value = SmallVec; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a sequence") - } - - fn visit_seq(self, mut seq: B) -> Result - where - B: SeqAccess<'de>, - { - use serde::de::Error; - let len = seq.size_hint().unwrap_or(0); - let mut values = SmallVec::new(); - values.try_reserve(len).map_err(B::Error::custom)?; - - while let Some(value) = seq.next_element()? { - values.push(value); +impl Drop for IntoIter { + fn drop(&mut self) { + unsafe { + let is_zst = size_of::() == 0; + let on_heap = self.end.on_heap(is_zst); + let begin = self.begin; + let end = self.end.value(is_zst); + let ptr = self.as_mut_ptr(); + let _drop_dealloc = if on_heap { + let capacity = self.raw.heap.1; + Some(DropDealloc { + ptr: ptr as *mut u8, + size_bytes: capacity * size_of::(), + align: align_of::(), + }) + } else { + None + }; + core::ptr::slice_from_raw_parts_mut(ptr.add(begin), end - begin).drop_in_place(); } - - Ok(values) } } -#[cfg(feature = "specialization")] -trait SpecFrom { - fn spec_from(slice: S) -> SmallVec; -} +impl core::ops::Deref for SmallVec { + type Target = [T]; -#[cfg(feature = "specialization")] -mod specialization; - -#[cfg(feature = "arbitrary")] -mod arbitrary; - -#[cfg(feature = "specialization")] -impl<'a, A: Array> SpecFrom for SmallVec -where - A::Item: Copy, -{ #[inline] - fn spec_from(slice: &'a [A::Item]) -> SmallVec { - SmallVec::from_slice(slice) + fn deref(&self) -> &Self::Target { + self.as_slice() } } - -impl<'a, A: Array> From<&'a [A::Item]> for SmallVec -where - A::Item: Clone, -{ - #[cfg(not(feature = "specialization"))] +impl core::ops::DerefMut for SmallVec { #[inline] - fn from(slice: &'a [A::Item]) -> SmallVec { - slice.iter().cloned().collect() - } - - #[cfg(feature = "specialization")] - #[inline] - fn from(slice: &'a [A::Item]) -> SmallVec { - SmallVec::spec_from(slice) - } -} - -impl From> for SmallVec { - #[inline] - fn from(vec: Vec) -> SmallVec { - SmallVec::from_vec(vec) - } -} - -impl From for SmallVec { - #[inline] - fn from(array: A) -> SmallVec { - SmallVec::from_buf(array) - } -} - -impl> ops::Index for SmallVec { - type Output = I::Output; - - fn index(&self, index: I) -> &I::Output { - &(**self)[index] - } -} - -impl> ops::IndexMut for SmallVec { - fn index_mut(&mut self, index: I) -> &mut I::Output { - &mut (&mut **self)[index] - } -} - -#[allow(deprecated)] -impl ExtendFromSlice for SmallVec -where - A::Item: Copy, -{ - fn extend_from_slice(&mut self, other: &[A::Item]) { - SmallVec::extend_from_slice(self, other) + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_mut_slice() } } -impl FromIterator for SmallVec { +impl core::iter::FromIterator for SmallVec { #[inline] - fn from_iter>(iterable: I) -> SmallVec { - let mut v = SmallVec::new(); - v.extend(iterable); - v - } -} - -impl Extend for SmallVec { - fn extend>(&mut self, iterable: I) { - let mut iter = iterable.into_iter(); - let (lower_size_bound, _) = iter.size_hint(); - self.reserve(lower_size_bound); - - unsafe { - let (ptr, len_ptr, cap) = self.triple_mut(); - let mut len = SetLenOnDrop::new(len_ptr); - while len.get() < cap { - if let Some(out) = iter.next() { - ptr::write(ptr.add(len.get()), out); - len.increment_len(1); - } else { - return; - } - } - } - - for elem in iter { - self.push(elem); - } - } -} - -impl fmt::Debug for SmallVec -where - A::Item: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.iter()).finish() + fn from_iter>(iterable: I) -> Self { + let mut vec = Self::new(); + vec.extend_impl(iterable.into_iter()); + vec } } -impl Default for SmallVec { - #[inline] - fn default() -> SmallVec { - SmallVec::new() +impl<'a, T: Clone, const N: usize> From<&'a [T]> for SmallVec { + fn from(slice: &'a [T]) -> Self { + slice.iter().cloned().collect() } } - -#[cfg(feature = "may_dangle")] -unsafe impl<#[may_dangle] A: Array> Drop for SmallVec { - fn drop(&mut self) { - unsafe { - if self.spilled() { - let (ptr, len) = self.data.heap(); - Vec::from_raw_parts(ptr, len, self.capacity); - } else { - ptr::drop_in_place(&mut self[..]); - } - } +impl From<[T; N]> for SmallVec { + fn from(array: [T; N]) -> Self { + Self::from_buf(array) } } - -#[cfg(not(feature = "may_dangle"))] -impl Drop for SmallVec { - fn drop(&mut self) { - unsafe { - if self.spilled() { - let (ptr, len) = self.data.heap(); - Vec::from_raw_parts(ptr, len, self.capacity); - } else { - ptr::drop_in_place(&mut self[..]); - } - } +impl From> for SmallVec { + fn from(array: Vec) -> Self { + Self::from_vec(array) } } -impl Clone for SmallVec -where - A::Item: Clone, -{ +impl Clone for SmallVec { #[inline] - fn clone(&self) -> SmallVec { + fn clone(&self) -> SmallVec { SmallVec::from(self.as_slice()) } @@ -1837,7 +1353,8 @@ where // self.len <= other.len due to the truncate above, so the // slices here are always in-bounds. - let (init, tail) = source.split_at(self.len()); + let init = unsafe { source.get_unchecked(..self.len()) }; + let tail = unsafe { source.get_unchecked(self.len()..) }; // reuse the contained values' allocations/resources. self.clone_from_slice(init); @@ -1845,281 +1362,233 @@ where } } -impl PartialEq> for SmallVec -where - A::Item: PartialEq, -{ +impl Clone for IntoIter { #[inline] - fn eq(&self, other: &SmallVec) -> bool { - self[..] == other[..] + fn clone(&self) -> IntoIter { + SmallVec::from(self.as_slice()).into_iter() } } -impl Eq for SmallVec where A::Item: Eq {} +#[macro_export] +macro_rules! smallvec { + // count helper: transform any expression into 1 + (@one $x:expr) => (1usize); + ($elem:expr; $n:expr) => ({ + $crate::SmallVec::from_elem($elem, $n) + }); + ($($x:expr),*$(,)?) => ({ + let count = 0usize $(+ $crate::smallvec!(@one $x))*; + #[allow(unused_mut)] + let mut vec = $crate::SmallVec::new(); + if count <= vec.inline_size() { + $(vec.push($x);)* + vec + } else { + $crate::SmallVec::from_vec($crate::alloc::vec![$($x,)*]) + } + }); +} -impl PartialOrd for SmallVec -where - A::Item: PartialOrd, -{ - #[inline] - fn partial_cmp(&self, other: &SmallVec) -> Option { - PartialOrd::partial_cmp(&**self, &**other) - } +#[macro_export] +macro_rules! smallvec_inline { + // count helper: transform any expression into 1 + (@one $x:expr) => (1usize); + ($elem:expr; $n:expr) => ({ + $crate::SmallVec::<_, $n>::from_buf([$elem; $n]) + }); + ($($x:expr),+ $(,)?) => ({ + const N: usize = 0usize $(+ $crate::smallvec_inline!(@one $x))*; + $crate::SmallVec::<_, N>::from_buf([$($x,)*]) + }); } -impl Ord for SmallVec -where - A::Item: Ord, -{ - #[inline] - fn cmp(&self, other: &SmallVec) -> cmp::Ordering { - Ord::cmp(&**self, &**other) +impl IntoIterator for SmallVec { + type IntoIter = IntoIter; + type Item = T; + fn into_iter(self) -> Self::IntoIter { + unsafe { + // Set SmallVec len to zero as `IntoIter` drop handles dropping of the elements + let this = ManuallyDrop::new(self); + IntoIter { + raw: (&this.raw as *const RawSmallVec).read(), + begin: 0, + end: this.len, + } + } } } -impl Hash for SmallVec -where - A::Item: Hash, -{ - fn hash(&self, state: &mut H) { - (**self).hash(state) +impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { + type IntoIter = core::slice::Iter<'a, T>; + type Item = &'a T; + fn into_iter(self) -> Self::IntoIter { + self.iter() } } -unsafe impl Send for SmallVec where A::Item: Send {} - -/// An iterator that consumes a `SmallVec` and yields its items by value. -/// -/// Returned from [`SmallVec::into_iter`][1]. -/// -/// [1]: struct.SmallVec.html#method.into_iter -pub struct IntoIter { - data: SmallVec, - current: usize, - end: usize, +impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { + type IntoIter = core::slice::IterMut<'a, T>; + type Item = &'a mut T; + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } } -impl fmt::Debug for IntoIter +impl PartialEq> for SmallVec where - A::Item: fmt::Debug, + T: PartialEq, { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IntoIter").field(&self.as_slice()).finish() + #[inline] + fn eq(&self, other: &SmallVec) -> bool { + self.as_slice().eq(other.as_slice()) } } +impl Eq for SmallVec where T: Eq {} -impl Clone for IntoIter +impl PartialOrd for SmallVec where - A::Item: Clone, + T: PartialOrd, { - fn clone(&self) -> IntoIter { - SmallVec::from(self.as_slice()).into_iter() + #[inline] + fn partial_cmp(&self, other: &SmallVec) -> Option { + self.as_slice().partial_cmp(other.as_slice()) } } -impl Drop for IntoIter { - fn drop(&mut self) { - for _ in self {} +impl Ord for SmallVec +where + T: Ord, +{ + #[inline] + fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { + self.as_slice().cmp(other.as_slice()) } } -impl Iterator for IntoIter { - type Item = A::Item; - +impl Borrow<[T]> for SmallVec { #[inline] - fn next(&mut self) -> Option { - if self.current == self.end { - None - } else { - unsafe { - let current = self.current; - self.current += 1; - Some(ptr::read(self.data.as_ptr().add(current))) - } - } + fn borrow(&self) -> &[T] { + self.as_slice() } +} +impl BorrowMut<[T]> for SmallVec { #[inline] - fn size_hint(&self) -> (usize, Option) { - let size = self.end - self.current; - (size, Some(size)) + fn borrow_mut(&mut self) -> &mut [T] { + self.as_mut_slice() } } -impl DoubleEndedIterator for IntoIter { +impl AsRef<[T]> for SmallVec { #[inline] - fn next_back(&mut self) -> Option { - if self.current == self.end { - None - } else { - unsafe { - self.end -= 1; - Some(ptr::read(self.data.as_ptr().add(self.end))) - } - } + fn as_ref(&self) -> &[T] { + self.as_slice() } } -impl ExactSizeIterator for IntoIter {} -impl FusedIterator for IntoIter {} - -impl IntoIter { - /// Returns the remaining items of this iterator as a slice. - pub fn as_slice(&self) -> &[A::Item] { - let len = self.end - self.current; - unsafe { core::slice::from_raw_parts(self.data.as_ptr().add(self.current), len) } +impl AsMut<[T]> for SmallVec { + #[inline] + fn as_mut(&mut self) -> &mut [T] { + self.as_mut_slice() } +} - /// Returns the remaining items of this iterator as a mutable slice. - pub fn as_mut_slice(&mut self) -> &mut [A::Item] { - let len = self.end - self.current; - unsafe { core::slice::from_raw_parts_mut(self.data.as_mut_ptr().add(self.current), len) } +impl Debug for SmallVec { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_list().entries(self.iter()).finish() } } -impl IntoIterator for SmallVec { - type IntoIter = IntoIter; - type Item = A::Item; - fn into_iter(mut self) -> Self::IntoIter { - unsafe { - // Set SmallVec len to zero as `IntoIter` drop handles dropping of the elements - let len = self.len(); - self.set_len(0); - IntoIter { - data: self, - current: 0, - end: len, - } - } +impl Debug for IntoIter { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_tuple("IntoIter").field(&self.as_slice()).finish() } } -impl<'a, A: Array> IntoIterator for &'a SmallVec { - type IntoIter = slice::Iter<'a, A::Item>; - type Item = &'a A::Item; - fn into_iter(self) -> Self::IntoIter { - self.iter() +impl<'a, T: Debug, const N: usize> Debug for Drain<'a, T, N> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() } } -impl<'a, A: Array> IntoIterator for &'a mut SmallVec { - type IntoIter = slice::IterMut<'a, A::Item>; - type Item = &'a mut A::Item; - fn into_iter(self) -> Self::IntoIter { - self.iter_mut() +#[cfg(feature = "serde")] +#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] +impl Serialize for SmallVec +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result { + let mut state = serializer.serialize_seq(Some(self.len()))?; + for item in self { + state.serialize_element(item)?; + } + state.end() } } -/// Types that can be used as the backing store for a SmallVec -pub unsafe trait Array { - /// The type of the array's elements. - type Item; - /// Returns the number of items the array can hold. - fn size() -> usize; +#[cfg(feature = "serde")] +#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] +impl<'de, T, const N: usize> Deserialize<'de> for SmallVec +where + T: Deserialize<'de>, +{ + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_seq(SmallVecVisitor { + phantom: PhantomData, + }) + } } -/// Set the length of the vec when the `SetLenOnDrop` value goes out of scope. -/// -/// Copied from https://github.com/rust-lang/rust/pull/36355 -struct SetLenOnDrop<'a> { - len: &'a mut usize, - local_len: usize, +#[cfg(feature = "serde")] +struct SmallVecVisitor { + phantom: PhantomData, } -impl<'a> SetLenOnDrop<'a> { - #[inline] - fn new(len: &'a mut usize) -> Self { - SetLenOnDrop { - local_len: *len, - len, - } - } - - #[inline] - fn get(&self) -> usize { - self.local_len - } +#[cfg(feature = "serde")] +impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor +where + T: Deserialize<'de>, +{ + type Value = SmallVec; - #[inline] - fn increment_len(&mut self, increment: usize) { - self.local_len += increment; + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter.write_str("a sequence") } -} -impl<'a> Drop for SetLenOnDrop<'a> { - #[inline] - fn drop(&mut self) { - *self.len = self.local_len; - } -} + fn visit_seq(self, mut seq: B) -> Result + where + B: SeqAccess<'de>, + { + use serde::de::Error; + let len = seq.size_hint().unwrap_or(0); + let mut values = SmallVec::new(); + values.try_reserve(len).map_err(B::Error::custom)?; -#[cfg(feature = "const_new")] -impl SmallVec<[T; N]> { - /// Construct an empty vector. - /// - /// This is a `const` version of [`SmallVec::new`] that is enabled by the feature `const_new`, with the limitation that it only works for arrays. - #[cfg_attr(docsrs, doc(cfg(feature = "const_new")))] - #[inline] - pub const fn new_const() -> Self { - SmallVec { - capacity: 0, - data: SmallVecData::from_const(MaybeUninit::uninit()), + while let Some(value) = seq.next_element()? { + values.push(value); } - } - /// The array passed as an argument is moved to be an inline version of `SmallVec`. - /// - /// This is a `const` version of [`SmallVec::from_buf`] that is enabled by the feature `const_new`, with the limitation that it only works for arrays. - #[cfg_attr(docsrs, doc(cfg(feature = "const_new")))] - #[inline] - pub const fn from_const(items: [T; N]) -> Self { - SmallVec { - capacity: N, - data: SmallVecData::from_const(MaybeUninit::new(items)), - } + Ok(values) } } -#[cfg(all(feature = "const_generics", not(doc)))] -#[cfg_attr(docsrs, doc(cfg(feature = "const_generics")))] -unsafe impl Array for [T; N] { - type Item = T; - fn size() -> usize { - N +#[cfg(feature = "write")] +#[cfg_attr(docsrs, doc(cfg(feature = "write")))] +impl io::Write for SmallVec { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + self.extend_from_slice(buf); + Ok(buf.len()) } -} -#[cfg(any(not(feature = "const_generics"), doc))] -macro_rules! impl_array( - ($($size:expr),+) => { - $( - unsafe impl Array for [T; $size] { - type Item = T; - fn size() -> usize { $size } - } - )+ + #[inline] + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.extend_from_slice(buf); + Ok(()) } -); - -#[cfg(any(not(feature = "const_generics"), doc))] -impl_array!( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 36, 0x40, 0x60, 0x80, 0x100, 0x200, 0x400, 0x600, 0x800, 0x1000, - 0x2000, 0x4000, 0x6000, 0x8000, 0x10000, 0x20000, 0x40000, 0x60000, 0x80000, 0x10_0000 -); -/// Convenience trait for constructing a `SmallVec` -pub trait ToSmallVec { - /// Construct a new `SmallVec` from a slice. - fn to_smallvec(&self) -> SmallVec; -} - -impl ToSmallVec for [A::Item] -where - A::Item: Copy, -{ #[inline] - fn to_smallvec(&self) -> SmallVec { - SmallVec::from_slice(self) + fn flush(&mut self) -> io::Result<()> { + Ok(()) } } diff --git a/src/tests.rs b/src/tests.rs index 7643fd7..d233701 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -9,7 +9,7 @@ use alloc::{vec, vec::Vec}; #[test] pub fn test_zero() { - let mut v = SmallVec::<[_; 0]>::new(); + let mut v = SmallVec::<_, 0>::new(); assert!(!v.spilled()); v.push(0usize); assert!(v.spilled()); @@ -20,7 +20,7 @@ pub fn test_zero() { #[test] pub fn test_inline() { - let mut v = SmallVec::<[_; 16]>::new(); + let mut v = SmallVec::<_, 16>::new(); v.push("hello".to_owned()); v.push("there".to_owned()); assert_eq!(&*v, &["hello".to_owned(), "there".to_owned(),][..]); @@ -28,7 +28,7 @@ pub fn test_inline() { #[test] pub fn test_spill() { - let mut v = SmallVec::<[_; 2]>::new(); + let mut v = SmallVec::<_, 2>::new(); v.push("hello".to_owned()); assert_eq!(v[0], "hello"); v.push("there".to_owned()); @@ -48,7 +48,7 @@ pub fn test_spill() { #[test] pub fn test_double_spill() { - let mut v = SmallVec::<[_; 2]>::new(); + let mut v = SmallVec::<_, 2>::new(); v.push("hello".to_owned()); v.push("there".to_owned()); v.push("burma".to_owned()); @@ -75,23 +75,23 @@ pub fn test_double_spill() { /// https://github.com/servo/rust-smallvec/issues/4 #[test] fn issue_4() { - SmallVec::<[Box; 2]>::new(); + SmallVec::, 2>::new(); } /// https://github.com/servo/rust-smallvec/issues/5 #[test] fn issue_5() { - assert!(Some(SmallVec::<[&u32; 2]>::new()).is_some()); + assert!(Some(SmallVec::<&u32, 2>::new()).is_some()); } #[test] fn test_with_capacity() { - let v: SmallVec<[u8; 3]> = SmallVec::with_capacity(1); + let v: SmallVec = SmallVec::with_capacity(1); assert!(v.is_empty()); assert!(!v.spilled()); assert_eq!(v.capacity(), 3); - let v: SmallVec<[u8; 3]> = SmallVec::with_capacity(10); + let v: SmallVec = SmallVec::with_capacity(10); assert!(v.is_empty()); assert!(v.spilled()); assert_eq!(v.capacity(), 10); @@ -99,7 +99,7 @@ fn test_with_capacity() { #[test] fn drain() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.drain(..).collect::>(), &[3]); @@ -114,7 +114,7 @@ fn drain() { // Exercise the tail-shifting code when in the inline state // This has the potential to produce UB due to aliasing - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(1); v.push(2); assert_eq!(v.drain(..1).collect::>(), &[1]); @@ -122,7 +122,7 @@ fn drain() { #[test] fn drain_rev() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.drain(..).rev().collect::>(), &[3]); @@ -135,19 +135,19 @@ fn drain_rev() { #[test] fn drain_forget() { - let mut v: SmallVec<[u8; 1]> = smallvec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6, 7]; std::mem::forget(v.drain(2..5)); assert_eq!(v.len(), 2); } #[test] fn into_iter() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.into_iter().collect::>(), &[3]); // spilling the vec - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(3); v.push(4); v.push(5); @@ -156,12 +156,12 @@ fn into_iter() { #[test] fn into_iter_rev() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.into_iter().rev().collect::>(), &[3]); // spilling the vec - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(3); v.push(4); v.push(5); @@ -182,7 +182,7 @@ fn into_iter_drop() { { let cell = Cell::new(0); - let mut v: SmallVec<[DropCounter<'_>; 2]> = SmallVec::new(); + let mut v: SmallVec, 2> = SmallVec::new(); v.push(DropCounter(&cell)); v.into_iter(); assert_eq!(cell.get(), 1); @@ -190,7 +190,7 @@ fn into_iter_drop() { { let cell = Cell::new(0); - let mut v: SmallVec<[DropCounter<'_>; 2]> = SmallVec::new(); + let mut v: SmallVec, 2> = SmallVec::new(); v.push(DropCounter(&cell)); v.push(DropCounter(&cell)); assert!(v.into_iter().next().is_some()); @@ -199,7 +199,7 @@ fn into_iter_drop() { { let cell = Cell::new(0); - let mut v: SmallVec<[DropCounter<'_>; 2]> = SmallVec::new(); + let mut v: SmallVec, 2> = SmallVec::new(); v.push(DropCounter(&cell)); v.push(DropCounter(&cell)); v.push(DropCounter(&cell)); @@ -208,7 +208,7 @@ fn into_iter_drop() { } { let cell = Cell::new(0); - let mut v: SmallVec<[DropCounter<'_>; 2]> = SmallVec::new(); + let mut v: SmallVec, 2> = SmallVec::new(); v.push(DropCounter(&cell)); v.push(DropCounter(&cell)); v.push(DropCounter(&cell)); @@ -223,7 +223,7 @@ fn into_iter_drop() { #[test] fn test_capacity() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.reserve(1); assert_eq!(v.capacity(), 2); assert!(!v.spilled()); @@ -242,7 +242,7 @@ fn test_capacity() { #[test] fn test_truncate() { - let mut v: SmallVec<[Box; 8]> = SmallVec::new(); + let mut v: SmallVec, 8> = SmallVec::new(); for x in 0..8 { v.push(Box::new(x)); @@ -261,7 +261,7 @@ fn test_truncate() { #[test] fn test_insert_many() { - let mut v: SmallVec<[u8; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); for x in 0..4 { v.push(x); } @@ -289,7 +289,7 @@ impl Iterator for MockHintIter { #[test] fn test_insert_many_short_hint() { - let mut v: SmallVec<[u8; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); for x in 0..4 { v.push(x); } @@ -309,7 +309,7 @@ fn test_insert_many_short_hint() { #[test] fn test_insert_many_long_hint() { - let mut v: SmallVec<[u8; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); for x in 0..4 { v.push(x); } @@ -373,7 +373,7 @@ mod insert_many_panic { #[test] fn panic_early_at_start() { - let mut vec: SmallVec<[PanicOnDoubleDrop; 0]> = + let mut vec: SmallVec = smallvec![PanicOnDoubleDrop::new(), PanicOnDoubleDrop::new(),]; let result = ::std::panic::catch_unwind(move || { vec.insert_many(0, BadIter { hint: 1, count: 0 }); @@ -383,7 +383,7 @@ mod insert_many_panic { #[test] fn panic_early_in_middle() { - let mut vec: SmallVec<[PanicOnDoubleDrop; 0]> = + let mut vec: SmallVec = smallvec![PanicOnDoubleDrop::new(), PanicOnDoubleDrop::new(),]; let result = ::std::panic::catch_unwind(move || { vec.insert_many(1, BadIter { hint: 4, count: 2 }); @@ -393,7 +393,7 @@ mod insert_many_panic { #[test] fn panic_early_at_end() { - let mut vec: SmallVec<[PanicOnDoubleDrop; 0]> = + let mut vec: SmallVec = smallvec![PanicOnDoubleDrop::new(), PanicOnDoubleDrop::new(),]; let result = ::std::panic::catch_unwind(move || { vec.insert_many(2, BadIter { hint: 3, count: 1 }); @@ -403,7 +403,7 @@ mod insert_many_panic { #[test] fn panic_late_at_start() { - let mut vec: SmallVec<[PanicOnDoubleDrop; 0]> = + let mut vec: SmallVec = smallvec![PanicOnDoubleDrop::new(), PanicOnDoubleDrop::new(),]; let result = ::std::panic::catch_unwind(move || { vec.insert_many(0, BadIter { hint: 3, count: 5 }); @@ -413,7 +413,7 @@ mod insert_many_panic { #[test] fn panic_late_at_end() { - let mut vec: SmallVec<[PanicOnDoubleDrop; 0]> = + let mut vec: SmallVec = smallvec![PanicOnDoubleDrop::new(), PanicOnDoubleDrop::new(),]; let result = ::std::panic::catch_unwind(move || { vec.insert_many(2, BadIter { hint: 3, count: 5 }); @@ -425,7 +425,7 @@ mod insert_many_panic { #[test] #[should_panic] fn test_invalid_grow() { - let mut v: SmallVec<[u8; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.extend(0..8); v.grow(5); } @@ -433,13 +433,13 @@ fn test_invalid_grow() { #[test] #[should_panic] fn drain_overflow() { - let mut v: SmallVec<[u8; 8]> = smallvec![0]; + let mut v: SmallVec = smallvec![0]; v.drain(..=std::usize::MAX); } #[test] fn test_insert_from_slice() { - let mut v: SmallVec<[u8; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); for x in 0..4 { v.push(x); } @@ -453,7 +453,7 @@ fn test_insert_from_slice() { #[test] fn test_extend_from_slice() { - let mut v: SmallVec<[u8; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); for x in 0..4 { v.push(x); } @@ -478,15 +478,15 @@ fn test_drop_panic_smallvec() { } } - let mut v = SmallVec::<[_; 1]>::new(); + let mut v = SmallVec::<_, 1>::new(); v.push(DropPanic); } #[test] fn test_eq() { - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); - let mut b: SmallVec<[u32; 2]> = SmallVec::new(); - let mut c: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); + let mut b: SmallVec = SmallVec::new(); + let mut c: SmallVec = SmallVec::new(); // a = [1, 2] a.push(1); a.push(2); @@ -503,9 +503,9 @@ fn test_eq() { #[test] fn test_ord() { - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); - let mut b: SmallVec<[u32; 2]> = SmallVec::new(); - let mut c: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); + let mut b: SmallVec = SmallVec::new(); + let mut c: SmallVec = SmallVec::new(); // a = [1] a.push(1); // b = [1, 1] @@ -527,14 +527,14 @@ fn test_hash() { use std::hash::Hash; { - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); let b = [1, 2]; a.extend(b.iter().cloned()); let mut hasher = DefaultHasher::new(); assert_eq!(a.hash(&mut hasher), b.hash(&mut hasher)); } { - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); let b = [1, 2, 11, 12]; a.extend(b.iter().cloned()); let mut hasher = DefaultHasher::new(); @@ -544,7 +544,7 @@ fn test_hash() { #[test] fn test_as_ref() { - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); a.push(1); assert_eq!(a.as_ref(), [1]); a.push(2); @@ -555,7 +555,7 @@ fn test_as_ref() { #[test] fn test_as_mut() { - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); a.push(1); assert_eq!(a.as_mut(), [1]); a.push(2); @@ -570,7 +570,7 @@ fn test_as_mut() { fn test_borrow() { use std::borrow::Borrow; - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); a.push(1); assert_eq!(a.borrow(), [1]); a.push(2); @@ -583,7 +583,7 @@ fn test_borrow() { fn test_borrow_mut() { use std::borrow::BorrowMut; - let mut a: SmallVec<[u32; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); a.push(1); assert_eq!(a.borrow_mut(), [1]); a.push(2); @@ -596,47 +596,47 @@ fn test_borrow_mut() { #[test] fn test_from() { - assert_eq!(&SmallVec::<[u32; 2]>::from(&[1][..])[..], [1]); - assert_eq!(&SmallVec::<[u32; 2]>::from(&[1, 2, 3][..])[..], [1, 2, 3]); + assert_eq!(&SmallVec::::from(&[1][..])[..], [1]); + assert_eq!(&SmallVec::::from(&[1, 2, 3][..])[..], [1, 2, 3]); let vec = vec![]; - let small_vec: SmallVec<[u8; 3]> = SmallVec::from(vec); + let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[]); drop(small_vec); let vec = vec![1, 2, 3, 4, 5]; - let small_vec: SmallVec<[u8; 3]> = SmallVec::from(vec); + let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); let vec = vec![1, 2, 3, 4, 5]; - let small_vec: SmallVec<[u8; 1]> = SmallVec::from(vec); + let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); let array = [1]; - let small_vec: SmallVec<[u8; 1]> = SmallVec::from(array); + let small_vec: SmallVec = SmallVec::from(array); assert_eq!(&*small_vec, &[1]); drop(small_vec); let array = [99; 128]; - let small_vec: SmallVec<[u8; 128]> = SmallVec::from(array); + let small_vec: SmallVec = SmallVec::from(array); assert_eq!(&*small_vec, vec![99u8; 128].as_slice()); drop(small_vec); } #[test] fn test_from_slice() { - assert_eq!(&SmallVec::<[u32; 2]>::from_slice(&[1][..])[..], [1]); + assert_eq!(&SmallVec::::from_slice(&[1][..])[..], [1]); assert_eq!( - &SmallVec::<[u32; 2]>::from_slice(&[1, 2, 3][..])[..], + &SmallVec::::from_slice(&[1, 2, 3][..])[..], [1, 2, 3] ); } #[test] fn test_exact_size_iterator() { - let mut vec = SmallVec::<[u32; 2]>::from(&[1, 2, 3][..]); + let mut vec = SmallVec::::from(&[1, 2, 3][..]); assert_eq!(vec.clone().into_iter().len(), 3); assert_eq!(vec.drain(..2).len(), 2); assert_eq!(vec.into_iter().len(), 1); @@ -644,7 +644,7 @@ fn test_exact_size_iterator() { #[test] fn test_into_iter_as_slice() { - let vec = SmallVec::<[u32; 2]>::from(&[1, 2, 3][..]); + let vec = SmallVec::::from(&[1, 2, 3][..]); let mut iter = vec.clone().into_iter(); assert_eq!(iter.as_slice(), &[1, 2, 3]); assert_eq!(iter.as_mut_slice(), &[1, 2, 3]); @@ -660,7 +660,7 @@ fn test_into_iter_as_slice() { fn test_into_iter_clone() { // Test that the cloned iterator yields identical elements and that it owns its own copy // (i.e. no use after move errors). - let mut iter = SmallVec::<[u8; 2]>::from_iter(0..3).into_iter(); + let mut iter = SmallVec::::from_iter(0..3).into_iter(); let mut clone_iter = iter.clone(); while let Some(x) = iter.next() { assert_eq!(x, clone_iter.next().unwrap()); @@ -671,7 +671,7 @@ fn test_into_iter_clone() { #[test] fn test_into_iter_clone_partially_consumed_iterator() { // Test that the cloned iterator only contains the remaining elements of the original iterator. - let mut iter = SmallVec::<[u8; 2]>::from_iter(0..3).into_iter().skip(1); + let mut iter = SmallVec::::from_iter(0..3).into_iter().skip(1); let mut clone_iter = iter.clone(); while let Some(x) = iter.next() { assert_eq!(x, clone_iter.next().unwrap()); @@ -681,7 +681,7 @@ fn test_into_iter_clone_partially_consumed_iterator() { #[test] fn test_into_iter_clone_empty_smallvec() { - let mut iter = SmallVec::<[u8; 2]>::new().into_iter(); + let mut iter = SmallVec::::new().into_iter(); let mut clone_iter = iter.clone(); assert_eq!(iter.next(), None); assert_eq!(clone_iter.next(), None); @@ -689,7 +689,7 @@ fn test_into_iter_clone_empty_smallvec() { #[test] fn shrink_to_fit_unspill() { - let mut vec = SmallVec::<[u8; 2]>::from_iter(0..3); + let mut vec = SmallVec::::from_iter(0..3); vec.pop(); assert!(vec.spilled()); vec.shrink_to_fit(); @@ -698,54 +698,54 @@ fn shrink_to_fit_unspill() { #[test] fn test_into_vec() { - let vec = SmallVec::<[u8; 2]>::from_iter(0..2); + let vec = SmallVec::::from_iter(0..2); assert_eq!(vec.into_vec(), vec![0, 1]); - let vec = SmallVec::<[u8; 2]>::from_iter(0..3); + let vec = SmallVec::::from_iter(0..3); assert_eq!(vec.into_vec(), vec![0, 1, 2]); } #[test] fn test_into_inner() { - let vec = SmallVec::<[u8; 2]>::from_iter(0..2); + let vec = SmallVec::::from_iter(0..2); assert_eq!(vec.into_inner(), Ok([0, 1])); - let vec = SmallVec::<[u8; 2]>::from_iter(0..1); + let vec = SmallVec::::from_iter(0..1); assert_eq!(vec.clone().into_inner(), Err(vec)); - let vec = SmallVec::<[u8; 2]>::from_iter(0..3); + let vec = SmallVec::::from_iter(0..3); assert_eq!(vec.clone().into_inner(), Err(vec)); } #[test] fn test_from_vec() { let vec = vec![]; - let small_vec: SmallVec<[u8; 3]> = SmallVec::from_vec(vec); + let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[]); drop(small_vec); let vec = vec![]; - let small_vec: SmallVec<[u8; 1]> = SmallVec::from_vec(vec); + let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[]); drop(small_vec); let vec = vec![1]; - let small_vec: SmallVec<[u8; 3]> = SmallVec::from_vec(vec); + let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1]); drop(small_vec); let vec = vec![1, 2, 3]; - let small_vec: SmallVec<[u8; 3]> = SmallVec::from_vec(vec); + let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3]); drop(small_vec); let vec = vec![1, 2, 3, 4, 5]; - let small_vec: SmallVec<[u8; 3]> = SmallVec::from_vec(vec); + let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); let vec = vec![1, 2, 3, 4, 5]; - let small_vec: SmallVec<[u8; 1]> = SmallVec::from_vec(vec); + let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); } @@ -753,7 +753,7 @@ fn test_from_vec() { #[test] fn test_retain() { // Test inline data storate - let mut sv: SmallVec<[i32; 5]> = SmallVec::from_slice(&[1, 2, 3, 3, 4]); + let mut sv: SmallVec = SmallVec::from_slice(&[1, 2, 3, 3, 4]); sv.retain(|&mut i| i != 3); assert_eq!(sv.pop(), Some(4)); assert_eq!(sv.pop(), Some(2)); @@ -761,7 +761,7 @@ fn test_retain() { assert_eq!(sv.pop(), None); // Test spilled data storage - let mut sv: SmallVec<[i32; 3]> = SmallVec::from_slice(&[1, 2, 3, 3, 4]); + let mut sv: SmallVec = SmallVec::from_slice(&[1, 2, 3, 3, 4]); sv.retain(|&mut i| i != 3); assert_eq!(sv.pop(), Some(4)); assert_eq!(sv.pop(), Some(2)); @@ -770,14 +770,14 @@ fn test_retain() { // Test that drop implementations are called for inline. let one = Rc::new(1); - let mut sv: SmallVec<[Rc; 3]> = SmallVec::new(); + let mut sv: SmallVec, 3> = SmallVec::new(); sv.push(Rc::clone(&one)); assert_eq!(Rc::strong_count(&one), 2); sv.retain(|_| false); assert_eq!(Rc::strong_count(&one), 1); // Test that drop implementations are called for spilled data. - let mut sv: SmallVec<[Rc; 1]> = SmallVec::new(); + let mut sv: SmallVec, 1> = SmallVec::new(); sv.push(Rc::clone(&one)); sv.push(Rc::new(2)); assert_eq!(Rc::strong_count(&one), 2); @@ -787,26 +787,26 @@ fn test_retain() { #[test] fn test_dedup() { - let mut dupes: SmallVec<[i32; 5]> = SmallVec::from_slice(&[1, 1, 2, 3, 3]); + let mut dupes: SmallVec = SmallVec::from_slice(&[1, 1, 2, 3, 3]); dupes.dedup(); assert_eq!(&*dupes, &[1, 2, 3]); - let mut empty: SmallVec<[i32; 5]> = SmallVec::new(); + let mut empty: SmallVec = SmallVec::new(); empty.dedup(); assert!(empty.is_empty()); - let mut all_ones: SmallVec<[i32; 5]> = SmallVec::from_slice(&[1, 1, 1, 1, 1]); + let mut all_ones: SmallVec = SmallVec::from_slice(&[1, 1, 1, 1, 1]); all_ones.dedup(); assert_eq!(all_ones.len(), 1); - let mut no_dupes: SmallVec<[i32; 5]> = SmallVec::from_slice(&[1, 2, 3, 4, 5]); + let mut no_dupes: SmallVec = SmallVec::from_slice(&[1, 2, 3, 4, 5]); no_dupes.dedup(); assert_eq!(no_dupes.len(), 5); } #[test] fn test_resize() { - let mut v: SmallVec<[i32; 8]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(1); v.resize(5, 0); assert_eq!(v[..], [1, 0, 0, 0, 0][..]); @@ -822,12 +822,12 @@ fn test_write() { let data = [1, 2, 3, 4, 5]; - let mut small_vec: SmallVec<[u8; 2]> = SmallVec::new(); + let mut small_vec: SmallVec = SmallVec::new(); let len = small_vec.write(&data[..]).unwrap(); assert_eq!(len, 5); assert_eq!(small_vec.as_ref(), data.as_ref()); - let mut small_vec: SmallVec<[u8; 2]> = SmallVec::new(); + let mut small_vec: SmallVec = SmallVec::new(); small_vec.write_all(&data[..]).unwrap(); assert_eq!(small_vec.as_ref(), data.as_ref()); } @@ -839,10 +839,10 @@ extern crate bincode; #[test] fn test_serde() { use self::bincode::{config, deserialize}; - let mut small_vec: SmallVec<[i32; 2]> = SmallVec::new(); + let mut small_vec: SmallVec = SmallVec::new(); small_vec.push(1); let encoded = config().limit(100).serialize(&small_vec).unwrap(); - let decoded: SmallVec<[i32; 2]> = deserialize(&encoded).unwrap(); + let decoded: SmallVec = deserialize(&encoded).unwrap(); assert_eq!(small_vec, decoded); small_vec.push(2); // Spill the vec @@ -850,13 +850,13 @@ fn test_serde() { small_vec.push(4); // Check again after spilling. let encoded = config().limit(100).serialize(&small_vec).unwrap(); - let decoded: SmallVec<[i32; 2]> = deserialize(&encoded).unwrap(); + let decoded: SmallVec = deserialize(&encoded).unwrap(); assert_eq!(small_vec, decoded); } #[test] fn grow_to_shrink() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(1); v.push(2); v.push(3); @@ -878,7 +878,7 @@ fn resumable_extend() { let it = s .chars() .scan(0, |_, ch| if ch.is_whitespace() { None } else { Some(ch) }); - let mut v: SmallVec<[char; 4]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.extend(it); assert_eq!(v[..], ['a']); } @@ -887,12 +887,12 @@ fn resumable_extend() { #[test] fn uninhabited() { enum Void {} - let _sv = SmallVec::<[Void; 8]>::new(); + let _sv = SmallVec::::new(); } #[test] fn grow_spilled_same_size() { - let mut v: SmallVec<[u8; 2]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(0); v.push(1); v.push(2); @@ -904,13 +904,11 @@ fn grow_spilled_same_size() { assert_eq!(v[..], [0, 1, 2]); } -#[cfg(feature = "const_generics")] #[test] fn const_generics() { - let _v = SmallVec::<[i32; 987]>::default(); + let _v = SmallVec::::default(); } -#[cfg(feature = "const_new")] #[test] fn const_new() { let v = const_new_inner(); @@ -926,32 +924,29 @@ fn const_new() { assert_eq!(v[0], 1); assert_eq!(v[1], 4); } -#[cfg(feature = "const_new")] -const fn const_new_inner() -> SmallVec<[i32; 4]> { - SmallVec::<[i32; 4]>::new_const() +const fn const_new_inner() -> SmallVec { + SmallVec::::new() } -#[cfg(feature = "const_new")] -const fn const_new_inline_sized() -> SmallVec<[i32; 4]> { +const fn const_new_inline_sized() -> SmallVec { crate::smallvec_inline![1; 4] } -#[cfg(feature = "const_new")] -const fn const_new_inline_args() -> SmallVec<[i32; 2]> { +const fn const_new_inline_args() -> SmallVec { crate::smallvec_inline![1, 4] } #[test] fn empty_macro() { - let _v: SmallVec<[u8; 1]> = smallvec![]; + let _v: SmallVec = smallvec![]; } #[test] fn zero_size_items() { - SmallVec::<[(); 0]>::new().push(()); + SmallVec::<(), 0>::new().push(()); } #[test] fn test_insert_many_overflow() { - let mut v: SmallVec<[u8; 1]> = SmallVec::new(); + let mut v: SmallVec = SmallVec::new(); v.push(123); // Prepare an iterator with small lower bound @@ -964,15 +959,15 @@ fn test_insert_many_overflow() { #[test] fn test_clone_from() { - let mut a: SmallVec<[u8; 2]> = SmallVec::new(); + let mut a: SmallVec = SmallVec::new(); a.push(1); a.push(2); a.push(3); - let mut b: SmallVec<[u8; 2]> = SmallVec::new(); + let mut b: SmallVec = SmallVec::new(); b.push(10); - let mut c: SmallVec<[u8; 2]> = SmallVec::new(); + let mut c: SmallVec = SmallVec::new(); c.push(20); c.push(21); c.push(22); diff --git a/tests/macro.rs b/tests/macro.rs index fa52e79..a5d3a71 100644 --- a/tests/macro.rs +++ b/tests/macro.rs @@ -3,7 +3,7 @@ #[test] fn smallvec() { - let mut vec: smallvec::SmallVec<[i32; 2]>; + let mut vec: smallvec::SmallVec; macro_rules! check { ($init:tt) => { From c182d102ffd8e4216d98b247fea81ab7d4535af1 Mon Sep 17 00:00:00 2001 From: sarah el kazdadi Date: Mon, 1 Aug 2022 17:48:07 +0200 Subject: [PATCH 2/7] cleanup --- Cargo.toml | 4 ---- src/arbitrary.rs | 19 ------------------- src/lib.rs | 17 ++++++----------- src/specialization.rs | 19 ------------------- 4 files changed, 6 insertions(+), 53 deletions(-) delete mode 100644 src/arbitrary.rs delete mode 100644 src/specialization.rs diff --git a/Cargo.toml b/Cargo.toml index a8a0c8d..745e986 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,16 +12,12 @@ readme = "README.md" documentation = "https://docs.rs/smallvec/" [features] -const_generics = [] -const_new = ["const_generics"] write = [] -union = [] specialization = [] may_dangle = [] [dependencies] serde = { version = "1", optional = true, default-features = false } -arbitrary = { version = "1", optional = true } [dev_dependencies] bincode = "1.0.1" diff --git a/src/arbitrary.rs b/src/arbitrary.rs deleted file mode 100644 index cbdfcb0..0000000 --- a/src/arbitrary.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::{Array, SmallVec}; -use arbitrary::{Arbitrary, Unstructured}; - -impl<'a, A: Array> Arbitrary<'a> for SmallVec -where - ::Item: Arbitrary<'a>, -{ - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - u.arbitrary_iter()?.collect() - } - - fn arbitrary_take_rest(u: Unstructured<'a>) -> arbitrary::Result { - u.arbitrary_take_rest_iter()?.collect() - } - - fn size_hint(depth: usize) -> (usize, Option) { - arbitrary::size_hint::and(::size_hint(depth), (0, None)) - } -} diff --git a/src/lib.rs b/src/lib.rs index 7b293e3..44bbb8d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -565,7 +565,7 @@ impl SmallVec { } #[inline] - const fn inline_capacity() -> usize { + pub const fn inline_size() -> usize { if Self::is_zst() { usize::MAX } else { @@ -573,11 +573,6 @@ impl SmallVec { } } - #[inline] - pub const fn inline_size(&self) -> usize { - Self::inline_capacity() - } - #[inline] pub const fn len(&self) -> usize { self.len.value(Self::is_zst()) @@ -594,7 +589,7 @@ impl SmallVec { if self.len.on_heap(Self::is_zst()) { unsafe { self.raw.heap.1 } } else { - self.inline_size() + Self::inline_size() } } @@ -694,7 +689,7 @@ impl SmallVec { let len = self.len(); assert!(new_capacity >= len); - if new_capacity > self.inline_size() { + if new_capacity > Self::inline_size() { let result = unsafe { self.raw.try_grow(self.len, new_capacity) }; if result.is_ok() { unsafe { self.set_on_heap() }; @@ -779,7 +774,7 @@ impl SmallVec { return; } let len = self.len(); - if len <= self.inline_size() { + if len <= Self::inline_size() { let (ptr, capacity) = unsafe { self.raw.heap }; self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); unsafe { copy_nonoverlapping(ptr, self.raw.as_mut_ptr_inline(), len) }; @@ -1128,7 +1123,7 @@ impl SmallVec { #[inline] pub fn from_elem(elem: T, n: usize) -> Self { - if n > Self::inline_capacity() { + if n > Self::inline_size() { Self::from_vec(vec![elem; n]) } else { let mut v = Self::new(); @@ -1380,7 +1375,7 @@ macro_rules! smallvec { let count = 0usize $(+ $crate::smallvec!(@one $x))*; #[allow(unused_mut)] let mut vec = $crate::SmallVec::new(); - if count <= vec.inline_size() { + if count <= vec.capacity() { $(vec.push($x);)* vec } else { diff --git a/src/specialization.rs b/src/specialization.rs deleted file mode 100644 index 658fa77..0000000 --- a/src/specialization.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Implementations that require `default fn`. - -use super::{Array, SmallVec, SpecFrom}; - -impl<'a, A: Array> SpecFrom for SmallVec -where - A::Item: Clone, -{ - #[inline] - default fn spec_from(slice: &'a [A::Item]) -> SmallVec { - slice.into_iter().cloned().collect() - } -} From c8edfa216db0ea97a4c213edf0e3375349da3cde Mon Sep 17 00:00:00 2001 From: sarah el kazdadi Date: Mon, 1 Aug 2022 19:42:01 +0200 Subject: [PATCH 3/7] test: fix fuzzing --- .gitignore | 2 +- fuzz/Cargo.lock | 192 ------------------------------ fuzz/fuzz_targets/smallvec_ops.rs | 14 +-- 3 files changed, 8 insertions(+), 200 deletions(-) delete mode 100644 fuzz/Cargo.lock diff --git a/.gitignore b/.gitignore index 4dec061..b059388 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ target -/Cargo.lock +Cargo.lock /fuzz/hfuzz_target diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock deleted file mode 100644 index 6add833..0000000 --- a/fuzz/Cargo.lock +++ /dev/null @@ -1,192 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "afl" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "arbitrary" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "atty" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bitflags" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cc" -version = "1.0.45" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "clap" -version = "2.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "honggfuzz" -version = "0.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arbitrary 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libc" -version = "0.2.62" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "smallvec" -version = "1.3.0" - -[[package]] -name = "smallvec-fuzz" -version = "0.1.0" -dependencies = [ - "afl 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", - "honggfuzz 0.5.47 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.3.0", -] - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicode-width" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "vec_map" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "xdg" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum afl 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2b15f7ed343b7446090d0de7789a640c22b163fb200d81cef958fcb9856692" -"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -"checksum arbitrary 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1148c9b25d393a07c4cc3ef5dd30f82a40a1c261018c4a670611ed8e76cad3ea" -"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" -"checksum bitflags 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a606a02debe2813760609f57a64a2ffd27d9fdf5b2f133eaca0b248dd92cdd2" -"checksum cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)" = "4fc9a35e1f4290eb9e5fc54ba6cf40671ed2a2514c3eeb2b2a908dda2ea5a1be" -"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" -"checksum honggfuzz 0.5.47 (registry+https://github.com/rust-lang/crates.io-index)" = "c3de2c3273ef7735df1c5a72128ca85b1d20105b9aac643cdfd7a6e581311150" -"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" -"checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -"checksum unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7007dbd421b92cc6e28410fe7362e2e0a2503394908f417b68ec8d1c364c4e20" -"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" -"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" diff --git a/fuzz/fuzz_targets/smallvec_ops.rs b/fuzz/fuzz_targets/smallvec_ops.rs index b5c932e..9d0af34 100644 --- a/fuzz/fuzz_targets/smallvec_ops.rs +++ b/fuzz/fuzz_targets/smallvec_ops.rs @@ -31,8 +31,8 @@ fn black_box_mut_slice(s: &mut [u8]) { black_box_iter((s as &[u8]).iter().copied()) } -fn do_test>(data: &[u8]) -> SmallVec { - let mut v = SmallVec::::new(); +fn do_test(data: &[u8]) -> SmallVec { + let mut v = SmallVec::::new(); let mut bytes = data.iter().copied(); @@ -170,11 +170,11 @@ fn do_test>(data: &[u8]) -> SmallVec { } fn do_test_all(data: &[u8]) { - do_test::<[u8; 0]>(data); - do_test::<[u8; 1]>(data); - do_test::<[u8; 2]>(data); - do_test::<[u8; 7]>(data); - do_test::<[u8; 8]>(data); + do_test::<0>(data); + do_test::<1>(data); + do_test::<2>(data); + do_test::<7>(data); + do_test::<8>(data); } #[cfg(feature = "afl")] From 3e2eecad9934b253f106a298f3b8551a23a9993c Mon Sep 17 00:00:00 2001 From: sarah el kazdadi Date: Mon, 1 Aug 2022 20:16:09 +0200 Subject: [PATCH 4/7] perf: implement specialization based dispatch for improved perf --- src/lib.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 44bbb8d..79e743b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,6 +47,10 @@ //! Tracking issue: [rust-lang/rust#34761](https://github.com/rust-lang/rust/issues/34761) #![no_std] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(feature = "specialization", allow(incomplete_features))] +#![cfg_attr(feature = "specialization", feature(specialization))] +#![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))] #[doc(hidden)] pub extern crate alloc; @@ -73,8 +77,8 @@ use core::ptr::addr_of; use core::ptr::addr_of_mut; use core::ptr::copy_nonoverlapping; -#[cfg(feature = "serde")] use core::marker::PhantomData; + #[cfg(feature = "serde")] use serde::{ de::{Deserialize, Deserializer, SeqAccess, Visitor}, @@ -279,6 +283,7 @@ impl TaggedLen { pub struct SmallVec { len: TaggedLen, raw: RawSmallVec, + _marker: PhantomData, } /// An iterator that removes the items from a `SmallVec` and yields them by value. @@ -361,6 +366,7 @@ pub struct IntoIter { raw: RawSmallVec, begin: usize, end: TaggedLen, + _marker: PhantomData, } impl IntoIter { @@ -468,13 +474,16 @@ impl SmallVec { Self { len: TaggedLen::new(0, false, Self::is_zst()), raw: RawSmallVec::new(), + _marker: PhantomData, } } #[inline] pub fn with_capacity(capacity: usize) -> Self { let mut this = Self::new(); - this.reserve_exact(capacity); + if capacity > Self::inline_size() { + this.grow(capacity); + } this } @@ -494,6 +503,7 @@ impl SmallVec { Self { len: TaggedLen::new(len, false, Self::is_zst()), raw: RawSmallVec::new(), + _marker: PhantomData, } } else { let mut vec = ManuallyDrop::new(vec); @@ -504,6 +514,7 @@ impl SmallVec { Self { len: TaggedLen::new(len, true, Self::is_zst()), raw: RawSmallVec::new_heap(ptr, cap), + _marker: PhantomData, } } } @@ -513,6 +524,7 @@ impl SmallVec { Self { len: TaggedLen::new(N, false, Self::is_zst()), raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), + _marker: PhantomData, } } @@ -522,6 +534,7 @@ impl SmallVec { let mut vec = Self { len: TaggedLen::new(len, false, Self::is_zst()), raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), + _marker: PhantomData, }; // Deallocate the remaining elements so no memory is leaked. unsafe { @@ -545,6 +558,7 @@ impl SmallVec { Self { len: TaggedLen::new(len, false, Self::is_zst()), raw: RawSmallVec::new_inline(buf), + _marker: PhantomData, } } @@ -684,7 +698,7 @@ impl SmallVec { infallible(self.try_grow(new_capacity)); } - #[inline] + #[cold] pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), CollectionAllocErr> { let len = self.len(); assert!(new_capacity >= len); @@ -1043,6 +1057,7 @@ impl SmallVec { SmallVec { len: TaggedLen::new(length, true, Self::is_zst()), raw: RawSmallVec::new_heap(ptr, capacity), + _marker: PhantomData, } } @@ -1070,14 +1085,23 @@ impl SmallVec { #[inline] pub fn from_slice(slice: &[T]) -> Self { let len = slice.len(); - - let mut this = Self::with_capacity(len); - let ptr = this.as_mut_ptr(); - unsafe { - copy_nonoverlapping(slice.as_ptr(), ptr, len); - this.set_len(len); + if len <= Self::inline_size() { + let mut this = Self::new(); + unsafe { + let ptr = this.raw.as_mut_ptr_inline(); + copy_nonoverlapping(slice.as_ptr(), ptr, len); + this.set_len(len); + } + this + } else { + let mut this = Vec::with_capacity(len); + unsafe { + let ptr = this.as_mut_ptr(); + copy_nonoverlapping(slice.as_ptr(), ptr, len); + this.set_len(len); + } + Self::from_vec(this) } - this } #[inline] @@ -1250,6 +1274,29 @@ impl Drop for DropDealloc { } } +#[cfg(feature = "may_dangle")] +unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { + fn drop(&mut self) { + let on_heap = self.spilled(); + let len = self.len(); + let ptr = self.as_mut_ptr(); + unsafe { + let _drop_dealloc = if on_heap { + let capacity = self.capacity(); + Some(DropDealloc { + ptr: ptr as *mut u8, + size_bytes: capacity * size_of::(), + align: align_of::(), + }) + } else { + None + }; + core::ptr::slice_from_raw_parts_mut(ptr, len).drop_in_place(); + } + } +} + +#[cfg(not(feature = "may_dangle"))] impl Drop for SmallVec { fn drop(&mut self) { let on_heap = self.spilled(); @@ -1318,6 +1365,36 @@ impl core::iter::FromIterator for SmallVec { } } +#[cfg(feature = "specialization")] +trait SpecFrom { + type Element; + fn spec_from(slice: &[Self::Element]) -> Self; +} + +#[cfg(feature = "specialization")] +impl SpecFrom for SmallVec { + type Element = T; + + default fn spec_from(slice: &[Self::Element]) -> Self { + slice.iter().cloned().collect() + } +} + +#[cfg(feature = "specialization")] +impl SpecFrom for SmallVec { + fn spec_from(slice: &[Self::Element]) -> Self { + Self::from_slice(slice) + } +} + +#[cfg(feature = "specialization")] +impl<'a, T: Clone, const N: usize> From<&'a [T]> for SmallVec { + fn from(slice: &'a [T]) -> Self { + ::spec_from(slice) + } +} + +#[cfg(not(feature = "specialization"))] impl<'a, T: Clone, const N: usize> From<&'a [T]> for SmallVec { fn from(slice: &'a [T]) -> Self { slice.iter().cloned().collect() @@ -1408,6 +1485,7 @@ impl IntoIterator for SmallVec { raw: (&this.raw as *const RawSmallVec).read(), begin: 0, end: this.len, + _marker: PhantomData, } } } From 095161732a581c1911a732d3edceb4ebac425df2 Mon Sep 17 00:00:00 2001 From: sarah el kazdadi Date: Mon, 1 Aug 2022 20:34:20 +0200 Subject: [PATCH 5/7] chore: import core::ptr::copy instead of qualifying it --- src/lib.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 79e743b..e18ed66 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -75,6 +75,7 @@ use core::mem::ManuallyDrop; use core::mem::MaybeUninit; use core::ptr::addr_of; use core::ptr::addr_of_mut; +use core::ptr::copy; use core::ptr::copy_nonoverlapping; use core::marker::PhantomData; @@ -349,7 +350,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { let ptr = source_vec.as_mut_ptr(); let src = ptr.add(tail); let dst = ptr.add(start); - core::ptr::copy(src, dst, self.tail_len); + copy(src, dst, self.tail_len); } source_vec.set_len(start + self.tail_len); } @@ -864,7 +865,7 @@ impl SmallVec { let ptr = self.as_mut_ptr(); let ith = ptr.add(index); let ith_item = ith.read(); - core::ptr::copy(ith.add(1), ith, new_len - index); + copy(ith.add(1), ith, new_len - index); ith_item } } @@ -877,7 +878,7 @@ impl SmallVec { let ptr = self.as_mut_ptr(); unsafe { if index < len { - core::ptr::copy(ptr.add(index), ptr.add(index + 1), len - index); + copy(ptr.add(index), ptr.add(index + 1), len - index); } ptr.add(index).write(value); self.set_len(len + 1); @@ -1114,7 +1115,7 @@ impl SmallVec { let base_ptr = self.as_mut_ptr(); let ith_ptr = base_ptr.add(index); let shifted_ptr = base_ptr.add(index + other_len); - core::ptr::copy(ith_ptr, shifted_ptr, len - index); + copy(ith_ptr, shifted_ptr, len - index); copy_nonoverlapping(slice.as_ptr(), ith_ptr, other_len); self.set_len(len + other_len); } @@ -1180,7 +1181,7 @@ impl Drop for DropShiftGuard { fn drop(&mut self) { unsafe { core::ptr::slice_from_raw_parts_mut(self.ptr, self.len).drop_in_place(); - core::ptr::copy(self.shifted_ptr, self.ptr, self.shifted_len); + copy(self.shifted_ptr, self.ptr, self.shifted_len); } } } @@ -1206,7 +1207,7 @@ unsafe fn insert_many_batch_phase>( iter: &mut I, ) -> usize { // shift elements to the right to make space for the initial elements from the iterator - core::ptr::copy(ptr.add(index), ptr.add(index + lower_bound), len - index); + copy(ptr.add(index), ptr.add(index + lower_bound), len - index); let ptr_ith = ptr.add(index); let mut guard = DropShiftGuard { ptr: ptr_ith, @@ -1222,7 +1223,7 @@ unsafe fn insert_many_batch_phase>( core::mem::forget(guard); if count < lower_bound { - core::ptr::copy(ptr_ith.add(lower_bound), ptr_ith.add(count), len - index); + copy(ptr_ith.add(lower_bound), ptr_ith.add(count), len - index); } count } From 741ab83a44e594760a96af582cfd337618b4782b Mon Sep 17 00:00:00 2001 From: sarah <> Date: Tue, 2 Aug 2022 09:11:02 +0200 Subject: [PATCH 6/7] doc,perf,bench: - document unsafe code - improve perf for extend - add benchmark for extend --- benches/bench.rs | 13 +- src/lib.rs | 429 +++++++++++++++++++++++++++++++++++++---------- src/tests.rs | 21 +++ 3 files changed, 370 insertions(+), 93 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index b677c19..a936fdd 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -11,7 +11,6 @@ use smallvec::SmallVec; const VEC_SIZE: usize = 16; const SPILLED_SIZE: usize = 100; - trait Vector: for<'a> From<&'a [T]> + Extend { fn new() -> Self; fn push(&mut self, val: T); @@ -114,6 +113,8 @@ make_benches! { bench_remove_small => gen_remove(VEC_SIZE as _), bench_extend => gen_extend(SPILLED_SIZE as _), bench_extend_small => gen_extend(VEC_SIZE as _), + bench_extend_filtered => gen_extend_filtered(SPILLED_SIZE as _), + bench_extend_filtered_small => gen_extend_filtered(VEC_SIZE as _), bench_from_iter => gen_from_iter(SPILLED_SIZE as _), bench_from_iter_small => gen_from_iter(VEC_SIZE as _), bench_from_slice => gen_from_slice(SPILLED_SIZE as _), @@ -138,6 +139,8 @@ make_benches! { bench_remove_vec_small => gen_remove(VEC_SIZE as _), bench_extend_vec => gen_extend(SPILLED_SIZE as _), bench_extend_vec_small => gen_extend(VEC_SIZE as _), + bench_extend_vec_filtered => gen_extend_filtered(SPILLED_SIZE as _), + bench_extend_vec_filtered_small => gen_extend_filtered(VEC_SIZE as _), bench_from_iter_vec => gen_from_iter(SPILLED_SIZE as _), bench_from_iter_vec_small => gen_from_iter(VEC_SIZE as _), bench_from_slice_vec => gen_from_slice(SPILLED_SIZE as _), @@ -221,6 +224,14 @@ fn gen_extend>(n: u64, b: &mut Bencher) { }); } +fn gen_extend_filtered>(n: u64, b: &mut Bencher) { + b.iter(|| { + let mut vec = V::new(); + vec.extend((0..n).filter(|i| i % 2 == 0)); + vec + }); +} + fn gen_from_iter>(n: u64, b: &mut Bencher) { let v: Vec = (0..n).collect(); b.iter(|| { diff --git a/src/lib.rs b/src/lib.rs index e18ed66..935ccf6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -77,6 +77,7 @@ use core::ptr::addr_of; use core::ptr::addr_of_mut; use core::ptr::copy; use core::ptr::copy_nonoverlapping; +use core::ptr::NonNull; use core::marker::PhantomData; @@ -113,7 +114,7 @@ impl core::fmt::Display for CollectionAllocErr { #[repr(C)] pub union RawSmallVec { inline: ManuallyDrop>, - heap: (*const T, usize), + heap: (*const T, usize), // this pointer is never null } #[inline] @@ -162,76 +163,81 @@ impl RawSmallVec { (unsafe { addr_of_mut!(self.inline) }) as *mut T } + /// # Safety + /// + /// The vector must be on the heap #[inline] const unsafe fn as_ptr_heap(&self) -> *const T { self.heap.0 } + /// # Safety + /// + /// The vector must be on the heap #[inline] unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { self.heap.0 as *mut T } - unsafe fn try_grow( + /// # Safety + /// + /// `new_capacity` must be non zero, and greater or equal to the length. + /// T must not be a ZST. + unsafe fn try_grow_raw( &mut self, len: TaggedLen, new_capacity: usize, ) -> Result<(), CollectionAllocErr> { use alloc::alloc::{alloc, realloc}; + debug_assert!(!Self::is_zst()); debug_assert!(new_capacity > 0); + debug_assert!(new_capacity >= len.value(Self::is_zst())); - if Self::is_zst() { - debug_assert_eq!(len.value(Self::is_zst()), usize::MAX); - Err(CollectionAllocErr::CapacityOverflow) + let was_on_heap = len.on_heap(Self::is_zst()); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() } else { - let was_on_heap = len.on_heap(Self::is_zst()); - let ptr = if was_on_heap { - self.as_mut_ptr_heap() - } else { - self.as_mut_ptr_inline() - }; - let len = len.value(Self::is_zst()); + self.as_mut_ptr_inline() + }; + let len = len.value(Self::is_zst()); - let new_layout = Layout::array::(new_capacity) - .map_err(|_| CollectionAllocErr::CapacityOverflow)?; - if new_layout.size() > isize::MAX as usize { - return Err(CollectionAllocErr::CapacityOverflow); - } + let new_layout = + Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(CollectionAllocErr::CapacityOverflow); + } - if len == 0 || !was_on_heap { - // get a fresh allocation + if len == 0 || !was_on_heap { + // get a fresh allocation - // layout has non zero size - let new_ptr = alloc(new_layout) as *mut T; - if new_ptr.is_null() { - Err(CollectionAllocErr::AllocErr { layout: new_layout }) - } else { - copy_nonoverlapping(ptr, new_ptr, len); - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) - } + // layout has non zero size + let new_ptr = alloc(new_layout) as *mut T; + if new_ptr.is_null() { + Err(CollectionAllocErr::AllocErr { layout: new_layout }) } else { - // use realloc - - // this can't overflow since we already constructed an equivalent layout during - // the previous allocation - let old_layout = Layout::from_size_align_unchecked( - self.heap.1 * size_of::(), - align_of::(), - ); - - // SAFETY: ptr was allocated with this allocator - // old_layout is the same as the layout used to allocate the previous memory block - // new_layout.size() is greater than zero - // does not overflow when rounded up to alignment. since it was constructed - // with Layout::array - let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; - if new_ptr.is_null() { - Err(CollectionAllocErr::AllocErr { layout: new_layout }) - } else { - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) - } + copy_nonoverlapping(ptr, new_ptr, len); + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } + } else { + // use realloc + + // this can't overflow since we already constructed an equivalent layout during + // the previous allocation + let old_layout = + Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); + + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the previous memory block + // new_layout.size() is greater than zero + // does not overflow when rounded up to alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + if new_ptr.is_null() { + Err(CollectionAllocErr::AllocErr { layout: new_layout }) + } else { + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) } } } @@ -293,6 +299,13 @@ pub struct SmallVec { /// /// [1]: struct.SmallVec.html#method.drain pub struct Drain<'a, T: 'a, const N: usize> { + // `vec` points to a valid object within its lifetime. + // This is ensured by the fact that we're holding an iterator to its items. + // + // # Safety + // + // Members in vec[tail_start..tail_start + tail_len] are initialized + // even though vec has length < tail_start tail_start: usize, tail_len: usize, iter: core::slice::Iter<'a, T>, @@ -304,6 +317,8 @@ impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { #[inline] fn next(&mut self) -> Option { + // SAFETY: we shrunk the length of the vector so it no longer owns these items, and we can + // take ownership of them. self.iter .next() .map(|reference| unsafe { core::ptr::read(reference) }) @@ -318,6 +333,7 @@ impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { #[inline] fn next_back(&mut self) -> Option { + // SAFETY: see above self.iter .next_back() .map(|reference| unsafe { core::ptr::read(reference) }) @@ -335,13 +351,16 @@ impl<'a, T, const N: usize> core::iter::FusedIterator for Drain<'a, T, N> {} impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { fn drop(&mut self) { - self.for_each(drop); + if core::mem::needs_drop::() { + self.for_each(drop); + } if self.tail_len > 0 { + // SAFETY: we're copying initialized members back to the end of the vector + // then updating its length unsafe { let source_vec = self.vec.as_mut(); - // memmove back untouched tail, update to new length let start = source_vec.len(); let tail = self.tail_start; if tail != start { @@ -364,6 +383,11 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { /// /// [1]: struct.SmallVec.html#method.into_iter pub struct IntoIter { + // # Safety + // + // `end` decides whether the data lives on the heap or not + // + // The members from begin..end are initialized raw: RawSmallVec, begin: usize, end: TaggedLen, @@ -380,6 +404,7 @@ impl IntoIter { const fn as_ptr(&self) -> *const T { let on_heap = self.end.on_heap(Self::is_zst()); if on_heap { + // SAFETY: vector is on the heap unsafe { self.raw.as_ptr_heap() } } else { self.raw.as_ptr_inline() @@ -390,6 +415,7 @@ impl IntoIter { fn as_mut_ptr(&mut self) -> *mut T { let on_heap = self.end.on_heap(Self::is_zst()); if on_heap { + // SAFETY: vector is on the heap unsafe { self.raw.as_mut_ptr_heap() } } else { self.raw.as_mut_ptr_inline() @@ -398,6 +424,8 @@ impl IntoIter { #[inline] pub fn as_slice(&self) -> &[T] { + // SAFETY: The members in self.begin..self.end.value() are all initialized + // So the pointer arithmetic is valid, and so is the construction of the slice unsafe { let ptr = self.as_ptr(); core::slice::from_raw_parts( @@ -409,6 +437,7 @@ impl IntoIter { #[inline] pub fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: see above unsafe { let ptr = self.as_mut_ptr(); core::slice::from_raw_parts_mut( @@ -427,6 +456,7 @@ impl Iterator for IntoIter { if self.begin == self.end.value(Self::is_zst()) { None } else { + // SAFETY: see above unsafe { let ptr = self.as_mut_ptr(); let value = ptr.add(self.begin).read(); @@ -450,6 +480,7 @@ impl DoubleEndedIterator for IntoIter { if self.begin == end { None } else { + // SAFETY: see above unsafe { let ptr = self.as_mut_ptr(); let on_heap = self.end.on_heap(Self::is_zst()); @@ -522,6 +553,7 @@ impl SmallVec { #[inline] pub const fn from_buf(buf: [T; N]) -> Self { + // SAFETY: all the members in 0..N are initialized Self { len: TaggedLen::new(N, false, Self::is_zst()), raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), @@ -532,6 +564,7 @@ impl SmallVec { #[inline] pub fn from_buf_and_len(buf: [T; N], len: usize) -> Self { assert!(len <= N); + // SAFETY: all the members in 0..len are initialized let mut vec = Self { len: TaggedLen::new(len, false, Self::is_zst()), raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), @@ -540,7 +573,7 @@ impl SmallVec { // Deallocate the remaining elements so no memory is leaked. unsafe { // SAFETY: both the input and output pointers are in range of the stack allocation - let remainder_ptr = addr_of_mut!(vec.raw.inline).add(len); + let remainder_ptr = vec.raw.as_mut_ptr_inline().add(len); let remainder_len = N - len; // SAFETY: the values are initialized, so dropping them here is fine. @@ -553,6 +586,25 @@ impl SmallVec { vec } + /// Constructs a new SmallVec on the stack from an A without copying elements. Also sets the length. The user is responsible for ensuring that len <= A::size(). + /// + /// # Examples + /// + /// ``` + /// use smallvec::SmallVec; + /// use std::mem::MaybeUninit; + /// + /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; + /// let small_vec = unsafe { + /// SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) + /// }; + /// + /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); + /// ``` + /// + /// # Safety + /// + /// `len <= N`, and all the elements in `buf[..len]` must be initialized #[inline] pub const unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit<[T; N]>, len: usize) -> Self { debug_assert!(len <= N); @@ -563,15 +615,35 @@ impl SmallVec { } } + /// Sets the tag to be on the heap + /// + /// # Safety + /// + /// The active union member must be the self.raw.heap #[inline] unsafe fn set_on_heap(&mut self) { self.len = TaggedLen::new(self.len(), true, Self::is_zst()); } + + /// Sets the tag to be inline + /// + /// # Safety + /// + /// The active union member must be the self.raw.inline #[inline] unsafe fn set_inline(&mut self) { self.len = TaggedLen::new(self.len(), false, Self::is_zst()); } + /// Sets the length of a vector. + /// + /// This will explicitly set the size of the vector, without actually modifying its buffers, so + /// it is up to the caller to ensure that the vector is actually the specified size. + /// + /// # Safety + /// + /// `new_len <= self.capacity()` must be true, and all the elements in the range `..self.len` + /// must be initialized. #[inline] pub unsafe fn set_len(&mut self, new_len: usize) { debug_assert!(new_len <= self.capacity()); @@ -602,6 +674,7 @@ impl SmallVec { #[inline] pub const fn capacity(&self) -> usize { if self.len.on_heap(Self::is_zst()) { + // SAFETY: raw.heap is active unsafe { self.raw.heap.1 } } else { Self::inline_size() @@ -635,10 +708,13 @@ impl SmallVec { assert!(end <= len); unsafe { + // SAFETY: `start <= len` self.set_len(start); + // SAFETY: all the elements in `start..end` are initialized let range_slice = core::slice::from_raw_parts(self.as_ptr().add(start), end - start); + // SAFETY: all the elements in `end..len` are initialized Drain { tail_start: end, tail_len: len - end, @@ -663,6 +739,20 @@ impl SmallVec { unsafe { self.set_len(len + 1) } } + #[inline] + unsafe fn push_heap(&mut self, value: T) { + // SAFETY: see above + debug_assert!(self.spilled()); + let len = self.len(); + let (ptr, cap) = self.raw.heap; + let ptr = ptr as *mut T; + if len == cap { + self.reserve(1); + } + ptr.add(len).write(value); + self.set_len(len + 1) + } + #[inline] pub fn pop(&mut self) -> Option { if self.is_empty() { @@ -671,6 +761,8 @@ impl SmallVec { let len = self.len() - 1; // SAFETY: len < old_len since this can't overflow, because the old length is non zero unsafe { self.set_len(len) }; + // SAFETY: this element was initialized and we just gave up ownership of it, so we can + // give it away let value = unsafe { self.as_mut_ptr().add(len).read() }; Some(value) } @@ -688,10 +780,10 @@ impl SmallVec { // SAFETY: see `Self::push` let ptr = unsafe { self.as_mut_ptr().add(len) }; + unsafe { other.set_len(0) } // SAFETY: we have a mutable reference to each vector and each uniquely owns its memory. // so the ranges can't overlap unsafe { copy_nonoverlapping(other.as_ptr(), ptr, other_len) }; - unsafe { other.set_len(0) } } #[inline] @@ -701,27 +793,38 @@ impl SmallVec { #[cold] pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), CollectionAllocErr> { + if Self::is_zst() { + return Ok(()); + } + let len = self.len(); assert!(new_capacity >= len); if new_capacity > Self::inline_size() { - let result = unsafe { self.raw.try_grow(self.len, new_capacity) }; + // SAFETY: we checked all the preconditions + let result = unsafe { self.raw.try_grow_raw(self.len, new_capacity) }; + if result.is_ok() { + // SAFETY: the allocation succeeded, so self.raw.heap is now active unsafe { self.set_on_heap() }; } result } else { + // new_capacity <= Self::inline_size() if self.spilled() { unsafe { + // SAFETY: heap member is active let (ptr, old_cap) = self.raw.heap; + // inline member is now active + + // SAFETY: len <= new_capacity <= Self::inline_size() + // so the copy is within bounds of the inline member copy_nonoverlapping(ptr, self.raw.as_mut_ptr_inline(), len); - { - let _drop_dealloc = DropDealloc { - ptr: ptr as *mut u8, - size_bytes: old_cap * size_of::(), - align: align_of::(), - }; - } + drop(DropDealloc { + ptr: NonNull::new_unchecked(ptr as *mut u8), + size_bytes: old_cap * size_of::(), + align: align_of::(), + }); self.set_inline(); } } @@ -790,18 +893,22 @@ impl SmallVec { } let len = self.len(); if len <= Self::inline_size() { - let (ptr, capacity) = unsafe { self.raw.heap }; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); - unsafe { copy_nonoverlapping(ptr, self.raw.as_mut_ptr_inline(), len) }; - unsafe { self.set_inline() }; + // SAFETY: self.spilled() is true, so we're on the heap unsafe { + let (ptr, capacity) = self.raw.heap; + self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); + copy_nonoverlapping(ptr, self.raw.as_mut_ptr_inline(), len); + self.set_inline(); alloc::alloc::dealloc( ptr as *mut T as *mut u8, Layout::from_size_align_unchecked(capacity * size_of::(), align_of::()), - ) - }; + ); + } } else if len < self.capacity() { - self.grow(len); + // SAFETY: len > Self::inline_size() >= 0 + // so new capacity is non zero, it is equal to the length + // T can't be a ZST because SmallVec is never spilled. + unsafe { infallible(self.raw.try_grow_raw(self.len, len)) }; } } @@ -809,6 +916,8 @@ impl SmallVec { pub fn truncate(&mut self, len: usize) { let old_len = self.len(); if len < old_len { + // SAFETY: we set `len` to a smaller value + // then we drop the previously initialized elements unsafe { self.set_len(len); core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( @@ -823,6 +932,7 @@ impl SmallVec { pub fn as_slice(&self) -> &[T] { let len = self.len(); let ptr = self.as_ptr(); + // SAFETY: all the elements in `..len` are initialized unsafe { core::slice::from_raw_parts(ptr, len) } } @@ -830,6 +940,7 @@ impl SmallVec { pub fn as_mut_slice(&mut self) -> &mut [T] { let len = self.len(); let ptr = self.as_mut_ptr(); + // SAFETY: see above unsafe { core::slice::from_raw_parts_mut(ptr, len) } } @@ -837,14 +948,22 @@ impl SmallVec { pub fn swap_remove(&mut self, index: usize) -> T { let len = self.len(); assert!(index < len); + // This can't overflow since `len > index >= 0` let new_len = len - 1; unsafe { + // SAFETY: we set len to a smaller value self.set_len(new_len); let ptr = self.as_mut_ptr(); let last = ptr.add(new_len); let ith = ptr.add(index); + // This item is initialized since it was in the vector just before let last_item = last.read(); + // This item is initialized since index < len let ith_item = ith.read(); + + // Note that these may be the same element. + // This is fine since in this case we just write it back to the pointer past the end of + // the vector, so the vector no longer owns it ith.write(last_item); ith_item } @@ -861,9 +980,11 @@ impl SmallVec { assert!(index < len); let new_len = len - 1; unsafe { + // SAFETY: new_len < len self.set_len(new_len); let ptr = self.as_mut_ptr(); let ith = ptr.add(index); + // This item is initialized since index < len let ith_item = ith.read(); copy(ith.add(1), ith, new_len - index); ith_item @@ -877,26 +998,34 @@ impl SmallVec { self.reserve(1); let ptr = self.as_mut_ptr(); unsafe { + // the elements at `index + 1..len + 1` are now initialized if index < len { copy(ptr.add(index), ptr.add(index + 1), len - index); } + // the element at `index` is now initialized ptr.add(index).write(value); + + // SAFETY: all the elements are initialized self.set_len(len + 1); } } - fn insert_many_impl>(&mut self, mut index: usize, mut iter: I) { + fn insert_many_impl>(&mut self, mut index: usize, iter: I) { let len = self.len(); if index == len { return self.extend(iter); } + let mut iter = iter.fuse(); let (lower_bound, _) = iter.size_hint(); self.reserve(lower_bound); let count = unsafe { let ptr = self.as_mut_ptr(); - let count = insert_many_batch_phase(ptr, index, lower_bound, len, &mut iter); + // SAFETY: ptr is valid for `lower_bound` writes since we just reserved that much + let count = insert_many_batch(ptr, index, lower_bound, len, &mut iter); + // SAFETY: insert_many_batch_phase returns the number of elements it initialized, and + // leaves the vector in a valid state, without setting the new length self.set_len(len + count); count }; @@ -914,6 +1043,7 @@ impl SmallVec { #[inline] pub const fn as_ptr(&self) -> *const T { if self.len.on_heap(Self::is_zst()) { + // SAFETY: heap member is active unsafe { self.raw.as_ptr_heap() } } else { self.raw.as_ptr_inline() @@ -923,6 +1053,7 @@ impl SmallVec { #[inline] pub fn as_mut_ptr(&mut self) -> *mut T { if self.len.on_heap(Self::is_zst()) { + // SAFETY: see above unsafe { self.raw.as_mut_ptr_heap() } } else { self.raw.as_mut_ptr_inline() @@ -935,6 +1066,9 @@ impl SmallVec { if !self.spilled() { let mut vec = Vec::with_capacity(len); let this = ManuallyDrop::new(self); + // SAFETY: we create a new vector with sufficient capacity, copy our elements into it + // to transfer ownership and then set the length + // we don't drop the elements we previously held unsafe { copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); vec.set_len(len); @@ -942,6 +1076,7 @@ impl SmallVec { vec } else { let this = ManuallyDrop::new(self); + // SAFETY: ptr was created with the global allocator unsafe { let (ptr, cap) = this.raw.heap; Vec::from_raw_parts(ptr as *mut T, len, cap) @@ -959,8 +1094,14 @@ impl SmallVec { if self.len() != N { Err(self) } else { - let this = ManuallyDrop::new(self); + // when `this` is dropped, the memory is released if it's on the heap. + let mut this = self; + // SAFETY: we release ownership of the elements we hold + unsafe { + this.set_len(0); + } let ptr = this.as_ptr() as *const [T; N]; + // SAFETY: these elements are initialized since the length was `N` unsafe { Ok(ptr.read()) } } } @@ -968,13 +1109,15 @@ impl SmallVec { pub fn retain bool>(&mut self, mut f: F) { let mut del = 0; let len = self.len(); + let ptr = self.as_mut_ptr(); for i in 0..len { + // SAFETY: all the pointers are in bounds + // `i - del` never overflows since `del <= i` is a maintained invariant unsafe { - let ptr = self.as_mut_ptr().add(i); - if !f(&mut *ptr) { + if !f(&mut *ptr.add(i)) { del += 1; } else if del > 0 { - core::mem::swap(&mut *ptr, &mut *ptr.sub(del)); + core::mem::swap(&mut *ptr.add(i), &mut *ptr.add(i - del)); } } } @@ -1052,6 +1195,61 @@ impl SmallVec { } } + /// Creates a SmallVec directly from the raw components of another SmallVec. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren’t checked: + /// + /// - ptr needs to have been previously allocated via SmallVec from its spilled storage (at least, it’s highly likely to be incorrect if it wasn’t). + /// - ptr’s A::Item type needs to be the same size and alignment that it was allocated with + /// - length needs to be less than or equal to capacity. + /// - capacity needs to be the capacity that the pointer was allocated with. + /// + /// Violating these may cause problems like corrupting the allocator’s internal data structures. + /// + /// Additionally, capacity must be greater than the amount of inline storage A has; that is, the new SmallVec must need to spill over into heap allocated storage. This condition is asserted against. + /// + /// The ownership of ptr is effectively transferred to the SmallVec which may then deallocate, reallocate or change the contents of memory pointed to by the pointer at will. Ensure that nothing else uses the pointer after calling this function. + /// + /// # Examples + /// + /// ``` + /// use std::mem; + /// use std::ptr; + /// use smallvec::{SmallVec, smallvec}; + /// + /// fn main() { + /// let mut v: SmallVec<_, 1> = smallvec![1, 2, 3]; + /// + /// // Pull out the important parts of `v`. + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// let spilled = v.spilled(); + /// + /// unsafe { + /// // Forget all about `v`. The heap allocation that stored the + /// // three values won't be deallocated. + /// mem::forget(v); + /// + /// // Overwrite memory with [4, 5, 6]. + /// // + /// // This is only safe if `spilled` is true! Otherwise, we are + /// // writing into the old `SmallVec`'s inline storage on the + /// // stack. + /// assert!(spilled); + /// for i in 0..len { + /// ptr::write(p.add(i), 4 + i); + /// } + /// + /// // Put everything back together into a SmallVec with a different + /// // amount of inline storage, but which is still less than `cap`. + /// let rebuilt = SmallVec::<_, 2>::from_raw_parts(p, len, cap); + /// assert_eq!(&*rebuilt, &[4, 5, 6]); + /// } + /// } + /// ``` #[inline] pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> SmallVec { assert!(!Self::is_zst()); @@ -1062,16 +1260,43 @@ impl SmallVec { } } - fn extend_impl>(&mut self, mut iter: I) { + fn extend_impl>(&mut self, iter: I) { + let mut iter = iter.fuse(); let len = self.len(); let (lower_bound, _) = iter.size_hint(); self.reserve(lower_bound); + let capacity = self.capacity(); unsafe { let ptr = self.as_mut_ptr(); - let count = extend_batch_phase(ptr, lower_bound, len, &mut iter); + // SAFETY: ptr is valid for `capacity - len` writes + let count = extend_batch(ptr, capacity - len, len, &mut iter); self.set_len(len + count); } - iter.for_each(|item| self.push(item)); + + if let Some(item) = iter.next() { + self.push(item); + } else { + return; + } + + // either we ran out of items, in which case this loop doesn't get executed. or we still + // have items to push, and in that case we must be on the heap, since we filled up the + // capacity and then pushed one item + unsafe { + loop { + if let Some(item) = iter.next() { + self.push_heap(item); + } else { + break; + } + let len = self.len(); + let (ptr, capacity) = self.raw.heap; + let ptr = ptr as *mut T; + // SAFETY: ptr is valid for `capacity - len` writes + let count = extend_batch(ptr, capacity - len, len, &mut iter); + self.set_len(len + count); + } + } } } @@ -1115,8 +1340,12 @@ impl SmallVec { let base_ptr = self.as_mut_ptr(); let ith_ptr = base_ptr.add(index); let shifted_ptr = base_ptr.add(index + other_len); + // elements at `index + other_len..len + other_len` are now initialized copy(ith_ptr, shifted_ptr, len - index); + // elements at `index..index + other_len` are now initialized copy_nonoverlapping(slice.as_ptr(), ith_ptr, other_len); + + // SAFETY: all the elements are initialized self.set_len(len + other_len); } } @@ -1126,6 +1355,7 @@ impl SmallVec { let len = self.len(); let other_len = slice.len(); self.reserve(other_len); + // SAFETY: see above unsafe { let base_ptr = self.as_mut_ptr(); let end_ptr = base_ptr.add(len); @@ -1154,15 +1384,16 @@ impl SmallVec { let mut v = Self::new(); unsafe { - let ptr = v.as_mut_ptr(); + let ptr = v.raw.as_mut_ptr_inline(); let mut guard = DropGuard { ptr, len: 0 }; - // assume T is expensive to clone + // SAFETY: `n <= Self::inline_size()` so we can write `n` elements for i in 0..n { guard.len = i; ptr.add(i).write(elem.clone()); } core::mem::forget(guard); + // SAFETY: we just initialized `n` elements in the vector v.set_len(n); } v @@ -1199,7 +1430,11 @@ impl Drop for DropGuard { } } -unsafe fn insert_many_batch_phase>( +// Safety: +// +// `ptr..ptr + lower_bound` must be valid for writes +#[inline] +unsafe fn insert_many_batch>( ptr: *mut T, index: usize, lower_bound: usize, @@ -1228,9 +1463,11 @@ unsafe fn insert_many_batch_phase>( count } -unsafe fn extend_batch_phase>( +// `ptr..ptr + remaining_capacity` must be valid for writes +#[inline] +unsafe fn extend_batch>( ptr: *mut T, - lower_bound: usize, + remaining_capacity: usize, len: usize, iter: &mut I, ) -> usize { @@ -1239,10 +1476,12 @@ unsafe fn extend_batch_phase>( ptr: ptr_end, len: 0, }; - iter.take(lower_bound).enumerate().for_each(|(i, item)| { - ptr_end.add(i).write(item); - guard.len = i + 1; - }); + iter.take(remaining_capacity) + .enumerate() + .for_each(|(i, item)| { + ptr_end.add(i).write(item); + guard.len = i + 1; + }); let count = guard.len; core::mem::forget(guard); count @@ -1256,7 +1495,7 @@ impl Extend for SmallVec { } struct DropDealloc { - ptr: *mut u8, + ptr: NonNull, size_bytes: usize, align: usize, } @@ -1267,7 +1506,7 @@ impl Drop for DropDealloc { unsafe { if self.size_bytes > 0 { alloc::alloc::dealloc( - self.ptr, + self.ptr.as_ptr(), Layout::from_size_align_unchecked(self.size_bytes, self.align), ); } @@ -1281,11 +1520,13 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { let on_heap = self.spilled(); let len = self.len(); let ptr = self.as_mut_ptr(); + // SAFETY: we first drop the elements, then `_drop_dealloc` is dropped, releasing memory we + // used to own unsafe { let _drop_dealloc = if on_heap { let capacity = self.capacity(); Some(DropDealloc { - ptr: ptr as *mut u8, + ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), align: align_of::(), }) @@ -1303,11 +1544,12 @@ impl Drop for SmallVec { let on_heap = self.spilled(); let len = self.len(); let ptr = self.as_mut_ptr(); + // SAFETY: see above unsafe { let _drop_dealloc = if on_heap { let capacity = self.capacity(); Some(DropDealloc { - ptr: ptr as *mut u8, + ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), align: align_of::(), }) @@ -1321,6 +1563,7 @@ impl Drop for SmallVec { impl Drop for IntoIter { fn drop(&mut self) { + // SAFETY: see above unsafe { let is_zst = size_of::() == 0; let on_heap = self.end.on_heap(is_zst); @@ -1330,7 +1573,7 @@ impl Drop for IntoIter { let _drop_dealloc = if on_heap { let capacity = self.raw.heap.1; Some(DropDealloc { - ptr: ptr as *mut u8, + ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), align: align_of::(), }) @@ -1424,7 +1667,7 @@ impl Clone for SmallVec { // drop anything that will not be overwritten self.truncate(source.len()); - // self.len <= other.len due to the truncate above, so the + // SAFETY: self.len <= other.len due to the truncate above, so the // slices here are always in-bounds. let init = unsafe { source.get_unchecked(..self.len()) }; let tail = unsafe { source.get_unchecked(self.len()..) }; @@ -1479,6 +1722,8 @@ impl IntoIterator for SmallVec { type IntoIter = IntoIter; type Item = T; fn into_iter(self) -> Self::IntoIter { + // SAFETY: we move out of this.raw by reading the value at its address, which is fine since + // we don't drop it unsafe { // Set SmallVec len to zero as `IntoIter` drop handles dropping of the elements let this = ManuallyDrop::new(self); diff --git a/src/tests.rs b/src/tests.rs index d233701..eee6fb0 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -259,6 +259,27 @@ fn test_truncate() { assert_eq!(&v.iter().map(|v| **v).collect::>(), &[0, 3, 2]); } +#[test] +fn test_truncate_references() { + let mut v = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut i = 8; + let mut v: SmallVec<&mut u8, 8> = v.iter_mut().collect(); + + v.truncate(4); + + assert_eq!(v.len(), 4); + assert!(!v.spilled()); + + assert_eq!(*v.swap_remove(1), 1); + assert_eq!(*v.remove(1), 3); + v.insert(1, &mut i); + + assert_eq!( + &v.iter_mut().map(|v| &mut **v).collect::>(), + &[&mut 0, &mut 8, &mut 2] + ); +} + #[test] fn test_insert_many() { let mut v: SmallVec = SmallVec::new(); From 731f6bcad036cbf338d1f975abc0cd595a216cb3 Mon Sep 17 00:00:00 2001 From: sarah <> Date: Mon, 27 Mar 2023 20:32:37 +0200 Subject: [PATCH 7/7] impl Send/Sync for SmallVec --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 935ccf6..067f230 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -293,6 +293,9 @@ pub struct SmallVec { _marker: PhantomData, } +unsafe impl Send for SmallVec {} +unsafe impl Sync for SmallVec {} + /// An iterator that removes the items from a `SmallVec` and yields them by value. /// /// Returned from [`SmallVec::drain`][1].