diff --git a/src/doc/book b/src/doc/book index 7db393dae740d..08e79609ce885 160000 --- a/src/doc/book +++ b/src/doc/book @@ -1 +1 @@ -Subproject commit 7db393dae740d84775b73f403123c866e94e3a5b +Subproject commit 08e79609ce88583fa7286157dfe497486a09fabe diff --git a/src/doc/nomicon b/src/doc/nomicon index 1625e0b8c8708..0ee3f7265e9d0 160000 --- a/src/doc/nomicon +++ b/src/doc/nomicon @@ -1 +1 @@ -Subproject commit 1625e0b8c870891b84b0969777a974bf87be579b +Subproject commit 0ee3f7265e9d09746d901cef6f1f300baff1d923 diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index dbf1fb1367dda..1f159bb36b187 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -13,7 +13,7 @@ use core::mem; use core::ops::Drop; use core::ptr::{self, Unique}; use core::slice; -use heap::{Alloc, Layout, Heap}; +use heap::{Alloc, Layout, Heap, Excess}; use super::boxed::Box; /// A low-level utility for more ergonomically allocating, reallocating, and deallocating @@ -165,6 +165,79 @@ impl RawVec { } } +/// Given a `current_capacity` and a desired `capacity_increase` returns a +/// suitable capacity for the `RawVec` such that `suitable_capacity >= +/// current_capacity + capacity_increase`. +/// +/// # Panics +/// +/// Panics on overflow if `current_capacity + capacity_increase > +/// std::usize::MAX`. +/// +/// # Growth strategy +/// +/// RawVec grows differently depending on: +/// +/// - 1. initial size: grows from zero to four elements or at least 64 bytes; +/// use `with_capacity` to avoid a growth from zero. +/// +/// - 2. vector size: +/// - small vectors (<= 4096 bytes) grow with a factor of 2x. +/// - medium-sized and large vectors grow with a growth factor of 1.5x. +/// +/// # Growth factor +/// +/// The growth factor of medium-sized and large vectors is chosen to allow +/// reusing memory from previously-freed allocations in subsequent allocations. +/// +/// Depending on the growth factor, previously freed memory can be reused after +/// +/// - 4 reallocations for a growth factor of 1.5x, +/// - 3 reallocations for a growth factor of 1.45x, +/// - 2 reallocations for a growth factor of 1.3x, +/// +/// Which growth-factor is better [is application +/// dependent](https://stackoverflow.com/questions/1100311/ +/// what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), +/// also some claim that [the golden ration (1.618) is +/// optimal](https://crntaylor.wordpress.com/2011/07/15/ +/// optimal-memory-reallocation-and-the-golden-ratio/). +/// The trade-off is having to wait for many reallocations to be able to +/// reuse old memory. +/// +/// A factor of 2x _never_ allows reusing any previously-freed memory but +/// `jemalloc`'s memory blocks < 4096 bytes cannot grow in place, that is, +/// trying to grow a vector < 4096 bytes is always going to require allocating +/// new memory and copying the contents over. Since `jemalloc`'s memory pools +/// for small allocations grow with powers of 2 it makes sense to keep a +/// growth-factor of 2x for these allocations. +/// +#[inline(always)] +fn amortized_new_capacity(elem_size: usize, current_capacity: usize, + capacity_increase: usize) -> usize { + // Computes the capacity from the `current_capacity` following the + // growth-strategy. The function `alloc_guard` ensures that + // `current_capacity <= isize::MAX` so that `current_capacity * N` where `N + // <= 2` cannot overflow. + let growth_capacity = match current_capacity { + // Empty vector => 4 elements or at least 64 bytes + 0 => (64 / elem_size).max(4), + + // Small vectors: jemalloc cannot grow in place blocks smaller than 4096 + // bytes so until then the memory of previously-freed allocations + // cannot be reused by subsequent allocations: + c if c < 4096 / elem_size => 2 * c, + + // Medium and large vectors (>= 4096 bytes): a growth factor of 1.5 + // allows the memory of a previously-freeed allocation to be reused + // after 4 subsequent allocations: + c => (c / 2 + 1) * 3, + }; + cmp::max(growth_capacity, + current_capacity.checked_add(capacity_increase).unwrap()) +} + + impl RawVec { /// Reconstitutes a RawVec from a pointer, capacity. /// @@ -439,19 +512,6 @@ impl RawVec { } } - /// Calculates the buffer's new size given that it'll hold `used_cap + - /// needed_extra_cap` elements. This logic is used in amortized reserve methods. - /// Returns `(new_capacity, new_alloc_size)`. - fn amortized_new_size(&self, used_cap: usize, needed_extra_cap: usize) -> usize { - // Nothing we can really do about these checks :( - let required_cap = used_cap.checked_add(needed_extra_cap) - .expect("capacity overflow"); - // Cannot overflow, because `cap <= isize::MAX`, and type of `cap` is `usize`. - let double_cap = self.cap * 2; - // `double_cap` guarantees exponential growth. - cmp::max(double_cap, required_cap) - } - /// Ensures that the buffer contains at least enough space to hold /// `used_cap + needed_extra_cap` elements. If it doesn't already have /// enough capacity, will reallocate enough space plus comfortable slack @@ -504,6 +564,8 @@ impl RawVec { /// # vector.push_all(&[1, 3, 5, 7, 9]); /// # } /// ``` + #[inline] + #[cold] pub fn reserve(&mut self, used_cap: usize, needed_extra_cap: usize) { unsafe { // NOTE: we don't early branch on ZSTs here because we want this @@ -517,7 +579,10 @@ impl RawVec { return; } - let new_cap = self.amortized_new_size(used_cap, needed_extra_cap); + let elem_size = mem::size_of::(); + let new_cap = amortized_new_capacity(elem_size, + used_cap, + needed_extra_cap); let new_layout = match Layout::array::(new_cap) { Some(layout) => layout, @@ -525,19 +590,19 @@ impl RawVec { }; // FIXME: may crash and burn on over-reserve alloc_guard(new_layout.size()); - let res = match self.current_layout() { + let Excess(res, usable_size) = match self.current_layout() { Some(layout) => { let old_ptr = self.ptr.as_ptr() as *mut u8; - self.a.realloc(old_ptr, layout, new_layout) + self.a.realloc_excess(old_ptr, layout, new_layout) } - None => self.a.alloc(new_layout), + None => self.a.alloc_excess(new_layout), }; let uniq = match res { Ok(ptr) => Unique::new_unchecked(ptr as *mut T), Err(e) => self.a.oom(e), }; self.ptr = uniq; - self.cap = new_cap; + self.cap = usable_size / mem::size_of::(); } } @@ -576,7 +641,10 @@ impl RawVec { return false; } - let new_cap = self.amortized_new_size(used_cap, needed_extra_cap); + let elem_size = mem::size_of::(); + let new_cap = amortized_new_capacity(elem_size, + used_cap, + needed_extra_cap); // Here, `cap < used_cap + needed_extra_cap <= new_cap` // (regardless of whether `self.cap - used_cap` wrapped). @@ -588,7 +656,10 @@ impl RawVec { alloc_guard(new_layout.size()); match self.a.grow_in_place(ptr, old_layout, new_layout) { Ok(_) => { - self.cap = new_cap; + // FIXME: grow_in_place swallows the usable_size, so we + // need to recompute it here: + let (_, new_usable_size) = self.a.usable_size(&new_layout); + self.cap = new_usable_size / mem::size_of::(); true } Err(_) => { @@ -760,38 +831,79 @@ mod tests { v.reserve(50, 150); // (causes a realloc, thus using 50 + 150 = 200 units of fuel) assert_eq!(v.a.fuel, 250); } +/* + #[test] + fn amortized_new_capacity_tests() { + // empty vector: + assert_eq!(amortized_new_capacity(1, 0, 1), 64); + assert_eq!(amortized_new_capacity(64, 0, 1), 1); + assert_eq!(amortized_new_capacity(128, 0, 1), 1); + + // small vector: + assert_eq!(amortized_new_capacity(1, 2048, 1), 4096); // 2x + + // medium vector + assert_eq!(amortized_new_capacity(1, 5000, 1), 7500); // 1.5x + + // large vector + assert_eq!(amortized_new_capacity(1, 140000, 1), 280000); // 2x + } #[test] - fn reserve_does_not_overallocate() { + fn reserve_() { + // Allocation from empty vector: { - let mut v: RawVec = RawVec::new(); - // First `reserve` allocates like `reserve_exact` - v.reserve(0, 9); - assert_eq!(9, v.cap()); - } + let mut v: RawVec = RawVec::new(); + assert_eq!(v.cap(), 0); + // empty vectors allocate at least 64 bytes + v.reserve(0, 63); + assert_eq!(v.cap(), 64); + } { let mut v: RawVec = RawVec::new(); - v.reserve(0, 7); - assert_eq!(7, v.cap()); - // 97 if more than double of 7, so `reserve` should work - // like `reserve_exact`. - v.reserve(7, 90); - assert_eq!(97, v.cap()); - } + // empty vectors allocate at least 64 bytes + v.reserve(0, 9); + assert_eq!(16, v.cap()); + } + // Exponential growth: { - let mut v: RawVec = RawVec::new(); - v.reserve(0, 12); - assert_eq!(12, v.cap()); - v.reserve(12, 3); - // 3 is less than half of 12, so `reserve` must grow - // exponentially. At the time of writing this test grow - // factor is 2, so new capacity is 24, however, grow factor - // of 1.5 is OK too. Hence `>= 18` in assert. - assert!(v.cap() >= 12 + 12 / 2); + let mut v: RawVec = RawVec::new(); + assert_eq!(v.cap(), 0); + + // empty vectors allocate at least 64 bytes + v.reserve(0, 1); + assert_eq!(v.cap(), 64); + + // small vectors grow with a factor of two: + v.reserve(64, 1); + assert_eq!(v.cap(), 128); + + // The required capacity is 128 + 3968 = 4096 which is larger than + // 2*128 = 256, so here `reserve` allocates "exactly" 4096 elements + // modulo extra size returned by the allocator: + v.reserve(128, 3968); + assert!(v.cap() >= 4096 && v.cap() <= 4096 + 128); + + // 1 <= 1.5 * cap, so here the "medium" sized vector uses a growth + // factor of 1.5: + let cap = v.cap(); + v.reserve(cap, 1); + assert!(v.cap() >= cap * 3 / 2 && v.cap() <= cap * 3 / 2 + 512); + + // we reserve enough to make the vector "large" + let cap = v.cap(); + v.reserve(cap, 4096 * 32); + assert!(v.cap() >= cap + 4096 * 32 + && v.cap() <= cap + 4096 * 40); + + // large vectors grow with a growth-factor equals 2x: + let cap = v.cap(); + v.reserve(cap, 1); + assert_eq!(v.cap(), cap * 2); + assert!(v.cap() >= cap * 2 && v.cap() <= cap * 2 + 4096); } } - - + */ } diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 5aca199cf40c0..7b8b39ba29ae7 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -79,6 +79,7 @@ use core::ops; use core::ptr; use core::ptr::Shared; use core::slice; +use core::intrinsics; use borrow::ToOwned; use borrow::Cow; @@ -707,6 +708,11 @@ impl Vec { self.pop().unwrap() } + #[inline(always)] + fn is_full(&self) -> bool { + self.len == self.buf.cap() + } + /// Inserts an element at position `index` within the vector, shifting all /// elements after it to the right. /// @@ -729,8 +735,8 @@ impl Vec { assert!(index <= len); // space for the new element - if len == self.buf.cap() { - self.buf.double(); + if unsafe { intrinsics::unlikely(self.is_full()) } { + self.buf.reserve(self.len, 1); } unsafe { @@ -964,8 +970,8 @@ impl Vec { pub fn push(&mut self, value: T) { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. - if self.len == self.buf.cap() { - self.buf.double(); + if unsafe { intrinsics::unlikely(self.is_full()) } { + self.buf.reserve(self.len, 1); } unsafe { let end = self.as_mut_ptr().offset(self.len as isize); @@ -2532,8 +2538,9 @@ impl<'a, T> Placer for PlaceBack<'a, T> { fn make_place(self) -> Self { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. - if self.vec.len == self.vec.buf.cap() { - self.vec.buf.double(); + if unsafe { intrinsics::unlikely(self.vec.is_full()) } { + let len = self.vec.len(); + self.vec.buf.reserve(len, 1); } self } diff --git a/src/liballoc/vec_deque.rs b/src/liballoc/vec_deque.rs index f56aa23a4eb2f..c4848fc1097bf 100644 --- a/src/liballoc/vec_deque.rs +++ b/src/liballoc/vec_deque.rs @@ -25,6 +25,7 @@ use core::ops::{Index, IndexMut, Place, Placer, InPlace}; use core::ptr; use core::ptr::Shared; use core::slice; +use core::intrinsics; use core::hash::{Hash, Hasher}; use core::cmp; @@ -1752,7 +1753,7 @@ impl VecDeque { // This may panic or abort #[inline] fn grow_if_necessary(&mut self) { - if self.is_full() { + if unsafe { intrinsics::unlikely(self.is_full()) } { let old_cap = self.cap(); self.buf.double(); unsafe { diff --git a/src/liblibc b/src/liblibc index 68f9959e53da6..44e4018e1a377 160000 --- a/src/liblibc +++ b/src/liblibc @@ -1 +1 @@ -Subproject commit 68f9959e53da6c70bed7119cd09342859d431266 +Subproject commit 44e4018e1a37716286ec98cb5b7dd7d33ecaf940 diff --git a/src/tools/cargo b/src/tools/cargo index b83550edc300e..e447ac7e94b7f 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit b83550edc300e3d80dd16d0440123ffc1ad77bb9 +Subproject commit e447ac7e94b7f56ab13e361f9e324dafe3eb0a34 diff --git a/src/tools/clippy b/src/tools/clippy index f76225e388717..b62b1b68edcdf 160000 --- a/src/tools/clippy +++ b/src/tools/clippy @@ -1 +1 @@ -Subproject commit f76225e3887170743403af9204887918b5db5a80 +Subproject commit b62b1b68edcdf23a70cb12f31403c80e97f13634 diff --git a/src/tools/rls b/src/tools/rls index 9ad92e134ff56..93b47d14cef57 160000 --- a/src/tools/rls +++ b/src/tools/rls @@ -1 +1 @@ -Subproject commit 9ad92e134ff56df52481cf19dc3da14b9e735061 +Subproject commit 93b47d14cef5720bba7cfb4dcb8078fbf1f706c1 diff --git a/src/tools/rustfmt b/src/tools/rustfmt index 51b03c3aaf5e6..16a478368c8dc 160000 --- a/src/tools/rustfmt +++ b/src/tools/rustfmt @@ -1 +1 @@ -Subproject commit 51b03c3aaf5e69afbb7508e566c5da2bf0bc3662 +Subproject commit 16a478368c8dcc0c0ee47372a9f663b23d28b097