From 401cc19b7a89ae8b13d0ecd0fbae9eaebf4facd2 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Fri, 25 Apr 2014 02:19:34 -0400 Subject: [PATCH 1/2] add back jemalloc to the tree This adds a `std::rt::heap` module with a nice allocator API. It's a step towards fixing #13094 and is a starting point for working on a generic allocator trait. The revision used for the jemalloc submodule is the stable 3.6.0 release. --- .gitmodules | 3 ++ configure | 1 + mk/crates.mk | 2 +- mk/rt.mk | 48 ++++++++++++++++++++-- mk/tests.mk | 4 +- src/jemalloc | 1 + src/libstd/rt/heap.rs | 93 +++++++++++++++++++++++++++++++++++++++++++ src/libstd/rt/mod.rs | 3 ++ 8 files changed, 149 insertions(+), 6 deletions(-) create mode 160000 src/jemalloc create mode 100644 src/libstd/rt/heap.rs diff --git a/.gitmodules b/.gitmodules index 597086229b8df..bb50c6e87b263 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,3 +12,6 @@ [submodule "src/compiler-rt"] path = src/compiler-rt url = https://github.com/rust-lang/compiler-rt.git +[submodule "src/jemalloc"] + path = src/jemalloc + url = https://github.com/jemalloc/jemalloc.git diff --git a/configure b/configure index 983be4e8a0c0a..18be689287d20 100755 --- a/configure +++ b/configure @@ -762,6 +762,7 @@ do for s in 0 1 2 3 do make_dir $t/rt/stage$s + make_dir $t/rt/jemalloc make_dir $t/rt/libuv make_dir $t/rt/libuv/src/ares make_dir $t/rt/libuv/src/eio diff --git a/mk/crates.mk b/mk/crates.mk index 28330010dc2ac..50c0fc5494864 100644 --- a/mk/crates.mk +++ b/mk/crates.mk @@ -56,7 +56,7 @@ HOST_CRATES := syntax rustc rustdoc fourcc hexfloat CRATES := $(TARGET_CRATES) $(HOST_CRATES) TOOLS := compiletest rustdoc rustc -DEPS_std := libc native:rustrt native:compiler-rt native:backtrace +DEPS_std := libc native:rustrt native:compiler-rt native:backtrace native:jemalloc DEPS_green := std rand native:context_switch DEPS_rustuv := std native:uv native:uv_support DEPS_native := std diff --git a/mk/rt.mk b/mk/rt.mk index d0fc577458987..79952ca172dfe 100644 --- a/mk/rt.mk +++ b/mk/rt.mk @@ -121,10 +121,13 @@ $(foreach lib,$(NATIVE_LIBS), \ ################################################################################ # Building third-party targets with external build systems # -# The only current member of this section is libuv, but long ago this used to -# also be occupied by jemalloc. This location is meant for dependencies which -# have external build systems. It is still assumed that the output of each of -# these steps is a static library in the correct location. +# This location is meant for dependencies which have external build systems. It +# is still assumed that the output of each of these steps is a static library +# in the correct location. +################################################################################ + +################################################################################ +# libuv ################################################################################ define DEF_LIBUV_ARCH_VAR @@ -160,6 +163,7 @@ else ifeq ($(OSTYPE_$(1)), unknown-freebsd) else ifeq ($(OSTYPE_$(1)), linux-androideabi) LIBUV_OSTYPE_$(1) := android LIBUV_ARGS_$(1) := PLATFORM=android host=android OS=linux + JEMALLOC_ARGS_$(1) := --disable-tls else LIBUV_OSTYPE_$(1) := linux endif @@ -219,6 +223,42 @@ $$(LIBUV_DIR_$(1))/Release/libuv.a: $$(LIBUV_DEPS) $$(LIBUV_MAKEFILE_$(1)) \ endif +################################################################################ +# jemalloc +################################################################################ + +ifdef CFG_ENABLE_FAST_MAKE +JEMALLOC_DEPS := $(S)/.gitmodules +else +JEMALLOC_DEPS := $(wildcard \ + $(S)src/jemalloc/* \ + $(S)src/jemalloc/*/* \ + $(S)src/jemalloc/*/*/* \ + $(S)src/jemalloc/*/*/*/*) +endif + +JEMALLOC_NAME_$(1) := $$(call CFG_STATIC_LIB_NAME_$(1),jemalloc) +ifeq ($$(CFG_WINDOWSY_$(1)),1) + JEMALLOC_REAL_NAME_$(1) := $$(call CFG_STATIC_LIB_NAME_$(1),jemalloc) +else + JEMALLOC_REAL_NAME_$(1) := $$(call CFG_STATIC_LIB_NAME_$(1),jemalloc_pic) +endif +JEMALLOC_LIB_$(1) := $$(RT_OUTPUT_DIR_$(1))/$$(JEMALLOC_NAME_$(1)) +JEMALLOC_BUILD_DIR_$(1) := $$(RT_OUTPUT_DIR_$(1))/jemalloc + +$$(JEMALLOC_LIB_$(1)): $$(JEMALLOC_DEPS) $$(MKFILE_DEPS) + @$$(call E, make: jemalloc) + cd "$(S)src/jemalloc"; autoconf + cd "$$(JEMALLOC_BUILD_DIR_$(1))"; "$(S)src/jemalloc/configure" \ + $$(JEMALLOC_ARGS_$(1)) --with-jemalloc-prefix=je --enable-autogen \ + --disable-experimental --build=$(CFG_BUILD_TRIPLE) --host=$(1) \ + CC="$$(CC_$(1))" \ + AR="$$(AR_$(1))" \ + RANLIB="$$(AR_$(1)) s" \ + EXTRA_CFLAGS="$$(CFG_GCCISH_CFLAGS)" + $$(Q)$$(MAKE) -C "$$(JEMALLOC_BUILD_DIR_$(1))" build_lib_static + $$(Q)cp $$(JEMALLOC_BUILD_DIR_$(1))/lib/$$(JEMALLOC_REAL_NAME_$(1)) $$(JEMALLOC_LIB_$(1)) + ################################################################################ # compiler-rt ################################################################################ diff --git a/mk/tests.mk b/mk/tests.mk index 9fc1c7390ccd0..0d799d5b61e24 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -240,6 +240,7 @@ ALL_HS := $(filter-out $(S)src/rt/vg/valgrind.h \ tidy: @$(call E, check: formatting) $(Q)find $(S)src -name '*.r[sc]' \ + | grep '^$(S)src/jemalloc' -v \ | grep '^$(S)src/libuv' -v \ | grep '^$(S)src/llvm' -v \ | grep '^$(S)src/gyp' -v \ @@ -264,8 +265,9 @@ tidy: $(Q)find $(S)src -type f -perm +111 \ -not -name '*.rs' -and -not -name '*.py' \ -and -not -name '*.sh' \ - | grep '^$(S)src/llvm' -v \ + | grep '^$(S)src/jemalloc' -v \ | grep '^$(S)src/libuv' -v \ + | grep '^$(S)src/llvm' -v \ | grep '^$(S)src/gyp' -v \ | grep '^$(S)src/etc' -v \ | grep '^$(S)src/doc' -v \ diff --git a/src/jemalloc b/src/jemalloc new file mode 160000 index 0000000000000..46c0af68bd248 --- /dev/null +++ b/src/jemalloc @@ -0,0 +1 @@ +Subproject commit 46c0af68bd248b04df75e4f92d5fb804c3d75340 diff --git a/src/libstd/rt/heap.rs b/src/libstd/rt/heap.rs new file mode 100644 index 0000000000000..b69be7f984791 --- /dev/null +++ b/src/libstd/rt/heap.rs @@ -0,0 +1,93 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use intrinsics::{abort, cttz32}; +use libc::{c_int, c_void, size_t}; +use ptr::RawPtr; + +#[link(name = "jemalloc", kind = "static")] +#[link(name = "pthread")] +extern { + fn jemallocx(size: size_t, flags: c_int) -> *mut c_void; + fn jerallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void; + fn jexallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t; + fn jedallocx(ptr: *mut c_void, flags: c_int); + fn jenallocx(size: size_t, flags: c_int) -> size_t; +} + +// MALLOCX_ALIGN(a) macro +#[inline(always)] +fn mallocx_align(a: uint) -> c_int { unsafe { cttz32(a as u32) as c_int } } + +/// Return a pointer to `size` bytes of memory. +/// +/// Behavior is undefined if the requested size is 0 or the alignment is not a power of 2. The +/// alignment must be no larger than the largest supported page size on the platform. +#[inline] +pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 { + let ptr = jemallocx(size as size_t, mallocx_align(align)) as *mut u8; + if ptr.is_null() { + abort() + } + ptr +} + +/// Extend or shrink the allocation referenced by `ptr` to `size` bytes of memory. +/// +/// Behavior is undefined if the requested size is 0 or the alignment is not a power of 2. The +/// alignment must be no larger than the largest supported page size on the platform. +/// +/// The `old_size` and `align` parameters are the parameters that were used to create the +/// allocation referenced by `ptr`. The `old_size` parameter may also be the value returned by +/// `usable_size` for the requested size. +#[inline] +#[allow(unused_variable)] // for the parameter names in the documentation +pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint, old_size: uint) -> *mut u8 { + let ptr = jerallocx(ptr as *mut c_void, size as size_t, mallocx_align(align)) as *mut u8; + if ptr.is_null() { + abort() + } + ptr +} + +/// Extend or shrink the allocation referenced by `ptr` to `size` bytes of memory in-place. +/// +/// Return true if successful, otherwise false if the allocation was not altered. +/// +/// Behavior is undefined if the requested size is 0 or the alignment is not a power of 2. The +/// alignment must be no larger than the largest supported page size on the platform. +/// +/// The `old_size` and `align` parameters are the parameters that were used to +/// create the allocation referenced by `ptr`. The `old_size` parameter may be +/// any value in range_inclusive(requested_size, usable_size). +#[inline] +#[allow(unused_variable)] // for the parameter names in the documentation +pub unsafe fn reallocate_inplace(ptr: *mut u8, size: uint, align: uint, old_size: uint) -> bool { + jexallocx(ptr as *mut c_void, size as size_t, 0, mallocx_align(align)) == size as size_t +} + +/// Deallocate the memory referenced by `ptr`. +/// +/// The `ptr` parameter must not be null. +/// +/// The `size` and `align` parameters are the parameters that were used to create the +/// allocation referenced by `ptr`. The `size` parameter may also be the value returned by +/// `usable_size` for the requested size. +#[inline] +#[allow(unused_variable)] // for the parameter names in the documentation +pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) { + jedallocx(ptr as *mut c_void, mallocx_align(align)) +} + +/// Return the usable size of an allocation created with the specified the `size` and `align`. +#[inline] +pub fn usable_size(size: uint, align: uint) -> uint { + unsafe { jenallocx(size as size_t, mallocx_align(align)) as uint } +} diff --git a/src/libstd/rt/mod.rs b/src/libstd/rt/mod.rs index a61443d335acb..905afceb50c8a 100644 --- a/src/libstd/rt/mod.rs +++ b/src/libstd/rt/mod.rs @@ -88,6 +88,9 @@ mod macros; /// The global (exchange) heap. pub mod global_heap; +/// The low-level memory allocation API. +pub mod heap; + /// Implementations of language-critical runtime features like @. pub mod task; From 15672a56f2e709b727e0d0c349006af839d83609 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Fri, 25 Apr 2014 21:24:51 -0400 Subject: [PATCH 2/2] add an align parameter to exchange_malloc --- src/libarena/lib.rs | 30 ++++++++++++++++++++++++++---- src/librustc/middle/trans/base.rs | 5 +++-- src/librustc/middle/trans/expr.rs | 5 +++-- src/librustc/middle/trans/tvec.rs | 4 +++- src/libstd/rt/global_heap.rs | 19 ++++++++++++++++++- 5 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/libarena/lib.rs b/src/libarena/lib.rs index ec14820fc5db9..bf7fa2d74229d 100644 --- a/src/libarena/lib.rs +++ b/src/libarena/lib.rs @@ -30,6 +30,7 @@ use std::cast::{transmute, transmute_mut, transmute_mut_lifetime}; use std::cast; use std::cell::{Cell, RefCell}; use std::mem; +use std::mem::min_align_of; use std::ptr::read; use std::cmp; use std::num; @@ -204,7 +205,7 @@ impl Arena { #[inline] fn alloc_copy<'a, T>(&'a mut self, op: || -> T) -> &'a T { unsafe { - let ptr = self.alloc_copy_inner(mem::size_of::(), mem::min_align_of::()); + let ptr = self.alloc_copy_inner(mem::size_of::(), min_align_of::()); let ptr: *mut T = transmute(ptr); mem::move_val_init(&mut (*ptr), op()); return transmute(ptr); @@ -261,7 +262,7 @@ impl Arena { unsafe { let tydesc = get_tydesc::(); let (ty_ptr, ptr) = - self.alloc_noncopy_inner(mem::size_of::(), mem::min_align_of::()); + self.alloc_noncopy_inner(mem::size_of::(), min_align_of::()); let ty_ptr: *mut uint = transmute(ty_ptr); let ptr: *mut T = transmute(ptr); // Write in our tydesc along with a bit indicating that it @@ -354,9 +355,10 @@ struct TypedArenaChunk { impl TypedArenaChunk { #[inline] + #[cfg(stage0)] fn new(next: Option<~TypedArenaChunk>, capacity: uint) -> ~TypedArenaChunk { let mut size = mem::size_of::>(); - size = round_up(size, mem::min_align_of::()); + size = round_up(size, min_align_of::()); let elem_size = mem::size_of::(); let elems_size = elem_size.checked_mul(&capacity).unwrap(); size = size.checked_add(&elems_size).unwrap(); @@ -372,6 +374,26 @@ impl TypedArenaChunk { chunk } + #[inline] + #[cfg(not(stage0))] + fn new(next: Option<~TypedArenaChunk>, capacity: uint) -> ~TypedArenaChunk { + let mut size = mem::size_of::>(); + size = round_up(size, mem::min_align_of::()); + let elem_size = mem::size_of::(); + let elems_size = elem_size.checked_mul(&capacity).unwrap(); + size = size.checked_add(&elems_size).unwrap(); + + let mut chunk = unsafe { + let chunk = global_heap::exchange_malloc(size, min_align_of::>()); + let mut chunk: ~TypedArenaChunk = cast::transmute(chunk); + mem::move_val_init(&mut chunk.next, next); + chunk + }; + + chunk.capacity = capacity; + chunk + } + /// Destroys this arena chunk. If the type descriptor is supplied, the /// drop glue is called; otherwise, drop glue is not called. #[inline] @@ -401,7 +423,7 @@ impl TypedArenaChunk { fn start(&self) -> *u8 { let this: *TypedArenaChunk = self; unsafe { - cast::transmute(round_up(this.offset(1) as uint, mem::min_align_of::())) + cast::transmute(round_up(this.offset(1) as uint, min_align_of::())) } } diff --git a/src/librustc/middle/trans/base.rs b/src/librustc/middle/trans/base.rs index fda51e744ce09..882d6c3e1c4cc 100644 --- a/src/librustc/middle/trans/base.rs +++ b/src/librustc/middle/trans/base.rs @@ -352,7 +352,8 @@ fn require_alloc_fn(bcx: &Block, info_ty: ty::t, it: LangItem) -> ast::DefId { pub fn malloc_raw_dyn<'a>(bcx: &'a Block<'a>, ptr_ty: ty::t, - size: ValueRef) + size: ValueRef, + align: ValueRef) -> Result<'a> { let _icx = push_ctxt("malloc_raw_exchange"); let ccx = bcx.ccx(); @@ -360,7 +361,7 @@ pub fn malloc_raw_dyn<'a>(bcx: &'a Block<'a>, // Allocate space: let r = callee::trans_lang_call(bcx, require_alloc_fn(bcx, ptr_ty, ExchangeMallocFnLangItem), - [size], + [size, align], None); let llty_ptr = type_of::type_of(ccx, ptr_ty); diff --git a/src/librustc/middle/trans/expr.rs b/src/librustc/middle/trans/expr.rs index 9bb0375bf44d2..37ee367de54f7 100644 --- a/src/librustc/middle/trans/expr.rs +++ b/src/librustc/middle/trans/expr.rs @@ -67,7 +67,7 @@ use middle::typeck::MethodCall; use util::common::indenter; use util::ppaux::Repr; use util::nodemap::NodeMap; -use middle::trans::machine::{llsize_of, llsize_of_alloc}; +use middle::trans::machine::{llalign_of_min, llsize_of, llsize_of_alloc}; use middle::trans::type_::Type; use syntax::ast; @@ -1170,10 +1170,11 @@ fn trans_uniq_expr<'a>(bcx: &'a Block<'a>, let fcx = bcx.fcx; let llty = type_of::type_of(bcx.ccx(), contents_ty); let size = llsize_of(bcx.ccx(), llty); + let align = C_uint(bcx.ccx(), llalign_of_min(bcx.ccx(), llty) as uint); // We need to a make a pointer type because box_ty is ty_bot // if content_ty is, e.g. ~fail!(). let real_box_ty = ty::mk_uniq(bcx.tcx(), contents_ty); - let Result { bcx, val } = malloc_raw_dyn(bcx, real_box_ty, size); + let Result { bcx, val } = malloc_raw_dyn(bcx, real_box_ty, size, align); // Unique boxes do not allocate for zero-size types. The standard library may assume // that `free` is never called on the pointer returned for `~ZeroSizeType`. let bcx = if llsize_of_alloc(bcx.ccx(), llty) == 0 { diff --git a/src/librustc/middle/trans/tvec.rs b/src/librustc/middle/trans/tvec.rs index 4bcb609df616c..d56eeb5f8f023 100644 --- a/src/librustc/middle/trans/tvec.rs +++ b/src/librustc/middle/trans/tvec.rs @@ -278,7 +278,9 @@ pub fn trans_uniq_vstore<'a>(bcx: &'a Block<'a>, let vecsize = Add(bcx, alloc, llsize_of(ccx, ccx.opaque_vec_type)); - let Result { bcx: bcx, val: val } = malloc_raw_dyn(bcx, vec_ty, vecsize); + // ~[T] is not going to be changed to support alignment, since it's obsolete. + let align = C_uint(ccx, 8); + let Result { bcx: bcx, val: val } = malloc_raw_dyn(bcx, vec_ty, vecsize, align); Store(bcx, fill, GEPi(bcx, val, [0u, abi::vec_elt_fill])); Store(bcx, alloc, GEPi(bcx, val, [0u, abi::vec_elt_alloc])); diff --git a/src/libstd/rt/global_heap.rs b/src/libstd/rt/global_heap.rs index 01949a7057bc4..60341b4500e63 100644 --- a/src/libstd/rt/global_heap.rs +++ b/src/libstd/rt/global_heap.rs @@ -65,7 +65,7 @@ pub unsafe fn realloc_raw(ptr: *mut u8, size: uint) -> *mut u8 { } /// The allocator for unique pointers without contained managed pointers. -#[cfg(not(test))] +#[cfg(not(test), stage0)] #[lang="exchange_malloc"] #[inline] pub unsafe fn exchange_malloc(size: uint) -> *mut u8 { @@ -81,6 +81,23 @@ pub unsafe fn exchange_malloc(size: uint) -> *mut u8 { } } +/// The allocator for unique pointers without contained managed pointers. +#[cfg(not(test), not(stage0))] +#[lang="exchange_malloc"] +#[inline] +pub unsafe fn exchange_malloc(size: uint, _align: uint) -> *mut u8 { + // The compiler never calls `exchange_free` on ~ZeroSizeType, so zero-size + // allocations can point to this `static`. It would be incorrect to use a null + // pointer, due to enums assuming types like unique pointers are never null. + static EMPTY: () = (); + + if size == 0 { + &EMPTY as *() as *mut u8 + } else { + malloc_raw(size) + } +} + // FIXME: #7496 #[cfg(not(test))] #[lang="closure_exchange_malloc"]