diff --git a/src/array.c b/src/array.c index c6cefbebceb20..86b1056ef4d07 100644 --- a/src/array.c +++ b/src/array.c @@ -59,15 +59,6 @@ JL_DLLEXPORT char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset; } -STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT -{ - if (a->flags.how == 3) { - a = (jl_array_t*)jl_array_data_owner(a); - assert(jl_is_string(a) || a->flags.how != 3); - } - return (jl_value_t*)a; -} - #if defined(_P64) && defined(UINT128MAX) typedef __uint128_t wideint_t; #else @@ -1198,69 +1189,11 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary) return new_ary; } -// Copy element by element until we hit a young object, at which point -// we can finish by using `memmove`. -static NOINLINE ssize_t jl_array_ptr_copy_forward(jl_value_t *owner, - void **src_p, void **dest_p, - ssize_t n) JL_NOTSAFEPOINT -{ - _Atomic(void*) *src_pa = (_Atomic(void*)*)src_p; - _Atomic(void*) *dest_pa = (_Atomic(void*)*)dest_p; - for (ssize_t i = 0; i < n; i++) { - void *val = jl_atomic_load_relaxed(src_pa + i); - jl_atomic_store_release(dest_pa + i, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { - jl_gc_queue_root(owner); - return i; - } - } - return n; -} - -static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner, - void **src_p, void **dest_p, - ssize_t n) JL_NOTSAFEPOINT -{ - _Atomic(void*) *src_pa = (_Atomic(void*)*)src_p; - _Atomic(void*) *dest_pa = (_Atomic(void*)*)dest_p; - for (ssize_t i = 0; i < n; i++) { - void *val = jl_atomic_load_relaxed(src_pa + n - i - 1); - jl_atomic_store_release(dest_pa + n - i - 1, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { - jl_gc_queue_root(owner); - return i; - } - } - return n; -} - // Unsafe, assume inbounds and that dest and src have the same eltype JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT { - assert(dest->flags.ptrarray && src->flags.ptrarray); - jl_value_t *owner = jl_array_owner(dest); - // Destination is old and doesn't refer to any young object - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { - jl_value_t *src_owner = jl_array_owner(src); - // Source is young or being promoted or might refer to young objects - // (i.e. source is not an old object that doesn't have wb triggered) - if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { - ssize_t done; - if (dest_p < src_p || dest_p > src_p + n) { - done = jl_array_ptr_copy_forward(owner, src_p, dest_p, n); - dest_p += done; - src_p += done; - } - else { - done = jl_array_ptr_copy_backward(owner, src_p, dest_p, n); - } - n -= done; - } - } - memmove_refs(dest_p, src_p, n); + jl_gc_array_ptr_copy(dest, dest_p, src, src_p, n); } JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item) diff --git a/src/gc.c b/src/gc.c index 69ec08b6cf9d4..0f6c13777e265 100644 --- a/src/gc.c +++ b/src/gc.c @@ -114,6 +114,71 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_external_free(jl_gc_cb_notify_external_fre jl_gc_deregister_callback(&gc_cblist_notify_external_free, (jl_gc_cb_func_t)cb); } +// Copy element by element until we hit a young object, at which point +// we can finish by using `memmove`. +static NOINLINE ssize_t jl_array_ptr_copy_forward(jl_value_t *owner, + void **src_p, void **dest_p, + ssize_t n) JL_NOTSAFEPOINT +{ + _Atomic(void*) *src_pa = (_Atomic(void*)*)src_p; + _Atomic(void*) *dest_pa = (_Atomic(void*)*)dest_p; + for (ssize_t i = 0; i < n; i++) { + void *val = jl_atomic_load_relaxed(src_pa + i); + jl_atomic_store_release(dest_pa + i, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { + jl_gc_queue_root(owner); + return i; + } + } + return n; +} + +static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner, + void **src_p, void **dest_p, + ssize_t n) JL_NOTSAFEPOINT +{ + _Atomic(void*) *src_pa = (_Atomic(void*)*)src_p; + _Atomic(void*) *dest_pa = (_Atomic(void*)*)dest_p; + for (ssize_t i = 0; i < n; i++) { + void *val = jl_atomic_load_relaxed(src_pa + n - i - 1); + jl_atomic_store_release(dest_pa + n - i - 1, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { + jl_gc_queue_root(owner); + return i; + } + } + return n; +} + +// Unsafe, assume inbounds and that dest and src have the same eltype +JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, + jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT +{ + assert(dest->flags.ptrarray && src->flags.ptrarray); + jl_value_t *owner = jl_array_owner(dest); + // Destination is old and doesn't refer to any young object + if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { + jl_value_t *src_owner = jl_array_owner(src); + // Source is young or being promoted or might refer to young objects + // (i.e. source is not an old object that doesn't have wb triggered) + if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { + ssize_t done; + if (dest_p < src_p || dest_p > src_p + n) { + done = jl_array_ptr_copy_forward(owner, src_p, dest_p, n); + dest_p += done; + src_p += done; + } + else { + done = jl_array_ptr_copy_backward(owner, src_p, dest_p, n); + } + n -= done; + } + } + memmove_refs(dest_p, src_p, n); +} + // Perm gen allocator // 2M pool #define GC_PERM_POOL_SIZE (2 * 1024 * 1024) diff --git a/src/julia.h b/src/julia.h index 710fd11cf9372..2396b7a38a00d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1345,6 +1345,14 @@ STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT return jl_is_array_type(t); } +STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + if (a->flags.how == 3) { + a = (jl_array_t*)jl_array_data_owner(a); + assert(jl_is_string(a) || a->flags.how != 3); + } + return (jl_value_t*)a; +} STATIC_INLINE int jl_is_opaque_closure_type(void *t) JL_NOTSAFEPOINT { diff --git a/src/julia_internal.h b/src/julia_internal.h index 65e1966385039..6d456b470a116 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -630,6 +630,8 @@ STATIC_INLINE void jl_gc_wb_buf(void *parent, void *bufptr, size_t minsz) JL_NOT } #endif // MMTK_GC +JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT; + void jl_gc_debug_print_status(void) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_gc_debug_critical_error(void) JL_NOTSAFEPOINT; void jl_print_gc_stats(JL_STREAM *s); diff --git a/src/mmtk-gc.c b/src/mmtk-gc.c index 06a0a028303ab..9dc21c2ad48db 100644 --- a/src/mmtk-gc.c +++ b/src/mmtk-gc.c @@ -480,6 +480,12 @@ void objprofile_reset(void) { } +JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT +{ + jl_ptls_t ptls = jl_current_task->ptls; + mmtk_memory_region_copy(ptls->mmtk_mutator_ptr, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n); +} + // No inline write barrier -- only used for debugging JL_DLLEXPORT void jl_gc_wb1_noinline(const void *parent) JL_NOTSAFEPOINT {