Push pinning roots (#142)

Supporting processing roots in the shadow stack that are not
transitively pinned (only the root object is pinned), and therefore
should enable moving more objects.

NB: Should be merged with mmtk/julia#43.

Author: udesou

.github/scripts/common.sh

@@ -10,6 +10,8 @@ export MMTK_JULIA_DIR=$BINDING_PATH
 # Make sure we have enough heap to build Julia
 export MMTK_MIN_HSIZE_G=0.5
 export MMTK_MAX_HSIZE_G=4
+# Print out a backtrace in case of an error
+export RUST_BACKTRACE=1
 # Make sure we do not get OOM killed.
 total_mem=$(free -m | awk '/^Mem:/ {print $2}')
 export JULIA_TEST_MAXRSS_MB=$total_mem

mmtk/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2018"
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
 julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "09e025b25c65e5842b6eeeb3c81ba4cab97f5192"
+julia_version = "0ac54e680f9a8b57bb1fd8ef04919ce9898ae77b"
 
 [lib]
 crate-type = ["cdylib"]

mmtk/src/julia_scanning.rs

@@ -278,7 +278,9 @@ pub unsafe fn scan_julia_object<EV: EdgeVisitor<JuliaVMEdge>>(obj: Address, clos
 
         let ta = obj.to_ptr::<mmtk_jl_task_t>();
 
-        mmtk_scan_gcstack(ta, closure);
+        // transitively pinnig of stack roots happens during root
+        // processing so it's fine to have only one closure here
+        mmtk_scan_gcstack(ta, closure, None);
 
         let layout = (*jl_task_type).layout;
         debug_assert!((*layout).fielddesc_type() == 0);
@@ -354,9 +356,10 @@ pub unsafe fn scan_julia_object<EV: EdgeVisitor<JuliaVMEdge>>(obj: Address, clos
     }
 }
 
-pub unsafe fn mmtk_scan_gcstack<EV: EdgeVisitor<JuliaVMEdge>>(
+pub unsafe fn mmtk_scan_gcstack<'a, EV: EdgeVisitor<JuliaVMEdge>>(
     ta: *const mmtk_jl_task_t,
-    closure: &mut EV,
+    mut closure: &'a mut EV,
+    mut pclosure: Option<&'a mut EV>,
 ) {
     let stkbuf = (*ta).stkbuf;
     let copy_stack = (*ta).copy_stack_custom();
@@ -385,16 +388,28 @@ pub unsafe fn mmtk_scan_gcstack<EV: EdgeVisitor<JuliaVMEdge>>(
         let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots);
         let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub);
         debug_assert!(nroots.as_usize() as u32 <= UINT32_MAX);
-        let mut nr = nroots >> 2;
+        let mut nr = nroots >> 3;
 
         loop {
+            // if the 'pin' bit on the root type is not set, must transitively pin
+            // and therefore use transitive pinning closure
+            let closure_to_use: &mut &mut EV = if (nroots.as_usize() & 4) == 0 {
+                &mut closure
+            } else {
+                // otherwise, use the pinning closure (if available)
+                match &mut pclosure {
+                    Some(c) => c,
+                    None => &mut closure,
+                }
+            };
+
             let rts = Address::from_mut_ptr(s).shift::<Address>(2);
             let mut i = 0;
             while i < nr {
                 if (nroots.as_usize() & 1) != 0 {
                     let slot = read_stack(rts.shift::<Address>(i as isize), offset, lb, ub);
                     let real_addr = get_stack_addr(slot, offset, lb, ub);
-                    process_edge(closure, real_addr);
+                    process_edge(*closure_to_use, real_addr);
                 } else {
                     let real_addr =
                         get_stack_addr(rts.shift::<Address>(i as isize), offset, lb, ub);
@@ -410,12 +425,12 @@ pub unsafe fn mmtk_scan_gcstack<EV: EdgeVisitor<JuliaVMEdge>>(
 
                     // pointer is not malloced but function is native, so skip it
                     if gc_ptr_tag(slot, 1) {
-                        process_offset_edge(closure, real_addr, 1);
+                        process_offset_edge(*closure_to_use, real_addr, 1);
                         i += 2;
                         continue;
                     }
 
-                    process_edge(closure, real_addr);
+                    process_edge(*closure_to_use, real_addr);
                 }
 
                 i += 1;
@@ -431,7 +446,7 @@ pub unsafe fn mmtk_scan_gcstack<EV: EdgeVisitor<JuliaVMEdge>>(
             let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots);
             let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub);
             nroots = new_nroots;
-            nr = nroots >> 2;
+            nr = nroots >> 3;
             continue;
         }
     }
@@ -447,14 +462,14 @@ pub unsafe fn mmtk_scan_gcstack<EV: EdgeVisitor<JuliaVMEdge>>(
 }
 
 #[inline(always)]
-unsafe fn read_stack(addr: Address, offset: isize, lb: u64, ub: u64) -> Address {
+pub unsafe fn read_stack(addr: Address, offset: isize, lb: u64, ub: u64) -> Address {
     let real_addr = get_stack_addr(addr, offset, lb, ub);
 
     real_addr.load::<Address>()
 }
 
 #[inline(always)]
-fn get_stack_addr(addr: Address, offset: isize, lb: u64, ub: u64) -> Address {
+pub fn get_stack_addr(addr: Address, offset: isize, lb: u64, ub: u64) -> Address {
     if addr.as_usize() >= lb as usize && addr.as_usize() < ub as usize {
         return addr + offset;
     } else {

mmtk/src/scanning.rs

@@ -2,8 +2,8 @@ use crate::edges::JuliaVMEdge;
 use crate::{SINGLETON, UPCALLS};
 use mmtk::memory_manager;
 use mmtk::scheduler::*;
-use mmtk::util::opaque_pointer::*;
 use mmtk::util::ObjectReference;
+use mmtk::util::{opaque_pointer::*, Address};
 use mmtk::vm::edge_shape::Edge;
 use mmtk::vm::EdgeVisitor;
 use mmtk::vm::ObjectTracerContext;
@@ -50,17 +50,21 @@ impl Scanning<JuliaVM> for VMScanning {
 
         use crate::julia_scanning::*;
         use crate::julia_types::*;
-        use mmtk::util::Address;
 
         let ptls: &mut mmtk__jl_tls_states_t = unsafe { std::mem::transmute(mutator.mutator_tls) };
-        let mut edge_buffer = EdgeBuffer { buffer: vec![] }; // need to be tpinned as they're all from the shadow stack
+        let mut tpinning_edge_buffer = EdgeBuffer { buffer: vec![] }; // need to be transitively pinned
+        let mut pinning_edge_buffer = EdgeBuffer { buffer: vec![] }; // roots from the shadow stack that we know that do not need to be transitively pinned
         let mut node_buffer = vec![];
 
         // Scan thread local from ptls: See gc_queue_thread_local in gc.c
         let mut root_scan_task = |task: *const mmtk__jl_task_t, task_is_root: bool| {
             if !task.is_null() {
                 unsafe {
-                    crate::julia_scanning::mmtk_scan_gcstack(task, &mut edge_buffer);
+                    mmtk_scan_gcstack(
+                        task,
+                        &mut tpinning_edge_buffer,
+                        Some(&mut pinning_edge_buffer),
+                    );
                 }
                 if task_is_root {
                     // captures wrong root nodes before creating the work
@@ -127,13 +131,20 @@ impl Scanning<JuliaVM> for VMScanning {
 
         // Push work
         const CAPACITY_PER_PACKET: usize = 4096;
-        for tpinning_roots in edge_buffer
+        for tpinning_roots in tpinning_edge_buffer
             .buffer
             .chunks(CAPACITY_PER_PACKET)
             .map(|c| c.to_vec())
         {
             factory.create_process_tpinning_roots_work(tpinning_roots);
         }
+        for pinning_roots in pinning_edge_buffer
+            .buffer
+            .chunks(CAPACITY_PER_PACKET)
+            .map(|c| c.to_vec())
+        {
+            factory.create_process_pinning_roots_work(pinning_roots);
+        }
         for nodes in node_buffer.chunks(CAPACITY_PER_PACKET).map(|c| c.to_vec()) {
             factory.create_process_pinning_roots_work(nodes);
         }