Grab bag of runtime optimizations

Makefile.in

@@ -96,7 +96,8 @@ ifdef CFG_DISABLE_OPTIMIZE
   $(info cfg: disabling rustc optimization (CFG_DISABLE_OPTIMIZE))
   CFG_RUSTC_FLAGS +=
 else
-  CFG_RUSTC_FLAGS += -O
+  # The rtopt cfg turns off runtime sanity checks
+  CFG_RUSTC_FLAGS += -O --cfg rtopt
 endif
 
 ifdef CFG_ENABLE_DEBUG

src/libstd/macros.rs

@@ -27,8 +27,10 @@ macro_rules! rtdebug (
 
 macro_rules! rtassert (
     ( $arg:expr ) => ( {
-        if !$arg {
-            rtabort!("assertion failed: %s", stringify!($arg));
+        if ::rt::util::ENFORCE_SANITY {
+            if !$arg {
+                rtabort!("assertion failed: %s", stringify!($arg));
+            }
         }
     } )
 )

src/libstd/rt/comm.rs

@@ -125,7 +125,7 @@ impl<T> ChanOne<T> {
         unsafe {
 
             // Install the payload
-            assert!((*packet).payload.is_none());
+            rtassert!((*packet).payload.is_none());
             (*packet).payload = Some(val);
 
             // Atomically swap out the old state to figure out what
@@ -144,16 +144,8 @@ impl<T> ChanOne<T> {
             match oldstate {
                 STATE_BOTH => {
                     // Port is not waiting yet. Nothing to do
-                    do Local::borrow::<Scheduler, ()> |sched| {
-                        rtdebug!("non-rendezvous send");
-                        sched.metrics.non_rendezvous_sends += 1;
-                    }
                 }
                 STATE_ONE => {
-                    do Local::borrow::<Scheduler, ()> |sched| {
-                        rtdebug!("rendezvous send");
-                        sched.metrics.rendezvous_sends += 1;
-                    }
                     // Port has closed. Need to clean up.
                     let _packet: ~Packet<T> = cast::transmute(this.void_packet);
                     recvr_active = false;
@@ -251,7 +243,6 @@ impl<T> SelectInner for PortOne<T> {
                 STATE_BOTH => {
                     // Data has not been sent. Now we're blocked.
                     rtdebug!("non-rendezvous recv");
-                    sched.metrics.non_rendezvous_recvs += 1;
                     false
                 }
                 STATE_ONE => {
@@ -267,7 +258,6 @@ impl<T> SelectInner for PortOne<T> {
                     (*self.packet()).state.store(STATE_ONE, Relaxed);
 
                     rtdebug!("rendezvous recv");
-                    sched.metrics.rendezvous_recvs += 1;
 
                     // Channel is closed. Switch back and check the data.
                     // NB: We have to drop back into the scheduler event loop here
@@ -307,7 +297,7 @@ impl<T> SelectInner for PortOne<T> {
                         STATE_ONE  => true, // Lost the race. Data available.
                         same_ptr   => {
                             // We successfully unblocked our task pointer.
-                            assert!(task_as_state == same_ptr);
+                            rtassert!(task_as_state == same_ptr);
                             let handle = BlockedTask::cast_from_uint(task_as_state);
                             // Because we are already awake, the handle we
                             // gave to this port shall already be empty.
@@ -341,7 +331,8 @@ impl<T> SelectPortInner<T> for PortOne<T> {
         unsafe {
             // See corresponding store() above in block_on for rationale.
             // FIXME(#8130) This can happen only in test builds.
-            assert!((*packet).state.load(Relaxed) == STATE_ONE);
+            // This load is not required for correctness and may be compiled out.
+            rtassert!((*packet).state.load(Relaxed) == STATE_ONE);
 
             let payload = (*packet).payload.take();
 
@@ -387,7 +378,7 @@ impl<T> Drop for ChanOne<T> {
                 },
                 task_as_state => {
                     // The port is blocked waiting for a message we will never send. Wake it.
-                    assert!((*this.packet()).payload.is_none());
+                    rtassert!((*this.packet()).payload.is_none());
                     let recvr = BlockedTask::cast_from_uint(task_as_state);
                     do recvr.wake().map_move |woken_task| {
                         Scheduler::run_task(woken_task);

src/libstd/rt/local.rs

@@ -21,12 +21,15 @@ pub trait Local {
     fn take() -> ~Self;
     fn exists() -> bool;
     fn borrow<T>(f: &fn(&mut Self) -> T) -> T;
+    unsafe fn unsafe_take() -> ~Self;
     unsafe fn unsafe_borrow() -> *mut Self;
     unsafe fn try_unsafe_borrow() -> Option<*mut Self>;
 }
 
 impl Local for Task {
+    #[inline]
     fn put(value: ~Task) { unsafe { local_ptr::put(value) } }
+    #[inline]
     fn take() -> ~Task { unsafe { local_ptr::take() } }
     fn exists() -> bool { local_ptr::exists() }
     fn borrow<T>(f: &fn(&mut Task) -> T) -> T {
@@ -43,7 +46,11 @@ impl Local for Task {
             None => { rtabort!("function failed in local_borrow") }
         }
     }
+    #[inline]
+    unsafe fn unsafe_take() -> ~Task { local_ptr::unsafe_take() }
+    #[inline]
     unsafe fn unsafe_borrow() -> *mut Task { local_ptr::unsafe_borrow() }
+    #[inline]
     unsafe fn try_unsafe_borrow() -> Option<*mut Task> {
         local_ptr::try_unsafe_borrow()
     }
@@ -57,12 +64,12 @@ impl Local for Scheduler {
             task.sched = Some(value.take());
         };
     }
+    #[inline]
     fn take() -> ~Scheduler {
-        do Local::borrow::<Task,~Scheduler> |task| {
-            let sched = task.sched.take_unwrap();
-            let task = task;
-            task.sched = None;
-            sched
+        unsafe {
+            // XXX: Unsafe for speed
+            let task = Local::unsafe_borrow::<Task>();
+            (*task).sched.take_unwrap()
         }
     }
     fn exists() -> bool {
@@ -85,6 +92,7 @@ impl Local for Scheduler {
             }
         }
     }
+    unsafe fn unsafe_take() -> ~Scheduler { rtabort!("unimpl") }
     unsafe fn unsafe_borrow() -> *mut Scheduler {
         match (*Local::unsafe_borrow::<Task>()).sched {
             Some(~ref mut sched) => {
@@ -97,10 +105,17 @@ impl Local for Scheduler {
         }
     }
     unsafe fn try_unsafe_borrow() -> Option<*mut Scheduler> {
-        if Local::exists::<Scheduler>() {
-            Some(Local::unsafe_borrow())
-        } else {
-            None
+        match Local::try_unsafe_borrow::<Task>() {
+            Some(task) => {
+                match (*task).sched {
+                    Some(~ref mut sched) => {
+                        let s: *mut Scheduler = &mut *sched;
+                        Some(s)
+                    }
+                    None => None
+                }
+            }
+            None => None
         }
     }
 }
@@ -111,6 +126,8 @@ impl Local for IoFactoryObject {
     fn take() -> ~IoFactoryObject { rtabort!("unimpl") }
     fn exists() -> bool { rtabort!("unimpl") }
     fn borrow<T>(_f: &fn(&mut IoFactoryObject) -> T) -> T { rtabort!("unimpl") }
+
+    unsafe fn unsafe_take() -> ~IoFactoryObject { rtabort!("unimpl") }
     unsafe fn unsafe_borrow() -> *mut IoFactoryObject {
         let sched = Local::unsafe_borrow::<Scheduler>();
         let io: *mut IoFactoryObject = (*sched).event_loop.io().unwrap();

src/libstd/rt/local_ptr.rs

@@ -23,14 +23,16 @@ use option::{Option, Some, None};
 use unstable::finally::Finally;
 use tls = rt::thread_local_storage;
 
+static mut RT_TLS_KEY: tls::Key = -1;
+
 /// Initialize the TLS key. Other ops will fail if this isn't executed first.
 #[fixed_stack_segment]
 #[inline(never)]
 pub fn init_tls_key() {
     unsafe {
-        rust_initialize_rt_tls_key();
+        rust_initialize_rt_tls_key(&mut RT_TLS_KEY);
         extern {
-            fn rust_initialize_rt_tls_key();
+            fn rust_initialize_rt_tls_key(key: *mut tls::Key);
         }
     }
 }
@@ -40,6 +42,7 @@ pub fn init_tls_key() {
 /// # Safety note
 ///
 /// Does not validate the pointer type.
+#[inline]
 pub unsafe fn put<T>(sched: ~T) {
     let key = tls_key();
     let void_ptr: *mut c_void = cast::transmute(sched);
@@ -51,6 +54,7 @@ pub unsafe fn put<T>(sched: ~T) {
 /// # Safety note
 ///
 /// Does not validate the pointer type.
+#[inline]
 pub unsafe fn take<T>() -> ~T {
     let key = tls_key();
     let void_ptr: *mut c_void = tls::get(key);
@@ -62,6 +66,23 @@ pub unsafe fn take<T>() -> ~T {
     return ptr;
 }
 
+/// Take ownership of a pointer from thread-local storage.
+///
+/// # Safety note
+///
+/// Does not validate the pointer type.
+/// Leaves the old pointer in TLS for speed.
+#[inline]
+pub unsafe fn unsafe_take<T>() -> ~T {
+    let key = tls_key();
+    let void_ptr: *mut c_void = tls::get(key);
+    if void_ptr.is_null() {
+        rtabort!("thread-local pointer is null. bogus!");
+    }
+    let ptr: ~T = cast::transmute(void_ptr);
+    return ptr;
+}
+
 /// Check whether there is a thread-local pointer installed.
 pub fn exists() -> bool {
     unsafe {
@@ -99,10 +120,15 @@ pub unsafe fn borrow<T>(f: &fn(&mut T)) {
 /// Because this leaves the value in thread-local storage it is possible
 /// For the Scheduler pointer to be aliased
 pub unsafe fn unsafe_borrow<T>() -> *mut T {
-    match try_unsafe_borrow() {
-        Some(p) => p,
-        None => rtabort!("thread-local pointer is null. bogus!")
+    let key = tls_key();
+    let mut void_ptr: *mut c_void = tls::get(key);
+    if void_ptr.is_null() {
+        rtabort!("thread-local pointer is null. bogus!")
     }
+    let ptr: *mut *mut c_void = &mut void_ptr;
+    let ptr: *mut ~T = ptr as *mut ~T;
+    let ptr: *mut T = &mut **ptr;
+    return ptr;
 }
 
 pub unsafe fn try_unsafe_borrow<T>() -> Option<*mut T> {
@@ -119,22 +145,18 @@ pub unsafe fn try_unsafe_borrow<T>() -> Option<*mut T> {
     }
 }
 
+#[inline]
 fn tls_key() -> tls::Key {
     match maybe_tls_key() {
         Some(key) => key,
         None => rtabort!("runtime tls key not initialized")
     }
 }
 
-#[fixed_stack_segment]
-#[inline(never)]
-fn maybe_tls_key() -> Option<tls::Key> {
+#[inline]
+#[cfg(not(test))]
+pub fn maybe_tls_key() -> Option<tls::Key> {
     unsafe {
-        let key: *mut c_void = rust_get_rt_tls_key();
-        let key: &mut tls::Key = cast::transmute(key);
-        let key = *key;
-        // Check that the key has been initialized.
-
         // NB: This is a little racy because, while the key is
         // initalized under a mutex and it's assumed to be initalized
         // in the Scheduler ctor by any thread that needs to use it,
@@ -145,14 +167,19 @@ fn maybe_tls_key() -> Option<tls::Key> {
         // another thread. I think this is fine since the only action
         // they could take if it was initialized would be to check the
         // thread-local value and see that it's not set.
-        if key != -1 {
-            return Some(key);
+        if RT_TLS_KEY != -1 {
+            return Some(RT_TLS_KEY);
         } else {
             return None;
         }
     }
+}
 
-    extern {
-        fn rust_get_rt_tls_key() -> *mut c_void;
-    }
+// XXX: The boundary between the running runtime and the testing runtime
+// seems to be fuzzy at the moment, and trying to use two different keys
+// results in disaster. This should not be necessary.
+#[inline]
+#[cfg(test)]
+pub fn maybe_tls_key() -> Option<tls::Key> {
+    unsafe { ::cast::transmute(::realstd::rt::local_ptr::maybe_tls_key()) }
 }

src/libstd/rt/message_queue.rs

@@ -16,33 +16,66 @@ use kinds::Send;
 use vec::OwnedVector;
 use cell::Cell;
 use option::*;
-use unstable::sync::Exclusive;
+use unstable::sync::{UnsafeAtomicRcBox, LittleLock};
 use clone::Clone;
 
 pub struct MessageQueue<T> {
-    // XXX: Another mystery bug fixed by boxing this lock
-    priv queue: ~Exclusive<~[T]>
+    priv state: UnsafeAtomicRcBox<State<T>>
+}
+
+struct State<T> {
+    count: uint,
+    queue: ~[T],
+    lock: LittleLock
 }
 
 impl<T: Send> MessageQueue<T> {
     pub fn new() -> MessageQueue<T> {
         MessageQueue {
-            queue: ~Exclusive::new(~[])
+            state: UnsafeAtomicRcBox::new(State {
+                count: 0,
+                queue: ~[],
+                lock: LittleLock::new()
+            })
         }
     }
 
     pub fn push(&mut self, value: T) {
         unsafe {
             let value = Cell::new(value);
-            self.queue.with(|q| q.push(value.take()) );
+            let state = self.state.get();
+            do (*state).lock.lock {
+                (*state).count += 1;
+                (*state).queue.push(value.take());
+            }
         }
     }
 
     pub fn pop(&mut self) -> Option<T> {
         unsafe {
-            do self.queue.with |q| {
-                if !q.is_empty() {
-                    Some(q.shift())
+            let state = self.state.get();
+            do (*state).lock.lock {
+                if !(*state).queue.is_empty() {
+                    (*state).count += 1;
+                    Some((*state).queue.shift())
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    /// A pop that may sometimes miss enqueued elements, but is much faster
+    /// to give up without doing any synchronization
+    pub fn casual_pop(&mut self) -> Option<T> {
+        unsafe {
+            let state = self.state.get();
+            // NB: Unsynchronized check
+            if (*state).count == 0 { return None; }
+            do (*state).lock.lock {
+                if !(*state).queue.is_empty() {
+                    (*state).count += 1;
+                    Some((*state).queue.shift())
                 } else {
                     None
                 }
@@ -51,10 +84,10 @@ impl<T: Send> MessageQueue<T> {
     }
 }
 
-impl<T> Clone for MessageQueue<T> {
+impl<T: Send> Clone for MessageQueue<T> {
     fn clone(&self) -> MessageQueue<T> {
         MessageQueue {
-            queue: self.queue.clone()
+            state: self.state.clone()
         }
     }
 }