Improve MPI_Waitsome performance

by avoiding extra atomic exchanges.

The fix is based on MPI spec:
    12.4.2 Multiple threads completing the same request. A program in which two threads block, waiting
on the same request, is erroneous. Similarly, the same request cannot appear in the
array of requests of two concurrent MPI_{WAIT|TEST}{ANY|SOME|ALL} calls. In MPI, a
request can only be completed once. Any combination of wait or test that violates this rule
is erroneous."

We add marked flag to the request structure. Only MPI_Waitsome thread will use/access it by any means.
PML threads will not see/touch it. So given that any particular request can be used no more than in one
MPI_Waitsome we are safe to do this change.

Author: artpol84

ompi/request/req_wait.c

@@ -367,8 +367,8 @@ int ompi_request_default_wait_some(size_t count,
     ompi_request_t **rptr = NULL;
     ompi_request_t *request = NULL;
     ompi_wait_sync_t sync;
-    size_t sync_sets = 0, sync_unsets = 0;
-    
+    bool will_be_signalled = false;
+
     WAIT_SYNC_INIT(&sync, 1);
 
     *outcount = 0;
@@ -390,9 +390,12 @@ int ompi_request_default_wait_some(size_t count,
         if( !OPAL_ATOMIC_CMPSET_PTR(&request->req_complete, REQUEST_PENDING, &sync) ) {
             /* If the request is completed go ahead and mark it as such */
             assert( REQUEST_COMPLETE(request) );
+            /* TODO: make sure MPI spec is not strict about index order */
+            indices[num_requests_done] = i;
             num_requests_done++;
+            REQUEST_UNMARK(request);
         } else {
-            sync_sets++;
+            REQUEST_MARK(request);
         }
     }
 
@@ -412,29 +415,29 @@ int ompi_request_default_wait_some(size_t count,
     /* Clean up the synchronization primitives */
 
     rptr = requests;
-    num_requests_done = 0;
     for (size_t i = 0; i < count; i++, rptr++) {
         request = *rptr;
 
-        if( request->req_state == OMPI_REQUEST_INACTIVE ) {
+        /* Skip inactive and already accounted requests */
+        if( request->req_state == OMPI_REQUEST_INACTIVE || !REQUEST_MARKED(request) ) {
             continue;
         }
-        /* Atomically mark the request as pending. If this succeed
-         * then the request was not completed, and it is now marked as
-         * pending. Otherwise, the request is complete )either it was
-         * before or it has been meanwhile). The major drawback here
-         * is that we will do all the atomics operations in all cases.
+        /* Atomically mark the request as pending. 
+         * If this succeed - then the request was not completed, 
+         * and it is now marked as pending.
+         * Otherwise, the request is complete meanwhile.
          */
-        if( !OPAL_ATOMIC_CMPSET_PTR(&request->req_complete, &sync, REQUEST_PENDING) ) {
+         if( !OPAL_ATOMIC_CMPSET_PTR(&request->req_complete, &sync, REQUEST_PENDING) ) {
             indices[num_requests_done] = i;
             num_requests_done++;
-        } else {
-            /* request wasn't finished during this call */
-            sync_unsets++;
-        }
+            /* at least one of requests was completed during this call
+             * corresponding thread will signal us
+             */
+            will_be_signalled = true;
+        } 
     }
 
-    if( sync_sets == sync_unsets ){
+    if( !will_be_signalled ){
         /* nobody knows about us,
          * set signa-in-progress flag to false
          */

ompi/request/request.h

@@ -15,6 +15,7 @@
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2016      Mellanox Technologies. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -112,6 +113,7 @@ struct ompi_request_t {
     ompi_request_complete_fn_t req_complete_cb; /**< Called when the request is MPI completed */
     void *req_complete_cb_data;
     ompi_mpi_object_t req_mpi_object;           /**< Pointer to MPI object that created this request */
+    bool marked;
 };
 
 /**
@@ -151,6 +153,12 @@ typedef struct ompi_predefined_request_t ompi_predefined_request_t;
 
 
 #define REQUEST_COMPLETE(req)        (REQUEST_COMPLETED == (req)->req_complete)
+
+#define REQUEST_MARK(req)  ( (req)->marked = true )
+#define REQUEST_UNMARK(req)  ( (req)->marked = false )
+#define REQUEST_MARKED(req)  ( (req)->marked )
+
+
 /**
  * Finalize a request.  This is a macro to avoid function call
  * overhead, since this is typically invoked in the critical