opal/asm: add opal_atomic_compare_exchange_strong functions

ompi/datatype/ompi_datatype_args.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2016 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -487,7 +487,8 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
     void* recursive_buffer;
 
     if (NULL == packed_description) {
-        if (opal_atomic_bool_cmpset (&datatype->packed_description, NULL, (void *) 1)) {
+        void *_tmp_ptr = NULL;
+        if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (void *) &_tmp_ptr, (void *) 1)) {
             if( ompi_datatype_is_predefined(datatype) ) {
                 packed_description = malloc(2 * sizeof(int));
             } else if( NULL == args ) {

ompi/group/group.h

@@ -14,7 +14,7 @@
  * Copyright (c) 2007-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -356,7 +356,7 @@ static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group,
         ompi_proc_t *real_proc =
             (ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc));
 
-        if (opal_atomic_bool_cmpset_ptr (group->grp_proc_pointers + peer_id, proc, real_proc)) {
+        if (opal_atomic_compare_exchange_strong_ptr (group->grp_proc_pointers + peer_id, &proc, real_proc)) {
             OBJ_RETAIN(real_proc);
         }
 

ompi/mca/mtl/portals4/mtl_portals4_flowctl.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -296,9 +296,10 @@ ompi_mtl_portals4_flowctl_add_procs(size_t me,
 int
 ompi_mtl_portals4_flowctl_trigger(void)
 {
+    int32_t _tmp_value = 0;
     int ret;
 
-    if (true == OPAL_ATOMIC_BOOL_CMPSET_32(&ompi_mtl_portals4.flowctl.flowctl_active, false, true)) {
+    if (true == OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32(&ompi_mtl_portals4.flowctl.flowctl_active, &_tmp_value, 1)) {
         /* send trigger to root */
         ret = PtlPut(ompi_mtl_portals4.zero_md_h,
                      0,

ompi/mca/osc/pt2pt/osc_pt2pt.h

@@ -8,7 +8,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2016 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
@@ -145,15 +145,11 @@ static inline bool ompi_osc_pt2pt_peer_eager_active (ompi_osc_pt2pt_peer_t *peer
 
 static inline void ompi_osc_pt2pt_peer_set_flag (ompi_osc_pt2pt_peer_t *peer, int32_t flag, bool value)
 {
-    int32_t peer_flags, new_flags;
-    do {
-        peer_flags = peer->flags;
-        if (value) {
-            new_flags = peer_flags | flag;
-        } else {
-            new_flags = peer_flags & ~flag;
-        }
-    } while (!OPAL_ATOMIC_BOOL_CMPSET_32 (&peer->flags, peer_flags, new_flags));
+    if (value) {
+        OPAL_ATOMIC_OR32 (&peer->flags, flag);
+    } else {
+        OPAL_ATOMIC_AND32 (&peer->flags, ~flag);
+    }
 }
 
 static inline void ompi_osc_pt2pt_peer_set_locked (ompi_osc_pt2pt_peer_t *peer, bool value)

ompi/mca/osc/pt2pt/osc_pt2pt_frag.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -105,7 +105,7 @@ static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, om
                          "osc pt2pt: flushing active fragment to target %d. pending: %d",
                          active_frag->target, active_frag->pending));
 
-    if (opal_atomic_bool_cmpset (&peer->active_frag, active_frag, NULL)) {
+    if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &active_frag, NULL)) {
         if (0 != OPAL_THREAD_ADD32(&active_frag->pending, -1)) {
             /* communication going on while synchronizing; this is an rma usage bug */
             return OMPI_ERR_RMA_SYNC;

ompi/mca/osc/pt2pt/osc_pt2pt_frag.h

@@ -67,7 +67,7 @@ static inline ompi_osc_pt2pt_frag_t *ompi_osc_pt2pt_frag_alloc_non_buffered (omp
 
     /* to ensure ordering flush the buffer on the peer */
     curr = peer->active_frag;
-    if (NULL != curr && opal_atomic_bool_cmpset (&peer->active_frag, curr, NULL)) {
+    if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &curr, NULL)) {
         /* If there's something pending, the pending finish will
            start the buffer.  Otherwise, we need to start it now. */
         int ret = ompi_osc_pt2pt_frag_finish (module, curr);

ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c

@@ -744,14 +744,13 @@ static bool ompi_osc_pt2pt_lock_try_acquire (ompi_osc_pt2pt_module_t* module, in
                 break;
             }
 
-            if (opal_atomic_bool_cmpset_32 (&module->lock_status, lock_status, lock_status + 1)) {
+            if (opal_atomic_compare_exchange_strong_32 (&module->lock_status, &lock_status, lock_status + 1)) {
                 break;
             }
-
-            lock_status = module->lock_status;
         } while (1);
     } else {
-        queue = !opal_atomic_bool_cmpset_32 (&module->lock_status, 0, -1);
+        int32_t _tmp_value = 0;
+        queue = !opal_atomic_compare_exchange_strong_32 (&module->lock_status, &_tmp_value, -1);
     }
 
     if (queue) {

ompi/mca/osc/rdma/osc_rdma_active_target.c

@@ -8,7 +8,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
@@ -285,7 +285,9 @@ int ompi_osc_rdma_post_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
                 ret = ompi_osc_rdma_lock_btl_cswap (module, peer, target, 0, 1 + (int64_t) my_rank, &result);
                 assert (OMPI_SUCCESS == ret);
             } else {
-                result = !ompi_osc_rdma_lock_cmpset ((osc_rdma_counter_t *) target, 0, 1 + (osc_rdma_counter_t) my_rank);
+                ompi_osc_rdma_lock_t _tmp_value = 0;
+
+                result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_counter_t *) target, &_tmp_value, 1 + (osc_rdma_counter_t) my_rank);
             }
 
             if (OPAL_LIKELY(0 == result)) {

ompi/mca/osc/rdma/osc_rdma_lock.h

@@ -17,7 +17,8 @@
 
 static inline int ompi_osc_rdma_trylock_local (volatile ompi_osc_rdma_lock_t *lock)
 {
-    return !ompi_osc_rdma_lock_cmpset (lock, 0, OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
+    ompi_osc_rdma_lock_t _tmp_value = 0;
+    return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
 }
 
 static inline void ompi_osc_rdma_unlock_local (volatile ompi_osc_rdma_lock_t *lock)

ompi/mca/osc/rdma/osc_rdma_peer.h

@@ -201,14 +201,13 @@ static inline bool ompi_osc_rdma_peer_test_set_flag (ompi_osc_rdma_peer_t *peer,
     int32_t flags;
 
     opal_atomic_mb ();
+    flags = peer->flags;
 
     do {
-        flags = peer->flags;
         if (flags & flag) {
             return false;
         }
-
-    } while (!OPAL_THREAD_BOOL_CMPSET_32 (&peer->flags, flags, flags | flag));
+    } while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 (&peer->flags, &flags, flags | flag));
 
     return true;
 }

ompi/mca/osc/rdma/osc_rdma_types.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2014-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -54,12 +54,12 @@ static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value
     return new;
 }
 
-static inline int ompi_osc_rdma_lock_cmpset (volatile int64_t *p, int64_t comp, int64_t value)
+static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int64_t *comp, int64_t value)
 {
     int ret;
 
     opal_atomic_mb ();
-    ret = opal_atomic_bool_cmpset_64 (p, comp, value);
+    ret = opal_atomic_compare_exchange_strong_64 (p, comp, value);
     opal_atomic_mb ();
 
     return ret;
@@ -83,12 +83,12 @@ static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value
     return new;
 }
 
-static inline int ompi_osc_rdma_lock_cmpset (volatile int32_t *p, int32_t comp, int32_t value)
+static inline int ompi_osc_rdma_lock_compare_exchange (volatile int32_t *p, int32_t *comp, int32_t value)
 {
     int ret;
 
     opal_atomic_mb ();
-    ret = opal_atomic_bool_cmpset_32 (p, comp, value);
+    ret = opal_atomic_compare_exchange_strong_32 (p, comp, value);
     opal_atomic_mb ();
 
     return ret;

ompi/mca/osc/sm/osc_sm_active_target.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014-2017 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
@@ -130,10 +130,11 @@ ompi_osc_sm_start(struct ompi_group_t *group,
     ompi_osc_sm_module_t *module =
         (ompi_osc_sm_module_t*) win->w_osc_module;
     int my_rank = ompi_comm_rank (module->comm);
+    void *_tmp_ptr = NULL;
 
     OBJ_RETAIN(group);
 
-    if (!OPAL_ATOMIC_BOOL_CMPSET_PTR(&module->start_group, NULL, group)) {
+    if (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&module->start_group, (void *) &_tmp_ptr, group)) {
         OBJ_RELEASE(group);
         return OMPI_ERR_RMA_SYNC;
     }
@@ -160,9 +161,11 @@ ompi_osc_sm_start(struct ompi_group_t *group,
 
             opal_atomic_rmb ();
 
-            do {
-                old = module->posts[my_rank][rank_byte];
-            } while (!opal_atomic_bool_cmpset ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, old, old ^ rank_bit));
+#if OPAL_HAVE_ATOMIC_MATH_64
+            opal_atomic_xor_64 ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, rank_bit);
+#else
+            opal_atomic_xor_32 ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, rank_bit);
+#endif
        }
 
         free (ranks);
@@ -185,7 +188,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win)
     opal_atomic_mb();
 
     group = module->start_group;
-    if (NULL == group || !OPAL_ATOMIC_BOOL_CMPSET_PTR(&module->start_group, group, NULL)) {
+    if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&module->start_group, &group, NULL)) {
         return OMPI_ERR_RMA_SYNC;
     }
 

ompi/request/req_wait.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Mellanox Technologies. All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
@@ -100,6 +100,8 @@ int ompi_request_default_wait_any(size_t count,
 
     num_requests_null_inactive = 0;
     for (i = 0; i < count; i++) {
+        void *_tmp_ptr = REQUEST_PENDING;
+
         request = requests[i];
 
         /* Check for null or completed persistent request. For
@@ -110,7 +112,7 @@ int ompi_request_default_wait_any(size_t count,
             continue;
         }
 
-        if( !OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, REQUEST_PENDING, &sync) ) {
+        if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync) ) {
             assert(REQUEST_COMPLETE(request));
             completed = i;
             *index = i;
@@ -136,6 +138,8 @@ int ompi_request_default_wait_any(size_t count,
      * user.
      */
     for(i = completed-1; (i+1) > 0; i--) {
+        void *tmp_ptr = &sync;
+
         request = requests[i];
 
         if( request->req_state == OMPI_REQUEST_INACTIVE ) {
@@ -146,7 +150,7 @@ int ompi_request_default_wait_any(size_t count,
          * Otherwise, the request has been completed meanwhile, and it
          * has been atomically marked as REQUEST_COMPLETE.
          */
-        if( !OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, &sync, REQUEST_PENDING) ) {
+        if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &tmp_ptr, REQUEST_PENDING) ) {
             *index = i;
         }
     }
@@ -211,14 +215,16 @@ int ompi_request_default_wait_all( size_t count,
     WAIT_SYNC_INIT(&sync, count);
     rptr = requests;
     for (i = 0; i < count; i++) {
+        void *_tmp_ptr = REQUEST_PENDING;
+
         request = *rptr++;
 
         if( request->req_state == OMPI_REQUEST_INACTIVE ) {
             completed++;
             continue;
         }
 
-        if (!OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, REQUEST_PENDING, &sync)) {
+        if (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync)) {
             if( OPAL_UNLIKELY( MPI_SUCCESS != request->req_status.MPI_ERROR ) ) {
                 failed++;
             }
@@ -246,6 +252,8 @@ int ompi_request_default_wait_all( size_t count,
     if (MPI_STATUSES_IGNORE != statuses) {
         /* fill out status and free request if required */
         for( i = 0; i < count; i++, rptr++ ) {
+            void *_tmp_ptr = &sync;
+
             request = *rptr;
 
             if( request->req_state == OMPI_REQUEST_INACTIVE ) {
@@ -260,7 +268,7 @@ int ompi_request_default_wait_all( size_t count,
                  * mark the request as pending then it is neither failed nor complete, and
                  * we must stop altering it.
                  */
-                if( OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, &sync, REQUEST_PENDING ) ) {
+                if( OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING ) ) {
                     /*
                      * Per MPI 2.2 p 60:
                      * Allows requests to be marked as MPI_ERR_PENDING if they are
@@ -306,6 +314,8 @@ int ompi_request_default_wait_all( size_t count,
         int rc;
         /* free request if required */
         for( i = 0; i < count; i++, rptr++ ) {
+            void *_tmp_ptr = &sync;
+
             request = *rptr;
 
             if( request->req_state == OMPI_REQUEST_INACTIVE ) {
@@ -320,7 +330,7 @@ int ompi_request_default_wait_all( size_t count,
                 /* If the request is still pending due to a failed request
                  * then skip it in this loop.
                  */
-                 if( OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, &sync, REQUEST_PENDING ) ) {
+                 if( OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING ) ) {
                     /*
                      * Per MPI 2.2 p 60:
                      * Allows requests to be marked as MPI_ERR_PENDING if they are
@@ -398,6 +408,8 @@ int ompi_request_default_wait_some(size_t count,
     num_requests_null_inactive = 0;
     num_requests_done = 0;
     for (size_t i = 0; i < count; i++, rptr++) {
+        void *_tmp_ptr = REQUEST_PENDING;
+
         request = *rptr;
         /*
          * Check for null or completed persistent request.
@@ -407,7 +419,7 @@ int ompi_request_default_wait_some(size_t count,
             num_requests_null_inactive++;
             continue;
         }
-        indices[i] = OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, REQUEST_PENDING, &sync);
+        indices[i] = OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, &sync);
         if( !indices[i] ) {
             /* If the request is completed go ahead and mark it as such */
             assert( REQUEST_COMPLETE(request) );
@@ -434,6 +446,8 @@ int ompi_request_default_wait_some(size_t count,
     rptr = requests;
     num_requests_done = 0;
     for (size_t i = 0; i < count; i++, rptr++) {
+        void *_tmp_ptr = &sync;
+
         request = *rptr;
 
         if( request->req_state == OMPI_REQUEST_INACTIVE ) {
@@ -454,7 +468,7 @@ int ompi_request_default_wait_some(size_t count,
          */
         if( !indices[i] ){
             indices[num_requests_done++] = i;
-        } else if( !OPAL_ATOMIC_BOOL_CMPSET_PTR(&request->req_complete, &sync, REQUEST_PENDING) ) {
+        } else if( !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&request->req_complete, &_tmp_ptr, REQUEST_PENDING) ) {
             indices[num_requests_done++] = i;
         }
     }