Add atomic fetch-and-op and compare-exchange functions

ompi/datatype/ompi_datatype_args.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2016 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -378,7 +378,7 @@ int32_t ompi_datatype_copy_args( const ompi_datatype_t* source_data,
      * a read only memory).
      */
     if( NULL != pArgs ) {
-        OPAL_THREAD_ADD32(&pArgs->ref_count, 1);
+        OPAL_THREAD_ADD_FETCH32(&pArgs->ref_count, 1);
         dest_data->args = pArgs;
     }
     return OMPI_SUCCESS;
@@ -396,7 +396,7 @@ int32_t ompi_datatype_release_args( ompi_datatype_t* pData )
     ompi_datatype_args_t* pArgs = (ompi_datatype_args_t*)pData->args;
 
     assert( 0 < pArgs->ref_count );
-    OPAL_THREAD_ADD32(&pArgs->ref_count, -1);
+    OPAL_THREAD_ADD_FETCH32(&pArgs->ref_count, -1);
     if( 0 == pArgs->ref_count ) {
         /* There are some duplicated datatypes around that have a pointer to this
          * args. We will release them only when the last datatype will dissapear.
@@ -487,7 +487,8 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
     void* recursive_buffer;
 
     if (NULL == packed_description) {
-        if (opal_atomic_bool_cmpset (&datatype->packed_description, NULL, (void *) 1)) {
+        void *_tmp_ptr = NULL;
+        if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (void *) &_tmp_ptr, (void *) 1)) {
             if( ompi_datatype_is_predefined(datatype) ) {
                 packed_description = malloc(2 * sizeof(int));
             } else if( NULL == args ) {

ompi/group/group.h

@@ -14,7 +14,7 @@
  * Copyright (c) 2007-2017 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2017 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -356,7 +356,7 @@ static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group,
         ompi_proc_t *real_proc =
             (ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc));
 
-        if (opal_atomic_bool_cmpset_ptr (group->grp_proc_pointers + peer_id, proc, real_proc)) {
+        if (opal_atomic_compare_exchange_strong_ptr (group->grp_proc_pointers + peer_id, &proc, real_proc)) {
             OBJ_RETAIN(real_proc);
         }
 

ompi/mca/coll/libnbc/coll_libnbc_component.c

@@ -314,7 +314,7 @@ libnbc_module_destruct(ompi_coll_libnbc_module_t *module)
     /* if we ever were used for a collective op, do the progress cleanup. */
     if (true == module->comm_registered) {
         int32_t tmp =
-            OPAL_THREAD_ADD32(&mca_coll_libnbc_component.active_comms, -1);
+            OPAL_THREAD_ADD_FETCH32(&mca_coll_libnbc_component.active_comms, -1);
         if (0 == tmp) {
             opal_progress_unregister(ompi_coll_libnbc_progress);
         }

ompi/mca/coll/libnbc/nbc.c

@@ -618,7 +618,7 @@ int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t
   /* register progress */
   if (need_register) {
       int32_t tmp =
-          OPAL_THREAD_ADD32(&mca_coll_libnbc_component.active_comms, 1);
+          OPAL_THREAD_ADD_FETCH32(&mca_coll_libnbc_component.active_comms, 1);
       if (tmp == 1) {
           opal_progress_register(ompi_coll_libnbc_progress);
       }

ompi/mca/coll/monitoring/coll_monitoring_component.c

@@ -120,7 +120,7 @@ static int
 mca_coll_monitoring_module_enable(mca_coll_base_module_t*module, struct ompi_communicator_t*comm)
 {
     mca_coll_monitoring_module_t*monitoring_module = (mca_coll_monitoring_module_t*) module;
-    if( 1 == opal_atomic_add_32(&monitoring_module->is_initialized, 1) ) {
+    if( 1 == opal_atomic_add_fetch_32(&monitoring_module->is_initialized, 1) ) {
         MONITORING_SAVE_FULL_PREV_COLL_API(monitoring_module, comm);
         monitoring_module->data = mca_common_monitoring_coll_new(comm);
         OPAL_MONITORING_PRINT_INFO("coll_module_enabled");    
@@ -132,7 +132,7 @@ static int
 mca_coll_monitoring_module_disable(mca_coll_base_module_t*module, struct ompi_communicator_t*comm)
 {
     mca_coll_monitoring_module_t*monitoring_module = (mca_coll_monitoring_module_t*) module;
-    if( 0 == opal_atomic_sub_32(&monitoring_module->is_initialized, 1) ) {
+    if( 0 == opal_atomic_sub_fetch_32(&monitoring_module->is_initialized, 1) ) {
         MONITORING_RELEASE_FULL_PREV_COLL_API(monitoring_module, comm);
         mca_common_monitoring_coll_release(monitoring_module->data);
         monitoring_module->data = NULL;

ompi/mca/coll/portals4/coll_portals4_allreduce.c

@@ -68,7 +68,7 @@ allreduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
     zero_md_h = mca_coll_portals4_component.zero_md_h;
     data_md_h = mca_coll_portals4_component.data_md_h;
 
-    internal_count = opal_atomic_add_size_t(&module->coll_count, 1);
+    internal_count = opal_atomic_add_fetch_size_t(&module->coll_count, 1);
 
     /*
      ** DATATYPE and SIZES

ompi/mca/coll/portals4/coll_portals4_barrier.c

@@ -44,7 +44,7 @@ barrier_hypercube_top(struct ompi_communicator_t *comm,
 
     request->type = OMPI_COLL_PORTALS4_TYPE_BARRIER;
 
-    count = opal_atomic_add_size_t(&portals4_module->coll_count, 1);
+    count = opal_atomic_add_fetch_size_t(&portals4_module->coll_count, 1);
 
     ret = PtlCTAlloc(mca_coll_portals4_component.ni_h,
             &request->u.barrier.rtr_ct_h);

ompi/mca/coll/portals4/coll_portals4_bcast.c

@@ -176,7 +176,7 @@ bcast_kary_tree_top(void *buff, int count,
     zero_md_h = mca_coll_portals4_component.zero_md_h;
     data_md_h = mca_coll_portals4_component.data_md_h;
 
-    internal_count = opal_atomic_add_size_t(&portals4_module->coll_count, 1);
+    internal_count = opal_atomic_add_fetch_size_t(&portals4_module->coll_count, 1);
 
 
     /*
@@ -513,7 +513,7 @@ bcast_pipeline_top(void *buff, int count,
     zero_md_h = mca_coll_portals4_component.zero_md_h;
     data_md_h = mca_coll_portals4_component.data_md_h;
 
-    internal_count = opal_atomic_add_size_t(&portals4_module->coll_count, 1);
+    internal_count = opal_atomic_add_fetch_size_t(&portals4_module->coll_count, 1);
 
     /*
      ** DATATYPE and SIZES

ompi/mca/coll/portals4/coll_portals4_gather.c

@@ -582,7 +582,7 @@ ompi_coll_portals4_gather_intra_binomial_top(const void *sbuf, int scount, struc
     /* Setup Common Parameters        */
     /**********************************/
 
-    request->u.gather.coll_count = opal_atomic_add_size_t(&portals4_module->coll_count, 1);
+    request->u.gather.coll_count = opal_atomic_add_fetch_size_t(&portals4_module->coll_count, 1);
 
     COLL_PORTALS4_UPDATE_IN_ORDER_BMTREE( comm, portals4_module, request->u.gather.root_rank );
     bmtree = portals4_module->cached_in_order_bmtree;
@@ -879,7 +879,7 @@ ompi_coll_portals4_gather_intra_linear_top(const void *sbuf, int scount, struct
 
     i_am_root = (request->u.gather.my_rank == request->u.gather.root_rank);
 
-    request->u.gather.coll_count = opal_atomic_add_size_t(&portals4_module->coll_count, 1);
+    request->u.gather.coll_count = opal_atomic_add_fetch_size_t(&portals4_module->coll_count, 1);
 
     ret = setup_gather_buffers_linear(comm, request, portals4_module);
     if (MPI_SUCCESS != ret) { line = __LINE__; goto err_hdlr; }

ompi/mca/coll/portals4/coll_portals4_reduce.c

@@ -69,7 +69,7 @@ reduce_kary_tree_top(const void *sendbuf, void *recvbuf, int count,
     zero_md_h = mca_coll_portals4_component.zero_md_h;
     data_md_h = mca_coll_portals4_component.data_md_h;
 
-    internal_count = opal_atomic_add_size_t(&module->coll_count, 1);
+    internal_count = opal_atomic_add_fetch_size_t(&module->coll_count, 1);
 
     /*
      ** DATATYPE and SIZES

ompi/mca/coll/portals4/coll_portals4_scatter.c

@@ -399,7 +399,7 @@ ompi_coll_portals4_scatter_intra_linear_top(const void *sbuf, int scount, struct
 
     i_am_root = (request->u.scatter.my_rank == request->u.scatter.root_rank);
 
-    request->u.scatter.coll_count = opal_atomic_add_size_t(&portals4_module->coll_count, 1);
+    request->u.scatter.coll_count = opal_atomic_add_fetch_size_t(&portals4_module->coll_count, 1);
 
     ret = setup_scatter_buffers_linear(comm, request, portals4_module);
     if (MPI_SUCCESS != ret) { line = __LINE__; goto err_hdlr; }

ompi/mca/coll/sm/coll_sm.h

@@ -358,7 +358,7 @@ extern uint32_t mca_coll_sm_one;
  * Macro to release an in-use flag from this process
  */
 #define FLAG_RELEASE(flag) \
-    (void)opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1)
+    opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1)
 
 /**
  * Macro to copy a single segment in from a user buffer to a shared

ompi/mca/coll/sm/coll_sm_barrier.c

@@ -101,7 +101,7 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
     if (0 != rank) {
         /* Get parent *in* buffer */
         parent = &data->mcb_barrier_control_parent[buffer_set];
-        (void)opal_atomic_add(parent, 1);
+        opal_atomic_add (parent, 1);
 
         SPIN_CONDITION(0 != *me_out, exit_label2);
         *me_out = 0;

ompi/mca/coll/sm/coll_sm_module.c

@@ -463,7 +463,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
     OBJ_RETAIN(sm_module->previous_reduce_module);
 
     /* Indicate that we have successfully attached and setup */
-    (void)opal_atomic_add(&(data->sm_bootstrap_meta->module_seg->seg_inited), 1);
+    opal_atomic_add (&(data->sm_bootstrap_meta->module_seg->seg_inited), 1);
 
     /* Wait for everyone in this communicator to attach and setup */
     opal_output_verbose(10, ompi_coll_base_framework.framework_output,

ompi/mca/common/monitoring/common_monitoring.c

@@ -209,7 +209,7 @@ static int mca_common_monitoring_comm_size_notify(mca_base_pvar_t *pvar,
 int mca_common_monitoring_init( void )
 {
     if( !mca_common_monitoring_enabled ) return OMPI_ERROR;
-    if( 1 < opal_atomic_add_32(&mca_common_monitoring_hold, 1) ) return OMPI_SUCCESS; /* Already initialized */
+    if( 1 < opal_atomic_add_fetch_32(&mca_common_monitoring_hold, 1) ) return OMPI_SUCCESS; /* Already initialized */
 
     char hostname[OPAL_MAXHOSTNAMELEN] = "NA";
     /* Initialize constant */
@@ -229,7 +229,7 @@ int mca_common_monitoring_init( void )
 void mca_common_monitoring_finalize( void )
 {
     if( ! mca_common_monitoring_enabled || /* Don't release if not last */
-        0 < opal_atomic_sub_32(&mca_common_monitoring_hold, 1) ) return;
+        0 < opal_atomic_sub_fetch_32(&mca_common_monitoring_hold, 1) ) return;
 
     OPAL_MONITORING_PRINT_INFO("common_component_finish");
     /* Dump monitoring informations */
@@ -503,21 +503,21 @@ void mca_common_monitoring_record_pml(int world_rank, size_t data_size, int tag)
 
     /* Keep tracks of the data_size distribution */
     if( 0 == data_size ) {
-        opal_atomic_add_size_t(&size_histogram[world_rank * max_size_histogram], 1);
+        opal_atomic_add_fetch_size_t(&size_histogram[world_rank * max_size_histogram], 1);
     } else {
         int log2_size = log10(data_size)/log10_2;
         if(log2_size > max_size_histogram - 2) /* Avoid out-of-bound write */
             log2_size = max_size_histogram - 2;
-        opal_atomic_add_size_t(&size_histogram[world_rank * max_size_histogram + log2_size + 1], 1);
+        opal_atomic_add_fetch_size_t(&size_histogram[world_rank * max_size_histogram + log2_size + 1], 1);
     }
 
     /* distinguishses positive and negative tags if requested */
     if( (tag < 0) && (mca_common_monitoring_filter()) ) {
-        opal_atomic_add_size_t(&filtered_pml_data[world_rank], data_size);
-        opal_atomic_add_size_t(&filtered_pml_count[world_rank], 1);
+        opal_atomic_add_fetch_size_t(&filtered_pml_data[world_rank], data_size);
+        opal_atomic_add_fetch_size_t(&filtered_pml_count[world_rank], 1);
     } else { /* if filtered monitoring is not activated data is aggregated indifferently */
-        opal_atomic_add_size_t(&pml_data[world_rank], data_size);
-        opal_atomic_add_size_t(&pml_count[world_rank], 1);
+        opal_atomic_add_fetch_size_t(&pml_data[world_rank], data_size);
+        opal_atomic_add_fetch_size_t(&pml_count[world_rank], 1);
     }
 }
 
@@ -564,11 +564,11 @@ void mca_common_monitoring_record_osc(int world_rank, size_t data_size,
     if( 0 == mca_common_monitoring_current_state ) return;  /* right now the monitoring is not started */
 
     if( SEND == dir ) {
-        opal_atomic_add_size_t(&osc_data_s[world_rank], data_size);
-        opal_atomic_add_size_t(&osc_count_s[world_rank], 1);
+        opal_atomic_add_fetch_size_t(&osc_data_s[world_rank], data_size);
+        opal_atomic_add_fetch_size_t(&osc_count_s[world_rank], 1);
     } else {
-        opal_atomic_add_size_t(&osc_data_r[world_rank], data_size);
-        opal_atomic_add_size_t(&osc_count_r[world_rank], 1);
+        opal_atomic_add_fetch_size_t(&osc_data_r[world_rank], data_size);
+        opal_atomic_add_fetch_size_t(&osc_count_r[world_rank], 1);
     }
 }
 
@@ -650,8 +650,8 @@ void mca_common_monitoring_record_coll(int world_rank, size_t data_size)
 {
     if( 0 == mca_common_monitoring_current_state ) return;  /* right now the monitoring is not started */
 
-    opal_atomic_add_size_t(&coll_data[world_rank], data_size);
-    opal_atomic_add_size_t(&coll_count[world_rank], 1);
+    opal_atomic_add_fetch_size_t(&coll_data[world_rank], data_size);
+    opal_atomic_add_fetch_size_t(&coll_count[world_rank], 1);
 }
 
 static int mca_common_monitoring_get_coll_count(const struct mca_base_pvar_t *pvar,

ompi/mca/common/monitoring/common_monitoring_coll.c

@@ -236,8 +236,8 @@ void mca_common_monitoring_coll_o2a(size_t size, mca_monitoring_coll_data_t*data
         return;
     }
 #endif /* OPAL_ENABLE_DEBUG */
-    opal_atomic_add_size_t(&data->o2a_size, size);
-    opal_atomic_add_size_t(&data->o2a_count, 1);
+    opal_atomic_add_fetch_size_t(&data->o2a_size, size);
+    opal_atomic_add_fetch_size_t(&data->o2a_count, 1);
 }
 
 int mca_common_monitoring_coll_get_o2a_count(const struct mca_base_pvar_t *pvar,
@@ -277,8 +277,8 @@ void mca_common_monitoring_coll_a2o(size_t size, mca_monitoring_coll_data_t*data
         return;
     }
 #endif /* OPAL_ENABLE_DEBUG */
-    opal_atomic_add_size_t(&data->a2o_size, size);
-    opal_atomic_add_size_t(&data->a2o_count, 1);
+    opal_atomic_add_fetch_size_t(&data->a2o_size, size);
+    opal_atomic_add_fetch_size_t(&data->a2o_count, 1);
 }
 
 int mca_common_monitoring_coll_get_a2o_count(const struct mca_base_pvar_t *pvar,
@@ -318,8 +318,8 @@ void mca_common_monitoring_coll_a2a(size_t size, mca_monitoring_coll_data_t*data
         return;
     }
 #endif /* OPAL_ENABLE_DEBUG */
-    opal_atomic_add_size_t(&data->a2a_size, size);
-    opal_atomic_add_size_t(&data->a2a_count, 1);
+    opal_atomic_add_fetch_size_t(&data->a2a_size, size);
+    opal_atomic_add_fetch_size_t(&data->a2a_count, 1);
 }
 
 int mca_common_monitoring_coll_get_a2a_count(const struct mca_base_pvar_t *pvar,

ompi/mca/mtl/portals4/mtl_portals4_flowctl.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -296,9 +296,10 @@ ompi_mtl_portals4_flowctl_add_procs(size_t me,
 int
 ompi_mtl_portals4_flowctl_trigger(void)
 {
+    int32_t _tmp_value = 0;
     int ret;
 
-    if (true == OPAL_ATOMIC_BOOL_CMPSET_32(&ompi_mtl_portals4.flowctl.flowctl_active, false, true)) {
+    if (true == OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32(&ompi_mtl_portals4.flowctl.flowctl_active, &_tmp_value, 1)) {
         /* send trigger to root */
         ret = PtlPut(ompi_mtl_portals4.zero_md_h,
                      0,
@@ -346,7 +347,7 @@ start_recover(void)
     int64_t epoch_counter;
 
     ompi_mtl_portals4.flowctl.flowctl_active = true;
-    epoch_counter = opal_atomic_add_64(&ompi_mtl_portals4.flowctl.epoch_counter, 1);
+    epoch_counter = opal_atomic_add_fetch_64(&ompi_mtl_portals4.flowctl.epoch_counter, 1);
 
     opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                         "Entering flowctl_start_recover %ld",

ompi/mca/mtl/portals4/mtl_portals4_recv.c

@@ -53,14 +53,14 @@ read_msg(void *start, ptl_size_t length, ptl_process_t target,
     int32_t frag_count;
 
 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
-    while (OPAL_UNLIKELY(OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, -1) < 0)) {
-        OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
+    while (OPAL_UNLIKELY(OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, -1) < 0)) {
+        OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
         ompi_mtl_portals4_progress();
     }
 #endif
 
     frag_count = (length + ompi_mtl_portals4.max_msg_size_mtl - 1) / ompi_mtl_portals4.max_msg_size_mtl;
-    ret = OPAL_THREAD_ADD32(&(request->pending_reply), frag_count);
+    ret = OPAL_THREAD_ADD_FETCH32(&(request->pending_reply), frag_count);
 
     for (i = 0 ; i < frag_count ; i++) {
         opal_free_list_item_t *tmp;
@@ -385,14 +385,14 @@ ompi_mtl_portals4_rndv_get_frag_progress(ptl_event_t *ev,
     opal_free_list_return (&ompi_mtl_portals4.fl_rndv_get_frag,
                            &rndv_get_frag->super);
 
-    ret = OPAL_THREAD_ADD32(&(ptl_request->pending_reply), -1);
+    ret = OPAL_THREAD_ADD_FETCH32(&(ptl_request->pending_reply), -1);
     if (ret > 0) {
         return OMPI_SUCCESS;
     }
     assert(ptl_request->pending_reply == 0);
 
 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
-    OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
+    OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 #endif
 
     /* make sure the data is in the right place.  Use _ucount for
@@ -468,7 +468,7 @@ ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
     ptl_request->super.type = portals4_req_recv;
     ptl_request->super.event_callback = ompi_mtl_portals4_recv_progress;
 #if OPAL_ENABLE_DEBUG
-    ptl_request->opcount = OPAL_THREAD_ADD64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
+    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
     ptl_request->hdr_data = 0;
 #endif
     ptl_request->buffer_ptr = (free_after) ? start : NULL;
@@ -549,7 +549,7 @@ ompi_mtl_portals4_imrecv(struct mca_mtl_base_module_t* mtl,
     }
 
 #if OPAL_ENABLE_DEBUG
-    ptl_request->opcount = OPAL_THREAD_ADD64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
+    ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1);
     ptl_request->hdr_data = 0;
 #endif
     ptl_request->super.type = portals4_req_recv;