Skip to content

Commit c5cf418

Browse files
devrealdevreal
committed
Atomics: add relaxed load and store
The _Atomic qualifier requires accesses to such variables to ensure sequential consistency, leading to unnecessary atomic operations. OPAL_ATOMIC_RELAXED_STORE and OPAL_ATOMIC_RELAXED_LOAD avoid atomic operations in paths where atomicity and memory ordering is not required. Signed-off-by: Joseph Schuchart <[email protected]>
1 parent c21033e commit c5cf418

File tree

7 files changed

+38
-27
lines changed

7 files changed

+38
-27
lines changed

‎ompi/mca/pml/ob1/pml_ob1_recvfrag.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,7 @@ void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
594594
iov,
595595
&iov_count,
596596
&bytes_received );
597-
match->req_bytes_received = bytes_received;
597+
OPAL_ATOMIC_RELAXED_STORE(&match->req_bytes_received, bytes_received);
598598
SPC_USER_OR_MPI(match->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
599599
OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
600600
/*

‎ompi/mca/pml/ob1/pml_ob1_recvreq.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,9 +1280,9 @@ void mca_pml_ob1_recv_req_start(mca_pml_ob1_recv_request_t *req)
12801280
#endif
12811281

12821282
/* init/re-init the request */
1283-
req->req_lock = 0;
1284-
req->req_pipeline_depth = 0;
1285-
req->req_bytes_received = 0;
1283+
OPAL_ATOMIC_RELAXED_STORE(&req->req_lock, 0);
1284+
OPAL_ATOMIC_RELAXED_STORE(&req->req_pipeline_depth, 0);
1285+
OPAL_ATOMIC_RELAXED_STORE(&req->req_bytes_received, 0);
12861286
req->req_bytes_expected = 0;
12871287
/* What about req_rdma_cnt ? */
12881288
req->req_rdma_idx = 0;

‎opal/class/opal_lifo.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -295,15 +295,15 @@ static inline opal_list_item_t *opal_lifo_push_st(opal_lifo_t *lifo, opal_list_i
295295
{
296296
item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
297297
item->item_free = 0;
298-
lifo->opal_lifo_head.data.item = (intptr_t) item;
298+
OPAL_ATOMIC_RELAXED_STORE(&lifo->opal_lifo_head.data.item, (intptr_t) item);
299299
return (opal_list_item_t *) item->opal_list_next;
300300
}
301301

302302
static inline opal_list_item_t *opal_lifo_pop_st(opal_lifo_t *lifo)
303303
{
304304
opal_list_item_t *item;
305-
item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
306-
lifo->opal_lifo_head.data.item = (intptr_t) item->opal_list_next;
305+
item = (opal_list_item_t *) OPAL_ATOMIC_RELAXED_LOAD(&lifo->opal_lifo_head.data.item);
306+
OPAL_ATOMIC_RELAXED_STORE(&lifo->opal_lifo_head.data.item, (intptr_t) item->opal_list_next);
307307
if (item == &lifo->opal_lifo_ghost) {
308308
return NULL;
309309
}

‎opal/class/opal_object.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t *type, const char *
377377
opal_class_initialize((type)); \
378378
} \
379379
((opal_object_t *) (object))->obj_class = (type); \
380-
((opal_object_t *) (object))->obj_reference_count = 1; \
380+
OPAL_ATOMIC_RELAXED_STORE(&((opal_object_t *) (object))->obj_reference_count, 1); \
381381
opal_obj_run_constructors((opal_object_t *) (object)); \
382382
OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__); \
383383
} while (0)

‎opal/include/opal/sys/atomic.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,12 @@ static inline void opal_atomic_sc_ptr(opal_atomic_intptr_t *addr, intptr_t newva
463463
# define OPAL_HAVE_ATOMIC_LLSC_PTR 0
464464
#endif
465465

466+
/****** Relaxed load and store ********/
467+
468+
#if !defined(OPAL_ATOMIC_HAVE_RELAXED_LOAD_STORE)
469+
#define OPAL_ATOMIC_RELAXED_LOAD(a) *(a)
470+
#define OPAL_ATOMIC_RELAXED_STORE(a, v) *(a) = (v);
471+
#endif // !OPAL_ATOMIC_HAVE_RELAXED_LOAD_STORE
466472
END_C_DECLS
467473

468474
#endif /* OPAL_SYS_ATOMIC_H */

‎opal/include/opal/sys/atomic_stdc.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,4 +212,8 @@ OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
212212

213213
#include "opal/sys/atomic_impl_minmax_math.h"
214214

215+
#define OPAL_ATOMIC_HAVE_RELAXED_LOAD_STORE 1
216+
#define OPAL_ATOMIC_RELAXED_LOAD(a) atomic_load_explicit((a), memory_order_relaxed)
217+
#define OPAL_ATOMIC_RELAXED_STORE(a, v) atomic_store_explicit((a), (v), memory_order_relaxed)
218+
215219
#endif /* !defined(OPAL_ATOMIC_STDC_H) */

‎opal/mca/threads/thread_usage.h

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,9 @@ static inline bool opal_set_using_threads(bool have)
101101
return opal_atomic_##name##_fetch_##suffix(addr, delta); \
102102
} \
103103
\
104-
*addr = *addr operator delta; \
105-
return *addr; \
104+
type value = OPAL_ATOMIC_RELAXED_LOAD(addr) operator delta; \
105+
OPAL_ATOMIC_RELAXED_STORE(addr, value); \
106+
return value; \
106107
} \
107108
\
108109
static inline type opal_thread_fetch_##name##_##suffix(opal_atomic_##type *addr, type delta) \
@@ -111,8 +112,8 @@ static inline bool opal_set_using_threads(bool have)
111112
return opal_atomic_fetch_##name##_##suffix(addr, delta); \
112113
} \
113114
\
114-
type old = *addr; \
115-
*addr = old operator delta; \
115+
type old = OPAL_ATOMIC_RELAXED_LOAD(addr); \
116+
OPAL_ATOMIC_RELAXED_STORE(addr, old operator delta); \
116117
return old; \
117118
}
118119

@@ -124,28 +125,28 @@ static inline bool opal_set_using_threads(bool have)
124125
return opal_atomic_compare_exchange_strong_##suffix(addr, (addr_type *) compare, \
125126
(addr_type) value); \
126127
} \
127-
\
128-
if ((type) *addr == *compare) { \
129-
((type *) addr)[0] = value; \
128+
type old = OPAL_ATOMIC_RELAXED_LOAD(addr); \
129+
if (old == *compare) { \
130+
OPAL_ATOMIC_RELAXED_STORE(addr, value); \
130131
return true; \
131132
} \
132133
\
133-
*compare = ((type *) addr)[0]; \
134+
*compare = old; \
134135
\
135136
return false; \
136137
}
137138

138-
#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \
139-
static inline type opal_thread_swap_##suffix(opal_atomic_##addr_type *ptr, type newvalue) \
140-
{ \
141-
if (opal_using_threads()) { \
142-
return (type) opal_atomic_swap_##suffix(ptr, (addr_type) newvalue); \
143-
} \
144-
\
145-
type old = ((type *) ptr)[0]; \
146-
((type *) ptr)[0] = newvalue; \
147-
\
148-
return old; \
139+
#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \
140+
static inline type opal_thread_swap_##suffix(opal_atomic_##addr_type *addr, type newvalue) \
141+
{ \
142+
if (opal_using_threads()) { \
143+
return (type) opal_atomic_swap_##suffix(addr, (addr_type) newvalue); \
144+
} \
145+
\
146+
type old = OPAL_ATOMIC_RELAXED_LOAD(addr); \
147+
OPAL_ATOMIC_RELAXED_STORE(addr, newvalue); \
148+
\
149+
return old; \
149150
}
150151

151152
OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32)

0 commit comments

Comments
 (0)