From bcd376a1158b0960746e592080e430446b5f2218 Mon Sep 17 00:00:00 2001 From: Shintaro Iwasaki Date: Thu, 8 Apr 2021 16:50:04 -0500 Subject: [PATCH] opal/mca/threads/pthreads: remove opal_pthread_mutex_t opal_pthread_mutex_t is not used. This patch removes it. Signed-off-by: Shintaro Iwasaki opal/mca/threads/qthreads,argobots: remove m_recursive m_recursive is not used. This patch removes it. Signed-off-by: Shintaro Iwasaki opal/mca/threads/qthreads,argobots: update wait_sync implementation This patch is corresponding to changes to the following commit for Pthreads: 6a406fb3c9ea74aa371bdff82e8eb01bf9820293 Signed-off-by: Shintaro Iwasaki opal/mca/threads/pthreads: implement opal_thread_internal_mutex/cond_t opal_thread_internal_mutex_t and opal_thread_internal_cond_t are new objects that are directly mapped to raw mutex/condition variable implementations (e.g., pthread_mutex_t for Pthreads). This abstraction is different from opal_mutex_t, which has more functionalities. Introducing this abstraction helps removal of code duplication among other threading layers. Signed-off-by: Shintaro Iwasaki opal/mca/threads/pthreads: use opal_thread_internal_mutex/cond_t opal_thread_internal_mutex/cond routines are static inline functions, so there should be no additional overhead. Signed-off-by: Shintaro Iwasaki opal/mca/threads/argobots: implement opal_thread_internal_mutex/cond_t Signed-off-by: Shintaro Iwasaki opal/mca/threads/argobots: use opal_thread_internal_mutex/cond_t Signed-off-by: Shintaro Iwasaki opal/mca/threads/qthreads: implement opal_thread_internal_mutex/cond_t Signed-off-by: Shintaro Iwasaki opal/mca/threads/qthreads: use opal_thread_internal_mutex/cond_t Signed-off-by: Shintaro Iwasaki opal/mca/threads: move mutex implementation to threads/base Now implementations of opal_mutex_xxx() and opal_cond_xxx() are the same. This patch moves them to base/mutex.c and mutex.h and removes the duplication. Note that those implementations are copied from pthread/threads_pthreads_mutex.c and pthread/threads_pthreads_mutex.h. Signed-off-by: Shintaro Iwasaki opal/mca/threads: use opal_thread_yield() in busy loop if requested This commit is prerequisite for the next commit that unifies the implementation of wait_sync. Nonpreemptive threads like Qthreads and Argobots need a yield in a busy loop to avoid a deadlock, while Pthreads does not. To unify the implementation in the next commit, this patch adds opal_thread_yield() if opal_progress_yield_when_idle is true. At the same time, this patch removes a yield operation after opal_progress() since opal_progress() internally yields when it does not make any progress. Signed-off-by: Shintaro Iwasaki opal/mca/threads: move wait_sync implementation to threads/base Now implementations of wait_sync_xxx() functions and macros are the same. This patch moves them to base/wait_sync.c and wait_sync.h and removes the duplication. Note that those implementations are copied from pthread/threads_pthreads_wait_sync.c and pthread/threads_pthreads_wait_sync.h. Signed-off-by: Shintaro Iwasaki opal/mca/threads: change Pthread-specific names This patch renames variables and functions that are no longer specific to Pthread. Signed-off-by: Shintaro Iwasaki opal/mca/threads: remove MCA_threads_wait_sync_base_include_xxx MCA_threads_wait_sync_base_include_xxx is no longer used. This patch removes them. Signed-off-by: Shintaro Iwasaki opal/mca/threads: minor cleanup - Remove unnecessary "include" - Remove weird empty lines - Clean up parentheses for macros - Remove EDEADLK check for trylock() and unlock() Signed-off-by: Shintaro Iwasaki --- opal/mca/threads/argobots/Makefile.am | 7 +- opal/mca/threads/argobots/configure.m4 | 3 - .../threads/argobots/threads_argobots_mutex.h | 182 ++++++-------- .../argobots/threads_argobots_wait_sync.c | 142 ----------- .../argobots/threads_argobots_wait_sync.h | 114 --------- opal/mca/threads/base/Makefile.am | 5 +- .../threads_argobots_mutex.c => base/mutex.c} | 64 ++--- .../wait_sync.c} | 21 +- opal/mca/threads/mutex.h | 104 +++++++- opal/mca/threads/pthreads/Makefile.am | 5 +- opal/mca/threads/pthreads/configure.m4 | 3 - .../threads/pthreads/threads_pthreads_mutex.c | 128 ---------- .../threads/pthreads/threads_pthreads_mutex.h | 212 ++++++++-------- .../pthreads/threads_pthreads_wait_sync.h | 118 --------- opal/mca/threads/qthreads/Makefile.am | 6 +- opal/mca/threads/qthreads/configure.m4 | 3 - .../threads/qthreads/threads_qthreads_mutex.c | 147 ------------ .../threads/qthreads/threads_qthreads_mutex.h | 227 +++++++----------- .../qthreads/threads_qthreads_wait_sync.c | 115 --------- .../qthreads/threads_qthreads_wait_sync.h | 102 -------- opal/mca/threads/wait_sync.h | 109 ++++++++- 21 files changed, 516 insertions(+), 1301 deletions(-) delete mode 100644 opal/mca/threads/argobots/threads_argobots_wait_sync.c delete mode 100644 opal/mca/threads/argobots/threads_argobots_wait_sync.h rename opal/mca/threads/{argobots/threads_argobots_mutex.c => base/mutex.c} (55%) rename opal/mca/threads/{pthreads/threads_pthreads_wait_sync.c => base/wait_sync.c} (86%) delete mode 100644 opal/mca/threads/pthreads/threads_pthreads_mutex.c delete mode 100644 opal/mca/threads/pthreads/threads_pthreads_wait_sync.h delete mode 100644 opal/mca/threads/qthreads/threads_qthreads_mutex.c delete mode 100644 opal/mca/threads/qthreads/threads_qthreads_wait_sync.c delete mode 100644 opal/mca/threads/qthreads/threads_qthreads_wait_sync.h diff --git a/opal/mca/threads/argobots/Makefile.am b/opal/mca/threads/argobots/Makefile.am index b0c90fd78de..d7a0b9ada2f 100644 --- a/opal/mca/threads/argobots/Makefile.am +++ b/opal/mca/threads/argobots/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2019 Sandia National Laboratories. All rights reserved. +# Copyright (c) 2021 Argonne National Laboratory. All rights reserved. +# # $COPYRIGHT$ # # Additional copyrights may follow @@ -25,12 +27,9 @@ sources = \ threads_argobots_component.c \ threads_argobots_condition.c \ threads_argobots_module.c \ - threads_argobots_mutex.c \ threads_argobots_mutex.h \ threads_argobots_threads.h \ - threads_argobots_tsd.h \ - threads_argobots_wait_sync.c \ - threads_argobots_wait_sync.h + threads_argobots_tsd.h #lib = libmca_threads_argobots.la lib_sources = $(sources) diff --git a/opal/mca/threads/argobots/configure.m4 b/opal/mca/threads/argobots/configure.m4 index 6a02735a8a1..d3fee7482a3 100644 --- a/opal/mca/threads/argobots/configure.m4 +++ b/opal/mca/threads/argobots/configure.m4 @@ -123,9 +123,6 @@ AC_DEFUN([MCA_opal_threads_argobots_POST_CONFIG],[ AC_DEFINE_UNQUOTED([MCA_threads_tsd_base_include_HEADER], ["opal/mca/threads/argobots/threads_argobots_tsd.h"], [Header to include for tsd implementation]) - AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER], - ["opal/mca/threads/argobots/threads_argobots_wait_sync.h"], - [Header to include for wait_sync implementation]) THREAD_CFLAGS="$TPKG_CFLAGS" THREAD_FCFLAGS="$TPKG_FCFLAGS" THREAD_CXXFLAGS="$TPKG_CXXFLAGS" diff --git a/opal/mca/threads/argobots/threads_argobots_mutex.h b/opal/mca/threads/argobots/threads_argobots_mutex.h index adf2b2c8230..a0b2b2d6d85 100644 --- a/opal/mca/threads/argobots/threads_argobots_mutex.h +++ b/opal/mca/threads/argobots/threads_argobots_mutex.h @@ -31,108 +31,71 @@ #include "opal_config.h" -#include #include +#include #include "opal/class/opal_object.h" +#include "opal/constants.h" #include "opal/mca/threads/argobots/threads_argobots.h" #include "opal/mca/threads/mutex.h" -#include "opal/sys/atomic.h" #include "opal/util/output.h" BEGIN_C_DECLS -struct opal_mutex_t { - opal_object_t super; +typedef ABT_mutex_memory opal_thread_internal_mutex_t; - ABT_mutex_memory m_lock_argobots; - int m_recursive; +#define OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER ABT_MUTEX_INITIALIZER +#define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER ABT_RECURSIVE_MUTEX_INITIALIZER -#if OPAL_ENABLE_DEBUG - int m_lock_debug; - const char *m_lock_file; - int m_lock_line; -#endif - - opal_atomic_lock_t m_lock_atomic; -}; - -OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); -OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); - -#if OPAL_ENABLE_DEBUG -# define OPAL_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = ABT_MUTEX_INITIALIZER, \ - .m_recursive = 0, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \ - .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -#else -# define OPAL_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = ABT_MUTEX_INITIALIZER, \ - .m_recursive = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -#endif +static inline int opal_thread_internal_mutex_init(opal_thread_internal_mutex_t *p_mutex, + bool recursive) +{ + if (recursive) { + const ABT_mutex_memory init_mutex = ABT_RECURSIVE_MUTEX_INITIALIZER; + memcpy(p_mutex, &init_mutex, sizeof(ABT_mutex_memory)); + } else { + const ABT_mutex_memory init_mutex = ABT_MUTEX_INITIALIZER; + memcpy(p_mutex, &init_mutex, sizeof(ABT_mutex_memory)); + } + return OPAL_SUCCESS; +} +static inline void opal_thread_internal_mutex_lock(opal_thread_internal_mutex_t *p_mutex) +{ + ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex); #if OPAL_ENABLE_DEBUG -# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ - .m_lock_argobots = ABT_RECURSIVE_MUTEX_INITIALIZER, .m_recursive = 1, \ - .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \ - .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } + int ret = ABT_mutex_lock(mutex); + if (ABT_SUCCESS != ret) { + opal_output(0, "opal_thread_internal_mutex_lock()"); + } #else -# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ - .m_lock_argobots = ABT_RECURSIVE_MUTEX_INITIALIZER, .m_recursive = 1, \ - .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } + ABT_mutex_lock(mutex); #endif +} -/************************************************************************ - * - * mutex operations (non-atomic versions) - * - ************************************************************************/ - -static inline int opal_mutex_trylock(opal_mutex_t *m) +static inline int opal_thread_internal_mutex_trylock(opal_thread_internal_mutex_t *p_mutex) { - ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots); + ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex); int ret = ABT_mutex_trylock(mutex); if (ABT_ERR_MUTEX_LOCKED == ret) { return 1; } else if (ABT_SUCCESS != ret) { #if OPAL_ENABLE_DEBUG - opal_output(0, "opal_mutex_trylock()"); + opal_output(0, "opal_thread_internal_mutex_trylock()"); #endif return 1; + } else { + return 0; } - return 0; } -static inline void opal_mutex_lock(opal_mutex_t *m) +static inline void opal_thread_internal_mutex_unlock(opal_thread_internal_mutex_t *p_mutex) { - ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots); -#if OPAL_ENABLE_DEBUG - int ret = ABT_mutex_lock(mutex); - if (ABT_SUCCESS != ret) { - opal_output(0, "opal_mutex_lock()"); - } -#else - ABT_mutex_lock(mutex); -#endif -} - -static inline void opal_mutex_unlock(opal_mutex_t *m) -{ - ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots); + ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex); #if OPAL_ENABLE_DEBUG int ret = ABT_mutex_unlock(mutex); if (ABT_SUCCESS != ret) { - opal_output(0, "opal_mutex_unlock()"); + opal_output(0, "opal_thread_internal_mutex_unlock()"); } #else ABT_mutex_unlock(mutex); @@ -141,65 +104,62 @@ static inline void opal_mutex_unlock(opal_mutex_t *m) ABT_thread_yield(); } -/************************************************************************ - * - * mutex operations (atomic versions) - * - ************************************************************************/ +static inline void opal_thread_internal_mutex_destroy(opal_thread_internal_mutex_t *p_mutex) +{ + /* No specific operation is needed to destroy opal_thread_internal_mutex_t. */ +} -#if OPAL_HAVE_ATOMIC_SPINLOCKS +typedef ABT_cond_memory opal_thread_internal_cond_t; -/************************************************************************ - * Spin Locks - ************************************************************************/ +#define OPAL_THREAD_INTERNAL_COND_INITIALIZER ABT_COND_INITIALIZER -static inline int opal_mutex_atomic_trylock(opal_mutex_t *m) +static inline int opal_thread_internal_cond_init(opal_thread_internal_cond_t *p_cond) { - return opal_atomic_trylock(&m->m_lock_atomic); + const ABT_cond_memory init_cond = ABT_COND_INITIALIZER; + memcpy(p_cond, &init_cond, sizeof(ABT_cond_memory)); + return OPAL_SUCCESS; } -static inline void opal_mutex_atomic_lock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_wait(opal_thread_internal_cond_t *p_cond, + opal_thread_internal_mutex_t *p_mutex) { - opal_atomic_lock(&m->m_lock_atomic); + ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(p_mutex); + ABT_cond cond = ABT_COND_MEMORY_GET_HANDLE(p_cond); +#if OPAL_ENABLE_DEBUG + int ret = ABT_cond_wait(cond, mutex); + assert(ABT_SUCCESS == ret); +#else + ABT_cond_wait(cond, mutex); +#endif } -static inline void opal_mutex_atomic_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_broadcast(opal_thread_internal_cond_t *p_cond) { - opal_atomic_unlock(&m->m_lock_atomic); -} - + ABT_cond cond = ABT_COND_MEMORY_GET_HANDLE(p_cond); +#if OPAL_ENABLE_DEBUG + int ret = ABT_cond_broadcast(cond); + assert(ABT_SUCCESS == ret); #else - -/************************************************************************ - * Standard locking - ************************************************************************/ - -static inline int opal_mutex_atomic_trylock(opal_mutex_t *m) -{ - return opal_mutex_trylock(m); + ABT_cond_broadcast(cond); +#endif } -static inline void opal_mutex_atomic_lock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_signal(opal_thread_internal_cond_t *p_cond) { - opal_mutex_lock(m); + ABT_cond cond = ABT_COND_MEMORY_GET_HANDLE(p_cond); +#if OPAL_ENABLE_DEBUG + int ret = ABT_cond_signal(cond); + assert(ABT_SUCCESS == ret); +#else + ABT_cond_signal(cond); +#endif } -static inline void opal_mutex_atomic_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_destroy(opal_thread_internal_cond_t *p_cond) { - opal_mutex_unlock(m); + /* No destructor is needed. */ } -#endif - -typedef ABT_cond_memory opal_cond_t; -#define OPAL_CONDITION_STATIC_INIT ABT_COND_INITIALIZER - -int opal_cond_init(opal_cond_t *cond); -int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock); -int opal_cond_broadcast(opal_cond_t *cond); -int opal_cond_signal(opal_cond_t *cond); -int opal_cond_destroy(opal_cond_t *cond); - END_C_DECLS #endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_MUTEX_H */ diff --git a/opal/mca/threads/argobots/threads_argobots_wait_sync.c b/opal/mca/threads/argobots/threads_argobots_wait_sync.c deleted file mode 100644 index ceb5eef0ce9..00000000000 --- a/opal/mca/threads/argobots/threads_argobots_wait_sync.c +++ /dev/null @@ -1,142 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2014-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2016 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2017 IBM Corporation. All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal/mca/threads/argobots/threads_argobots.h" -#include "opal/mca/threads/wait_sync.h" - -static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT; -static ompi_wait_sync_t *wait_sync_list = NULL; - -void wait_sync_global_wakeup_st(int status) -{ - ompi_wait_sync_t *sync; - for (sync = wait_sync_list; sync != NULL; sync = sync->next) { - wait_sync_update(sync, 0, status); - } -} - -void wait_sync_global_wakeup_mt(int status) -{ - ompi_wait_sync_t *sync; - opal_mutex_lock(&wait_sync_lock); - for (sync = wait_sync_list; sync != NULL; sync = sync->next) { - /* sync_update is going to take the sync->lock from within - * the wait_sync_lock. Thread lightly here: Idealy we should - * find a way to not take a lock in a lock as this is deadlock prone, - * but as of today we are the only place doing this so it is safe. - */ - wait_sync_update(sync, 0, status); - if (sync->next == wait_sync_list) { - break; /* special case for rings */ - } - } - opal_mutex_unlock(&wait_sync_lock); -} - -static opal_atomic_int32_t num_thread_in_progress = 0; - -#define WAIT_SYNC_PASS_OWNERSHIP(who) \ - do { \ - ABT_mutex_lock((who)->lock); \ - ABT_cond_signal((who)->condition); \ - ABT_mutex_unlock((who)->lock); \ - } while (0) - -int ompi_sync_wait_mt(ompi_wait_sync_t *sync) -{ - /* Don't stop if the waiting synchronization is completed. We avoid the - * race condition around the release of the synchronization using the - * signaling field. - */ - if (sync->count <= 0) { - return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; - } - - /* lock so nobody can signal us during the list updating */ - ABT_mutex_lock(sync->lock); - - /* Now that we hold the lock make sure another thread has not already - * call cond_signal. - */ - if (sync->count <= 0) { - ABT_mutex_unlock(sync->lock); - return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; - } - - /* Insert sync on the list of pending synchronization constructs */ - OPAL_THREAD_LOCK(&wait_sync_lock); - if (NULL == wait_sync_list) { - sync->next = sync->prev = sync; - wait_sync_list = sync; - } else { - sync->prev = wait_sync_list->prev; - sync->prev->next = sync; - sync->next = wait_sync_list; - wait_sync_list->prev = sync; - } - OPAL_THREAD_UNLOCK(&wait_sync_lock); - - /** - * If we are not responsible for progressing, go silent until something - * worth noticing happen: - * - this thread has been promoted to take care of the progress - * - our sync has been triggered. - */ -check_status: - if (sync != wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) { - ABT_cond_wait(sync->condition, sync->lock); - - /** - * At this point either the sync was completed in which case - * we should remove it from the wait list, or/and I was - * promoted as the progress manager. - */ - - if (sync->count <= 0) { /* Completed? */ - ABT_mutex_unlock(sync->lock); - goto i_am_done; - } - /* either promoted, or spurious wakeup ! */ - goto check_status; - } - ABT_mutex_unlock(sync->lock); - - OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1); - while (sync->count > 0) { /* progress till completion */ - /* don't progress with the sync lock locked or you'll deadlock */ - opal_progress(); - ABT_thread_yield(); - } - OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1); - -i_am_done: - /* My sync is now complete. Trim the list: remove self, wake next */ - OPAL_THREAD_LOCK(&wait_sync_lock); - sync->prev->next = sync->next; - sync->next->prev = sync->prev; - /* In case I am the progress manager, pass the duties on */ - if (sync == wait_sync_list) { - wait_sync_list = (sync == sync->next) ? NULL : sync->next; - if (NULL != wait_sync_list) { - WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list); - } - } - OPAL_THREAD_UNLOCK(&wait_sync_lock); - - return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; -} diff --git a/opal/mca/threads/argobots/threads_argobots_wait_sync.h b/opal/mca/threads/argobots/threads_argobots_wait_sync.h deleted file mode 100644 index 375264597d3..00000000000 --- a/opal/mca/threads/argobots/threads_argobots_wait_sync.h +++ /dev/null @@ -1,114 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H -#define OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H - -#include "opal/mca/threads/argobots/threads_argobots.h" - -typedef struct ompi_wait_sync_t { - opal_atomic_int32_t count; - int32_t status; - ABT_cond condition; - ABT_mutex lock; - struct ompi_wait_sync_t *next; - struct ompi_wait_sync_t *prev; - volatile bool signaling; -} ompi_wait_sync_t; - -#define SYNC_WAIT(sync) (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync)) - -/* The loop in release handles a race condition between the signaling - * thread and the destruction of the condition variable. The signaling - * member will be set to false after the final signaling thread has - * finished operating on the sync object. This is done to avoid - * extra atomics in the signalling function and keep it as fast - * as possible. Note that the race window is small so spinning here - * is more optimal than sleeping since this macro is called in - * the critical path. */ -#define WAIT_SYNC_RELEASE(sync) \ - if (opal_using_threads()) { \ - while ((sync)->signaling) { \ - ABT_thread_yield(); \ - continue; \ - } \ - ABT_cond_free(&(sync)->condition); \ - ABT_mutex_free(&(sync)->lock); \ - } - -#define WAIT_SYNC_RELEASE_NOWAIT(sync) \ - if (opal_using_threads()) { \ - ABT_cond_free(&(sync)->condition); \ - ABT_mutex_free(&(sync)->lock); \ - } - -#define WAIT_SYNC_SIGNAL(sync) \ - if (opal_using_threads()) { \ - ABT_mutex_lock(sync->lock); \ - ABT_cond_signal(sync->condition); \ - ABT_mutex_unlock(sync->lock); \ - sync->signaling = false; \ - } - -#define WAIT_SYNC_SIGNALLED(sync) \ - { \ - (sync)->signaling = false; \ - } - -OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync); -static inline int sync_wait_st(ompi_wait_sync_t *sync) -{ - while (sync->count > 0) { - opal_progress(); - ABT_thread_yield(); - } - return sync->status; -} - -#define WAIT_SYNC_INIT(sync, c) \ - do { \ - (sync)->count = (c); \ - (sync)->next = NULL; \ - (sync)->prev = NULL; \ - (sync)->status = 0; \ - (sync)->signaling = (0 != (c)); \ - if (opal_using_threads()) { \ - ABT_cond_create(&(sync)->condition); \ - ABT_mutex_create(&(sync)->lock); \ - } \ - } while (0) - -/** - * Wake up all syncs with a particular status. If status is OMPI_SUCCESS this - * operation is a NO-OP. Otherwise it will trigger the "error condition" from - * all registered sync. - */ -OPAL_DECLSPEC void wait_sync_global_wakeup_st(int status); -OPAL_DECLSPEC void wait_sync_global_wakeup_mt(int status); -#define wait_sync_global_wakeup(st) \ - (opal_using_threads() ? wait_sync_global_wakeup_mt(st) : wait_sync_global_wakeup_st(st)) - -#endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H */ diff --git a/opal/mca/threads/base/Makefile.am b/opal/mca/threads/base/Makefile.am index fcf7c0b3a67..4cbd5c742b1 100644 --- a/opal/mca/threads/base/Makefile.am +++ b/opal/mca/threads/base/Makefile.am @@ -10,6 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2019 Sandia National Laboratories. All rights reserved. +# Copyright (c) 2021 Argonne National Laboratory. All rights reserved. # # $COPYRIGHT$ # @@ -22,5 +23,7 @@ headers += \ base/base.h libmca_threads_la_SOURCES += \ + base/mutex.c \ base/threads_base.c \ - base/tsd.c + base/tsd.c \ + base/wait_sync.c diff --git a/opal/mca/threads/argobots/threads_argobots_mutex.c b/opal/mca/threads/base/mutex.c similarity index 55% rename from opal/mca/threads/argobots/threads_argobots_mutex.c rename to opal/mca/threads/base/mutex.c index c4cf2820875..fec6cee1d98 100644 --- a/opal/mca/threads/argobots/threads_argobots_mutex.c +++ b/opal/mca/threads/base/mutex.c @@ -25,13 +25,8 @@ */ #include "opal_config.h" -#include "opal/mca/threads/argobots/threads_argobots.h" - -#include -#include #include "opal/constants.h" -#include "opal/mca/threads/argobots/threads_argobots_mutex.h" #include "opal/mca/threads/mutex.h" /* @@ -40,68 +35,75 @@ */ bool opal_uses_threads = false; -static void mca_threads_argobots_mutex_constructor(opal_mutex_t *p_mutex) +static void mca_threads_mutex_constructor(opal_mutex_t *p_mutex) { - opal_threads_argobots_ensure_init(); - const ABT_mutex_memory init_mutex = ABT_MUTEX_INITIALIZER; - memcpy(&p_mutex->m_lock_argobots, &init_mutex, sizeof(ABT_mutex_memory)); - p_mutex->m_recursive = 0; #if OPAL_ENABLE_DEBUG + int ret = opal_thread_internal_mutex_init(&p_mutex->m_lock, false); + assert(0 == ret); p_mutex->m_lock_debug = 0; p_mutex->m_lock_file = NULL; p_mutex->m_lock_line = 0; +#else + opal_thread_internal_mutex_init(&p_mutex->m_lock, false); #endif opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0); } -static void mca_threads_argobots_recursive_mutex_constructor(opal_recursive_mutex_t *p_mutex) +static void mca_threads_mutex_destructor(opal_mutex_t *p_mutex) +{ + opal_thread_internal_mutex_destroy(&p_mutex->m_lock); +} + +static void mca_threads_recursive_mutex_constructor(opal_recursive_mutex_t *p_mutex) { - opal_threads_argobots_ensure_init(); - const ABT_mutex_memory init_mutex = ABT_RECURSIVE_MUTEX_INITIALIZER; - memcpy(&p_mutex->m_lock_argobots, &init_mutex, sizeof(ABT_mutex_memory)); - p_mutex->m_recursive = 1; #if OPAL_ENABLE_DEBUG + int ret = opal_thread_internal_mutex_init(&p_mutex->m_lock, true); + assert(0 == ret); p_mutex->m_lock_debug = 0; p_mutex->m_lock_file = NULL; p_mutex->m_lock_line = 0; +#else + opal_thread_internal_mutex_init(&p_mutex->m_lock, true); #endif opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0); } -OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_argobots_mutex_constructor, NULL); -OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t, - mca_threads_argobots_recursive_mutex_constructor, NULL); +static void mca_threads_recursive_mutex_destructor(opal_recursive_mutex_t *p_mutex) +{ + opal_thread_internal_mutex_destroy(&p_mutex->m_lock); +} + +OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_mutex_constructor, + mca_threads_mutex_destructor); + +OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t, mca_threads_recursive_mutex_constructor, + mca_threads_recursive_mutex_destructor); int opal_cond_init(opal_cond_t *cond) { - const ABT_cond_memory init_cond = ABT_COND_INITIALIZER; - memcpy(cond, &init_cond, sizeof(ABT_cond_memory)); - return OPAL_SUCCESS; + return opal_thread_internal_cond_init(cond); } int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock) { - ABT_mutex abt_mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&lock->m_lock_argobots); - ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond); - int ret = ABT_cond_wait(abt_cond, abt_mutex); - return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR; + opal_thread_internal_cond_wait(cond, &lock->m_lock); + return OPAL_SUCCESS; } int opal_cond_broadcast(opal_cond_t *cond) { - ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond); - int ret = ABT_cond_broadcast(abt_cond); - return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR; + opal_thread_internal_cond_broadcast(cond); + return OPAL_SUCCESS; } int opal_cond_signal(opal_cond_t *cond) { - ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond); - int ret = ABT_cond_signal(abt_cond); - return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR; + opal_thread_internal_cond_signal(cond); + return OPAL_SUCCESS; } int opal_cond_destroy(opal_cond_t *cond) { + opal_thread_internal_cond_destroy(cond); return OPAL_SUCCESS; } diff --git a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.c b/opal/mca/threads/base/wait_sync.c similarity index 86% rename from opal/mca/threads/pthreads/threads_pthreads_wait_sync.c rename to opal/mca/threads/base/wait_sync.c index eb777d95dd3..2451419620e 100644 --- a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.c +++ b/opal/mca/threads/base/wait_sync.c @@ -7,6 +7,7 @@ * reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. * * $COPYRIGHT$ * @@ -48,11 +49,11 @@ void wait_sync_global_wakeup_mt(int status) static opal_atomic_int32_t num_thread_in_progress = 0; -#define WAIT_SYNC_PASS_OWNERSHIP(who) \ - do { \ - pthread_mutex_lock(&(who)->lock); \ - pthread_cond_signal(&(who)->condition); \ - pthread_mutex_unlock(&(who)->lock); \ +#define WAIT_SYNC_PASS_OWNERSHIP(who) \ + do { \ + opal_thread_internal_mutex_lock(&(who)->lock); \ + opal_thread_internal_cond_signal(&(who)->condition); \ + opal_thread_internal_mutex_unlock(&(who)->lock); \ } while (0) int ompi_sync_wait_mt(ompi_wait_sync_t *sync) @@ -66,13 +67,13 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) } /* lock so nobody can signal us during the list updating */ - pthread_mutex_lock(&sync->lock); + opal_thread_internal_mutex_lock(&sync->lock); /* Now that we hold the lock make sure another thread has not already * call cond_signal. */ if (sync->count <= 0) { - pthread_mutex_unlock(&sync->lock); + opal_thread_internal_mutex_unlock(&sync->lock); return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; } @@ -97,7 +98,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) */ check_status: if (sync != wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) { - pthread_cond_wait(&sync->condition, &sync->lock); + opal_thread_internal_cond_wait(&sync->condition, &sync->lock); /** * At this point either the sync was completed in which case @@ -106,13 +107,13 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) */ if (sync->count <= 0) { /* Completed? */ - pthread_mutex_unlock(&sync->lock); + opal_thread_internal_mutex_unlock(&sync->lock); goto i_am_done; } /* either promoted, or spurious wakeup ! */ goto check_status; } - pthread_mutex_unlock(&sync->lock); + opal_thread_internal_mutex_unlock(&sync->lock); OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1); while (sync->count > 0) { /* progress till completion */ diff --git a/opal/mca/threads/mutex.h b/opal/mca/threads/mutex.h index 552f9f115ad..e6c9dc3f5bf 100644 --- a/opal/mca/threads/mutex.h +++ b/opal/mca/threads/mutex.h @@ -11,11 +11,13 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2007 Voltaire. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2020 Triad National Security, LLC. All rights reserved. + * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. * * $COPYRIGHT$ * @@ -28,6 +30,7 @@ #define OPAL_MCA_THREADS_MUTEX_H #include "opal_config.h" +#include "opal/sys/atomic.h" BEGIN_C_DECLS @@ -48,8 +51,53 @@ typedef struct opal_mutex_t opal_recursive_mutex_t; #include MCA_threads_mutex_base_include_HEADER -OBJ_CLASS_DECLARATION(opal_mutex_t); -OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); +struct opal_mutex_t { + opal_object_t super; + opal_thread_internal_mutex_t m_lock; +#if OPAL_ENABLE_DEBUG + int m_lock_debug; + const char *m_lock_file; + int m_lock_line; +#endif + opal_atomic_lock_t m_lock_atomic; +}; + +OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); +OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); + +#if OPAL_ENABLE_DEBUG +# define OPAL_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock = OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER, .m_lock_debug = 0, \ + .m_lock_file = NULL, .m_lock_line = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ + } +#else +# define OPAL_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock = OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ + } +#endif + +#if defined(OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER) +# if OPAL_ENABLE_DEBUG +# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock = OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER, .m_lock_debug = 0, \ + .m_lock_file = NULL, .m_lock_line = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ + } +# else +# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ + { \ + .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ + .m_lock = OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ + } +# endif +#endif /* OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER */ /** * Try to acquire a mutex. @@ -57,21 +105,30 @@ OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); * @param mutex Address of the mutex. * @return 0 if the mutex was acquired, 1 otherwise. */ -static inline int opal_mutex_trylock(opal_mutex_t *mutex); +static inline int opal_mutex_trylock(opal_mutex_t *mutex) +{ + return opal_thread_internal_mutex_trylock(&mutex->m_lock); +} /** * Acquire a mutex. * * @param mutex Address of the mutex. */ -static inline void opal_mutex_lock(opal_mutex_t *mutex); +static inline void opal_mutex_lock(opal_mutex_t *mutex) +{ + opal_thread_internal_mutex_lock(&mutex->m_lock); +} /** * Release a mutex. * * @param mutex Address of the mutex. */ -static inline void opal_mutex_unlock(opal_mutex_t *mutex); +static inline void opal_mutex_unlock(opal_mutex_t *mutex) +{ + opal_thread_internal_mutex_unlock(&mutex->m_lock); +} /** * Try to acquire a mutex using atomic operations. @@ -79,21 +136,42 @@ static inline void opal_mutex_unlock(opal_mutex_t *mutex); * @param mutex Address of the mutex. * @return 0 if the mutex was acquired, 1 otherwise. */ -static inline int opal_mutex_atomic_trylock(opal_mutex_t *mutex); +static inline int opal_mutex_atomic_trylock(opal_mutex_t *mutex) +{ +#if OPAL_HAVE_ATOMIC_SPINLOCKS + return opal_atomic_trylock(&mutex->m_lock_atomic); +#else + return opal_mutex_trylock(mutex); +#endif +} /** * Acquire a mutex using atomic operations. * * @param mutex Address of the mutex. */ -static inline void opal_mutex_atomic_lock(opal_mutex_t *mutex); +static inline void opal_mutex_atomic_lock(opal_mutex_t *mutex) +{ +#if OPAL_HAVE_ATOMIC_SPINLOCKS + opal_atomic_lock(&mutex->m_lock_atomic); +#else + opal_mutex_lock(mutex); +#endif +} /** * Release a mutex using atomic operations. * * @param mutex Address of the mutex. */ -static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex); +static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex) +{ +#if OPAL_HAVE_ATOMIC_SPINLOCKS + opal_atomic_unlock(&mutex->m_lock_atomic); +#else + opal_mutex_unlock(mutex); +#endif +} /** * Lock a mutex if opal_using_threads() says that multiple threads may @@ -179,6 +257,14 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *mutex); } \ } while (0) +typedef opal_thread_internal_cond_t opal_cond_t; +#define OPAL_CONDITION_STATIC_INIT OPAL_THREAD_INTERNAL_COND_INITIALIZER +int opal_cond_init(opal_cond_t *cond); +int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock); +int opal_cond_broadcast(opal_cond_t *cond); +int opal_cond_signal(opal_cond_t *cond); +int opal_cond_destroy(opal_cond_t *cond); + END_C_DECLS #endif /* OPAL_MCA_THREADS_MUTEX_H */ diff --git a/opal/mca/threads/pthreads/Makefile.am b/opal/mca/threads/pthreads/Makefile.am index 0f5aa1d45de..2930a292cce 100644 --- a/opal/mca/threads/pthreads/Makefile.am +++ b/opal/mca/threads/pthreads/Makefile.am @@ -11,6 +11,8 @@ # All rights reserved. # Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2019 Sandia National Laboratories. All rights reserved. +# Copyright (c) 2021 Argonne National Laboratory. All rights reserved. +# # $COPYRIGHT$ # # Additional copyrights may follow @@ -24,11 +26,8 @@ libmca_threads_pthreads_la_SOURCES = \ threads_pthreads_component.c \ threads_pthreads_condition.c \ threads_pthreads_module.c \ - threads_pthreads_mutex.c \ threads_pthreads_mutex.h \ threads_pthreads_threads.h \ threads_pthreads_tsd.h \ - threads_pthreads_wait_sync.c \ - threads_pthreads_wait_sync.h \ threads_pthreads_yield.c \ threads_pthreads.h diff --git a/opal/mca/threads/pthreads/configure.m4 b/opal/mca/threads/pthreads/configure.m4 index 93f05d862f9..0bf263e7570 100644 --- a/opal/mca/threads/pthreads/configure.m4 +++ b/opal/mca/threads/pthreads/configure.m4 @@ -716,9 +716,6 @@ AC_DEFUN([MCA_opal_threads_pthreads_POST_CONFIG],[ AC_DEFINE_UNQUOTED([MCA_threads_tsd_base_include_HEADER], ["opal/mca/threads/pthreads/threads_pthreads_tsd.h"], [Header to include for tsd implementation]) - AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER], - ["opal/mca/threads/pthreads/threads_pthreads_wait_sync.h"], - [Header to include for wait_sync implementation]) THREAD_CFLAGS="$TPKG_CFLAGS" THREAD_FCFLAGS="$TPKG_FCFLAGS" THREAD_CXXFLAGS="$TPKG_CXXFLAGS" diff --git a/opal/mca/threads/pthreads/threads_pthreads_mutex.c b/opal/mca/threads/pthreads/threads_pthreads_mutex.c deleted file mode 100644 index 8072cfccd49..00000000000 --- a/opal/mca/threads/pthreads/threads_pthreads_mutex.c +++ /dev/null @@ -1,128 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include -#include - -#include "opal/constants.h" -#include "opal/mca/threads/mutex.h" -#include "opal/mca/threads/pthreads/threads_pthreads_mutex.h" - -/* - * Wait and see if some upper layer wants to use threads, if support - * exists. - */ -bool opal_uses_threads = false; - -struct opal_pthread_mutex_t { - opal_object_t super; - - pthread_mutex_t m_lock_pthread; - opal_atomic_lock_t m_lock_atomic; - -#if OPAL_ENABLE_DEBUG - int m_lock_debug; - const char *m_lock_file; - int m_lock_line; -#endif -}; - -typedef struct opal_pthread_mutex_t opal_pthread_mutex_t; -typedef struct opal_pthread_mutex_t opal_pthread_recursive_mutex_t; - -static void mca_threads_pthreads_mutex_constructor(opal_mutex_t *p_mutex) -{ - pthread_mutex_init(&p_mutex->m_lock_pthread, NULL); -#if OPAL_ENABLE_DEBUG - p_mutex->m_lock_debug = 0; - p_mutex->m_lock_file = NULL; - p_mutex->m_lock_line = 0; -#endif - opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0); -} - -static void mca_threads_pthreads_mutex_destructor(opal_mutex_t *p_mutex) -{ - pthread_mutex_destroy(&p_mutex->m_lock_pthread); -} - -static void mca_threads_pthreads_recursive_mutex_constructor(opal_recursive_mutex_t *p_mutex) -{ - pthread_mutexattr_t mutex_attr; - pthread_mutexattr_init(&mutex_attr); - pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&p_mutex->m_lock_pthread, &mutex_attr); - pthread_mutexattr_destroy(&mutex_attr); -#if OPAL_ENABLE_DEBUG - p_mutex->m_lock_debug = 0; - p_mutex->m_lock_file = NULL; - p_mutex->m_lock_line = 0; -#endif - opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0); -} - -static void mca_threads_pthreads_recursive_mutex_destructor(opal_recursive_mutex_t *p_mutex) -{ - pthread_mutex_destroy(&p_mutex->m_lock_pthread); -} - -OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_pthreads_mutex_constructor, - mca_threads_pthreads_mutex_destructor); - -OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t, - mca_threads_pthreads_recursive_mutex_constructor, - mca_threads_pthreads_recursive_mutex_destructor); - -int opal_cond_init(opal_cond_t *cond) -{ - int ret = pthread_cond_init(cond, NULL); - return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; -} - -int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock) -{ - int ret = pthread_cond_wait(cond, &lock->m_lock_pthread); - return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; -} - -int opal_cond_broadcast(opal_cond_t *cond) -{ - int ret = pthread_cond_broadcast(cond); - return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; -} - -int opal_cond_signal(opal_cond_t *cond) -{ - int ret = pthread_cond_signal(cond); - return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; -} - -int opal_cond_destroy(opal_cond_t *cond) -{ - int ret = pthread_cond_destroy(cond); - return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; -} diff --git a/opal/mca/threads/pthreads/threads_pthreads_mutex.h b/opal/mca/threads/pthreads/threads_pthreads_mutex.h index d60ed8dff54..82dc8ef63e0 100644 --- a/opal/mca/threads/pthreads/threads_pthreads_mutex.h +++ b/opal/mca/threads/pthreads/threads_pthreads_mutex.h @@ -17,6 +17,7 @@ * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. * Copyright (c) 2020 Triad National Security, LLC. All rights * reserved. + * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. * * $COPYRIGHT$ * @@ -46,170 +47,153 @@ #include #include "opal/class/opal_object.h" -#include "opal/sys/atomic.h" +#include "opal/constants.h" #include "opal/util/output.h" BEGIN_C_DECLS -struct opal_mutex_t { - opal_object_t super; - - pthread_mutex_t m_lock_pthread; - -#if OPAL_ENABLE_DEBUG - int m_lock_debug; - const char *m_lock_file; - int m_lock_line; -#endif - - opal_atomic_lock_t m_lock_atomic; -}; -OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); -OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); +typedef pthread_mutex_t opal_thread_internal_mutex_t; +#define OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER #if defined(PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP) -# define OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP +# define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP #elif defined(PTHREAD_RECURSIVE_MUTEX_INITIALIZER) -# define OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER +# define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER PTHREAD_RECURSIVE_MUTEX_INITIALIZER #endif +static inline int opal_thread_internal_mutex_init(opal_thread_internal_mutex_t *p_mutex, + bool recursive) +{ + int ret; #if OPAL_ENABLE_DEBUG -# define OPAL_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ - .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, .m_lock_debug = 0, .m_lock_file = NULL, \ - .m_lock_line = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ + if (recursive) { + pthread_mutexattr_t mutex_attr; + ret = pthread_mutexattr_init(&mutex_attr); + if (0 != ret) + return OPAL_ERR_IN_ERRNO; + ret = pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); + if (0 != ret) { + ret = pthread_mutexattr_destroy(&mutex_attr); + assert(0 == ret); + return OPAL_ERR_IN_ERRNO; + } + ret = pthread_mutex_init(p_mutex, &mutex_attr); + if (0 != ret) { + ret = pthread_mutexattr_destroy(&mutex_attr); + assert(0 == ret); + return OPAL_ERR_IN_ERRNO; } + ret = pthread_mutexattr_destroy(&mutex_attr); + assert(0 == ret); + } else { + ret = pthread_mutex_init(p_mutex, NULL); + } #else -# define OPAL_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ - .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ + if (recursive) { + pthread_mutexattr_t mutex_attr; + ret = pthread_mutexattr_init(&mutex_attr); + if (0 != ret) { + return OPAL_ERR_IN_ERRNO; } + pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); + ret = pthread_mutex_init(p_mutex, &mutex_attr); + pthread_mutexattr_destroy(&mutex_attr); + } else { + ret = pthread_mutex_init(p_mutex, NULL); + } #endif + return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; +} -#if defined(OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER) - -# if OPAL_ENABLE_DEBUG -# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ - .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, .m_lock_debug = 0, \ - .m_lock_file = NULL, .m_lock_line = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -# else -# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ - .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ - .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -# endif - -#endif - -/************************************************************************ - * - * mutex operations (non-atomic versions) - * - ************************************************************************/ - -static inline int opal_mutex_trylock(opal_mutex_t *m) +static inline void opal_thread_internal_mutex_lock(opal_thread_internal_mutex_t *p_mutex) { - int ret = pthread_mutex_trylock(&m->m_lock_pthread); - if (EDEADLK == ret) { #if OPAL_ENABLE_DEBUG - opal_output(0, "opal_mutex_trylock() %d", ret); -#endif - return 1; + int ret = pthread_mutex_lock(p_mutex); + if (EDEADLK == ret) { + opal_output(0, "opal_thread_internal_mutex_lock() %d", ret); } + assert(0 == ret); +#else + pthread_mutex_lock(p_mutex); +#endif +} + +static inline int opal_thread_internal_mutex_trylock(opal_thread_internal_mutex_t *p_mutex) +{ + int ret = pthread_mutex_trylock(p_mutex); return 0 == ret ? 0 : 1; } -static inline void opal_mutex_lock(opal_mutex_t *m) +static inline void opal_thread_internal_mutex_unlock(opal_thread_internal_mutex_t *p_mutex) { #if OPAL_ENABLE_DEBUG - int ret = pthread_mutex_lock(&m->m_lock_pthread); - if (EDEADLK == ret) { - errno = ret; - opal_output(0, "opal_mutex_lock() %d", ret); - } + int ret = pthread_mutex_unlock(p_mutex); + assert(0 == ret); #else - pthread_mutex_lock(&m->m_lock_pthread); + pthread_mutex_unlock(p_mutex); #endif } -static inline void opal_mutex_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_mutex_destroy(opal_thread_internal_mutex_t *p_mutex) { #if OPAL_ENABLE_DEBUG - int ret = pthread_mutex_unlock(&m->m_lock_pthread); - if (EPERM == ret) { - errno = ret; - opal_output(0, "opal_mutex_unlock() %d", ret); - } + int ret = pthread_mutex_destroy(p_mutex); + assert(0 == ret); #else - pthread_mutex_unlock(&m->m_lock_pthread); + pthread_mutex_destroy(p_mutex); #endif } -/************************************************************************ - * - * mutex operations (atomic versions) - * - ************************************************************************/ +typedef pthread_cond_t opal_thread_internal_cond_t; -#if OPAL_HAVE_ATOMIC_SPINLOCKS +#define OPAL_THREAD_INTERNAL_COND_INITIALIZER PTHREAD_COND_INITIALIZER -/************************************************************************ - * Spin Locks - ************************************************************************/ - -static inline int opal_mutex_atomic_trylock(opal_mutex_t *m) +static inline int opal_thread_internal_cond_init(opal_thread_internal_cond_t *p_cond) { - return opal_atomic_trylock(&m->m_lock_atomic); + int ret = pthread_cond_init(p_cond, NULL); + return 0 == ret ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO; } -static inline void opal_mutex_atomic_lock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_wait(opal_thread_internal_cond_t *p_cond, + opal_thread_internal_mutex_t *p_mutex) { - opal_atomic_lock(&m->m_lock_atomic); +#if OPAL_ENABLE_DEBUG + int ret = pthread_cond_wait(p_cond, p_mutex); + assert(0 == ret); +#else + pthread_cond_wait(p_cond, p_mutex); +#endif } -static inline void opal_mutex_atomic_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_broadcast(opal_thread_internal_cond_t *p_cond) { - opal_atomic_unlock(&m->m_lock_atomic); -} - +#if OPAL_ENABLE_DEBUG + int ret = pthread_cond_broadcast(p_cond); + assert(0 == ret); #else - -/************************************************************************ - * Standard locking - ************************************************************************/ - -static inline int opal_mutex_atomic_trylock(opal_mutex_t *m) -{ - return opal_mutex_trylock(m); + pthread_cond_broadcast(p_cond); +#endif } -static inline void opal_mutex_atomic_lock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_signal(opal_thread_internal_cond_t *p_cond) { - opal_mutex_lock(m); +#if OPAL_ENABLE_DEBUG + int ret = pthread_cond_signal(p_cond); + assert(0 == ret); +#else + pthread_cond_signal(p_cond); +#endif } -static inline void opal_mutex_atomic_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_destroy(opal_thread_internal_cond_t *p_cond) { - opal_mutex_unlock(m); -} - +#if OPAL_ENABLE_DEBUG + int ret = pthread_cond_destroy(p_cond); + assert(0 == ret); +#else + pthread_cond_destroy(p_cond); #endif - -typedef pthread_cond_t opal_cond_t; -#define OPAL_CONDITION_STATIC_INIT PTHREAD_COND_INITIALIZER - -int opal_cond_init(opal_cond_t *cond); -int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock); -int opal_cond_broadcast(opal_cond_t *cond); -int opal_cond_signal(opal_cond_t *cond); -int opal_cond_destroy(opal_cond_t *cond); +} END_C_DECLS diff --git a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.h b/opal/mca/threads/pthreads/threads_pthreads_wait_sync.h deleted file mode 100644 index e84a24e12e2..00000000000 --- a/opal/mca/threads/pthreads/threads_pthreads_wait_sync.h +++ /dev/null @@ -1,118 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2020 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2015-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H -#define OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H - -typedef struct ompi_wait_sync_t { - opal_atomic_int32_t count; - int32_t status; - pthread_cond_t condition; - pthread_mutex_t lock; - struct ompi_wait_sync_t *next; - struct ompi_wait_sync_t *prev; - volatile bool signaling; -} ompi_wait_sync_t; - -#define SYNC_WAIT(sync) (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync)) - -/* The loop in release handles a race condition between the signaling - * thread and the destruction of the condition variable. The signaling - * member will be set to false after the final signaling thread has - * finished operating on the sync object. This is done to avoid - * extra atomics in the signalling function and keep it as fast - * as possible. Note that the race window is small so spinning here - * is more optimal than sleeping since this macro is called in - * the critical path. */ -#define WAIT_SYNC_RELEASE(sync) \ - if (opal_using_threads()) { \ - while ((sync)->signaling) { \ - continue; \ - } \ - pthread_cond_destroy(&(sync)->condition); \ - pthread_mutex_destroy(&(sync)->lock); \ - } - -#define WAIT_SYNC_RELEASE_NOWAIT(sync) \ - if (opal_using_threads()) { \ - pthread_cond_destroy(&(sync)->condition); \ - pthread_mutex_destroy(&(sync)->lock); \ - } - -#define WAIT_SYNC_SIGNAL(sync) \ - if (opal_using_threads()) { \ - pthread_mutex_lock(&(sync->lock)); \ - pthread_cond_signal(&sync->condition); \ - pthread_mutex_unlock(&(sync->lock)); \ - sync->signaling = false; \ - } - -#define WAIT_SYNC_SIGNALLED(sync) \ - { \ - (sync)->signaling = false; \ - } - -/* not static for inline "wait_sync_st" */ -OPAL_DECLSPEC extern ompi_wait_sync_t *wait_sync_list; - -OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync); -static inline int sync_wait_st(ompi_wait_sync_t *sync) -{ - assert(NULL == wait_sync_list); - assert(NULL == sync->next); - wait_sync_list = sync; - - while (sync->count > 0) { - opal_progress(); - } - wait_sync_list = NULL; - - return sync->status; -} - -#define WAIT_SYNC_INIT(sync, c) \ - do { \ - (sync)->count = (c); \ - (sync)->next = NULL; \ - (sync)->prev = NULL; \ - (sync)->status = 0; \ - (sync)->signaling = (0 != (c)); \ - if (opal_using_threads()) { \ - pthread_cond_init(&(sync)->condition, NULL); \ - pthread_mutex_init(&(sync)->lock, NULL); \ - } \ - } while (0) - -/** - * Wake up all syncs with a particular status. If status is OMPI_SUCCESS this - * operation is a NO-OP. Otherwise it will trigger the "error condition" from - * all registered sync. - */ -OPAL_DECLSPEC void wait_sync_global_wakeup_st(int status); -OPAL_DECLSPEC void wait_sync_global_wakeup_mt(int status); -#define wait_sync_global_wakeup(st) \ - (opal_using_threads() ? wait_sync_global_wakeup_mt(st) : wait_sync_global_wakeup_st(st)) - -#endif /* OPAL_MCA_THREADS_PTHREADS_THREADS_PTHREADS_WAIT_SYNC_H */ diff --git a/opal/mca/threads/qthreads/Makefile.am b/opal/mca/threads/qthreads/Makefile.am index 683b9cc435e..0e375b4c930 100644 --- a/opal/mca/threads/qthreads/Makefile.am +++ b/opal/mca/threads/qthreads/Makefile.am @@ -11,6 +11,7 @@ # All rights reserved. # Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2019 Sandia National Laboratories. All rights reserved. +# Copyright (c) 2021 Argonne National Laboratory. All rights reserved. # # $COPYRIGHT$ # @@ -26,12 +27,9 @@ sources = \ threads_qthreads_component.c \ threads_qthreads_condition.c \ threads_qthreads_module.c \ - threads_qthreads_mutex.c \ - threads_qthreads_wait_sync.c \ threads_qthreads_mutex.h \ threads_qthreads_threads.h \ - threads_qthreads_tsd.h \ - threads_qthreads_wait_sync.h + threads_qthreads_tsd.h lib_sources = $(sources) diff --git a/opal/mca/threads/qthreads/configure.m4 b/opal/mca/threads/qthreads/configure.m4 index 4121711685a..247ddde6640 100644 --- a/opal/mca/threads/qthreads/configure.m4 +++ b/opal/mca/threads/qthreads/configure.m4 @@ -112,9 +112,6 @@ AC_DEFUN([MCA_opal_threads_qthreads_POST_CONFIG],[ AC_DEFINE_UNQUOTED([MCA_threads_tsd_base_include_HEADER], ["opal/mca/threads/qthreads/threads_qthreads_tsd.h"], [Header to include for tsd implementation]) - AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER], - ["opal/mca/threads/qthreads/threads_qthreads_wait_sync.h"], - [Header to include for wait_sync implementation]) THREAD_CFLAGS="$TPKG_CFLAGS" THREAD_FCFLAGS="$TPKG_FCFLAGS" THREAD_CXXFLAGS="$TPKG_CXXFLAGS" diff --git a/opal/mca/threads/qthreads/threads_qthreads_mutex.c b/opal/mca/threads/qthreads/threads_qthreads_mutex.c deleted file mode 100644 index 6c002ccd093..00000000000 --- a/opal/mca/threads/qthreads/threads_qthreads_mutex.c +++ /dev/null @@ -1,147 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal_config.h" - -#include "opal/mca/threads/mutex.h" -#include "opal/constants.h" - -/* - * Wait and see if some upper layer wants to use threads, if support - * exists. - */ -bool opal_uses_threads = false; - -static void opal_mutex_construct(opal_mutex_t *m) -{ - opal_mutex_create(m); -} - -static void opal_mutex_destruct(opal_mutex_t *m) -{ -} - -static void opal_recursive_mutex_construct(opal_mutex_t *m) -{ - opal_mutex_recursive_create(m); -} - -static void opal_recursive_mutex_destruct(opal_mutex_t *m) -{ -} - -OBJ_CLASS_INSTANCE(opal_mutex_t, - opal_object_t, - opal_mutex_construct, - opal_mutex_destruct); - -static void opal_recursive_mutex_construct(opal_recursive_mutex_t *m) -{ -} - -OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, - opal_object_t, - opal_recursive_mutex_construct, - opal_recursive_mutex_destruct); - -int opal_cond_init(opal_cond_t *cond) -{ - opal_atomic_lock_init(&cond->m_lock, 0); - cond->m_waiter_head = NULL; - cond->m_waiter_tail = NULL; - return OPAL_SUCCESS; -} - -typedef struct { - int m_signaled; - void * m_prev; -} cond_waiter_t; - -int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock) -{ - opal_threads_ensure_init_qthreads(); - /* This thread is taking "lock", so only this thread can access this - * condition variable. */ - opal_atomic_lock(&cond->m_lock); - cond_waiter_t waiter = {0, NULL}; - if (NULL == cond->m_waiter_head) { - cond->m_waiter_tail = (void *)&waiter; - } else { - ((cond_waiter_t *)cond->m_waiter_head)->m_prev = (void *)&waiter; - } - cond->m_waiter_head = (void *)&waiter; - opal_atomic_unlock(&cond->m_lock); - - while (1) { - opal_mutex_unlock(lock); - qthread_yield(); - opal_mutex_lock(lock); - /* Check if someone woke me up. */ - opal_atomic_lock(&cond->m_lock); - int signaled = waiter.m_signaled; - opal_atomic_unlock(&cond->m_lock); - if (1 == signaled) { - break; - } - /* Unlock the lock again. */ - } - return OPAL_SUCCESS; -} - -int opal_cond_broadcast(opal_cond_t *cond) -{ - opal_atomic_lock(&cond->m_lock); - while (NULL != cond->m_waiter_tail) { - cond_waiter_t *p_cur_tail = (cond_waiter_t *)cond->m_waiter_tail; - cond->m_waiter_tail = p_cur_tail->m_prev; - /* Awaken one of threads in a FIFO manner. */ - p_cur_tail->m_signaled = 1; - } - /* No waiters. */ - cond->m_waiter_head = NULL; - opal_atomic_unlock(&cond->m_lock); - return OPAL_SUCCESS; -} - -int opal_cond_signal(opal_cond_t *cond) -{ - opal_atomic_lock(&cond->m_lock); - if (NULL != cond->m_waiter_tail) { - cond_waiter_t *p_cur_tail = (cond_waiter_t *)cond->m_waiter_tail; - cond->m_waiter_tail = p_cur_tail->m_prev; - /* Awaken one of threads. */ - p_cur_tail->m_signaled = 1; - if (NULL == cond->m_waiter_tail) { - cond->m_waiter_head = NULL; - } - } - opal_atomic_unlock(&cond->m_lock); - return OPAL_SUCCESS; -} - -int opal_cond_destroy(opal_cond_t *cond) -{ - return OPAL_SUCCESS; -} diff --git a/opal/mca/threads/qthreads/threads_qthreads_mutex.h b/opal/mca/threads/qthreads/threads_qthreads_mutex.h index e44dcf66dff..f7cb85d7a15 100644 --- a/opal/mca/threads/qthreads/threads_qthreads_mutex.h +++ b/opal/mca/threads/qthreads/threads_qthreads_mutex.h @@ -19,6 +19,7 @@ * reserved. * * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,7 +32,6 @@ #include "opal_config.h" -#include #include #include "opal/class/opal_object.h" @@ -42,93 +42,34 @@ BEGIN_C_DECLS -struct opal_mutex_t { - opal_object_t super; +typedef opal_atomic_lock_t opal_thread_internal_mutex_t; - opal_atomic_lock_t m_lock; - int m_recursive; - -#if OPAL_ENABLE_DEBUG - int m_lock_debug; - const char *m_lock_file; - int m_lock_line; -#endif - - opal_atomic_lock_t m_lock_atomic; -}; - -OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); -OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); - -#if OPAL_ENABLE_DEBUG -# define OPAL_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock = OPAL_ATOMIC_LOCK_INIT, \ - .m_recursive = 0, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \ - .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -#else -# define OPAL_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock = OPAL_ATOMIC_LOCK_INIT, \ - .m_recursive = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -#endif - -#if OPAL_ENABLE_DEBUG -# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock = OPAL_ATOMIC_LOCK_INIT, \ - .m_recursive = 1, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \ - .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -#else -# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ - { \ - .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock = OPAL_ATOMIC_LOCK_INIT, \ - .m_recursive = 1, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ - } -#endif - -/************************************************************************ - * - * mutex operations (non-atomic versions) - * - ************************************************************************/ +#define OPAL_THREAD_INTERNAL_MUTEX_INITIALIZER OPAL_ATOMIC_LOCK_INIT +#define OPAL_THREAD_INTERNAL_RECURSIVE_MUTEX_INITIALIZER OPAL_ATOMIC_LOCK_INIT -static inline void opal_mutex_create(struct opal_mutex_t *m) +static inline int opal_thread_internal_mutex_init(opal_thread_internal_mutex_t *p_mutex, + bool recursive) { - opal_threads_ensure_init_qthreads(); - - opal_atomic_lock_init(&m->m_lock, 0); - opal_atomic_lock_init(&m->m_lock_atomic, 0); - m->m_recursive = 0; -#if OPAL_ENABLE_DEBUG - m->m_lock_debug = 0; - m->m_lock_file = NULL; - m->m_lock_line = 0; -#endif + opal_atomic_lock_init(p_mutex, 0); + return OPAL_SUCCESS; } -static inline void opal_mutex_recursive_create(struct opal_mutex_t *m) +static inline void opal_thread_internal_mutex_lock(opal_thread_internal_mutex_t *p_mutex) { opal_threads_ensure_init_qthreads(); - opal_atomic_lock_init(&m->m_lock, 0); - opal_atomic_lock_init(&m->m_lock_atomic, 0); - m->m_recursive = 1; -#if OPAL_ENABLE_DEBUG - m->m_lock_debug = 0; - m->m_lock_file = NULL; - m->m_lock_line = 0; -#endif + int ret = opal_atomic_trylock(p_mutex); + while (0 != ret) { + qthread_yield(); + ret = opal_atomic_trylock(p_mutex); + } } -static inline int opal_mutex_trylock(opal_mutex_t *m) +static inline int opal_thread_internal_mutex_trylock(opal_thread_internal_mutex_t *p_mutex) { opal_threads_ensure_init_qthreads(); - int ret = opal_atomic_trylock(&m->m_lock); + int ret = opal_atomic_trylock(p_mutex); if (0 != ret) { /* Yield to avoid a deadlock. */ qthread_yield(); @@ -136,93 +77,109 @@ static inline int opal_mutex_trylock(opal_mutex_t *m) return ret; } -static inline void opal_mutex_lock(opal_mutex_t *m) +static inline void opal_thread_internal_mutex_unlock(opal_thread_internal_mutex_t *p_mutex) { opal_threads_ensure_init_qthreads(); - int ret = opal_atomic_trylock(&m->m_lock); - while (0 != ret) { - qthread_yield(); - ret = opal_atomic_trylock(&m->m_lock); - } + opal_atomic_unlock(p_mutex); + /* For fairness of locking. */ + qthread_yield(); } -static inline void opal_mutex_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_mutex_destroy(opal_thread_internal_mutex_t *p_mutex) { - opal_threads_ensure_init_qthreads(); - - opal_atomic_unlock(&m->m_lock); - /* For fairness of locking. */ - qthread_yield(); + /* No specific operation is needed to destroy opal_thread_internal_mutex_t. */ } -/************************************************************************ - * - * mutex operations (atomic versions) - * - ************************************************************************/ +typedef struct opal_thread_cond_waiter_t { + int m_signaled; + struct opal_thread_cond_waiter_t *m_prev; +} opal_thread_cond_waiter_t; -#if OPAL_HAVE_ATOMIC_SPINLOCKS +typedef struct { + opal_atomic_lock_t m_lock; + opal_thread_cond_waiter_t *m_waiter_head; + opal_thread_cond_waiter_t *m_waiter_tail; +} opal_thread_internal_cond_t; -/************************************************************************ - * Spin Locks - ************************************************************************/ +#define OPAL_THREAD_INTERNAL_COND_INITIALIZER \ + { \ + .m_lock = OPAL_ATOMIC_LOCK_INIT, .m_waiter_head = NULL, .m_waiter_tail = NULL, \ + } -static inline int opal_mutex_atomic_trylock(opal_mutex_t *m) +static inline int opal_thread_internal_cond_init(opal_thread_internal_cond_t *p_cond) { - return opal_atomic_trylock(&m->m_lock_atomic); + opal_atomic_lock_init(&p_cond->m_lock, 0); + p_cond->m_waiter_head = NULL; + p_cond->m_waiter_tail = NULL; + return OPAL_SUCCESS; } -static inline void opal_mutex_atomic_lock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_wait(opal_thread_internal_cond_t *p_cond, + opal_thread_internal_mutex_t *p_mutex) { - opal_atomic_lock(&m->m_lock_atomic); -} + opal_threads_ensure_init_qthreads(); + /* This thread is taking "lock", so only this thread can access this + * condition variable. */ + opal_atomic_lock(&p_cond->m_lock); + opal_thread_cond_waiter_t waiter = {0, NULL}; + if (NULL == p_cond->m_waiter_head) { + p_cond->m_waiter_tail = &waiter; + } else { + p_cond->m_waiter_head->m_prev = &waiter; + } + p_cond->m_waiter_head = &waiter; + opal_atomic_unlock(&p_cond->m_lock); -static inline void opal_mutex_atomic_unlock(opal_mutex_t *m) -{ - opal_atomic_unlock(&m->m_lock_atomic); + while (1) { + opal_thread_internal_mutex_unlock(p_mutex); + qthread_yield(); + opal_thread_internal_mutex_lock(p_mutex); + /* Check if someone woke me up. */ + opal_atomic_lock(&p_cond->m_lock); + int signaled = waiter.m_signaled; + opal_atomic_unlock(&p_cond->m_lock); + if (1 == signaled) { + break; + } + /* Unlock the lock again. */ + } } -#else - -/************************************************************************ - * Standard locking - ************************************************************************/ - -static inline int opal_mutex_atomic_trylock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_broadcast(opal_thread_internal_cond_t *p_cond) { - return opal_mutex_trylock(m); + opal_atomic_lock(&p_cond->m_lock); + while (NULL != p_cond->m_waiter_tail) { + opal_thread_cond_waiter_t *p_cur_tail = p_cond->m_waiter_tail; + p_cond->m_waiter_tail = p_cur_tail->m_prev; + /* Awaken one of threads in a FIFO manner. */ + p_cur_tail->m_signaled = 1; + } + /* No waiters. */ + p_cond->m_waiter_head = NULL; + opal_atomic_unlock(&p_cond->m_lock); } -static inline void opal_mutex_atomic_lock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_signal(opal_thread_internal_cond_t *p_cond) { - opal_mutex_lock(m); + opal_atomic_lock(&p_cond->m_lock); + if (NULL != p_cond->m_waiter_tail) { + opal_thread_cond_waiter_t *p_cur_tail = p_cond->m_waiter_tail; + p_cond->m_waiter_tail = p_cur_tail->m_prev; + /* Awaken one of threads. */ + p_cur_tail->m_signaled = 1; + if (NULL == p_cond->m_waiter_tail) { + p_cond->m_waiter_head = NULL; + } + } + opal_atomic_unlock(&p_cond->m_lock); } -static inline void opal_mutex_atomic_unlock(opal_mutex_t *m) +static inline void opal_thread_internal_cond_destroy(opal_thread_internal_cond_t *p_cond) { - opal_mutex_unlock(m); + /* No destructor is needed. */ } -#endif - -typedef struct opal_cond_t { - opal_atomic_lock_t m_lock; - void *m_waiter_head; - void *m_waiter_tail; -} opal_cond_t; - -#define OPAL_CONDITION_STATIC_INIT \ - { \ - .m_lock = OPAL_ATOMIC_LOCK_INIT, .m_waiter_head = NULL, .m_waiter_tail = NULL, \ - } - -int opal_cond_init(opal_cond_t *cond); -int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock); -int opal_cond_broadcast(opal_cond_t *cond); -int opal_cond_signal(opal_cond_t *cond); -int opal_cond_destroy(opal_cond_t *cond); - END_C_DECLS #endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_MUTEX_H */ diff --git a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.c b/opal/mca/threads/qthreads/threads_qthreads_wait_sync.c deleted file mode 100644 index cbdc56c174f..00000000000 --- a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.c +++ /dev/null @@ -1,115 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2014-2016 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2016 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2017 IBM Corporation. All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal/mca/threads/qthreads/threads_qthreads.h" -#include "opal/mca/threads/wait_sync.h" - -static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT; -static ompi_wait_sync_t *wait_sync_list = NULL; - -static opal_atomic_int32_t num_thread_in_progress = 0; - -#define WAIT_SYNC_PASS_OWNERSHIP(who) \ - do { \ - opal_mutex_lock(&(who)->lock); \ - opal_cond_signal(&(who)->condition); \ - opal_mutex_unlock(&(who)->lock); \ - } while (0) - -int ompi_sync_wait_mt(ompi_wait_sync_t *sync) -{ - /* Don't stop if the waiting synchronization is completed. We avoid the - * race condition around the release of the synchronization using the - * signaling field. - */ - if (sync->count <= 0) { - return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; - } - - /* lock so nobody can signal us during the list updating */ - opal_mutex_lock(&sync->lock); - - /* Now that we hold the lock make sure another thread has not already - * call cond_signal. - */ - if (sync->count <= 0) { - opal_mutex_unlock(&sync->lock); - return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; - } - - /* Insert sync on the list of pending synchronization constructs */ - OPAL_THREAD_LOCK(&wait_sync_lock); - if (NULL == wait_sync_list) { - sync->next = sync->prev = sync; - wait_sync_list = sync; - } else { - sync->prev = wait_sync_list->prev; - sync->prev->next = sync; - sync->next = wait_sync_list; - wait_sync_list->prev = sync; - } - OPAL_THREAD_UNLOCK(&wait_sync_lock); - - /** - * If we are not responsible for progressing, go silent until something - * worth noticing happen: - * - this thread has been promoted to take care of the progress - * - our sync has been triggered. - */ -check_status: - if (sync != wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) { - opal_cond_wait(&sync->condition, &sync->lock); - - /** - * At this point either the sync was completed in which case - * we should remove it from the wait list, or/and I was - * promoted as the progress manager. - */ - - if (sync->count <= 0) { /* Completed? */ - opal_mutex_unlock(&sync->lock); - goto i_am_done; - } - /* either promoted, or spurious wakeup ! */ - goto check_status; - } - opal_mutex_unlock(&sync->lock); - - OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1); - while (sync->count > 0) { /* progress till completion */ - /* don't progress with the sync lock locked or you'll deadlock */ - opal_progress(); - qthread_yield(); - } - OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1); - -i_am_done: - /* My sync is now complete. Trim the list: remove self, wake next */ - OPAL_THREAD_LOCK(&wait_sync_lock); - sync->prev->next = sync->next; - sync->next->prev = sync->prev; - /* In case I am the progress manager, pass the duties on */ - if (sync == wait_sync_list) { - wait_sync_list = (sync == sync->next) ? NULL : sync->next; - if (NULL != wait_sync_list) { - WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list); - } - } - OPAL_THREAD_UNLOCK(&wait_sync_lock); - - return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; -} diff --git a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.h b/opal/mca/threads/qthreads/threads_qthreads_wait_sync.h deleted file mode 100644 index 7f6f6b5dd83..00000000000 --- a/opal/mca/threads/qthreads/threads_qthreads_wait_sync.h +++ /dev/null @@ -1,102 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H -#define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H 1 - -#include "opal/mca/threads/mutex.h" -#include "opal/mca/threads/qthreads/threads_qthreads.h" - -typedef struct ompi_wait_sync_t { - opal_atomic_int32_t count; - int32_t status; - opal_cond_t condition; - opal_mutex_t lock; - struct ompi_wait_sync_t *next; - struct ompi_wait_sync_t *prev; - volatile bool signaling; -} ompi_wait_sync_t; - -#define SYNC_WAIT(sync) (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync)) - -/* The loop in release handles a race condition between the signaling - * thread and the destruction of the condition variable. The signaling - * member will be set to false after the final signaling thread has - * finished operating on the sync object. This is done to avoid - * extra atomics in the signalling function and keep it as fast - * as possible. Note that the race window is small so spinning here - * is more optimal than sleeping since this macro is called in - * the critical path. */ -#define WAIT_SYNC_RELEASE(sync) \ - if (opal_using_threads()) { \ - while ((sync)->signaling) { \ - qthread_yield(); \ - continue; \ - } \ - opal_cond_destroy(&(sync)->condition); \ - } - -#define WAIT_SYNC_RELEASE_NOWAIT(sync) \ - if (opal_using_threads()) { \ - opal_cond_destroy(&(sync)->condition); \ - } - -#define WAIT_SYNC_SIGNAL(sync) \ - if (opal_using_threads()) { \ - opal_mutex_lock(&(sync)->lock); \ - opal_cond_signal(&(sync)->condition); \ - opal_mutex_unlock(&(sync)->lock); \ - (sync)->signaling = false; \ - } - -#define WAIT_SYNC_SIGNALLED(sync) \ - { \ - (sync)->signaling = false; \ - } - -OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync); -static inline int sync_wait_st(ompi_wait_sync_t *sync) -{ - while (sync->count > 0) { - opal_progress(); - qthread_yield(); - } - return sync->status; -} - -#define WAIT_SYNC_INIT(sync, c) \ - do { \ - (sync)->count = (c); \ - (sync)->next = NULL; \ - (sync)->prev = NULL; \ - (sync)->status = 0; \ - (sync)->signaling = (0 != (c)); \ - if (opal_using_threads()) { \ - opal_cond_init(&(sync)->condition); \ - opal_mutex_create(&(sync)->lock); \ - } \ - } while (0) - -#endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H */ diff --git a/opal/mca/threads/wait_sync.h b/opal/mca/threads/wait_sync.h index 611d37c7628..c90e3d52a5c 100644 --- a/opal/mca/threads/wait_sync.h +++ b/opal/mca/threads/wait_sync.h @@ -1,15 +1,23 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2020 The University of Tennessee and The University + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016 Mellanox Technologies. All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science + * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2021 Argonne National Laboratory. All rights reserved. * * $COPYRIGHT$ * @@ -22,13 +30,106 @@ #define OPAL_MCA_THREADS_WAIT_SYNC_H #include "opal/mca/threads/condition.h" +#include "opal/mca/threads/mutex.h" +#include "opal/mca/threads/threads.h" +#include "opal/runtime/opal_progress.h" #include "opal/sys/atomic.h" BEGIN_C_DECLS extern int opal_max_thread_in_progress; -#include MCA_threads_wait_sync_base_include_HEADER +typedef struct ompi_wait_sync_t { + opal_atomic_int32_t count; + int32_t status; + opal_thread_internal_cond_t condition; + opal_thread_internal_mutex_t lock; + struct ompi_wait_sync_t *next; + struct ompi_wait_sync_t *prev; + volatile bool signaling; +} ompi_wait_sync_t; + +#define SYNC_WAIT(sync) (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync)) + +/* The loop in release handles a race condition between the signaling + * thread and the destruction of the condition variable. The signaling + * member will be set to false after the final signaling thread has + * finished operating on the sync object. This is done to avoid + * extra atomics in the signalling function and keep it as fast + * as possible. Note that the race window is small so spinning here + * is more optimal than sleeping since this macro is called in + * the critical path. */ +#define WAIT_SYNC_RELEASE(sync) \ + if (opal_using_threads()) { \ + while ((sync)->signaling) { \ + if (opal_progress_yield_when_idle) { \ + opal_thread_yield(); \ + } \ + continue; \ + } \ + opal_thread_internal_cond_destroy(&(sync)->condition); \ + opal_thread_internal_mutex_destroy(&(sync)->lock); \ + } + +#define WAIT_SYNC_RELEASE_NOWAIT(sync) \ + if (opal_using_threads()) { \ + opal_thread_internal_cond_destroy(&(sync)->condition); \ + opal_thread_internal_mutex_destroy(&(sync)->lock); \ + } + +#define WAIT_SYNC_SIGNAL(sync) \ + if (opal_using_threads()) { \ + opal_thread_internal_mutex_lock(&(sync)->lock); \ + opal_thread_internal_cond_signal(&(sync)->condition); \ + opal_thread_internal_mutex_unlock(&(sync)->lock); \ + (sync)->signaling = false; \ + } + +#define WAIT_SYNC_SIGNALLED(sync) \ + { \ + (sync)->signaling = false; \ + } + +/* not static for inline "wait_sync_st" */ +OPAL_DECLSPEC extern ompi_wait_sync_t *wait_sync_list; + +OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync); +static inline int sync_wait_st(ompi_wait_sync_t *sync) +{ + assert(NULL == wait_sync_list); + assert(NULL == sync->next); + wait_sync_list = sync; + + while (sync->count > 0) { + opal_progress(); + } + wait_sync_list = NULL; + + return sync->status; +} + +#define WAIT_SYNC_INIT(sync, c) \ + do { \ + (sync)->count = (c); \ + (sync)->next = NULL; \ + (sync)->prev = NULL; \ + (sync)->status = 0; \ + (sync)->signaling = (0 != (c)); \ + if (opal_using_threads()) { \ + opal_thread_internal_cond_init(&(sync)->condition); \ + opal_thread_internal_mutex_init(&(sync)->lock, false); \ + } \ + } while (0) + +/** + * Wake up all syncs with a particular status. If status is OMPI_SUCCESS this + * operation is a NO-OP. Otherwise it will trigger the "error condition" from + * all registered sync. + */ +OPAL_DECLSPEC void wait_sync_global_wakeup_st(int status); +OPAL_DECLSPEC void wait_sync_global_wakeup_mt(int status); +#define wait_sync_global_wakeup(st) \ + (opal_using_threads() ? wait_sync_global_wakeup_mt(st) : wait_sync_global_wakeup_st(st)) /** * Update the status of the synchronization primitive. If an error is