Skip to content

Add wait_sync_global_wakeup() to Qthreads and Argobots threading MCA #8479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Add wait_sync_global_wakeup() to Qthreads and Argobots threading MCA #8479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cc16959
Add wait_sync_global_wakeup() to Qthreads and Argobots threading MCA.
janciesko Feb 12, 2021
1c8ed58
Merge branch 'master' into wait_sync_global_wakeup
Apr 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion opal/mca/threads/argobots/threads_argobots_wait_sync.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2017 IBM Corporation. All rights reserved.
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2019-2021 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
*
* $COPYRIGHT$
Expand Down
26 changes: 25 additions & 1 deletion opal/mca/threads/qthreads/threads_qthreads_wait_sync.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2017 IBM Corporation. All rights reserved.
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2019-2021 Sandia National Laboratories. All rights reserved.
*
* $COPYRIGHT$
*
Expand All @@ -21,6 +21,30 @@
static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
static ompi_wait_sync_t *wait_sync_list = NULL;

void wait_sync_global_wakeup_st(int status)
{
ompi_wait_sync_t* sync;
for( sync = wait_sync_list; sync != NULL; sync = sync->next ) {
wait_sync_update(sync, 0, status);
}
}

void wait_sync_global_wakeup_mt(int status)
{
ompi_wait_sync_t* sync;
opal_mutex_lock(&wait_sync_lock);
for( sync = wait_sync_list; sync != NULL; sync = sync->next ) {
/* sync_update is going to take the sync->lock from within
* the wait_sync_lock. Thread lightly here: Idealy we should
* find a way to not take a lock in a lock as this is deadlock prone,
* but as of today we are the only place doing this so it is safe.
*/
wait_sync_update(sync, 0, status);
if( sync->next == wait_sync_list ) break; /* special case for rings */
}
opal_mutex_unlock(&wait_sync_lock);
}

static opal_atomic_int32_t num_thread_in_progress = 0;

#define WAIT_SYNC_PASS_OWNERSHIP(who) \
Expand Down
56 changes: 34 additions & 22 deletions opal/mca/threads/qthreads/threads_qthreads_wait_sync.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2019-2021 Sandia National Laboratories. All rights reserved.
*
* $COPYRIGHT$
*
Expand All @@ -26,8 +26,8 @@
#ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H
#define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H 1

#include "opal/mca/threads/mutex.h"
#include "opal/mca/threads/qthreads/threads_qthreads.h"
#include "opal/mca/threads/mutex.h"

typedef struct ompi_wait_sync_t {
opal_atomic_int32_t count;
Expand All @@ -39,7 +39,8 @@ typedef struct ompi_wait_sync_t {
volatile bool signaling;
} ompi_wait_sync_t;

#define SYNC_WAIT(sync) (opal_using_threads() ? ompi_sync_wait_mt(sync) : sync_wait_st(sync))
#define SYNC_WAIT(sync) \
(opal_using_threads() ? ompi_sync_wait_mt (sync) : sync_wait_st (sync))

/* The loop in release handles a race condition between the signaling
* thread and the destruction of the condition variable. The signaling
Expand All @@ -49,31 +50,31 @@ typedef struct ompi_wait_sync_t {
* as possible. Note that the race window is small so spinning here
* is more optimal than sleeping since this macro is called in
* the critical path. */
#define WAIT_SYNC_RELEASE(sync) \
if (opal_using_threads()) { \
while ((sync)->signaling) { \
qthread_yield(); \
continue; \
} \
opal_cond_destroy(&(sync)->condition); \
#define WAIT_SYNC_RELEASE(sync) \
if (opal_using_threads()) { \
while ((sync)->signaling) { \
qthread_yield(); \
continue; \
} \
opal_cond_destroy(&(sync)->condition); \
}

#define WAIT_SYNC_RELEASE_NOWAIT(sync) \
if (opal_using_threads()) { \
opal_cond_destroy(&(sync)->condition); \
#define WAIT_SYNC_RELEASE_NOWAIT(sync) \
if (opal_using_threads()) { \
opal_cond_destroy(&(sync)->condition); \
}

#define WAIT_SYNC_SIGNAL(sync) \
if (opal_using_threads()) { \
opal_mutex_lock(&(sync)->lock); \
opal_cond_signal(&(sync)->condition); \
opal_mutex_unlock(&(sync)->lock); \
(sync)->signaling = false; \
#define WAIT_SYNC_SIGNAL(sync) \
if (opal_using_threads()) { \
opal_mutex_lock(&(sync)->lock); \
opal_cond_signal(&(sync)->condition); \
opal_mutex_unlock(&(sync)->lock); \
(sync)->signaling = false; \
}

#define WAIT_SYNC_SIGNALLED(sync) \
{ \
(sync)->signaling = false; \
#define WAIT_SYNC_SIGNALLED(sync) \
{ \
(sync)->signaling = false; \
}

OPAL_DECLSPEC int ompi_sync_wait_mt(ompi_wait_sync_t *sync);
Expand All @@ -99,4 +100,15 @@ static inline int sync_wait_st(ompi_wait_sync_t *sync)
} \
} while (0)


/**
* Wake up all syncs with a particular status. If status is OMPI_SUCCESS this
* operation is a NO-OP. Otherwise it will trigger the "error condition" from
* all registered sync.
*/
OPAL_DECLSPEC void wait_sync_global_wakeup_st(int status);
OPAL_DECLSPEC void wait_sync_global_wakeup_mt(int status);
#define wait_sync_global_wakeup(st) (opal_using_threads()? wait_sync_global_wakeup_mt(st): wait_sync_global_wakeup_st(st))


#endif /* OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_WAIT_SYNC_H */