From c6fab321379dfe59f5ef36b1c401b8a28e90dbb7 Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Tue, 1 Oct 2019 17:32:39 -0400 Subject: [PATCH] Fix the sigkill timeout sleep to prevent SIGCHLD from preventing completion. * The user can set `-mca odls_base_sigkill_timeout 30` to have ORTE wait 30 seconds before sending SIGTERM then another 30 seconds before sending SIGKILL to remaining processes. This usually happens on an abnormal termination. Sometimes the user wants to delay the cleanup to give the system time to write out corefile or run other diagnostics. * The problem is that child processes may be completing while ORTE is in this loop. The SIGCHLD will interrupt the `sleep` system call. Without the loop the sleep could effectively be ignored in this case. - Sleep returns the amount of time remaining to sleep. If it was interrupted by a signal then it is a positive number less than or equal to the parameter passed to it. If it slept the whole time then it returns 0. Signed-off-by: Joshua Hursey (cherry picked from commit 0e8a97c598d841d472047ea1025931813c3ef8a9) --- orte/mca/odls/base/odls_base_default_fns.c | 28 ++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/orte/mca/odls/base/odls_base_default_fns.c b/orte/mca/odls/base/odls_base_default_fns.c index a9282dd4bbd..69974bb5790 100644 --- a/orte/mca/odls/base/odls_base_default_fns.c +++ b/orte/mca/odls/base/odls_base_default_fns.c @@ -1767,7 +1767,7 @@ int orte_odls_base_default_kill_local_procs(opal_pointer_array_t *procs, orte_proc_t *child; opal_list_t procs_killed; orte_proc_t *proc, proctmp; - int i, j; + int i, j, ret; opal_pointer_array_t procarray, *procptr; bool do_cleanup; orte_odls_quick_caddy_t *cd; @@ -1913,7 +1913,17 @@ int orte_odls_base_default_kill_local_procs(opal_pointer_array_t *procs, /* if we are issuing signals, then we need to wait a little * and send the next in sequence */ if (0 < opal_list_get_size(&procs_killed)) { - sleep(orte_odls_globals.timeout_before_sigkill); + /* Wait a little. Do so in a loop since sleep() can be interrupted by a + * signal. Most likely SIGCHLD in this case */ + ret = orte_odls_globals.timeout_before_sigkill; + while( ret > 0 ) { + OPAL_OUTPUT_VERBOSE((5, orte_odls_base_framework.framework_output, + "%s Sleep %d sec (total = %d)", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + ret, orte_odls_globals.timeout_before_sigkill)); + ret = sleep(ret); + } + /* issue a SIGTERM to all */ OPAL_LIST_FOREACH(cd, &procs_killed, orte_odls_quick_caddy_t) { OPAL_OUTPUT_VERBOSE((5, orte_odls_base_framework.framework_output, @@ -1922,8 +1932,18 @@ int orte_odls_base_default_kill_local_procs(opal_pointer_array_t *procs, ORTE_NAME_PRINT(&cd->child->name))); kill_local(cd->child->pid, SIGTERM); } - /* wait a little again */ - sleep(orte_odls_globals.timeout_before_sigkill); + + /* Wait a little. Do so in a loop since sleep() can be interrupted by a + * signal. Most likely SIGCHLD in this case */ + ret = orte_odls_globals.timeout_before_sigkill; + while( ret > 0 ) { + OPAL_OUTPUT_VERBOSE((5, orte_odls_base_framework.framework_output, + "%s Sleep %d sec (total = %d)", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + ret, orte_odls_globals.timeout_before_sigkill)); + ret = sleep(ret); + } + /* issue a SIGKILL to all */ OPAL_LIST_FOREACH(cd, &procs_killed, orte_odls_quick_caddy_t) { OPAL_OUTPUT_VERBOSE((5, orte_odls_base_framework.framework_output,