From fb2ccd33ed5b8009775b4920517b30651014fb13 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Wed, 4 May 2022 18:37:22 +0000 Subject: [PATCH 1/2] Split out an opal-core library for tools * Fixes Issue #9869 * Split `libopen-pal.so` into two libraries: - `libopen-pal_core.la` : Internal "core" portion of OPAL containing the essential source and MCA needed for mpicc/mpirun tools to link against. The "core" library is not installed. - `libopen-pal.la` : Includes "core" plus all of the other OPAL project sources. The `.so` version of this is installed. * The "core" library contains the following: - `opal/class` - `opal/mca/backtrace` - `opal/mca/dl` - `opal/mca/installdirs` - `opal/mca/threads` - `opal/mca/timer` - `opal/runtime/*_core.[c|h]` - `opal/runtime/opal_info_support.c` - `opal/util (most - see Makefile.am)` - `opal/util/keyval` * The "core" library is linked into the following tools instead of the full `libopen-pal.so`: - `ompi/tools/mpirun` - `ompi/tools/wrappers` (by extension of `opal/tools/wrappers`) - `opal/tools/wrappers` * The `opal/runtime` files were divided into a 'core' set vs 'main' set Co-authored-by: George Bosilca Co-authored-by: Brian Barrett Signed-off-by: Joshua Hursey (cherry picked from commit fe1c3844ac0bd7ffafa5a7cb300eace0299a724c) --- config/opal_mca.m4 | 12 +- ompi/tools/mpirun/Makefile.am | 2 +- opal/Makefile.am | 20 +- opal/class/Makefile.am | 5 +- opal/mca/backtrace/configure.m4 | 4 + opal/mca/dl/configure.m4 | 4 + opal/mca/installdirs/configure.m4 | 4 + opal/mca/pmix/base/pmix_base_frame.c | 32 ++ opal/mca/threads/configure.m4 | 3 + opal/mca/timer/configure.m4 | 4 + opal/runtime/Makefile.am | 13 +- opal/runtime/opal_finalize.c | 117 +----- opal/runtime/opal_finalize_core.c | 154 ++++++++ opal/runtime/opal_init.c | 535 +++----------------------- opal/runtime/opal_init_core.c | 544 +++++++++++++++++++++++++++ opal/runtime/opal_params.c | 289 +------------- opal/runtime/opal_params.h | 66 +--- opal/runtime/opal_params_core.c | 352 +++++++++++++++++ opal/runtime/opal_params_core.h | 101 +++++ opal/runtime/opal_progress.h | 4 + opal/tools/wrappers/Makefile.am | 5 +- opal/util/Makefile.am | 22 +- opal/util/few.c | 2 - opal/util/output.c | 60 ++- opal/util/output.h | 8 + opal/util/proc.c | 52 ++- opal/util/proc.h | 4 +- test/datatype/unpack_hetero.c | 5 +- 28 files changed, 1391 insertions(+), 1032 deletions(-) create mode 100644 opal/runtime/opal_finalize_core.c create mode 100644 opal/runtime/opal_init_core.c create mode 100644 opal/runtime/opal_params_core.c create mode 100644 opal/runtime/opal_params_core.h diff --git a/config/opal_mca.m4 b/config/opal_mca.m4 index 9fd54bc9678..f9d33b66ffa 100644 --- a/config/opal_mca.m4 +++ b/config/opal_mca.m4 @@ -265,6 +265,9 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[ # can't use a variable rename here because these need to be evaled # at auto* time. + AS_LITERAL_IF([$1], [], + [m4_fatal([MCA_CONFIGURE_PROJECT argument must be a literal])]) + opal_show_subtitle "Configuring MCA for $1" AC_MSG_CHECKING([for frameworks for $1]) @@ -291,6 +294,7 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[ MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS= MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS= MCA_$1_FRAMEWORK_LIBS= + MCA_$1_FRAMEWORK_CORE_LIBS= m4_foreach(mca_framework, [mca_$1_framework_list], [m4_ifval(mca_framework, @@ -301,14 +305,17 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[ MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS="[\$(MCA_]$1[_]mca_framework[_ALL_SUBDIRS)] $MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS" MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS="[\$(MCA_]$1[_]mca_framework[_DSO_SUBDIRS)] $MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS" MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS="[\$(MCA_]$1[_]mca_framework[_STATIC_SUBDIRS)] $MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS" + mca_$1_framework_base_lib= ], [ MCA_$1_FRAMEWORKS="$MCA_$1_FRAMEWORKS mca_framework" MCA_$1_FRAMEWORKS_SUBDIRS="$MCA_$1_FRAMEWORKS_SUBDIRS [mca/]mca_framework" MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS="$MCA_$1_FRAMEWORK_COMPONENT_ALL_SUBDIRS [\$(MCA_]$1[_]mca_framework[_ALL_SUBDIRS)]" MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS="$MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS [\$(MCA_]$1[_]mca_framework[_DSO_SUBDIRS)]" MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS="$MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS [\$(MCA_]$1[_]mca_framework[_STATIC_SUBDIRS)]" - MCA_$1_FRAMEWORK_LIBS="$MCA_$1_FRAMEWORK_LIBS [mca/]mca_framework[/libmca_]mca_framework[.la]"]) - MCA_$1_FRAMEWORK_LIBS="$MCA_$1_FRAMEWORK_LIBS [\$(MCA_]$1[_]mca_framework[_STATIC_LTLIBS)]" + mca_$1_framework_base_lib="[mca/]mca_framework[/libmca_]mca_framework[.la]"]) + m4_ifdef([MCA_]$1[_]mca_framework[_CORE_LIB], + [MCA_$1_FRAMEWORK_CORE_LIBS="$MCA_$1_FRAMEWORK_CORE_LIBS ${mca_$1_framework_base_lib} [\$(MCA_]$1[_]mca_framework[_STATIC_LTLIBS)]"], + [MCA_$1_FRAMEWORK_LIBS="$MCA_$1_FRAMEWORK_LIBS ${mca_$1_framework_base_lib} [\$(MCA_]$1[_]mca_framework[_STATIC_LTLIBS)]"]) m4_ifdef([MCA_]$1[_]mca_framework[_CONFIG], [MCA_]$1[_]mca_framework[_CONFIG]($1, mca_framework), [MCA_CONFIGURE_FRAMEWORK($1, mca_framework, 1)])])]) @@ -324,6 +331,7 @@ AC_DEFUN([MCA_CONFIGURE_PROJECT],[ AC_SUBST(MCA_$1_FRAMEWORK_COMPONENT_DSO_SUBDIRS) AC_SUBST(MCA_$1_FRAMEWORK_COMPONENT_STATIC_SUBDIRS) AC_SUBST(MCA_$1_FRAMEWORK_LIBS) + AC_SUBST(MCA_$1_FRAMEWORK_CORE_LIBS) ]) # MCA_ORDER_COMPONENT_LIST(project_name, framework_name) diff --git a/ompi/tools/mpirun/Makefile.am b/ompi/tools/mpirun/Makefile.am index 1953eab4a6f..45d3b75a5c7 100644 --- a/ompi/tools/mpirun/Makefile.am +++ b/ompi/tools/mpirun/Makefile.am @@ -20,7 +20,7 @@ mpirun_SOURCES = \ main.c mpirun_LDADD = \ - $(top_builddir)/opal/libopen-pal.la + $(top_builddir)/opal/libopen-pal_core.la install-exec-hook: (cd $(DESTDIR)$(bindir); rm -f mpiexec$(EXEEXT); $(LN_S) mpirun$(EXEEXT) mpiexec$(EXEEXT)) diff --git a/opal/Makefile.am b/opal/Makefile.am index 9e150e81b82..4b52e7c85d3 100644 --- a/opal/Makefile.am +++ b/opal/Makefile.am @@ -2,7 +2,7 @@ # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. -# Copyright (c) 2004-2009 The University of Tennessee and The University +# Copyright (c) 2004-2022 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, @@ -55,18 +55,29 @@ DIST_SUBDIRS = \ $(MCA_opal_FRAMEWORKS_SUBDIRS) \ $(MCA_opal_FRAMEWORK_COMPONENT_ALL_SUBDIRS) -# Build the main OPAL library +noinst_LTLIBRARIES = libopen-pal_core.la lib_LTLIBRARIES = lib@OPAL_LIB_NAME@.la + +libopen_pal_core_la_SOURCES = +libopen_pal_core_la_LIBADD = \ + mca/base/libmca_base.la \ + util/libopalutil_core.la \ + $(MCA_opal_FRAMEWORK_CORE_LIBS) +libopen_pal_core_la_DEPENDENCIES = \ + mca/base/libmca_base.la \ + util/libopalutil_core.la \ + $(MCA_opal_FRAMEWORK_CORE_LIBS) + lib@OPAL_LIB_NAME@_la_SOURCES = lib@OPAL_LIB_NAME@_la_LIBADD = \ + libopen-pal_core.la \ datatype/libdatatype.la \ - mca/base/libmca_base.la \ util/libopalutil.la \ $(LIBOPAL_GPU_LA) \ $(MCA_opal_FRAMEWORK_LIBS) lib@OPAL_LIB_NAME@_la_DEPENDENCIES = \ + libopen-pal_core.la \ datatype/libdatatype.la \ - mca/base/libmca_base.la \ util/libopalutil.la \ $(LIBOPAL_GPU_LA) \ $(MCA_opal_FRAMEWORK_LIBS) @@ -74,7 +85,6 @@ lib@OPAL_LIB_NAME@_la_LDFLAGS = -version-info @libopen_pal_so_version@ # included subdirectory Makefile.am's and appended-to variables headers = -noinst_LTLIBRARIES = dist_opaldata_DATA = lib@OPAL_LIB_NAME@_la_SOURCES += $(headers) diff --git a/opal/class/Makefile.am b/opal/class/Makefile.am index ed49054c8ea..1d3beb50a78 100644 --- a/opal/class/Makefile.am +++ b/opal/class/Makefile.am @@ -3,7 +3,7 @@ # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. -# Copyright (c) 2004-2007 The University of Tennessee and The University +# Copyright (c) 2004-2022 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -14,6 +14,7 @@ # Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights # reserved. # Copyright (c) 2021 Nanook Consulting. All rights reserved. +# Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -42,7 +43,7 @@ headers += \ class/opal_rb_tree.h \ class/opal_interval_tree.h -lib@OPAL_LIB_NAME@_la_SOURCES += \ +libopen_pal_core_la_SOURCES += \ class/opal_bitmap.c \ class/opal_cstring.c \ class/opal_free_list.c \ diff --git a/opal/mca/backtrace/configure.m4 b/opal/mca/backtrace/configure.m4 index 0ee4261f3e8..4f6f8478d3a 100644 --- a/opal/mca/backtrace/configure.m4 +++ b/opal/mca/backtrace/configure.m4 @@ -10,6 +10,7 @@ dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2006 The Regents of the University of California. dnl All rights reserved. +dnl Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -17,5 +18,8 @@ dnl dnl $HEADER$ dnl +dnl need in the core library +AC_DEFUN([MCA_opal_backtrace_CORE_LIB], [1]) + dnl we only want one :) m4_define(MCA_opal_backtrace_CONFIGURE_MODE, STOP_AT_FIRST) diff --git a/opal/mca/dl/configure.m4 b/opal/mca/dl/configure.m4 index b1da4cd09dc..289b351a31d 100644 --- a/opal/mca/dl/configure.m4 +++ b/opal/mca/dl/configure.m4 @@ -1,6 +1,7 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. +dnl Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -8,6 +9,9 @@ dnl dnl $HEADER$ dnl +dnl need in the core library +AC_DEFUN([MCA_opal_dl_CORE_LIB], [1]) + dnl There will only be one component used in this framework, and it will dnl be selected at configure time by priority. Components must set dnl their priorities in their configure.m4 file. diff --git a/opal/mca/installdirs/configure.m4 b/opal/mca/installdirs/configure.m4 index 081a3dcfc67..0d8934675ec 100644 --- a/opal/mca/installdirs/configure.m4 +++ b/opal/mca/installdirs/configure.m4 @@ -1,6 +1,7 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2006-2010 Sandia National Laboratories. All rights reserved. +dnl Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -8,4 +9,7 @@ dnl dnl $HEADER$ dnl +dnl need in the core library +AC_DEFUN([MCA_opal_installdirs_CORE_LIB], [1]) + AC_DEFUN([MCA_opal_installdirs_CONFIGURE_MODE], [PRIORITY]) diff --git a/opal/mca/pmix/base/pmix_base_frame.c b/opal/mca/pmix/base/pmix_base_frame.c index 319b21ccddc..cda84d77aba 100644 --- a/opal/mca/pmix/base/pmix_base_frame.c +++ b/opal/mca/pmix/base/pmix_base_frame.c @@ -1,6 +1,9 @@ /* * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2022 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -16,6 +19,7 @@ #include "opal/mca/threads/thread_usage.h" #include "opal/util/argv.h" #include "opal/util/output.h" +#include "opal/util/proc.h" #include "opal/mca/pmix/base/base.h" #include "opal/mca/pmix/pmix-internal.h" @@ -59,6 +63,32 @@ static int opal_pmix_base_frame_register(mca_base_register_flag_t flags) return OPAL_SUCCESS; } +static char* +opal_get_proc_hostname_using_pmix(const opal_proc_t *proc) +{ + int ret; + char *hostname; + + /* if the proc is NULL, then we can't know */ + if (NULL == proc) { + return strdup("unknown"); + } + + /* if it is my own hostname we are after, then just hand back + * the value in opal_process_info */ + if (proc == opal_proc_local_get()) { + return strdup(opal_process_info.nodename); + } + /* if we don't already have it, then try to get it */ + OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_HOSTNAME, &proc->proc_name, (char **) &hostname, + PMIX_STRING); + if (OPAL_SUCCESS != ret) { + return strdup("unknown"); // return something so the caller doesn't segfault + } + /* user is not allowed to release the data */ + return hostname; +} + static int opal_pmix_base_frame_close(void) { int rc; @@ -77,6 +107,8 @@ static int opal_pmix_base_frame_open(mca_base_open_flag_t flags) opal_pmix_base.evbase = opal_sync_event_base; /* pass across the verbosity */ opal_pmix_verbose_output = opal_pmix_base_framework.framework_output; + /* Set the distributed name service via PMIx */ + opal_get_proc_hostname = opal_get_proc_hostname_using_pmix; return rc; } diff --git a/opal/mca/threads/configure.m4 b/opal/mca/threads/configure.m4 index 1edaa8b2f34..6325401ad31 100644 --- a/opal/mca/threads/configure.m4 +++ b/opal/mca/threads/configure.m4 @@ -22,6 +22,9 @@ dnl dnl $HEADER$ dnl +dnl need in the core library +AC_DEFUN([MCA_opal_threads_CORE_LIB], [1]) + dnl we only want one :) m4_define(MCA_opal_threads_CONFIGURE_MODE, STOP_AT_FIRST) diff --git a/opal/mca/timer/configure.m4 b/opal/mca/timer/configure.m4 index 14ec0ec9694..9bc41ea20c9 100644 --- a/opal/mca/timer/configure.m4 +++ b/opal/mca/timer/configure.m4 @@ -11,6 +11,7 @@ dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. dnl Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +dnl Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -18,6 +19,9 @@ dnl dnl $HEADER$ dnl +dnl need in the core library +AC_DEFUN([MCA_opal_timer_CORE_LIB], [1]) + dnl we only want one :) m4_define(MCA_opal_timer_CONFIGURE_MODE, STOP_AT_FIRST) diff --git a/opal/runtime/Makefile.am b/opal/runtime/Makefile.am index 0f3abc64e06..5d28b4f3d6c 100644 --- a/opal/runtime/Makefile.am +++ b/opal/runtime/Makefile.am @@ -3,7 +3,7 @@ # Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. -# Copyright (c) 2004-2020 The University of Tennessee and The University +# Copyright (c) 2004-2022 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -14,6 +14,7 @@ # All rights reserved. # Copyright (c) 2014 Intel, Inc. All rights reserved # Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -35,12 +36,18 @@ headers += \ runtime/opal.h \ runtime/opal_info_support.h \ runtime/opal_params.h \ + runtime/opal_params_core.h \ runtime/opal_progress_threads.h +libopen_pal_core_la_SOURCES += \ + runtime/opal_params_core.c \ + runtime/opal_finalize_core.c \ + runtime/opal_init_core.c \ + runtime/opal_info_support.c + lib@OPAL_LIB_NAME@_la_SOURCES += \ + runtime/opal_params.c \ runtime/opal_progress.c \ runtime/opal_finalize.c \ runtime/opal_init.c \ - runtime/opal_params.c \ - runtime/opal_info_support.c \ runtime/opal_progress_threads.c diff --git a/opal/runtime/opal_finalize.c b/opal/runtime/opal_finalize.c index 94cd775af5b..dd51b840616 100644 --- a/opal/runtime/opal_finalize.c +++ b/opal/runtime/opal_finalize.c @@ -3,7 +3,7 @@ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2022 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -43,121 +43,6 @@ #include "opal/util/show_help.h" extern int opal_initialized; -extern int opal_util_initialized; -extern bool opal_init_called; - -static opal_mutex_t opal_finalize_cleanup_fns_lock = OPAL_MUTEX_STATIC_INIT; -opal_list_t opal_finalize_cleanup_fns = {{0}}; - -struct opal_cleanup_fn_item_t { - opal_list_item_t super; - opal_cleanup_fn_t cleanup_fn; - void *user_data; -#if OPAL_ENABLE_DEBUG - char *cleanup_fn_name; -#endif -}; - -typedef struct opal_cleanup_fn_item_t opal_cleanup_fn_item_t; -OBJ_CLASS_DECLARATION(opal_cleanup_fn_item_t); - -static void opal_cleanup_fn_item_construct(opal_cleanup_fn_item_t *item) -{ -#if OPAL_ENABLE_DEBUG - item->cleanup_fn_name = NULL; -#endif -} - -static void opal_cleanup_fn_item_destruct(opal_cleanup_fn_item_t *item) -{ -#if OPAL_ENABLE_DEBUG - free(item->cleanup_fn_name); - item->cleanup_fn_name = NULL; -#endif -} - -OBJ_CLASS_INSTANCE(opal_cleanup_fn_item_t, opal_list_item_t, opal_cleanup_fn_item_construct, - opal_cleanup_fn_item_destruct); - -static void opal_finalize_domain_construct(opal_finalize_domain_t *domain) -{ - domain->domain_name = NULL; -} - -static void opal_finalize_domain_destruct(opal_finalize_domain_t *domain) -{ - free(domain->domain_name); - domain->domain_name = NULL; -} - -OBJ_CLASS_INSTANCE(opal_finalize_domain_t, opal_list_t, opal_finalize_domain_construct, - opal_finalize_domain_destruct); - -static opal_finalize_domain_t *current_finalize_domain; -opal_finalize_domain_t opal_init_util_domain = {{{0}}}; -opal_finalize_domain_t opal_init_domain = {{{0}}}; - -void opal_finalize_append_cleanup(opal_cleanup_fn_t cleanup_fn, const char *fn_name, - void *user_data) -{ - opal_cleanup_fn_item_t *cleanup_item = OBJ_NEW(opal_cleanup_fn_item_t); - assert(NULL != cleanup_item); - cleanup_item->cleanup_fn = cleanup_fn; - cleanup_item->user_data = user_data; -#if OPAL_ENABLE_DEBUG - cleanup_item->cleanup_fn_name = strdup(fn_name); - assert(NULL != cleanup_item->cleanup_fn_name); -#else - (void) fn_name; -#endif - - opal_mutex_lock(&opal_finalize_cleanup_fns_lock); - opal_list_append(¤t_finalize_domain->super, &cleanup_item->super); - opal_mutex_unlock(&opal_finalize_cleanup_fns_lock); -} - -void opal_finalize_domain_init(opal_finalize_domain_t *domain, const char *domain_name) -{ - free(domain->domain_name); - domain->domain_name = domain_name ? strdup(domain_name) : NULL; -} - -void opal_finalize_set_domain(opal_finalize_domain_t *domain) -{ - current_finalize_domain = domain; -} - -void opal_finalize_cleanup_domain(opal_finalize_domain_t *domain) -{ - opal_cleanup_fn_item_t *cleanup_item, *next; - /* call any registered cleanup functions before tearing down OPAL */ - OPAL_LIST_FOREACH_SAFE_REV (cleanup_item, next, &domain->super, opal_cleanup_fn_item_t) { - cleanup_item->cleanup_fn(cleanup_item->user_data); - opal_list_remove_item(&domain->super, &cleanup_item->super); - OBJ_RELEASE(cleanup_item); - } -} - -int opal_finalize_util(void) -{ - if (--opal_util_initialized != 0) { - if (opal_util_initialized < 0) { - return OPAL_ERROR; - } - return OPAL_SUCCESS; - } - - opal_finalize_cleanup_domain(&opal_init_util_domain); - OBJ_DESTRUCT(&opal_init_util_domain); - - /* finalize the class/object system */ - opal_class_finalize(); - - free(opal_process_info.nodename); - opal_process_info.nodename = NULL; - - return OPAL_SUCCESS; -} int opal_finalize(void) { diff --git a/opal/runtime/opal_finalize_core.c b/opal/runtime/opal_finalize_core.c new file mode 100644 index 00000000000..c7516477d44 --- /dev/null +++ b/opal/runtime/opal_finalize_core.c @@ -0,0 +1,154 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2022 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2015 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 Amazon.com, Inc. or its affiliates. + * All Rights reserved. + * Copyright (c) 2018 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file **/ + +#include "opal_config.h" + +#include "opal/class/opal_object.h" +#include "opal/constants.h" +#include "opal/runtime/opal.h" +#include "opal/util/output.h" +#include "opal/util/proc.h" + +extern int opal_util_initialized; + +static opal_mutex_t opal_finalize_cleanup_fns_lock = OPAL_MUTEX_STATIC_INIT; +opal_list_t opal_finalize_cleanup_fns = {{0}}; + +struct opal_cleanup_fn_item_t { + opal_list_item_t super; + opal_cleanup_fn_t cleanup_fn; + void *user_data; +#if OPAL_ENABLE_DEBUG + char *cleanup_fn_name; +#endif +}; + +typedef struct opal_cleanup_fn_item_t opal_cleanup_fn_item_t; +OBJ_CLASS_DECLARATION(opal_cleanup_fn_item_t); + +static void opal_cleanup_fn_item_construct(opal_cleanup_fn_item_t *item) +{ +#if OPAL_ENABLE_DEBUG + item->cleanup_fn_name = NULL; +#endif +} + +static void opal_cleanup_fn_item_destruct(opal_cleanup_fn_item_t *item) +{ +#if OPAL_ENABLE_DEBUG + free(item->cleanup_fn_name); + item->cleanup_fn_name = NULL; +#endif +} + +OBJ_CLASS_INSTANCE(opal_cleanup_fn_item_t, opal_list_item_t, opal_cleanup_fn_item_construct, + opal_cleanup_fn_item_destruct); + +static void opal_finalize_domain_construct(opal_finalize_domain_t *domain) +{ + domain->domain_name = NULL; +} + +static void opal_finalize_domain_destruct(opal_finalize_domain_t *domain) +{ + free(domain->domain_name); + domain->domain_name = NULL; +} + +OBJ_CLASS_INSTANCE(opal_finalize_domain_t, opal_list_t, opal_finalize_domain_construct, + opal_finalize_domain_destruct); + +static opal_finalize_domain_t *current_finalize_domain; +opal_finalize_domain_t opal_init_util_domain = {{{0}}}; +opal_finalize_domain_t opal_init_domain = {{{0}}}; + +void opal_finalize_append_cleanup(opal_cleanup_fn_t cleanup_fn, const char *fn_name, + void *user_data) +{ + opal_cleanup_fn_item_t *cleanup_item = OBJ_NEW(opal_cleanup_fn_item_t); + assert(NULL != cleanup_item); + cleanup_item->cleanup_fn = cleanup_fn; + cleanup_item->user_data = user_data; +#if OPAL_ENABLE_DEBUG + cleanup_item->cleanup_fn_name = strdup(fn_name); + assert(NULL != cleanup_item->cleanup_fn_name); +#else + (void) fn_name; +#endif + + opal_mutex_lock(&opal_finalize_cleanup_fns_lock); + opal_list_append(¤t_finalize_domain->super, &cleanup_item->super); + opal_mutex_unlock(&opal_finalize_cleanup_fns_lock); +} + +void opal_finalize_domain_init(opal_finalize_domain_t *domain, const char *domain_name) +{ + free(domain->domain_name); + domain->domain_name = domain_name ? strdup(domain_name) : NULL; +} + +void opal_finalize_set_domain(opal_finalize_domain_t *domain) +{ + current_finalize_domain = domain; +} + +void opal_finalize_cleanup_domain(opal_finalize_domain_t *domain) +{ + opal_cleanup_fn_item_t *cleanup_item, *next; + /* call any registered cleanup functions before tearing down OPAL */ + OPAL_LIST_FOREACH_SAFE_REV (cleanup_item, next, &domain->super, opal_cleanup_fn_item_t) { + cleanup_item->cleanup_fn(cleanup_item->user_data); + opal_list_remove_item(&domain->super, &cleanup_item->super); + OBJ_RELEASE(cleanup_item); + } +} + +int opal_finalize_util(void) +{ + if (--opal_util_initialized != 0) { + if (opal_util_initialized < 0) { + return OPAL_ERROR; + } + return OPAL_SUCCESS; + } + + opal_finalize_cleanup_domain(&opal_init_util_domain); + OBJ_DESTRUCT(&opal_init_util_domain); + + /* finalize the class/object system */ + opal_class_finalize(); + + free(opal_process_info.nodename); + opal_process_info.nodename = NULL; + + return OPAL_SUCCESS; +} + diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index f5eb3983779..e9fdd406cde 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -3,7 +3,7 @@ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2020 The University of Tennessee and The University + * Copyright (c) 2004-2022 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -18,7 +18,7 @@ * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates. + * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates. All Rights reserved. * All Rights reserved. * Copyright (c) 2018 Mellanox Technologies, Inc. * All rights reserved. @@ -86,339 +86,13 @@ const char opal_version_string[] = OPAL_IDENT_STRING; int opal_initialized = 0; -bool opal_init_called = false; -int opal_util_initialized = 0; /* We have to put a guess in here in case hwloc is not available. If hwloc is available, this value will be overwritten when the hwloc data is loaded. */ int opal_cache_line_size = 128; -bool opal_warn_on_fork = true; -/* If there is a preprocessor macro that redefined the call to - * gethostname, we undefine that here */ -#ifdef gethostname -# undef gethostname -#endif - -#define NUM_TRIES_FOR_NULL_HOSTNAME 8 - -/* - * This gethostname wrapper does not return the full-length hostname in - * those rare cases where it is too long for the buffer. It does, however, - * guarantee a null-terminated hostname is returned, even if it's - * truncated. It also tries again in the case where gethostname returns an - * error because the buffer is initially too short. - */ -int opal_init_gethostname(void) -{ - size_t count, length = OPAL_LOCAL_MAXHOSTNAMELEN; - int ret_val, num_tries = 0; - - char *buf = calloc(1, length); - if (NULL == buf) { - return OPAL_ERR_OUT_OF_RESOURCE; - } - - while (num_tries < NUM_TRIES_FOR_NULL_HOSTNAME) { - ++num_tries; - - /* - * Offer all but the last byte of the buffer to gethostname. - */ - ret_val = gethostname(buf, length - 1); - /* - * Terminate the buffer in the last position. - */ - buf[length - 1] = '\0'; - if (0 == ret_val) { - count = strlen(buf); - /* The result was not truncated */ - if (count > 0 && count < length - 1) { - /* - * If we got a good result, save it. This value may - * be longer than what callers to opal_gethostname() - * are expecting, so that should be checked by the - * caller. - */ - opal_process_info.nodename = buf; - return OPAL_SUCCESS; - } - /* - * "Good" cases: - * - * 0 == count: The buffer is empty. In some gethostname - * implementations, this can be because the - * buffer was too small. - * (length-1) == count: The result *may* be truncated. - * - * If it's one of these cases, we'll fall through to - * increase the length of the buffer and try again. - * - * If it's not one of these good cases, it's an error: - * return. - */ - else if (!(0 == count || count == length - 1)) { - free(buf); - return OPAL_ERR_IN_ERRNO; - } - } - /* - * "Good" cases: - * - * errno == EINVAL or ENAMETOOLONG: hostname was truncated and - * there was an error. Perhaps there is something - * in the buffer and perhaps not. - * - * If it's one of these cases, we'll fall through to - * increase the length of the buffer and try again. - * - * If it's not one of these good cases, it's an error: return. - */ - else if (!(EINVAL == errno || ENAMETOOLONG == errno)) { - free(buf); - return OPAL_ERR_IN_ERRNO; - } - - /* - * If we get here, it means we want to double the length of - * the buffer and try again. - */ - length *= 2; - buf = realloc(buf, length); - if (NULL == buf) { - return OPAL_ERR_OUT_OF_RESOURCE; - } - } /* end while */ - - /* If we got here, it means that we tried too many times and are - * giving up. */ - free(buf); - return OPAL_ERR_NOT_FOUND; -} - -static int opal_err2str(int errnum, const char **errmsg) -{ - const char *retval; - - switch (errnum) { - case OPAL_SUCCESS: - retval = "Success"; - break; - case OPAL_ERROR: - retval = "Error"; - break; - case OPAL_ERR_OUT_OF_RESOURCE: - retval = "Out of resource"; - break; - case OPAL_ERR_TEMP_OUT_OF_RESOURCE: - retval = "Temporarily out of resource"; - break; - case OPAL_ERR_RESOURCE_BUSY: - retval = "Resource busy"; - break; - case OPAL_ERR_BAD_PARAM: - retval = "Bad parameter"; - break; - case OPAL_ERR_FATAL: - retval = "Fatal"; - break; - case OPAL_ERR_NOT_IMPLEMENTED: - retval = "Not implemented"; - break; - case OPAL_ERR_NOT_SUPPORTED: - retval = "Not supported"; - break; - case OPAL_ERR_INTERRUPTED: - retval = "Interrupted"; - break; - case OPAL_ERR_WOULD_BLOCK: - retval = "Would block"; - break; - case OPAL_ERR_IN_ERRNO: - retval = "In errno"; - break; - case OPAL_ERR_UNREACH: - retval = "Unreachable"; - break; - case OPAL_ERR_NOT_FOUND: - retval = "Not found"; - break; - case OPAL_EXISTS: - retval = "Exists"; - break; - case OPAL_ERR_TIMEOUT: - retval = "Timeout"; - break; - case OPAL_ERR_NOT_AVAILABLE: - retval = "Not available"; - break; - case OPAL_ERR_PERM: - retval = "No permission"; - break; - case OPAL_ERR_VALUE_OUT_OF_BOUNDS: - retval = "Value out of bounds"; - break; - case OPAL_ERR_FILE_READ_FAILURE: - retval = "File read failure"; - break; - case OPAL_ERR_FILE_WRITE_FAILURE: - retval = "File write failure"; - break; - case OPAL_ERR_FILE_OPEN_FAILURE: - retval = "File open failure"; - break; - case OPAL_ERR_PACK_MISMATCH: - retval = "Pack data mismatch"; - break; - case OPAL_ERR_PACK_FAILURE: - retval = "Data pack failed"; - break; - case OPAL_ERR_UNPACK_FAILURE: - retval = "Data unpack failed"; - break; - case OPAL_ERR_UNPACK_INADEQUATE_SPACE: - retval = "Data unpack had inadequate space"; - break; - case OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - retval = "Data unpack would read past end of buffer"; - break; - case OPAL_ERR_OPERATION_UNSUPPORTED: - retval = "Requested operation is not supported on referenced data type"; - break; - case OPAL_ERR_UNKNOWN_DATA_TYPE: - retval = "Unknown data type"; - break; - case OPAL_ERR_BUFFER: - retval = "Buffer type (described vs non-described) mismatch - operation not allowed"; - break; - case OPAL_ERR_DATA_TYPE_REDEF: - retval = "Attempt to redefine an existing data type"; - break; - case OPAL_ERR_DATA_OVERWRITE_ATTEMPT: - retval = "Attempt to overwrite a data value"; - break; - case OPAL_ERR_MODULE_NOT_FOUND: - retval = "Framework requires at least one active module, but none found"; - break; - case OPAL_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED: - retval = "OS topology does not support slot_list process affinity"; - break; - case OPAL_ERR_TOPO_SOCKET_NOT_SUPPORTED: - retval = "Could not obtain socket topology information"; - break; - case OPAL_ERR_TOPO_CORE_NOT_SUPPORTED: - retval = "Could not obtain core topology information"; - break; - case OPAL_ERR_NOT_ENOUGH_SOCKETS: - retval = "Not enough sockets to meet request"; - break; - case OPAL_ERR_NOT_ENOUGH_CORES: - retval = "Not enough cores to meet request"; - break; - case OPAL_ERR_INVALID_PHYS_CPU: - retval = "Invalid physical cpu number returned"; - break; - case OPAL_ERR_MULTIPLE_AFFINITIES: - retval = "Multiple methods for assigning process affinity were specified"; - break; - case OPAL_ERR_SLOT_LIST_RANGE: - retval = "Provided slot_list range is invalid"; - break; - case OPAL_ERR_NETWORK_NOT_PARSEABLE: - retval = "Provided network specification is not parseable"; - break; - case OPAL_ERR_SILENT: - retval = NULL; - break; - case OPAL_ERR_NOT_INITIALIZED: - retval = "Not initialized"; - break; - case OPAL_ERR_NOT_BOUND: - retval = "Not bound"; - break; - case OPAL_ERR_TAKE_NEXT_OPTION: - retval = "Take next option"; - break; - case OPAL_ERR_PROC_ENTRY_NOT_FOUND: - retval = "Database entry not found"; - break; - case OPAL_ERR_DATA_VALUE_NOT_FOUND: - retval = "Data for specified key not found"; - break; - case OPAL_ERR_CONNECTION_FAILED: - retval = "Connection failed"; - break; - case OPAL_ERR_AUTHENTICATION_FAILED: - retval = "Authentication failed"; - break; - case OPAL_ERR_COMM_FAILURE: - retval = "Comm failure"; - break; - case OPAL_ERR_SERVER_NOT_AVAIL: - retval = "Server not available"; - break; - case OPAL_ERR_IN_PROCESS: - retval = "Operation in process"; - break; - case OPAL_ERR_DEBUGGER_RELEASE: - retval = "Release debugger"; - break; - case OPAL_ERR_HANDLERS_COMPLETE: - retval = "Event handlers complete"; - break; - case OPAL_ERR_PARTIAL_SUCCESS: - retval = "Partial success"; - break; - case OPAL_ERR_PROC_ABORTED: - retval = "Process abnormally terminated"; - break; - case OPAL_ERR_PROC_REQUESTED_ABORT: - retval = "Process requested abort"; - break; - case OPAL_ERR_PROC_ABORTING: - retval = "Process is aborting"; - break; - case OPAL_ERR_NODE_DOWN: - retval = "Node has gone down"; - break; - case OPAL_ERR_NODE_OFFLINE: - retval = "Node has gone offline"; - break; - case OPAL_ERR_JOB_TERMINATED: - retval = "Job terminated"; - break; - case OPAL_ERR_PROC_RESTART: - retval = "Process restarted"; - break; - case OPAL_ERR_PROC_CHECKPOINT: - retval = "Process checkpoint"; - break; - case OPAL_ERR_PROC_MIGRATE: - retval = "Process migrate"; - break; - case OPAL_ERR_EVENT_REGISTRATION: - retval = "Event registration"; - break; - case OPAL_ERR_HEARTBEAT_ALERT: - retval = "Heartbeat not received"; - break; - case OPAL_ERR_FILE_ALERT: - retval = "File alert - proc may have stalled"; - break; - case OPAL_ERR_MODEL_DECLARED: - retval = "Model declared"; - break; - case OPAL_PMIX_LAUNCH_DIRECTIVE: - retval = "Launch directive"; - break; - - default: - retval = "UNRECOGNIZED"; - } - - *errmsg = retval; - return OPAL_SUCCESS; -} +/* Defined in opal_util_init.c and part of the opal_util.la library */ +int opal_init_error(const char *error, int ret); int opal_init_psm(void) { @@ -447,90 +121,54 @@ int opal_init_psm(void) return OPAL_SUCCESS; } -static int opal_init_error(const char *error, int ret) -{ - if (OPAL_ERR_SILENT != ret) { - opal_show_help("help-opal-runtime.txt", "opal_init:startup:internal-failure", true, error, - ret); - } - return ret; -} - -static mca_base_framework_t *opal_init_util_frameworks[] = { - &opal_installdirs_base_framework, +/* the memcpy component should be one of the first who get + * loaded in order to make sure we have all the available + * versions of memcpy correctly configured. + */ +static mca_base_framework_t *opal_init_frameworks[] = { &opal_if_base_framework, + &opal_threads_base_framework, &opal_hwloc_base_framework, + &opal_memcpy_base_framework, &opal_memchecker_base_framework, + &opal_backtrace_base_framework, &opal_timer_base_framework, + &opal_shmem_base_framework, &opal_reachable_base_framework, + &opal_pmix_base_framework, NULL, }; -int opal_init_util(int *pargc, char ***pargv) +int opal_init(int *pargc, char ***pargv) { int ret; - char *error = NULL; - OPAL_TIMING_ENV_INIT(otmng); - if (opal_util_initialized != 0) { - if (opal_util_initialized < 0) { + if (opal_initialized != 0) { + if (opal_initialized < 0) { return OPAL_ERROR; } - ++opal_util_initialized; + ++opal_initialized; return OPAL_SUCCESS; } - OBJ_CONSTRUCT(&opal_init_util_domain, opal_finalize_domain_t); - (void) opal_finalize_domain_init(&opal_init_util_domain, "opal_init_util"); - opal_finalize_set_domain(&opal_init_util_domain); - - opal_thread_set_main(); - - opal_init_called = true; - - /* register for */ - opal_finalize_register_cleanup_arg (mca_base_framework_close_list, opal_init_util_frameworks); - - /* set the nodename right away so anyone who needs it has it. Note - * that we don't bother with fqdn and prefix issues here - we let - * the RTE later replace this with a modified name if the user - * requests it */ - ret = opal_init_gethostname(); - if (OPAL_SUCCESS != ret) { - fprintf(stderr, - "opal_init_gethostname() failed -- process will likely abort (%s:%d, returned %d " - "instead of OPAL_SUCCESS)\n", - __FILE__, __LINE__, ret); + /* initialize util code */ + if (OPAL_SUCCESS != (ret = opal_init_util(pargc, pargv))) { return ret; } - /* initialize the memory allocator */ - opal_malloc_init(); - - OPAL_TIMING_ENV_NEXT(otmng, "opal_malloc_init"); - - /* initialize the output system */ - opal_output_init(); - - /* initialize install dirs code */ - if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_installdirs_base_framework, 0))) { + /* initialize if framework */ + if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_if_base_framework, 0))) { fprintf(stderr, - "opal_installdirs_base_open() failed -- process will likely abort (%s:%d, returned " - "%d instead of OPAL_SUCCESS)\n", + "opal_if_base_open() failed -- process will likely abort (%s:%d, returned %d " + "instead of OPAL_SUCCESS)\n", __FILE__, __LINE__, ret); return ret; } - /* initialize the help system */ - opal_show_help_init(); - - OPAL_TIMING_ENV_NEXT(otmng, "opal_show_help_init"); + OPAL_TIMING_ENV_NEXT(otmng, "opal_if_init"); - /* register handler for errnum -> string conversion */ - if (OPAL_SUCCESS - != (ret = opal_error_register("OPAL", OPAL_ERR_BASE, OPAL_ERR_MAX, opal_err2str))) { - return opal_init_error("opal_error_register", ret); - } + /* register PMIx cleanup function for output streams */ + opal_output_register_pmix_cleanup_fn(&opal_pmix_register_cleanup); - /* keyval lex-based parser */ - if (OPAL_SUCCESS != (ret = opal_util_keyval_parse_init())) { - return opal_init_error("opal_util_keyval_parse_init", ret); + /* register non-core MCA parameters for opal */ + if (OPAL_SUCCESS != (ret = opal_register_params())) { + return opal_init_error("opal_register_params", ret); } // Disable PSM signal hijacking (see comment in function for more @@ -539,128 +177,45 @@ int opal_init_util(int *pargc, char ***pargv) OPAL_TIMING_ENV_NEXT(otmng, "opal_init_psm"); - /* Setup the parameter system */ - if (OPAL_SUCCESS != (ret = mca_base_var_init())) { - return opal_init_error("mca_base_var_init", ret); - } - OPAL_TIMING_ENV_NEXT(otmng, "opal_var_init"); - - /* read any param files that were provided */ - if (OPAL_SUCCESS != (ret = mca_base_var_cache_files(false))) { - return opal_init_error("failed to cache files", ret); - } - - OPAL_TIMING_ENV_NEXT(otmng, "opal_var_cache"); - - /* register params for opal */ - if (OPAL_SUCCESS != (ret = opal_register_params())) { - return opal_init_error("opal_register_params", ret); - } - #if OPAL_ROCM_SUPPORT /* register params for opal/rocm. This is temporarily done here. */ mca_common_rocm_register_mca_variables(); #endif - + if (OPAL_SUCCESS != (ret = opal_net_init())) { return opal_init_error("opal_net_init", ret); } OPAL_TIMING_ENV_NEXT(otmng, "opal_net_init"); - /* pretty-print stack handlers */ - if (OPAL_SUCCESS != (ret = opal_util_register_stackhandlers())) { - return opal_init_error("opal_util_register_stackhandlers", ret); - } + OBJ_CONSTRUCT(&opal_init_domain, opal_finalize_domain_t); + (void) opal_finalize_domain_init(&opal_init_domain, "opal_init"); + opal_finalize_set_domain(&opal_init_domain); - /* set system resource limits - internally protected against - * doing so twice in cases where the launch agent did it for us - */ - if (OPAL_SUCCESS != (ret = opal_util_init_sys_limits(&error))) { - opal_show_help("help-opal-runtime.txt", "opal_init:syslimit", false, error); - return OPAL_ERR_SILENT; - } + /* register for cleanup */ + opal_finalize_register_cleanup_arg(mca_base_framework_close_list, opal_init_frameworks); - /* initialize the arch string */ - if (OPAL_SUCCESS != (ret = opal_arch_init())) { - return opal_init_error("opal_arch_init", ret); + ret = mca_base_framework_open_list(opal_init_frameworks, 0); + if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) { + return opal_init_error("opal_init framework open", ret); } - OPAL_TIMING_ENV_NEXT(otmng, "opal_arch_init"); - /* initialize the datatype engine */ if (OPAL_SUCCESS != (ret = opal_datatype_init())) { return opal_init_error("opal_datatype_init", ret); } - - OPAL_TIMING_ENV_NEXT(otmng, "opal_datatype_init"); - - /* initialize the mca */ - if (OPAL_SUCCESS != (ret = mca_base_open())) { - return opal_init_error("mca_base_open", ret); - } - - OPAL_TIMING_ENV_NEXT(otmng, "mca_base_open"); - - /* initialize if framework */ - if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_if_base_framework, 0))) { - fprintf(stderr, - "opal_if_base_open() failed -- process will likely abort (%s:%d, returned %d " - "instead of OPAL_SUCCESS)\n", - __FILE__, __LINE__, ret); + /* The ddt engine has a few parameters */ + ret = opal_datatype_register_params(); + if (OPAL_SUCCESS != ret) { return ret; } - /* register for */ - opal_finalize_register_cleanup_arg(mca_base_framework_close_list, opal_init_util_frameworks); - - OPAL_TIMING_ENV_NEXT(otmng, "opal_if_init"); - - ++opal_util_initialized; - - return OPAL_SUCCESS; -} - -/* the memcpy component should be one of the first who get - * loaded in order to make sure we have all the available - * versions of memcpy correctly configured. - */ -static mca_base_framework_t *opal_init_frameworks[] = { - &opal_threads_base_framework, &opal_hwloc_base_framework, - &opal_memcpy_base_framework, &opal_memchecker_base_framework, - &opal_backtrace_base_framework, &opal_timer_base_framework, - &opal_shmem_base_framework, &opal_reachable_base_framework, - &opal_pmix_base_framework, - NULL, -}; - -int opal_init(int *pargc, char ***pargv) -{ - int ret; - - if (opal_initialized != 0) { - if (opal_initialized < 0) { - return OPAL_ERROR; - } - ++opal_initialized; - return OPAL_SUCCESS; - } - - /* initialize util code */ - if (OPAL_SUCCESS != (ret = opal_init_util(pargc, pargv))) { + ret = opal_event_register_params(); + if (OPAL_SUCCESS != ret) { return ret; } - OBJ_CONSTRUCT(&opal_init_domain, opal_finalize_domain_t); - (void) opal_finalize_domain_init(&opal_init_domain, "opal_init"); - opal_finalize_set_domain(&opal_init_domain); - - opal_finalize_register_cleanup_arg(mca_base_framework_close_list, opal_init_frameworks); - - ret = mca_base_framework_open_list(opal_init_frameworks, 0); - if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) { - return opal_init_error("opal_init framework open", ret); - } + OPAL_TIMING_ENV_NEXT(otmng, "opal_datatype_init"); /* initialize libevent */ if (OPAL_SUCCESS != (ret = opal_event_init())) { diff --git a/opal/runtime/opal_init_core.c b/opal/runtime/opal_init_core.c new file mode 100644 index 00000000000..b634a88e3f2 --- /dev/null +++ b/opal/runtime/opal_init_core.c @@ -0,0 +1,544 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2022 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2010-2015 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2017-2022 Amazon.com, Inc. or its affiliates. All Rights reserved. + * All Rights reserved. + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018-2019 Triad National Security, LLC. All rights + * reserved. + * Copyright (c) 2020 FUJITSU LIMITED. All rights reserved. + * Copyright (c) 2021 Nanook Consulting. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file **/ + +#ifdef HAVE_UNISTD_H +# include +#endif + +#include "opal/include/opal_config.h" + +#include "opal/mca/base/base.h" +#include "opal/mca/base/mca_base_var.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/mca/installdirs/base/base.h" +#include "opal/mca/memchecker/base/base.h" +#include "opal/mca/memcpy/base/base.h" +#include "opal/mca/memory/base/base.h" +#include "opal/mca/patcher/base/base.h" +#include "opal/mca/pmix/base/base.h" +#include "opal/mca/reachable/base/base.h" +#include "opal/mca/shmem/base/base.h" +#include "opal/mca/smsc/base/base.h" +#include "opal/mca/threads/threads.h" +#include "opal/mca/threads/tsd.h" +#include "opal/mca/timer/base/base.h" +#include "opal/memoryhooks/memory.h" +#include "opal/runtime/opal.h" +#include "opal/util/arch.h" +#include "opal/util/malloc.h" +#include "opal/util/net.h" +#include "opal/util/output.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" + +#include "opal/mca/backtrace/base/base.h" +#include "opal/mca/threads/base/base.h" +#include "opal/runtime/opal_progress.h" +#include "opal/runtime/opal_params_core.h" +#include "opal/util/opal_environ.h" + +#include "opal/constants.h" +#include "opal/util/error.h" +#include "opal/util/event.h" +#include "opal/util/keyval_parse.h" +#include "opal/util/stacktrace.h" +#include "opal/util/sys_limits.h" +#include "opal/util/timings.h" + +int opal_util_initialized = 0; +bool opal_warn_on_fork = true; + +int opal_init_error(const char *error, int ret); + +static int opal_err2str(int errnum, const char **errmsg) +{ + const char *retval; + + switch (errnum) { + case OPAL_SUCCESS: + retval = "Success"; + break; + case OPAL_ERROR: + retval = "Error"; + break; + case OPAL_ERR_OUT_OF_RESOURCE: + retval = "Out of resource"; + break; + case OPAL_ERR_TEMP_OUT_OF_RESOURCE: + retval = "Temporarily out of resource"; + break; + case OPAL_ERR_RESOURCE_BUSY: + retval = "Resource busy"; + break; + case OPAL_ERR_BAD_PARAM: + retval = "Bad parameter"; + break; + case OPAL_ERR_FATAL: + retval = "Fatal"; + break; + case OPAL_ERR_NOT_IMPLEMENTED: + retval = "Not implemented"; + break; + case OPAL_ERR_NOT_SUPPORTED: + retval = "Not supported"; + break; + case OPAL_ERR_INTERRUPTED: + retval = "Interrupted"; + break; + case OPAL_ERR_WOULD_BLOCK: + retval = "Would block"; + break; + case OPAL_ERR_IN_ERRNO: + retval = "In errno"; + break; + case OPAL_ERR_UNREACH: + retval = "Unreachable"; + break; + case OPAL_ERR_NOT_FOUND: + retval = "Not found"; + break; + case OPAL_EXISTS: + retval = "Exists"; + break; + case OPAL_ERR_TIMEOUT: + retval = "Timeout"; + break; + case OPAL_ERR_NOT_AVAILABLE: + retval = "Not available"; + break; + case OPAL_ERR_PERM: + retval = "No permission"; + break; + case OPAL_ERR_VALUE_OUT_OF_BOUNDS: + retval = "Value out of bounds"; + break; + case OPAL_ERR_FILE_READ_FAILURE: + retval = "File read failure"; + break; + case OPAL_ERR_FILE_WRITE_FAILURE: + retval = "File write failure"; + break; + case OPAL_ERR_FILE_OPEN_FAILURE: + retval = "File open failure"; + break; + case OPAL_ERR_PACK_MISMATCH: + retval = "Pack data mismatch"; + break; + case OPAL_ERR_PACK_FAILURE: + retval = "Data pack failed"; + break; + case OPAL_ERR_UNPACK_FAILURE: + retval = "Data unpack failed"; + break; + case OPAL_ERR_UNPACK_INADEQUATE_SPACE: + retval = "Data unpack had inadequate space"; + break; + case OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER: + retval = "Data unpack would read past end of buffer"; + break; + case OPAL_ERR_OPERATION_UNSUPPORTED: + retval = "Requested operation is not supported on referenced data type"; + break; + case OPAL_ERR_UNKNOWN_DATA_TYPE: + retval = "Unknown data type"; + break; + case OPAL_ERR_BUFFER: + retval = "Buffer type (described vs non-described) mismatch - operation not allowed"; + break; + case OPAL_ERR_DATA_TYPE_REDEF: + retval = "Attempt to redefine an existing data type"; + break; + case OPAL_ERR_DATA_OVERWRITE_ATTEMPT: + retval = "Attempt to overwrite a data value"; + break; + case OPAL_ERR_MODULE_NOT_FOUND: + retval = "Framework requires at least one active module, but none found"; + break; + case OPAL_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED: + retval = "OS topology does not support slot_list process affinity"; + break; + case OPAL_ERR_TOPO_SOCKET_NOT_SUPPORTED: + retval = "Could not obtain socket topology information"; + break; + case OPAL_ERR_TOPO_CORE_NOT_SUPPORTED: + retval = "Could not obtain core topology information"; + break; + case OPAL_ERR_NOT_ENOUGH_SOCKETS: + retval = "Not enough sockets to meet request"; + break; + case OPAL_ERR_NOT_ENOUGH_CORES: + retval = "Not enough cores to meet request"; + break; + case OPAL_ERR_INVALID_PHYS_CPU: + retval = "Invalid physical cpu number returned"; + break; + case OPAL_ERR_MULTIPLE_AFFINITIES: + retval = "Multiple methods for assigning process affinity were specified"; + break; + case OPAL_ERR_SLOT_LIST_RANGE: + retval = "Provided slot_list range is invalid"; + break; + case OPAL_ERR_NETWORK_NOT_PARSEABLE: + retval = "Provided network specification is not parseable"; + break; + case OPAL_ERR_SILENT: + retval = NULL; + break; + case OPAL_ERR_NOT_INITIALIZED: + retval = "Not initialized"; + break; + case OPAL_ERR_NOT_BOUND: + retval = "Not bound"; + break; + case OPAL_ERR_TAKE_NEXT_OPTION: + retval = "Take next option"; + break; + case OPAL_ERR_PROC_ENTRY_NOT_FOUND: + retval = "Database entry not found"; + break; + case OPAL_ERR_DATA_VALUE_NOT_FOUND: + retval = "Data for specified key not found"; + break; + case OPAL_ERR_CONNECTION_FAILED: + retval = "Connection failed"; + break; + case OPAL_ERR_AUTHENTICATION_FAILED: + retval = "Authentication failed"; + break; + case OPAL_ERR_COMM_FAILURE: + retval = "Comm failure"; + break; + case OPAL_ERR_SERVER_NOT_AVAIL: + retval = "Server not available"; + break; + case OPAL_ERR_IN_PROCESS: + retval = "Operation in process"; + break; + case OPAL_ERR_DEBUGGER_RELEASE: + retval = "Release debugger"; + break; + case OPAL_ERR_HANDLERS_COMPLETE: + retval = "Event handlers complete"; + break; + case OPAL_ERR_PARTIAL_SUCCESS: + retval = "Partial success"; + break; + case OPAL_ERR_PROC_ABORTED: + retval = "Process abnormally terminated"; + break; + case OPAL_ERR_PROC_REQUESTED_ABORT: + retval = "Process requested abort"; + break; + case OPAL_ERR_PROC_ABORTING: + retval = "Process is aborting"; + break; + case OPAL_ERR_NODE_DOWN: + retval = "Node has gone down"; + break; + case OPAL_ERR_NODE_OFFLINE: + retval = "Node has gone offline"; + break; + case OPAL_ERR_JOB_TERMINATED: + retval = "Job terminated"; + break; + case OPAL_ERR_PROC_RESTART: + retval = "Process restarted"; + break; + case OPAL_ERR_PROC_CHECKPOINT: + retval = "Process checkpoint"; + break; + case OPAL_ERR_PROC_MIGRATE: + retval = "Process migrate"; + break; + case OPAL_ERR_EVENT_REGISTRATION: + retval = "Event registration"; + break; + case OPAL_ERR_HEARTBEAT_ALERT: + retval = "Heartbeat not received"; + break; + case OPAL_ERR_FILE_ALERT: + retval = "File alert - proc may have stalled"; + break; + case OPAL_ERR_MODEL_DECLARED: + retval = "Model declared"; + break; + case OPAL_PMIX_LAUNCH_DIRECTIVE: + retval = "Launch directive"; + break; + + default: + retval = "UNRECOGNIZED"; + } + + *errmsg = retval; + return OPAL_SUCCESS; +} + +int opal_init_error(const char *error, int ret) +{ + if (OPAL_ERR_SILENT != ret) { + opal_show_help("help-opal-runtime.txt", "opal_init:startup:internal-failure", true, error, + ret); + } + return ret; +} + +/* If there is a preprocessor macro that redefined the call to + * gethostname, we undefine that here */ +#ifdef gethostname +# undef gethostname +#endif + +#define NUM_TRIES_FOR_NULL_HOSTNAME 8 + +/* + * This gethostname wrapper does not return the full-length hostname in + * those rare cases where it is too long for the buffer. It does, however, + * guarantee a null-terminated hostname is returned, even if it's + * truncated. It also tries again in the case where gethostname returns an + * error because the buffer is initially too short. + */ +int opal_init_gethostname(void) +{ + size_t count, length = OPAL_LOCAL_MAXHOSTNAMELEN; + int ret_val, num_tries = 0; + + char *buf = calloc(1, length); + if (NULL == buf) { + return OPAL_ERR_OUT_OF_RESOURCE; + } + + while (num_tries < NUM_TRIES_FOR_NULL_HOSTNAME) { + ++num_tries; + + /* + * Offer all but the last byte of the buffer to gethostname. + */ + ret_val = gethostname(buf, length - 1); + /* + * Terminate the buffer in the last position. + */ + buf[length - 1] = '\0'; + if (0 == ret_val) { + count = strlen(buf); + /* The result was not truncated */ + if (count > 0 && count < length - 1) { + /* + * If we got a good result, save it. This value may + * be longer than what callers to opal_gethostname() + * are expecting, so that should be checked by the + * caller. + */ + opal_process_info.nodename = buf; + return OPAL_SUCCESS; + } + /* + * "Good" cases: + * + * 0 == count: The buffer is empty. In some gethostname + * implementations, this can be because the + * buffer was too small. + * (length-1) == count: The result *may* be truncated. + * + * If it's one of these cases, we'll fall through to + * increase the length of the buffer and try again. + * + * If it's not one of these good cases, it's an error: + * return. + */ + else if (!(0 == count || count == length - 1)) { + free(buf); + return OPAL_ERR_IN_ERRNO; + } + } + /* + * "Good" cases: + * + * errno == EINVAL or ENAMETOOLONG: hostname was truncated and + * there was an error. Perhaps there is something + * in the buffer and perhaps not. + * + * If it's one of these cases, we'll fall through to + * increase the length of the buffer and try again. + * + * If it's not one of these good cases, it's an error: return. + */ + else if (!(EINVAL == errno || ENAMETOOLONG == errno)) { + free(buf); + return OPAL_ERR_IN_ERRNO; + } + + /* + * If we get here, it means we want to double the length of + * the buffer and try again. + */ + length *= 2; + buf = realloc(buf, length); + if (NULL == buf) { + return OPAL_ERR_OUT_OF_RESOURCE; + } + } /* end while */ + + /* If we got here, it means that we tried too many times and are + * giving up. */ + free(buf); + return OPAL_ERR_NOT_FOUND; +} + +static mca_base_framework_t *opal_init_util_frameworks[] = { + &opal_installdirs_base_framework, + NULL, +}; + +int opal_init_util(int *pargc, char ***pargv) +{ + int ret; + char *error = NULL; + OPAL_TIMING_ENV_INIT(otmng); + + if (opal_util_initialized != 0) { + if (opal_util_initialized < 0) { + return OPAL_ERROR; + } + ++opal_util_initialized; + return OPAL_SUCCESS; + } + + OBJ_CONSTRUCT(&opal_init_util_domain, opal_finalize_domain_t); + (void) opal_finalize_domain_init(&opal_init_util_domain, "opal_init_util"); + opal_finalize_set_domain(&opal_init_util_domain); + + opal_thread_set_main(); + + /* set the nodename right away so anyone who needs it has it. Note + * that we don't bother with fqdn and prefix issues here - we let + * the RTE later replace this with a modified name if the user + * requests it */ + ret = opal_init_gethostname(); + if (OPAL_SUCCESS != ret) { + fprintf(stderr, + "opal_init_gethostname() failed -- process will likely abort (%s:%d, returned %d " + "instead of OPAL_SUCCESS)\n", + __FILE__, __LINE__, ret); + return ret; + } + + /* initialize the memory allocator */ + opal_malloc_init(); + + OPAL_TIMING_ENV_NEXT(otmng, "opal_malloc_init"); + + /* initialize the output system */ + opal_output_init(); + + /* initialize install dirs code */ + if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_installdirs_base_framework, 0))) { + fprintf(stderr, + "opal_installdirs_base_open() failed -- process will likely abort (%s:%d, returned " + "%d instead of OPAL_SUCCESS)\n", + __FILE__, __LINE__, ret); + return ret; + } + + /* initialize the help system */ + opal_show_help_init(); + + OPAL_TIMING_ENV_NEXT(otmng, "opal_show_help_init"); + + /* register handler for errnum -> string converstion */ + if (OPAL_SUCCESS + != (ret = opal_error_register("OPAL", OPAL_ERR_BASE, OPAL_ERR_MAX, opal_err2str))) { + return opal_init_error("opal_error_register", ret); + } + + /* keyval lex-based parser */ + if (OPAL_SUCCESS != (ret = opal_util_keyval_parse_init())) { + return opal_init_error("opal_util_keyval_parse_init", ret); + } + + /* Setup the parameter system */ + if (OPAL_SUCCESS != (ret = mca_base_var_init())) { + return opal_init_error("mca_base_var_init", ret); + } + OPAL_TIMING_ENV_NEXT(otmng, "opal_var_init"); + + /* read any param files that were provided */ + if (OPAL_SUCCESS != (ret = mca_base_var_cache_files(false))) { + return opal_init_error("failed to cache files", ret); + } + + OPAL_TIMING_ENV_NEXT(otmng, "opal_var_cache"); + + /* register util params for opal */ + if (OPAL_SUCCESS != (ret = opal_register_util_params())) { + return opal_init_error("opal_register_util_params", ret); + } + + /* pretty-print stack handlers */ + if (OPAL_SUCCESS != (ret = opal_util_register_stackhandlers())) { + return opal_init_error("opal_util_register_stackhandlers", ret); + } + + /* set system resource limits - internally protected against + * doing so twice in cases where the launch agent did it for us + */ + if (OPAL_SUCCESS != (ret = opal_util_init_sys_limits(&error))) { + opal_show_help("help-opal-runtime.txt", "opal_init:syslimit", false, error); + return OPAL_ERR_SILENT; + } + + /* initialize the arch string */ + if (OPAL_SUCCESS != (ret = opal_arch_init())) { + return opal_init_error("opal_arch_init", ret); + } + + OPAL_TIMING_ENV_NEXT(otmng, "opal_arch_init"); + + /* initialize the mca */ + if (OPAL_SUCCESS != (ret = mca_base_open())) { + return opal_init_error("mca_base_open", ret); + } + + OPAL_TIMING_ENV_NEXT(otmng, "mca_base_open"); + + /* register for cleanup */ + opal_finalize_register_cleanup_arg(mca_base_framework_close_list, opal_init_util_frameworks); + + ++opal_util_initialized; + + return OPAL_SUCCESS; +} diff --git a/opal/runtime/opal_params.c b/opal/runtime/opal_params.c index 1e7ccc635bc..379b30e02f2 100644 --- a/opal/runtime/opal_params.c +++ b/opal/runtime/opal_params.c @@ -22,7 +22,7 @@ * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. - * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. + * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ @@ -50,49 +50,17 @@ #include "opal/util/show_help.h" #include "opal/util/timings.h" -char *opal_signal_string = NULL; -char *opal_stacktrace_output_filename = NULL; -char *opal_net_private_ipv4 = NULL; -char *opal_set_max_sys_limits = NULL; - -#if OPAL_ENABLE_TIMING -char *opal_timing_sync_file = NULL; -char *opal_timing_output = NULL; -bool opal_timing_overhead = true; -#endif - -bool opal_built_with_cuda_support = OPAL_INT_TO_BOOL(OPAL_CUDA_SUPPORT); -bool opal_cuda_runtime_initialized = false; -bool opal_cuda_support = false; -bool opal_warn_on_missing_libcuda = true; - -bool opal_built_with_rocm_support = OPAL_INT_TO_BOOL(OPAL_ROCM_SUPPORT); -bool opal_rocm_runtime_initialized = false; - - -/** - * Globals imported from the OMPI layer. - */ -int opal_leave_pinned = -1; -bool opal_leave_pinned_pipeline = false; -bool opal_abort_print_stack = false; -int opal_abort_delay = 0; - -int opal_max_thread_in_progress = 1; static bool opal_register_done = false; static void opal_deregister_params(void) { - /* The MCA variable system will be torn down shortly so reset the registered - * flag. */ opal_register_done = false; } int opal_register_params(void) { int ret; - char *string = NULL; if (opal_register_done) { return OPAL_SUCCESS; @@ -100,75 +68,6 @@ int opal_register_params(void) opal_register_done = true; - /* - * This string is going to be used in opal/util/stacktrace.c - */ - { - int j; - int signals[] = { -#ifdef SIGABRT - SIGABRT, -#endif -#ifdef SIGBUS - SIGBUS, -#endif -#ifdef SIGFPE - SIGFPE, -#endif -#ifdef SIGSEGV - SIGSEGV, -#endif - -1}; - for (j = 0; signals[j] != -1; ++j) { - if (j == 0) { - opal_asprintf(&string, "%d", signals[j]); - } else { - char *tmp; - opal_asprintf(&tmp, "%s,%d", string, signals[j]); - free(string); - string = tmp; - } - } - - opal_signal_string = string; - ret = mca_base_var_register( - "opal", "opal", NULL, "signal", - "Comma-delimited list of integer signal numbers to Open MPI to attempt to intercept. " - "Upon receipt of the intercepted signal, Open MPI will display a stack trace and " - "abort. Open MPI will *not* replace signals if handlers are already installed by the " - "time MPI_INIT is invoked. Optionally append \":complain\" to any signal number in " - "the comma-delimited list to make Open MPI complain if it detects another signal " - "handler (and therefore does not insert its own).", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_LOCAL, &opal_signal_string); - free(string); - if (0 > ret) { - return ret; - } - } - - /* - * Where should the stack trace output be directed - * This string is going to be used in opal/util/stacktrace.c - */ - string = strdup("stderr"); - opal_stacktrace_output_filename = string; - ret = mca_base_var_register( - "opal", "opal", NULL, "stacktrace_output", - "Specifies where the stack trace output stream goes. " - "Accepts one of the following: none (disabled), stderr (default), stdout, file[:filename]. " - " " - "If 'filename' is not specified, a default filename of 'stacktrace' is used. " - "The 'filename' is appended with either '.PID' or '.RANK.PID', if RANK is available. " - "The 'filename' can be an absolute path or a relative path to the current working " - "directory.", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_LOCAL, &opal_stacktrace_output_filename); - free(string); - if (0 > ret) { - return ret; - } - #if defined(HAVE_SCHED_YIELD) opal_progress_yield_when_idle = false; ret = mca_base_var_register("opal", "opal", "progress", "yield_when_idle", @@ -187,193 +86,7 @@ int opal_register_params(void) if (0 > ret) { return ret; } - - opal_debug_threads = false; - ret = mca_base_var_register("opal", "opal", "debug", "threads", - "Debug thread usage within OPAL. Reports out " - "when threads are acquired and released.", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, - OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL, &opal_debug_threads); - if (0 > ret) { - return ret; - } -#endif - - /* RFC1918 defines - - 10.0.0./8 - - 172.16.0.0/12 - - 192.168.0.0/16 - - RFC3330 also mentions - - 169.254.0.0/16 for DHCP onlink iff there's no DHCP server - */ - opal_net_private_ipv4 = "10.0.0.0/8;172.16.0.0/12;192.168.0.0/16;169.254.0.0/16"; - ret = mca_base_var_register( - "opal", "opal", "net", "private_ipv4", - "Semicolon-delimited list of CIDR notation entries specifying what networks are considered " - "\"private\" (default value based on RFC1918 and RFC3330)", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_net_private_ipv4); - if (0 > ret) { - return ret; - } - - opal_set_max_sys_limits = NULL; - ret = mca_base_var_register( - "opal", "opal", NULL, "set_max_sys_limits", - "Set the specified system-imposed limits to the specified value, including \"unlimited\"." - "Supported params: core, filesize, maxmem, openfiles, stacksize, maxchildren", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_set_max_sys_limits); - if (0 > ret) { - return ret; - } - - ret = mca_base_var_register("opal", "opal", NULL, "built_with_cuda_support", - "Whether CUDA GPU buffer support is built into library or not", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, - OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_CONSTANT, - &opal_built_with_cuda_support); - if (0 > ret) { - return ret; - } - - /* Current default is to enable CUDA support if it is built into library */ - opal_cuda_support = opal_built_with_cuda_support; - ret = mca_base_var_register("opal", "opal", NULL, "cuda_support", - "Whether CUDA GPU buffer support is enabled or not", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, - OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_cuda_support); - if (0 > ret) { - return ret; - } - - opal_warn_on_missing_libcuda = true; - ret = mca_base_var_register( - "opal", "opal", NULL, "warn_on_missing_libcuda", - "Whether to print a message when CUDA support is enabled but libcuda is not found", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_warn_on_missing_libcuda); - if (0 > ret) { - return ret; - } - - /* Leave pinned parameter */ - opal_leave_pinned = -1; - ret = mca_base_var_register( - "ompi", "mpi", NULL, "leave_pinned", - "Whether to use the \"leave pinned\" protocol or not. Enabling this setting can help " - "bandwidth performance when repeatedly sending and receiving large messages with the same " - "buffers over RDMA-based networks (false = do not use \"leave pinned\" protocol, true = " - "use \"leave pinned\" protocol, auto = allow network to choose at runtime).", - MCA_BASE_VAR_TYPE_INT, &mca_base_var_enum_auto_bool, 0, 0, OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &opal_leave_pinned); - mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned", - MCA_BASE_VAR_SYN_FLAG_DEPRECATED); - - opal_leave_pinned_pipeline = false; - ret = mca_base_var_register("ompi", "mpi", NULL, "leave_pinned_pipeline", - "Whether to use the \"leave pinned pipeline\" protocol or not.", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &opal_leave_pinned_pipeline); - mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned_pipeline", - MCA_BASE_VAR_SYN_FLAG_DEPRECATED); - - if (opal_leave_pinned > 0 && opal_leave_pinned_pipeline) { - opal_leave_pinned_pipeline = 0; - opal_show_help("help-opal-runtime.txt", "mpi-params:leave-pinned-and-pipeline-selected", - true); - } - - opal_warn_on_fork = true; - (void) mca_base_var_register("ompi", "mpi", NULL, "warn_on_fork", - "If nonzero, issue a warning if program forks under conditions " - "that could cause system errors", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, - MCA_BASE_VAR_SCOPE_READONLY, &opal_warn_on_fork); - - opal_abort_delay = 0; - ret = mca_base_var_register( - "opal", "opal", NULL, "abort_delay", - "If nonzero, print out an identifying message when abort operation is invoked (hostname, " - "PID of the process that called abort) and delay for that many seconds before exiting (a " - "negative delay value means to never abort). This allows attaching of a debugger before " - "quitting the job.", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, - &opal_abort_delay); - if (0 > ret) { - return ret; - } - - opal_abort_print_stack = false; - ret = mca_base_var_register("opal", "opal", NULL, "abort_print_stack", - "If nonzero, print out a stack trace when abort is invoked", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, - /* If we do not have stack trace - capability, make this a constant - MCA variable */ -#if OPAL_WANT_PRETTY_PRINT_STACKTRACE - 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, -#else - MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5, - MCA_BASE_VAR_SCOPE_CONSTANT, #endif - &opal_abort_print_stack); - if (0 > ret) { - return ret; - } - - /* register the envar-forwarding params */ - (void) mca_base_var_register("opal", "mca", "base", "env_list", "Set SHELL env variables", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list); - - mca_base_env_list_sep = MCA_BASE_ENV_LIST_SEP_DEFAULT; - (void) mca_base_var_register("opal", "mca", "base", "env_list_delimiter", - "Set SHELL env variables delimiter. Default: semicolon ';'", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_sep); - - /* Set OMPI_MCA_mca_base_env_list variable, it might not be set before - * if mca variable was taken from amca conf file. Need to set it - * here because mca_base_var_process_env_list is called from schizo_ompi.c - * only when this env variable was set. - */ - if (NULL != mca_base_env_list) { - char *name = NULL; - (void) mca_base_var_env_name("mca_base_env_list", &name); - if (NULL != name) { - opal_setenv(name, mca_base_env_list, false, &environ); - free(name); - } - } - - /* Register internal MCA variable mca_base_env_list_internal. It can be set only during - * parsing of amca conf file and contains SHELL env variables specified via -x there. - * Its format is the same as for mca_base_env_list. - */ - (void) mca_base_var_register("opal", "mca", "base", "env_list_internal", - "Store SHELL env variables from amca conf file", - MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, - OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY, - &mca_base_env_list_internal); - - /* Number of threads allowed in opal_progress. This might increase multithreaded performance. */ - (void) mca_base_var_register("opal", "opal", NULL, "max_thread_in_progress", - "Number of thread allowed in opal_progress. Default: 1", - MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8, - MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress); - - /* The ddt engine has a few parameters */ - ret = opal_datatype_register_params(); - if (OPAL_SUCCESS != ret) { - return ret; - } - - ret = opal_event_register_params(); - if (OPAL_SUCCESS != ret) { - return ret; - } opal_finalize_register_cleanup(opal_deregister_params); diff --git a/opal/runtime/opal_params.h b/opal/runtime/opal_params.h index 2a8e99ac041..67fc2fb9e15 100644 --- a/opal/runtime/opal_params.h +++ b/opal/runtime/opal_params.h @@ -30,70 +30,6 @@ #if !defined(OPAL_PARAMS_H) # define OPAL_PARAMS_H -extern char *opal_signal_string; -extern char *opal_stacktrace_output_filename; -extern char *opal_net_private_ipv4; -extern char *opal_set_max_sys_limits; - -# if OPAL_ENABLE_TIMING -extern char *opal_timing_sync_file; -extern char *opal_timing_output; -extern bool opal_timing_overhead; -# endif - -OPAL_DECLSPEC extern int opal_initialized; -OPAL_DECLSPEC extern bool opal_built_with_cuda_support; -OPAL_DECLSPEC extern bool opal_built_with_rocm_support; - -/** - * * Whether we want to enable CUDA GPU buffer send and receive support. - * */ -OPAL_DECLSPEC extern bool opal_cuda_support; - -/** - * Whether cuda runtime support is initialized or not. - */ -OPAL_DECLSPEC extern bool opal_cuda_runtime_initialized; - -/** - * Whether rocm runtime support is initialized or not. - */ -OPAL_DECLSPEC extern bool opal_rocm_runtime_initialized; - -/** - * * Whether we want to warn the user when libcuda is missing. - * */ -OPAL_DECLSPEC extern bool opal_warn_on_missing_libcuda; - -/** - * Whether to use the "leave pinned" protocol or not (0 = no, 1 = yes, - * -1 = determine at runtime). - */ -OPAL_DECLSPEC extern int opal_leave_pinned; - -/** - * Whether to use the "leave pinned pipeline" protocol or not. - */ -OPAL_DECLSPEC extern bool opal_leave_pinned_pipeline; - -/** - * Whether an abort operation should print out a stack trace or not. - */ -OPAL_DECLSPEC extern bool opal_abort_print_stack; - -/** - * Whether abort operation should print out an identifying message - * (e.g., hostname and PID) and loop waiting for a debugger to - * attach. The value of the integer is how many seconds to wait: - * - * 0 = do not print the message and do not loop - * negative value = print the message and loop forever - * positive value = print the message and delay for that many seconds - */ -OPAL_DECLSPEC extern int opal_abort_delay; - -# if OPAL_ENABLE_DEBUG -extern bool opal_progress_debug; -# endif +#include "opal/runtime/opal_params_core.h" #endif diff --git a/opal/runtime/opal_params_core.c b/opal/runtime/opal_params_core.c new file mode 100644 index 00000000000..a27a2e1a406 --- /dev/null +++ b/opal/runtime/opal_params_core.c @@ -0,0 +1,352 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2010-2014 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2014 Hochschule Esslingen. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates. All Rights reserved. + * Copyright (c) 2021 Nanook Consulting. All rights reserved. + * Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" + +#include +#include + +#include "opal/constants.h" +#include "opal/datatype/opal_datatype.h" +#include "opal/mca/base/mca_base_var.h" +#include "opal/mca/shmem/base/base.h" +#include "opal/mca/threads/mutex.h" +#include "opal/mca/threads/threads.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_params_core.h" +#include "opal/util/opal_environ.h" +#include "opal/util/printf.h" +#include "opal/util/show_help.h" +#include "opal/util/timings.h" + +char *opal_signal_string = NULL; +char *opal_stacktrace_output_filename = NULL; +char *opal_net_private_ipv4 = NULL; +char *opal_set_max_sys_limits = NULL; + +#if OPAL_ENABLE_TIMING +char *opal_timing_sync_file = NULL; +char *opal_timing_output = NULL; +bool opal_timing_overhead = true; +#endif + +bool opal_built_with_cuda_support = OPAL_INT_TO_BOOL(OPAL_CUDA_SUPPORT); +bool opal_cuda_runtime_initialized = false; +bool opal_cuda_support = false; +bool opal_warn_on_missing_libcuda = true; + +bool opal_built_with_rocm_support = OPAL_INT_TO_BOOL(OPAL_ROCM_SUPPORT); +bool opal_rocm_runtime_initialized = false; + + +/** + * Globals imported from the OMPI layer. + */ +int opal_leave_pinned = -1; +bool opal_leave_pinned_pipeline = false; +bool opal_abort_print_stack = false; +int opal_abort_delay = 0; + +int opal_max_thread_in_progress = 1; + +static bool opal_register_util_done = false; + +static void opal_deregister_util_params(void) +{ + /* The MCA variable system will be torn down shortly so reset the registered + * flag. */ + opal_register_util_done = false; +} + +int opal_register_util_params(void) +{ + int ret; + char *string = NULL; + + if (opal_register_util_done) { + return OPAL_SUCCESS; + } + + opal_register_util_done = true; + + /* + * This string is going to be used in opal/util/stacktrace.c + */ + { + int j; + int signals[] = { +#ifdef SIGABRT + SIGABRT, +#endif +#ifdef SIGBUS + SIGBUS, +#endif +#ifdef SIGFPE + SIGFPE, +#endif +#ifdef SIGSEGV + SIGSEGV, +#endif + -1}; + for (j = 0; signals[j] != -1; ++j) { + if (j == 0) { + opal_asprintf(&string, "%d", signals[j]); + } else { + char *tmp; + opal_asprintf(&tmp, "%s,%d", string, signals[j]); + free(string); + string = tmp; + } + } + + opal_signal_string = string; + ret = mca_base_var_register( + "opal", "opal", NULL, "signal", + "Comma-delimited list of integer signal numbers to Open MPI to attempt to intercept. " + "Upon receipt of the intercepted signal, Open MPI will display a stack trace and " + "abort. Open MPI will *not* replace signals if handlers are already installed by the " + "time MPI_INIT is invoked. Optionally append \":complain\" to any signal number in " + "the comma-delimited list to make Open MPI complain if it detects another signal " + "handler (and therefore does not insert its own).", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_LOCAL, &opal_signal_string); + free(string); + if (0 > ret) { + return ret; + } + } + + /* + * Where should the stack trace output be directed + * This string is going to be used in opal/util/stacktrace.c + */ + string = strdup("stderr"); + opal_stacktrace_output_filename = string; + ret = mca_base_var_register( + "opal", "opal", NULL, "stacktrace_output", + "Specifies where the stack trace output stream goes. " + "Accepts one of the following: none (disabled), stderr (default), stdout, file[:filename]. " + " " + "If 'filename' is not specified, a default filename of 'stacktrace' is used. " + "The 'filename' is appended with either '.PID' or '.RANK.PID', if RANK is available. " + "The 'filename' can be an absolute path or a relative path to the current working " + "directory.", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_LOCAL, &opal_stacktrace_output_filename); + free(string); + if (0 > ret) { + return ret; + } + +#if OPAL_ENABLE_DEBUG + opal_debug_threads = false; + ret = mca_base_var_register("opal", "opal", "debug", "threads", + "Debug thread usage within OPAL. Reports out " + "when threads are acquired and released.", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, + OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL, &opal_debug_threads); + if (0 > ret) { + return ret; + } +#endif + + /* RFC1918 defines + - 10.0.0./8 + - 172.16.0.0/12 + - 192.168.0.0/16 + + RFC3330 also mentions + - 169.254.0.0/16 for DHCP onlink iff there's no DHCP server + */ + opal_net_private_ipv4 = "10.0.0.0/8;172.16.0.0/12;192.168.0.0/16;169.254.0.0/16"; + ret = mca_base_var_register( + "opal", "opal", "net", "private_ipv4", + "Semicolon-delimited list of CIDR notation entries specifying what networks are considered " + "\"private\" (default value based on RFC1918 and RFC3330)", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_net_private_ipv4); + if (0 > ret) { + return ret; + } + + opal_set_max_sys_limits = NULL; + ret = mca_base_var_register( + "opal", "opal", NULL, "set_max_sys_limits", + "Set the specified system-imposed limits to the specified value, including \"unlimited\"." + "Supported params: core, filesize, maxmem, openfiles, stacksize, maxchildren", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_set_max_sys_limits); + if (0 > ret) { + return ret; + } + + ret = mca_base_var_register("opal", "opal", NULL, "built_with_cuda_support", + "Whether CUDA GPU buffer support is built into library or not", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, + OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_CONSTANT, + &opal_built_with_cuda_support); + if (0 > ret) { + return ret; + } + + /* Current default is to enable CUDA support if it is built into library */ + opal_cuda_support = opal_built_with_cuda_support; + ret = mca_base_var_register("opal", "opal", NULL, "cuda_support", + "Whether CUDA GPU buffer support is enabled or not", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, + OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_cuda_support); + if (0 > ret) { + return ret; + } + + opal_warn_on_missing_libcuda = true; + ret = mca_base_var_register( + "opal", "opal", NULL, "warn_on_missing_libcuda", + "Whether to print a message when CUDA support is enabled but libcuda is not found", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_ALL_EQ, &opal_warn_on_missing_libcuda); + if (0 > ret) { + return ret; + } + + /* Leave pinned parameter */ + opal_leave_pinned = -1; + ret = mca_base_var_register( + "ompi", "mpi", NULL, "leave_pinned", + "Whether to use the \"leave pinned\" protocol or not. Enabling this setting can help " + "bandwidth performance when repeatedly sending and receiving large messages with the same " + "buffers over RDMA-based networks (false = do not use \"leave pinned\" protocol, true = " + "use \"leave pinned\" protocol, auto = allow network to choose at runtime).", + MCA_BASE_VAR_TYPE_INT, &mca_base_var_enum_auto_bool, 0, 0, OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &opal_leave_pinned); + mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned", + MCA_BASE_VAR_SYN_FLAG_DEPRECATED); + + opal_leave_pinned_pipeline = false; + ret = mca_base_var_register("ompi", "mpi", NULL, "leave_pinned_pipeline", + "Whether to use the \"leave pinned pipeline\" protocol or not.", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &opal_leave_pinned_pipeline); + mca_base_var_register_synonym(ret, "opal", "opal", NULL, "leave_pinned_pipeline", + MCA_BASE_VAR_SYN_FLAG_DEPRECATED); + + if (opal_leave_pinned > 0 && opal_leave_pinned_pipeline) { + opal_leave_pinned_pipeline = 0; + opal_show_help("help-opal-runtime.txt", "mpi-params:leave-pinned-and-pipeline-selected", + true); + } + + opal_warn_on_fork = true; + (void) mca_base_var_register("ompi", "mpi", NULL, "warn_on_fork", + "If nonzero, issue a warning if program forks under conditions " + "that could cause system errors", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, &opal_warn_on_fork); + + opal_abort_delay = 0; + ret = mca_base_var_register( + "opal", "opal", NULL, "abort_delay", + "If nonzero, print out an identifying message when abort operation is invoked (hostname, " + "PID of the process that called abort) and delay for that many seconds before exiting (a " + "negative delay value means to never abort). This allows attaching of a debugger before " + "quitting the job.", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, + &opal_abort_delay); + if (0 > ret) { + return ret; + } + + opal_abort_print_stack = false; + ret = mca_base_var_register("opal", "opal", NULL, "abort_print_stack", + "If nonzero, print out a stack trace when abort is invoked", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, + /* If we do not have stack trace + capability, make this a constant + MCA variable */ +#if OPAL_WANT_PRETTY_PRINT_STACKTRACE + 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY, +#else + MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5, + MCA_BASE_VAR_SCOPE_CONSTANT, +#endif + &opal_abort_print_stack); + if (0 > ret) { + return ret; + } + + /* register the envar-forwarding params */ + (void) mca_base_var_register("opal", "mca", "base", "env_list", "Set SHELL env variables", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list); + + mca_base_env_list_sep = MCA_BASE_ENV_LIST_SEP_DEFAULT; + (void) mca_base_var_register("opal", "mca", "base", "env_list_delimiter", + "Set SHELL env variables delimiter. Default: semicolon ';'", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_sep); + + /* Set OMPI_MCA_mca_base_env_list variable, it might not be set before + * if mca variable was taken from amca conf file. Need to set it + * here because mca_base_var_process_env_list is called from schizo_ompi.c + * only when this env variable was set. + */ + if (NULL != mca_base_env_list) { + char *name = NULL; + (void) mca_base_var_env_name("mca_base_env_list", &name); + if (NULL != name) { + opal_setenv(name, mca_base_env_list, false, &environ); + free(name); + } + } + + /* Register internal MCA variable mca_base_env_list_internal. It can be set only during + * parsing of amca conf file and contains SHELL env variables specified via -x there. + * Its format is the same as for mca_base_env_list. + */ + (void) mca_base_var_register("opal", "mca", "base", "env_list_internal", + "Store SHELL env variables from amca conf file", + MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, + OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_READONLY, + &mca_base_env_list_internal); + + /* Number of threads allowed in opal_progress. This might increase multithreaded performance. */ + (void) mca_base_var_register("opal", "opal", NULL, "max_thread_in_progress", + "Number of thread allowed in opal_progress. Default: 1", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8, + MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress); + + opal_finalize_register_cleanup(opal_deregister_util_params); + + return OPAL_SUCCESS; +} diff --git a/opal/runtime/opal_params_core.h b/opal/runtime/opal_params_core.h new file mode 100644 index 00000000000..8e756699aaa --- /dev/null +++ b/opal/runtime/opal_params_core.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2010-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2014 Hochschule Esslingen. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(OPAL_PARAMS_UTIL_H) +# define OPAL_PARAMS_UTIL_H + +extern char *opal_signal_string; +extern char *opal_stacktrace_output_filename; +extern char *opal_net_private_ipv4; +extern char *opal_set_max_sys_limits; + +# if OPAL_ENABLE_TIMING +extern char *opal_timing_sync_file; +extern char *opal_timing_output; +extern bool opal_timing_overhead; +# endif + +OPAL_DECLSPEC extern int opal_initialized; +OPAL_DECLSPEC extern bool opal_built_with_cuda_support; +OPAL_DECLSPEC extern bool opal_built_with_rocm_support; + +/** + * * Whether we want to enable CUDA GPU buffer send and receive support. + * */ +OPAL_DECLSPEC extern bool opal_cuda_support; + +/** + * Whether cuda runtime support is initialized or not. + */ +OPAL_DECLSPEC extern bool opal_cuda_runtime_initialized; + +/** + * Whether rocm runtime support is initialized or not. + */ +OPAL_DECLSPEC extern bool opal_rocm_runtime_initialized; + +/** + * * Whether we want to warn the user when libcuda is missing. + * */ +OPAL_DECLSPEC extern bool opal_warn_on_missing_libcuda; + +/** + * Whether to use the "leave pinned" protocol or not (0 = no, 1 = yes, + * -1 = determine at runtime). + */ +OPAL_DECLSPEC extern int opal_leave_pinned; + +/** + * Whether to use the "leave pinned pipeline" protocol or not. + */ +OPAL_DECLSPEC extern bool opal_leave_pinned_pipeline; + +/** + * Whether an abort operation should print out a stack trace or not. + */ +OPAL_DECLSPEC extern bool opal_abort_print_stack; + +/** + * Whether abort operation should print out an identifying message + * (e.g., hostname and PID) and loop waiting for a debugger to + * attach. The value of the integer is how many seconds to wait: + * + * 0 = do not print the message and do not loop + * negative value = print the message and loop forever + * positive value = print the message and delay for that many seconds + */ +OPAL_DECLSPEC extern int opal_abort_delay; + + +/** + * Register OPAL MCA parameters from the core + */ +OPAL_DECLSPEC int opal_register_util_params(void); + +#endif diff --git a/opal/runtime/opal_progress.h b/opal/runtime/opal_progress.h index d81adfbcb46..2eef5aecef2 100644 --- a/opal/runtime/opal_progress.h +++ b/opal/runtime/opal_progress.h @@ -161,6 +161,10 @@ OPAL_DECLSPEC int opal_progress_register_lp(opal_progress_callback_t cb); */ OPAL_DECLSPEC int opal_progress_unregister(opal_progress_callback_t cb); +#if OPAL_ENABLE_DEBUG +OPAL_DECLSPEC extern bool opal_progress_debug; +#endif + OPAL_DECLSPEC extern int opal_progress_spin_count; /* do we want to call sched_yield() if nothing happened */ diff --git a/opal/tools/wrappers/Makefile.am b/opal/tools/wrappers/Makefile.am index 16c874ccd99..70308b1b4b4 100644 --- a/opal/tools/wrappers/Makefile.am +++ b/opal/tools/wrappers/Makefile.am @@ -2,7 +2,7 @@ # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. -# Copyright (c) 2004-2006 The University of Tennessee and The University +# Copyright (c) 2004-2022 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, @@ -13,6 +13,7 @@ # Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. # Copyright (c) 2014 Research Organization for Information Science # and Technology (RIST). All rights reserved. +# Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,4 +45,4 @@ endif # OPAL_INSTALL_BINARIES endif # OPAL_WANT_SCRIPT_WRAPPER_COMPILERS opal_wrapper_SOURCES = opal_wrapper.c -opal_wrapper_LDADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la +opal_wrapper_LDADD = $(top_builddir)/opal/libopen-pal_core.la diff --git a/opal/util/Makefile.am b/opal/util/Makefile.am index 4a457d7e1b3..74cd6efd694 100644 --- a/opal/util/Makefile.am +++ b/opal/util/Makefile.am @@ -34,7 +34,7 @@ dist_opaldata_DATA = help-opal-util.txt AM_LFLAGS = -Popal_show_help_yy LEX_OUTPUT_ROOT = lex.opal_show_help_yy -noinst_LTLIBRARIES = libopalutil.la +noinst_LTLIBRARIES = libopalutil.la libopalutil_core.la AM_CPPFLAGS = $(LTDLINCL) @@ -84,6 +84,14 @@ headers = \ libopalutil_la_SOURCES = \ $(headers) \ + ethtool.c \ + event.c \ + if.c \ + net.c \ + info_subscriber.c \ + info.c + +libopalutil_core_la_SOURCES = \ alfg.c \ arch.c \ argv.c \ @@ -91,15 +99,11 @@ libopalutil_la_SOURCES = \ bipartite_graph.c \ cmd_line.c \ crc.c \ - ethtool.c \ error.c \ - event.c \ fd.c \ few.c \ - if.c \ keyval_parse.c \ malloc.c \ - net.c \ numtostr.c \ opal_environ.c \ opal_getcwd.c \ @@ -116,17 +120,15 @@ libopalutil_la_SOURCES = \ stacktrace.c \ string_copy.c \ sys_limits.c \ - uri.c \ - info_subscriber.c \ - info.c + uri.c if OPAL_COMPILE_TIMING libopalutil_la_SOURCES += timings.c endif -libopalutil_la_LIBADD = \ +libopalutil_core_la_LIBADD = \ keyval/libopalutilkeyval.la -libopalutil_la_DEPENDENCIES = \ +libopalutil_core_la_DEPENDENCIES = \ keyval/libopalutilkeyval.la # Conditionally install the header files diff --git a/opal/util/few.c b/opal/util/few.c index 94e14167f17..d029e0e63cc 100644 --- a/opal/util/few.c +++ b/opal/util/few.c @@ -29,8 +29,6 @@ #endif #include "opal/constants.h" -#include "opal/util/argv.h" -#include "opal/util/basename.h" #include "opal/util/few.h" int opal_few(char *argv[], int *status) diff --git a/opal/util/output.c b/opal/util/output.c index 778a0dc826c..ede0200cbae 100644 --- a/opal/util/output.c +++ b/opal/util/output.c @@ -3,7 +3,7 @@ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2017 The University of Tennessee and The University + * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart, @@ -47,7 +47,6 @@ #endif #include "opal/constants.h" -#include "opal/mca/pmix/pmix-internal.h" #include "opal/mca/threads/mutex.h" #include "opal/runtime/opal.h" #include "opal/util/opal_environ.h" @@ -63,6 +62,8 @@ static opal_output_stream_t verbose; static char *output_dir = NULL; static char *output_prefix = NULL; +static opal_output_pmix_cleanup_fn_t output_pmix_cleanup_fn = NULL; + /* * Internal data structures and helpers for the generalized output * stream mechanism. @@ -90,6 +91,7 @@ typedef struct { char *ldi_file_suffix; int ldi_fd; int ldi_file_num_lines_lost; + char *ldi_filename; } output_desc_t; /* @@ -205,6 +207,7 @@ bool opal_output_init(void) info[i].ldi_file_suffix = NULL; info[i].ldi_file_want_append = false; info[i].ldi_fd = -1; + info[i].ldi_filename = NULL; info[i].ldi_file_num_lines_lost = 0; } @@ -474,6 +477,22 @@ void opal_output_set_output_file_info(const char *dir, const char *prefix, char } } +void opal_output_register_pmix_cleanup_fn(opal_output_pmix_cleanup_fn_t cleanup_fn) +{ + int n; + + OPAL_THREAD_LOCK(&mutex); + /* Register any filenames setup prior to this call */ + for (n = 0; n < OPAL_OUTPUT_MAX_STREAMS; n++) { + if (NULL != info[n].ldi_filename) { + output_pmix_cleanup_fn(info[n].ldi_filename, false, true, false); + } + } + /* Save the function pointer for later calls */ + output_pmix_cleanup_fn = cleanup_fn; + OPAL_THREAD_UNLOCK(&mutex); +} + /* * Shut down the output stream system */ @@ -710,7 +729,6 @@ static int do_open(int output_id, opal_output_stream_t *lds) static int open_file(int i) { int flags; - char *filename; int n; /* first check to see if this file is already open @@ -748,20 +766,23 @@ static int open_file(int i) /* Setup the filename and open flags */ if (NULL != output_dir) { - filename = (char *) malloc(OPAL_PATH_MAX); - if (NULL == filename) { + if (NULL != info[i].ldi_filename) { + free(info[i].ldi_filename); + } + info[i].ldi_filename = (char *) malloc(OPAL_PATH_MAX); + if (NULL == info[i].ldi_filename) { return OPAL_ERR_OUT_OF_RESOURCE; } - opal_string_copy(filename, output_dir, OPAL_PATH_MAX); - strcat(filename, "/"); + opal_string_copy(info[i].ldi_filename, output_dir, OPAL_PATH_MAX); + strcat(info[i].ldi_filename, "/"); if (NULL != output_prefix) { - strcat(filename, output_prefix); + strcat(info[i].ldi_filename, output_prefix); } if (info[i].ldi_file_suffix != NULL) { - strcat(filename, info[i].ldi_file_suffix); + strcat(info[i].ldi_filename, info[i].ldi_file_suffix); } else { info[i].ldi_file_suffix = NULL; - strcat(filename, "output.txt"); + strcat(info[i].ldi_filename, "output.txt"); } flags = O_CREAT | O_RDWR; if (!info[i].ldi_file_want_append) { @@ -769,23 +790,28 @@ static int open_file(int i) } /* Actually open the file */ - info[i].ldi_fd = open(filename, flags, 0644); + info[i].ldi_fd = open(info[i].ldi_filename, flags, 0644); if (-1 == info[i].ldi_fd) { info[i].ldi_used = false; - free(filename); /* release the filename in all cases */ + /* release the filename in all error cases */ + free(info[i].ldi_filename); + info[i].ldi_filename = NULL; return OPAL_ERR_IN_ERRNO; } /* Make the file be close-on-exec to prevent child inheritance * problems */ if (-1 == fcntl(info[i].ldi_fd, F_SETFD, 1)) { - free(filename); /* release the filename in all cases */ + /* release the filename in all error cases */ + free(info[i].ldi_filename); + info[i].ldi_filename = NULL; return OPAL_ERR_IN_ERRNO; } /* register it to be ignored */ - opal_pmix_register_cleanup(filename, false, true, false); - free(filename); /* release the filename in all cases */ + if (NULL != output_pmix_cleanup_fn) { + output_pmix_cleanup_fn(info[i].ldi_filename, false, true, false); + } } /* Return successfully even if the session dir did not exist yet; @@ -808,6 +834,10 @@ static void free_descriptor(int output_id) if (-1 != ldi->ldi_fd) { close(ldi->ldi_fd); } + if (NULL != ldi->ldi_filename) { + free(ldi->ldi_filename); + ldi->ldi_filename = NULL; + } ldi->ldi_used = false; /* If we strduped a prefix, suffix, or syslog ident, free it */ diff --git a/opal/util/output.h b/opal/util/output.h index 4659240d9b5..ff14622f2e5 100644 --- a/opal/util/output.h +++ b/opal/util/output.h @@ -579,6 +579,14 @@ static inline int opal_best_effort_write(int fd, const void *buf, size_t count) return OPAL_SUCCESS; } +/** + * Cleanup output files function: + * Signature: opal_pmix_register_cleanup + */ +typedef int (*opal_output_pmix_cleanup_fn_t)(char *path, bool directory, bool ignore, + bool jobscope); +void opal_output_register_pmix_cleanup_fn(opal_output_pmix_cleanup_fn_t cleanup_fn); + END_C_DECLS #endif /* OPAL_OUTPUT_H_ */ diff --git a/opal/util/proc.c b/opal/util/proc.c index c309e1a7f98..628a90f399f 100644 --- a/opal/util/proc.c +++ b/opal/util/proc.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2013 The University of Tennessee and The University + * Copyright (c) 2013-2022 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2013 Inria. All rights reserved. @@ -185,6 +185,28 @@ static struct opal_proc_t *opal_proc_for_name_should_never_be_called(opal_proces return NULL; } +/** + * Return the hostname of the target proc. The default implementation + * only support the current proc. Once a RTE is initialized it must replace + * the default function with one that can handle multiple, potentially + * distributed, processes. + */ +static char* +opal_get_proc_hostname_local_only(const opal_proc_t *proc) +{ + /* if the proc is NULL, then we can't know */ + if (NULL == proc) { + return strdup("unknown"); + } + + /* if it is my own hostname we are after, then just hand back + * the value in opal_process_info */ + if (proc == opal_proc_my_name) { + return strdup(opal_process_info.nodename); + } + return strdup("unknown"); +} + char *(*opal_process_name_print)(const opal_process_name_t) = opal_process_name_print_should_never_be_called; char *(*opal_vpid_print)(const opal_vpid_t) = opal_vpid_print_should_never_be_called; @@ -199,30 +221,6 @@ int (*opal_convert_string_to_jobid)(opal_jobid_t *jobid, const char *jobid_strin = opal_convert_string_to_jobid_should_never_be_called; struct opal_proc_t *(*opal_proc_for_name)(const opal_process_name_t name) = opal_proc_for_name_should_never_be_called; +char *(*opal_get_proc_hostname)(const opal_proc_t *proc) + = opal_get_proc_hostname_local_only; -char *opal_get_proc_hostname(const opal_proc_t *proc) -{ - int ret; - char *hostname; - - /* if the proc is NULL, then we can't know */ - if (NULL == proc) { - return strdup("unknown"); - } - - /* if it is my own hostname we are after, then just hand back - * the value in opal_process_info */ - if (proc == opal_proc_my_name) { - return strdup(opal_process_info.nodename); - } - - /* if we don't already have it, then try to get it */ - OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_HOSTNAME, &proc->proc_name, (char **) &hostname, - PMIX_STRING); - if (OPAL_SUCCESS != ret) { - return strdup("unknown"); // return something so the caller doesn't segfault - } - - /* user is not allowed to release the data */ - return hostname; -} diff --git a/opal/util/proc.h b/opal/util/proc.h index ad3282ae654..fee320b38d8 100644 --- a/opal/util/proc.h +++ b/opal/util/proc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 The University of Tennessee and The University + * Copyright (c) 2013-2022 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2013 Inria. All rights reserved. @@ -172,6 +172,6 @@ OPAL_DECLSPEC extern struct opal_proc_t *(*opal_proc_for_name)(const opal_proces * our own. This is to be used by all BTLs so we don't retrieve hostnames * unless needed. The returned value MUST NOT be free'd as it is * owned by the proc_t */ -OPAL_DECLSPEC char *opal_get_proc_hostname(const opal_proc_t *proc); +OPAL_DECLSPEC extern char *(*opal_get_proc_hostname)(const opal_proc_t *proc); #endif /* OPAL_PROC_H */ diff --git a/test/datatype/unpack_hetero.c b/test/datatype/unpack_hetero.c index 49e31b006dd..adad520afb7 100644 --- a/test/datatype/unpack_hetero.c +++ b/test/datatype/unpack_hetero.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2022 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -39,7 +40,7 @@ uint32_t remote_arch = 0xffffffff; */ int main( int argc, char* argv[] ) { - opal_init_util (NULL, NULL); + opal_init(NULL, NULL); /** * By default simulate homogeneous architectures. @@ -93,7 +94,7 @@ int main( int argc, char* argv[] ) OBJ_RELEASE(pConv); /* clean-ups all data allocations */ - opal_finalize_util (); + opal_finalize(); return OPAL_SUCCESS; } From d835977492fb8b10196ad653f85b1baa899d2923 Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Wed, 7 Sep 2022 15:43:32 -0500 Subject: [PATCH 2/2] doc: Add notes about libopen-pal_core Signed-off-by: Joshua Hursey (cherry picked from commit 8112e9b66e388cfdbb81b2d1ecab00da5572868d) --- docs/developers/source-code.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/developers/source-code.rst b/docs/developers/source-code.rst index e8260910f49..2e55c2ad74f 100644 --- a/docs/developers/source-code.rst +++ b/docs/developers/source-code.rst @@ -209,6 +209,16 @@ Each of the three main source directories (``oshmem``, ``ompi``, and either static or shared libraries. Executables are also produced in subdirectories of some of the trees. +The ``libopen-pal`` top-level library is built internally in two parts: + +* ``libopen-pal_core`` Internal "core" portion of OPAL containing the essential source and MCA needed for tools like mpicc/mpirun to link against. The "core" library is not installed. + + * Includes the following MCA frameworks: ``backtrace``, ``dl``, ``installdirs``, ``threads``, ``timer`` + * Includes all of the source under ``opal/class`` and most of ``opal/util`` + * Includes the files suffixed with ``_core`` in ``opal/runtime`` + +* ``libopen-pal`` Includes "core" plus all of the other OPAL project sources. This is installed. + Each of the sub-project source directories have similar (but not identical) directory structures under them: