Import ULFM Fault Tolerance

The historical repositories contain the full history and
attribution and are available from
  https://bitbucket.org/icldistcomp/ulfm2/src/ulfm/
and prior
  https://github.com/ICLDisco/ulfm-legacy

Signed-off-by: Aurelien Bouteiller <[email protected]>
Signed-off-by: George Bosilca <[email protected]>
Signed-off-by: Josh Hursey <[email protected]>
Signed-off-by: Thomas Herault <[email protected]>
Signed-off-by: Wesley Bland <[email protected]>
Signed-off-by: Nuria Losada <[email protected]>
Signed-off-by: Nathan T. Weeks <[email protected]>

Squashed commit of the following:

commit 878bf118703eadec73f3fee09342c6c33c19c00b
Merge: 4c24553d a7e91d8
Author: Aurelien Bouteiller <[email protected]>
Date:   Tue Jan 5 10:53:44 2021 -0500

    Merge branch 'master' into export/ulfm-to-ompi5-expanded

commit 4c24553daf7db2028f87a01b0769226abb417992
Author: Aurelien Bouteiller <[email protected]>
Date:   Tue Jan 5 10:53:26 2021 -0500

    Revert "TEMPORARY: Use the corrected prrte by default to avoid confusion"

    This reverts commit e5769efcf00ac0faa7f556ed57bcd7b4064fe262.

  ...

commit 69ab6b8
Author: Aurélien Bouteiller <[email protected]>
Date:   Thu Feb 18 20:03:12 2016 -0500

    Importing ULFM ompi layer: snapshot of WIP
    Missing BTL and COLL imports.
    Almost compiles w/o --with-ft

Author: abouteiller

README.FT.ULFM.md

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -45,6 +45,12 @@ AC_DEFUN([OMPI_SETUP_PRRTE],[
 
     AM_CONDITIONAL([OMPI_WANT_PRRTE], [test "$internal_prrte_build" = "1"])
 
+    AS_IF([test "$internal_prrte_build" = "0" -a "$opal_want_ft_type" = "mpi"], [
+        AS_IF([prte_info | $GREP "Resilience support: yes"], [], [
+            AC_MSG_ERROR([Requested enabling fault-tolerance and using external launcher, but external PRTE doesn't support resilience; you can either use the internal PRTE, recompile the external PRTE with fault-tolerance, or disable fault-tolerance. Aborting.])
+        ])
+    ])
+
     OPAL_VAR_SCOPE_POP
 ])
 
@@ -139,6 +145,9 @@ AC_DEFUN([OMPI_SETUP_PRRTE_INTERNAL], [
 
     # add the extra libs
     internal_prrte_args="$internal_prrte_args --with-prte-extra-lib=\"$internal_prrte_libs\" --with-prte-extra-ltlib=\"$internal_prrte_libs\""
+    AS_IF([test "$opal_want_ft_type" = "mpi"],
+          [internal_prrte_args="--enable-prte-ft $internal_prrte_args"],
+          [])
 
     # Pass all our compiler/linker flags to PRRTE, so that it
     # picks up how to build an internal HWLOC, libevent, and PMIx, plus

config/ompi_setup_prrte.m4

@@ -1,4 +1,8 @@
 dnl
+dnl Copyright (c) 2004-2020 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2009-2012 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2013-2014 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
@@ -12,11 +16,15 @@ dnl
 #
 # --with-ft=TYPE
 #  TYPE:
+#    - mpi (synonym for 'ulfm')
 #    - LAM (synonym for 'cr' currently)
 #    - cr
 #  /* General FT sections */
 #  #if OPAL_ENABLE_FT == 0 /* FT Disabled globaly */
 #  #if OPAL_ENABLE_FT == 1 /* FT Enabled globaly */
+#  /* ULFM Specific sections */
+#  #if OPAL_ENABLE_FT_MPI == 0 /* FT ULFM Disabled */
+#  #if OPAL_ENABLE_FT_MPI == 1 /* FT ULFM Enabled */
 #  /* CR Specific sections */
 #  #if OPAL_ENABLE_FT_CR == 0 /* FT Ckpt/Restart Disabled */
 #  #if OPAL_ENABLE_FT_CR == 1 /* FT Ckpt/Restart Enabled */
@@ -33,9 +41,9 @@ AC_DEFUN([OPAL_SETUP_FT_OPTIONS],[
     opal_setup_ft_options="yes"
     AC_ARG_WITH(ft,
                 [AC_HELP_STRING([--with-ft=TYPE],
-                [Specify the type of fault tolerance to enable. Options: LAM (LAM/MPI-like), cr (Checkpoint/Restart), (default: disabled)])],
-                [opal_want_ft=1],
-                [opal_want_ft=0])
+                [Specify the type of fault tolerance to enable. Options: mpi (ULFM), LAM (LAM/MPI-like), cr (Checkpoint/Restart) (default: mpi)])],
+                [],
+                [with_ft=mpi]) # If not specified act as if --with-ft=mpi
 
     #
     # Checkpoint/restart enabled debugging
@@ -63,9 +71,11 @@ AC_DEFUN([OPAL_SETUP_FT],[
     if test "$opal_setup_ft_options" = "yes"; then
         AC_MSG_CHECKING([if want fault tolerance])
     fi
-    if test "x$with_ft" != "x" || test "$opal_want_ft" = "1"; then
+
+    if test x"$with_ft" != "xno"; then
         opal_want_ft=1
         opal_want_ft_cr=0
+        opal_want_ft_mpi=0
         opal_want_ft_type=none
 
         as_save_IFS=$IFS
@@ -74,8 +84,16 @@ AC_DEFUN([OPAL_SETUP_FT],[
             IFS=$as_save_IFS
 
             # Default value
-            if test "$opt" = "" || test "$opt" = "yes"; then
-                opal_want_ft_cr=1
+            if test "$opt" = "yes"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ULFM"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ulfm"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "MPI"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "mpi"; then
+                opal_want_ft_mpi=1
             elif test "$opt" = "LAM"; then
                 opal_want_ft_cr=1
             elif test "$opt" = "lam"; then
@@ -89,28 +107,35 @@ AC_DEFUN([OPAL_SETUP_FT],[
                 AC_MSG_ERROR([Cannot continue])
             fi
         done
-        if test "$opal_want_ft_cr" = 1; then
+        if test "$opal_want_ft_mpi" = 1; then
+            opal_want_ft_type="mpi"
+        elif test "$opal_want_ft_cr" = 1; then
             opal_want_ft_type="cr"
         fi
 
         AC_MSG_RESULT([Enabled $opal_want_ft_type (Specified $with_ft)])
         AC_MSG_WARN([**************************************************])
         AC_MSG_WARN([*** Fault Tolerance Integration into Open MPI is *])
-        AC_MSG_WARN([*** a research quality implementation, and care  *])
-        AC_MSG_WARN([*** should be used when choosing to enable it.   *])
+        AC_MSG_WARN([*** compiled-in, but off by default. Use mpiexec *])
+        AC_MSG_WARN([*** and MCA parameters to turn it on.            *])
+        AC_MSG_WARN([*** Not all components support fault tolerance.  *])
         AC_MSG_WARN([**************************************************])
     else
         opal_want_ft=0
+        opal_want_ft_mpi=0
         opal_want_ft_cr=0
         if test "$opal_setup_ft_options" = "yes"; then
             AC_MSG_RESULT([Disabled fault tolerance])
         fi
     fi
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT], [$opal_want_ft],
                        [Enable fault tolerance general components and logic])
+    AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_MPI], [$opal_want_ft_mpi],
+                       [Enable fault tolerance MPI ULFM components and logic])
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_CR], [$opal_want_ft_cr],
                        [Enable fault tolerance checkpoint/restart components and logic])
     AM_CONDITIONAL(WANT_FT, test "$opal_want_ft" = "1")
+    AM_CONDITIONAL(WANT_FT_MPI, test "$opal_want_ft_mpi" = "1")
     AM_CONDITIONAL(WANT_FT_CR,  test "$opal_want_ft_cr" = "1")
 
     if test "$opal_setup_ft_options" = "yes"; then
@@ -175,4 +200,5 @@ AC_DEFUN([OPAL_SETUP_FT],[
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_THREAD], [$opal_want_ft_thread],
                        [Enable fault tolerance thread in Open PAL])
     AM_CONDITIONAL(WANT_FT_THREAD, test "$opal_want_ft_thread" = "1")
+    OPAL_SUMMARY_ADD([[Miscellaneous]],[[Fault Tolerance support]],[unnecessary], [$with_ft])
 ])

config/opal_setup_ft.m4

@@ -1128,7 +1128,7 @@ AC_CACHE_SAVE
 # visible again
 #
 ###########################################################
-dnl OPAL_SETUP_FT_OPTIONS
+OPAL_SETUP_FT_OPTIONS
 ###########################################################
 # The following line is always required as it contains the
 # AC_DEFINE and AM_CONDITIONAL calls that set variables used

configure.ac

@@ -27,6 +27,10 @@
 amca_paramdir = $(AMCA_PARAM_SETS_DIR)
 dist_amca_param_DATA = amca-param-sets/example.conf
 
+if WANT_FT_MPI
+dist_amca_param_DATA += amca-param-sets/ft-mpi
+endif # WANT_FT_MPI
+
 if WANT_FT_CR
 dist_amca_param_DATA += \
 	amca-param-sets/ft-enable-cr \

contrib/Makefile.am

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2020     The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# An Aggregate MCA Parameter Set to setup an environment that can support
+# User-Level Failure Mitigation (ULFM) fault tolerance (must also be 
+# compiled in with --with-ft=mpi).
+#
+# Usage:
+#   shell$ mpirun --tune ft-mpi ./app
+#
+
+mpi_ft_enable=true
+
+# Since failures are expected, reduce the verbosity of the transport errors
+btl_base_warn_peer_error=false
+
+#
+# Performance tuning parameters (default shown)
+# By default the PRTE failure detector is used (see README.ULFM.md)
+#mpi_ft_detector=false
+#mpi_ft_detector_thread=false
+#mpi_ft_detector_rdma_heartbeat=false
+#mpi_ft_detector_period=3.
+#mpi_ft_detector_timeout=10.
+#
+
+
+#
+# Select only ULFM ready components
+# disabling non-tested and known broken components in FT-MPI builds
+#
+
+#
+# The following frameworks/components are TESTED
+# They handle faults amd should be prefered when running with FT.
+#   pml     ob1
+#   btl     tcp, self, sm(+xpmem,+cma), ugni, uct
+#   coll    base/basic, tuned, ftagree, libnbc
+pml=ob1
+threads=pthreads
+
+#
+# The following frameworks/components are UNTESTED, but **may** work.
+# They should run without faults, and **may** work with faults.
+# You may try and report if successfull.
+#   btl     ofi, portals4, smcuda, usnic, sm(+knem)
+#   coll    inter, sm, sync, cuda, monitoring
+#   pml     monitoring, v/vprotocol
+# We will disable only the components for which good components are known to exist.
+btl=^usnic
+# older versions of xpmem generate bus errors when the other end is dead.
+#btl_sm_single_copy_mechanism=cma
+
+
+#
+# The following frameworks/components are UNTESTED, and probably won't work.
+# They should run without faults, and will probably crash/deadlock after a fault.
+# You may try at your own risk.
+#   coll    hcoll, portals4
+#   topo    (all)
+#   osc     (all)
+#   io      (all)
+#   fcoll   (all)
+#   fbtl    (all)
+# We will disable only the components for which good components are known to exist.
+# Other untested components are selectable but will issue a runtime warning at
+# initiation if FT is enabled.
+coll=^hcoll,portals4
+
+#
+# The following frameworks/components are NOT WORKING. Do not enable these with FT.
+#   mtl     (all)
+#   pml     cm, crcpw, ucx
+mtl=^ofi,portals4,psm2
+# allready enforced by pml=ob1 above
+#pml=^cm,crcpw,ucx
+# allready enforced by threads=pthreads above
+#threads=^argobots,qthreads
+# There is a bug in libevent with the "select" backend that causes an infinite loop
+# when an unplanned disconnect happens. Use something else, or bail.
+opal_event_include=epoll,devpoll,kqueue,evport,poll
+

contrib/amca-param-sets/ft-mpi

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -52,6 +52,9 @@
  *
  * MPI_WTIME_IS_GLOBAL is set to 0 (a conservative answer).
  *
+ * MPI_FT is set to 0 or 1 (according to OPAL_ENABLE_FT_MPI and
+ * ompi_ftmpi_enabled)
+ *
  * MPI_APPNUM is set as the result of a GPR subscription.
  *
  * MPI_LASTUSEDCODE is set to an initial value and is reset every time
@@ -123,7 +126,11 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL))) {
+        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
+#if 1 /* not conditional on OPAL_ENABLE_FT_MPI for ABI */
+        OMPI_SUCCESS != (ret = create_comm(MPI_FT, true)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
+        0) {
         return ret;
     }
 
@@ -133,6 +140,14 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = set_f(MPI_HOST, MPI_PROC_NULL)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_IO, MPI_ANY_SOURCE)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_WTIME_IS_GLOBAL, 0)) ||
+#if OPAL_ENABLE_FT_MPI
+        /* Although we always define the key to ease fortran integration,
+         * lets not set a default value to the attribute if we do not 
+         * have fault tolerance built in. */
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, ompi_ftmpi_enabled)) ||
+#else
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, false)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = set_f(MPI_LASTUSEDCODE,
                                      ompi_mpi_errcode_lastused))) {
         return ret;
@@ -161,6 +176,9 @@ int ompi_attr_free_predefined(void)
         OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||
+#if 1 /* not conditional on OPAL_ENABLE_FT_MPI for ABI */
+        OMPI_SUCCESS != (ret = free_comm(MPI_FT)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_BASE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_DISP_UNIT)) ||

ompi/attribute/attribute_predefined.c

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -34,3 +34,8 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
 	communicator/comm_cid.c \
 	communicator/comm_request.c
 
+if WANT_FT_MPI
+lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+	communicator/ft/comm_ft.c communicator/ft/comm_ft_reliable_bcast.c communicator/ft/comm_ft_propagator.c communicator/ft/comm_ft_detector.c communicator/ft/comm_ft_revoke.c
+endif # WANT_FT_MPI
+