Import ULFM Fault Tolerance

The historical repositories contain the full history and
attribution and are available
from https://bitbucket.org/icldistcomp/ulfm2/src/ulfm/
and prior https://github.com/ICLDisco/ulfm-legacy

Squashed commit of the following:

commit 618d5653442bde65bff97ac93f8c26eaeec3cfba
Merge: c4bb8155 8a2127b
Author: Aurelien Bouteiller <[email protected]>
Date:   Fri Aug 7 17:35:13 2020 -0400

    Merge branch 'master' into export/ulfm-to-ompi5-expanded

...

commit 69ab6b8
Author: Aurélien Bouteiller <[email protected]>
Date:   Thu Feb 18 20:03:12 2016 -0500

    Importing ULFM ompi layer: snapshot of WIP
    Missing BTL and COLL imports.
    Almost compiles w/o --with-ft

Signed-off-by: Aurelien Bouteiller <[email protected]>
Signed-off-by: George Bosilca <[email protected]>
Signed-off-by: Josh Hursey <[email protected]>
Signed-off-by: Thomas Herault <[email protected]>
Signed-off-by: Wesley Bland <[email protected]>
Signed-off-by: Nuria Losada <[email protected]>
Signed-off-by: Nathan T. Weeks <[email protected]>

Author: abouteiller

README.FT.ULFM.md

@@ -1,4 +1,8 @@
 dnl
+dnl Copyright (c) 2004-2018 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2009-2012 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2013-2014 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
@@ -12,11 +16,15 @@ dnl
 #
 # --with-ft=TYPE
 #  TYPE:
+#    - mpi (synonym for 'ulfm')
 #    - LAM (synonym for 'cr' currently)
 #    - cr
 #  /* General FT sections */
 #  #if OPAL_ENABLE_FT == 0 /* FT Disabled globaly */
 #  #if OPAL_ENABLE_FT == 1 /* FT Enabled globaly */
+#  /* ULFM Specific sections */
+#  #if OPAL_ENABLE_FT_MPI == 0 /* FT ULFM Disabled */
+#  #if OPAL_ENABLE_FT_MPI == 1 /* FT ULFM Enabled */
 #  /* CR Specific sections */
 #  #if OPAL_ENABLE_FT_CR == 0 /* FT Ckpt/Restart Disabled */
 #  #if OPAL_ENABLE_FT_CR == 1 /* FT Ckpt/Restart Enabled */
@@ -33,9 +41,9 @@ AC_DEFUN([OPAL_SETUP_FT_OPTIONS],[
     opal_setup_ft_options="yes"
     AC_ARG_WITH(ft,
                 [AC_HELP_STRING([--with-ft=TYPE],
-                [Specify the type of fault tolerance to enable. Options: LAM (LAM/MPI-like), cr (Checkpoint/Restart), (default: disabled)])],
-                [opal_want_ft=1],
-                [opal_want_ft=0])
+                [Specify the type of fault tolerance to enable. Options: mpi (ULFM), LAM (LAM/MPI-like), cr (Checkpoint/Restart) (default: mpi)])],
+                [],
+                [with_ft=mpi]) # If not specified act as if --with-ft=mpi
 
     #
     # Checkpoint/restart enabled debugging
@@ -63,9 +71,11 @@ AC_DEFUN([OPAL_SETUP_FT],[
     if test "$opal_setup_ft_options" = "yes"; then
         AC_MSG_CHECKING([if want fault tolerance])
     fi
-    if test "x$with_ft" != "x" || test "$opal_want_ft" = "1"; then
+
+    if test x"$with_ft" != "xno"; then
         opal_want_ft=1
         opal_want_ft_cr=0
+        opal_want_ft_mpi=0
         opal_want_ft_type=none
 
         as_save_IFS=$IFS
@@ -74,8 +84,16 @@ AC_DEFUN([OPAL_SETUP_FT],[
             IFS=$as_save_IFS
 
             # Default value
-            if test "$opt" = "" || test "$opt" = "yes"; then
-                opal_want_ft_cr=1
+            if test "$opt" = "yes"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ULFM"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ulfm"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "MPI"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "mpi"; then
+                opal_want_ft_mpi=1
             elif test "$opt" = "LAM"; then
                 opal_want_ft_cr=1
             elif test "$opt" = "lam"; then
@@ -89,7 +107,9 @@ AC_DEFUN([OPAL_SETUP_FT],[
                 AC_MSG_ERROR([Cannot continue])
             fi
         done
-        if test "$opal_want_ft_cr" = 1; then
+        if test "$opal_want_ft_mpi" = 1; then
+            opal_want_ft_type="mpi"
+        elif test "$opal_want_ft_cr" = 1; then
             opal_want_ft_type="cr"
         fi
 
@@ -101,18 +121,33 @@ AC_DEFUN([OPAL_SETUP_FT],[
         AC_MSG_WARN([**************************************************])
     else
         opal_want_ft=0
+        opal_want_ft_mpi=0
         opal_want_ft_cr=0
         if test "$opal_setup_ft_options" = "yes"; then
             AC_MSG_RESULT([Disabled fault tolerance])
         fi
     fi
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT], [$opal_want_ft],
                        [Enable fault tolerance general components and logic])
+    AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_MPI], [$opal_want_ft_mpi],
+                       [Enable fault tolerance MPI ULFM components and logic])
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_CR], [$opal_want_ft_cr],
                        [Enable fault tolerance checkpoint/restart components and logic])
     AM_CONDITIONAL(WANT_FT, test "$opal_want_ft" = "1")
+    AM_CONDITIONAL(WANT_FT_MPI, test "$opal_want_ft_mpi" = "1")
     AM_CONDITIONAL(WANT_FT_CR,  test "$opal_want_ft_cr" = "1")
 
+    if test "$opal_want_ft_mpi" = "1"; then
+        AC_MSG_CHECKING([loading UFLM FT MPI platform file additions])
+        if test -r "${srcdir}/contrib/platform/ft_mpi_ulfm" ; then
+            . ${srcdir}/contrib/platform/ft_mpi_ulfm
+            AC_MSG_RESULT([Loaded contrib/platform/ft_mpi_ulfm])
+            enable_mca_no_build="$enable_mca_no_build_ft,$enable_mca_no_build"
+        else
+            AC_MSG_RESULT([Not found])
+        fi
+    fi
+
     if test "$opal_setup_ft_options" = "yes"; then
         AC_MSG_CHECKING([if want checkpoint/restart enabled debugging option])
     fi
@@ -175,4 +210,5 @@ AC_DEFUN([OPAL_SETUP_FT],[
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_THREAD], [$opal_want_ft_thread],
                        [Enable fault tolerance thread in Open PAL])
     AM_CONDITIONAL(WANT_FT_THREAD, test "$opal_want_ft_thread" = "1")
+    OPAL_SUMMARY_ADD([[Miscellaneous]],[[Fault Tolerance support]],[unnecessary], [$with_ft (disabling components: {${enable_mca_no_build_ft//,/ }})])
 ])

config/opal_setup_ft.m4

@@ -1087,7 +1087,7 @@ AC_CACHE_SAVE
 # visible again
 #
 ###########################################################
-dnl OPAL_SETUP_FT_OPTIONS
+OPAL_SETUP_FT_OPTIONS
 ###########################################################
 # The following line is always required as it contains the
 # AC_DEFINE and AM_CONDITIONAL calls that set variables used

configure.ac

@@ -27,6 +27,10 @@
 amca_paramdir = $(AMCA_PARAM_SETS_DIR)
 dist_amca_param_DATA = amca-param-sets/example.conf
 
+if WANT_FT_MPI
+dist_amca_param_DATA += amca-param-sets/ft-mpi
+endif # WANT_FT_MPI
+
 if WANT_FT_CR
 dist_amca_param_DATA += \
 	amca-param-sets/ft-enable-cr \

contrib/Makefile.am

@@ -0,0 +1,83 @@
+#
+# Copyright (c) 2020     The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# An Aggregate MCA Parameter Set to setup an environment that can support
+# User-Level Failure Mitigation (ULFM) fault tolerance (must also be 
+# compiled in with --with-ft=mpi).
+#
+# Usage:
+#   shell$ mpirun --tune ft-mpi ./app
+#
+
+mpi_ft_enable=true
+
+#
+# Performance tuning parameters (default shown)
+#mpi_ft_detector=true
+#mpi_ft_detector_thread=false
+#mpi_ft_detector_rdma_heartbeat=false
+#mpi_ft_detector_period=3.
+#mpi_ft_detector_timeout=10.
+#
+
+
+#
+# Select only ULFM ready components
+# disabling non-tested and known broken components in FT-MPI builds
+#
+
+#
+# The following frameworks/components are TESTED
+# They handle faults amd should be prefered when running with FT.
+#   pml     ob1
+#   btl     tcp, self, sm(+xpmem,+cma), ugni, uct
+#   coll    base/basic, tuned, ftagree, libnbc
+pml=ob1
+threads=pthreads
+
+#
+# The following frameworks/components are UNTESTED, but **may** work.
+# They should run without faults, and **may** work with faults.
+# You may try and report if successfull.
+#   btl     ofi, portals4, smcuda, usnic, sm(+knem)
+#   coll    inter, sm, sync, cuda, monitoring
+#   pml     monitoring, v/vprotocol
+# We will disable only the components for which good components are known to exist.
+btl=^usnic
+# older versions of xpmem generate bus errors when the other end is dead.
+#btl_sm_single_copy_mechanism=cma
+
+
+#
+# The following frameworks/components are UNTESTED, and probably won't work.
+# They should run without faults, and will probably crash/deadlock after a fault.
+# You may try at your own risk.
+#   coll    hcoll, portals4
+#   topo    (all)
+#   osc     (all)
+#   io      (all)
+#   fcoll   (all)
+#   fbtl    (all)
+# We will disable only the components for which good components are known to exist.
+# Other untested components are selectable but will issue a runtime warning at
+# initiation if FT is enabled.
+coll=^hcoll,portals4
+
+#
+# The following frameworks/components are NOT WORKING. Do not enable these with FT.
+# They should run without faults, but will completely bypass FT capabilities.
+#   mtl     (all)
+#   pml     cm, crcpw, ucx
+mtl=^ofi,portals4,psm2
+# allready enforced by pml=ob1 above
+#pml=^cm,crcpw,ucx
+# allready enforced by threads=pthreads above
+#threads=^argobots,qthreads
+

contrib/amca-param-sets/ft-mpi

@@ -0,0 +1,19 @@
+# configure time selection of components.
+
+# we have small modifications to libevent, better use the internal one.
+with_libevent=internal
+# at the moment using any PMIx4 should work.
+#with_pmix=internal
+
+# older versions of xpmem generate bus errors when the other end is dead.
+#with_cray_xpmem=no
+
+# These components are runtime-disabled when using
+#   `mpirun --tune ft-mpi`
+# You may also want to disable compilation alltogether.
+#enable_mca_no_build_ft=\
+#mtl,pml-cm,pml-crcpw,pml-yalla,pml-ucx,\
+#coll-cuda,coll-fca,coll-hcoll,coll-portals4,\
+#btl-usnic,btl-portals4,btl-scif,btl-smcuda,\
+#pml-monitoring,pml-v,vprotocol,crcp\
+

contrib/platform/ft_mpi_ulfm

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2019 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -52,6 +52,9 @@
  *
  * MPI_WTIME_IS_GLOBAL is set to 0 (a conservative answer).
  *
+ * MPI_FT is set to 0 or 1 (according to OPAL_ENABLE_FT_MPI and
+ * ompi_ftmpi_enabled)
+ *
  * MPI_APPNUM is set as the result of a GPR subscription.
  *
  * MPI_LASTUSEDCODE is set to an initial value and is reset every time
@@ -123,7 +126,11 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL))) {
+        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
+#if OPAL_ENABLE_FT_MPI
+        OMPI_SUCCESS != (ret = create_comm(MPI_FT, true)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
+        0) {
         return ret;
     }
 
@@ -133,6 +140,12 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = set_f(MPI_HOST, MPI_PROC_NULL)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_IO, MPI_ANY_SOURCE)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_WTIME_IS_GLOBAL, 0)) ||
+#if OPAL_ENABLE_FT_MPI
+        /* Although we always define the key to ease fortran integration,
+         * lets not set a default value to the attribute if we do not 
+         * have fault tolerance built in. */
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, ompi_ftmpi_enabled)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = set_f(MPI_LASTUSEDCODE,
                                      ompi_mpi_errcode_lastused))) {
         return ret;
@@ -161,6 +174,9 @@ int ompi_attr_free_predefined(void)
         OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||
+#if OPAL_ENABLE_FT_MPI
+        OMPI_SUCCESS != (ret = free_comm(MPI_FT)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_BASE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_DISP_UNIT)) ||

ompi/attribute/attribute_predefined.c

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -34,3 +34,8 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
 	communicator/comm_cid.c \
 	communicator/comm_request.c
 
+if WANT_FT_MPI
+lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+	communicator/ft/comm_ft.c communicator/ft/comm_ft_reliable_bcast.c communicator/ft/comm_ft_propagator.c communicator/ft/comm_ft_detector.c communicator/ft/comm_ft_revoke.c
+endif # WANT_FT_MPI
+