Importing ULFM ompi layer: snapshot of WIP

Missing BTL and COLL imports.
Almost compiles w/o --with-ft

Author: abouteiller

config/opal_setup_ft.m4

@@ -1,4 +1,8 @@
 dnl
+dnl Copyright (c) 2004-2016 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2009-2012 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2013-2014 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
@@ -12,11 +16,15 @@ dnl
 #
 # --with-ft=TYPE
 #  TYPE:
+#    - mpi (synonym for 'ulfm')
 #    - LAM (synonym for 'cr' currently)
 #    - cr
 #  /* General FT sections */
 #  #if OPAL_ENABLE_FT == 0 /* FT Disabled globaly */
 #  #if OPAL_ENABLE_FT == 1 /* FT Enabled globaly */
+#  /* ULFM Specific sections */
+#  #if OPAL_ENABLE_FT_MPI == 0 /* FT ULFM Disabled */
+#  #if OPAL_ENABLE_FT_MPI == 1 /* FT ULFM Enabled */
 #  /* CR Specific sections */
 #  #if OPAL_ENABLE_FT_CR == 0 /* FT Ckpt/Restart Disabled */
 #  #if OPAL_ENABLE_FT_CR == 1 /* FT Ckpt/Restart Enabled */
@@ -33,7 +41,7 @@ AC_DEFUN([OPAL_SETUP_FT_OPTIONS],[
     opal_setup_ft_options="yes"
     AC_ARG_WITH(ft,
                 [AC_HELP_STRING([--with-ft=TYPE],
-                [Specify the type of fault tolerance to enable. Options: LAM (LAM/MPI-like), cr (Checkpoint/Restart), (default: disabled)])],
+                [Specify the type of fault tolerance to enable. Options: LAM (LAM/MPI-like), cr (Checkpoint/Restart), mpi (ULFM) (default: disabled)])],
                 [opal_want_ft=1],
                 [opal_want_ft=0])
 
@@ -66,6 +74,7 @@ AC_DEFUN([OPAL_SETUP_FT],[
     if test "x$with_ft" != "x" || test "$opal_want_ft" = "1"; then
         opal_want_ft=1
         opal_want_ft_cr=0
+        opal_want_ft_mpi=0
         opal_want_ft_type=none
 
         as_save_IFS=$IFS
@@ -75,7 +84,15 @@ AC_DEFUN([OPAL_SETUP_FT],[
 
             # Default value
             if test "$opt" = "" || test "$opt" = "yes"; then
-                opal_want_ft_cr=1
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ULFM"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ulfm"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "MPI"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "mpi"; then
+                opal_want_ft_mpi=1
             elif test "$opt" = "LAM"; then
                 opal_want_ft_cr=1
             elif test "$opt" = "lam"; then
@@ -89,7 +106,9 @@ AC_DEFUN([OPAL_SETUP_FT],[
                 AC_MSG_ERROR([Cannot continue])
             fi
         done
-        if test "$opal_want_ft_cr" = 1; then
+        if test "$opal_want_ft_mpi" = 1; then
+            opal_want_ft_type="mpi"
+        elif test "$opal_want_ft_cr" = 1; then
             opal_want_ft_type="cr"
         fi
 
@@ -101,16 +120,20 @@ AC_DEFUN([OPAL_SETUP_FT],[
         AC_MSG_WARN([**************************************************])
     else
         opal_want_ft=0
+        opal_want_ft_mpi=0
         opal_want_ft_cr=0
         if test "$opal_setup_ft_options" = "yes"; then
             AC_MSG_RESULT([Disabled fault tolerance])
         fi
     fi
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT], [$opal_want_ft],
                        [Enable fault tolerance general components and logic])
+    AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_MPI], [$opal_want_ft_mpi],
+                       [Enable fault tolerance MPI ULFM components and logic])
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_CR], [$opal_want_ft_cr],
                        [Enable fault tolerance checkpoint/restart components and logic])
     AM_CONDITIONAL(WANT_FT, test "$opal_want_ft" = "1")
+    AM_CONDITIONAL(WANT_FT_MPI, test "$opal_want_ft_mpi" = "1")
     AM_CONDITIONAL(WANT_FT_CR,  test "$opal_want_ft_cr" = "1")
 
     if test "$opal_setup_ft_options" = "yes"; then

contrib/amca-param-sets/ft-enable-mpi

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+# Copyright (c) 2012-2016 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+#
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+# An Aggregate MCA Parameter Set to enable MPI Layer fault tolerance
+# capabilities.
+#
+# Usage:
+#   shell$ mpirun -am ft-enable-mpi ./app
+#
+
+#
+# OPAL Parameters
+#
+
+# ORTE Parameters
+# - Use the modified 'cm' routed component - 'rts'. It is the only one that is currently able to
+#   handle process and daemon loss.
+#
+# JJH: routed=binomial
+routed=rts
+#plm=rsh
+#rmaps=resilient
+
+# Disable tree spawn for now
+plm_rsh_no_tree_spawn = 1
+
+# Stablization at the runtime layer
+errmgr_rts_hnp_priority=5000
+errmgr_rts_orted_priority=5000
+errmgr_rts_app_priority=5000
+
+#
+# OMPI Parameters
+# - Only fully tested with the listed btls
+# - ftbasic provides agreement.
+#
+ompi_ftmpi_enable=1
+# UGNI, OpenIB, TCP, SM have been tested to work well and support most failures gracefully.
+#   if you want perfectly safe, the following restriction can be set, but we believe its' unecessary.
+#   XPMEM is broken and it's build is disabled in this fork.
+#btl=tcp,sm,self
+# Tuned has been tested and works well in many cases; however, it does have some
+#   corner cases. The safer setup "basic,ftbasic" imparts a huge performance 
+#   penalty, we encourage you to try "tuned,basic,ftbasic" and see if it works for you, in most cases, it will.
+coll=tuned,basic,ftbasic

ompi/attribute/attribute_predefined.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -51,6 +51,9 @@
  *
  * MPI_WTIME_IS_GLOBAL is set to 0 (a conservative answer).
  *
+ * MPI_FT is set to 0 or 1 (according to OPAL_ENABLE_FT_MPI and
+ * ompi_ftmpi_enabled)
+ *
  * MPI_APPNUM is set as the result of a GPR subscription.
  *
  * MPI_LASTUSEDCODE is set to an initial value and is reset every time
@@ -129,6 +132,9 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
+#if OPAL_ENABLE_FT_MPI
+        OMPI_SUCCESS != (ret = create_comm(MPI_FT, true)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
 #if 0
         /* JMS For when we implement IMPI */
         OMPI_SUCCESS != (ret = create_comm(IMPI_CLIENT_SIZE, true)) ||
@@ -148,6 +154,12 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = set_f(MPI_WTIME_IS_GLOBAL, 0)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_LASTUSEDCODE,
                                      ompi_mpi_errcode_lastused)) ||
+#if OPAL_ENABLE_FT_MPI
+        /* Although we always define the key to ease fortran integration,
+         * lets not set a default value to the attribute if we do not 
+         * have fault tolerance built in. */
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, ompi_ftmpi_enabled)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
 #if 0
         /* JMS For when we implement IMPI */
         OMPI_SUCCESS != (ret = set(IMPI_CLIENT_SIZE,
@@ -189,6 +201,9 @@ int ompi_attr_free_predefined(void)
         OMPI_SUCCESS != (ret = free_comm(MPI_HOST)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_IO)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_WTIME_IS_GLOBAL)) ||
+#if OPAL_ENABLE_FT_MPI
+        OMPI_SUCCESS != (ret = free_comm(MPI_FT)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||

ompi/communicator/Makefile.am

@@ -3,13 +3,14 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2016 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
 # Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2014      Research Organization for Information Science
@@ -34,3 +35,8 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
 	communicator/comm_cid.c \
 	communicator/comm_request.c
 
+if WANT_FT_MPI
+lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+	communicator/comm_ft.c communicator/comm_ft_reliable_bcast.c communicator/comm_ft_propagate.c communicator/comm_ft_revoke.c
+endif # WANT_FT_MPI
+