Bring the RML modifications across. This is the first step in a revamp of the ORTE messaging subsystem to support fabric-based communications during launch and wireup phases. When completed, the grpcomm and plm frameworks will each have their own "conduit" for communication - each conduit corresponds to a particular RML messaging transport. This can be the active OOB-based component, or a provider from within the RML/OFI component. Messages sent down the conduit will flow across the associated transport.

Multiple conduits can exist at the same time, and can even point to the same base transport. Each conduit can have its own characteristics (e.g., flow control) based on the info keys provided to the "open_conduit" call. For ease during the transition period, the "legacy" RML interfaces remain as wrappers over the new conduit-based APIs using a default conduit opened during orte_init - this default conduit is tied to the OOB framework so that current behaviors are preserved. Once the transition has been completed, a one-time cleanup will be done to update all RML calls to the new APIs and the "legacy" interfaces will be deleted.

While we are at it: Remove oob/usock component to eliminate the TMPDIR length problem - get all working, including oob_stress

Author: Ralph Castain

config/ltmain_nag_pthread.diff

@@ -2,7 +2,7 @@
 +++ config/ltmain.sh
 @@ -6417,8 +6417,14 @@
  	func_source "$lib"
- 
+
  	# Convert "-framework foo" to "foo.ltframework"
 +        # and "-pthread" to "-Wl,-pthread" if NAG compiler
  	if test -n "$inherited_linker_flags"; then

config/ompi_check_psm.m4

@@ -17,7 +17,7 @@ dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 dnl                         reserved.
 dnl Copyright (c) 2016      Intel Corporation. All rights reserved.
-dnl 
+dnl
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow

config/ompi_check_ucx.m4

@@ -15,7 +15,7 @@
 
 # OMPI_CHECK_UCX(prefix, [action-if-found], [action-if-not-found])
 # --------------------------------------------------------
-# check if UCX support can be found.  sets prefix_{CPPFLAGS, 
+# check if UCX support can be found.  sets prefix_{CPPFLAGS,
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
 AC_DEFUN([OMPI_CHECK_UCX],[

config/opal_check_xpmem.m4

@@ -30,7 +30,7 @@
 # --------------------------------------------------------
 AC_DEFUN([OPAL_CHECK_CRAY_XPMEM],[
     if test -z "$opal_check_cray_xpmem_happy" ; then
-        AC_ARG_WITH([cray_xpmem],		    
+        AC_ARG_WITH([cray_xpmem],
                     [AC_HELP_STRING([--with-cray-xpmem(=yes/no)],
                     [Build Cray XPMEM support(default: auto)])],
                     [], with_cray_xpmem=auto)

config/opal_setup_wrappers.m4

@@ -263,7 +263,7 @@ EOF
     chmod +x $rpath_script
     wl_fc=`./$rpath_script`
     rm -f $rpath_script
-    
+
     LDFLAGS="$LDFLAGS_save ${wl_fc}--enable-new-dtags"
     AC_LANG_PUSH([Fortran])
     AC_LINK_IFELSE([AC_LANG_SOURCE([[program test

contrib/ompi-time.sh

@@ -287,10 +287,10 @@ function do_analysis() {
 
   if [ ! -e $tesdir ]; then
     do_err "can not find testdir: $testdir"
-  fi 
+  fi
   if [ -z $basefile -o ! -f $basefile ]; then
     do_err "can not find basefile: $basefile"
-  fi 
+  fi
   if [ -z $outfile -o ! -f $outfile ]; then
     do_err "can not find outfile: $outfile"
   fi
@@ -323,7 +323,7 @@ function do_analysis() {
   max_line=`sort -n $outfile1 | tail -n1`
   if [ -z "$min_line" -o -z "$max_line" ]; then
     do_err "can not find max/min lines in : $outfile1"
-  fi 
+  fi
   min_t=$( echo "$min_line" | cut -f1 -d$'\t')
   max_t=$( echo "$max_line" | cut -f1 -d$'\t')
   echo -e "`bc -l <<< "scale=3; (($min_t - $start_t) / 1000000)"`\t`echo "$min_line" | cut -f4 -d$'\t'`\t`echo "$min_line" | cut -f5 -d$'\t'`" >> $resultfile 2>&1
@@ -341,7 +341,7 @@ function do_report() {
 
   if [ -z $resultfile -o ! -f $resultfile ]; then
     do_err "can not find resultfile: $resultfile"
-  fi 
+  fi
   min_t=`awk -F $'\t' '{ if (NR == 1) print $1 }' $resultfile`
   max_t=`awk -F $'\t' '{ if (NR == 2) print $1 }' $resultfile`
   echo -e "${nodes}\t${ppn}\t${min_t}\t${max_t}" >> $reportfile 2>&1
@@ -374,7 +374,7 @@ static inline void timestamp(FILE *file)
     int procid = -1;
     char *str = NULL;
 
-    gettimeofday(&tv, NULL); 
+    gettimeofday(&tv, NULL);
 
     if (gethostname(name, sizeof(name)) != 0)
         strcpy(name, "localhost");
@@ -485,7 +485,7 @@ function do_parse() {
       test_list="${test_list} ${test}"
     done
   done
-    
+
   result_list=`echo $result_list | tr " " "\n" | sort | uniq | tr "\n" " "`
   test_list=`echo $test_list | tr " " "\n" | sort | uniq | tr "\n" " "`
 
@@ -994,7 +994,7 @@ int orte_rml_base_update_contact_info(void * data)
             FILE *fd = NULL;
             char filename[1024];
             char *str = getenv("SLURM_PROCID");
- 
+
             if (str) {
                 sprintf(filename, "%s.%s", "$outfile", str);
                 fd = fopen(filename, "a");
@@ -1070,7 +1070,7 @@ int orte_rml_base_update_contact_info(void * data)
             FILE *fd = NULL;
             char filename[1024];
             char *str = getenv("SLURM_PROCID");
- 
+
             if (str) {
                 sprintf(filename, "%s.%s", "$outfile", str);
                 fd = fopen(filename, "a");

ompi/communicator/comm_cid.c

@@ -340,7 +340,7 @@ static int ompi_comm_allreduce_getnextcid (ompi_comm_request_t *request)
         if (flag) {
             opal_pointer_array_test_and_set_item(&ompi_mpi_communicators, context->nextlocal_cid, NULL);
         }
- 
+
         ompi_comm_cid_lowest_id = INT64_MAX;
         OPAL_THREAD_UNLOCK(&ompi_cid_lock);
         return OMPI_ERR_OUT_OF_RESOURCE;

orte/include/orte/constants.h

@@ -154,7 +154,8 @@ enum {
     ORTE_ERR_OUT_OF_ORDER_MSG               = (ORTE_ERR_BASE - 55),
     ORTE_ERR_OPEN_CHANNEL_DUPLICATE         = (ORTE_ERR_BASE - 56),
     ORTE_ERR_FORCE_SELECT                   = (ORTE_ERR_BASE - 57),
-    ORTE_ERR_JOB_CANCELLED                  = (ORTE_ERR_BASE - 58)
+    ORTE_ERR_JOB_CANCELLED                  = (ORTE_ERR_BASE - 58),
+    ORTE_ERR_CONDUIT_SEND_FAIL              = (ORTE_ERR_BASE - 59)
 };
 
 #define ORTE_ERR_MAX                      (ORTE_ERR_BASE - 100)

orte/mca/ess/base/ess_base_std_app.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -201,12 +201,6 @@ int orte_ess_base_app_setup(bool db_restrict_local)
         error = "orte_grpcomm_base_select";
         goto error;
     }
-    /* enable communication via the rml */
-    if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_rml.enable_comm";
-        goto error;
-    }
     /* setup the routed info  */
     if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
         ORTE_ERROR_LOG(ret);

orte/mca/ess/base/ess_base_std_orted.c

@@ -356,6 +356,29 @@ int orte_ess_base_orted_setup(char **hosts)
     /* obviously, we have "reported" */
     jdata->num_reported = 1;
 
+    /* setup the PMIx framework - ensure it skips all non-PMIx components,
+     * but do not override anything we were given */
+    opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray,isolated", false, &environ);
+    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_pmix_base_framework, 0))) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_pmix_base_open";
+        goto error;
+    }
+    if (ORTE_SUCCESS != (ret = opal_pmix_base_select())) {
+        ORTE_ERROR_LOG(ret);
+        error = "opal_pmix_base_select";
+        goto error;
+    }
+    /* set the event base */
+    opal_pmix_base_set_evbase(orte_event_base);
+    /* setup the PMIx server */
+    if (ORTE_SUCCESS != (ret = pmix_server_init())) {
+        /* the server code already barked, so let's be quiet */
+        ret = ORTE_ERR_SILENT;
+        error = "pmix_server_init";
+        goto error;
+    }
+
     /* Setup the communication infrastructure */
     if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_oob_base_framework, 0))) {
         ORTE_ERROR_LOG(ret);
@@ -380,6 +403,13 @@ int orte_ess_base_orted_setup(char **hosts)
     /* add our contact info */
     proc->rml_uri = orte_rml.get_contact_info();
 
+    /* setup the PMIx server */
+    if (ORTE_SUCCESS != (ret = pmix_server_init())) {
+        ORTE_ERROR_LOG(ret);
+        error = "pmix server init";
+        goto error;
+    }
+
     /* select the errmgr */
     if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
         ORTE_ERROR_LOG(ret);
@@ -440,49 +470,7 @@ int orte_ess_base_orted_setup(char **hosts)
         error = "orte_rtc_base_select";
         goto error;
     }
-    /* enable communication with the rml */
-    if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_rml.enable_comm";
-        goto error;
-    }
-    /* Now provide a chance for the PLM
-     * to perform any module-specific init functions. This
-     * needs to occur AFTER the communications are setup
-     * as it may involve starting a non-blocking recv
-     * Do this only if a specific PLM was given to us - the
-     * orted has no need of the proxy PLM at all
-     */
-    if (plm_in_use) {
-        if (ORTE_SUCCESS != (ret = orte_plm.init())) {
-            ORTE_ERROR_LOG(ret);
-            error = "orte_plm_init";
-            goto error;
-        }
-    }
 
-    /* setup the PMIx framework - ensure it skips all non-PMIx components,
-     * but do not override anything we were given */
-    opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray,isolated", false, &environ);
-    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_pmix_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_pmix_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = opal_pmix_base_select())) {
-        ORTE_ERROR_LOG(ret);
-        error = "opal_pmix_base_select";
-        goto error;
-    }
-    /* set the event base */
-    opal_pmix_base_set_evbase(orte_event_base);
-    /* setup the PMIx server */
-    if (ORTE_SUCCESS != (ret = pmix_server_init())) {
-        /* the server code already barked, so let's be quiet */
-        ret = ORTE_ERR_SILENT;
-        error = "pmix_server_init";
-        goto error;
-    }
 #if ORTE_ENABLE_STATIC_PORTS
     /* if we are using static ports, then we need to setup
      * the daemon info so the RML can function properly
@@ -511,6 +499,21 @@ int orte_ess_base_orted_setup(char **hosts)
      */
     orte_routed.update_routing_plan();
 
+    /* Now provide a chance for the PLM
+     * to perform any module-specific init functions. This
+     * needs to occur AFTER the communications are setup
+     * as it may involve starting a non-blocking recv
+     * Do this only if a specific PLM was given to us - the
+     * orted has no need of the proxy PLM at all
+     */
+    if (plm_in_use) {
+        if (ORTE_SUCCESS != (ret = orte_plm.init())) {
+            ORTE_ERROR_LOG(ret);
+            error = "orte_plm_init";
+            goto error;
+        }
+    }
+
     /* setup the routed info - the selected routed component
      * will know what to do.
      */

orte/mca/ess/base/ess_base_std_tool.c

@@ -134,12 +134,7 @@ int orte_ess_base_tool_setup(void)
      * to which I want to communicate and setting up a route for
      * that link is my responsibility
      */
-    /* enable communication via the rml */
-    if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_rml.enable_comm";
-        goto error;
-    }
+
     /* we -may- need to know the name of the head
      * of our session directory tree, particularly the
      * tmp base where any other session directories on

orte/mca/ess/hnp/ess_hnp_module.c

@@ -551,12 +551,7 @@ static int rte_init(void)
         error = "orte_rtc_base_select";
         goto error;
     }
-    /* enable communication with the rml */
-    if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_rml.enable_comm";
-        goto error;
-    }
+
     /* we are an hnp, so update the contact info field for later use */
     orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
     proc->rml_uri = strdup(orte_process_info.my_hnp_uri);

orte/mca/iof/hnp/iof_hnp.c

@@ -56,9 +56,6 @@
 /* LOCAL FUNCTIONS */
 static void stdin_write_handler(int fd, short event, void *cbdata);
 
-static void
-orte_iof_hnp_exception_handler(orte_process_name_t* peer, orte_rml_exception_t reason);
-
 /* API FUNCTIONS */
 static int init(void);
 
@@ -94,8 +91,6 @@ orte_iof_base_module_t orte_iof_hnp_module = {
 /* Initialize the module */
 static int init(void)
 {
-    int rc;
-
     /* post non-blocking recv to catch forwarded IO from
      * the orteds
      */
@@ -105,12 +100,6 @@ static int init(void)
                             orte_iof_hnp_recv,
                             NULL);
 
-    if (ORTE_SUCCESS != (rc = orte_rml.add_exception_handler(orte_iof_hnp_exception_handler))) {
-        ORTE_ERROR_LOG(rc);
-        orte_rml.recv_cancel(ORTE_NAME_WILDCARD, ORTE_RML_TAG_IOF_HNP);
-        return rc;
-    }
-
     OBJ_CONSTRUCT(&mca_iof_hnp_component.procs, opal_list_t);
     mca_iof_hnp_component.stdinev = NULL;
 
@@ -610,37 +599,3 @@ static void stdin_write_handler(int fd, short event, void *cbdata)
         }
     }
 }
-
-/**
- * Callback when peer is disconnected
- */
-
-static void
-orte_iof_hnp_exception_handler(orte_process_name_t* peer, orte_rml_exception_t reason)
-{
-#if 0
-    orte_iof_base_endpoint_t *endpoint;
-    opal_output_verbose(1, orte_iof_base_framework.framework_output,
-                        "iof svc exception handler! %s\n",
-                        ORTE_NAME_PRINT((orte_process_name_t*)peer));
-
-    /* If we detect an exception on the RML connection to a peer,
-     delete all of its subscriptions and publications.  Note that
-     exceptions can be detected during a normal RML shutdown; they
-     are recoverable events (no need to abort). */
-    orte_iof_hnp_sub_delete_all(peer);
-    orte_iof_hnp_pub_delete_all(peer);
-    opal_output_verbose(1, orte_iof_base_framework.framework_output, "deleted all pubs and subs\n");
-
-    /* Find any streams on any endpoints for this peer and close them */
-    while (NULL !=
-           (endpoint = orte_iof_base_endpoint_match(peer, ORTE_NS_CMP_ALL,
-                                                    ORTE_IOF_ANY))) {
-        orte_iof_base_endpoint_closed(endpoint);
-
-        /* Delete the endpoint that we just matched */
-        orte_iof_base_endpoint_delete(peer, ORTE_NS_CMP_ALL, ORTE_IOF_ANY);
-    }
-#endif
-    opal_output_verbose(1, orte_iof_base_framework.framework_output, "done with exception handler\n");
-}

orte/mca/oob/base/base.h

@@ -181,6 +181,12 @@ OBJ_CLASS_DECLARATION(mca_oob_uri_req_t);
     }while(0);
 ORTE_DECLSPEC void orte_oob_base_set_addr(int fd, short args, void *cbdata);
 
+
+/* Get the available transports and their attributes */
+#define ORTE_OOB_GET_TRANSPORTS(u) orte_oob_base_get_transports(u)
+ORTE_DECLSPEC void orte_oob_base_get_transports(opal_list_t *transports);
+
+
 #if OPAL_ENABLE_FT_CR == 1
 ORTE_DECLSPEC void orte_oob_base_ft_event(int fd, short args, void *cbdata);
 #endif