WIP: Add smsc endpoints to HAN

Signed-off-by: Luke Robison <[email protected]>

Author: lrbison

ompi/mca/coll/han/coll_han.h

@@ -41,6 +41,7 @@
 #include "coll_han_trigger.h"
 #include "ompi/mca/coll/han/coll_han_dynamic.h"
 #include "coll_han_algorithms.h"
+#include "opal/mca/smsc/smsc.h"
 
 /*
  * Today;
@@ -341,6 +342,7 @@ typedef struct mca_coll_han_module_t {
 
     /* Sub-communicator */
     struct ompi_communicator_t *sub_comm[NB_TOPO_LVL];
+    mca_smsc_endpoint_t **local_smsc_eps;
 } mca_coll_han_module_t;
 OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
 

ompi/mca/coll/han/coll_han_module.c

@@ -138,6 +138,12 @@ mca_coll_han_module_destruct(mca_coll_han_module_t * module)
         free(module->cached_topo);
         module->cached_topo = NULL;
     }
+    if (module->local_smsc_eps != NULL) {
+        int low_size = ompi_comm_size(module->sub_comm[0]);
+        for (i=0; i<low_size; i++) {
+            mca_smsc->return_endpoint( module->local_smsc_eps[i] );
+        }
+    }
     for(i=0 ; i<NB_TOPO_LVL ; i++) {
         if(NULL != module->sub_comm[i]) {
             ompi_comm_free(&(module->sub_comm[i]));

ompi/mca/coll/han/coll_han_subcomms.c

@@ -25,6 +25,7 @@
 #include "mpi.h"
 #include "coll_han.h"
 #include "coll_han_dynamic.h"
+#include "opal/mca/smsc/smsc.h"
 
 #define HAN_SUBCOM_SAVE_COLLECTIVE(FALLBACKS, COMM, HANM, COLL)                  \
     do {                                                                         \
@@ -183,6 +184,20 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
     HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, gather);
     HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, scatter);
 
+    if (mca_smsc) {
+        han_module->local_smsc_eps = malloc( sizeof(*han_module->local_smsc_eps) * low_size);
+        for (int jlow=0; jlow<low_size; jlow++) {
+            struct ompi_proc_t* ompi_proc = ompi_comm_peer_lookup(*low_comm, jlow);
+            han_module->local_smsc_eps[jlow] = mca_smsc->get_endpoint(&ompi_proc->super);
+        }
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "Han created SMSC endpoints for low_comm\n"));
+    } else {
+        han_module->local_smsc_eps = NULL;
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "Han did not find any SMSC components\n"));
+    }
+
     OBJ_DESTRUCT(&comm_info);
     return OMPI_SUCCESS;
 
@@ -350,6 +365,20 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, gather);
     HAN_SUBCOM_LOAD_COLLECTIVE(fallbacks, comm, han_module, scatter);
 
+    if (mca_smsc) {
+        han_module->local_smsc_eps = malloc( sizeof(*han_module->local_smsc_eps) * low_size);
+        for (int jlow=0; jlow<low_size; jlow++) {
+            struct ompi_proc_t* ompi_proc = ompi_comm_peer_lookup(low_comms[1], jlow);
+            han_module->local_smsc_eps[jlow] = mca_smsc->get_endpoint(&ompi_proc->super);
+        }
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "Han created SMSC endpoints for low_comm\n"));
+    } else {
+        han_module->local_smsc_eps = NULL;
+        OPAL_OUTPUT_VERBOSE((30, mca_coll_han_component.han_output,
+                             "Han did not find any SMSC components\n"));
+    }
+
     OBJ_DESTRUCT(&comm_info);
     return OMPI_SUCCESS;
 }