MPI 4: Add MPI_COMM_TYPE_HW_UNGUIDED and MPI_COMM_TYPE_HW_GUIDED

 * `MPI_COMM_TYPE_HW_GUIDED` supports all of the existing `OMPI_COMM_TYPE_`
   options.
 * `MPI_COMM_TYPE_HW_UNGUIDED` is recognized, but not supported so it returns
   `MPI_COMM_NULL` indidicating that the MPI library cannot split the
   communicator any further.

Signed-off-by: Joshua Hursey <[email protected]>

Author: jjhursey

ompi/communicator/comm.c

@@ -57,6 +57,29 @@
 
 #include "ompi/runtime/params.h"
 
+#if MPI_VERSION >= 4
+struct ompi_comm_split_type_hw_guided_t {
+    const char *info_value;
+    int split_type;
+};
+typedef struct ompi_comm_split_type_hw_guided_t ompi_comm_split_type_hw_guided_t;
+static const ompi_comm_split_type_hw_guided_t ompi_comm_split_type_hw_guided_support[] = {
+    {.info_value = "mpi_shared_memory", .split_type = MPI_COMM_TYPE_SHARED},
+    {.info_value = "hwthread", .split_type = OMPI_COMM_TYPE_HWTHREAD},
+    {.info_value = "core",     .split_type = OMPI_COMM_TYPE_CORE},
+    {.info_value = "l1cache",  .split_type = OMPI_COMM_TYPE_L1CACHE},
+    {.info_value = "l2cache",  .split_type = OMPI_COMM_TYPE_L2CACHE},
+    {.info_value = "l3cache",  .split_type = OMPI_COMM_TYPE_L3CACHE},
+    {.info_value = "socket",   .split_type = OMPI_COMM_TYPE_SOCKET},
+    {.info_value = "numanode", .split_type = OMPI_COMM_TYPE_NUMA},
+    {.info_value = "board",    .split_type = OMPI_COMM_TYPE_BOARD},
+    {.info_value = "host",     .split_type = OMPI_COMM_TYPE_HOST},
+    {.info_value = "cu",       .split_type = OMPI_COMM_TYPE_CU},
+    {.info_value = "cluster",  .split_type = OMPI_COMM_TYPE_CLUSTER},
+    {.info_value = NULL},
+};
+#endif
+
 /*
 ** sort-function for MPI_Comm_split
 */
@@ -764,6 +787,17 @@ static int ompi_comm_split_type_get_part (ompi_group_t *group, const int split_t
         case OMPI_COMM_TYPE_CLUSTER:
             include = OPAL_PROC_ON_LOCAL_CLUSTER(locality);
             break;
+#if MPI_VERSION >= 4
+        case MPI_COMM_TYPE_HW_GUIDED:
+        case MPI_COMM_TYPE_HW_UNGUIDED:
+            /*
+             * MPI_COMM_TYPE_HW_(UN)GUIDED handled in calling function.
+             * We should not get here as the split type will be changed
+             * at a higher level.
+             */
+            opal_output(0, "Error: in ompi_comm_split_type_get_part() unexpected split_type=%d", split_type);
+            return OMPI_ERR_BAD_PARAM;
+#endif
         }
 
         if (include) {
@@ -899,6 +933,75 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
         return OMPI_SUCCESS;
     }
 
+#if MPI_VERSION >= 4
+    if (MPI_COMM_TYPE_HW_GUIDED == global_split_type) {
+        int flag;
+        opal_cstring_t *value = NULL;
+
+        opal_info_get(info, "mpi_hw_resource_type", &value, &flag);
+        /* If key is not in the 'info', then return MPI_COMM_NULL.
+         * This is caught at the MPI interface level, but it doesn't hurt to
+         * check it again.
+         */
+        if (!flag) {
+            *newcomm = MPI_COMM_NULL;
+            return OMPI_SUCCESS;
+        }
+
+        /* Verify the value associated with the "mpi_hw_resource_type" key
+         * - is supported, and
+         * - is the same value at all ranks
+         *
+         * If not supported, then return MPI_COMM_NULL.
+         * If not the same at all ranks, throw an error.
+         */
+        flag = 0;
+        for (int i = 0; ompi_comm_split_type_hw_guided_support[i].info_value; ++i) {
+            if (0 == strncasecmp(value->string, ompi_comm_split_type_hw_guided_support[i].info_value, strlen(ompi_comm_split_type_hw_guided_support[i].info_value))) {
+                global_split_type = ompi_comm_split_type_hw_guided_support[i].split_type;
+                flag = 1;
+                break;
+            }
+        }
+        /* If not supported, then return MPI_COMM_NULL. */
+        if (0 == flag) {
+            *newcomm = MPI_COMM_NULL;
+            return OMPI_SUCCESS;
+        }
+
+        /* Verify all ranks have supplied the same info 'value' represented at
+         * this point by global_split_type since we swapped out 'MPI_COMM_TYPE_HW_GUIDED'
+         * for the specific OMPI_COMM_TYPE_ represented by the info 'value'.
+         */
+        tmp[0] = global_split_type;
+        tmp[1] = -global_split_type;
+        rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &tmp, 2, MPI_INT, MPI_MAX, comm,
+                                           comm->c_coll->coll_allreduce_module);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+            return rc;
+        }
+        /* If not the same at all ranks, throw an error. */
+        if (tmp[0] != -tmp[1]) {
+            if (0 == ompi_comm_rank(comm)) {
+                opal_output(0, "Error: Mismatched info values for MPI_COMM_TYPE_HW_GUIDED");
+            }
+            return OMPI_ERR_BAD_PARAM;
+        }
+    }
+
+    /* TODO: Make this better...
+     *
+     * See Example 7.4 in the MPI 4.0 standard for example usage.
+     *
+     * Stage 0: Recognized, but not implemented.
+     * Stage 1: Do better than that
+     */
+    if (MPI_COMM_TYPE_HW_UNGUIDED == global_split_type) {
+        *newcomm = MPI_COMM_NULL;
+        return OMPI_SUCCESS;
+    }
+#endif
+
     /* Step 2: Build potential communicator groups. If any ranks will not be part of
      * the ultimate communicator we will drop them later. This saves doing an extra
      * allgather on the whole communicator. By using ompi_comm_split() later only

ompi/include/mpi.h.in

@@ -852,7 +852,11 @@ enum {
   OMPI_COMM_TYPE_BOARD,
   OMPI_COMM_TYPE_HOST,
   OMPI_COMM_TYPE_CU,
-  OMPI_COMM_TYPE_CLUSTER
+  OMPI_COMM_TYPE_CLUSTER,
+#if MPI_VERSION >= 4
+  MPI_COMM_TYPE_HW_UNGUIDED,
+  MPI_COMM_TYPE_HW_GUIDED
+#endif
 };
 #define OMPI_COMM_TYPE_NODE MPI_COMM_TYPE_SHARED
 

ompi/include/mpif-values.pl

@@ -395,6 +395,10 @@ sub write_file {
 $constants->{OMPI_COMM_TYPE_HOST} = 9;
 $constants->{OMPI_COMM_TYPE_CU} = 10;
 $constants->{OMPI_COMM_TYPE_CLUSTER} = 11;
+if ($constants->{MPI_VERSION} >= 4) {
+    $constants->{MPI_COMM_TYPE_HW_UNGUIDED} = 12;
+    $constants->{MPI_COMM_TYPE_HW_GUIDED} = 13;
+}
 
 #----------------------------------------------------------------------------
 

ompi/mpi/c/comm_split_type.c

@@ -77,6 +77,10 @@ int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key,
 	     OMPI_COMM_TYPE_L1CACHE != split_type &&
 	     OMPI_COMM_TYPE_CORE != split_type &&
 	     OMPI_COMM_TYPE_HWTHREAD != split_type &&
+#if MPI_VERSION >= 4
+             MPI_COMM_TYPE_HW_UNGUIDED != split_type &&
+             MPI_COMM_TYPE_HW_GUIDED != split_type &&
+#endif
              MPI_UNDEFINED != split_type ) {
             return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
                                           FUNC_NAME);
@@ -99,6 +103,32 @@ int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key,
     }
 #endif
 
+#if MPI_VERSION >= 4
+    if ( MPI_COMM_TYPE_HW_GUIDED == split_type ) {
+        int flag;
+        opal_cstring_t *value = NULL;
+
+        /* MPI_Info is required for this split_type.
+         * Not an error condition, per MPI 4.0.
+         */
+        if ( MPI_INFO_NULL == info ) {
+            *newcomm = MPI_COMM_NULL;
+            rc = MPI_SUCCESS;
+            OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+        }
+
+        /* MPI_Info with key "mpi_hw_resource_type" is required for this split_type.
+         * Not an error condition, per MPI 4.0.
+         */
+        ompi_info_get(info, "mpi_hw_resource_type", &value, &flag);
+        if ( !flag ) {
+            *newcomm = MPI_COMM_NULL;
+            rc = MPI_SUCCESS;
+            OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+        }
+    }
+#endif
+
     if( (MPI_COMM_SELF == comm) && (MPI_UNDEFINED == split_type) ) {
         *newcomm = MPI_COMM_NULL;
         rc = MPI_SUCCESS;