accelerator: add interface to retrieve memkind

add an API to the accelerator component to retrieve the
memory_alloc_kind information that is supported by the component.
The values stored/returned are based on the side document that is
about to be ratified, see

https://github.com/mpi-forum/mem-alloc/blob/main/mem_alloc.tex

Signed-off-by: Edgar Gabriel <[email protected]>

Author: edgargabriel

ompi/info/info_memkind.c

@@ -138,11 +138,9 @@ static int ompi_info_memkind_get_available(int *num_memkinds, ompi_memkind_t **m
     }
 
     int tmp_num = 2;
-#if 0
     if (0 != strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name, "null")) {
         tmp_num++;
     }
-#endif
 
     ompi_info_memkind_available = (ompi_memkind_t *) malloc (tmp_num * sizeof(ompi_memkind_t));
     if (NULL == ompi_info_memkind_available) {
@@ -164,11 +162,9 @@ static int ompi_info_memkind_get_available(int *num_memkinds, ompi_memkind_t **m
     ompi_info_memkind_available[1].im_restrictors[1] = strdup ("win_allocate");
     ompi_info_memkind_available[1].im_restrictors[2] = strdup ("win_allocate_shared");
 
-#if 0
     if (tmp_num > 2) {
-        opal_accelerator.get_memkind_info (&ompi_info_memkind_available[2]);
+        opal_accelerator.get_memkind (&ompi_info_memkind_available[2]);
     }
-#endif
     ompi_info_memkind_num_available = tmp_num;
 
  exit:

opal/mca/accelerator/accelerator.h

@@ -4,7 +4,7 @@
  *                         reserved.
  * Copyright (c)           Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
- * Copyright (c) 2023      Advanced Micro Devices, Inc. All Rights reserved.
+ * Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All Rights reserved.
  * Copyright (c) 2024      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
@@ -81,6 +81,7 @@
 
 #include "opal/class/opal_object.h"
 #include "opal/mca/mca.h"
+#include "ompi/info/info_memkind.h"
 
 BEGIN_C_DECLS
 
@@ -654,6 +655,12 @@ typedef int (*opal_accelerator_base_module_get_num_devices_fn_t)(int *num_device
  */
 typedef int (*opal_accelerator_base_module_get_mem_bw_fn_t)(int device, float *bw);
 
+/**
+ * Get the memkind information of the accelerator component.
+ * @param[OUT] supported     Memory alloc kinds supported by component
+ *
+ */
+typedef void (*opal_accelerator_base_module_get_memkind_fn_t)(ompi_memkind_t *memkind);
 
 /*
  * the standard public API data structure
@@ -700,6 +707,7 @@ typedef struct {
 
     opal_accelerator_base_module_get_num_devices_fn_t num_devices;
     opal_accelerator_base_module_get_mem_bw_fn_t get_mem_bw;
+    opal_accelerator_base_module_get_memkind_fn_t get_memkind;
 } opal_accelerator_base_module_t;
 
 /**

opal/mca/accelerator/cuda/accelerator_cuda.c

@@ -26,6 +26,7 @@
 #include "opal/mca/rcache/rcache.h"
 #include "opal/util/show_help.h"
 #include "opal/util/proc.h"
+#include "ompi/info/info_memkind.h"
 /* Accelerator API's */
 static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *flags);
 static int accelerator_cuda_create_stream(int dev_id, opal_accelerator_stream_t **stream);
@@ -80,6 +81,7 @@ static int accelerator_cuda_get_buffer_id(int dev_id, const void *addr, opal_acc
 static int accelerator_cuda_sync_stream(opal_accelerator_stream_t *stream);
 static int accelerator_cuda_get_num_devices(int *num_devices);
 static int accelerator_cuda_get_mem_bw(int device, float *bw);
+static void accelerator_cuda_get_memkind(ompi_memkind_t *memkind);
 
 #define GET_STREAM(_stream) \
     ((_stream) == MCA_ACCELERATOR_STREAM_DEFAULT ? 0 : *((CUstream *) (_stream)->stream))
@@ -125,7 +127,8 @@ opal_accelerator_base_module_t opal_accelerator_cuda_module =
     accelerator_cuda_get_buffer_id,
 
     accelerator_cuda_get_num_devices,
-    accelerator_cuda_get_mem_bw
+    accelerator_cuda_get_mem_bw,
+    accelerator_cuda_get_memkind
 };
 
 static inline int opal_accelerator_cuda_delayed_init_check(void)
@@ -1218,3 +1221,15 @@ static int accelerator_cuda_get_mem_bw(int device, float *bw)
     *bw = opal_accelerator_cuda_mem_bw[device];
     return OPAL_SUCCESS;
 }
+
+static void mca_accelerator_cuda_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = strdup("cuda");
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 3;
+  memkind->im_restrictors[0] = strdup("host");
+  memkind->im_restrictors[1] = strdup("device");
+  memkind->im_restrictors[2] = strdup("managed");
+
+  return;
+}

opal/mca/accelerator/null/accelerator_null_component.c

@@ -23,6 +23,7 @@
 
 #include "accelerator_null_component.h"
 #include "opal/constants.h"
+#include "ompi/info/info_memkind.h"
 #include <string.h>
 
 /*
@@ -94,6 +95,7 @@ static int accelerator_null_sync_stream(opal_accelerator_stream_t *stream);
 static int accelerator_null_get_num_devices(int *num_devices);
 
 static int accelerator_null_get_mem_bw(int device, float *bw);
+static void accelerator_null_get_memkind(ompi_memkind_t *memkind);
 
 /*
  * Instantiate the public struct with all of our public information
@@ -174,7 +176,8 @@ opal_accelerator_base_module_t opal_accelerator_null_module =
     accelerator_null_get_buffer_id,
 
     accelerator_null_get_num_devices,
-    accelerator_null_get_mem_bw
+    accelerator_null_get_mem_bw,
+    accelerator_null_get_memkind
 };
 
 static int accelerator_null_open(void)
@@ -393,3 +396,12 @@ static int accelerator_null_get_mem_bw(int device, float *bw)
     *bw = 1.0; // return something that is not 0
     return OPAL_SUCCESS;
 }
+
+static void accelerator_null_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = NULL;
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 0;
+
+  return;
+}

opal/mca/accelerator/rocm/accelerator_rocm_module.c

@@ -16,6 +16,7 @@
 #include "opal/mca/accelerator/base/base.h"
 #include "opal/constants.h"
 #include "opal/util/output.h"
+#include "ompi/info/info_memkind.h"
 
 /* Accelerator API's */
 static int mca_accelerator_rocm_check_addr(const void *addr, int *dev_id, uint64_t *flags);
@@ -74,6 +75,7 @@ static int mca_accelerator_rocm_sync_stream(opal_accelerator_stream_t *stream);
 static int mca_accelerator_rocm_get_num_devices(int *num_devices);
 
 static int mca_accelerator_rocm_get_mem_bw(int device, float *bw);
+static void mca_accelerator_rocm_get_memkind(ompi_memkind_t *memkind);
 
 #define GET_STREAM(_stream) (_stream == MCA_ACCELERATOR_STREAM_DEFAULT ? 0 : *((hipStream_t *)_stream->stream))
 
@@ -118,7 +120,8 @@ opal_accelerator_base_module_t opal_accelerator_rocm_module =
     mca_accelerator_rocm_get_buffer_id,
 
     mca_accelerator_rocm_get_num_devices,
-    mca_accelerator_rocm_get_mem_bw
+    mca_accelerator_rocm_get_mem_bw,
+    mca_accelerator_rocm_get_memkind
 };
 
 
@@ -946,3 +949,15 @@ static int mca_accelerator_rocm_get_mem_bw(int device, float *bw)
     *bw = opal_accelerator_rocm_mem_bw[device];
     return OPAL_SUCCESS;
 }
+
+static void mca_accelerator_rocm_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = strdup("rocm");
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 3;
+  memkind->im_restrictors[0] = strdup("host");
+  memkind->im_restrictors[1] = strdup("device");
+  memkind->im_restrictors[2] = strdup("managed");
+
+  return;
+}

opal/mca/accelerator/ze/accelerator_ze_module.c

@@ -18,6 +18,7 @@
 #include "opal/util/printf.h"
 #include "opal/constants.h"
 #include "opal/util/output.h"
+#include "ompi/info/info_memkind.h"
 
 /* Accelerator API's */
 static int mca_accelerator_ze_check_addr(const void *addr, int *dev_id, uint64_t *flags);
@@ -77,6 +78,7 @@ static int mca_accelerator_ze_sync_stream(opal_accelerator_stream_t *stream);
 static int mca_accelerator_ze_get_num_devices(int *num_devices);
 
 static int mca_accelerator_ze_get_mem_bw(int device, float *bw);
+static void mca_accelerator_ze_get_memkind(ompi_memkind_t *memkind);
 
 opal_accelerator_base_module_t opal_accelerator_ze_module =
 {
@@ -118,7 +120,8 @@ opal_accelerator_base_module_t opal_accelerator_ze_module =
 
     .get_buffer_id = mca_accelerator_ze_get_buffer_id,
     .num_devices = mca_accelerator_ze_get_num_devices,
-    .get_mem_bw = mca_accelerator_ze_get_mem_bw
+    .get_mem_bw = mca_accelerator_ze_get_mem_bw,
+    .get_memkind = mca_accelerator_ze_get_memkind
 };
 
 static int accelerator_ze_dev_handle_to_dev_id(ze_device_handle_t hDevice)
@@ -872,4 +875,16 @@ static int mca_accelerator_ze_get_mem_bw(int device, float *bw)
      * TODO
      */
     return OPAL_ERR_NOT_IMPLEMENTED;
-}
+}
+
+static void mca_accelerator_ze_get_memkind (ompi_memkind_t *memkind)
+{
+  memkind->im_name = strdup("level_zero");
+  memkind->im_no_restrictors = false;
+  memkind->im_num_restrictors = 3;
+  memkind->im_restrictors[0] = strdup("host");
+  memkind->im_restrictors[1] = strdup("device");
+  memkind->im_restrictors[2] = strdup("shared");
+
+  return;
+}