mpi/cxx: isolate internal headers from C++ bindings

ompi/errhandler/errhandler.h

@@ -1,4 +1,4 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,7 +12,11 @@
  *                         All rights reserved.
  * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -32,7 +36,6 @@
 #include "opal/class/opal_object.h"
 #include "opal/class/opal_pointer_array.h"
 
-#include "ompi/mca/rte/rte.h"
 #include "ompi/runtime/mpiruntime.h"
 #include "ompi/errhandler/errhandler_predefined.h"
 #include "ompi/errhandler/errcode-internal.h"
@@ -91,9 +94,8 @@ struct ompi_errhandler_t;
 /**
  * C++ invocation function signature
  */
-typedef void (ompi_errhandler_cxx_dispatch_fn_t)(struct ompi_errhandler_t *errhandler,
-                                                 void *handle, int *err_code,
-                                                 const char *message);
+typedef void (ompi_errhandler_cxx_dispatch_fn_t)(void *handle, int *err_code,
+                                                 const char *message, ompi_errhandler_generic_handler_fn_t *fn);
 
 /**
  * Back-end type for MPI_Errorhandler.

ompi/errhandler/errhandler_invoke.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +12,10 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,8 +59,8 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
             break;
 
         case OMPI_ERRHANDLER_LANG_CXX:
-            errhandler->eh_cxx_dispatch_fn(errhandler, &comm,
-                                           &err_code, message);
+            errhandler->eh_cxx_dispatch_fn(&comm, &err_code, message,
+                                           (ompi_errhandler_generic_handler_fn_t *)errhandler->eh_comm_fn);
             break;
 
         case OMPI_ERRHANDLER_LANG_FORTRAN:
@@ -74,8 +79,8 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
             break;
 
         case OMPI_ERRHANDLER_LANG_CXX:
-            errhandler->eh_cxx_dispatch_fn(errhandler, &win,
-                                           &err_code, message);
+            errhandler->eh_cxx_dispatch_fn(&win, &err_code, message,
+                                           (ompi_errhandler_generic_handler_fn_t *)errhandler->eh_win_fn);
             break;
 
         case OMPI_ERRHANDLER_LANG_FORTRAN:
@@ -94,8 +99,8 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
             break;
 
         case OMPI_ERRHANDLER_LANG_CXX:
-            errhandler->eh_cxx_dispatch_fn(errhandler, &file,
-                                           &err_code, message);
+            errhandler->eh_cxx_dispatch_fn(&file, &err_code, message,
+                                           (ompi_errhandler_generic_handler_fn_t *)errhandler->eh_file_fn);
             break;
 
         case OMPI_ERRHANDLER_LANG_FORTRAN:

ompi/mpi/cxx/Makefile.am

@@ -36,7 +36,8 @@ lib@OMPI_LIBMPI_NAME@_cxx_la_SOURCES = \
 	intercepts.cc \
 	comm.cc \
 	datatype.cc \
-	win.cc
+	win.cc \
+	cxx_glue.c
 
 if OMPI_PROVIDE_MPI_FILE_INTERFACE
 lib@OMPI_LIBMPI_NAME@_cxx_la_SOURCES += \
@@ -77,7 +78,8 @@ headers = \
         group_inln.h \
         op_inln.h \
         errhandler_inln.h \
-        status_inln.h
+        status_inln.h \
+        cxx_glue.h
 
 ompidir = $(ompiincludedir)/ompi/mpi/cxx
 ompi_HEADERS = \

ompi/mpi/cxx/comm.cc

@@ -11,6 +11,8 @@
 // Copyright (c) 2004-2005 The Regents of the University of California.
 //                         All rights reserved.
 // Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+// Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+//                         reserved.
 // $COPYRIGHT$
 //
 // Additional copyrights may follow
@@ -20,16 +22,9 @@
 
 // do not include ompi_config.h because it kills the free/malloc defines
 #include "mpi.h"
+#include "ompi/constants.h"
 #include "ompi/mpi/cxx/mpicxx.h"
-
-#ifdef HAVE_SCHED_H
-#include <sched.h>
-#endif
-
-
-#include "ompi/communicator/communicator.h"
-#include "ompi/attribute/attribute.h"
-#include "ompi/errhandler/errhandler.h"
+#include "cxx_glue.h"
 
 
 //
@@ -56,14 +51,7 @@ MPI::Comm::Comm(const Comm_Null& data) : Comm_Null(data)
 MPI::Errhandler
 MPI::Comm::Create_errhandler(MPI::Comm::_MPI2CPP_ERRHANDLERFN_* function)
 {
-    MPI_Errhandler c_errhandler =
-        ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_COMM,
-                               (ompi_errhandler_generic_handler_fn_t*) function,
-                               OMPI_ERRHANDLER_LANG_CXX);
-    c_errhandler->eh_cxx_dispatch_fn =
-        (ompi_errhandler_cxx_dispatch_fn_t*)
-        ompi_mpi_cxx_comm_errhandler_invoke;
-    return c_errhandler;
+    return ompi_cxx_errhandler_create_comm ((ompi_cxx_dummy_fn_t *) function);
 }
 
 
@@ -77,20 +65,15 @@ MPI::Comm::do_create_keyval(MPI_Comm_copy_attr_function* c_copy_fn,
                             void* extra_state, int &keyval)
 {
     int ret, count = 0;
-    ompi_attribute_fn_ptr_union_t copy_fn;
-    ompi_attribute_fn_ptr_union_t delete_fn;
     keyval_intercept_data_t *cxx_extra_state;
 
     // If both the callbacks are C, then do the simple thing -- no
     // need for all the C++ machinery.
     if (NULL != c_copy_fn && NULL != c_delete_fn) {
-        copy_fn.attr_communicator_copy_fn =
-            (MPI_Comm_internal_copy_attr_function*) c_copy_fn;
-        delete_fn.attr_communicator_delete_fn = c_delete_fn;
-        ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn, delete_fn,
-                                      &keyval, extra_state, 0, NULL);
+        ret = ompi_cxx_attr_create_keyval_comm (c_copy_fn, c_delete_fn, &keyval,
+                                                extra_state, 0, NULL);
         if (MPI_SUCCESS != ret) {
-            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret,
+            return ompi_cxx_errhandler_invoke_comm(MPI_COMM_WORLD, ret,
                                           "MPI::Comm::Create_keyval");
         }
     }
@@ -107,8 +90,8 @@ MPI::Comm::do_create_keyval(MPI_Comm_copy_attr_function* c_copy_fn,
     cxx_extra_state = (keyval_intercept_data_t*)
         malloc(sizeof(keyval_intercept_data_t));
     if (NULL == cxx_extra_state) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_NO_MEM,
-                                      "MPI::Comm::Create_keyval");
+        return ompi_cxx_errhandler_invoke_comm (MPI_COMM_WORLD, MPI_ERR_NO_MEM,
+                                                "MPI::Comm::Create_keyval");
     }
     cxx_extra_state->c_copy_fn = c_copy_fn;
     cxx_extra_state->cxx_copy_fn = cxx_copy_fn;
@@ -131,26 +114,20 @@ MPI::Comm::do_create_keyval(MPI_Comm_copy_attr_function* c_copy_fn,
     }
     if (2 != count) {
         free(cxx_extra_state);
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
-                                      "MPI::Comm::Create_keyval");
+        return ompi_cxx_errhandler_invoke_comm (MPI_COMM_WORLD, MPI_ERR_ARG,
+                                                "MPI::Comm::Create_keyval");
     }
 
     // We do not call MPI_Comm_create_keyval() here because we need to
     // pass in the cxx_extra_state to the backend keyval creation so
     // that when the keyval is destroyed (i.e., when its refcount goes
     // to 0), the cxx_extra_state is free()'ed.
-
-    copy_fn.attr_communicator_copy_fn =
-        (MPI_Comm_internal_copy_attr_function*)
-        ompi_mpi_cxx_comm_copy_attr_intercept;
-    delete_fn.attr_communicator_delete_fn =
-        ompi_mpi_cxx_comm_delete_attr_intercept;
-    ret = ompi_attr_create_keyval(COMM_ATTR, copy_fn, delete_fn,
-                                  &keyval, cxx_extra_state, 0,
-                                  cxx_extra_state);
+    ret = ompi_cxx_attr_create_keyval_comm ((MPI_Comm_copy_attr_function *) ompi_mpi_cxx_comm_copy_attr_intercept,
+                                            ompi_mpi_cxx_comm_delete_attr_intercept,
+                                            &keyval, cxx_extra_state, 0, cxx_extra_state);
     if (OMPI_SUCCESS != ret) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret,
-                                      "MPI::Comm::Create_keyval");
+        return ompi_cxx_errhandler_invoke_comm (MPI_COMM_WORLD, ret,
+                                                "MPI::Comm::Create_keyval");
     }
 
     return MPI_SUCCESS;

ompi/mpi/cxx/cxx_glue.c

@@ -0,0 +1,158 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2016      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "ompi/communicator/communicator.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/file/file.h"
+#include "opal/class/opal_list.h"
+#include "cxx_glue.h"
+
+typedef struct ompi_cxx_intercept_file_extra_state_item_t {
+    opal_list_item_t super;
+    ompi_cxx_intercept_file_extra_state_t state;
+} ompi_cxx_intercept_file_extra_state_item_t;
+
+OBJ_CLASS_DECLARATION(ompi_cxx_intercept_file_extra_state_item_t);
+OBJ_CLASS_INSTANCE(ompi_cxx_intercept_file_extra_state_item_t, opal_list_item_t,
+                   NULL, NULL);
+
+ompi_cxx_communicator_type_t ompi_cxx_comm_get_type (MPI_Comm comm)
+{
+    if (OMPI_COMM_IS_GRAPH(comm)) {
+        return OMPI_CXX_COMM_TYPE_GRAPH;
+    } else if (OMPI_COMM_IS_CART(comm)) {
+        return OMPI_CXX_COMM_TYPE_CART;
+    } else if (OMPI_COMM_IS_INTRA(comm)) {
+        return OMPI_CXX_COMM_TYPE_INTRACOMM;
+    } else if (OMPI_COMM_IS_INTER(comm)) {
+        return OMPI_CXX_COMM_TYPE_INTERCOMM;
+    }
+
+    return OMPI_CXX_COMM_TYPE_UNKNOWN;
+}
+
+int ompi_cxx_errhandler_invoke_comm (MPI_Comm comm, int ret, const char *message)
+{
+    return OMPI_ERRHANDLER_INVOKE (comm, ret, message);
+}
+
+#if OMPI_PROVIDE_MPI_FILE_INTERFACE
+int ompi_cxx_errhandler_invoke_file (MPI_File file, int ret, const char *message)
+{
+    return OMPI_ERRHANDLER_INVOKE (file, ret, message);
+}
+#endif
+
+int ompi_cxx_attr_create_keyval_comm (MPI_Comm_copy_attr_function *copy_fn,
+                                      MPI_Comm_delete_attr_function* delete_fn, int *keyval, void *extra_state,
+                                      int flags, void *bindings_extra_state)
+{
+    ompi_attribute_fn_ptr_union_t copy_fn_u = {.attr_communicator_copy_fn =
+                                               (MPI_Comm_internal_copy_attr_function *) copy_fn};
+    ompi_attribute_fn_ptr_union_t delete_fn_u = {.attr_communicator_delete_fn =
+                                                 (MPI_Comm_delete_attr_function *) delete_fn};
+
+    return ompi_attr_create_keyval (COMM_ATTR, copy_fn_u, delete_fn_u, keyval, extra_state, 0, bindings_extra_state);
+}
+
+int ompi_cxx_attr_create_keyval_win (MPI_Win_copy_attr_function *copy_fn,
+                                      MPI_Win_delete_attr_function* delete_fn, int *keyval, void *extra_state,
+                                      int flags, void *bindings_extra_state)
+{
+    ompi_attribute_fn_ptr_union_t copy_fn_u = {.attr_win_copy_fn =
+                                               (MPI_Win_internal_copy_attr_function *) copy_fn};
+    ompi_attribute_fn_ptr_union_t delete_fn_u = {.attr_win_delete_fn =
+                                                 (MPI_Win_delete_attr_function *) delete_fn};
+
+    return ompi_attr_create_keyval (WIN_ATTR, copy_fn_u, delete_fn_u, keyval, extra_state, 0, NULL);
+}
+
+int ompi_cxx_attr_create_keyval_type (MPI_Type_copy_attr_function *copy_fn,
+                                      MPI_Type_delete_attr_function* delete_fn, int *keyval, void *extra_state,
+                                      int flags, void *bindings_extra_state)
+{
+    ompi_attribute_fn_ptr_union_t copy_fn_u = {.attr_datatype_copy_fn =
+                                               (MPI_Type_internal_copy_attr_function *) copy_fn};
+    ompi_attribute_fn_ptr_union_t delete_fn_u = {.attr_datatype_delete_fn =
+                                                 (MPI_Type_delete_attr_function *) delete_fn};
+
+    return ompi_attr_create_keyval (TYPE_ATTR, copy_fn_u, delete_fn_u, keyval, extra_state, 0, NULL);
+}
+
+MPI_Errhandler ompi_cxx_errhandler_create_comm (ompi_cxx_dummy_fn_t *fn)
+{
+    ompi_errhandler_t *errhandler;
+    errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_COMM,
+                                        (ompi_errhandler_generic_handler_fn_t *) fn,
+                                        OMPI_ERRHANDLER_LANG_CXX);
+    errhandler->eh_cxx_dispatch_fn =
+        (ompi_errhandler_cxx_dispatch_fn_t *) ompi_mpi_cxx_comm_errhandler_invoke;
+    return errhandler;
+}
+
+MPI_Errhandler ompi_cxx_errhandler_create_win (ompi_cxx_dummy_fn_t *fn)
+{
+    ompi_errhandler_t *errhandler;
+    errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_WIN,
+                                        (ompi_errhandler_generic_handler_fn_t *) fn,
+                                        OMPI_ERRHANDLER_LANG_CXX);
+    errhandler->eh_cxx_dispatch_fn =
+        (ompi_errhandler_cxx_dispatch_fn_t *) ompi_mpi_cxx_win_errhandler_invoke;
+    return errhandler;
+}
+
+#if OMPI_PROVIDE_MPI_FILE_INTERFACE
+MPI_Errhandler ompi_cxx_errhandler_create_file (ompi_cxx_dummy_fn_t *fn)
+{
+    ompi_errhandler_t *errhandler;
+    errhandler = ompi_errhandler_create(OMPI_ERRHANDLER_TYPE_FILE,
+                                        (ompi_errhandler_generic_handler_fn_t *) fn,
+                                        OMPI_ERRHANDLER_LANG_CXX);
+    errhandler->eh_cxx_dispatch_fn =
+        (ompi_errhandler_cxx_dispatch_fn_t *) ompi_mpi_cxx_file_errhandler_invoke;
+    return errhandler;
+}
+#endif
+
+ompi_cxx_intercept_file_extra_state_t
+*ompi_cxx_new_intercept_state (void *read_fn_cxx, void *write_fn_cxx, void *extent_fn_cxx,
+                               void *extra_state_cxx)
+{
+    ompi_cxx_intercept_file_extra_state_item_t *intercept;
+
+    intercept = OBJ_NEW(ompi_cxx_intercept_file_extra_state_item_t);
+    if (NULL == intercept) {
+        return NULL;
+    }
+
+    opal_list_append(&ompi_registered_datareps, &intercept->super);
+    intercept->state.read_fn_cxx = read_fn_cxx;
+    intercept->state.write_fn_cxx = write_fn_cxx;
+    intercept->state.extent_fn_cxx = extent_fn_cxx;
+    intercept->state.extra_state_cxx = extra_state_cxx;
+
+    return &intercept->state;
+}
+
+void ompi_cxx_errhandler_set_callbacks (struct ompi_errhandler_t *errhandler, MPI_Comm_errhandler_function *eh_comm_fn,
+                                        ompi_file_errhandler_fn *eh_file_fn, MPI_Win_errhandler_function *eh_win_fn)
+{
+    errhandler->eh_comm_fn = eh_comm_fn;
+#if OMPI_PROVIDE_MPI_FILE_INTERFACE
+    errhandler->eh_file_fn = eh_file_fn;
+#endif
+    errhandler->eh_win_fn = eh_win_fn;
+}